From a2665937ee7ea08fb908a7778092bfcf8886295c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 5 Apr 2018 14:38:29 -0600 Subject: [PATCH 001/272] grid-manager proposal --- docs/configuration.rst | 3 + docs/proposed/grid-manager/managed-grid.rst | 247 ++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 docs/proposed/grid-manager/managed-grid.rst diff --git a/docs/configuration.rst b/docs/configuration.rst index 3d7d68ef3..1f5a862b0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -958,6 +958,9 @@ If you omit the introducer definitions from both ``tahoe.cfg`` and "introducerless" clients must be configured with static servers (described below), or they will not be able to upload and download files. + +.. _server_list: + Static Server Definitions ========================= diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst new file mode 100644 index 000000000..05844d9be --- /dev/null +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -0,0 +1,247 @@ +(This document is "in-progress", with feedback and input from two +devchats with Brain Warner and exarkun as well as other input, +discussion and edits from exarkun. It is NOT done). Search for +"DECIDE" for open questions. + + +Managed Grid +============ + +In a grid using an Introducer, a client will use any storage-server +the Introducer announces. This means that anyone with the Introducer +fURL can connect storage to the grid. + +Sometimes, this is just what you want! + +For some use-cases, though, you want to have clients only use certain +servers. One case might be a "managed" grid, where some entity runs +the grid; clients of this grid don't want their uploads to go to +"unmanaged" storage if some other client decides to provide storage. + +One way to limit which storage servers a client connects to is via the +"server list" (:ref:`server_list`) (aka "Introducer-less" +mode). Clients are given static lists of storage-servers, and connect +only to those. This means manually updating these lists if the storage +servers change, however. + +Another method is for clients to use `[client] peers.preferred=` +configuration option (XXX link? appears undocumented), which suffers +from a similar disadvantage. + + +Grid Manager +------------ + +A "grid-manager" consists of some data defining a keypair along with +some other details and Tahoe sub-commands to manipulate the data and +produce certificates to give to storage-servers. Certificates assert +the statement: "Grid Manager X suggests you use storage-server Y to +upload shares to" (X and Y are public-keys). Such a certificate +consists of: + + - a version (currently 1) + - the public-key of a storage-server + - an expiry timestamp + - a signature of the above + +A client will always use any storage-server for downloads (expired +certificate, or no certificate) because we check the ciphertext and +re-assembled plaintext agains the keys in the capability; +"grid-manager" certificates only control uploads. + + +Grid Manager Data Storage +------------------------- + +The data defining the grid-manager is stored in an arbitrary +directory, which you indicate with the ``--config`` option (in the +future, we may add the ability to store the data directly in a grid, +at which time you may be able to pass a directory-capability to this +option). + +If you don't want to store the configuration on disk at all, you may +use ``--config -`` (that's a dash) and write a valid JSON (YAML? I'm +guessing JSON is easier to deal with here, more-interoperable?) +configuration to stdin. + +All commands take the ``--config`` option, and they all behave +similarly for "data from stdin" versus "data from disk". + +DECIDE: + - config is YAML or JSON? + + +tahoe grid-manager create +````````````````````````` + +Create a new grid-manager. + +If you specify ``--config -`` then a new grid-manager configuration is +written to stdout. Otherwise, a new grid-manager is created in the +directory specified by the ``--config`` option. It is an error if the +directory already exists. + + +tahoe grid-manager show-identity +```````````````````````````````` + +Print out a grid-manager's public key. This key is derived from the +private-key of the grid-manager, so a valid grid-manager config must +be given via ``--config`` + +This public key is what is put in clients' configuration to actually +validate and use grid-manager certificates. + + +tahoe grid-manager add +`````````````````````` + +Takes two args: ``name pubkey``. The ``name`` is an arbitrary local +identifier and the pubkey is the encoded key from a ``node.pubkey`` +file in the storage-server's node directory (with no whitespace). + +This adds a new storage-server to a Grid Manager's +configuration. (Since it mutates the configuration, if you used +``--config -`` the new configuration will be printed to stdout). + + +tahoe grid-manager sign +``````````````````````` + +Takes one arg: ``name``, the petname used previous in a ``tahoe +grid-manager add`` command. + +Note that this mutates the state of the grid-manager if it is on disk, +by adding this certificate to our collection of issued +certificates. If you used ``--config -``, the certificate isn't +persisted anywhere except to stdout (so if you wish to keep it +somewhere, that is up to you). + +This command creates a new "version 1" certificate for a +storage-server (identified by its public key). The new certificate is +printed to stdout. If you stored the config on disk, the new +certificate will (also) be in a file named like +``pub-v0-kioayfth3g7zaitcskln64ddx7tkd6xoe7dbr62uogmlwxtxudpq.cert``. + + +Enrolling a Storage Server +-------------------------- + +tahoe admin add-grid-manager-cert +````````````````````````````````` + +- `--filename`: the file to read the cert from (default: stdin) +- `--name`: the name of this certificate (default: "default") + +Import a "version 1" storage-certificate produced by a grid-manager +(probably: a storage server may have zero or more such certificates +installed; for now just one is sufficient). You will have to re-start +your node after this. Subsequent announcements to the Introducer will +include this certificate. + + +Enrolling a Client +------------------ + +tahoe add-grid-manager +`````````````````````` + +- ``--name``: a petname to call this Grid Manager (default: "default") + +For clients to start using a Grid Manager, they must add a +public-key. A client may have any number of grid-managers, so each one +has a name. If you don't supply ``--name`` then ``"default"`` is used. + +This command takes a single argument, which is the hex-encoded public +key of the Grid Manager. The client will have to be re-started once +this change is made. + + +Example Setup of a New Managed Grid +----------------------------------- + +We'll store our Grid Manager configuration on disk, in +``~/grid-manager``. To initialize this directory:: + + tahoe grid-manager create --config ~/grid-manager + +This example creates an actual grid, but it's all just on one machine +with different "node directories". Usually of course each one would be +on a separate computer. + +(If you already have a grid, you can :ref:`skip ahead `.) + +First of all, create an Introducer. Note that we actually have to run +it briefly before it creates the "Introducer fURL" we want for the +next steps. + + tahoe create-introducer --listen=tcp --port=5555 --location=tcp:localhost:5555 ./introducer + tahoe -d introducer run + (Ctrl-C to stop it after a bit) + +Next, we attach a couple of storage nodes:: + + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage0 --webport 6001 --webport 6002 --location tcp:localhost:6003 --port 6003 ./storage0 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage1 --webport 6101 --webport 6102 --location tcp:localhost:6103 --port 6103 ./storage1 + daemonize tahoe -d storage0 run + daemonize tahoe -d storage1 run + +.. _skip_ahead: + +We can now ask the Grid Manager to create certificates for our new +storage servers:: + + tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage0/node.pubkey) > storage0.cert + tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage1/node.pubkey) > storage1.cert + + # enroll server0 (using file) + kill $(cat storage0/twistd.pid) + tahoe -d storage0 admin add-grid-manager-cert --filename storage0.cert + daemonize tahoe -d storage0 run + + # enroll server1 (using stdin) + kill $(cat storage1/twistd.pid) + cat storage1.cert | tahoe -d storage1 admin add-grid-manager-cert + daemonize tahoe -d storage1 run + +Now try adding a new storage server ``storage2``. This client can join +the grid just fine, and announce itself to the Introducer as providing +storage:: + + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage2 --webport 6301 --webport 6302 --location tcp:localhost:6303 --port 6303 ./storage2 + daemonize tahoe -d storage2 run + +At this point any client will upload to any of these three +storage-servers. Make a client "alice" and try! + +:: + + tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6301 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice + daemonize tahoe -d alice run + tahoe -d alice mkdir # prints out a dir-cap + find storage2/storage/shares # confirm storage2 has a share + +Now we want to make Alice only upload to the storage servers that the +grid-manager has given certificates to (``storage0`` and +``storage1``). We need the grid-manager's public key to put in Alice's +configuration:: + + kill $(cat alice/twistd.pid) + tahoe -d alice add-grid-manager --name work-grid $(tahoe grid-manager --config ~/grid-manager show-identity) + daemonize tahoe -d alice start + +DECIDE: + - should the grid-manager be identified by a certificate? exarkun + points out: --name seems like the hint of the beginning of a + use-case for certificates rather than bare public keys?). + +Since we made Alice's parameters require 3 storage servers to be +reachable (`--happy=3`), all their uploads should now fail (so `tahoe +mkdir` will fail) because they won't use storage2 and can't "achieve +happiness". + +You can check Alice's "Welcome" page (where the list of connected servers +is) at http://localhost:6301/ and should be able to see details about +the "work-grid" Grid Manager that you added. When any Grid Managers +are enabled, each storage-server line will show whether it has a valid +cerifiticate or not (and how much longer it's valid until). From a4be9494afd6419281a1841e1b4a934dff6aa80a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Apr 2018 13:42:51 -0600 Subject: [PATCH 002/272] first cut and implementing some grid-manager commands --- src/allmydata/scripts/runner.py | 5 +- src/allmydata/scripts/tahoe_grid_manager.py | 286 ++++++++++++++++++++ 2 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 src/allmydata/scripts/tahoe_grid_manager.py diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 327325e95..164b17334 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -9,7 +9,7 @@ from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ stats_gatherer, admin, magic_folder_cli, tahoe_daemonize, tahoe_start, \ - tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite + tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite, tahoe_grid_manager from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding from allmydata.util.eliotutil import ( opt_eliot_destination, @@ -62,6 +62,7 @@ class Options(usage.Options): + cli.subCommands + magic_folder_cli.subCommands + tahoe_invite.subCommands + + tahoe_grid_manager.subCommands ) optFlags = [ @@ -159,6 +160,8 @@ def dispatch(config, # same f0 = magic_folder_cli.dispatch[command] f = lambda so: threads.deferToThread(f0, so) + elif command in tahoe_grid_manager.dispatch: + f = tahoe_grid_manager.dispatch[command] elif command in tahoe_invite.dispatch: f = tahoe_invite.dispatch[command] else: diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py new file mode 100644 index 000000000..3f62efc46 --- /dev/null +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -0,0 +1,286 @@ + +import os +import sys +import json + +from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though + +from allmydata.scripts.common import BasedirOptions +from twisted.scripts import twistd +from twisted.python import usage +from twisted.python.reflect import namedAny +from twisted.python.filepath import FilePath +from allmydata.scripts.default_nodedir import _default_nodedir +from allmydata.util import fileutil +from allmydata.util import base32 +from allmydata.util import keyutil +from allmydata.node import read_config +from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path +from twisted.application.service import Service +from twisted.internet.defer import inlineCallbacks, returnValue + + +class CreateOptions(BasedirOptions): + description = ( + "Create a new identity key and configuration of a Grid Manager" + ) + + +class ShowIdentityOptions(BasedirOptions): + description = ( + "Create a new identity key and configuration of a Grid Manager" + ) + + +class AddOptions(BasedirOptions): + description = ( + "Add a new storage-server's key to a Grid Manager configuration" + ) + + def parseArgs(self, *args, **kw): + BasedirOptions.parseArgs(self, **kw) + if len(args) != 2: + raise usage.UsageError( + "Requires two arguments: name pubkey" + ) + self['name'] = unicode(args[0]) + try: + # WTF?! why does it want 'str' and not six.text_type? + self['storage_pubkey'] = keyutil.parse_pubkey(args[1]) + except Exception as e: + raise usage.UsageError( + "Invalid pubkey argument: {}".format(e) + ) + + +class SignOptions(BasedirOptions): + description = ( + "Create and sign a new certificate for a storage-server" + ) + + def parseArgs(self, *args, **kw): + BasedirOptions.parseArgs(self, **kw) + if len(args) != 1: + raise usage.UsageError( + "Requires one argument: name" + ) + self['name'] = unicode(args[0]) + + +class GridManagerOptions(BasedirOptions): + subCommands = [ + ["create", None, CreateOptions, "Create a Grid Manager."], + ["show-identity", None, ShowIdentityOptions, "Show public-key for Grid Manager."], + ["add", None, AddOptions, "Add a storage server to a Grid Manager."], + ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], + ] + + optParameters = [ + ("config", "c", None, "How to find the Grid Manager's configuration") + ] + + def postOptions(self): + if not hasattr(self, 'subOptions'): + raise usage.UsageError("must specify a subcommand") + if self['config'] is None: + raise usage.UsageError("Must supply configuration with --config") + + description = ( + "A Grid Manager creates certificates for Storage Servers certifying " + "them for use by clients to upload shares to. Configuration may be " + "passed in on stdin or stored in a directory." + ) + + +def _create_gridmanager(): + return { + "grid_manager_config_version": 0, + "privkey": ed25519.SigningKey(os.urandom(32)), + } + +def _create(gridoptions, options): + gm_config = gridoptions['config'] + assert gm_config is not None + + # pre-conditions check + fp = None + if gm_config.strip() != '-': + fp = FilePath(gm_config.strip()) + if fp.exists(): + raise usage.UsageError( + "The directory '{}' already exists.".format(gm_config) + ) + + gm = _create_gridmanager() + _save_gridmanager_config(fp, gm) + + +def _save_gridmanager_config(file_path, grid_manager): + """ + Writes a Grid Manager configuration to the place specified by + 'file_path' (if None, stdout is used). + """ + # FIXME probably want a GridManagerConfig class or something with + # .save and .load instead of this crap + raw_data = { + k: v + for k, v in grid_manager.items() + } + raw_data['privkey'] = base32.b2a(raw_data['privkey'].sk_and_vk[:32]) + data = json.dumps(raw_data, indent=4) + + if file_path is None: + print("{}\n".format(data)) + else: + fileutil.make_dirs(file_path.path, mode=0o700) + with file_path.child("config.json").open("w") as f: + f.write("{}\n".format(data)) + return 0 + + +# XXX should take a FilePath or None +def _load_gridmanager_config(gm_config): + """ + Loads a Grid Manager configuration and returns it (a dict) after + validating. Exceptions if the config can't be found, or has + problems. + """ + fp = None + if gm_config.strip() != '-': + fp = FilePath(gm_config.strip()) + if not fp.exists(): + raise RuntimeError( + "No such directory '{}'".format(gm_config) + ) + + if fp is None: + gm = json.load(sys.stdin) + else: + with fp.child("config.json").open("r") as f: + gm = json.load(f) + + if 'privkey' not in gm: + raise RuntimeError( + "Grid Manager config from '{}' requires a 'privkey'".format( + gm_config + ) + ) + + privkey_str = gm['privkey'] + try: + privkey_bytes = base32.a2b(privkey_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? + gm['privkey'] = ed25519.SigningKey(privkey_bytes) + except Exception as e: + raise RuntimeError( + "Invalid Grid Manager privkey: {}".format(e) + ) + + gm_version = gm.get('grid_manager_config_version', None) + if gm_version != 0: + raise RuntimeError( + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) + ) + return gm + + +def _show_identity(gridoptions, options): + """ + Output the public-key of a Grid Manager + """ + gm_config = gridoptions['config'].strip() + assert gm_config is not None + + gm = _load_gridmanager_config(gm_config) + verify_key_bytes = gm['privkey'].get_verifying_key_bytes() + print(base32.b2a(verify_key_bytes)) + + +def _add(gridoptions, options): + """ + Add a new storage-server by name to a Grid Manager + """ + gm_config = gridoptions['config'].strip() + assert gm_config is not None + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + + gm = _load_gridmanager_config(gm_config) + if options['name'] in gm.get('storage_severs', set()): + raise usage.UsageError( + "A storage-server called '{}' already exists".format(options['name']) + ) + if 'storage_servers' not in gm: + gm['storage_servers'] = dict() + gm['storage_servers'][options['name']] = base32.b2a(options['storage_pubkey'].vk_bytes) + _save_gridmanager_config(fp, gm) + + +def _sign(gridoptions, options): + """ + sign a new certificate + """ + gm_config = gridoptions['config'].strip() + assert gm_config is not None + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + gm = _load_gridmanager_config(gm_config) + + if options['name'] not in gm.get('storage_servers', dict()): + raise usage.UsageError( + "No storage-server called '{}' exists".format(options['name']) + ) + + pubkey = gm['storage_servers'][options['name']] + import time + cert_info = { + "expires": int(time.time() + 86400), # XXX FIXME + "pubkey": pubkey, + "version": 1, + } + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) + sig = gm['privkey'].sign(cert_data) + certificate = { + "certificate": cert_data, + "signature": base32.b2a(sig), + } + certificate_data = json.dumps(certificate, indent=4) + print(certificate_data) + if fp is not None: + with fp.child('{}.cert'.format(options['name'])).open('w') as f: + f.write(certificate_data) + + +grid_manager_commands = { + CreateOptions: _create, + ShowIdentityOptions: _show_identity, + AddOptions: _add, + SignOptions: _sign, +} + +@inlineCallbacks +def gridmanager(config): + """ + Runs the 'tahoe grid-manager' command. + """ + if config.subCommand is None: + print(config) + returnValue(1) + + try: + f = grid_manager_commands[config.subOptions.__class__] + except KeyError: + print(config.subOptions, grid_manager_commands.keys()) + print("Unknown command 'tahoe grid-manager {}': no such grid-manager subcommand".format(config.subCommand)) + returnValue(2) + + x = yield f(config, config.subOptions) + returnValue(x) + +subCommands = [ + ["grid-manager", None, GridManagerOptions, + "Grid Manager subcommands: use 'tahoe grid-manager' for a list."], +] + +dispatch = { + "grid-manager": gridmanager, +} From 31246c84ab8769218e822f7b0e1085e5aa6666d5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 14:58:32 -0600 Subject: [PATCH 003/272] pubkey -> public_key, privkey -> private_key and doc fixups --- docs/proposed/grid-manager/managed-grid.rst | 8 ++-- src/allmydata/scripts/tahoe_grid_manager.py | 52 ++++++++++++--------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 05844d9be..342bcf29d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -32,8 +32,8 @@ from a similar disadvantage. Grid Manager ------------ -A "grid-manager" consists of some data defining a keypair along with -some other details and Tahoe sub-commands to manipulate the data and +A "grid-manager" consists of some data defining a keypair (along with +some other details) and Tahoe sub-commands to manipulate the data and produce certificates to give to storage-servers. Certificates assert the statement: "Grid Manager X suggests you use storage-server Y to upload shares to" (X and Y are public-keys). Such a certificate @@ -45,8 +45,8 @@ consists of: - a signature of the above A client will always use any storage-server for downloads (expired -certificate, or no certificate) because we check the ciphertext and -re-assembled plaintext agains the keys in the capability; +certificate, or no certificate) because clients check the ciphertext +and re-assembled plaintext against the keys in the capability; "grid-manager" certificates only control uploads. diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 3f62efc46..9f87adc98 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -28,7 +28,10 @@ class CreateOptions(BasedirOptions): class ShowIdentityOptions(BasedirOptions): description = ( - "Create a new identity key and configuration of a Grid Manager" + "Show the public identity key of a Grid Manager\n" + "\n" + "This is what you give to clients to add to their configuration" + " so they use announcements from this Grid Manager" ) @@ -41,15 +44,15 @@ class AddOptions(BasedirOptions): BasedirOptions.parseArgs(self, **kw) if len(args) != 2: raise usage.UsageError( - "Requires two arguments: name pubkey" + "Requires two arguments: name public_key" ) self['name'] = unicode(args[0]) try: # WTF?! why does it want 'str' and not six.text_type? - self['storage_pubkey'] = keyutil.parse_pubkey(args[1]) + self['storage_public_key'] = keyutil.parse_public_key(args[1]) except Exception as e: raise usage.UsageError( - "Invalid pubkey argument: {}".format(e) + "Invalid public_key argument: {}".format(e) ) @@ -70,8 +73,8 @@ class SignOptions(BasedirOptions): class GridManagerOptions(BasedirOptions): subCommands = [ ["create", None, CreateOptions, "Create a Grid Manager."], - ["show-identity", None, ShowIdentityOptions, "Show public-key for Grid Manager."], - ["add", None, AddOptions, "Add a storage server to a Grid Manager."], + ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], + ["add", None, AddOptions, "Add a storage server to this Grid Manager."], ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], ] @@ -86,16 +89,21 @@ class GridManagerOptions(BasedirOptions): raise usage.UsageError("Must supply configuration with --config") description = ( - "A Grid Manager creates certificates for Storage Servers certifying " - "them for use by clients to upload shares to. Configuration may be " - "passed in on stdin or stored in a directory." + 'A "grid-manager" consists of some data defining a keypair (along with ' + 'some other details) and Tahoe sub-commands to manipulate the data and ' + 'produce certificates to give to storage-servers. Certificates assert ' + 'the statement: "Grid Manager X suggests you use storage-server Y to ' + 'upload shares to" (X and Y are public-keys).' + '\n\n' + 'Clients can use Grid Managers to decide which storage servers to ' + 'upload shares to.' ) def _create_gridmanager(): return { "grid_manager_config_version": 0, - "privkey": ed25519.SigningKey(os.urandom(32)), + "private_key": ed25519.SigningKey(os.urandom(32)), } def _create(gridoptions, options): @@ -126,7 +134,7 @@ def _save_gridmanager_config(file_path, grid_manager): k: v for k, v in grid_manager.items() } - raw_data['privkey'] = base32.b2a(raw_data['privkey'].sk_and_vk[:32]) + raw_data['private_key'] = base32.b2a(raw_data['private_key'].sk_and_vk[:32]) data = json.dumps(raw_data, indent=4) if file_path is None: @@ -159,20 +167,20 @@ def _load_gridmanager_config(gm_config): with fp.child("config.json").open("r") as f: gm = json.load(f) - if 'privkey' not in gm: + if 'private_key' not in gm: raise RuntimeError( - "Grid Manager config from '{}' requires a 'privkey'".format( + "Grid Manager config from '{}' requires a 'private_key'".format( gm_config ) ) - privkey_str = gm['privkey'] + private_key_str = gm['private_key'] try: - privkey_bytes = base32.a2b(privkey_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? - gm['privkey'] = ed25519.SigningKey(privkey_bytes) + private_key_bytes = base32.a2b(private_key_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? + gm['private_key'] = ed25519.SigningKey(private_key_bytes) except Exception as e: raise RuntimeError( - "Invalid Grid Manager privkey: {}".format(e) + "Invalid Grid Manager private_key: {}".format(e) ) gm_version = gm.get('grid_manager_config_version', None) @@ -193,7 +201,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - verify_key_bytes = gm['privkey'].get_verifying_key_bytes() + verify_key_bytes = gm['private_key'].get_verifying_key_bytes() print(base32.b2a(verify_key_bytes)) @@ -212,7 +220,7 @@ def _add(gridoptions, options): ) if 'storage_servers' not in gm: gm['storage_servers'] = dict() - gm['storage_servers'][options['name']] = base32.b2a(options['storage_pubkey'].vk_bytes) + gm['storage_servers'][options['name']] = base32.b2a(options['storage_public_key'].vk_bytes) _save_gridmanager_config(fp, gm) @@ -230,15 +238,15 @@ def _sign(gridoptions, options): "No storage-server called '{}' exists".format(options['name']) ) - pubkey = gm['storage_servers'][options['name']] + public_key = gm['storage_servers'][options['name']] import time cert_info = { "expires": int(time.time() + 86400), # XXX FIXME - "pubkey": pubkey, + "public_key": public_key, "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) - sig = gm['privkey'].sign(cert_data) + sig = gm['private_key'].sign(cert_data) certificate = { "certificate": cert_data, "signature": base32.b2a(sig), From c874a388ce6c7f49d3514727d30ba511873dcc62 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 16:33:10 -0600 Subject: [PATCH 004/272] wip --- src/allmydata/scripts/tahoe_grid_manager.py | 140 ++++++++++++++------ 1 file changed, 100 insertions(+), 40 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 9f87adc98..9a39ab567 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -49,13 +49,19 @@ class AddOptions(BasedirOptions): self['name'] = unicode(args[0]) try: # WTF?! why does it want 'str' and not six.text_type? - self['storage_public_key'] = keyutil.parse_public_key(args[1]) + self['storage_public_key'] = keyutil.parse_pubkey(args[1]) except Exception as e: raise usage.UsageError( "Invalid public_key argument: {}".format(e) ) +class ListOptions(BasedirOptions): + description = ( + "List all storage servers in this Grid Manager" + ) + + class SignOptions(BasedirOptions): description = ( "Create and sign a new certificate for a storage-server" @@ -75,6 +81,7 @@ class GridManagerOptions(BasedirOptions): ["create", None, CreateOptions, "Create a Grid Manager."], ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], ["add", None, AddOptions, "Add a storage server to this Grid Manager."], + ["list", None, ListOptions, "List all storage servers in this Grid Manager."], ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], ] @@ -107,8 +114,10 @@ def _create_gridmanager(): } def _create(gridoptions, options): + """ + Create a new Grid Manager + """ gm_config = gridoptions['config'] - assert gm_config is not None # pre-conditions check fp = None @@ -123,19 +132,78 @@ def _create(gridoptions, options): _save_gridmanager_config(fp, gm) +class _GridManager(object): + """ + A Grid Manager's configuration. + """ + + def __init__(self, config, config_location): + if 'private_key' not in config: + raise RuntimeError( + "Grid Manager config from '{}' requires a 'private_key'".format( + config_config + ) + ) + + private_key_str = config['private_key'] + try: + self._private_key_bytes = base32.a2b(private_key_str.encode('ascii')) + self._private_key = ed25519.SigningKey(self._private_key_bytes) + except Exception as e: + raise RuntimeError( + "Invalid Grid Manager private_key: {}".format(e) + ) + + gm_version = config.get('grid_manager_config_version', None) + if gm_version != 0: + raise RuntimeError( + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) + ) + self._version = 0 + self._storage_servers = dict() + + @property + def storage_servers(self): + return self._storage_servers + + def add_storage_server(self, name, public_key): + """ + :param name: a user-meaningful name for the server + :param public_key: ed25519.VerifyingKey the public-key of the + storage provider (e.g. from the contents of node.pubkey + for the client) + """ + if name in self._storage_servers: + raise KeyError( + "Already have a storage server called '{}'".format(name) + ) + assert public_key.vk_bytes + self._storage_servers[name] = public_key + + def marshal(self): + data = { + u"grid_manager_config_version": self._version, + u"private_key": base32.b2a(self._private_key.sk_and_vk[:32]), + } + if self._storage_servers: + data[u"storage_servers"] = { + name: base32.b2a(public_key.vk_bytes) + for name, public_key + in self._storage_servers.items() + } + + def _save_gridmanager_config(file_path, grid_manager): """ Writes a Grid Manager configuration to the place specified by 'file_path' (if None, stdout is used). """ - # FIXME probably want a GridManagerConfig class or something with - # .save and .load instead of this crap - raw_data = { - k: v - for k, v in grid_manager.items() - } - raw_data['private_key'] = base32.b2a(raw_data['private_key'].sk_and_vk[:32]) - data = json.dumps(raw_data, indent=4) + data = json.dumps( + grid_manager.marshal(), + indent=4, + ) if file_path is None: print("{}\n".format(data)) @@ -167,30 +235,7 @@ def _load_gridmanager_config(gm_config): with fp.child("config.json").open("r") as f: gm = json.load(f) - if 'private_key' not in gm: - raise RuntimeError( - "Grid Manager config from '{}' requires a 'private_key'".format( - gm_config - ) - ) - - private_key_str = gm['private_key'] - try: - private_key_bytes = base32.a2b(private_key_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? - gm['private_key'] = ed25519.SigningKey(private_key_bytes) - except Exception as e: - raise RuntimeError( - "Invalid Grid Manager private_key: {}".format(e) - ) - - gm_version = gm.get('grid_manager_config_version', None) - if gm_version != 0: - raise RuntimeError( - "Missing or unknown version '{}' of Grid Manager config".format( - gm_version - ) - ) - return gm + return _GridManager(gm, gm_config) def _show_identity(gridoptions, options): @@ -210,26 +255,40 @@ def _add(gridoptions, options): Add a new storage-server by name to a Grid Manager """ gm_config = gridoptions['config'].strip() - assert gm_config is not None fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) - if options['name'] in gm.get('storage_severs', set()): + try: + gm.add_storage_server( + options['name'], + options['storage_public_key'], + ) + except KeyError: raise usage.UsageError( "A storage-server called '{}' already exists".format(options['name']) ) - if 'storage_servers' not in gm: - gm['storage_servers'] = dict() - gm['storage_servers'][options['name']] = base32.b2a(options['storage_public_key'].vk_bytes) + _save_gridmanager_config(fp, gm) +def _list(gridoptions, options): + """ + List all storage-servers known to a Grid Manager + """ + gm_config = gridoptions['config'].strip() + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + + gm = _load_gridmanager_config(gm_config) + for name in sorted(gm.storage_servers.keys()): + key = "pub-v0-" + gm.storage_servers[name].vk_bytes + print("{}: {}".format(name, key)) + + def _sign(gridoptions, options): """ sign a new certificate """ gm_config = gridoptions['config'].strip() - assert gm_config is not None fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) @@ -262,6 +321,7 @@ grid_manager_commands = { CreateOptions: _create, ShowIdentityOptions: _show_identity, AddOptions: _add, + ListOptions: _list, SignOptions: _sign, } From a40364adba45090a43ec192ad426e809534a0db4 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:12:28 -0600 Subject: [PATCH 005/272] do something sensible on UsageError --- src/allmydata/scripts/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 164b17334..421eca5d5 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -175,7 +175,13 @@ def dispatch(config, # 4: return a Deferred that does 1 or 2 or 3 def _raise_sys_exit(rc): sys.exit(rc) + + def _error(f): + f.trap(usage.UsageError) + print("Error: {}".format(f.value.message)) + sys.exit(1) d.addCallback(_raise_sys_exit) + d.addErrback(_error) return d def _maybe_enable_eliot_logging(options, reactor): From 4d409776650cd3454b7ec0202513f9fae61eaee7 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:12:46 -0600 Subject: [PATCH 006/272] refactor .sign() into GridManager, better GM construction --- src/allmydata/scripts/tahoe_grid_manager.py | 76 +++++++++++++-------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 9a39ab567..6775b38f1 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -2,6 +2,7 @@ import os import sys import json +import time from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though @@ -108,10 +109,7 @@ class GridManagerOptions(BasedirOptions): def _create_gridmanager(): - return { - "grid_manager_config_version": 0, - "private_key": ed25519.SigningKey(os.urandom(32)), - } + return _GridManager(ed25519.SigningKey(os.urandom(32)), {}) def _create(gridoptions, options): """ @@ -137,9 +135,10 @@ class _GridManager(object): A Grid Manager's configuration. """ - def __init__(self, config, config_location): + @staticmethod + def from_config(config, config_location): if 'private_key' not in config: - raise RuntimeError( + raise ValueError( "Grid Manager config from '{}' requires a 'private_key'".format( config_config ) @@ -147,27 +146,56 @@ class _GridManager(object): private_key_str = config['private_key'] try: - self._private_key_bytes = base32.a2b(private_key_str.encode('ascii')) - self._private_key = ed25519.SigningKey(self._private_key_bytes) + private_key_bytes = base32.a2b(private_key_str.encode('ascii')) + private_key = ed25519.SigningKey(private_key_bytes) except Exception as e: - raise RuntimeError( + raise ValueError( "Invalid Grid Manager private_key: {}".format(e) ) - gm_version = config.get('grid_manager_config_version', None) + storage_servers = dict() + for name, pubkey in config.get(u'storage_servers', {}).items(): + pubkey_bytes = base32.a2b(pubkey.encode('ascii')) + storage_servers[name] = ed25519.VerifyingKey(pubkey_bytes) + + gm_version = config.get(u'grid_manager_config_version', None) if gm_version != 0: - raise RuntimeError( + raise ValueError( "Missing or unknown version '{}' of Grid Manager config".format( gm_version ) ) + return _GridManager(private_key, storage_servers) + + def __init__(self, private_key, storage_servers): + self._storage_servers = dict() if storage_servers is None else storage_servers + self._private_key = private_key self._version = 0 - self._storage_servers = dict() @property def storage_servers(self): return self._storage_servers + def sign(self, name): + try: + public_key = self._storage_servers[name] + except KeyError: + raise KeyError( + u"No storage server named '{}'".format(name) + ) + cert_info = { + "expires": int(time.time() + 86400), # XXX FIXME + "public_key": base32.b2a(public_key.vk_bytes), + "version": 1, + } + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) + sig = self._private_key.sign(cert_data) + certificate = { + u"certificate": cert_data, + u"signature": base32.b2a(sig), + } + return certificate + def add_storage_server(self, name, public_key): """ :param name: a user-meaningful name for the server @@ -193,6 +221,7 @@ class _GridManager(object): for name, public_key in self._storage_servers.items() } + return data def _save_gridmanager_config(file_path, grid_manager): @@ -211,7 +240,6 @@ def _save_gridmanager_config(file_path, grid_manager): fileutil.make_dirs(file_path.path, mode=0o700) with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) - return 0 # XXX should take a FilePath or None @@ -235,7 +263,7 @@ def _load_gridmanager_config(gm_config): with fp.child("config.json").open("r") as f: gm = json.load(f) - return _GridManager(gm, gm_config) + return _GridManager.from_config(gm, gm_config) def _show_identity(gridoptions, options): @@ -269,6 +297,7 @@ def _add(gridoptions, options): ) _save_gridmanager_config(fp, gm) + return 0 def _list(gridoptions, options): @@ -280,7 +309,7 @@ def _list(gridoptions, options): gm = _load_gridmanager_config(gm_config) for name in sorted(gm.storage_servers.keys()): - key = "pub-v0-" + gm.storage_servers[name].vk_bytes + key = "pub-v0-" + base32.b2a(gm.storage_servers[name].vk_bytes) print("{}: {}".format(name, key)) @@ -292,24 +321,13 @@ def _sign(gridoptions, options): fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) - if options['name'] not in gm.get('storage_servers', dict()): + try: + certificate = gm.sign(options['name']) + except KeyError: raise usage.UsageError( "No storage-server called '{}' exists".format(options['name']) ) - public_key = gm['storage_servers'][options['name']] - import time - cert_info = { - "expires": int(time.time() + 86400), # XXX FIXME - "public_key": public_key, - "version": 1, - } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) - sig = gm['private_key'].sign(cert_data) - certificate = { - "certificate": cert_data, - "signature": base32.b2a(sig), - } certificate_data = json.dumps(certificate, indent=4) print(certificate_data) if fp is not None: From 7803febf2da0ec1674d2dd46654619acccee7268 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:17:20 -0600 Subject: [PATCH 007/272] refactor; make doc match code --- docs/proposed/grid-manager/managed-grid.rst | 11 +++++++++-- src/allmydata/scripts/tahoe_grid_manager.py | 7 +++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 342bcf29d..304273fe2 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -82,8 +82,8 @@ directory specified by the ``--config`` option. It is an error if the directory already exists. -tahoe grid-manager show-identity -```````````````````````````````` +tahoe grid-manager public-identity +`````````````````````````````````` Print out a grid-manager's public key. This key is derived from the private-key of the grid-manager, so a valid grid-manager config must @@ -105,6 +105,13 @@ configuration. (Since it mutates the configuration, if you used ``--config -`` the new configuration will be printed to stdout). +tahoe grid-manager list +``````````````````````` + +Lists all storage-servers that have previously been added using +``tahoe grid-manager add``. + + tahoe grid-manager sign ``````````````````````` diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 6775b38f1..1a8c627ac 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -176,6 +176,10 @@ class _GridManager(object): def storage_servers(self): return self._storage_servers + def public_identity(self): + verify_key_bytes = self._private_key.get_verifying_key_bytes() + return base32.b2a(verify_key_bytes) + def sign(self, name): try: public_key = self._storage_servers[name] @@ -274,8 +278,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - verify_key_bytes = gm['private_key'].get_verifying_key_bytes() - print(base32.b2a(verify_key_bytes)) + print(gm.public_identity()) def _add(gridoptions, options): From 870b0c40a0a230d195b7fd0e497ee4916980fdbc Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:49:24 -0600 Subject: [PATCH 008/272] tahoe grid-manager list and other fixups --- docs/proposed/grid-manager/managed-grid.rst | 3 +- src/allmydata/scripts/tahoe_grid_manager.py | 59 +++++++++++++++++---- src/allmydata/util/abbreviate.py | 2 +- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 304273fe2..d13cfef33 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -127,8 +127,7 @@ somewhere, that is up to you). This command creates a new "version 1" certificate for a storage-server (identified by its public key). The new certificate is printed to stdout. If you stored the config on disk, the new -certificate will (also) be in a file named like -``pub-v0-kioayfth3g7zaitcskln64ddx7tkd6xoe7dbr62uogmlwxtxudpq.cert``. +certificate will (also) be in a file named like ``alice.cert.0``. Enrolling a Storage Server diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 1a8c627ac..f9dc90280 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -3,10 +3,12 @@ import os import sys import json import time +from datetime import datetime from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though from allmydata.scripts.common import BasedirOptions +from allmydata.util.abbreviate import abbreviate_time from twisted.scripts import twistd from twisted.python import usage from twisted.python.reflect import namedAny @@ -130,6 +132,25 @@ def _create(gridoptions, options): _save_gridmanager_config(fp, gm) +class _GridManagerStorageServer(object): + """ + A Grid Manager's notion of a storage server + """ + + def __init__(self, name, public_key, certificates): + self.name = name + self._public_key = public_key + self._certificates = [] if certificates is None else certificates + + def add_certificate(self, certificate): + self._certificates.append(certificate) + + def public_key(self): + return "pub-v0-" + base32.b2a(self._public_key.vk_bytes) + + def marshal(self): + pass + class _GridManager(object): """ A Grid Manager's configuration. @@ -156,7 +177,9 @@ class _GridManager(object): storage_servers = dict() for name, pubkey in config.get(u'storage_servers', {}).items(): pubkey_bytes = base32.a2b(pubkey.encode('ascii')) - storage_servers[name] = ed25519.VerifyingKey(pubkey_bytes) + storage_servers[name] = _GridManagerStorageServer( + name, ed25519.VerifyingKey(pubkey_bytes), None + ) gm_version = config.get(u'grid_manager_config_version', None) if gm_version != 0: @@ -182,14 +205,14 @@ class _GridManager(object): def sign(self, name): try: - public_key = self._storage_servers[name] + srv = self._storage_servers[name] except KeyError: raise KeyError( u"No storage server named '{}'".format(name) ) cert_info = { "expires": int(time.time() + 86400), # XXX FIXME - "public_key": base32.b2a(public_key.vk_bytes), + "public_key": srv.public_key(), "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) @@ -212,7 +235,9 @@ class _GridManager(object): "Already have a storage server called '{}'".format(name) ) assert public_key.vk_bytes - self._storage_servers[name] = public_key + ss = _GridManagerStorageServer(name, public_key, None) + self._storage_servers[name] = ss + return ss def marshal(self): data = { @@ -221,8 +246,8 @@ class _GridManager(object): } if self._storage_servers: data[u"storage_servers"] = { - name: base32.b2a(public_key.vk_bytes) - for name, public_key + name: srv.marshal() + for name, srv in self._storage_servers.items() } return data @@ -312,8 +337,20 @@ def _list(gridoptions, options): gm = _load_gridmanager_config(gm_config) for name in sorted(gm.storage_servers.keys()): - key = "pub-v0-" + base32.b2a(gm.storage_servers[name].vk_bytes) - print("{}: {}".format(name, key)) + print("{}: {}".format(name, gm.storage_servers[name].public_key())) + if fp: + cert_count = 0 + while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): + container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) + cert_data = json.loads(container['certificate']) + expires = datetime.fromtimestamp(cert_data['expires']) + delta = datetime.utcnow() - expires + if delta.total_seconds() < 0: + print(" {}: valid until {} ({})".format(cert_count, expires, abbreviate_time(delta))) + else: + print(" {}: expired ({})".format(cert_count, abbreviate_time(delta))) + cert_count += 1 + certs = ' ({} certificates)'.format(cert_count) def _sign(gridoptions, options): @@ -334,7 +371,11 @@ def _sign(gridoptions, options): certificate_data = json.dumps(certificate, indent=4) print(certificate_data) if fp is not None: - with fp.child('{}.cert'.format(options['name'])).open('w') as f: + next_serial = 0 + p = fp.child('{}.cert'.format(options['name'])).path + while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists(): + next_serial += 1 + with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f: f.write(certificate_data) diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 6fdf894ce..b3f481d5e 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -23,7 +23,7 @@ def abbreviate_time(s): if s >= 0.0: postfix = ' ago' else: - postfix = ' in the future' + postfix = ' from now' s = -s def _plural(count, unit): count = int(count) From c76d25ed8734096be5bc15d0394eac57f70bffb1 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 19 May 2018 19:17:19 -0600 Subject: [PATCH 009/272] Add 'tahoe grid-manager remove' command --- src/allmydata/scripts/tahoe_grid_manager.py | 65 +++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f9dc90280..d75b293cd 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -59,6 +59,20 @@ class AddOptions(BasedirOptions): ) +class RemoveOptions(BasedirOptions): + description = ( + "Remove a storage-server from a Grid Manager configuration" + ) + + def parseArgs(self, *args, **kw): + BasedirOptions.parseArgs(self, **kw) + if len(args) != 1: + raise usage.UsageError( + "Requires one arguments: name" + ) + self['name'] = unicode(args[0]) + + class ListOptions(BasedirOptions): description = ( "List all storage servers in this Grid Manager" @@ -84,6 +98,7 @@ class GridManagerOptions(BasedirOptions): ["create", None, CreateOptions, "Create a Grid Manager."], ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], ["add", None, AddOptions, "Add a storage server to this Grid Manager."], + ["remove", None, RemoveOptions, "Remove a storage server from this Grid Manager."], ["list", None, ListOptions, "List all storage servers in this Grid Manager."], ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], ] @@ -149,7 +164,9 @@ class _GridManagerStorageServer(object): return "pub-v0-" + base32.b2a(self._public_key.vk_bytes) def marshal(self): - pass + return { + u"public_key": self.public_key(), + } class _GridManager(object): """ @@ -175,10 +192,15 @@ class _GridManager(object): ) storage_servers = dict() - for name, pubkey in config.get(u'storage_servers', {}).items(): - pubkey_bytes = base32.a2b(pubkey.encode('ascii')) + for name, srv_config in config.get(u'storage_servers', {}).items(): + if not 'public_key' in srv_config: + raise ValueError( + "No 'public_key' for storage server '{}'".format(name) + ) storage_servers[name] = _GridManagerStorageServer( - name, ed25519.VerifyingKey(pubkey_bytes), None + name, + keyutil.parse_pubkey(srv_config['public_key'].encode('ascii')), + None, ) gm_version = config.get(u'grid_manager_config_version', None) @@ -239,6 +261,17 @@ class _GridManager(object): self._storage_servers[name] = ss return ss + def remove_storage_server(self, name): + """ + :param name: a user-meaningful name for the server + """ + try: + del self._storage_servers[name] + except KeyError: + raise KeyError( + "No storage server called '{}'".format(name) + ) + def marshal(self): data = { u"grid_manager_config_version": self._version, @@ -328,6 +361,29 @@ def _add(gridoptions, options): return 0 +def _remove(gridoptions, options): + """ + Remove an existing storage-server by name from a Grid Manager + """ + gm_config = gridoptions['config'].strip() + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + gm = _load_gridmanager_config(gm_config) + + try: + gm.remove_storage_server(options['name']) + except KeyError: + raise usage.UsageError( + "No storage-server called '{}' exists".format(options['name']) + ) + cert_count = 0 + while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): + fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() + cert_count += 1 + + _save_gridmanager_config(fp, gm) + return 0 + + def _list(gridoptions, options): """ List all storage-servers known to a Grid Manager @@ -383,6 +439,7 @@ grid_manager_commands = { CreateOptions: _create, ShowIdentityOptions: _show_identity, AddOptions: _add, + RemoveOptions: _remove, ListOptions: _list, SignOptions: _sign, } From 64eb9d7c30f752a6b2e0f2205e676cc6645d726e Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 19 May 2018 19:17:47 -0600 Subject: [PATCH 010/272] hack in grid-manager announcements to storage-servers --- src/allmydata/client.py | 66 +++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e521f08bc..4702436ed 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,4 +1,9 @@ -import os, stat, time, weakref +import os +import stat +import time +import weakref +import json +from allmydata import node from base64 import urlsafe_b64encode from functools import partial from errno import ENOENT, EPERM @@ -564,23 +569,54 @@ class _Client(node.Node, pollmixin.PollMixin): sharetypes.append("mutable") expiration_sharetypes = tuple(sharetypes) - ss = StorageServer(storedir, self.nodeid, - reserved_space=reserved, - discard_storage=discard, - readonly_storage=readonly, - stats_provider=self.stats_provider, - expiration_enabled=expire, - expiration_mode=mode, - expiration_override_lease_duration=o_l_d, - expiration_cutoff_date=cutoff_date, - expiration_sharetypes=expiration_sharetypes) + ss = StorageServer( + storedir, self.nodeid, + reserved_space=reserved, + discard_storage=discard, + readonly_storage=readonly, + stats_provider=self.stats_provider, + expiration_enabled=expire, + expiration_mode=mode, + expiration_override_lease_duration=o_l_d, + expiration_cutoff_date=cutoff_date, + expiration_sharetypes=expiration_sharetypes, + ) ss.setServiceParent(self) - furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) + grid_manager_certificates = [] + cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") + for fname in cert_fnames.split(): + fname = abspath_expanduser_unicode(fname.decode('ascii'), base=self.basedir) + if not os.path.exists(fname): + raise ValueError( + "Grid Manager certificate file '{}' doesn't exist".format( + fname + ) + ) + with open(fname, 'r') as f: + cert = json.load(f) + if set(cert.keys()) != {"certificate", "signature"}: + raise ValueError( + "Unknown key in Grid Manager certificate '{}'".format( + fname + ) + ) + grid_manager_certificates.append(cert) + + # XXX we should probably verify that the certificates are + # valid and not expired, as that could be confusing for the + # storage-server operator -- but then we need the public key + # of the Grid Manager (should that go in the config too, + # then? How to handle multiple grid-managers?) + + + furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(ss, furlFile=furl_file) - ann = {"anonymous-storage-FURL": furl, - "permutation-seed-base32": self._init_permutation_seed(ss), - } + ann = { + "anonymous-storage-FURL": furl, + "permutation-seed-base32": self._init_permutation_seed(ss), + "grid-manager-certificates": grid_manager_certificates, + } for ic in self.introducer_clients: ic.publish("storage", ann, self._node_key) From ee3e1cbcf23a179d8d125a66d12dc7b382aa31d9 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 May 2018 10:42:34 -0600 Subject: [PATCH 011/272] wip; updates to grid-manager impl --- src/allmydata/client.py | 21 +++++++ src/allmydata/immutable/upload.py | 2 +- src/allmydata/storage_client.py | 96 ++++++++++++++++++++++++++++--- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 4702436ed..b9225e169 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -15,6 +15,7 @@ from twisted.application.internet import TimerService from twisted.python.filepath import FilePath from twisted.python.failure import Failure from pycryptopp.publickey import rsa +from pycryptopp.publickey import ed25519 import allmydata from allmydata.storage.server import StorageServer @@ -31,6 +32,7 @@ from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider +from allmydata.util.base32 import a2b, b2a from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import IStatsProducer, SDMF_VERSION, MDMF_VERSION, DEFAULT_MAX_SEGMENT_SIZE @@ -58,6 +60,7 @@ def _valid_config_sections(): "shares.needed", "shares.total", "stats_gatherer.furl", + "grid_managers", ), "drop_upload": ( # deprecated already? "enabled", @@ -380,10 +383,28 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con **kwargs ) + # grid manager setup + + grid_manager_keys = [] + gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') + for gm_key in gm_keydata.strip().split(): + grid_manager_keys.append( + keyutil.parse_pubkey(a2b(gm_key)) + ) + + my_pubkey = keyutil.parse_pubkey( + self.get_config_from_file("node.pubkey") + ) + + # create the actual storage-broker + sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, preferred_peers=preferred_peers, + grid_manager_keys=grid_manager_keys, + node_pubkey=my_pubkey, + ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 6720e4195..79c632c91 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -515,7 +515,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): # 0. Start with an ordered list of servers. Maybe *2N* of them. # - all_servers = storage_broker.get_servers_for_psi(storage_index) + all_servers = storage_broker.get_servers_for_psi(storage_index, for_upload=True) if not all_servers: raise NoServersError("client gave us zero servers") diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 65c65f535..34e96f38a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -29,7 +29,11 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . # 6: implement other sorts of IStorageClient classes: S3, etc -import re, time, hashlib +import re +import time +import hashlib +import json + from zope.interface import implementer from twisted.internet import defer from twisted.application import service @@ -67,12 +71,14 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=()): + def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[], node_pubkey=None): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker self.preferred_peers = preferred_peers + self._grid_manager_keys = grid_manager_keys + self._node_pubkey = node_pubkey # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its @@ -91,7 +97,7 @@ class StorageFarmBroker(service.MultiService): self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) s = NativeStorageServer(server_id, server["ann"], - self._tub_maker, handler_overrides) + self._tub_maker, handler_overrides, [], self._node_pubkey) s.on_status_changed(lambda _: self._got_connection()) s.setServiceParent(self) self.servers[server_id] = s @@ -110,7 +116,7 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}) + s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], self._node_pubkey) s.rref = rref s._is_connected = True self.servers[serverid] = s @@ -151,7 +157,7 @@ class StorageFarmBroker(service.MultiService): facility="tahoe.storage_broker", umid="AlxzqA", level=log.UNUSUAL) return - s = NativeStorageServer(server_id, ann, self._tub_maker, {}) + s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, self._node_pubkey) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() old = self.servers.get(server_id) @@ -192,11 +198,26 @@ class StorageFarmBroker(service.MultiService): for dsc in self.servers.values(): dsc.try_to_connect() - def get_servers_for_psi(self, peer_selection_index): + def get_servers_for_psi(self, peer_selection_index, for_upload=False): + """ + :param for_upload: used to determine if we should include any + servers that are invalid according to Grid Manager + processing. When for_upload is True and we have any Grid + Manager keys configured, any storage servers with invalid or + missing certificates will be excluded. + """ # return a list of server objects (IServers) assert self.permute_peers == True connected_servers = self.get_connected_servers() preferred_servers = frozenset(s for s in connected_servers if s.get_longname() in self.preferred_peers) + if for_upload: + print("upload processing: {}".format([srv.upload_permitted() for srv in connected_servers])) + connected_servers = [ + srv + for srv in connected_servers + if srv.upload_permitted() + ] + def _permuted(server): seed = server.get_permutation_seed() is_unpreferred = server not in preferred_servers @@ -248,6 +269,38 @@ class StubServer(object): def get_nickname(self): return "?" + +def _validate_grid_manager_certificate(gm_key, alleged_cert, storage_pubkey): + """ + :param gm_key: a VerifyingKey instance, a Grid Manager's public + key. + + :param cert: dict with "certificate" and "signature" keys, where + "certificate" contains a JSON-serialized certificate for a Storage + Server (comes from a Grid Manager). + + :return: False if the signature is invalid or the certificate is + expired. + """ + try: + gm_key.verify( + alleged_cert['signature'], + alleged_cert['certificate'], + ) + except Exception: + return False + # signature is valid; now we can load the actual data + cert = json.loads(alleged_cert['certificate']) + now = datetime.utcnow() + expires = datetime.fromordinal(cert['expires']) + cert_pubkey = keyutil.parse_pubkey(cert['public_key']) + if cert_pubkey != storage_pubkey: + return False # certificate is for wrong server + if expires < now: + return False # certificate is expired + return True + + @implementer(IServer) class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. @@ -276,13 +329,15 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, server_id, ann, tub_maker, handler_overrides): + def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, node_pubkey): service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id self.announcement = ann self._tub_maker = tub_maker self._handler_overrides = handler_overrides + self._node_pubkey = node_pubkey + self._grid_manager_keys = grid_manager_keys assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) @@ -321,6 +376,33 @@ class NativeStorageServer(service.MultiService): self._trigger_cb = None self._on_status_changed = ObserverList() + def upload_permitted(self): + """ + If our client is configured with Grid Manager public-keys, we will + only upload to storage servers that have a currently-valid + certificate signed by at least one of the Grid Managers we + accept. + + :return: True if we should use this server for uploads, False + otherwise. + """ + # if we have no Grid Manager keys configured, choice is easy + if not self._grid_manager_keys: + return True + + # XXX probably want to cache the answer to this? (ignoring + # that for now because certificates expire, so .. slightly + # more complex) + certificates = self.announcements.get("grid-manager-certificates", None) + if not certificates: + return False + for gm_key in self._grid_manager_keys: + for cert in certificates: + if _validate_grid_manager_certificate(gm_key, cert, self._pubkey): + return True + return False + + def on_status_changed(self, status_changed): """ :param status_changed: a callable taking a single arg (the From 3f1a8d64d8ba79e3164c9fa8bd91fc81bd374867 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 May 2018 10:44:25 -0600 Subject: [PATCH 012/272] reformatting/whitespace --- src/allmydata/introducer/client.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index a22c46103..4d41b5655 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -35,12 +35,13 @@ class IntroducerClient(service.Service, Referenceable): self._sequencer = sequencer self._cache_filepath = cache_filepath - self._my_subscriber_info = { "version": 0, - "nickname": self._nickname, - "app-versions": self._app_versions, - "my-version": self._my_version, - "oldest-supported": self._oldest_supported, - } + self._my_subscriber_info = { + "version": 0, + "nickname": self._nickname, + "app-versions": self._app_versions, + "my-version": self._my_version, + "oldest-supported": self._oldest_supported, + } self._outbound_announcements = {} # not signed self._published_announcements = {} # signed From 2031b723b8bf087ec31e980a5ae2ff78ce397b0d Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 May 2018 16:19:43 -0600 Subject: [PATCH 013/272] various fixes (flake8, errors) --- src/allmydata/client.py | 1 - src/allmydata/scripts/tahoe_grid_manager.py | 10 +--------- src/allmydata/storage_client.py | 2 ++ src/allmydata/test/no_network.py | 2 +- src/allmydata/test/test_checker.py | 2 +- src/allmydata/test/test_storage_client.py | 2 +- src/allmydata/test/test_util.py | 2 +- src/allmydata/test/web/test_root.py | 2 +- 8 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b9225e169..335d5b31d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -15,7 +15,6 @@ from twisted.application.internet import TimerService from twisted.python.filepath import FilePath from twisted.python.failure import Failure from pycryptopp.publickey import rsa -from pycryptopp.publickey import ed25519 import allmydata from allmydata.storage.server import StorageServer diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index d75b293cd..7cbfcd22d 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -9,17 +9,11 @@ from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code use from allmydata.scripts.common import BasedirOptions from allmydata.util.abbreviate import abbreviate_time -from twisted.scripts import twistd from twisted.python import usage -from twisted.python.reflect import namedAny from twisted.python.filepath import FilePath -from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util import fileutil from allmydata.util import base32 from allmydata.util import keyutil -from allmydata.node import read_config -from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path -from twisted.application.service import Service from twisted.internet.defer import inlineCallbacks, returnValue @@ -178,7 +172,7 @@ class _GridManager(object): if 'private_key' not in config: raise ValueError( "Grid Manager config from '{}' requires a 'private_key'".format( - config_config + config_location, ) ) @@ -406,7 +400,6 @@ def _list(gridoptions, options): else: print(" {}: expired ({})".format(cert_count, abbreviate_time(delta))) cert_count += 1 - certs = ' ({} certificates)'.format(cert_count) def _sign(gridoptions, options): @@ -428,7 +421,6 @@ def _sign(gridoptions, options): print(certificate_data) if fp is not None: next_serial = 0 - p = fp.child('{}.cert'.format(options['name'])).path while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists(): next_serial += 1 with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 34e96f38a..f0928d46b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,6 +33,7 @@ import re import time import hashlib import json +from datetime import datetime from zope.interface import implementer from twisted.internet import defer @@ -41,6 +42,7 @@ from twisted.application import service from foolscap.api import eventually from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer from allmydata.util import log, base32, connection_status +from allmydata.util import keyutil from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 1fcfab1f9..de15d32bf 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -171,7 +171,7 @@ class NoNetworkServer(object): @implementer(IStorageBroker) class NoNetworkStorageBroker(object): - def get_servers_for_psi(self, peer_selection_index): + def get_servers_for_psi(self, peer_selection_index, for_upload=True): def _permuted(server): seed = server.get_permutation_seed() return permute_server_hash(peer_selection_index, seed) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 46dec8c24..7f89e9e08 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -41,7 +41,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "my-version": "ver", "oldest-supported": "oldest", } - s = NativeStorageServer(server_id, ann, None, None) + s = NativeStorageServer(server_id, ann, None, None, {}, None) sb.test_add_server(server_id, s) c = FakeClient() c.storage_broker = sb diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index a78f91acb..6a4ae5d06 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -40,7 +40,7 @@ class TestNativeStorageServer(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - nss = NativeStorageServer("server_id", ann, None, {}) + nss = NativeStorageServer("server_id", ann, None, {}, {}, None) self.assertEqual(nss.get_nickname(), "") class TestStorageFarmBroker(unittest.TestCase): diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 381add043..76e3f8155 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -96,7 +96,7 @@ class HumanReadable(unittest.TestCase): def test_abbrev_time_future_5_minutes(self): diff = timedelta(minutes=-5) s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes in the future', s) + self.assertEqual('5 minutes from now', s) def test_abbrev_time_hours(self): diff = timedelta(hours=4) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 727189421..81b531113 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -26,7 +26,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - s = NativeStorageServer("server_id", ann, None, {}) + s = NativeStorageServer("server_id", ann, None, {}, {}, None) cs = ConnectionStatus(False, "summary", {}, 0, 0) s.get_connection_status = lambda: cs From 4afebbd88cfd302131097d7684a854dff7719165 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 31 Jul 2018 15:23:58 -0600 Subject: [PATCH 014/272] better docs and make load consistent with save internal-API --- src/allmydata/scripts/tahoe_grid_manager.py | 43 ++++++++++++++------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 7cbfcd22d..60d8b0f63 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -282,8 +282,12 @@ class _GridManager(object): def _save_gridmanager_config(file_path, grid_manager): """ - Writes a Grid Manager configuration to the place specified by - 'file_path' (if None, stdout is used). + Writes a Grid Manager configuration. + + :param file_path: a FilePath specifying where to write the config + (if None, stdout is used) + + :param grid_manager: a _GridManager instance """ data = json.dumps( grid_manager.marshal(), @@ -298,20 +302,33 @@ def _save_gridmanager_config(file_path, grid_manager): f.write("{}\n".format(data)) -# XXX should take a FilePath or None -def _load_gridmanager_config(gm_config): +def _config_to_filepath(gm_config_location): """ - Loads a Grid Manager configuration and returns it (a dict) after - validating. Exceptions if the config can't be found, or has - problems. + Converts a command-line string specifying the GridManager + configuration's location into a readable file-like object. + + :param gm_config_location str: a valid GridManager directory or + '-' (a single dash) to use stdin. """ fp = None if gm_config.strip() != '-': - fp = FilePath(gm_config.strip()) + fp = FilePath(gm_config_location.strip()) if not fp.exists(): raise RuntimeError( "No such directory '{}'".format(gm_config) ) + return fp + + +def _load_gridmanager_config(file_path) + """ + Loads a Grid Manager configuration and returns it (a dict) after + validating. Exceptions if the config can't be found, or has + problems. + + :param file_path: a FilePath to a vlid GridManager directory or + None to load from stdin. + """ if fp is None: gm = json.load(sys.stdin) @@ -329,7 +346,7 @@ def _show_identity(gridoptions, options): gm_config = gridoptions['config'].strip() assert gm_config is not None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) print(gm.public_identity()) @@ -340,7 +357,7 @@ def _add(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) try: gm.add_storage_server( options['name'], @@ -361,7 +378,7 @@ def _remove(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) try: gm.remove_storage_server(options['name']) @@ -385,7 +402,7 @@ def _list(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) for name in sorted(gm.storage_servers.keys()): print("{}: {}".format(name, gm.storage_servers[name].public_key())) if fp: @@ -408,7 +425,7 @@ def _sign(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) try: certificate = gm.sign(options['name']) From e5f608f80e1f99d79b0317954271d418c25546f9 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 13:09:39 -0600 Subject: [PATCH 015/272] fixes to Options use --- src/allmydata/client.py | 3 +- src/allmydata/scripts/tahoe_grid_manager.py | 77 ++++++++++++--------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 335d5b31d..478a3fc98 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -387,8 +387,9 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') for gm_key in gm_keydata.strip().split(): + # XXX FIXME this needs pub-v0- prefix then ... grid_manager_keys.append( - keyutil.parse_pubkey(a2b(gm_key)) + keyutil.parse_pubkey(gm_key) ) my_pubkey = keyutil.parse_pubkey( diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 60d8b0f63..5dd084c01 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -7,7 +7,7 @@ from datetime import datetime from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though -from allmydata.scripts.common import BasedirOptions +from allmydata.scripts.common import BaseOptions from allmydata.util.abbreviate import abbreviate_time from twisted.python import usage from twisted.python.filepath import FilePath @@ -17,13 +17,13 @@ from allmydata.util import keyutil from twisted.internet.defer import inlineCallbacks, returnValue -class CreateOptions(BasedirOptions): +class CreateOptions(BaseOptions): description = ( "Create a new identity key and configuration of a Grid Manager" ) -class ShowIdentityOptions(BasedirOptions): +class ShowIdentityOptions(BaseOptions): description = ( "Show the public identity key of a Grid Manager\n" "\n" @@ -32,13 +32,17 @@ class ShowIdentityOptions(BasedirOptions): ) -class AddOptions(BasedirOptions): +class AddOptions(BaseOptions): description = ( - "Add a new storage-server's key to a Grid Manager configuration" + "Add a new storage-server's key to a Grid Manager configuration\n" + "using NAME and PUBIC_KEY (comes from a node.pubkey file)" ) + def getSynopsis(self): + return "{} add NAME PUBLIC_KEY".format(BaseOptions.getSynopsis()) + def parseArgs(self, *args, **kw): - BasedirOptions.parseArgs(self, **kw) + BaseOptions.parseArgs(self, **kw) if len(args) != 2: raise usage.UsageError( "Requires two arguments: name public_key" @@ -53,13 +57,13 @@ class AddOptions(BasedirOptions): ) -class RemoveOptions(BasedirOptions): +class RemoveOptions(BaseOptions): description = ( "Remove a storage-server from a Grid Manager configuration" ) def parseArgs(self, *args, **kw): - BasedirOptions.parseArgs(self, **kw) + BaseOptions.parseArgs(self, **kw) if len(args) != 1: raise usage.UsageError( "Requires one arguments: name" @@ -67,19 +71,22 @@ class RemoveOptions(BasedirOptions): self['name'] = unicode(args[0]) -class ListOptions(BasedirOptions): +class ListOptions(BaseOptions): description = ( "List all storage servers in this Grid Manager" ) -class SignOptions(BasedirOptions): +class SignOptions(BaseOptions): description = ( "Create and sign a new certificate for a storage-server" ) + def getSynopsis(self): + return "{} NAME".format(super(SignOptions, self).getSynopsis()) + def parseArgs(self, *args, **kw): - BasedirOptions.parseArgs(self, **kw) + BaseOptions.parseArgs(self, **kw) if len(args) != 1: raise usage.UsageError( "Requires one argument: name" @@ -87,7 +94,7 @@ class SignOptions(BasedirOptions): self['name'] = unicode(args[0]) -class GridManagerOptions(BasedirOptions): +class GridManagerOptions(BaseOptions): subCommands = [ ["create", None, CreateOptions, "Create a Grid Manager."], ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], @@ -115,7 +122,8 @@ class GridManagerOptions(BasedirOptions): 'upload shares to" (X and Y are public-keys).' '\n\n' 'Clients can use Grid Managers to decide which storage servers to ' - 'upload shares to.' + 'upload shares to. They do this by adding one or more Grid Manager ' + 'public keys to their config.' ) @@ -169,6 +177,10 @@ class _GridManager(object): @staticmethod def from_config(config, config_location): + if not config: + raise ValueError( + "Invalid Grid Manager config in '{}'".format(config_location) + ) if 'private_key' not in config: raise ValueError( "Grid Manager config from '{}' requires a 'private_key'".format( @@ -307,28 +319,26 @@ def _config_to_filepath(gm_config_location): Converts a command-line string specifying the GridManager configuration's location into a readable file-like object. - :param gm_config_location str: a valid GridManager directory or - '-' (a single dash) to use stdin. + :param gm_config_location str: a valid path, or '-' (a single + dash) to use stdin. """ - fp = None - if gm_config.strip() != '-': - fp = FilePath(gm_config_location.strip()) - if not fp.exists(): - raise RuntimeError( - "No such directory '{}'".format(gm_config) - ) - return fp -def _load_gridmanager_config(file_path) +def _load_gridmanager_config(gm_config): """ Loads a Grid Manager configuration and returns it (a dict) after validating. Exceptions if the config can't be found, or has problems. - :param file_path: a FilePath to a vlid GridManager directory or - None to load from stdin. + :param gm_config str: "-" (a single dash) for stdin or a filename """ + fp = None + if gm_config.strip() != '-': + fp = FilePath(gm_config.strip()) + if not fp.exists(): + raise RuntimeError( + "No such directory '{}'".format(gm_config) + ) if fp is None: gm = json.load(sys.stdin) @@ -336,7 +346,10 @@ def _load_gridmanager_config(file_path) with fp.child("config.json").open("r") as f: gm = json.load(f) - return _GridManager.from_config(gm, gm_config) + try: + return _GridManager.from_config(gm, gm_config) + except ValueError as e: + raise usage.UsageError(str(e)) def _show_identity(gridoptions, options): @@ -346,7 +359,7 @@ def _show_identity(gridoptions, options): gm_config = gridoptions['config'].strip() assert gm_config is not None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) print(gm.public_identity()) @@ -357,7 +370,7 @@ def _add(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) try: gm.add_storage_server( options['name'], @@ -378,7 +391,7 @@ def _remove(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) try: gm.remove_storage_server(options['name']) @@ -402,7 +415,7 @@ def _list(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) for name in sorted(gm.storage_servers.keys()): print("{}: {}".format(name, gm.storage_servers[name].public_key())) if fp: @@ -425,7 +438,7 @@ def _sign(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) try: certificate = gm.sign(options['name']) From 2692c2caea89e786eaa7c3c937b47208bd8f2f36 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:08:22 -0600 Subject: [PATCH 016/272] verify certs properly --- src/allmydata/scripts/tahoe_grid_manager.py | 2 +- src/allmydata/storage_client.py | 46 +++++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 5dd084c01..91dc4f890 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -243,7 +243,7 @@ class _GridManager(object): "public_key": srv.public_key(), "version": 1, } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') sig = self._private_key.sign(cert_data) certificate = { u"certificate": cert_data, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f0928d46b..dcf92a61c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -38,8 +38,9 @@ from datetime import datetime from zope.interface import implementer from twisted.internet import defer from twisted.application import service - from foolscap.api import eventually +from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though + from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer from allmydata.util import log, base32, connection_status from allmydata.util import keyutil @@ -99,7 +100,7 @@ class StorageFarmBroker(service.MultiService): self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) s = NativeStorageServer(server_id, server["ann"], - self._tub_maker, handler_overrides, [], self._node_pubkey) + self._tub_maker, handler_overrides, []) s.on_status_changed(lambda _: self._got_connection()) s.setServiceParent(self) self.servers[server_id] = s @@ -118,7 +119,7 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], self._node_pubkey) + s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, []) s.rref = rref s._is_connected = True self.servers[serverid] = s @@ -159,7 +160,9 @@ class StorageFarmBroker(service.MultiService): facility="tahoe.storage_broker", umid="AlxzqA", level=log.UNUSUAL) return - s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, self._node_pubkey) + + grid_manager_certs = ann.get("grid-manager-certificates", []) + s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, grid_manager_certs) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() old = self.servers.get(server_id) @@ -272,7 +275,7 @@ class StubServer(object): return "?" -def _validate_grid_manager_certificate(gm_key, alleged_cert, storage_pubkey): +def _validate_grid_manager_certificate(gm_key, alleged_cert): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. @@ -286,18 +289,16 @@ def _validate_grid_manager_certificate(gm_key, alleged_cert, storage_pubkey): """ try: gm_key.verify( - alleged_cert['signature'], - alleged_cert['certificate'], + base32.a2b(alleged_cert['signature'].encode('ascii')), + alleged_cert['certificate'].encode('ascii'), ) - except Exception: + except ed25519.BadSignatureError: return False # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) now = datetime.utcnow() - expires = datetime.fromordinal(cert['expires']) - cert_pubkey = keyutil.parse_pubkey(cert['public_key']) - if cert_pubkey != storage_pubkey: - return False # certificate is for wrong server + expires = datetime.utcfromtimestamp(cert['expires']) + # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) if expires < now: return False # certificate is expired return True @@ -331,15 +332,25 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, node_pubkey): + def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id self.announcement = ann self._tub_maker = tub_maker self._handler_overrides = handler_overrides - self._node_pubkey = node_pubkey + + # XXX we should validate as much as we can about the + # certificates right now -- the only thing we HAVE to be lazy + # about is the expiry, which should be checked before any + # possible upload... + + # any public-keys which the user has configured (if none, it + # means use any storage servers) self._grid_manager_keys = grid_manager_keys + # any storage-certificates that this storage-server included + # in its announcement + self._grid_manager_certificates = grid_manager_certs assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) @@ -395,12 +406,11 @@ class NativeStorageServer(service.MultiService): # XXX probably want to cache the answer to this? (ignoring # that for now because certificates expire, so .. slightly # more complex) - certificates = self.announcements.get("grid-manager-certificates", None) - if not certificates: + if not self._grid_manager_certificates: return False for gm_key in self._grid_manager_keys: - for cert in certificates: - if _validate_grid_manager_certificate(gm_key, cert, self._pubkey): + for cert in self._grid_manager_certificates: + if _validate_grid_manager_certificate(gm_key, cert): return True return False From 36c471fdef25e67a9d0979bb87394ae6f0a5c742 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:13:30 -0600 Subject: [PATCH 017/272] don't need our pubkey for anything --- src/allmydata/client.py | 8 +++++--- src/allmydata/scripts/tahoe_grid_manager.py | 6 ++++++ src/allmydata/storage_client.py | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 478a3fc98..7d7723b6d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -392,9 +392,11 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con keyutil.parse_pubkey(gm_key) ) - my_pubkey = keyutil.parse_pubkey( - self.get_config_from_file("node.pubkey") - ) + # we don't actually use this keypair for anything (yet) as far + # as I can see. + # my_pubkey = keyutil.parse_pubkey( + # self.get_config_from_file("node.pubkey") + # ) # create the actual storage-broker diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 91dc4f890..01cb72356 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -249,6 +249,12 @@ class _GridManager(object): u"certificate": cert_data, u"signature": base32.b2a(sig), } + + if True: + verify_key_bytes = self._private_key.get_verifying_key_bytes() + vk = ed25519.VerifyingKey(verify_key_bytes) + assert vk.verify(sig, cert_data) is None, "cert should verify" + return certificate def add_storage_server(self, name, public_key): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index dcf92a61c..33c72e5e2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -74,14 +74,13 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[], node_pubkey=None): + def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[]): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker self.preferred_peers = preferred_peers self._grid_manager_keys = grid_manager_keys - self._node_pubkey = node_pubkey # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its From df88b9059f410a83e9e491d024d210a847ddd1c4 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:24:12 -0600 Subject: [PATCH 018/272] put 'header' on pubkey --- src/allmydata/scripts/tahoe_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 01cb72356..8e1d1093c 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -366,7 +366,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - print(gm.public_identity()) + print("pub-v0-{}".format(gm.public_identity())) def _add(gridoptions, options): From 62a6277429a51024d4002ba2a533654362832292 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:24:25 -0600 Subject: [PATCH 019/272] bring tutorial/doc in line with implementation --- docs/proposed/grid-manager/managed-grid.rst | 56 ++++++++++++--------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index d13cfef33..bcc7f7164 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -169,17 +169,17 @@ Example Setup of a New Managed Grid We'll store our Grid Manager configuration on disk, in ``~/grid-manager``. To initialize this directory:: - tahoe grid-manager create --config ~/grid-manager + tahoe grid-manager --config ~/grid-manager create This example creates an actual grid, but it's all just on one machine -with different "node directories". Usually of course each one would be -on a separate computer. +with different "node directories". Usually of course each storage +server would be on a separate computer. (If you already have a grid, you can :ref:`skip ahead `.) First of all, create an Introducer. Note that we actually have to run it briefly before it creates the "Introducer fURL" we want for the -next steps. +next steps:: tahoe create-introducer --listen=tcp --port=5555 --location=tcp:localhost:5555 ./introducer tahoe -d introducer run @@ -194,21 +194,26 @@ Next, we attach a couple of storage nodes:: .. _skip_ahead: -We can now ask the Grid Manager to create certificates for our new -storage servers:: +We can now tell the Grid Manager about our new storage servers:: - tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage0/node.pubkey) > storage0.cert - tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage1/node.pubkey) > storage1.cert + tahoe grid-manager --config ~/grid-manager add storage0 $(cat storage0/node.pubkey) + tahoe grid-manager --config ~/grid-manager add storage1 $(cat storage1/node.pubkey) - # enroll server0 (using file) - kill $(cat storage0/twistd.pid) - tahoe -d storage0 admin add-grid-manager-cert --filename storage0.cert - daemonize tahoe -d storage0 run - # enroll server1 (using stdin) - kill $(cat storage1/twistd.pid) - cat storage1.cert | tahoe -d storage1 admin add-grid-manager-cert - daemonize tahoe -d storage1 run +To produce a new certificate for each node, we do this:: + + tahoe grid-manager --config ~/grid-manager sign storage0 > ./storage0/gridmanager.cert + tahoe grid-manager --config ~/grid-manager sign storage1 > ./storage1/gridmanager.cert + +Now, we want our storage servers to actually announce these +certificates into the grid. We do this by adding some configuration +(in ``tahoe.cfg``):: + + [storage] + grid_manager_certificate_files = gridmanager.cert + +Add the above bit to each node's ``tahoe.cfg`` and re-start the +storage nodes. Now try adding a new storage server ``storage2``. This client can join the grid just fine, and announce itself to the Introducer as providing @@ -232,19 +237,24 @@ grid-manager has given certificates to (``storage0`` and ``storage1``). We need the grid-manager's public key to put in Alice's configuration:: - kill $(cat alice/twistd.pid) - tahoe -d alice add-grid-manager --name work-grid $(tahoe grid-manager --config ~/grid-manager show-identity) - daemonize tahoe -d alice start + tahoe grid-manager --config ~/grid-manager public-identity + +Put the key printed out above into Alice's ``tahoe.cfg`` in section +``client``:: + + [client] + grid_manager_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + DECIDE: - should the grid-manager be identified by a certificate? exarkun points out: --name seems like the hint of the beginning of a use-case for certificates rather than bare public keys?). -Since we made Alice's parameters require 3 storage servers to be -reachable (`--happy=3`), all their uploads should now fail (so `tahoe -mkdir` will fail) because they won't use storage2 and can't "achieve -happiness". +Now, re-start the "alice" client. Since we made Alice's parameters +require 3 storage servers to be reachable (``--happy=3``), all their +uploads should now fail (so ``tahoe mkdir`` will fail) because they +won't use storage2 and thus can't "achieve happiness". You can check Alice's "Welcome" page (where the list of connected servers is) at http://localhost:6301/ and should be able to see details about From 82a74e898645526212de921684d86605818b71a1 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 16 Aug 2018 17:26:49 -0600 Subject: [PATCH 020/272] updates and a bunch of debugging for cert/grid-manager handling --- src/allmydata/client.py | 2 +- src/allmydata/node.py | 13 +++++++++++++ src/allmydata/storage_client.py | 22 ++++++++++++++++++++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 7d7723b6d..c45a20698 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -386,7 +386,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') - for gm_key in gm_keydata.strip().split(): + for name, gm_key in self.config.enumerate_section('grid_managers').items(): # XXX FIXME this needs pub-v0- prefix then ... grid_manager_keys.append( keyutil.parse_pubkey(gm_key) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 426a0f796..c058fce4f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -287,6 +287,19 @@ class _Config(object): "Unable to write config file '{}'".format(fn), ) + def enumerate_section(self, section): + """ + returns a dict containing all items in a configuration section. an + empty dict is returned if the section doesn't exist. + """ + answer = dict() + try: + for k in self.config.options(section): + answer[k] = self.config.get(section, k) + except ConfigParser.NoSectionError: + pass + return answer + def get_config(self, section, option, default=_None, boolean=False): try: if boolean: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 33c72e5e2..1a928ea27 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -98,8 +98,15 @@ class StorageFarmBroker(service.MultiService): server_id = server_id.encode("ascii") self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) - s = NativeStorageServer(server_id, server["ann"], - self._tub_maker, handler_overrides, []) + s = NativeStorageServer( + server_id, + server["ann"], + self._tub_maker, + handler_overrides, + self._grid_manager_keys, + self._grid_manager_certificates, + ) + print("SET STATIC {}".format(s)) s.on_status_changed(lambda _: self._got_connection()) s.setServiceParent(self) self.servers[server_id] = s @@ -161,6 +168,7 @@ class StorageFarmBroker(service.MultiService): return grid_manager_certs = ann.get("grid-manager-certificates", []) + print("certs for {}: {}".format(key_s, grid_manager_certs)) s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, grid_manager_certs) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() @@ -332,6 +340,7 @@ class NativeStorageServer(service.MultiService): } def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): + print("CREATE {}: {}".format(server_id, grid_manager_certs)) service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id @@ -347,9 +356,11 @@ class NativeStorageServer(service.MultiService): # any public-keys which the user has configured (if none, it # means use any storage servers) self._grid_manager_keys = grid_manager_keys + print("keys: {}".format(self._grid_manager_keys)) # any storage-certificates that this storage-server included # in its announcement self._grid_manager_certificates = grid_manager_certs + print("certs: {}".format(self._grid_manager_certificates)) assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) @@ -398,19 +409,26 @@ class NativeStorageServer(service.MultiService): :return: True if we should use this server for uploads, False otherwise. """ + print("upload permitted? {}".format(self._server_id)) # if we have no Grid Manager keys configured, choice is easy if not self._grid_manager_keys: + print("{} no grid manager keys at all (so yes)".format(self._server_id)) return True # XXX probably want to cache the answer to this? (ignoring # that for now because certificates expire, so .. slightly # more complex) if not self._grid_manager_certificates: + print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) return False for gm_key in self._grid_manager_keys: for cert in self._grid_manager_certificates: if _validate_grid_manager_certificate(gm_key, cert): + print("valid: {}\n{}".format(gm_key, cert)) return True + else: + print("invalid: {}\n{}".format(gm_key, cert)) + print("didn't validate {} keys".format(len(self._grid_manager_keys))) return False From 5deb31ea2a0a698c4f838c993cca69e46aa079be Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:04:48 -0600 Subject: [PATCH 021/272] different directory for example grid-manager --- docs/proposed/grid-manager/managed-grid.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index bcc7f7164..5df37aff1 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -167,9 +167,9 @@ Example Setup of a New Managed Grid ----------------------------------- We'll store our Grid Manager configuration on disk, in -``~/grid-manager``. To initialize this directory:: +``./gm0``. To initialize this directory:: - tahoe grid-manager --config ~/grid-manager create + tahoe grid-manager --config ./gm0 create This example creates an actual grid, but it's all just on one machine with different "node directories". Usually of course each storage @@ -196,14 +196,13 @@ Next, we attach a couple of storage nodes:: We can now tell the Grid Manager about our new storage servers:: - tahoe grid-manager --config ~/grid-manager add storage0 $(cat storage0/node.pubkey) - tahoe grid-manager --config ~/grid-manager add storage1 $(cat storage1/node.pubkey) - + tahoe grid-manager --config ./gm0 add storage0 $(cat storage0/node.pubkey) + tahoe grid-manager --config ./gm0 add storage1 $(cat storage1/node.pubkey) To produce a new certificate for each node, we do this:: - tahoe grid-manager --config ~/grid-manager sign storage0 > ./storage0/gridmanager.cert - tahoe grid-manager --config ~/grid-manager sign storage1 > ./storage1/gridmanager.cert + tahoe grid-manager --config ./gm0 sign storage0 > ./storage0/gridmanager.cert + tahoe grid-manager --config ./gm0 sign storage1 > ./storage1/gridmanager.cert Now, we want our storage servers to actually announce these certificates into the grid. We do this by adding some configuration From a884c75ef6aac29036b1d487cdb9605403336037 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:04:58 -0600 Subject: [PATCH 022/272] proviso --- docs/proposed/grid-manager/managed-grid.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5df37aff1..9f392f292 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -236,19 +236,23 @@ grid-manager has given certificates to (``storage0`` and ``storage1``). We need the grid-manager's public key to put in Alice's configuration:: - tahoe grid-manager --config ~/grid-manager public-identity + tahoe grid-manager --config ./gm0 public-identity Put the key printed out above into Alice's ``tahoe.cfg`` in section ``client``:: [client] - grid_manager_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + grid_manager_public_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq DECIDE: - should the grid-manager be identified by a certificate? exarkun points out: --name seems like the hint of the beginning of a use-case for certificates rather than bare public keys?). + - (note the "--name" thing came from a former version of this + proposal that used CLI commands to add the public-keys -- but the + point remains, if there's to be metadata associated with "grid + managers" maybe they should be certificates..) Now, re-start the "alice" client. Since we made Alice's parameters require 3 storage servers to be reachable (``--happy=3``), all their From e453db879a46905e3c6cad90ef0dc8fce82a7fa0 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:06:03 -0600 Subject: [PATCH 023/272] make options work, config API usage --- src/allmydata/client.py | 17 ++++++++--------- src/allmydata/util/configutil.py | 17 +++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index c45a20698..d504f5fbe 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -59,8 +59,8 @@ def _valid_config_sections(): "shares.needed", "shares.total", "stats_gatherer.furl", - "grid_managers", ), + "grid_managers": None, # means "any options valid" "drop_upload": ( # deprecated already? "enabled", ), @@ -83,6 +83,7 @@ def _valid_config_sections(): "readonly", "reserved_space", "storage_dir", + "grid_manager_certificate_files", ), "sftpd": ( "accounts.file", @@ -385,9 +386,9 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con # grid manager setup grid_manager_keys = [] - gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') - for name, gm_key in self.config.enumerate_section('grid_managers').items(): + for name, gm_key in config.enumerate_section('grid_managers').items(): # XXX FIXME this needs pub-v0- prefix then ... + print("KEY: {}".format(gm_key)) grid_manager_keys.append( keyutil.parse_pubkey(gm_key) ) @@ -399,14 +400,13 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con # ) # create the actual storage-broker - + sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, preferred_peers=preferred_peers, grid_manager_keys=grid_manager_keys, - node_pubkey=my_pubkey, - +## node_pubkey=my_pubkey, ) for ic in introducer_clients: sb.use_introducer(ic) @@ -609,7 +609,7 @@ class _Client(node.Node, pollmixin.PollMixin): grid_manager_certificates = [] cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") for fname in cert_fnames.split(): - fname = abspath_expanduser_unicode(fname.decode('ascii'), base=self.basedir) + fname = self.config.get_config_path(fname.decode('ascii')) if not os.path.exists(fname): raise ValueError( "Grid Manager certificate file '{}' doesn't exist".format( @@ -632,8 +632,7 @@ class _Client(node.Node, pollmixin.PollMixin): # of the Grid Manager (should that go in the config too, # then? How to handle multiple grid-managers?) - - furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding()) + furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(ss, furlFile=furl_file) ann = { "anonymous-storage-FURL": furl, diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 78894e301..0e22767cb 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -51,12 +51,13 @@ def validate_config(fname, cfg, valid_sections): section=section, ) ) - for option in cfg.options(section): - if option not in valid_in_section: - raise UnknownConfigError( - "'{fname}' section [{section}] contains unknown option '{option}'".format( - fname=fname, - section=section, - option=option, + if valid_in_section is not None: + for option in cfg.options(section): + if option not in valid_in_section: + raise UnknownConfigError( + "'{fname}' section [{section}] contains unknown option '{option}'".format( + fname=fname, + section=section, + option=option, + ) ) - ) From 41233284047602d5033b41cc61e2711d1edbc817 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:07:57 -0600 Subject: [PATCH 024/272] for mutable publishes, ignore servers w/o valid grid-manager certiicates --- src/allmydata/mutable/publish.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 629d06bd5..8810bdf14 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -945,6 +945,17 @@ class Publish: serverid = server.get_serverid() if server in self.bad_servers: continue + # if we have >= 1 grid-managers, this checks that we have + # a valid certificate for this server + if not server.upload_permitted(): + self.log( + "No valid grid-manager certificates for '{}' while choosing slots for mutable".format( + server.get_serverid(), + ), + level=log.UNUSUAL, + ) + continue + entry = (len(old_assignments.get(server, [])), i, serverid, server) serverlist.append(entry) serverlist.sort() From 026c7735b6ef303c0140bf11a9e8e48d541e1a63 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 14:45:01 -0700 Subject: [PATCH 025/272] configuration is JSON --- docs/proposed/grid-manager/managed-grid.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 9f392f292..602d93611 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -60,16 +60,12 @@ at which time you may be able to pass a directory-capability to this option). If you don't want to store the configuration on disk at all, you may -use ``--config -`` (that's a dash) and write a valid JSON (YAML? I'm -guessing JSON is easier to deal with here, more-interoperable?) +use ``--config -`` (that's a dash) and write a valid JSON configuration to stdin. All commands take the ``--config`` option, and they all behave similarly for "data from stdin" versus "data from disk". -DECIDE: - - config is YAML or JSON? - tahoe grid-manager create ````````````````````````` From 02dfb51dba9333c0af5f22b8cba891ae2488e723 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 14:46:28 -0700 Subject: [PATCH 026/272] spelling --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 602d93611..4cec4a040 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -111,7 +111,7 @@ Lists all storage-servers that have previously been added using tahoe grid-manager sign ``````````````````````` -Takes one arg: ``name``, the petname used previous in a ``tahoe +Takes one arg: ``name``, the petname used previously in a ``tahoe grid-manager add`` command. Note that this mutates the state of the grid-manager if it is on disk, From 2415058dd5d39180ec761c65d666c4dd1917ac46 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 14:54:05 -0700 Subject: [PATCH 027/272] more-clear output from 'tahoe grid-manager lsit' --- src/allmydata/scripts/tahoe_grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 8e1d1093c..ee6080174 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -432,9 +432,9 @@ def _list(gridoptions, options): expires = datetime.fromtimestamp(cert_data['expires']) delta = datetime.utcnow() - expires if delta.total_seconds() < 0: - print(" {}: valid until {} ({})".format(cert_count, expires, abbreviate_time(delta))) + print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta))) else: - print(" {}: expired ({})".format(cert_count, abbreviate_time(delta))) + print("{}: cert {}: expired ({})".format(name, cert_count, abbreviate_time(delta))) cert_count += 1 From 71f96c9c9e340f7dcf68a0c95525c3cf2872c104 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 15:01:01 -0700 Subject: [PATCH 028/272] call super properly; docstring --- src/allmydata/scripts/tahoe_grid_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index ee6080174..f2c3aace6 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -39,7 +39,7 @@ class AddOptions(BaseOptions): ) def getSynopsis(self): - return "{} add NAME PUBLIC_KEY".format(BaseOptions.getSynopsis()) + return "{} add NAME PUBLIC_KEY".format(super(AddOptions, self).getSynopsis()) def parseArgs(self, *args, **kw): BaseOptions.parseArgs(self, **kw) @@ -128,7 +128,13 @@ class GridManagerOptions(BaseOptions): def _create_gridmanager(): - return _GridManager(ed25519.SigningKey(os.urandom(32)), {}) + """ + :return: an object providing the GridManager interface initialized + with a new random keypair + """ + private_key_bytes, public_key_bytes = keyutil.make_keypair() + secret_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) + return _GridManager(secret_key, {}) def _create(gridoptions, options): """ From 03258dcb9f4ce75979cca5c39d1368344b22b094 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 19:45:50 -0700 Subject: [PATCH 029/272] use keyutil --- src/allmydata/scripts/tahoe_grid_manager.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f2c3aace6..6fbdd4f7b 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -133,8 +133,7 @@ def _create_gridmanager(): with a new random keypair """ private_key_bytes, public_key_bytes = keyutil.make_keypair() - secret_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) - return _GridManager(secret_key, {}) + return _GridManager(private_key_bytes, {}) def _create(gridoptions, options): """ @@ -194,10 +193,9 @@ class _GridManager(object): ) ) - private_key_str = config['private_key'] + private_key_bytes = config['private_key'].encode('ascii') try: - private_key_bytes = base32.a2b(private_key_str.encode('ascii')) - private_key = ed25519.SigningKey(private_key_bytes) + private_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) except Exception as e: raise ValueError( "Invalid Grid Manager private_key: {}".format(e) @@ -222,11 +220,12 @@ class _GridManager(object): gm_version ) ) - return _GridManager(private_key, storage_servers) + return _GridManager(private_key_bytes, storage_servers) - def __init__(self, private_key, storage_servers): + def __init__(self, private_key_bytes, storage_servers): self._storage_servers = dict() if storage_servers is None else storage_servers - self._private_key = private_key + self._private_key_bytes = private_key_bytes + self._private_key, _ = keyutil.parse_privkey(self._private_key_bytes) self._version = 0 @property @@ -293,7 +292,7 @@ class _GridManager(object): def marshal(self): data = { u"grid_manager_config_version": self._version, - u"private_key": base32.b2a(self._private_key.sk_and_vk[:32]), + u"private_key": self._private_key_bytes.decode('ascii'), } if self._storage_servers: data[u"storage_servers"] = { From 4f481bbb4c963b44d99852e71c6db6fa0ee5b766 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 22:54:03 -0700 Subject: [PATCH 030/272] remove FIXME --- src/allmydata/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index d504f5fbe..d30b85694 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -387,8 +387,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] for name, gm_key in config.enumerate_section('grid_managers').items(): - # XXX FIXME this needs pub-v0- prefix then ... - print("KEY: {}".format(gm_key)) grid_manager_keys.append( keyutil.parse_pubkey(gm_key) ) From 43b446bacf53c953ba0ad6690e338395971c806e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 23:40:06 -0700 Subject: [PATCH 031/272] CLI vs. 'edit config' for server, client enrollment --- docs/proposed/grid-manager/managed-grid.rst | 56 +++++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 4cec4a040..3f2053814 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -126,8 +126,15 @@ printed to stdout. If you stored the config on disk, the new certificate will (also) be in a file named like ``alice.cert.0``. -Enrolling a Storage Server --------------------------- +Enrolling a Storage Server: CLI +------------------------------- + +DECIDE: is a command like this best, or should you have to edit the + config "by hand"? (below fits into warner's philosophy that "at some + point" it might be best to have all config in a database or similar + and the only way to view/edit it is via tahoe commands...) + if command: write it + if not: delete this section tahoe admin add-grid-manager-cert ````````````````````````````````` @@ -142,8 +149,31 @@ your node after this. Subsequent announcements to the Introducer will include this certificate. -Enrolling a Client ------------------- +Enrolling a Storage Server: Config +---------------------------------- + +You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to +include an entry ``grid_manager_certificate_files = `` whose value is +a space-separated list of paths to valid certificate files. These +certificate files are issued by the ``tahoe grid-manager sign`` +command; these should be securely transmitted to the storage +server. Relative paths are relative to the node directory. Example:: + + [storage] + grid_manager_certificate_files = example_grid.cert + +This will cause us to give this certificate to any Introducers we +connect to (and subsequently, the Introducer will give the certificate +out to clients). + + +Enrolling a Client: CLI +----------------------- + +DECIDE: is a command like this best, or should you have to edit the + config "by hand"? (below fits into warner's philosophy that "at some + point" it might be best to have all config in a database or similar + and the only way to view/edit it is via tahoe commands...) tahoe add-grid-manager `````````````````````` @@ -159,6 +189,24 @@ key of the Grid Manager. The client will have to be re-started once this change is made. +Enrolling a Client: Config +-------------------------- + +You may instruct a Tahoe client to use only storage servers from given +Grid Managers. If there are no such keys, any servers are used. If +there are one or more keys, the client will only upload to a storage +server that has a valid certificate (from any of the keys). + +To specify public-keys, add a ``[grid_managers]`` section to the +config. This consists of ``name = value`` pairs where ``name`` is an +arbitrary name and ``value`` is a public-key of a Grid +Manager. Example:: + + [grid_managers] + example_grid = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + + + Example Setup of a New Managed Grid ----------------------------------- From 85142acf97b2ffb957fd3cff93d326778e0d4ae3 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:06:53 -0700 Subject: [PATCH 032/272] add some integration tests --- integration/test_grid_manager.py | 62 ++++++++++++++++++++++++++++++++ integration/util.py | 21 ++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 integration/test_grid_manager.py diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py new file mode 100644 index 000000000..1a6e86266 --- /dev/null +++ b/integration/test_grid_manager.py @@ -0,0 +1,62 @@ +import sys +import time +import json +import shutil +from os import mkdir, unlink, listdir, utime +from os.path import join, exists, getmtime + +from allmydata.util import keyutil +from allmydata.util import base32 + +import util + +import pytest + + +@pytest.inlineCallbacks +def test_create_certificate(reactor): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) + pubkey = keyutil.parse_pubkey(pubkey_bytes) + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdin=gm_config, + ) + alice_cert_bytes = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "sign", "alice", + stdin=gm_config, + ) + alice_cert = json.loads(alice_cert_bytes) + + # confirm that alice's certificate is made by the Grid Manager + assert pubkey.verify( + base32.a2b(alice_cert['signature'].encode('ascii')), + alice_cert['certificate'].encode('ascii'), + ) + + +@pytest.inlineCallbacks +def test_remove_client(reactor): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdin=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].has_key("alice") + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "remove", + "alice", + stdin=gm_config, + ) + # there are no storage servers left at all now + assert not json.loads(gm_config).has_key('storage_servers') diff --git a/integration/util.py b/integration/util.py index 4dd20aa49..e2a198d34 100644 --- a/integration/util.py +++ b/integration/util.py @@ -37,9 +37,15 @@ class _CollectOutputProtocol(ProcessProtocol): self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self): + def __init__(self, stdin=None): self.done = Deferred() self.output = StringIO() + self._stdin = stdin + + def connectionMade(self): + if self._stdin is not None: + self.transport.write(self._stdin) + self.transport.closeStdin() def processEnded(self, reason): if not self.done.called: @@ -126,6 +132,19 @@ def _cleanup_twistd_process(twistd_process, exited): pass +def run_tahoe(reactor, *args, **kwargs): + stdin = kwargs.get('stdin', None) + protocol = _CollectOutputProtocol(stdin=stdin) + process = reactor.spawnProcess( + protocol, + sys.executable, + (sys.executable, '-m', 'allmydata.scripts.runner') + args + ) + process.exited = protocol.done + + return protocol.done + + def _run_node(reactor, node_dir, request, magic_text): if magic_text is None: magic_text = "client running" From c27964813f699730390ae9746cd674d850890654 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:06:58 -0700 Subject: [PATCH 033/272] fix a bug --- src/allmydata/scripts/tahoe_grid_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 6fbdd4f7b..f7193cbf2 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -411,9 +411,10 @@ def _remove(gridoptions, options): "No storage-server called '{}' exists".format(options['name']) ) cert_count = 0 - while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): - fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() - cert_count += 1 + if fp is not None: + while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): + fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() + cert_count += 1 _save_gridmanager_config(fp, gm) return 0 From 992314471e1c205bb3a9ec54ae9d6099c104c427 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:20:51 -0700 Subject: [PATCH 034/272] more integration tests --- integration/test_grid_manager.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 1a6e86266..103c3c897 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -46,6 +46,35 @@ def test_remove_client(reactor): reactor, "grid-manager", "--config", "-", "create", ) + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdin=gm_config, + ) + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "bob", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + stdin=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].has_key("alice") + assert json.loads(gm_config)['storage_servers'].has_key("bob") + return + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "remove", + "alice", + stdin=gm_config, + ) + assert not json.loads(gm_config)['storage_servers'].has_key('alice') + assert json.loads(gm_config)['storage_servers'].has_key('bob') + + +@pytest.inlineCallbacks +def test_remove_last_client(reactor): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", From 0432b7d42f3a280bfe7108a87c938ba4843c0f81 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:22:49 -0700 Subject: [PATCH 035/272] fix assert --- integration/test_grid_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 103c3c897..886bf76e9 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -34,7 +34,8 @@ def test_create_certificate(reactor): alice_cert = json.loads(alice_cert_bytes) # confirm that alice's certificate is made by the Grid Manager - assert pubkey.verify( + # (.verify returns None on success, raises exception on error) + pubkey.verify( base32.a2b(alice_cert['signature'].encode('ascii')), alice_cert['certificate'].encode('ascii'), ) From 7f195c9c9a9b20b95e5f94267c83645de871a2e5 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 02:54:49 -0700 Subject: [PATCH 036/272] test of certificates on not-enough-servers causing uploads to fail --- integration/test_grid_manager.py | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 886bf76e9..39e2fbe9b 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -7,6 +7,7 @@ from os.path import join, exists, getmtime from allmydata.util import keyutil from allmydata.util import base32 +from allmydata.util import configutil import util @@ -90,3 +91,81 @@ def test_remove_last_client(reactor): ) # there are no storage servers left at all now assert not json.loads(gm_config).has_key('storage_servers') + + +@pytest.inlineCallbacks +def test_reject_storage_server(reactor, request, alice, storage_nodes): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) + pubkey = keyutil.parse_pubkey(pubkey_bytes) + + # create certificates for first 2 storage-servers + for idx, storage in enumerate(storage_nodes[:2]): + pubkey_fname = join(storage._node_dir, "node.pubkey") + with open(pubkey_fname, 'r') as f: + pubkey = f.read().strip() + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "storage{}".format(idx), pubkey, + stdin=gm_config, + ) + assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] + + print("inserting certificates") + # insert their certificates + for idx, storage in enumerate(storage_nodes[:2]): + print(idx, storage) + cert = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "sign", + "storage{}".format(idx), + stdin=gm_config, + ) + with open(join(storage._node_dir, "gridmanager.cert"), "w") as f: + f.write(cert) + config = configutil.get_config(join(storage._node_dir, "tahoe.cfg")) + config.set("storage", "grid_manager_certificate_files", "gridmanager.cert") + config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) + + # re-start this storage server + storage.signalProcess('TERM') + yield storage._protocol.exited + time.sleep(1) + storage_nodes[idx] = yield util._run_node( + reactor, storage._node_dir, request, None, + ) + + # now only two storage-servers have certificates .. configure + # alice to have the grid-manager certificate + + config = configutil.get_config(join(alice._node_dir, "tahoe.cfg")) + print(dir(config)) + config.add_section("grid_managers") + config.set("grid_managers", "test", pubkey_bytes) + config.write(open(join(alice._node_dir, "tahoe.cfg"), "w")) + alice.signalProcess('TERM') + yield alice._protocol.exited + time.sleep(1) + alice = yield util._run_node( + reactor, alice._node_dir, request, None, + ) + time.sleep(5) + + # try to put something into the grid, which should fail (because + # alice has happy=3 but should only find storage0, storage1 to be + # acceptable to upload to) + + try: + yield util.run_tahoe( + reactor, "--node-directory", alice._node_dir, + "put", "-", + stdin="some content" * 200, + ) + assert False, "Should get a failure" + except Exception as e: + # XXX but how to tell if we failed for the right reason? + # (should get UploadUnhappinessError) + print("failure expected: {}".format(e)) From 3c679e92027a063d08f4213fdfb8ba867aa5ea77 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Dec 2018 15:57:21 -0700 Subject: [PATCH 037/272] tweak tests/error output --- integration/test_grid_manager.py | 7 ++++--- integration/util.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 39e2fbe9b..33710a457 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -166,6 +166,7 @@ def test_reject_storage_server(reactor, request, alice, storage_nodes): ) assert False, "Should get a failure" except Exception as e: - # XXX but how to tell if we failed for the right reason? - # (should get UploadUnhappinessError) - print("failure expected: {}".format(e)) + # depending on the full output being in the error-message + # here; see util.py + assert 'UploadUnhappinessError' in str(e) + print("found expected UploadUnhappinessError") diff --git a/integration/util.py b/integration/util.py index e2a198d34..c5bc04ab2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -53,7 +53,13 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(reason) + #self.done.errback(reason) + self.done.errback(RuntimeError( + "Process failed: {}\nOutput:\n{}".format( + reason, + self.output.getvalue(), + ) + )) def outReceived(self, data): self.output.write(data) From 7b5783536b9248e7adc17d023fb23074d1a62516 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Dec 2018 16:01:06 -0700 Subject: [PATCH 038/272] remove prints --- src/allmydata/storage_client.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1a928ea27..2a09efa8f 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -223,7 +223,7 @@ class StorageFarmBroker(service.MultiService): connected_servers = self.get_connected_servers() preferred_servers = frozenset(s for s in connected_servers if s.get_longname() in self.preferred_peers) if for_upload: - print("upload processing: {}".format([srv.upload_permitted() for srv in connected_servers])) + # print("upload processing: {}".format([srv.upload_permitted() for srv in connected_servers])) connected_servers = [ srv for srv in connected_servers @@ -409,26 +409,24 @@ class NativeStorageServer(service.MultiService): :return: True if we should use this server for uploads, False otherwise. """ - print("upload permitted? {}".format(self._server_id)) + # print("upload permitted? {}".format(self._server_id)) # if we have no Grid Manager keys configured, choice is easy if not self._grid_manager_keys: - print("{} no grid manager keys at all (so yes)".format(self._server_id)) + # print("{} no grid manager keys at all (so yes)".format(self._server_id)) return True # XXX probably want to cache the answer to this? (ignoring # that for now because certificates expire, so .. slightly # more complex) if not self._grid_manager_certificates: - print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) + # print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) return False for gm_key in self._grid_manager_keys: for cert in self._grid_manager_certificates: if _validate_grid_manager_certificate(gm_key, cert): - print("valid: {}\n{}".format(gm_key, cert)) + # print("valid: {}\n{}".format(gm_key, cert)) return True - else: - print("invalid: {}\n{}".format(gm_key, cert)) - print("didn't validate {} keys".format(len(self._grid_manager_keys))) + # print("didn't validate {} keys".format(len(self._grid_manager_keys))) return False From 75fee995b6777bec17e79e16c65ac63bc92693f0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 12:48:37 -0700 Subject: [PATCH 039/272] some tweaks and fixes for integration tests --- integration/test_grid_manager.py | 27 +++++++++++++++++------- integration/test_servers_of_happiness.py | 2 +- integration/test_tor.py | 3 +++ integration/util.py | 4 ++++ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 33710a457..f68f97941 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -94,7 +94,7 @@ def test_remove_last_client(reactor): @pytest.inlineCallbacks -def test_reject_storage_server(reactor, request, alice, storage_nodes): +def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) @@ -115,6 +115,15 @@ def test_reject_storage_server(reactor, request, alice, storage_nodes): ) assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] + + # XXX FIXME need to shut-down and nuke carol when we're done this + # test (i.d. request.addfinalizer) + carol = yield util._create_node( + reactor, request, temp_dir, introducer_furl, flog_gatherer, "carol", + web_port="tcp:9982:interface=localhost", + storage=False, + ) + print("inserting certificates") # insert their certificates for idx, storage in enumerate(storage_nodes[:2]): @@ -141,26 +150,28 @@ def test_reject_storage_server(reactor, request, alice, storage_nodes): # now only two storage-servers have certificates .. configure # alice to have the grid-manager certificate - config = configutil.get_config(join(alice._node_dir, "tahoe.cfg")) + # XXX FIXME remove this cert when test ends (fail or not!) + + config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) print(dir(config)) config.add_section("grid_managers") config.set("grid_managers", "test", pubkey_bytes) - config.write(open(join(alice._node_dir, "tahoe.cfg"), "w")) - alice.signalProcess('TERM') - yield alice._protocol.exited + config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) + carol.signalProcess('TERM') + yield carol._protocol.exited time.sleep(1) alice = yield util._run_node( - reactor, alice._node_dir, request, None, + reactor, carol._node_dir, request, None, ) time.sleep(5) # try to put something into the grid, which should fail (because - # alice has happy=3 but should only find storage0, storage1 to be + # carol has happy=3 but should only find storage0, storage1 to be # acceptable to upload to) try: yield util.run_tahoe( - reactor, "--node-directory", alice._node_dir, + reactor, "--node-directory", carol._node_dir, "put", "-", stdin="some content" * 200, ) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 1f84094e2..1984dd507 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -43,7 +43,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto yield proto.done assert False, "should raise exception" except Exception as e: - assert isinstance(e, ProcessTerminated) + assert "UploadUnhappinessError" in str(e) output = proto.output.getvalue() assert "shares could be placed on only" in output diff --git a/integration/test_tor.py b/integration/test_tor.py index 187754f08..e0b991128 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -68,6 +68,9 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: + if exists(node_dir): + print("nuking '{}'".format(node_dir)) + shutil.rmtree(node_dir) print("creating", node_dir) mkdir(node_dir) proto = util._DumpOutputProtocol(None) diff --git a/integration/util.py b/integration/util.py index c5bc04ab2..703402aa8 100644 --- a/integration/util.py +++ b/integration/util.py @@ -4,6 +4,7 @@ from os import mkdir from os.path import exists, join from six.moves import StringIO from functools import partial +from shutil import rmtree from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol @@ -200,6 +201,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: + if exists(node_dir): + print("nuking: {}".format(node_dir)) + rmtree(node_dir) print("creating", node_dir) mkdir(node_dir) done_proto = _ProcessExitedProtocol() From fea86d7e95e692d9b8c52c7fd7ed0c5c1dee54a3 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 12:54:22 -0700 Subject: [PATCH 040/272] alice->carol --- integration/test_grid_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index f68f97941..3553ccc23 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -148,9 +148,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd ) # now only two storage-servers have certificates .. configure - # alice to have the grid-manager certificate - - # XXX FIXME remove this cert when test ends (fail or not!) + # carol to have the grid-manager certificate config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) print(dir(config)) @@ -160,7 +158,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd carol.signalProcess('TERM') yield carol._protocol.exited time.sleep(1) - alice = yield util._run_node( + carol = yield util._run_node( reactor, carol._node_dir, request, None, ) time.sleep(5) From 39a80f7939359ea1bc4bfe9ef90eaf0cc3c092e0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 13:40:58 -0700 Subject: [PATCH 041/272] utest fixes --- src/allmydata/storage_client.py | 2 +- src/allmydata/test/no_network.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2a09efa8f..75a08866c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -125,7 +125,7 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, []) + s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], []) s.rref = rref s._is_connected = True self.servers[serverid] = s diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index de15d32bf..fb1da0635 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -149,6 +149,10 @@ class NoNetworkServer(object): return self def __deepcopy__(self, memodict): return self + + def upload_permitted(self): + return True + def get_serverid(self): return self.serverid def get_permutation_seed(self): From d0791eb2b5303da2be9f6008f92abe77dadf37e9 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 18:45:42 -0700 Subject: [PATCH 042/272] fix storage-client --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 75a08866c..34b2c0caf 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -104,7 +104,7 @@ class StorageFarmBroker(service.MultiService): self._tub_maker, handler_overrides, self._grid_manager_keys, - self._grid_manager_certificates, + [], # XXX FIXME? need grid_manager_certs too? ) print("SET STATIC {}".format(s)) s.on_status_changed(lambda _: self._got_connection()) From 633f36a33cc1468b8f268d1062208877149b6721 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Dec 2018 19:37:43 -0700 Subject: [PATCH 043/272] Made A Decision, and start of 'admin add-grid-manager-cert' command --- docs/proposed/grid-manager/managed-grid.rst | 32 +++++++++++++++++---- src/allmydata/scripts/admin.py | 10 ++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 3f2053814..59cfb3e50 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -129,12 +129,6 @@ certificate will (also) be in a file named like ``alice.cert.0``. Enrolling a Storage Server: CLI ------------------------------- -DECIDE: is a command like this best, or should you have to edit the - config "by hand"? (below fits into warner's philosophy that "at some - point" it might be best to have all config in a database or similar - and the only way to view/edit it is via tahoe commands...) - if command: write it - if not: delete this section tahoe admin add-grid-manager-cert ````````````````````````````````` @@ -148,6 +142,32 @@ installed; for now just one is sufficient). You will have to re-start your node after this. Subsequent announcements to the Introducer will include this certificate. +.. note:: + + This command will simply edit the `tahoe.cfg` file and direct you + to re-start. In the Future(tm), we should consider (in exarkun's + words): + + "A python program you run as a new process" might not be the + best abstraction to layer on top of the configuration + persistence system, though. It's a nice abstraction for users + (although most users would probably rather have a GUI) but it's + not a great abstraction for automation. So at some point it + may be better if there is CLI -> public API -> configuration + persistence system. And maybe "public API" is even a network + API for the storage server so it's equally easy to access from + an agent implemented in essentially any language and maybe if + the API is exposed by the storage node itself then this also + gives you live-configuration-updates, avoiding the need for + node restarts (not that this is the only way to accomplish + this, but I think it's a good way because it avoids the need + for messes like inotify and it supports the notion that the + storage node process is in charge of its own configuration + persistence system, not just one consumer among many ... which + has some nice things going for it ... though how this interacts + exactly with further node management automation might bear + closer scrutiny). + Enrolling a Storage Server: Config ---------------------------------- diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index d8e0b77f0..690059265 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -45,6 +45,13 @@ def derive_pubkey(options): print("public:", pubkey_vs, file=out) return 0 + +def add_grid_manager_cert(options): + """ + Add a new Grid Manager certificate to our config + """ + return 0 + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, @@ -68,7 +75,8 @@ each subcommand. subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, - } + "add-grid-manager-cert": add_grid_manager_cert, +} def do_admin(options): so = options.subOptions From 08937c6acfb6af167eaa45b3223528b1c27713c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Dec 2018 19:38:22 -0700 Subject: [PATCH 044/272] whitespace --- src/allmydata/scripts/admin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 690059265..55ad224c2 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -52,6 +52,7 @@ def add_grid_manager_cert(options): """ return 0 + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, @@ -72,12 +73,14 @@ each subcommand. """ return t + subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, "add-grid-manager-cert": add_grid_manager_cert, } + def do_admin(options): so = options.subOptions so.stdout = options.stdout @@ -88,8 +91,8 @@ def do_admin(options): subCommands = [ ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] +] dispatch = { "admin": do_admin, - } +} From d302a9867210fad4f5523efe58b57cb88888eefd Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Dec 2018 22:38:06 -0700 Subject: [PATCH 045/272] 'tahoe admin add-grid-manager-cert' command --- src/allmydata/scripts/admin.py | 96 +++++++++++++++++++++++++++++++++ src/allmydata/storage_client.py | 36 ++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 55ad224c2..0a641728c 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,7 +1,17 @@ from __future__ import print_function +import sys +import json +from os.path import exists, join + from twisted.python import usage +#from allmydata.node import read_config +from allmydata.client import read_config +from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions +from allmydata.util.encodingutil import argv_to_abspath +from allmydata.util import fileutil + class GenerateKeypairOptions(BaseOptions): @@ -46,10 +56,93 @@ def derive_pubkey(options): return 0 +class AddGridManagerCertOptions(BaseOptions): + + optParameters = [ + ['filename', 'f', None, "Filename of the certificate ('-', a dash, for stdin)"], + ['name', 'n', "default", "Name to give this certificate"], + ] + + def getSynopsis(self): + return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" + + def postOptions(self): + if self['filename'] is None: + raise usage.UsageError( + "Must provide --filename option" + ) + if self['filename'] == '-': + print >>self.parent.parent.stderr, "reading certificate from stdin" + data = sys.stdin.read() + if len(data) == 0: + raise usage.UsageError( + "Reading certificate from stdin failed" + ) + from allmydata.storage_client import parse_grid_manager_data + try: + self.certificate_data = parse_grid_manager_data(data) + except ValueError as e: + print >>self.parent.parent.stderr, "Error parsing certificate: {}".format(e) + self.certificate_data = None + else: + with open(self['filename'], 'r') as f: + self.certificate_data = f.read() + + def getUsage(self, width=None): + t = BaseOptions.getUsage(self, width) + t += ( + "Adds a Grid Manager certificate to a Storage Server.\n\n" + "The certificate will be copied into the base-dir and config\n" + "will be added to 'tahoe.cfg', which will be re-written. A\n" + "restart is required for changes to take effect.\n\n" + "The human who operates a Grid Manager would produce such a\n" + "certificate and communicate it securely to you.\n" + ) + return t + + def add_grid_manager_cert(options): """ Add a new Grid Manager certificate to our config """ + if options.certificate_data is None: + return 1 + # XXX is there really not already a function for this? + if options.parent.parent['node-directory']: + nd = argv_to_abspath(options.parent.parent['node-directory']) + else: + nd = _default_nodedir + + config = read_config(nd, "portnum") + config_path = join(nd, "tahoe.cfg") + cert_fname = "{}.cert".format(options['name']) + cert_path = config.get_config_path(cert_fname) + cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' + cert_name = options['name'] + + if exists(cert_path): + print >>options.parent.parent.stderr, "Already have file '{}'".format(cert_path) + return 1 + + cfg = config.config # why aren't methods we call on cfg in _Config itself? + + gm_certs = config.get_config("storage", "grid_manager_certificate_files", "").split() + if cert_fname not in gm_certs: + gm_certs.append(cert_fname) + cfg.set("storage", "grid_manager_certificate_files", " ".join(gm_certs)) + + # print("grid_manager_certificate_files in {}: {}".format(config_path, len(gm_certs))) + + # write all the data out + + fileutil.write(cert_path, cert_bytes) + # print("created {}: {} bytes".format(cert_fname, len(cert_bytes))) + with open(config_path, "w") as f: + cfg.write(f) + # print("wrote {}".format(config_fname)) + + print >>options.parent.parent.stderr, "There are now {} certificates".format(len(gm_certs)) + return 0 @@ -59,6 +152,9 @@ class AdminCommand(BaseOptions): "Generate a public/private keypair, write to stdout."), ("derive-pubkey", None, DerivePubkeyOptions, "Derive a public key from a private key."), + ("add-grid-manager-cert", None, AddGridManagerCertOptions, + "Add a Grid Manager-provided certificate to a storage " + "server's config."), ] def postOptions(self): if not hasattr(self, 'subOptions'): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 34b2c0caf..53b0d5c53 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -282,7 +282,39 @@ class StubServer(object): return "?" -def _validate_grid_manager_certificate(gm_key, alleged_cert): +def parse_grid_manager_data(gm_data): + """ + :param gm_data: some data that might be JSON that might be a valid + Grid Manager Certificate + + :returns: json data of a valid Grid Manager certificate, or an + exception if the data is not valid. + """ + + required_keys = allowed_keys = [ + 'certificate', + 'signature', + ] + + js = json.loads(gm_data) + for k in js.keys(): + if k not in allowed_keys: + raise ValueError( + "Grid Manager certificate JSON may not contain '{}'".format( + k, + ) + ) + for k in allowed_keys: + if k not in js: + raise ValueError( + "Grid Manager certificate JSON must contain '{}'".format( + k, + ) + ) + return js + + +def validate_grid_manager_certificate(gm_key, alleged_cert): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. @@ -423,7 +455,7 @@ class NativeStorageServer(service.MultiService): return False for gm_key in self._grid_manager_keys: for cert in self._grid_manager_certificates: - if _validate_grid_manager_certificate(gm_key, cert): + if validate_grid_manager_certificate(gm_key, cert): # print("valid: {}\n{}".format(gm_key, cert)) return True # print("didn't validate {} keys".format(len(self._grid_manager_keys))) From c88409afc545f95bddc4758d6f99ad8a769e4ba8 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 27 Mar 2019 21:04:48 -0600 Subject: [PATCH 046/272] add a note --- src/allmydata/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index d30b85694..ad5e61c14 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -605,6 +605,10 @@ class _Client(node.Node, pollmixin.PollMixin): ss.setServiceParent(self) grid_manager_certificates = [] + + # XXX this is probably a bad idea for multiple fnames -- what + # about spaces in filenames? + cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") for fname in cert_fnames.split(): fname = self.config.get_config_path(fname.decode('ascii')) From 4665617818c37edf188a6a81ace3da24f62b3bfd Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 18:53:25 -0600 Subject: [PATCH 047/272] newsfragment --- newsfragments/2916.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2916.feature diff --git a/newsfragments/2916.feature b/newsfragments/2916.feature new file mode 100644 index 000000000..292d429ea --- /dev/null +++ b/newsfragments/2916.feature @@ -0,0 +1 @@ +A new 'Grid Manager' specification and implementation \ No newline at end of file From 2cf36cf88460e7f6b5e322668e38fa7bb58f8dd6 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 18:58:39 -0600 Subject: [PATCH 048/272] flake8 --- src/allmydata/client.py | 2 -- src/allmydata/scripts/admin.py | 10 +++++----- src/allmydata/scripts/tahoe_grid_manager.py | 2 +- src/allmydata/storage_client.py | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ad5e61c14..50ceb448f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -31,13 +31,11 @@ from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider -from allmydata.util.base32 import a2b, b2a from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import IStatsProducer, SDMF_VERSION, MDMF_VERSION, DEFAULT_MAX_SEGMENT_SIZE from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist -from allmydata import node KiB=1024 diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 0a641728c..ef1059a31 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -72,7 +72,7 @@ class AddGridManagerCertOptions(BaseOptions): "Must provide --filename option" ) if self['filename'] == '-': - print >>self.parent.parent.stderr, "reading certificate from stdin" + print("reading certificate from stdin", file=self.parent.parent.stderr) data = sys.stdin.read() if len(data) == 0: raise usage.UsageError( @@ -82,7 +82,7 @@ class AddGridManagerCertOptions(BaseOptions): try: self.certificate_data = parse_grid_manager_data(data) except ValueError as e: - print >>self.parent.parent.stderr, "Error parsing certificate: {}".format(e) + print("Error parsing certificate: {}".format(e), file=self.parent.parent.stderr) self.certificate_data = None else: with open(self['filename'], 'r') as f: @@ -118,10 +118,10 @@ def add_grid_manager_cert(options): cert_fname = "{}.cert".format(options['name']) cert_path = config.get_config_path(cert_fname) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' - cert_name = options['name'] + # cert_name = options['name'] if exists(cert_path): - print >>options.parent.parent.stderr, "Already have file '{}'".format(cert_path) + print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) return 1 cfg = config.config # why aren't methods we call on cfg in _Config itself? @@ -141,7 +141,7 @@ def add_grid_manager_cert(options): cfg.write(f) # print("wrote {}".format(config_fname)) - print >>options.parent.parent.stderr, "There are now {} certificates".format(len(gm_certs)) + print("There are now {} certificates".format(len(gm_certs)), file=options.parent.parent.stderr) return 0 diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f7193cbf2..b33552861 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -1,5 +1,5 @@ +from __future__ import print_function -import os import sys import json import time diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 53b0d5c53..5f65c6cec 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -43,7 +43,6 @@ from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code use from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer from allmydata.util import log, base32, connection_status -from allmydata.util import keyutil from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference @@ -304,7 +303,7 @@ def parse_grid_manager_data(gm_data): k, ) ) - for k in allowed_keys: + for k in required_keys: if k not in js: raise ValueError( "Grid Manager certificate JSON must contain '{}'".format( From cfda360294c84273ca2842cc11deeef5d035b61e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 18:59:14 -0600 Subject: [PATCH 049/272] space-separated paths is a bad idea --- docs/proposed/grid-manager/managed-grid.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 59cfb3e50..10d0c142a 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -173,14 +173,19 @@ Enrolling a Storage Server: Config ---------------------------------- You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to -include an entry ``grid_manager_certificate_files = `` whose value is -a space-separated list of paths to valid certificate files. These -certificate files are issued by the ``tahoe grid-manager sign`` -command; these should be securely transmitted to the storage -server. Relative paths are relative to the node directory. Example:: +turn on grid-management with ``grid_management = true``. You then must +also provide a ``[grid_management_keys]]`` section in the config-file which +lists ``name = path/to/certificate`` pairs. + +These certificate files are issued by the ``tahoe grid-manager sign`` +command; these should be **securely transmitted** to the storage +server. Relative paths are based from the node directory. Example:: [storage] - grid_manager_certificate_files = example_grid.cert + grid_management = true + + [grid_management_keys] + default = example_grid.cert This will cause us to give this certificate to any Introducers we connect to (and subsequently, the Introducer will give the certificate From 16e7bc5e07a2284f13e49f3b7f1585b5244ec603 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 19:42:39 -0600 Subject: [PATCH 050/272] pytest API --- integration/test_grid_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 3553ccc23..075a1004a 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -11,10 +11,10 @@ from allmydata.util import configutil import util -import pytest +import pytest_twisted -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_create_certificate(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", @@ -42,7 +42,7 @@ def test_create_certificate(reactor): ) -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_remove_client(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", @@ -71,7 +71,7 @@ def test_remove_client(reactor): assert json.loads(gm_config)['storage_servers'].has_key('bob') -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_remove_last_client(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", @@ -93,7 +93,7 @@ def test_remove_last_client(reactor): assert not json.loads(gm_config).has_key('storage_servers') -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", From 9220dd12efe2bcee0b1e320276e669e0179b19b8 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 19:42:50 -0600 Subject: [PATCH 051/272] get rid of debug --- src/allmydata/storage_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5f65c6cec..39545b3a9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -371,7 +371,7 @@ class NativeStorageServer(service.MultiService): } def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): - print("CREATE {}: {}".format(server_id, grid_manager_certs)) + # print("CREATE {}: {}".format(server_id, grid_manager_certs)) service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id @@ -387,11 +387,11 @@ class NativeStorageServer(service.MultiService): # any public-keys which the user has configured (if none, it # means use any storage servers) self._grid_manager_keys = grid_manager_keys - print("keys: {}".format(self._grid_manager_keys)) + # print("keys: {}".format(self._grid_manager_keys)) # any storage-certificates that this storage-server included # in its announcement self._grid_manager_certificates = grid_manager_certs - print("certs: {}".format(self._grid_manager_certificates)) + # print("certs: {}".format(self._grid_manager_certificates)) assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) From 3d7055711a6ecb8411528eb0fa6736d14cc6ec84 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 22:35:07 -0600 Subject: [PATCH 052/272] make and use set_config instead of internals --- src/allmydata/node.py | 15 +++++++++++++++ src/allmydata/scripts/admin.py | 4 +--- src/allmydata/test/test_node.py | 6 ++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index c058fce4f..083cbef2c 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -321,6 +321,21 @@ class _Config(object): ) return default + def set_config(self, section, option, value): + """ + Set a config options in a section and re-write the tahoe.cfg file + """ + if option.endswith(".furl") and self._contains_unescaped_hash(value): + raise UnescapedHashError(section, option, item) + + try: + self.config.add_section(section) + except ConfigParser.DuplicateSectionError: + pass + self.config.set(section, option, value) + with open(self._config_fname, "w") as f: + self.config.write(f) + def get_config_from_file(self, name, required=False): """Get the (string) contents of a config file, or None if the file did not exist. If required=True, raise an exception rather than diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index ef1059a31..60d31cfb8 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -124,12 +124,10 @@ def add_grid_manager_cert(options): print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) return 1 - cfg = config.config # why aren't methods we call on cfg in _Config itself? - gm_certs = config.get_config("storage", "grid_manager_certificate_files", "").split() if cert_fname not in gm_certs: gm_certs.append(cert_fname) - cfg.set("storage", "grid_manager_certificate_files", " ".join(gm_certs)) + config.set_config("storage", "grid_manager_certificate_files", " ".join(gm_certs)) # print("grid_manager_certificate_files in {}: {}".format(config_path, len(gm_certs))) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 096cd4f8f..a7ed6d528 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -316,6 +316,12 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): yield client.create_client(basedir) self.failUnless(ns.called) + def test_set_config_new_section(self): + basedir = "test_node/test_set_config_new_section" + config = config_from_string(basedir, "", "") + config.set_config("foo", "bar", "value1") + config.set_config("foo", "bar", "value2") + class TestMissingPorts(unittest.TestCase): """ From f0e3b69f90278c71a45e978b833f64ae0b3dfcd9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 23:33:40 -0600 Subject: [PATCH 053/272] switch around how we do config (avoid space-separated filenames) --- docs/proposed/grid-manager/managed-grid.rst | 7 ++- integration/test_grid_manager.py | 5 +- src/allmydata/client.py | 54 ++++++++++++--------- src/allmydata/scripts/admin.py | 11 ++--- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 10d0c142a..5c388ffa1 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -174,7 +174,7 @@ Enrolling a Storage Server: Config You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to turn on grid-management with ``grid_management = true``. You then must -also provide a ``[grid_management_keys]]`` section in the config-file which +also provide a ``[grid_management_keys]`` section in the config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``tahoe grid-manager sign`` @@ -278,7 +278,10 @@ certificates into the grid. We do this by adding some configuration (in ``tahoe.cfg``):: [storage] - grid_manager_certificate_files = gridmanager.cert + grid_management = true + + [grid_manager_certificates] + default = gridmanager.cert Add the above bit to each node's ``tahoe.cfg`` and re-start the storage nodes. diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 075a1004a..f25819092 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -115,7 +115,6 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd ) assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] - # XXX FIXME need to shut-down and nuke carol when we're done this # test (i.d. request.addfinalizer) carol = yield util._create_node( @@ -136,7 +135,9 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd with open(join(storage._node_dir, "gridmanager.cert"), "w") as f: f.write(cert) config = configutil.get_config(join(storage._node_dir, "tahoe.cfg")) - config.set("storage", "grid_manager_certificate_files", "gridmanager.cert") + config.set("storage", "grid_management", "True") + config.add_section("grid_manager_certificates") + config.set("grid_manager_certificates", "default", "gridmanager.cert") config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) # re-start this storage server diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 50ceb448f..35512d8f5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -59,6 +59,7 @@ def _valid_config_sections(): "stats_gatherer.furl", ), "grid_managers": None, # means "any options valid" + "grid_manager_certificates": None, "drop_upload": ( # deprecated already? "enabled", ), @@ -81,7 +82,7 @@ def _valid_config_sections(): "readonly", "reserved_space", "storage_dir", - "grid_manager_certificate_files", + "grid_management", ), "sftpd": ( "accounts.file", @@ -409,6 +410,34 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con return sb +def _load_grid_manager_certificates(config): + """ + Load all Grid Manager certificates in the config in a list. An + empty list is returned if there are none. + """ + grid_manager_certificates = [] + + cert_fnames = list(config.enumerate_section("grid_manager_certificates").values()) + for fname in cert_fnames: + fname = config.get_config_path(fname.decode('ascii')) + if not os.path.exists(fname): + raise ValueError( + "Grid Manager certificate file '{}' doesn't exist".format( + fname + ) + ) + with open(fname, 'r') as f: + cert = json.load(f) + if set(cert.keys()) != {"certificate", "signature"}: + raise ValueError( + "Unknown key in Grid Manager certificate '{}'".format( + fname + ) + ) + grid_manager_certificates.append(cert) + return grid_manager_certificates + + @implementer(IStatsProducer) class _Client(node.Node, pollmixin.PollMixin): @@ -604,27 +633,8 @@ class _Client(node.Node, pollmixin.PollMixin): grid_manager_certificates = [] - # XXX this is probably a bad idea for multiple fnames -- what - # about spaces in filenames? - - cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") - for fname in cert_fnames.split(): - fname = self.config.get_config_path(fname.decode('ascii')) - if not os.path.exists(fname): - raise ValueError( - "Grid Manager certificate file '{}' doesn't exist".format( - fname - ) - ) - with open(fname, 'r') as f: - cert = json.load(f) - if set(cert.keys()) != {"certificate", "signature"}: - raise ValueError( - "Unknown key in Grid Manager certificate '{}'".format( - fname - ) - ) - grid_manager_certificates.append(cert) + if self.config.get_config("storage", "grid_management", default=False, boolean=True): + grid_manager_certificates = _load_grid_manager_certificates(self.config) # XXX we should probably verify that the certificates are # valid and not expired, as that could be confusing for the diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 60d31cfb8..3ddbe8d22 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -118,18 +118,15 @@ def add_grid_manager_cert(options): cert_fname = "{}.cert".format(options['name']) cert_path = config.get_config_path(cert_fname) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' - # cert_name = options['name'] + cert_name = options['name'] if exists(cert_path): print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) return 1 - gm_certs = config.get_config("storage", "grid_manager_certificate_files", "").split() - if cert_fname not in gm_certs: - gm_certs.append(cert_fname) - config.set_config("storage", "grid_manager_certificate_files", " ".join(gm_certs)) - - # print("grid_manager_certificate_files in {}: {}".format(config_path, len(gm_certs))) + config.set_config("storage", "grid_management", "True") + config.add_section("grid_manager_certificates") + config.set_config("grid_manager_certificates", cert_name, cert_fname) # write all the data out From f0a9240f5f25b88ec8795dd8839ac13632a0e952 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 9 Apr 2019 11:32:48 -0600 Subject: [PATCH 054/272] cleanup --- src/allmydata/scripts/admin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 3ddbe8d22..47ca4a0eb 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -133,10 +133,12 @@ def add_grid_manager_cert(options): fileutil.write(cert_path, cert_bytes) # print("created {}: {} bytes".format(cert_fname, len(cert_bytes))) with open(config_path, "w") as f: - cfg.write(f) + # XXX probably want a _Config.write_tahoe_cfg() or something? or just set_config() does that automagically + config.config.write(f) # print("wrote {}".format(config_fname)) - print("There are now {} certificates".format(len(gm_certs)), file=options.parent.parent.stderr) + cert_count = len(config.enumerate_section("grid_manager_certificates")) + print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) return 0 From 11993a230b750fd7caa386bdd7ab41bf5344e58f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 9 Apr 2019 11:33:31 -0600 Subject: [PATCH 055/272] item -> value --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 083cbef2c..da18c0720 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -326,7 +326,7 @@ class _Config(object): Set a config options in a section and re-write the tahoe.cfg file """ if option.endswith(".furl") and self._contains_unescaped_hash(value): - raise UnescapedHashError(section, option, item) + raise UnescapedHashError(section, option, value) try: self.config.add_section(section) From 8e363ca26b33c7b56ff73c15348997c3eb172734 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 9 Apr 2019 17:08:44 -0600 Subject: [PATCH 056/272] can't re-write a static string --- src/allmydata/node.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index da18c0720..dcbc5088c 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -203,7 +203,13 @@ def config_from_string(basedir, portnumfile, config_str): # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) - return _Config(parser, portnumfile, basedir, '') + + def write_new_config(cfg): + """ + We throw away any attempt to persist + """ + pass + return _Config(parser, portnumfile, basedir, '', write_new_config) def get_app_versions(): @@ -248,7 +254,8 @@ class _Config(object): have better names. """ - def __init__(self, configparser, portnum_fname, basedir, config_fname): + def __init__(self, configparser, portnum_fname, basedir, config_fname, + write_new_tahoecfg=None): """ :param configparser: a ConfigParser instance @@ -260,12 +267,25 @@ class _Config(object): :param config_fname: the pathname actually used to create the configparser (might be 'fake' if using in-memory data) + + :param write_new_tahoecfg: callable taking one argument which + is a ConfigParser instance """ self.portnum_fname = portnum_fname self._basedir = abspath_expanduser_unicode(unicode(basedir)) self._config_fname = config_fname self.config = configparser + if write_new_tahoecfg is None: + def write_new_tahoecfg(config): + """ + Write to the default place, /tahoe.cfg + """ + fn = os.path.join(self._basedir, "tahoe.cfg") + with open(fn, "w") as f: + config.write(f) + self._write_config = write_new_tahoecfg + nickname_utf8 = self.get_config("node", "nickname", "") self.nickname = nickname_utf8.decode("utf-8") assert type(self.nickname) is unicode @@ -333,8 +353,7 @@ class _Config(object): except ConfigParser.DuplicateSectionError: pass self.config.set(section, option, value) - with open(self._config_fname, "w") as f: - self.config.write(f) + self._write_config(self.config) def get_config_from_file(self, name, required=False): """Get the (string) contents of a config file, or None if the file From 1b3bfd53d4076cba7c9db2db73b9eebafcf79b28 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 14 May 2019 05:55:34 -0600 Subject: [PATCH 057/272] grid-manager client config is different --- docs/proposed/grid-manager/managed-grid.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5c388ffa1..8a655271d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -313,8 +313,8 @@ configuration:: Put the key printed out above into Alice's ``tahoe.cfg`` in section ``client``:: - [client] - grid_manager_public_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + [grid_managers] + example_name = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq DECIDE: From 883a3ba12e63c74bbd6d91c86bbb1601aba6eb23 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:50:30 -0600 Subject: [PATCH 058/272] no mutable defaults --- src/allmydata/storage_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 39545b3a9..489892b7a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -73,13 +73,13 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[]): + def __init__(self, permute_peers, tub_maker, preferred_peers=None, grid_manager_keys=None): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker - self.preferred_peers = preferred_peers - self._grid_manager_keys = grid_manager_keys + self.preferred_peers = preferred_peers if preferred_peers else tuple() + self._grid_manager_keys = grid_manager_keys if grid_manager_keys else list() # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its From 3af6d74ba828f2c8427d1882c36c275fc0311d7d Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:52:48 -0600 Subject: [PATCH 059/272] clarify further --- docs/proposed/grid-manager/managed-grid.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 8a655271d..b97e97c7b 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -8,8 +8,9 @@ Managed Grid ============ In a grid using an Introducer, a client will use any storage-server -the Introducer announces. This means that anyone with the Introducer -fURL can connect storage to the grid. +the Introducer announces (and the Introducer will annoucne any +storage-server that connects to it). This means that anyone with the +Introducer fURL can connect storage to the grid. Sometimes, this is just what you want! From 8e8b2f4d8f3916b5d1f778ae7d3fcdb72f22bb7a Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:52:59 -0600 Subject: [PATCH 060/272] introducer-less -> introducerless --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index b97e97c7b..410172870 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -20,7 +20,7 @@ the grid; clients of this grid don't want their uploads to go to "unmanaged" storage if some other client decides to provide storage. One way to limit which storage servers a client connects to is via the -"server list" (:ref:`server_list`) (aka "Introducer-less" +"server list" (:ref:`server_list`) (aka "Introducerless" mode). Clients are given static lists of storage-servers, and connect only to those. This means manually updating these lists if the storage servers change, however. From 235d5debe1e281dafd559fa998e6007b9860f66d Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:07:18 -0600 Subject: [PATCH 061/272] added example and more explanation for 'add' --- docs/proposed/grid-manager/managed-grid.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 410172870..04832fd3f 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -94,12 +94,19 @@ tahoe grid-manager add `````````````````````` Takes two args: ``name pubkey``. The ``name`` is an arbitrary local -identifier and the pubkey is the encoded key from a ``node.pubkey`` -file in the storage-server's node directory (with no whitespace). +identifier for the new storage node (also sometimes called "a +petname"). The pubkey is the encoded key from a ``node.pubkey`` file +in the storage-server's node directory (with no whitespace). For +example, if ``~/storage0`` contains a storage-node, you might do +something like this: + + tahoe grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) This adds a new storage-server to a Grid Manager's configuration. (Since it mutates the configuration, if you used -``--config -`` the new configuration will be printed to stdout). +``--config -`` the new configuration will be printed to stdout). The +usefulness of the ``name`` is solely for reference within this Grid +Manager. tahoe grid-manager list From 18fead08dd819f3b34125867759fbc95843aa45c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:08:18 -0600 Subject: [PATCH 062/272] consistently use 'nickname' --- docs/proposed/grid-manager/managed-grid.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 04832fd3f..bae3bf990 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -94,9 +94,9 @@ tahoe grid-manager add `````````````````````` Takes two args: ``name pubkey``. The ``name`` is an arbitrary local -identifier for the new storage node (also sometimes called "a -petname"). The pubkey is the encoded key from a ``node.pubkey`` file -in the storage-server's node directory (with no whitespace). For +identifier for the new storage node (also sometimes called "a petname" +or "nickname"). The pubkey is the encoded key from a ``node.pubkey`` +file in the storage-server's node directory (with no whitespace). For example, if ``~/storage0`` contains a storage-node, you might do something like this: @@ -119,7 +119,7 @@ Lists all storage-servers that have previously been added using tahoe grid-manager sign ``````````````````````` -Takes one arg: ``name``, the petname used previously in a ``tahoe +Takes one arg: ``name``, the nickname used previously in a ``tahoe grid-manager add`` command. Note that this mutates the state of the grid-manager if it is on disk, @@ -211,7 +211,7 @@ DECIDE: is a command like this best, or should you have to edit the tahoe add-grid-manager `````````````````````` -- ``--name``: a petname to call this Grid Manager (default: "default") +- ``--name``: a nickname to call this Grid Manager (default: "default") For clients to start using a Grid Manager, they must add a public-key. A client may have any number of grid-managers, so each one From 4340cfa1b83f6aba6cba70e7b89f700b156127bc Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:16:12 -0600 Subject: [PATCH 063/272] decision made --- docs/proposed/grid-manager/managed-grid.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index bae3bf990..88f210d34 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -203,10 +203,6 @@ out to clients). Enrolling a Client: CLI ----------------------- -DECIDE: is a command like this best, or should you have to edit the - config "by hand"? (below fits into warner's philosophy that "at some - point" it might be best to have all config in a database or similar - and the only way to view/edit it is via tahoe commands...) tahoe add-grid-manager `````````````````````` From a98eae3c7c92e3c8779c8e266d704bcdd189607a Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:26:30 -0600 Subject: [PATCH 064/272] clarify --- docs/proposed/grid-manager/managed-grid.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 88f210d34..6f7676b3c 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -207,15 +207,17 @@ Enrolling a Client: CLI tahoe add-grid-manager `````````````````````` -- ``--name``: a nickname to call this Grid Manager (default: "default") +This takes two arguments: ``name`` and ``public-identity``. -For clients to start using a Grid Manager, they must add a -public-key. A client may have any number of grid-managers, so each one -has a name. If you don't supply ``--name`` then ``"default"`` is used. +The ``name`` argument is a nickname to call this Grid Manager. A +client may have any number of grid-managers, so each one has a name. A +client with zero Grid Managers will accept any announcement from an +Introducer. -This command takes a single argument, which is the hex-encoded public -key of the Grid Manager. The client will have to be re-started once -this change is made. +The ``public-identity`` argument is the encoded public key of the Grid +Manager (that is, the output of ``tahoe grid-manager +public-identity``). The client will have to be re-started once this +change is made. Enrolling a Client: Config From fb3f0b79e1721b4f3d4f11e7450cd1f15c939490 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:34:56 -0600 Subject: [PATCH 065/272] more words about 'daemonize' --- docs/proposed/grid-manager/managed-grid.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 6f7676b3c..1ece84e42 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -241,6 +241,11 @@ Manager. Example:: Example Setup of a New Managed Grid ----------------------------------- +Note that we use the ``daemonize`` command in the following but that's +only one way to handle "running a command in the background". You +could instead run commands that start with ``daemonize ...`` in their +own shell/terminal window or via something like ``systemd`` + We'll store our Grid Manager configuration on disk, in ``./gm0``. To initialize this directory:: @@ -257,8 +262,7 @@ it briefly before it creates the "Introducer fURL" we want for the next steps:: tahoe create-introducer --listen=tcp --port=5555 --location=tcp:localhost:5555 ./introducer - tahoe -d introducer run - (Ctrl-C to stop it after a bit) + daemonize tahoe -d introducer run Next, we attach a couple of storage nodes:: From 7feed4146e74286854e1beb68db94ffd86e2f425 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:27:49 -0600 Subject: [PATCH 066/272] redundant webport --- docs/proposed/grid-manager/managed-grid.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 1ece84e42..c3ac29409 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -266,8 +266,8 @@ next steps:: Next, we attach a couple of storage nodes:: - tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage0 --webport 6001 --webport 6002 --location tcp:localhost:6003 --port 6003 ./storage0 - tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage1 --webport 6101 --webport 6102 --location tcp:localhost:6103 --port 6103 ./storage1 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage0 --webport 6001 --location tcp:localhost:6003 --port 6003 ./storage0 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage1 --webport 6101 --location tcp:localhost:6103 --port 6103 ./storage1 daemonize tahoe -d storage0 run daemonize tahoe -d storage1 run @@ -300,7 +300,7 @@ Now try adding a new storage server ``storage2``. This client can join the grid just fine, and announce itself to the Introducer as providing storage:: - tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage2 --webport 6301 --webport 6302 --location tcp:localhost:6303 --port 6303 ./storage2 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage2 --webport 6301 --location tcp:localhost:6303 --port 6303 ./storage2 daemonize tahoe -d storage2 run At this point any client will upload to any of these three @@ -308,7 +308,7 @@ storage-servers. Make a client "alice" and try! :: - tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6301 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice + tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6401 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice daemonize tahoe -d alice run tahoe -d alice mkdir # prints out a dir-cap find storage2/storage/shares # confirm storage2 has a share From 6c15d67df64b8c22b37d22429db049122a047534 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:28:12 -0600 Subject: [PATCH 067/272] 'tahoe put' not 'tahoe mkdir' --- docs/proposed/grid-manager/managed-grid.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index c3ac29409..5dbb44e57 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -310,7 +310,7 @@ storage-servers. Make a client "alice" and try! tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6401 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice daemonize tahoe -d alice run - tahoe -d alice mkdir # prints out a dir-cap + tahoe -d alice put README.rst # prints out a read-cap find storage2/storage/shares # confirm storage2 has a share Now we want to make Alice only upload to the storage servers that the @@ -338,7 +338,7 @@ DECIDE: Now, re-start the "alice" client. Since we made Alice's parameters require 3 storage servers to be reachable (``--happy=3``), all their -uploads should now fail (so ``tahoe mkdir`` will fail) because they +uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". You can check Alice's "Welcome" page (where the list of connected servers From f1ae20a6771a93c8c9e1bf6f97920f2525072dbc Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:33:02 -0600 Subject: [PATCH 068/272] spelling --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5dbb44e57..845aff7cc 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -345,4 +345,4 @@ You can check Alice's "Welcome" page (where the list of connected servers is) at http://localhost:6301/ and should be able to see details about the "work-grid" Grid Manager that you added. When any Grid Managers are enabled, each storage-server line will show whether it has a valid -cerifiticate or not (and how much longer it's valid until). +certificate or not (and how much longer it's valid until). From 65403eb164c9e63d6c5de72832a8c4ee9932b4bb Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:37:45 -0600 Subject: [PATCH 069/272] alice -> zara, bob -> yakov and docstrings --- integration/test_grid_manager.py | 43 +++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index f25819092..2547b35be 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -16,6 +16,9 @@ import pytest_twisted @pytest_twisted.inlineCallbacks def test_create_certificate(reactor): + """ + The Grid Manager produces a valid, correctly-signed certificate. + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) @@ -23,52 +26,58 @@ def test_create_certificate(reactor): privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) pubkey = keyutil.parse_pubkey(pubkey_bytes) + # Note that zara + her key here are arbitrary and don't match any + # "actual" clients in the test-grid; we're just checking that the + # Grid Manager signs this properly. gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) - alice_cert_bytes = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "sign", "alice", + zara_cert_bytes = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "sign", "zara", stdin=gm_config, ) - alice_cert = json.loads(alice_cert_bytes) + zara_cert = json.loads(zara_cert_bytes) - # confirm that alice's certificate is made by the Grid Manager + # confirm that zara's certificate is made by the Grid Manager # (.verify returns None on success, raises exception on error) pubkey.verify( - base32.a2b(alice_cert['signature'].encode('ascii')), - alice_cert['certificate'].encode('ascii'), + base32.a2b(zara_cert['signature'].encode('ascii')), + zara_cert['certificate'].encode('ascii'), ) @pytest_twisted.inlineCallbacks def test_remove_client(reactor): + """ + A Grid Manager can add and successfully remove a client + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "bob", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("alice") - assert json.loads(gm_config)['storage_servers'].has_key("bob") + assert json.loads(gm_config)['storage_servers'].has_key("zara") + assert json.loads(gm_config)['storage_servers'].has_key("yakov") return gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", - "alice", + "zara", stdin=gm_config, ) - assert not json.loads(gm_config)['storage_servers'].has_key('alice') - assert json.loads(gm_config)['storage_servers'].has_key('bob') + assert not json.loads(gm_config)['storage_servers'].has_key('zara') + assert json.loads(gm_config)['storage_servers'].has_key('yakov') @pytest_twisted.inlineCallbacks @@ -79,14 +88,14 @@ def test_remove_last_client(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("alice") + assert json.loads(gm_config)['storage_servers'].has_key("zara") gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", - "alice", + "zara", stdin=gm_config, ) # there are no storage servers left at all now From 97a7672226010ef5de2715c0fc40f7e7fa6a2f92 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:40:10 -0600 Subject: [PATCH 070/272] .has_key -> 'in' --- integration/test_grid_manager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 2547b35be..6aa50b09e 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -67,8 +67,8 @@ def test_remove_client(reactor): "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("zara") - assert json.loads(gm_config)['storage_servers'].has_key("yakov") + assert "zara" in json.loads(gm_config)['storage_servers'] + assert "yakov" in json.loads(gm_config)['storage_servers'] return gm_config = yield util.run_tahoe( @@ -76,8 +76,8 @@ def test_remove_client(reactor): "zara", stdin=gm_config, ) - assert not json.loads(gm_config)['storage_servers'].has_key('zara') - assert json.loads(gm_config)['storage_servers'].has_key('yakov') + assert "zara" not in json.loads(gm_config)['storage_servers'] + assert "yakov" in json.loads(gm_config)['storage_servers'] @pytest_twisted.inlineCallbacks @@ -91,7 +91,7 @@ def test_remove_last_client(reactor): "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("zara") + assert "zara" in json.loads(gm_config)['storage_servers'] gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", @@ -99,7 +99,7 @@ def test_remove_last_client(reactor): stdin=gm_config, ) # there are no storage servers left at all now - assert not json.loads(gm_config).has_key('storage_servers') + assert "storage_servers" not in json.loads(gm_config) @pytest_twisted.inlineCallbacks From 5071da2c0560ffd2803b89415186e6c4271a0b4b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:47:18 -0600 Subject: [PATCH 071/272] remove early return --- integration/test_grid_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 6aa50b09e..06d763b4f 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -69,7 +69,6 @@ def test_remove_client(reactor): ) assert "zara" in json.loads(gm_config)['storage_servers'] assert "yakov" in json.loads(gm_config)['storage_servers'] - return gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", From 72a51e903a8a6951ff86b2be116f300aabd392bf Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:50:10 -0600 Subject: [PATCH 072/272] docstring --- integration/test_grid_manager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 06d763b4f..d9fac17a9 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -81,6 +81,9 @@ def test_remove_client(reactor): @pytest_twisted.inlineCallbacks def test_remove_last_client(reactor): + """ + A Grid Manager can remove all clients + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) @@ -103,6 +106,10 @@ def test_remove_last_client(reactor): @pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): + """ + A client using grid-manager refuses to upload to a storage-server + without a valid certificate + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) From 371fcd5b8600f7332fff777090acbeb054ff1a1c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:54:50 -0600 Subject: [PATCH 073/272] timeouts not required --- integration/test_grid_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d9fac17a9..6ef1e053b 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -173,11 +173,10 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) carol.signalProcess('TERM') yield carol._protocol.exited - time.sleep(1) + carol = yield util._run_node( reactor, carol._node_dir, request, None, ) - time.sleep(5) # try to put something into the grid, which should fail (because # carol has happy=3 but should only find storage0, storage1 to be From c823ff1195334ba61f3c8ae1ff66b5fe9f3524d8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:56:24 -0600 Subject: [PATCH 074/272] better docstring --- integration/test_grid_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 6ef1e053b..0a825f14a 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -107,8 +107,9 @@ def test_remove_last_client(reactor): @pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): """ - A client using grid-manager refuses to upload to a storage-server - without a valid certificate + A client with happines=3 fails to upload to a Grid when it is + using Grid Manager and there are only two storage-servers with + valid certificates. """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", From 4ece4e9dce3cf62ce33fb245f507d070255be3aa Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:24:07 -0600 Subject: [PATCH 075/272] specific exception for failing subprocess --- integration/test_grid_manager.py | 7 ++----- integration/test_servers_of_happiness.py | 4 ++-- integration/util.py | 22 +++++++++++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 0a825f14a..d46bb6b10 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -190,8 +190,5 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd stdin="some content" * 200, ) assert False, "Should get a failure" - except Exception as e: - # depending on the full output being in the error-message - # here; see util.py - assert 'UploadUnhappinessError' in str(e) - print("found expected UploadUnhappinessError") + except util.ProcessFailed as e: + assert 'UploadUnhappinessError' in e.output.getvalue() diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 1984dd507..c9b654d9c 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -42,8 +42,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto try: yield proto.done assert False, "should raise exception" - except Exception as e: - assert "UploadUnhappinessError" in str(e) + except util.ProcessFailed as e: + assert "UploadUnhappinessError" in e.output.getvalue() output = proto.output.getvalue() assert "shares could be placed on only" in output diff --git a/integration/util.py b/integration/util.py index 703402aa8..ffdf5a317 100644 --- a/integration/util.py +++ b/integration/util.py @@ -32,6 +32,20 @@ class _ProcessExitedProtocol(ProcessProtocol): self.done.callback(None) +class ProcessFailed(Exception): + """ + A subprocess has failed. + + :ivar ProcessTerminated reason: the original reason from .processExited + + :ivar StringIO output: all stdout and stderr collected to this point. + """ + + def __init__(self, reason, output): + self.reason = reason + self.output = output + + class _CollectOutputProtocol(ProcessProtocol): """ Internal helper. Collects all output (stdout + stderr) into @@ -54,13 +68,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - #self.done.errback(reason) - self.done.errback(RuntimeError( - "Process failed: {}\nOutput:\n{}".format( - reason, - self.output.getvalue(), - ) - )) + self.done.errback(ProcessFailed(reason, self.output)) def outReceived(self, data): self.output.write(data) From 4fc649fee16587846d0ccecb7f3fc81949fe30fe Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:24:26 -0600 Subject: [PATCH 076/272] error to create a duplicate node --- integration/test_tor.py | 42 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index e0b991128..801efc3ca 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -67,28 +67,28 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) - if True: - if exists(node_dir): - print("nuking '{}'".format(node_dir)) - shutil.rmtree(node_dir) - print("creating", node_dir) - mkdir(node_dir) - proto = util._DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'allmydata.scripts.runner', - 'create-node', - '--nickname', name, - '--introducer', introducer_furl, - '--hide-ip', - '--tor-control-port', 'tcp:localhost:{}'.format(control_port), - '--listen', 'tor', - node_dir, - ) + if exists(node_dir): + raise RuntimeError( + "A node already exists in '{}'".format(node_dir) ) - yield proto.done + print("creating", node_dir) + mkdir(node_dir) + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--tor-control-port', 'tcp:localhost:{}'.format(control_port), + '--listen', 'tor', + node_dir, + ) + ) + yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write(''' From dfa8b37a152246abc6150b1a697afe41ddbd4884 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:26:22 -0600 Subject: [PATCH 077/272] irrelevant code --- integration/util.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/util.py b/integration/util.py index ffdf5a317..e0454533f 100644 --- a/integration/util.py +++ b/integration/util.py @@ -209,9 +209,6 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - if exists(node_dir): - print("nuking: {}".format(node_dir)) - rmtree(node_dir) print("creating", node_dir) mkdir(node_dir) done_proto = _ProcessExitedProtocol() From 85d8e24421bd55a2b096aa852889a2032837b7be Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:27:47 -0600 Subject: [PATCH 078/272] ascii -> utf8 --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 35512d8f5..901004db4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -419,7 +419,7 @@ def _load_grid_manager_certificates(config): cert_fnames = list(config.enumerate_section("grid_manager_certificates").values()) for fname in cert_fnames: - fname = config.get_config_path(fname.decode('ascii')) + fname = config.get_config_path(fname.decode('utf8')) if not os.path.exists(fname): raise ValueError( "Grid Manager certificate file '{}' doesn't exist".format( From c9f5ed7d6442a4fe00d89adc4d53217641c6071b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:39:51 -0600 Subject: [PATCH 079/272] redundant --- src/allmydata/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index dcbc5088c..f2f52d323 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -208,7 +208,6 @@ def config_from_string(basedir, portnumfile, config_str): """ We throw away any attempt to persist """ - pass return _Config(parser, portnumfile, basedir, '', write_new_config) From 7cb7cdfac90a45e9c3fcc5de35cbc93f7d96b805 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:44:25 -0600 Subject: [PATCH 080/272] method instead of nested function --- src/allmydata/node.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index f2f52d323..ae62e2fc3 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -276,19 +276,21 @@ class _Config(object): self.config = configparser if write_new_tahoecfg is None: - def write_new_tahoecfg(config): - """ - Write to the default place, /tahoe.cfg - """ - fn = os.path.join(self._basedir, "tahoe.cfg") - with open(fn, "w") as f: - config.write(f) + write_new_tahoecfg = self._default_write_new_tahoecfg self._write_config = write_new_tahoecfg nickname_utf8 = self.get_config("node", "nickname", "") self.nickname = nickname_utf8.decode("utf-8") assert type(self.nickname) is unicode + def _default_write_new_tahoecfg(self, config): + """ + Write to the default place, /tahoe.cfg + """ + fn = os.path.join(self._basedir, "tahoe.cfg") + with open(fn, "w") as f: + config.write(f) + def validate(self, valid_config_sections): configutil.validate_config(self._config_fname, self.config, valid_config_sections) From c0f0d765633edaad55cd1cfb1ba5e1a0768c70b5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:52:45 -0600 Subject: [PATCH 081/272] parametrize 'now' function --- src/allmydata/storage_client.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 489892b7a..e78470e88 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -313,18 +313,24 @@ def parse_grid_manager_data(gm_data): return js -def validate_grid_manager_certificate(gm_key, alleged_cert): +def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. - :param cert: dict with "certificate" and "signature" keys, where + :param alleged_cert: dict with "certificate" and "signature" keys, where "certificate" contains a JSON-serialized certificate for a Storage Server (comes from a Grid Manager). + :param now_fn: a zero-argument callable that returns a UTC + timestamp (will use datetime.utcnow by default) + :return: False if the signature is invalid or the certificate is expired. """ + if now_fn is None: + now_fn = datetime.utcnow + try: gm_key.verify( base32.a2b(alleged_cert['signature'].encode('ascii')), @@ -334,7 +340,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): return False # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) - now = datetime.utcnow() + now = now_fn() expires = datetime.utcfromtimestamp(cert['expires']) # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) if expires < now: From 026bcca6c562b32540ce458c6e5ad8b3fa94b8dd Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:54:23 -0600 Subject: [PATCH 082/272] upload_permitted in IServer --- src/allmydata/interfaces.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 36c3622d8..aced2b546 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -470,6 +470,12 @@ class IServer(IDisplayableServer): once the connection is lost. """ + def upload_permitted(): + """ + :return: True if we should use this server for uploads, False + otherwise. + """ + class IMutableSlotWriter(Interface): """ From a384df720a67e7deb765f482cae3c5d3805094be Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:56:10 -0600 Subject: [PATCH 083/272] link to docs --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 845aff7cc..15c728784 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -26,7 +26,7 @@ only to those. This means manually updating these lists if the storage servers change, however. Another method is for clients to use `[client] peers.preferred=` -configuration option (XXX link? appears undocumented), which suffers +configuration option (:ref:`Client Configuration`), which suffers from a similar disadvantage. From ac46fb24f97a7b7e619afdbdffc5dd0697c39f60 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:57:13 -0600 Subject: [PATCH 084/272] take -> require --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 15c728784..08ab07f70 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -64,7 +64,7 @@ If you don't want to store the configuration on disk at all, you may use ``--config -`` (that's a dash) and write a valid JSON configuration to stdin. -All commands take the ``--config`` option, and they all behave +All commands require the ``--config`` option, and they all behave similarly for "data from stdin" versus "data from disk". From 92bd22f2c611ba4c220cfbeb0ce696bfccf01f3f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 16:02:25 -0600 Subject: [PATCH 085/272] reword --- docs/proposed/grid-manager/managed-grid.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 08ab07f70..a3ad0d115 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -96,9 +96,9 @@ tahoe grid-manager add Takes two args: ``name pubkey``. The ``name`` is an arbitrary local identifier for the new storage node (also sometimes called "a petname" or "nickname"). The pubkey is the encoded key from a ``node.pubkey`` -file in the storage-server's node directory (with no whitespace). For -example, if ``~/storage0`` contains a storage-node, you might do -something like this: +file in the storage-server's node directory (minus any +whitespace). For example, if ``~/storage0`` contains a storage-node, +you might do something like this: tahoe grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) From 91af588c9be3bcc230ce308087f2d5a5050d20f5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:08:02 -0600 Subject: [PATCH 086/272] post-merge fixups (keyutil, preferred_peers) --- src/allmydata/client.py | 3 +-- src/allmydata/node.py | 2 +- src/allmydata/scripts/tahoe_grid_manager.py | 17 ++++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ced282900..24f217288 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -580,8 +580,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con tub_maker=tub_creator, node_config=config, storage_client_config=storage_client_config, - preferred_peers=preferred_peers, - grid_manager_keys=grid_manager_keys, + grid_manager_keys=grid_manager_keys, # XXX maybe roll into above storage_client_config? ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 404f8a125..676975728 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -205,7 +205,7 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None): # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) - configutil.validate_config(fname, parser, _valid_config) + configutil.validate_config('', parser, _valid_config) def write_new_config(cfg): """ diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index b33552861..6c40f0814 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -13,7 +13,7 @@ from twisted.python import usage from twisted.python.filepath import FilePath from allmydata.util import fileutil from allmydata.util import base32 -from allmydata.util import keyutil +from allmydata.crypto import ed25519 from twisted.internet.defer import inlineCallbacks, returnValue @@ -50,7 +50,7 @@ class AddOptions(BaseOptions): self['name'] = unicode(args[0]) try: # WTF?! why does it want 'str' and not six.text_type? - self['storage_public_key'] = keyutil.parse_pubkey(args[1]) + self['storage_public_key'] = ed25519.verifying_key_from_string(args[1]) except Exception as e: raise usage.UsageError( "Invalid public_key argument: {}".format(e) @@ -132,8 +132,11 @@ def _create_gridmanager(): :return: an object providing the GridManager interface initialized with a new random keypair """ - private_key_bytes, public_key_bytes = keyutil.make_keypair() - return _GridManager(private_key_bytes, {}) + private_key, public_key = ed25519.create_signing_keypair() + return _GridManager( + ed25519.string_from_signing_key(private_key), + {}, + ) def _create(gridoptions, options): """ @@ -195,7 +198,7 @@ class _GridManager(object): private_key_bytes = config['private_key'].encode('ascii') try: - private_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) + private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes) except Exception as e: raise ValueError( "Invalid Grid Manager private_key: {}".format(e) @@ -209,7 +212,7 @@ class _GridManager(object): ) storage_servers[name] = _GridManagerStorageServer( name, - keyutil.parse_pubkey(srv_config['public_key'].encode('ascii')), + ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), None, ) @@ -225,7 +228,7 @@ class _GridManager(object): def __init__(self, private_key_bytes, storage_servers): self._storage_servers = dict() if storage_servers is None else storage_servers self._private_key_bytes = private_key_bytes - self._private_key, _ = keyutil.parse_privkey(self._private_key_bytes) + self._private_key, _ = ed25519.signing_keypair_from_string(self._private_key_bytes) self._version = 0 @property From 38069e51fb65254418fd63f5c3e454b488e61e49 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:43:17 -0600 Subject: [PATCH 087/272] keyutil -> ed25519 --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 24f217288..54a4e9ee2 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -564,7 +564,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] for name, gm_key in config.enumerate_section('grid_managers').items(): grid_manager_keys.append( - keyutil.parse_pubkey(gm_key) + ed25519.verifying_key_from_string(gm_key) ) # we don't actually use this keypair for anything (yet) as far From e6cb700bcc2e33f743c0f100591ca6013a264243 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:43:37 -0600 Subject: [PATCH 088/272] incorrect merge conflict-resolution --- src/allmydata/client.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 54a4e9ee2..36f870a64 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -954,13 +954,6 @@ class _Client(node.Node, pollmixin.PollMixin): # of the Grid Manager (should that go in the config too, # then? How to handle multiple grid-managers?) - furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) - furl = self.tub.registerReference(ss, furlFile=furl_file) - ann = { - "anonymous-storage-FURL": furl, - "permutation-seed-base32": self._init_permutation_seed(ss), - "grid-manager-certificates": grid_manager_certificates, - } for ic in self.introducer_clients: ic.publish("storage", announcement, self._node_private_key) From 02c3401e016e48f0a1a080168cde28c57e7a6bea Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:43:49 -0600 Subject: [PATCH 089/272] make code more like master --- src/allmydata/node.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 676975728..32b78b094 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -205,13 +205,14 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None): # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) - configutil.validate_config('', parser, _valid_config) + fname = "" + configutil.validate_config(fname, parser, _valid_config) def write_new_config(cfg): """ We throw away any attempt to persist """ - return _Config(parser, portnumfile, basedir, '', write_new_config) + return _Config(parser, portnumfile, basedir, fname, write_new_config) def get_app_versions(): From 765d9daa8cfee588ee70167b32d44b5c9e412fd5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:57:06 -0600 Subject: [PATCH 090/272] keyutil -> ed25519 --- integration/test_grid_manager.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d46bb6b10..ffe6733d1 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -5,7 +5,7 @@ import shutil from os import mkdir, unlink, listdir, utime from os.path import join, exists, getmtime -from allmydata.util import keyutil +from allmydata.crypto import ed25519 from allmydata.util import base32 from allmydata.util import configutil @@ -23,8 +23,7 @@ def test_create_certificate(reactor): reactor, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) - pubkey = keyutil.parse_pubkey(pubkey_bytes) + privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) # Note that zara + her key here are arbitrary and don't match any # "actual" clients in the test-grid; we're just checking that the @@ -115,8 +114,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd reactor, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) - pubkey = keyutil.parse_pubkey(pubkey_bytes) + privkey, _ = ed25519.signing_keypair_from_string(privkey_bytes) # create certificates for first 2 storage-servers for idx, storage in enumerate(storage_nodes[:2]): @@ -170,7 +168,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) print(dir(config)) config.add_section("grid_managers") - config.set("grid_managers", "test", pubkey_bytes) + config.set("grid_managers", "test", ed25519.string_from_verifying_key(pubkey)) config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) carol.signalProcess('TERM') yield carol._protocol.exited From 37a23d8b5e551e0a194b2c5a96357ce8fee67209 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 21:02:39 -0600 Subject: [PATCH 091/272] debug --- src/allmydata/storage_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c076624ab..7d8df34c7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -233,7 +233,6 @@ class StorageFarmBroker(service.MultiService): assert isinstance(server_id, unicode) # from YAML server_id = server_id.encode("ascii") handler_overrides = server.get("connections", {}) - print("ANN", server["ann"]) s = NativeStorageServer( server_id, server["ann"], From 5e1e90df89655abdddd113e107c6fbe461d4fac5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 21:02:50 -0600 Subject: [PATCH 092/272] pass on stdin= --- integration/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 0c41c6bd3..d85050d4e 100644 --- a/integration/util.py +++ b/integration/util.py @@ -151,11 +151,10 @@ def _cleanup_tahoe_process(tahoe_transport, exited): pass -def run_tahoe(reactor, request, *args): +def run_tahoe(reactor, request, stdin=None, *args): """ Helper to run tahoe with optional coverage """ - stdin = kwargs.get('stdin', None) protocol = _CollectOutputProtocol(stdin=stdin) process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) process.exited = protocol.done From f21c1f4b4f4cfa8d23322c853903e20e0b458a98 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:29:04 -0600 Subject: [PATCH 093/272] pass on request arg --- integration/test_grid_manager.py | 34 ++++++++++++++++---------------- integration/util.py | 8 ++++++-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index ffe6733d1..4910bbfa2 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -15,12 +15,12 @@ import pytest_twisted @pytest_twisted.inlineCallbacks -def test_create_certificate(reactor): +def test_create_certificate(reactor, request): """ The Grid Manager produces a valid, correctly-signed certificate. """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) @@ -29,12 +29,12 @@ def test_create_certificate(reactor): # "actual" clients in the test-grid; we're just checking that the # Grid Manager signs this properly. gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) zara_cert_bytes = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "sign", "zara", + reactor, request, "grid-manager", "--config", "-", "sign", "zara", stdin=gm_config, ) zara_cert = json.loads(zara_cert_bytes) @@ -48,21 +48,21 @@ def test_create_certificate(reactor): @pytest_twisted.inlineCallbacks -def test_remove_client(reactor): +def test_remove_client(reactor, request): """ A Grid Manager can add and successfully remove a client """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdin=gm_config, ) @@ -70,7 +70,7 @@ def test_remove_client(reactor): assert "yakov" in json.loads(gm_config)['storage_servers'] gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "remove", + reactor, request, "grid-manager", "--config", "-", "remove", "zara", stdin=gm_config, ) @@ -79,23 +79,23 @@ def test_remove_client(reactor): @pytest_twisted.inlineCallbacks -def test_remove_last_client(reactor): +def test_remove_last_client(reactor, request): """ A Grid Manager can remove all clients """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) assert "zara" in json.loads(gm_config)['storage_servers'] gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "remove", + reactor, request, "grid-manager", "--config", "-", "remove", "zara", stdin=gm_config, ) @@ -111,7 +111,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd valid certificates. """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') privkey, _ = ed25519.signing_keypair_from_string(privkey_bytes) @@ -142,7 +142,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd for idx, storage in enumerate(storage_nodes[:2]): print(idx, storage) cert = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "sign", + reactor, request, "grid-manager", "--config", "-", "sign", "storage{}".format(idx), stdin=gm_config, ) @@ -183,9 +183,9 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd try: yield util.run_tahoe( - reactor, "--node-directory", carol._node_dir, + reactor, request, "--node-directory", carol._node_dir, "put", "-", - stdin="some content" * 200, + stdin="some content\n" * 200, ) assert False, "Should get a failure" except util.ProcessFailed as e: diff --git a/integration/util.py b/integration/util.py index d85050d4e..0ad02aed9 100644 --- a/integration/util.py +++ b/integration/util.py @@ -151,10 +151,14 @@ def _cleanup_tahoe_process(tahoe_transport, exited): pass -def run_tahoe(reactor, request, stdin=None, *args): +def run_tahoe(reactor, request, *args, **kwargs): """ - Helper to run tahoe with optional coverage + Helper to run tahoe with optional coverage. + + :returns: a Deferred that fires when the command is done (or a + ProcessFailed exception if it exits non-zero) """ + stdin = kwargs.get("stdin", None) protocol = _CollectOutputProtocol(stdin=stdin) process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) process.exited = protocol.done From fea9fc03d0d73b24658f6334027f85a90bf071a8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:29:31 -0600 Subject: [PATCH 094/272] fix more keyutil -> ed25519 --- integration/test_grid_manager.py | 6 +++--- src/allmydata/scripts/tahoe_grid_manager.py | 18 ++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 4910bbfa2..0f3070f91 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -120,11 +120,11 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd for idx, storage in enumerate(storage_nodes[:2]): pubkey_fname = join(storage._node_dir, "node.pubkey") with open(pubkey_fname, 'r') as f: - pubkey = f.read().strip() + pubkey_str = f.read().strip() gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", - "storage{}".format(idx), pubkey, + reactor, request, "grid-manager", "--config", "-", "add", + "storage{}".format(idx), pubkey_str, stdin=gm_config, ) assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 6c40f0814..4403b9f48 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -5,8 +5,6 @@ import json import time from datetime import datetime -from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though - from allmydata.scripts.common import BaseOptions from allmydata.util.abbreviate import abbreviate_time from twisted.python import usage @@ -171,7 +169,7 @@ class _GridManagerStorageServer(object): self._certificates.append(certificate) def public_key(self): - return "pub-v0-" + base32.b2a(self._public_key.vk_bytes) + return ed25519.string_from_verifying_key(self._public_key) def marshal(self): return { @@ -228,7 +226,7 @@ class _GridManager(object): def __init__(self, private_key_bytes, storage_servers): self._storage_servers = dict() if storage_servers is None else storage_servers self._private_key_bytes = private_key_bytes - self._private_key, _ = ed25519.signing_keypair_from_string(self._private_key_bytes) + self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) self._version = 0 @property @@ -236,8 +234,7 @@ class _GridManager(object): return self._storage_servers def public_identity(self): - verify_key_bytes = self._private_key.get_verifying_key_bytes() - return base32.b2a(verify_key_bytes) + return ed25519.string_from_verifying_key(self._public_key) def sign(self, name): try: @@ -258,10 +255,8 @@ class _GridManager(object): u"signature": base32.b2a(sig), } - if True: - verify_key_bytes = self._private_key.get_verifying_key_bytes() - vk = ed25519.VerifyingKey(verify_key_bytes) - assert vk.verify(sig, cert_data) is None, "cert should verify" + vk = ed25519.verifying_key_from_signing_key(self._private_key) + assert vk.verify(sig, cert_data) is None, "cert should verify" return certificate @@ -276,7 +271,6 @@ class _GridManager(object): raise KeyError( "Already have a storage server called '{}'".format(name) ) - assert public_key.vk_bytes ss = _GridManagerStorageServer(name, public_key, None) self._storage_servers[name] = ss return ss @@ -374,7 +368,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - print("pub-v0-{}".format(gm.public_identity())) + print(gm.public_identity()) def _add(gridoptions, options): From 5bd5d8145e459bcf1d02c8d7ec18a82a01ed1af8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:29:45 -0600 Subject: [PATCH 095/272] kill process --- integration/test_grid_manager.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 0f3070f91..448c641c0 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -155,8 +155,8 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) # re-start this storage server - storage.signalProcess('TERM') - yield storage._protocol.exited + storage.transport.signalProcess('TERM') + yield storage.transport._protocol.exited time.sleep(1) storage_nodes[idx] = yield util._run_node( reactor, storage._node_dir, request, None, @@ -166,12 +166,11 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd # carol to have the grid-manager certificate config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) - print(dir(config)) config.add_section("grid_managers") - config.set("grid_managers", "test", ed25519.string_from_verifying_key(pubkey)) + config.set("grid_managers", "test", pubkey_str) config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) - carol.signalProcess('TERM') - yield carol._protocol.exited + carol.transport.signalProcess('TERM') + yield carol.transport._protocol.exited carol = yield util._run_node( reactor, carol._node_dir, request, None, From 0bdfae845ed623f5e926dbef1041a6606b4f6bb9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:30:05 -0600 Subject: [PATCH 096/272] actually put grid-manager-certificates in announcement --- src/allmydata/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 36f870a64..8e063a1f5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -943,10 +943,9 @@ class _Client(node.Node, pollmixin.PollMixin): announcement.update(plugins_announcement) - grid_manager_certificates = [] - if self.config.get_config("storage", "grid_management", default=False, boolean=True): grid_manager_certificates = _load_grid_manager_certificates(self.config) + announcement["grid-manager-certificates"] = grid_manager_certificates # XXX we should probably verify that the certificates are # valid and not expired, as that could be confusing for the From 38e6557f817db2bf6819964cc1b75ff73be42f20 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:30:23 -0600 Subject: [PATCH 097/272] fix imports --- src/allmydata/storage_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7d8df34c7..09da110db 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -29,7 +29,11 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . # 6: implement other sorts of IStorageClient classes: S3, etc -import re, time, hashlib +import re +import time +import json +import hashlib +from datetime import datetime from ConfigParser import ( NoSectionError, ) From 975491b519d0f3f8c7b840726daf4fadd73b64d0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:30:35 -0600 Subject: [PATCH 098/272] allow 'anything valid' sections --- src/allmydata/util/configutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index d58bc4217..ae2a568f5 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -95,8 +95,11 @@ class ValidConfiguration(object): :return: True if the given section name, ite name pair is valid, False otherwise. """ + valid_items = self._static_valid_sections.get(section_name, ()) + if valid_items is None: + return True return ( - item_name in self._static_valid_sections.get(section_name, ()) or + item_name in valid_items or self._is_valid_item(section_name, item_name) ) From c029698435ecb05129e6f5fa549a179ee08a3f00 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 8 May 2020 00:47:17 -0600 Subject: [PATCH 099/272] fix more keyutil things and key-handling in test --- integration/test_grid_manager.py | 15 ++++++++------- integration/util.py | 6 +++--- src/allmydata/scripts/tahoe_grid_manager.py | 4 ++-- src/allmydata/storage_client.py | 6 ++++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 448c641c0..ba0debd5c 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -113,8 +113,8 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd gm_config = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "create", ) - privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, _ = ed25519.signing_keypair_from_string(privkey_bytes) + gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) # create certificates for first 2 storage-servers for idx, storage in enumerate(storage_nodes[:2]): @@ -140,7 +140,6 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd print("inserting certificates") # insert their certificates for idx, storage in enumerate(storage_nodes[:2]): - print(idx, storage) cert = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "sign", "storage{}".format(idx), @@ -152,12 +151,12 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config.set("storage", "grid_management", "True") config.add_section("grid_manager_certificates") config.set("grid_manager_certificates", "default", "gridmanager.cert") - config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) + with open(join(storage._node_dir, "tahoe.cfg"), "w") as f: + config.write(f) # re-start this storage server storage.transport.signalProcess('TERM') yield storage.transport._protocol.exited - time.sleep(1) storage_nodes[idx] = yield util._run_node( reactor, storage._node_dir, request, None, ) @@ -167,14 +166,16 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) config.add_section("grid_managers") - config.set("grid_managers", "test", pubkey_str) - config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) + config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) + with open(join(carol._node_dir, "tahoe.cfg"), "w") as f: + config.write(f) carol.transport.signalProcess('TERM') yield carol.transport._protocol.exited carol = yield util._run_node( reactor, carol._node_dir, request, None, ) + yield util.await_client_ready(carol, servers=5) # try to put something into the grid, which should fail (because # carol has happy=3 but should only find storage0, storage1 to be diff --git a/integration/util.py b/integration/util.py index 0ad02aed9..7178e1d67 100644 --- a/integration/util.py +++ b/integration/util.py @@ -478,7 +478,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2): +def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -502,8 +502,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2): time.sleep(1) continue - if len(js['servers']) == 0: - print("waiting because no servers at all") + if len(js['servers']) < servers: + print("waiting because fewer than {} server(s)".format(servers)) time.sleep(1) continue server_times = [ diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 4403b9f48..64561b95e 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -249,14 +249,14 @@ class _GridManager(object): "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') - sig = self._private_key.sign(cert_data) + sig = ed25519.sign_data(self._private_key, cert_data) certificate = { u"certificate": cert_data, u"signature": base32.b2a(sig), } vk = ed25519.verifying_key_from_signing_key(self._private_key) - assert vk.verify(sig, cert_data) is None, "cert should verify" + ed25519.verify_signature(vk, sig, cert_data) return certificate diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 09da110db..523dfa066 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -67,6 +67,7 @@ from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash +from allmydata.crypto import ed25519 # who is responsible for de-duplication? # both? @@ -473,11 +474,12 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): now_fn = datetime.utcnow try: - gm_key.verify( + ed25519.verify_signature( + gm_key, base32.a2b(alleged_cert['signature'].encode('ascii')), alleged_cert['certificate'].encode('ascii'), ) - except ed25519.BadSignatureError: + except ed25519.BadSignature: return False # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) From 0540df8ab1af6f04dd97b4b4cfbfede48b8e89cb Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 10 May 2020 00:56:52 -0600 Subject: [PATCH 100/272] Re-factor grid fixtures to be mostly helpers This lets us use them to create our own tiny grid to test the grid-manager certificates (instead of messing with "the" grid) --- integration/conftest.py | 121 ++------------ integration/grid.py | 262 +++++++++++++++++++++++++++++++ integration/test_grid_manager.py | 124 ++++++++------- integration/test_web.py | 10 +- integration/util.py | 9 +- 5 files changed, 352 insertions(+), 174 deletions(-) create mode 100644 integration/grid.py diff --git a/integration/conftest.py b/integration/conftest.py index 5395d7c5f..462ac8b60 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -36,6 +36,7 @@ from util import ( await_client_ready, TahoeProcess, ) +import grid # pytest customization hooks @@ -106,60 +107,10 @@ def flog_binary(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): - out_protocol = _CollectOutputProtocol() - gather_dir = join(temp_dir, 'flog_gather') - reactor.spawnProcess( - out_protocol, - flog_binary, - ( - 'flogtool', 'create-gatherer', - '--location', 'tcp:localhost:3117', - '--port', '3117', - gather_dir, - ) + fg = pytest_twisted.blockon( + grid.create_flog_gatherer(reactor, request, temp_dir, flog_binary) ) - pytest_twisted.blockon(out_protocol.done) - - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") - twistd_process = reactor.spawnProcess( - twistd_protocol, - which('twistd')[0], - ( - 'twistd', '--nodaemon', '--python', - join(gather_dir, 'gatherer.tac'), - ), - path=gather_dir, - ) - pytest_twisted.blockon(twistd_protocol.magic_seen) - - def cleanup(): - _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) - - flog_file = mktemp('.flog_dump') - flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) - flog_dir = join(temp_dir, 'flog_gather') - flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] - - print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) - reactor.spawnProcess( - flog_protocol, - flog_binary, - ( - 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) - ), - ) - print("Waiting for flogtool to complete") - try: - pytest_twisted.blockon(flog_protocol.done) - except ProcessTerminated as e: - print("flogtool exited unexpectedly: {}".format(str(e))) - print("Flogtool completed") - - request.addfinalizer(cleanup) - - with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: - furl = f.read().strip() - return furl + return fg @pytest.fixture(scope='session') @@ -169,64 +120,14 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): include_result=False, ) def introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer0 -web.port = 4560 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - - intro_dir = join(temp_dir, 'introducer') - print("making introducer", intro_dir) - - if not exists(intro_dir): - mkdir(intro_dir) - done_proto = _ProcessExitedProtocol() - _tahoe_runner_optional_coverage( - done_proto, - reactor, - request, - ( - 'create-introducer', - '--listen=tcp', - '--hostname=localhost', - intro_dir, - ), - ) - pytest_twisted.blockon(done_proto.done) - - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) - - # on windows, "tahoe start" means: run forever in the foreground, - # but on linux it means daemonize. "tahoe run" is consistent - # between platforms. - protocol = _MagicTextProtocol('introducer running') - transport = _tahoe_runner_optional_coverage( - protocol, - reactor, - request, - ( - 'run', - intro_dir, - ), - ) - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) - - pytest_twisted.blockon(protocol.magic_seen) - return TahoeProcess(transport, intro_dir) + intro = pytest_twisted.blockon(grid.create_introducer(reactor, request, temp_dir, flog_gatherer)) + return intro @pytest.fixture(scope='session') @log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"]) def introducer_furl(introducer, temp_dir): - furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl') - while not exists(furl_fname): - print("Don't see {} yet".format(furl_fname)) - sleep(.1) - furl = open(furl_fname, 'r').read() - return furl + return introducer.furl @pytest.fixture(scope='session') @@ -313,12 +214,10 @@ def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, # start all 5 nodes in parallel for x in range(5): name = 'node{}'.format(x) - web_port= 9990 + x + web_port = 'tcp:{}:interface=localhost'.format(9990 + x) nodes_d.append( - _create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, name, - web_port="tcp:{}:interface=localhost".format(web_port), - storage=True, + grid.create_storage_server( + reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, ) ) nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) diff --git a/integration/grid.py b/integration/grid.py new file mode 100644 index 000000000..7df86d248 --- /dev/null +++ b/integration/grid.py @@ -0,0 +1,262 @@ +from os import mkdir, listdir, environ +from os.path import join, exists +from tempfile import mkdtemp, mktemp + +from twisted.python.procutils import which +from twisted.internet.defer import ( + inlineCallbacks, + returnValue, +) +from twisted.internet.task import ( + deferLater, +) +from twisted.internet.interfaces import ( + IProcessTransport, + IProcessProtocol, + IProtocol, +) + +from util import ( + _CollectOutputProtocol, + _MagicTextProtocol, + _DumpOutputProtocol, + _ProcessExitedProtocol, + _create_node, + _run_node, + _cleanup_tahoe_process, + _tahoe_runner_optional_coverage, + await_client_ready, + TahoeProcess, +) + +import attr +import pytest_twisted + + +@attr.s +class FlogGatherer(object): + """ + Flog Gatherer process. + """ + + process = attr.ib( + validator=attr.validators.provides(IProcessTransport) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + furl = attr.ib() + + +@inlineCallbacks +def create_flog_gatherer(reactor, request, temp_dir, flog_binary): + out_protocol = _CollectOutputProtocol() + gather_dir = join(temp_dir, 'flog_gather') + reactor.spawnProcess( + out_protocol, + flog_binary, + ( + 'flogtool', 'create-gatherer', + '--location', 'tcp:localhost:3117', + '--port', '3117', + gather_dir, + ) + ) + yield out_protocol.done + + twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_process = reactor.spawnProcess( + twistd_protocol, + which('twistd')[0], + ( + 'twistd', '--nodaemon', '--python', + join(gather_dir, 'gatherer.tac'), + ), + path=gather_dir, + ) + yield twistd_protocol.magic_seen + + def cleanup(): + _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) + + flog_file = mktemp('.flog_dump') + flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) + flog_dir = join(temp_dir, 'flog_gather') + flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] + + print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) + reactor.spawnProcess( + flog_protocol, + flog_binary, + ( + 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) + ), + ) + print("Waiting for flogtool to complete") + try: + pytest_twisted.blockon(flog_protocol.done) + except ProcessTerminated as e: + print("flogtool exited unexpectedly: {}".format(str(e))) + print("Flogtool completed") + + request.addfinalizer(cleanup) + + with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: + furl = f.read().strip() + returnValue( + FlogGatherer( + protocol=twistd_protocol, + process=twistd_process, + furl=furl, + ) + ) + + +@attr.s +class StorageServer(object): + """ + Represents a Tahoe Storage Server + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + + # XXX needs a restart() probably .. or at least a stop() and + # start() + + +@inlineCallbacks +def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, + needed=2, happy=3, total=4): + """ + Create a new storage server + """ + from util import _create_node + node_process = yield _create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, + name, web_port, storage=True, needed=needed, happy=happy, total=total, + ) + storage = StorageServer( + process=node_process, + protocol=node_process.transport._protocol, + ) + returnValue(storage) + +@attr.s +class Introducer(object): + """ + Reprsents a running introducer + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + furl = attr.ib() + + +_introducer_num = 0 + + +@inlineCallbacks +def create_introducer(reactor, request, temp_dir, flog_gatherer): + """ + Run a new Introducer and return an Introducer instance. + """ + global _introducer_num + config = ( + '[node]\n' + 'nickname = introducer{num}\n' + 'web.port = {port}\n' + 'log_gatherer.furl = {log_furl}\n' + ).format( + num=_introducer_num, + log_furl=flog_gatherer.furl, + port=4560 + _introducer_num, + ) + _introducer_num += 1 + + intro_dir = join(temp_dir, 'introducer{}'.format(_introducer_num)) + print("making introducer", intro_dir, _introducer_num) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = _ProcessExitedProtocol() + _tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=tcp', + '--hostname=localhost', + intro_dir, + ), + ) + yield done_proto.done + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # on windows, "tahoe start" means: run forever in the foreground, + # but on linux it means daemonize. "tahoe run" is consistent + # between platforms. + protocol = _MagicTextProtocol('introducer running') + transport = _tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def clean(): + return _cleanup_tahoe_process(transport, protocol.exited) + request.addfinalizer(clean) + + yield protocol.magic_seen + + furl_fname = join(intro_dir, 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + yield deferLater(reactor, .1, lambda: None) + furl = open(furl_fname, 'r').read() + + returnValue( + Introducer( + process=TahoeProcess(transport, intro_dir), + protocol=protocol, + furl=furl, + ) + ) + + +@attr.s +class Grid(object): + """ + Represents an entire Tahoe Grid setup + + A Grid includes an Introducer, Flog Gatherer and some number of + Storage Servers. + """ + + introducer = attr.ib(default=None) + flog_gatherer = attr.ib(default=None) + storage_servers = attr.ib(factory=list) + + @storage_servers.validator + def check(self, attribute, value): + for server in value: + if not isinstance(server, StorageServer): + raise ValueError( + "storage_servers must be StorageServer" + ) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index ba0debd5c..d0fdcdd31 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -104,86 +104,102 @@ def test_remove_last_client(reactor, request): @pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): +def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): """ - A client with happines=3 fails to upload to a Grid when it is - using Grid Manager and there are only two storage-servers with - valid certificates. + A client with happines=2 fails to upload to a Grid when it is + using Grid Manager and there is only 1 storage server with a valid + certificate. """ + import grid + introducer = yield grid.create_introducer(reactor, request, temp_dir, flog_gatherer) + storage0 = yield grid.create_storage_server( + reactor, request, temp_dir, introducer, flog_gatherer, + name="gm_storage0", + web_port="tcp:9995:interface=localhost", + needed=2, + happy=2, + total=2, + ) + storage1 = yield grid.create_storage_server( + reactor, request, temp_dir, introducer, flog_gatherer, + name="gm_storage1", + web_port="tcp:9996:interface=localhost", + needed=2, + happy=2, + total=2, + ) + gm_config = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "create", ) gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) - # create certificates for first 2 storage-servers - for idx, storage in enumerate(storage_nodes[:2]): - pubkey_fname = join(storage._node_dir, "node.pubkey") - with open(pubkey_fname, 'r') as f: - pubkey_str = f.read().strip() + # create certificate for the first storage-server + pubkey_fname = join(storage0.process.node_dir, "node.pubkey") + with open(pubkey_fname, 'r') as f: + pubkey_str = f.read().strip() - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "storage{}".format(idx), pubkey_str, - stdin=gm_config, - ) - assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] + gm_config = yield util.run_tahoe( + reactor, request, "grid-manager", "--config", "-", "add", + "storage0", pubkey_str, + stdin=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - # XXX FIXME need to shut-down and nuke carol when we're done this - # test (i.d. request.addfinalizer) - carol = yield util._create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, "carol", - web_port="tcp:9982:interface=localhost", + # XXX FIXME want a grid.create_client() or similar + diana = yield util._create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, "diana", + web_port="tcp:9984:interface=localhost", storage=False, ) - print("inserting certificates") - # insert their certificates - for idx, storage in enumerate(storage_nodes[:2]): - cert = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", - "storage{}".format(idx), - stdin=gm_config, - ) - with open(join(storage._node_dir, "gridmanager.cert"), "w") as f: - f.write(cert) - config = configutil.get_config(join(storage._node_dir, "tahoe.cfg")) - config.set("storage", "grid_management", "True") - config.add_section("grid_manager_certificates") - config.set("grid_manager_certificates", "default", "gridmanager.cert") - with open(join(storage._node_dir, "tahoe.cfg"), "w") as f: - config.write(f) + print("inserting certificate") + cert = yield util.run_tahoe( + reactor, request, "grid-manager", "--config", "-", "sign", "storage0", + stdin=gm_config, + ) + with open(join(storage0.process.node_dir, "gridmanager.cert"), "w") as f: + f.write(cert) + config = configutil.get_config(join(storage0.process.node_dir, "tahoe.cfg")) + config.set("storage", "grid_management", "True") + config.add_section("grid_manager_certificates") + config.set("grid_manager_certificates", "default", "gridmanager.cert") + with open(join(storage0.process.node_dir, "tahoe.cfg"), "w") as f: + config.write(f) - # re-start this storage server - storage.transport.signalProcess('TERM') - yield storage.transport._protocol.exited - storage_nodes[idx] = yield util._run_node( - reactor, storage._node_dir, request, None, - ) + # re-start this storage server + storage0.process.transport.signalProcess('TERM') + yield storage0.protocol.exited + yield util._run_node( + reactor, storage0.process.node_dir, request, None, cleanup=False, + ) - # now only two storage-servers have certificates .. configure - # carol to have the grid-manager certificate + yield util.await_client_ready(diana, servers=2) - config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) + # now only one storage-server has the certificate .. configure + # diana to have the grid-manager certificate + + config = configutil.get_config(join(diana.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) - with open(join(carol._node_dir, "tahoe.cfg"), "w") as f: + with open(join(diana.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - carol.transport.signalProcess('TERM') - yield carol.transport._protocol.exited + diana.transport.signalProcess('TERM') + yield diana.transport._protocol.exited - carol = yield util._run_node( - reactor, carol._node_dir, request, None, + diana = yield util._run_node( + reactor, diana._node_dir, request, None, cleanup=False, ) - yield util.await_client_ready(carol, servers=5) + yield util.await_client_ready(diana, servers=2) # try to put something into the grid, which should fail (because - # carol has happy=3 but should only find storage0, storage1 to be - # acceptable to upload to) + # diana has happy=2 but should only find storage0 to be acceptable + # to upload to) try: yield util.run_tahoe( - reactor, request, "--node-directory", carol._node_dir, + reactor, request, "--node-directory", diana._node_dir, "put", "-", stdin="some content\n" * 200, ) diff --git a/integration/test_web.py b/integration/test_web.py index 4ba0a6fd1..ac7f5f285 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -96,7 +96,7 @@ def test_helper_status(storage_nodes): successfully GET the /helper_status page """ - url = util.node_url(storage_nodes[0].node_dir, "helper_status") + url = util.node_url(storage_nodes[0].process.node_dir, "helper_status") resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") @@ -418,7 +418,7 @@ def test_storage_info(storage_nodes): storage0 = storage_nodes[0] requests.get( - util.node_url(storage0.node_dir, u"storage"), + util.node_url(storage0.process.node_dir, u"storage"), ) @@ -429,7 +429,7 @@ def test_storage_info_json(storage_nodes): storage0 = storage_nodes[0] resp = requests.get( - util.node_url(storage0.node_dir, u"storage"), + util.node_url(storage0.process.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) @@ -441,12 +441,12 @@ def test_introducer_info(introducer): retrieve and confirm /introducer URI for the introducer """ resp = requests.get( - util.node_url(introducer.node_dir, u""), + util.node_url(introducer.process.node_dir, u""), ) assert "Introducer" in resp.content resp = requests.get( - util.node_url(introducer.node_dir, u""), + util.node_url(introducer.process.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) diff --git a/integration/util.py b/integration/util.py index 7178e1d67..f5c3f3462 100644 --- a/integration/util.py +++ b/integration/util.py @@ -144,7 +144,7 @@ def _cleanup_tahoe_process(tahoe_transport, exited): try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') - print("signaled, blocking on exit") + print("signaled, blocking on exit {}".format(exited)) pytest_twisted.blockon(exited) print("exited, goodbye") except ProcessExitedAlready: @@ -210,7 +210,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text): +def _run_node(reactor, node_dir, request, magic_text, cleanup=True): """ Run a tahoe process from its node_dir. @@ -236,7 +236,8 @@ def _run_node(reactor, node_dir, request, magic_text): ) transport.exited = protocol.exited - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + if cleanup: + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) # XXX abusing the Deferred; should use .when_magic_seen() pattern @@ -291,7 +292,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam def created(_): config_path = join(node_dir, 'tahoe.cfg') config = get_config(config_path) - set_config(config, 'node', 'log_gatherer.furl', flog_gatherer) + set_config(config, 'node', 'log_gatherer.furl', flog_gatherer.furl) write_config(config_path, config) created_d.addCallback(created) From 9a62b1f93f784e213a352a3f7df6cc19939f0792 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 10 May 2020 22:05:47 -0600 Subject: [PATCH 101/272] cleanup is non-optional --- integration/test_grid_manager.py | 4 ++-- integration/util.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d0fdcdd31..dc25c3b29 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -172,7 +172,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): storage0.process.transport.signalProcess('TERM') yield storage0.protocol.exited yield util._run_node( - reactor, storage0.process.node_dir, request, None, cleanup=False, + reactor, storage0.process.node_dir, request, None, ) yield util.await_client_ready(diana, servers=2) @@ -189,7 +189,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): yield diana.transport._protocol.exited diana = yield util._run_node( - reactor, diana._node_dir, request, None, cleanup=False, + reactor, diana._node_dir, request, None, ) yield util.await_client_ready(diana, servers=2) diff --git a/integration/util.py b/integration/util.py index f5c3f3462..2cdcb2563 100644 --- a/integration/util.py +++ b/integration/util.py @@ -210,7 +210,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text, cleanup=True): +def _run_node(reactor, node_dir, request, magic_text): """ Run a tahoe process from its node_dir. @@ -236,8 +236,7 @@ def _run_node(reactor, node_dir, request, magic_text, cleanup=True): ) transport.exited = protocol.exited - if cleanup: - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) # XXX abusing the Deferred; should use .when_magic_seen() pattern From a9fe12063a6463bd9395f03db5191abf0927de52 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 11 May 2020 00:20:48 -0600 Subject: [PATCH 102/272] docs --- integration/grid.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/integration/grid.py b/integration/grid.py index 7df86d248..726c22e40 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -1,3 +1,13 @@ +""" +Classes which directly represent various kinds of Tahoe processes +that co-operate to for "a Grid". + +These methods and objects are used by conftest.py fixtures but may +also be used as direct helpers for tests that don't want to (or can't) +rely on 'the' global grid as provided by fixtures like 'alice' or +'storage_servers'. +""" + from os import mkdir, listdir, environ from os.path import join, exists from tempfile import mkdtemp, mktemp @@ -33,6 +43,15 @@ import attr import pytest_twisted +# further directions: +# - "Grid" is unused, basically -- tie into the rest? +# - could make a Grid instance mandatory for create_* calls +# - could instead make create_* calls methods of Grid +# - Bring more 'util' or 'conftest' code into here +# - stop()/start()/restart() methods on StorageServer etc +# - more-complex stuff like config changes (which imply a restart too)? + + @attr.s class FlogGatherer(object): """ From c5fb2b5a8b0f874e4992ca392bc234224d232371 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 12:22:35 -0600 Subject: [PATCH 103/272] refactor more code into grid.py --- integration/conftest.py | 55 +++++---- integration/grid.py | 191 ++++++++++++++++++++++++++++--- integration/test_grid_manager.py | 47 +++----- integration/util.py | 2 +- 4 files changed, 223 insertions(+), 72 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 462ac8b60..11ee61d11 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -36,7 +36,11 @@ from util import ( await_client_ready, TahoeProcess, ) -import grid +from grid import ( + create_port_allocator, + create_flog_gatherer, + create_grid, +) # pytest customization hooks @@ -73,6 +77,12 @@ def reactor(): return _reactor +@pytest.fixture(scope='session') +@log_call(action_type=u"integration:port_allocator", include_result=False) +def port_allocator(reactor): + return create_port_allocator(start_port=45000) + + @pytest.fixture(scope='session') @log_call(action_type=u"integration:temp_dir", include_args=[]) def temp_dir(request): @@ -108,20 +118,23 @@ def flog_binary(): @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): fg = pytest_twisted.blockon( - grid.create_flog_gatherer(reactor, request, temp_dir, flog_binary) + create_flog_gatherer(reactor, request, temp_dir, flog_binary) ) return fg @pytest.fixture(scope='session') -@log_call( - action_type=u"integration:introducer", - include_args=["temp_dir", "flog_gatherer"], - include_result=False, -) -def introducer(reactor, temp_dir, flog_gatherer, request): - intro = pytest_twisted.blockon(grid.create_introducer(reactor, request, temp_dir, flog_gatherer)) - return intro +@log_call(action_type=u"integration:grid", include_args=[]) +def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + g = pytest_twisted.blockon( + create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) + ) + return g + + +@pytest.fixture(scope='session') +def introducer(grid): + return grid.introducer @pytest.fixture(scope='session') @@ -206,26 +219,20 @@ def tor_introducer_furl(tor_introducer, temp_dir): @pytest.fixture(scope='session') @log_call( action_type=u"integration:storage_nodes", - include_args=["temp_dir", "introducer_furl", "flog_gatherer"], + include_args=["grid"], include_result=False, ) -def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request): +def storage_nodes(grid): nodes_d = [] # start all 5 nodes in parallel for x in range(5): - name = 'node{}'.format(x) - web_port = 'tcp:{}:interface=localhost'.format(9990 + x) - nodes_d.append( - grid.create_storage_server( - reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, - ) - ) + #nodes_d.append(grid.add_storage_node()) + pytest_twisted.blockon(grid.add_storage_node()) + nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) - nodes = [] - for ok, process in nodes_status: - assert ok, "Storage node creation failed: {}".format(process) - nodes.append(process) - return nodes + for ok, value in nodes_status: + assert ok, "Storage node creation failed: {}".format(value) + return grid.storage_servers @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py index 726c22e40..166cf940d 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -12,10 +12,15 @@ from os import mkdir, listdir, environ from os.path import join, exists from tempfile import mkdtemp, mktemp +from eliot import ( + log_call, +) + from twisted.python.procutils import which from twisted.internet.defer import ( inlineCallbacks, returnValue, + maybeDeferred, ) from twisted.internet.task import ( deferLater, @@ -25,6 +30,13 @@ from twisted.internet.interfaces import ( IProcessProtocol, IProtocol, ) +from twisted.internet.endpoints import ( + TCP4ServerEndpoint, +) +from twisted.internet.protocol import ( + Factory, + Protocol, +) from util import ( _CollectOutputProtocol, @@ -35,7 +47,6 @@ from util import ( _run_node, _cleanup_tahoe_process, _tahoe_runner_optional_coverage, - await_client_ready, TahoeProcess, ) @@ -165,6 +176,43 @@ def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, ) returnValue(storage) + +@attr.s +class Client(object): + """ + Represents a Tahoe client + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + + # XXX add stop / start / restart + # ...maybe "reconfig" of some kind? + + +@inlineCallbacks +def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, + needed=2, happy=3, total=4): + """ + Create a new storage server + """ + from util import _create_node + node_process = yield _create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, + name, web_port, storage=False, needed=needed, happy=happy, total=total, + ) + returnValue( + Client( + process=node_process, + protocol=node_process.transport._protocol, + ) + ) + + @attr.s class Introducer(object): """ @@ -180,29 +228,27 @@ class Introducer(object): furl = attr.ib() -_introducer_num = 0 - - @inlineCallbacks -def create_introducer(reactor, request, temp_dir, flog_gatherer): +@log_call( + action_type=u"integration:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def create_introducer(reactor, request, temp_dir, flog_gatherer, port): """ Run a new Introducer and return an Introducer instance. """ - global _introducer_num config = ( '[node]\n' - 'nickname = introducer{num}\n' + 'nickname = introducer{port}\n' 'web.port = {port}\n' 'log_gatherer.furl = {log_furl}\n' ).format( - num=_introducer_num, + port=port, log_furl=flog_gatherer.furl, - port=4560 + _introducer_num, ) - _introducer_num += 1 - intro_dir = join(temp_dir, 'introducer{}'.format(_introducer_num)) - print("making introducer", intro_dir, _introducer_num) + intro_dir = join(temp_dir, 'introducer{}'.format(port)) if not exists(intro_dir): mkdir(intro_dir) @@ -268,9 +314,14 @@ class Grid(object): Storage Servers. """ - introducer = attr.ib(default=None) - flog_gatherer = attr.ib(default=None) + _reactor = attr.ib() + _request = attr.ib() + _temp_dir = attr.ib() + _port_allocator = attr.ib() + introducer = attr.ib() + flog_gatherer = attr.ib() storage_servers = attr.ib(factory=list) + clients = attr.ib(factory=dict) @storage_servers.validator def check(self, attribute, value): @@ -279,3 +330,115 @@ class Grid(object): raise ValueError( "storage_servers must be StorageServer" ) + + @inlineCallbacks + def add_storage_node(self): + """ + Creates a new storage node, returns a StorageServer instance + (which will already be added to our .storage_servers list) + """ + port = yield self._port_allocator() + print("make {}".format(port)) + name = 'node{}'.format(port) + web_port = 'tcp:{}:interface=localhost'.format(port) + server = yield create_storage_server( + self._reactor, + self._request, + self._temp_dir, + self.introducer, + self.flog_gatherer, + name, + web_port, + ) + self.storage_servers.append(server) + returnValue(server) + + @inlineCallbacks + def add_client(self, name, needed=2, happy=3, total=4): + """ + Create a new client node + """ + port = yield self._port_allocator() + web_port = 'tcp:{}:interface=localhost'.format(port) + client = yield create_client( + self._reactor, + self._request, + self._temp_dir, + self.introducer, + self.flog_gatherer, + name, + web_port, + needed=needed, + happy=happy, + total=total, + ) + self.clients[name] = client + returnValue(client) + + + +# XXX THINK can we tie a whole *grid* to a single request? (I think +# that's all that makes sense) +@inlineCallbacks +def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + """ + """ + intro_port = yield port_allocator() + introducer = yield create_introducer(reactor, request, temp_dir, flog_gatherer, intro_port) + grid = Grid( + reactor, + request, + temp_dir, + port_allocator, + introducer, + flog_gatherer, + ) + returnValue(grid) + + +def create_port_allocator(start_port): + """ + Returns a new port-allocator .. which is a zero-argument function + that returns Deferreds that fire with new, sequential ports + starting at `start_port` skipping any that already appear to have + a listener. + + There can still be a race against other processes allocating ports + -- between the time when we check the status of the port and when + our subprocess starts up. This *could* be mitigated by instructing + the OS to not randomly-allocate ports in some range, and then + using that range here (explicitly, ourselves). + + NB once we're Python3-only this could be an async-generator + """ + port = [start_port - 1] + + # import stays here to not interfere with reactor selection -- but + # maybe this function should be arranged to be called once from a + # fixture (with the reactor)? + from twisted.internet import reactor + + class NothingProtocol(Protocol): + """ + I do nothing. + """ + + def port_generator(): + print("Checking port {}".format(port)) + port[0] += 1 + ep = TCP4ServerEndpoint(reactor, port[0], interface="localhost") + d = ep.listen(Factory.forProtocol(NothingProtocol)) + + def good(listening_port): + unlisten_d = maybeDeferred(listening_port.stopListening) + def return_port(_): + return port[0] + unlisten_d.addBoth(return_port) + return unlisten_d + + def try_again(fail): + return port_generator() + + d.addCallbacks(good, try_again) + return d + return port_generator diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index dc25c3b29..863a806a6 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -104,30 +104,16 @@ def test_remove_last_client(reactor, request): @pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): +def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): """ A client with happines=2 fails to upload to a Grid when it is using Grid Manager and there is only 1 storage server with a valid certificate. """ - import grid - introducer = yield grid.create_introducer(reactor, request, temp_dir, flog_gatherer) - storage0 = yield grid.create_storage_server( - reactor, request, temp_dir, introducer, flog_gatherer, - name="gm_storage0", - web_port="tcp:9995:interface=localhost", - needed=2, - happy=2, - total=2, - ) - storage1 = yield grid.create_storage_server( - reactor, request, temp_dir, introducer, flog_gatherer, - name="gm_storage1", - web_port="tcp:9996:interface=localhost", - needed=2, - happy=2, - total=2, - ) + from grid import create_grid + grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) + storage0 = yield grid.add_storage_node() + storage1 = yield grid.add_storage_node() gm_config = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "create", @@ -147,12 +133,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): ) assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - # XXX FIXME want a grid.create_client() or similar - diana = yield util._create_node( - reactor, request, temp_dir, introducer.furl, flog_gatherer, "diana", - web_port="tcp:9984:interface=localhost", - storage=False, - ) + diana = yield grid.add_client("diana", needed=2, happy=2, total=2) print("inserting certificate") cert = yield util.run_tahoe( @@ -175,23 +156,23 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): reactor, storage0.process.node_dir, request, None, ) - yield util.await_client_ready(diana, servers=2) + yield util.await_client_ready(diana.process, servers=2) # now only one storage-server has the certificate .. configure # diana to have the grid-manager certificate - config = configutil.get_config(join(diana.node_dir, "tahoe.cfg")) + config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) - with open(join(diana.node_dir, "tahoe.cfg"), "w") as f: + with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - diana.transport.signalProcess('TERM') - yield diana.transport._protocol.exited + diana.process.transport.signalProcess('TERM') + yield diana.protocol.exited diana = yield util._run_node( - reactor, diana._node_dir, request, None, + reactor, diana.process.node_dir, request, None, ) - yield util.await_client_ready(diana, servers=2) + yield util.await_client_ready(diana.process, servers=2) # try to put something into the grid, which should fail (because # diana has happy=2 but should only find storage0 to be acceptable @@ -199,7 +180,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): try: yield util.run_tahoe( - reactor, request, "--node-directory", diana._node_dir, + reactor, request, "--node-directory", diana.process.node_dir, "put", "-", stdin="some content\n" * 200, ) diff --git a/integration/util.py b/integration/util.py index 2cdcb2563..72689d387 100644 --- a/integration/util.py +++ b/integration/util.py @@ -266,7 +266,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - print("creating", node_dir) + print("creating: {}".format(node_dir)) mkdir(node_dir) done_proto = _ProcessExitedProtocol() args = [ From c9de1ee757d7943bbd51231a7eec8f3e45b69467 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 14:22:32 -0600 Subject: [PATCH 104/272] add a .restart() to Client and StorageServer --- integration/grid.py | 20 ++++++++++++++++++-- integration/test_grid_manager.py | 19 ++++--------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 166cf940d..d7b104e16 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -48,6 +48,7 @@ from util import ( _cleanup_tahoe_process, _tahoe_runner_optional_coverage, TahoeProcess, + await_client_ready, ) import attr @@ -155,8 +156,14 @@ class StorageServer(object): validator=attr.validators.provides(IProcessProtocol) ) - # XXX needs a restart() probably .. or at least a stop() and - # start() + @inlineCallbacks + def restart(self, reactor, request): + self.process.transport.signalProcess('TERM') + yield self.protocol.exited + self.process = yield _run_node( + reactor, self.process.node_dir, request, None, + ) + self.protocol = self.process.transport._protocol @inlineCallbacks @@ -190,6 +197,14 @@ class Client(object): validator=attr.validators.provides(IProcessProtocol) ) + @inlineCallbacks + def restart(self, reactor, request): + self.process.transport.signalProcess('TERM') + yield self.protocol.exited + x = yield _run_node( + reactor, self.process.node_dir, request, None, + ) + # XXX add stop / start / restart # ...maybe "reconfig" of some kind? @@ -373,6 +388,7 @@ class Grid(object): total=total, ) self.clients[name] = client + yield await_client_ready(client.process) returnValue(client) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 863a806a6..38c2dd78a 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -133,8 +133,6 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - diana = yield grid.add_client("diana", needed=2, happy=2, total=2) - print("inserting certificate") cert = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "sign", "storage0", @@ -150,29 +148,20 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a config.write(f) # re-start this storage server - storage0.process.transport.signalProcess('TERM') - yield storage0.protocol.exited - yield util._run_node( - reactor, storage0.process.node_dir, request, None, - ) - - yield util.await_client_ready(diana.process, servers=2) + yield storage0.restart(reactor, request) # now only one storage-server has the certificate .. configure # diana to have the grid-manager certificate + diana = yield grid.add_client("diana", needed=2, happy=2, total=2) + config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - diana.process.transport.signalProcess('TERM') - yield diana.protocol.exited - diana = yield util._run_node( - reactor, diana.process.node_dir, request, None, - ) - yield util.await_client_ready(diana.process, servers=2) + yield diana.restart(reactor, request) # try to put something into the grid, which should fail (because # diana has happy=2 but should only find storage0 to be acceptable From f52cd4363ad9921ba8b6d85d64a486f6390b0cb9 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 16:56:35 -0600 Subject: [PATCH 105/272] set the stage for the example in the intro --- docs/proposed/grid-manager/managed-grid.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index a3ad0d115..ff4b6e5c4 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -241,6 +241,11 @@ Manager. Example:: Example Setup of a New Managed Grid ----------------------------------- +This example creates an actual grid, but it's all just on one machine +with different "node directories" and a separate tahoe process for +each node. Usually of course each storage server would be on a +separate computer. + Note that we use the ``daemonize`` command in the following but that's only one way to handle "running a command in the background". You could instead run commands that start with ``daemonize ...`` in their @@ -251,10 +256,6 @@ We'll store our Grid Manager configuration on disk, in tahoe grid-manager --config ./gm0 create -This example creates an actual grid, but it's all just on one machine -with different "node directories". Usually of course each storage -server would be on a separate computer. - (If you already have a grid, you can :ref:`skip ahead `.) First of all, create an Introducer. Note that we actually have to run From 7c8a7f973a9eed2dd98de69af68665617884506b Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 16:56:46 -0600 Subject: [PATCH 106/272] better re-start --- integration/grid.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index d7b104e16..58eafe6b6 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -158,6 +158,13 @@ class StorageServer(object): @inlineCallbacks def restart(self, reactor, request): + """ + re-start our underlying process by issuing a TERM, waiting and + then running again. await_client_ready() will be done as well + + Note that self.process and self.protocol will be new instances + after this. + """ self.process.transport.signalProcess('TERM') yield self.protocol.exited self.process = yield _run_node( @@ -165,6 +172,9 @@ class StorageServer(object): ) self.protocol = self.process.transport._protocol + @inlineCallbacks + def run( + @inlineCallbacks def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, @@ -198,12 +208,25 @@ class Client(object): ) @inlineCallbacks - def restart(self, reactor, request): + def restart(self, reactor, request, servers=1): + """ + re-start our underlying process by issuing a TERM, waiting and + then running again. + + :param int servers: number of server connections we will wait + for before being 'ready' + + Note that self.process and self.protocol will be new instances + after this. + """ self.process.transport.signalProcess('TERM') yield self.protocol.exited - x = yield _run_node( + process = yield _run_node( reactor, self.process.node_dir, request, None, ) + self.process = process + self.protocol = self.process.transport._protocol + # XXX add stop / start / restart # ...maybe "reconfig" of some kind? From aa2066e2214269d0cd9e0f8ffdba06d50a274b12 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 16:57:37 -0600 Subject: [PATCH 107/272] move imports --- integration/test_grid_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 38c2dd78a..07b8a9083 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -10,6 +10,9 @@ from allmydata.util import base32 from allmydata.util import configutil import util +from grid import ( + create_grid, +) import pytest_twisted @@ -110,7 +113,6 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a using Grid Manager and there is only 1 storage server with a valid certificate. """ - from grid import create_grid grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) storage0 = yield grid.add_storage_node() storage1 = yield grid.add_storage_node() @@ -161,7 +163,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - yield diana.restart(reactor, request) + yield diana.restart(reactor, request, servers=2) # try to put something into the grid, which should fail (because # diana has happy=2 but should only find storage0 to be acceptable From 114c2faf1bb66019cb57d327a79fa4910b8a3350 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 17:12:51 -0600 Subject: [PATCH 108/272] oops --- integration/grid.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 58eafe6b6..f39f04d3c 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -172,9 +172,6 @@ class StorageServer(object): ) self.protocol = self.process.transport._protocol - @inlineCallbacks - def run( - @inlineCallbacks def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, From d4e2c668432aa3b092a14f20a43bccfd72228c41 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 18:17:24 -0600 Subject: [PATCH 109/272] use eliot logging --- src/allmydata/mutable/publish.py | 43 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index aa97c2242..5477e3077 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -23,6 +23,9 @@ from allmydata.mutable.layout import get_version_from_checkstring,\ MDMFSlotWriteProxy, \ SDMFSlotWriteProxy +import eliot + + KiB = 1024 DEFAULT_MAX_SEGMENT_SIZE = 128 * KiB PUSHING_BLOCKS_STATE = 0 @@ -942,23 +945,31 @@ class Publish(object): old_assignments.add(server, shnum) serverlist = [] - for i, server in enumerate(self.full_serverlist): - serverid = server.get_serverid() - if server in self.bad_servers: - continue - # if we have >= 1 grid-managers, this checks that we have - # a valid certificate for this server - if not server.upload_permitted(): - self.log( - "No valid grid-manager certificates for '{}' while choosing slots for mutable".format( - server.get_serverid(), - ), - level=log.UNUSUAL, - ) - continue - entry = (len(old_assignments.get(server, [])), i, serverid, server) - serverlist.append(entry) + action = eliot.start_action( + action_type=u"mutable:upload:update_goal", + homeless_shares=len(homeless_shares), + ) + with action: + for i, server in enumerate(self.full_serverlist): + serverid = server.get_serverid() + if server in self.bad_servers: + action.log( + server_id=server.get_server_id(), + message="Server is bad", + ) + continue + # if we have >= 1 grid-managers, this checks that we have + # a valid certificate for this server + if not server.upload_permitted(): + action.log( + server_id=server.get_server_id(), + message="No valid grid-manager certificates", + ) + continue + + entry = (len(old_assignments.get(server, [])), i, serverid, server) + serverlist.append(entry) serverlist.sort() if not serverlist: From b13a688ea816bd93519fd08043aab6b6859f21b9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:21:09 -0600 Subject: [PATCH 110/272] better words --- docs/proposed/grid-manager/managed-grid.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index ff4b6e5c4..8d584e71d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -40,7 +40,7 @@ the statement: "Grid Manager X suggests you use storage-server Y to upload shares to" (X and Y are public-keys). Such a certificate consists of: - - a version (currently 1) + - a version - the public-key of a storage-server - an expiry timestamp - a signature of the above @@ -61,10 +61,10 @@ at which time you may be able to pass a directory-capability to this option). If you don't want to store the configuration on disk at all, you may -use ``--config -`` (that's a dash) and write a valid JSON -configuration to stdin. +use ``--config -`` (the last character is a dash) and write a valid +JSON configuration to stdin. -All commands require the ``--config`` option, and they all behave +All commands require the ``--config`` option and they all behave similarly for "data from stdin" versus "data from disk". @@ -144,11 +144,11 @@ tahoe admin add-grid-manager-cert - `--filename`: the file to read the cert from (default: stdin) - `--name`: the name of this certificate (default: "default") -Import a "version 1" storage-certificate produced by a grid-manager -(probably: a storage server may have zero or more such certificates -installed; for now just one is sufficient). You will have to re-start -your node after this. Subsequent announcements to the Introducer will -include this certificate. +Import a "version 1" storage-certificate produced by a grid-manager A +storage server may have zero or more such certificates installed; for +now just one is sufficient. You will have to re-start your node after +this. Subsequent announcements to the Introducer will include this +certificate. .. note:: From b0d48ddbc987c4919b47cc6686838eef9d5f6968 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:21:24 -0600 Subject: [PATCH 111/272] don't need to create section --- src/allmydata/scripts/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 0ac3c6479..d9e45149b 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -125,7 +125,6 @@ def add_grid_manager_cert(options): return 1 config.set_config("storage", "grid_management", "True") - config.add_section("grid_manager_certificates") config.set_config("grid_manager_certificates", cert_name, cert_fname) # write all the data out From 6b791cb3b5b47708bf4602e07a2a5023c88f4969 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:23:42 -0600 Subject: [PATCH 112/272] nicer error --- src/allmydata/scripts/admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index d9e45149b..e18b31672 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -121,7 +121,11 @@ def add_grid_manager_cert(options): cert_name = options['name'] if exists(cert_path): - print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) + msg = "Already have certificate for '{}' (at {})".format( + options['name'], + cert_path, + ) + print(msg, file=options.parent.parent.stderr) return 1 config.set_config("storage", "grid_management", "True") From e65b6ba9274ed9bed1002a4652e58f5f1bf75525 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:24:17 -0600 Subject: [PATCH 113/272] no default named 'default' --- src/allmydata/scripts/admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e18b31672..96c1aa916 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -60,13 +60,17 @@ class AddGridManagerCertOptions(BaseOptions): optParameters = [ ['filename', 'f', None, "Filename of the certificate ('-', a dash, for stdin)"], - ['name', 'n', "default", "Name to give this certificate"], + ['name', 'n', None, "Name to give this certificate"], ] def getSynopsis(self): return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" def postOptions(self): + if self['name'] is None: + raise usage.UsageError( + "Must provide --name option" + ) if self['filename'] is None: raise usage.UsageError( "Must provide --filename option" From 8deddc62725863e5f9fa697fd0557d784d7c912c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:46:43 -0600 Subject: [PATCH 114/272] no defaults --- docs/proposed/grid-manager/managed-grid.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 8d584e71d..d0284dd16 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -141,8 +141,8 @@ Enrolling a Storage Server: CLI tahoe admin add-grid-manager-cert ````````````````````````````````` -- `--filename`: the file to read the cert from (default: stdin) -- `--name`: the name of this certificate (default: "default") +- `--filename`: the file to read the cert from +- `--name`: the name of this certificate Import a "version 1" storage-certificate produced by a grid-manager A storage server may have zero or more such certificates installed; for From 2483e938dc235ec33892c1c79f3c78cea6a29c4a Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:47:03 -0600 Subject: [PATCH 115/272] tahoe add-grid-manager is only 'idea' currently --- docs/proposed/grid-manager/managed-grid.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index d0284dd16..72eec7660 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -204,8 +204,10 @@ Enrolling a Client: CLI ----------------------- -tahoe add-grid-manager -`````````````````````` +tahoe add-grid-manager (PROPOSED) +````````````````````````````````` + +DECIDE: this command hasn't actually been written yet. This takes two arguments: ``name`` and ``public-identity``. @@ -295,7 +297,8 @@ certificates into the grid. We do this by adding some configuration default = gridmanager.cert Add the above bit to each node's ``tahoe.cfg`` and re-start the -storage nodes. +storage nodes. (Alternatively, use the ``tahoe add-grid-manager`` +command). Now try adding a new storage server ``storage2``. This client can join the grid just fine, and announce itself to the Introducer as providing From d46c35e953e22cc21bbb43729cc5fb58b59e09e5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 2 Oct 2020 16:02:56 -0600 Subject: [PATCH 116/272] grid-manager unit-tests --- src/allmydata/test/test_grid_manager.py | 72 +++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/allmydata/test/test_grid_manager.py diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py new file mode 100644 index 000000000..459b360e5 --- /dev/null +++ b/src/allmydata/test/test_grid_manager.py @@ -0,0 +1,72 @@ + + +from allmydata.client import ( + _load_grid_manager_certificates, + create_storage_farm_broker, +) +from allmydata.node import ( + config_from_string, +) +from allmydata.client import ( + _valid_config as client_valid_config, +) + +from .common import SyncTestCase + + +class GridManagerUtilities(SyncTestCase): + """ + Confirm operation of utility functions used by GridManager + """ + + def test_client_grid_manager(self): + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + sfb = create_storage_farm_broker(config, {}, {}, {}, []) + # could introspect sfb._grid_manager_certificates, but that's + # "cheating"? even though _make_storage_sever is also + # "private"? + + # ...but, okay, a "real" client will call set_static_servers() + # with any configured/cached servers (thus causing + # _make_storage_server to be called). The other way + # _make_storage_server is called is when _got_announcement + # runs, which is when an introducer client gets an + # announcement... + + invalid_cert = { + "certificate": "foo", + "signature": "43564356435643564356435643564356", + } + announcement = { + "anonymous-storage-FURL": b"pb://abcde@nowhere/fake", + "grid-manager-certificates": [ + invalid_cert, + ] + } + static_servers = { + "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia": { + "ann": announcement, + } + } + sfb.set_static_servers(static_servers) + nss = sfb._make_storage_server(u"server0", {"ann": announcement}) + + # we have some grid-manager keys defined so the server should + # only upload if there's a valid certificate -- but the only + # one we have is invalid + self.assertFalse(nss.upload_permitted()) + + def test_load_certificates(self): + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + self.assertEqual( + 1, + len(config.enumerate_section("grid_managers")) + ) From 6a2f1ae00307781effc7054207044fca3d25641b Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 17:34:23 -0600 Subject: [PATCH 117/272] use 'tahoe admin add-grid-manager-cert' in tests --- integration/test_grid_manager.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 07b8a9083..8ecf72561 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -142,12 +142,13 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) with open(join(storage0.process.node_dir, "gridmanager.cert"), "w") as f: f.write(cert) - config = configutil.get_config(join(storage0.process.node_dir, "tahoe.cfg")) - config.set("storage", "grid_management", "True") - config.add_section("grid_manager_certificates") - config.set("grid_manager_certificates", "default", "gridmanager.cert") - with open(join(storage0.process.node_dir, "tahoe.cfg"), "w") as f: - config.write(f) + + yield util.run_tahoe( + reactor, request, "--node-directory", storage0.process.node_dir, + "admin", "add-grid-manager-cert", + "--name", "default", + "--filename", join(storage0.process.node_dir, "gridmanager.cert"), + ) # re-start this storage server yield storage0.restart(reactor, request) From 5eb1ad953bab89d460240f04a46ef0d42dc55ec6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 17:35:02 -0600 Subject: [PATCH 118/272] fix 'admin add-grid-manager-cert' --- src/allmydata/scripts/admin.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 96c1aa916..57cac39c7 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -7,6 +7,9 @@ from os.path import exists, join from twisted.python import usage #from allmydata.node import read_config from allmydata.client import read_config +from allmydata.storage_client import ( + parse_grid_manager_data, +) from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath @@ -82,15 +85,16 @@ class AddGridManagerCertOptions(BaseOptions): raise usage.UsageError( "Reading certificate from stdin failed" ) - from allmydata.storage_client import parse_grid_manager_data - try: - self.certificate_data = parse_grid_manager_data(data) - except ValueError as e: - print("Error parsing certificate: {}".format(e), file=self.parent.parent.stderr) - self.certificate_data = None else: with open(self['filename'], 'r') as f: - self.certificate_data = f.read() + data = f.read() + + try: + self.certificate_data = parse_grid_manager_data(data) + except ValueError as e: + raise UsageError( + "Error parsing certificate: {}".format(e) + ) def getUsage(self, width=None): t = BaseOptions.getUsage(self, width) From 370d4b0f8b801ea52b9a33915a59b96c1b0bb4da Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 17:35:50 -0600 Subject: [PATCH 119/272] load certs as well as keys in utest --- src/allmydata/test/test_grid_manager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 459b360e5..d5c98f2e7 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,4 +1,5 @@ +import json from allmydata.client import ( _load_grid_manager_certificates, @@ -61,9 +62,17 @@ class GridManagerUtilities(SyncTestCase): self.assertFalse(nss.upload_permitted()) def test_load_certificates(self): + cert_path = self.mktemp() + with open(cert_path, "w") as f: + f.write(json.dumps({ + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + })) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) self.assertEqual( From c43c84b6024733320150a63c59e6131c3ec0dbaa Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 18:02:46 -0600 Subject: [PATCH 120/272] fewer files in test --- integration/test_grid_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 8ecf72561..dca403d1d 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -140,14 +140,13 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a reactor, request, "grid-manager", "--config", "-", "sign", "storage0", stdin=gm_config, ) - with open(join(storage0.process.node_dir, "gridmanager.cert"), "w") as f: - f.write(cert) yield util.run_tahoe( reactor, request, "--node-directory", storage0.process.node_dir, "admin", "add-grid-manager-cert", "--name", "default", - "--filename", join(storage0.process.node_dir, "gridmanager.cert"), + "--filename", "-", + stdin=cert, ) # re-start this storage server From 84c9da4e7bc1c0a45498772a35b1f1c691a027c1 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 18:09:27 -0600 Subject: [PATCH 121/272] unused --- src/allmydata/scripts/tahoe_grid_manager.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 64561b95e..4b0a24d76 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -322,16 +322,6 @@ def _save_gridmanager_config(file_path, grid_manager): f.write("{}\n".format(data)) -def _config_to_filepath(gm_config_location): - """ - Converts a command-line string specifying the GridManager - configuration's location into a readable file-like object. - - :param gm_config_location str: a valid path, or '-' (a single - dash) to use stdin. - """ - - def _load_gridmanager_config(gm_config): """ Loads a Grid Manager configuration and returns it (a dict) after From c43955573bd9c278a0d1ff28544eefb68fb4f2a4 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 20:40:26 -0600 Subject: [PATCH 122/272] more gridmanager tests --- integration/test_grid_manager.py | 36 +++++++++++++++++++++++- integration/test_servers_of_happiness.py | 3 +- integration/util.py | 5 +++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index dca403d1d..d701172f4 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -5,6 +5,11 @@ import shutil from os import mkdir, unlink, listdir, utime from os.path import join, exists, getmtime +from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, +) + from allmydata.crypto import ed25519 from allmydata.util import base32 from allmydata.util import configutil @@ -177,4 +182,33 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) assert False, "Should get a failure" except util.ProcessFailed as e: - assert 'UploadUnhappinessError' in e.output.getvalue() + assert 'UploadUnhappinessError' in e.output + + +@pytest_twisted.inlineCallbacks +def test_identity(reactor, request, temp_dir): + """ + Dump public key to CLI + """ + gm_config = join(temp_dir, "test_identity") + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gm_config, "create", + ) + + # ask the CLI for the grid-manager pubkey + pubkey = yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gm_config, "public-identity", + ) + alleged_pubkey = ed25519.verifying_key_from_string(pubkey.strip()) + + # load the grid-manager pubkey "ourselves" + with open(join(gm_config, "config.json"), "r") as f: + real_config = json.load(f) + real_privkey, real_pubkey = ed25519.signing_keypair_from_string( + real_config["private_key"].encode("ascii"), + ) + + # confirm the CLI told us the correct thing + alleged_bytes = alleged_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) + real_bytes = real_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) + assert alleged_bytes == real_bytes, "Keys don't match" diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index fd8f75e39..dd0b3e32f 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -42,5 +42,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto except util.ProcessFailed as e: assert "UploadUnhappinessError" in e.output.getvalue() - output = proto.output.getvalue() - assert "shares could be placed on only" in output + assert "shares could be placed on only" in proto.output diff --git a/integration/util.py b/integration/util.py index 72689d387..5cad59c12 100644 --- a/integration/util.py +++ b/integration/util.py @@ -49,6 +49,9 @@ class ProcessFailed(Exception): self.reason = reason self.output = output + def __str__(self): + return ":\n{}".format(self.reason, self.output) + class _CollectOutputProtocol(ProcessProtocol): """ @@ -72,7 +75,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(ProcessFailed(reason, self.output)) + self.done.errback(ProcessFailed(reason, self.output.getvalue())) def outReceived(self, data): self.output.write(data) From 774ab72c9d4eb2d1d8ae41054c9b699a496c80b0 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 21:06:29 -0600 Subject: [PATCH 123/272] test user-management on files --- integration/test_grid_manager.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d701172f4..2c1f0d1c1 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -111,6 +111,37 @@ def test_remove_last_client(reactor, request): assert "storage_servers" not in json.loads(gm_config) +@pytest_twisted.inlineCallbacks +def test_add_remove_client_file(reactor, request, temp_dir): + """ + A Grid Manager can add and successfully remove a client (when + keeping data on disk) + """ + gmconfig = join(temp_dir, "gmtest") + gmconfig_file = join(temp_dir, "gmtest", "config.json") + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "create", + ) + + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "add", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + ) + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "add", + "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + ) + assert "zara" in json.load(open(gmconfig_file, "r"))['storage_servers'] + assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] + + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "remove", + "zara", + ) + assert "zara" not in json.load(open(gmconfig_file, "r"))['storage_servers'] + assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] + + @pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): """ From bf45f57b603d0bc0098a1d86e880689ea2ab40df Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 21:54:38 -0600 Subject: [PATCH 124/272] minor fixes --- integration/test_grid_manager.py | 4 +-- integration/test_servers_of_happiness.py | 4 +-- src/allmydata/scripts/tahoe_grid_manager.py | 27 +++++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 2c1f0d1c1..db93f17c7 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -42,7 +42,7 @@ def test_create_certificate(reactor, request): stdin=gm_config, ) zara_cert_bytes = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "zara", + reactor, request, "grid-manager", "--config", "-", "sign", "zara", "1", stdin=gm_config, ) zara_cert = json.loads(zara_cert_bytes) @@ -173,7 +173,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a print("inserting certificate") cert = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "storage0", + reactor, request, "grid-manager", "--config", "-", "sign", "storage0", "1", stdin=gm_config, ) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index dd0b3e32f..d98bfe0b8 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -40,6 +40,6 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto yield proto.done assert False, "should raise exception" except util.ProcessFailed as e: - assert "UploadUnhappinessError" in e.output.getvalue() + assert "UploadUnhappinessError" in e.output - assert "shares could be placed on only" in proto.output + assert "shares could be placed on only" in proto.output.getvalue() diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 4b0a24d76..67c8b631b 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -3,7 +3,7 @@ from __future__ import print_function import sys import json import time -from datetime import datetime +from datetime import datetime, timedelta from allmydata.scripts.common import BaseOptions from allmydata.util.abbreviate import abbreviate_time @@ -81,15 +81,22 @@ class SignOptions(BaseOptions): ) def getSynopsis(self): - return "{} NAME".format(super(SignOptions, self).getSynopsis()) + return "{} NAME EXPIRY_DAYS".format(super(SignOptions, self).getSynopsis()) def parseArgs(self, *args, **kw): BaseOptions.parseArgs(self, **kw) - if len(args) != 1: + if len(args) != 2: raise usage.UsageError( - "Requires one argument: name" + "Requires two arguments: name expiry_days" ) self['name'] = unicode(args[0]) + self['expiry_days'] = int(args[1]) + if self['expiry_days'] < 1 or self['expiry_days'] > 20*365: + raise usage.UsageError( + "Certificate expires in an unreasonable number of days: {}".format( + self['expiry_days'], + ) + ) class GridManagerOptions(BaseOptions): @@ -236,15 +243,17 @@ class _GridManager(object): def public_identity(self): return ed25519.string_from_verifying_key(self._public_key) - def sign(self, name): + def sign(self, name, expiry_seconds): try: srv = self._storage_servers[name] except KeyError: raise KeyError( u"No storage server named '{}'".format(name) ) + expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) + epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() cert_info = { - "expires": int(time.time() + 86400), # XXX FIXME + "expires": epoch_offset, "public_key": srv.public_key(), "version": 1, } @@ -422,7 +431,7 @@ def _list(gridoptions, options): while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) cert_data = json.loads(container['certificate']) - expires = datetime.fromtimestamp(cert_data['expires']) + expires = datetime.utcfromtimestamp(cert_data['expires']) delta = datetime.utcnow() - expires if delta.total_seconds() < 0: print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta))) @@ -439,8 +448,10 @@ def _sign(gridoptions, options): fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) + expiry_seconds = int(options['expiry_days']) * 86400 + try: - certificate = gm.sign(options['name']) + certificate = gm.sign(options['name'], expiry_seconds) except KeyError: raise usage.UsageError( "No storage-server called '{}' exists".format(options['name']) From c65a01fbe0692fb6c9f784a2f5efc64807e520e8 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 21:56:53 -0600 Subject: [PATCH 125/272] flake8 --- src/allmydata/scripts/admin.py | 2 +- src/allmydata/scripts/tahoe_grid_manager.py | 1 - src/allmydata/test/test_grid_manager.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 57cac39c7..e5b7dca2d 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -92,7 +92,7 @@ class AddGridManagerCertOptions(BaseOptions): try: self.certificate_data = parse_grid_manager_data(data) except ValueError as e: - raise UsageError( + raise usage.UsageError( "Error parsing certificate: {}".format(e) ) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 67c8b631b..f84f57766 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -2,7 +2,6 @@ from __future__ import print_function import sys import json -import time from datetime import datetime, timedelta from allmydata.scripts.common import BaseOptions diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index d5c98f2e7..a60739634 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -2,7 +2,6 @@ import json from allmydata.client import ( - _load_grid_manager_certificates, create_storage_farm_broker, ) from allmydata.node import ( From 817eef287d11cd14391986a201732c1b5ab23f62 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Oct 2020 10:16:10 -0600 Subject: [PATCH 126/272] typo --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 72eec7660..5720fd89d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -8,7 +8,7 @@ Managed Grid ============ In a grid using an Introducer, a client will use any storage-server -the Introducer announces (and the Introducer will annoucne any +the Introducer announces (and the Introducer will announce any storage-server that connects to it). This means that anyone with the Introducer fURL can connect storage to the grid. From 91e1fa3c52fee6947867ad5a7a3650d9dbbf8b63 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Oct 2020 10:16:23 -0600 Subject: [PATCH 127/272] correct config section --- docs/proposed/grid-manager/managed-grid.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5720fd89d..6583868cd 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -182,8 +182,8 @@ Enrolling a Storage Server: Config You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to turn on grid-management with ``grid_management = true``. You then must -also provide a ``[grid_management_keys]`` section in the config-file which -lists ``name = path/to/certificate`` pairs. +also provide a ``[grid_management_certificates]`` section in the +config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``tahoe grid-manager sign`` command; these should be **securely transmitted** to the storage @@ -192,7 +192,7 @@ server. Relative paths are based from the node directory. Example:: [storage] grid_management = true - [grid_management_keys] + [grid_management_certificates] default = example_grid.cert This will cause us to give this certificate to any Introducers we From 6ea3c6842bca31782f77fe8c825beb88d07ddfe0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 29 Oct 2020 19:32:27 -0600 Subject: [PATCH 128/272] encoded -> tahoe-encoded --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 6583868cd..01181d42a 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -95,7 +95,7 @@ tahoe grid-manager add Takes two args: ``name pubkey``. The ``name`` is an arbitrary local identifier for the new storage node (also sometimes called "a petname" -or "nickname"). The pubkey is the encoded key from a ``node.pubkey`` +or "nickname"). The pubkey is the tahoe-encoded key from a ``node.pubkey`` file in the storage-server's node directory (minus any whitespace). For example, if ``~/storage0`` contains a storage-node, you might do something like this: From f9231444784d56410ce84414b829ea3a3823c3ea Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 17:28:49 -0700 Subject: [PATCH 129/272] re-factor; use a predicate instead of expanding NativeStorageServer --- src/allmydata/storage_client.py | 124 +++++++++++++++++++------------- 1 file changed, 76 insertions(+), 48 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 93df74319..5ffb44092 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -242,6 +242,11 @@ class StorageFarmBroker(service.MultiService): assert isinstance(server_id, unicode) # from YAML server_id = server_id.encode("ascii") handler_overrides = server.get("connections", {}) + gm_verifier = create_grid_manager_verifier( + self._grid_manager_keys, + server["ann"].get("grid-manager-certificates", []), + ) + s = NativeStorageServer( server_id, server["ann"], @@ -249,8 +254,7 @@ class StorageFarmBroker(service.MultiService): handler_overrides, self.node_config, self.storage_client_config, - self._grid_manager_keys, - server["ann"].get("grid-manager-certificates", []), + gm_verifier, ) s.on_status_changed(lambda _: self._got_connection()) return s @@ -459,7 +463,7 @@ def parse_grid_manager_data(gm_data): return js -def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): +def _validate_grid_manager_certificate(gm_key, alleged_cert): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. @@ -468,15 +472,10 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): "certificate" contains a JSON-serialized certificate for a Storage Server (comes from a Grid Manager). - :param now_fn: a zero-argument callable that returns a UTC - timestamp (will use datetime.utcnow by default) - - :return: False if the signature is invalid or the certificate is - expired. + :return: a dict consisting of the deserialized certificate data or + None if the signature is invalid. Note we do NOT check the + expiry time in this function. """ - if now_fn is None: - now_fn = datetime.utcnow - try: ed25519.verify_signature( gm_key, @@ -484,15 +483,70 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): alleged_cert['certificate'].encode('ascii'), ) except ed25519.BadSignature: - return False + return None # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) - now = now_fn() - expires = datetime.utcfromtimestamp(cert['expires']) - # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) - if expires < now: - return False # certificate is expired - return True + return cert + + +def create_grid_manager_verifier(keys, certs, now_fn=None): + """ + Creates a predicate for confirming some Grid Manager-issued + certificates against Grid Manager keys. A predicate is used + (instead of just returning True/False here) so that the + expiry-time can be tested on each call. + + :param list keys: 0 or more `VerifyingKey` instances + + :param list certs: 1 or more Grid Manager certificates each of + which is a `dict` containing 'signature' and 'certificate' keys. + + :param callable now_fn: a callable which returns the current UTC + timestamp (or datetime.utcnow if None). + + :returns: a callable which will return True only-if there is at + least one valid certificate in `certs` signed by one of the keys + in `keys`. + """ + + now_fn = datetime.utcnow if now_fn is None else now_fn + valid_certs = [] + + # if we have zero grid-manager keys then everything is valid + if not keys: + return lambda: True + + # validate the signatures on any certificates we have (not yet the expiry dates) + for alleged_cert in certs: + for key in keys: + cert = _validate_grid_manager_certificate(key, alleged_cert) + if cert is not None: + valid_certs.append(cert) + else: + # we might want to let the user know about this + # failed-to-verify certificate .. but also if you have + # multiple grid-managers then a bunch of these + # messages would appear + print( + "Grid Manager certificate signature failed. Certificate: " + "\"{cert}\" for key \"{key}\".".format( + key=key, + cert=alleged_cert, + ) + ) + + def validate(): + now = now_fn() + # if *any* certificate is still valid then we consider the server valid + for cert in valid_certs: + expires = datetime.utcfromtimestamp(cert['expires']) + # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) + if expires > now: + # not-expired + return True + return False + + return validate class IFoolscapStorageServer(Interface): @@ -733,7 +787,7 @@ class NativeStorageServer(service.MultiService): } def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=None, - grid_manager_keys=None, grid_manager_certs=None): + grid_manager_verifier=None): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id @@ -744,18 +798,7 @@ class NativeStorageServer(service.MultiService): if config is None: config = StorageClientConfig() - # XXX we should validate as much as we can about the - # certificates right now -- the only thing we HAVE to be lazy - # about is the expiry, which should be checked before any - # possible upload... - - # any public-keys which the user has configured (if none, it - # means use any storage servers) - self._grid_manager_keys = grid_manager_keys or [] - - # any storage-certificates that this storage-server included - # in its announcement - self._grid_manager_certificates = grid_manager_certs or [] + self._grid_manager_verifier = grid_manager_verifier self._storage = self._make_storage_system(node_config, config, ann) @@ -778,25 +821,10 @@ class NativeStorageServer(service.MultiService): :return: True if we should use this server for uploads, False otherwise. """ - # print("upload permitted? {}".format(self._server_id)) # if we have no Grid Manager keys configured, choice is easy - if not self._grid_manager_keys: - # print("{} no grid manager keys at all (so yes)".format(self._server_id)) + if self._grid_manager_verifier is None: return True - - # XXX probably want to cache the answer to this? (ignoring - # that for now because certificates expire, so .. slightly - # more complex) - if not self._grid_manager_certificates: - # print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) - return False - for gm_key in self._grid_manager_keys: - for cert in self._grid_manager_certificates: - if validate_grid_manager_certificate(gm_key, cert): - # print("valid: {}\n{}".format(gm_key, cert)) - return True - # print("didn't validate {} keys".format(len(self._grid_manager_keys))) - return False + return self._grid_manager_verifier() def _make_storage_system(self, node_config, config, ann): """ From 30b7be6f1d0b7fe27e3ff93558fc7387312f926a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 18:07:30 -0700 Subject: [PATCH 130/272] remove integration tests/refactoring --- integration/conftest.py | 158 ++++++-- integration/grid.py | 480 ----------------------- integration/test_grid_manager.py | 245 ------------ integration/test_servers_of_happiness.py | 7 +- integration/test_tor.py | 39 +- integration/test_web.py | 10 +- integration/util.py | 54 +-- 7 files changed, 161 insertions(+), 832 deletions(-) delete mode 100644 integration/grid.py delete mode 100644 integration/test_grid_manager.py diff --git a/integration/conftest.py b/integration/conftest.py index 1fb3fd761..04e3dcb52 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -36,11 +36,6 @@ from util import ( await_client_ready, TahoeProcess, ) -from grid import ( - create_port_allocator, - create_flog_gatherer, - create_grid, -) # pytest customization hooks @@ -77,12 +72,6 @@ def reactor(): return _reactor -@pytest.fixture(scope='session') -@log_call(action_type=u"integration:port_allocator", include_result=False) -def port_allocator(reactor): - return create_port_allocator(start_port=45000) - - @pytest.fixture(scope='session') @log_call(action_type=u"integration:temp_dir", include_args=[]) def temp_dir(request): @@ -117,30 +106,127 @@ def flog_binary(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): - fg = pytest_twisted.blockon( - create_flog_gatherer(reactor, request, temp_dir, flog_binary) + out_protocol = _CollectOutputProtocol() + gather_dir = join(temp_dir, 'flog_gather') + reactor.spawnProcess( + out_protocol, + flog_binary, + ( + 'flogtool', 'create-gatherer', + '--location', 'tcp:localhost:3117', + '--port', '3117', + gather_dir, + ) ) - return fg + pytest_twisted.blockon(out_protocol.done) + + twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_process = reactor.spawnProcess( + twistd_protocol, + which('twistd')[0], + ( + 'twistd', '--nodaemon', '--python', + join(gather_dir, 'gatherer.tac'), + ), + path=gather_dir, + ) + pytest_twisted.blockon(twistd_protocol.magic_seen) + + def cleanup(): + _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) + + flog_file = mktemp('.flog_dump') + flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) + flog_dir = join(temp_dir, 'flog_gather') + flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] + + print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) + reactor.spawnProcess( + flog_protocol, + flog_binary, + ( + 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) + ), + ) + print("Waiting for flogtool to complete") + try: + pytest_twisted.blockon(flog_protocol.done) + except ProcessTerminated as e: + print("flogtool exited unexpectedly: {}".format(str(e))) + print("Flogtool completed") + + request.addfinalizer(cleanup) + + with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: + furl = f.read().strip() + return furl @pytest.fixture(scope='session') -@log_call(action_type=u"integration:grid", include_args=[]) -def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): - g = pytest_twisted.blockon( - create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) +@log_call( + action_type=u"integration:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer0 +web.port = 4560 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = _ProcessExitedProtocol() + _tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=tcp', + '--hostname=localhost', + intro_dir, + ), + ) + pytest_twisted.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # on windows, "tahoe start" means: run forever in the foreground, + # but on linux it means daemonize. "tahoe run" is consistent + # between platforms. + protocol = _MagicTextProtocol('introducer running') + transport = _tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), ) - return g + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) - -@pytest.fixture(scope='session') -def introducer(grid): - return grid.introducer + pytest_twisted.blockon(protocol.magic_seen) + return TahoeProcess(transport, intro_dir) @pytest.fixture(scope='session') @log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"]) def introducer_furl(introducer, temp_dir): - return introducer.furl + furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + return furl @pytest.fixture(scope='session') @@ -219,20 +305,28 @@ def tor_introducer_furl(tor_introducer, temp_dir): @pytest.fixture(scope='session') @log_call( action_type=u"integration:storage_nodes", - include_args=["grid"], + include_args=["temp_dir", "introducer_furl", "flog_gatherer"], include_result=False, ) -def storage_nodes(grid): +def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request): nodes_d = [] # start all 5 nodes in parallel for x in range(5): - #nodes_d.append(grid.add_storage_node()) - pytest_twisted.blockon(grid.add_storage_node()) - + name = 'node{}'.format(x) + web_port= 9990 + x + nodes_d.append( + _create_node( + reactor, request, temp_dir, introducer_furl, flog_gatherer, name, + web_port="tcp:{}:interface=localhost".format(web_port), + storage=True, + ) + ) nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) - for ok, value in nodes_status: - assert ok, "Storage node creation failed: {}".format(value) - return grid.storage_servers + nodes = [] + for ok, process in nodes_status: + assert ok, "Storage node creation failed: {}".format(process) + nodes.append(process) + return nodes @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py deleted file mode 100644 index f39f04d3c..000000000 --- a/integration/grid.py +++ /dev/null @@ -1,480 +0,0 @@ -""" -Classes which directly represent various kinds of Tahoe processes -that co-operate to for "a Grid". - -These methods and objects are used by conftest.py fixtures but may -also be used as direct helpers for tests that don't want to (or can't) -rely on 'the' global grid as provided by fixtures like 'alice' or -'storage_servers'. -""" - -from os import mkdir, listdir, environ -from os.path import join, exists -from tempfile import mkdtemp, mktemp - -from eliot import ( - log_call, -) - -from twisted.python.procutils import which -from twisted.internet.defer import ( - inlineCallbacks, - returnValue, - maybeDeferred, -) -from twisted.internet.task import ( - deferLater, -) -from twisted.internet.interfaces import ( - IProcessTransport, - IProcessProtocol, - IProtocol, -) -from twisted.internet.endpoints import ( - TCP4ServerEndpoint, -) -from twisted.internet.protocol import ( - Factory, - Protocol, -) - -from util import ( - _CollectOutputProtocol, - _MagicTextProtocol, - _DumpOutputProtocol, - _ProcessExitedProtocol, - _create_node, - _run_node, - _cleanup_tahoe_process, - _tahoe_runner_optional_coverage, - TahoeProcess, - await_client_ready, -) - -import attr -import pytest_twisted - - -# further directions: -# - "Grid" is unused, basically -- tie into the rest? -# - could make a Grid instance mandatory for create_* calls -# - could instead make create_* calls methods of Grid -# - Bring more 'util' or 'conftest' code into here -# - stop()/start()/restart() methods on StorageServer etc -# - more-complex stuff like config changes (which imply a restart too)? - - -@attr.s -class FlogGatherer(object): - """ - Flog Gatherer process. - """ - - process = attr.ib( - validator=attr.validators.provides(IProcessTransport) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - furl = attr.ib() - - -@inlineCallbacks -def create_flog_gatherer(reactor, request, temp_dir, flog_binary): - out_protocol = _CollectOutputProtocol() - gather_dir = join(temp_dir, 'flog_gather') - reactor.spawnProcess( - out_protocol, - flog_binary, - ( - 'flogtool', 'create-gatherer', - '--location', 'tcp:localhost:3117', - '--port', '3117', - gather_dir, - ) - ) - yield out_protocol.done - - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") - twistd_process = reactor.spawnProcess( - twistd_protocol, - which('twistd')[0], - ( - 'twistd', '--nodaemon', '--python', - join(gather_dir, 'gatherer.tac'), - ), - path=gather_dir, - ) - yield twistd_protocol.magic_seen - - def cleanup(): - _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) - - flog_file = mktemp('.flog_dump') - flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) - flog_dir = join(temp_dir, 'flog_gather') - flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] - - print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) - reactor.spawnProcess( - flog_protocol, - flog_binary, - ( - 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) - ), - ) - print("Waiting for flogtool to complete") - try: - pytest_twisted.blockon(flog_protocol.done) - except ProcessTerminated as e: - print("flogtool exited unexpectedly: {}".format(str(e))) - print("Flogtool completed") - - request.addfinalizer(cleanup) - - with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: - furl = f.read().strip() - returnValue( - FlogGatherer( - protocol=twistd_protocol, - process=twistd_process, - furl=furl, - ) - ) - - -@attr.s -class StorageServer(object): - """ - Represents a Tahoe Storage Server - """ - - process = attr.ib( - validator=attr.validators.instance_of(TahoeProcess) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - - @inlineCallbacks - def restart(self, reactor, request): - """ - re-start our underlying process by issuing a TERM, waiting and - then running again. await_client_ready() will be done as well - - Note that self.process and self.protocol will be new instances - after this. - """ - self.process.transport.signalProcess('TERM') - yield self.protocol.exited - self.process = yield _run_node( - reactor, self.process.node_dir, request, None, - ) - self.protocol = self.process.transport._protocol - - -@inlineCallbacks -def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, - needed=2, happy=3, total=4): - """ - Create a new storage server - """ - from util import _create_node - node_process = yield _create_node( - reactor, request, temp_dir, introducer.furl, flog_gatherer, - name, web_port, storage=True, needed=needed, happy=happy, total=total, - ) - storage = StorageServer( - process=node_process, - protocol=node_process.transport._protocol, - ) - returnValue(storage) - - -@attr.s -class Client(object): - """ - Represents a Tahoe client - """ - - process = attr.ib( - validator=attr.validators.instance_of(TahoeProcess) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - - @inlineCallbacks - def restart(self, reactor, request, servers=1): - """ - re-start our underlying process by issuing a TERM, waiting and - then running again. - - :param int servers: number of server connections we will wait - for before being 'ready' - - Note that self.process and self.protocol will be new instances - after this. - """ - self.process.transport.signalProcess('TERM') - yield self.protocol.exited - process = yield _run_node( - reactor, self.process.node_dir, request, None, - ) - self.process = process - self.protocol = self.process.transport._protocol - - - # XXX add stop / start / restart - # ...maybe "reconfig" of some kind? - - -@inlineCallbacks -def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, - needed=2, happy=3, total=4): - """ - Create a new storage server - """ - from util import _create_node - node_process = yield _create_node( - reactor, request, temp_dir, introducer.furl, flog_gatherer, - name, web_port, storage=False, needed=needed, happy=happy, total=total, - ) - returnValue( - Client( - process=node_process, - protocol=node_process.transport._protocol, - ) - ) - - -@attr.s -class Introducer(object): - """ - Reprsents a running introducer - """ - - process = attr.ib( - validator=attr.validators.instance_of(TahoeProcess) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - furl = attr.ib() - - -@inlineCallbacks -@log_call( - action_type=u"integration:introducer", - include_args=["temp_dir", "flog_gatherer"], - include_result=False, -) -def create_introducer(reactor, request, temp_dir, flog_gatherer, port): - """ - Run a new Introducer and return an Introducer instance. - """ - config = ( - '[node]\n' - 'nickname = introducer{port}\n' - 'web.port = {port}\n' - 'log_gatherer.furl = {log_furl}\n' - ).format( - port=port, - log_furl=flog_gatherer.furl, - ) - - intro_dir = join(temp_dir, 'introducer{}'.format(port)) - - if not exists(intro_dir): - mkdir(intro_dir) - done_proto = _ProcessExitedProtocol() - _tahoe_runner_optional_coverage( - done_proto, - reactor, - request, - ( - 'create-introducer', - '--listen=tcp', - '--hostname=localhost', - intro_dir, - ), - ) - yield done_proto.done - - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) - - # on windows, "tahoe start" means: run forever in the foreground, - # but on linux it means daemonize. "tahoe run" is consistent - # between platforms. - protocol = _MagicTextProtocol('introducer running') - transport = _tahoe_runner_optional_coverage( - protocol, - reactor, - request, - ( - 'run', - intro_dir, - ), - ) - - def clean(): - return _cleanup_tahoe_process(transport, protocol.exited) - request.addfinalizer(clean) - - yield protocol.magic_seen - - furl_fname = join(intro_dir, 'private', 'introducer.furl') - while not exists(furl_fname): - print("Don't see {} yet".format(furl_fname)) - yield deferLater(reactor, .1, lambda: None) - furl = open(furl_fname, 'r').read() - - returnValue( - Introducer( - process=TahoeProcess(transport, intro_dir), - protocol=protocol, - furl=furl, - ) - ) - - -@attr.s -class Grid(object): - """ - Represents an entire Tahoe Grid setup - - A Grid includes an Introducer, Flog Gatherer and some number of - Storage Servers. - """ - - _reactor = attr.ib() - _request = attr.ib() - _temp_dir = attr.ib() - _port_allocator = attr.ib() - introducer = attr.ib() - flog_gatherer = attr.ib() - storage_servers = attr.ib(factory=list) - clients = attr.ib(factory=dict) - - @storage_servers.validator - def check(self, attribute, value): - for server in value: - if not isinstance(server, StorageServer): - raise ValueError( - "storage_servers must be StorageServer" - ) - - @inlineCallbacks - def add_storage_node(self): - """ - Creates a new storage node, returns a StorageServer instance - (which will already be added to our .storage_servers list) - """ - port = yield self._port_allocator() - print("make {}".format(port)) - name = 'node{}'.format(port) - web_port = 'tcp:{}:interface=localhost'.format(port) - server = yield create_storage_server( - self._reactor, - self._request, - self._temp_dir, - self.introducer, - self.flog_gatherer, - name, - web_port, - ) - self.storage_servers.append(server) - returnValue(server) - - @inlineCallbacks - def add_client(self, name, needed=2, happy=3, total=4): - """ - Create a new client node - """ - port = yield self._port_allocator() - web_port = 'tcp:{}:interface=localhost'.format(port) - client = yield create_client( - self._reactor, - self._request, - self._temp_dir, - self.introducer, - self.flog_gatherer, - name, - web_port, - needed=needed, - happy=happy, - total=total, - ) - self.clients[name] = client - yield await_client_ready(client.process) - returnValue(client) - - - -# XXX THINK can we tie a whole *grid* to a single request? (I think -# that's all that makes sense) -@inlineCallbacks -def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): - """ - """ - intro_port = yield port_allocator() - introducer = yield create_introducer(reactor, request, temp_dir, flog_gatherer, intro_port) - grid = Grid( - reactor, - request, - temp_dir, - port_allocator, - introducer, - flog_gatherer, - ) - returnValue(grid) - - -def create_port_allocator(start_port): - """ - Returns a new port-allocator .. which is a zero-argument function - that returns Deferreds that fire with new, sequential ports - starting at `start_port` skipping any that already appear to have - a listener. - - There can still be a race against other processes allocating ports - -- between the time when we check the status of the port and when - our subprocess starts up. This *could* be mitigated by instructing - the OS to not randomly-allocate ports in some range, and then - using that range here (explicitly, ourselves). - - NB once we're Python3-only this could be an async-generator - """ - port = [start_port - 1] - - # import stays here to not interfere with reactor selection -- but - # maybe this function should be arranged to be called once from a - # fixture (with the reactor)? - from twisted.internet import reactor - - class NothingProtocol(Protocol): - """ - I do nothing. - """ - - def port_generator(): - print("Checking port {}".format(port)) - port[0] += 1 - ep = TCP4ServerEndpoint(reactor, port[0], interface="localhost") - d = ep.listen(Factory.forProtocol(NothingProtocol)) - - def good(listening_port): - unlisten_d = maybeDeferred(listening_port.stopListening) - def return_port(_): - return port[0] - unlisten_d.addBoth(return_port) - return unlisten_d - - def try_again(fail): - return port_generator() - - d.addCallbacks(good, try_again) - return d - return port_generator diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py deleted file mode 100644 index db93f17c7..000000000 --- a/integration/test_grid_manager.py +++ /dev/null @@ -1,245 +0,0 @@ -import sys -import time -import json -import shutil -from os import mkdir, unlink, listdir, utime -from os.path import join, exists, getmtime - -from cryptography.hazmat.primitives.serialization import ( - Encoding, - PublicFormat, -) - -from allmydata.crypto import ed25519 -from allmydata.util import base32 -from allmydata.util import configutil - -import util -from grid import ( - create_grid, -) - -import pytest_twisted - - -@pytest_twisted.inlineCallbacks -def test_create_certificate(reactor, request): - """ - The Grid Manager produces a valid, correctly-signed certificate. - """ - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) - - # Note that zara + her key here are arbitrary and don't match any - # "actual" clients in the test-grid; we're just checking that the - # Grid Manager signs this properly. - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - stdin=gm_config, - ) - zara_cert_bytes = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "zara", "1", - stdin=gm_config, - ) - zara_cert = json.loads(zara_cert_bytes) - - # confirm that zara's certificate is made by the Grid Manager - # (.verify returns None on success, raises exception on error) - pubkey.verify( - base32.a2b(zara_cert['signature'].encode('ascii')), - zara_cert['certificate'].encode('ascii'), - ) - - -@pytest_twisted.inlineCallbacks -def test_remove_client(reactor, request): - """ - A Grid Manager can add and successfully remove a client - """ - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - stdin=gm_config, - ) - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", - stdin=gm_config, - ) - assert "zara" in json.loads(gm_config)['storage_servers'] - assert "yakov" in json.loads(gm_config)['storage_servers'] - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "remove", - "zara", - stdin=gm_config, - ) - assert "zara" not in json.loads(gm_config)['storage_servers'] - assert "yakov" in json.loads(gm_config)['storage_servers'] - - -@pytest_twisted.inlineCallbacks -def test_remove_last_client(reactor, request): - """ - A Grid Manager can remove all clients - """ - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - stdin=gm_config, - ) - assert "zara" in json.loads(gm_config)['storage_servers'] - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "remove", - "zara", - stdin=gm_config, - ) - # there are no storage servers left at all now - assert "storage_servers" not in json.loads(gm_config) - - -@pytest_twisted.inlineCallbacks -def test_add_remove_client_file(reactor, request, temp_dir): - """ - A Grid Manager can add and successfully remove a client (when - keeping data on disk) - """ - gmconfig = join(temp_dir, "gmtest") - gmconfig_file = join(temp_dir, "gmtest", "config.json") - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "create", - ) - - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - ) - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "add", - "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", - ) - assert "zara" in json.load(open(gmconfig_file, "r"))['storage_servers'] - assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] - - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "remove", - "zara", - ) - assert "zara" not in json.load(open(gmconfig_file, "r"))['storage_servers'] - assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] - - -@pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): - """ - A client with happines=2 fails to upload to a Grid when it is - using Grid Manager and there is only 1 storage server with a valid - certificate. - """ - grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) - storage0 = yield grid.add_storage_node() - storage1 = yield grid.add_storage_node() - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) - - # create certificate for the first storage-server - pubkey_fname = join(storage0.process.node_dir, "node.pubkey") - with open(pubkey_fname, 'r') as f: - pubkey_str = f.read().strip() - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "storage0", pubkey_str, - stdin=gm_config, - ) - assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - - print("inserting certificate") - cert = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "storage0", "1", - stdin=gm_config, - ) - - yield util.run_tahoe( - reactor, request, "--node-directory", storage0.process.node_dir, - "admin", "add-grid-manager-cert", - "--name", "default", - "--filename", "-", - stdin=cert, - ) - - # re-start this storage server - yield storage0.restart(reactor, request) - - # now only one storage-server has the certificate .. configure - # diana to have the grid-manager certificate - - diana = yield grid.add_client("diana", needed=2, happy=2, total=2) - - config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) - config.add_section("grid_managers") - config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) - with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: - config.write(f) - - yield diana.restart(reactor, request, servers=2) - - # try to put something into the grid, which should fail (because - # diana has happy=2 but should only find storage0 to be acceptable - # to upload to) - - try: - yield util.run_tahoe( - reactor, request, "--node-directory", diana.process.node_dir, - "put", "-", - stdin="some content\n" * 200, - ) - assert False, "Should get a failure" - except util.ProcessFailed as e: - assert 'UploadUnhappinessError' in e.output - - -@pytest_twisted.inlineCallbacks -def test_identity(reactor, request, temp_dir): - """ - Dump public key to CLI - """ - gm_config = join(temp_dir, "test_identity") - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gm_config, "create", - ) - - # ask the CLI for the grid-manager pubkey - pubkey = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gm_config, "public-identity", - ) - alleged_pubkey = ed25519.verifying_key_from_string(pubkey.strip()) - - # load the grid-manager pubkey "ourselves" - with open(join(gm_config, "config.json"), "r") as f: - real_config = json.load(f) - real_privkey, real_pubkey = ed25519.signing_keypair_from_string( - real_config["private_key"].encode("ascii"), - ) - - # confirm the CLI told us the correct thing - alleged_bytes = alleged_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) - real_bytes = real_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) - assert alleged_bytes == real_bytes, "Keys don't match" diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index d98bfe0b8..e5e4eb565 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -39,7 +39,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto try: yield proto.done assert False, "should raise exception" - except util.ProcessFailed as e: - assert "UploadUnhappinessError" in e.output + except Exception as e: + assert isinstance(e, ProcessTerminated) - assert "shares could be placed on only" in proto.output.getvalue() + output = proto.output.getvalue() + assert "shares could be placed on only" in output diff --git a/integration/test_tor.py b/integration/test_tor.py index 2423ef7d8..28360207a 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -76,28 +76,25 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) - if exists(node_dir): - raise RuntimeError( - "A node already exists in '{}'".format(node_dir) + if True: + print("creating", node_dir) + mkdir(node_dir) + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--tor-control-port', 'tcp:localhost:{}'.format(control_port), + '--listen', 'tor', + node_dir, + ) ) - print("creating", node_dir) - mkdir(node_dir) - proto = util._DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'allmydata.scripts.runner', - 'create-node', - '--nickname', name, - '--introducer', introducer_furl, - '--hide-ip', - '--tor-control-port', 'tcp:localhost:{}'.format(control_port), - '--listen', 'tor', - node_dir, - ) - ) - yield proto.done + yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write(''' diff --git a/integration/test_web.py b/integration/test_web.py index 36a7d3757..575e4fc1a 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -96,7 +96,7 @@ def test_helper_status(storage_nodes): successfully GET the /helper_status page """ - url = util.node_url(storage_nodes[0].process.node_dir, "helper_status") + url = util.node_url(storage_nodes[0].node_dir, "helper_status") resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") @@ -416,7 +416,7 @@ def test_storage_info(storage_nodes): storage0 = storage_nodes[0] requests.get( - util.node_url(storage0.process.node_dir, u"storage"), + util.node_url(storage0.node_dir, u"storage"), ) @@ -427,7 +427,7 @@ def test_storage_info_json(storage_nodes): storage0 = storage_nodes[0] resp = requests.get( - util.node_url(storage0.process.node_dir, u"storage"), + util.node_url(storage0.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) @@ -439,12 +439,12 @@ def test_introducer_info(introducer): retrieve and confirm /introducer URI for the introducer """ resp = requests.get( - util.node_url(introducer.process.node_dir, u""), + util.node_url(introducer.node_dir, u""), ) assert "Introducer" in resp.content resp = requests.get( - util.node_url(introducer.process.node_dir, u""), + util.node_url(introducer.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) diff --git a/integration/util.py b/integration/util.py index 5cad59c12..bbcf5efc6 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,6 @@ from os import mkdir from os.path import exists, join from six.moves import StringIO from functools import partial -from shutil import rmtree from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol @@ -36,38 +35,15 @@ class _ProcessExitedProtocol(ProcessProtocol): self.done.callback(None) -class ProcessFailed(Exception): - """ - A subprocess has failed. - - :ivar ProcessTerminated reason: the original reason from .processExited - - :ivar StringIO output: all stdout and stderr collected to this point. - """ - - def __init__(self, reason, output): - self.reason = reason - self.output = output - - def __str__(self): - return ":\n{}".format(self.reason, self.output) - - class _CollectOutputProtocol(ProcessProtocol): """ Internal helper. Collects all output (stdout + stderr) into self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self, stdin=None): + def __init__(self): self.done = Deferred() self.output = StringIO() - self._stdin = stdin - - def connectionMade(self): - if self._stdin is not None: - self.transport.write(self._stdin) - self.transport.closeStdin() def processEnded(self, reason): if not self.done.called: @@ -75,7 +51,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(ProcessFailed(reason, self.output.getvalue())) + self.done.errback(reason) def outReceived(self, data): self.output.write(data) @@ -147,27 +123,13 @@ def _cleanup_tahoe_process(tahoe_transport, exited): try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') - print("signaled, blocking on exit {}".format(exited)) + print("signaled, blocking on exit") pytest_twisted.blockon(exited) print("exited, goodbye") except ProcessExitedAlready: pass -def run_tahoe(reactor, request, *args, **kwargs): - """ - Helper to run tahoe with optional coverage. - - :returns: a Deferred that fires when the command is done (or a - ProcessFailed exception if it exits non-zero) - """ - stdin = kwargs.get("stdin", None) - protocol = _CollectOutputProtocol(stdin=stdin) - process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) - process.exited = protocol.done - return protocol.done - - def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): """ Internal helper. Calls spawnProcess with `-m @@ -269,7 +231,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - print("creating: {}".format(node_dir)) + print("creating", node_dir) mkdir(node_dir) done_proto = _ProcessExitedProtocol() args = [ @@ -294,7 +256,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam def created(_): config_path = join(node_dir, 'tahoe.cfg') config = get_config(config_path) - set_config(config, 'node', 'log_gatherer.furl', flog_gatherer.furl) + set_config(config, 'node', 'log_gatherer.furl', flog_gatherer) write_config(config_path, config) created_d.addCallback(created) @@ -481,7 +443,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): +def await_client_ready(tahoe, timeout=10, liveness=60*2): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -505,8 +467,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): time.sleep(1) continue - if len(js['servers']) < servers: - print("waiting because fewer than {} server(s)".format(servers)) + if len(js['servers']) == 0: + print("waiting because no servers at all") time.sleep(1) continue server_times = [ From c7f4a1a1574f712b0b9b4b9074b5a370e6da387d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 18:33:54 -0700 Subject: [PATCH 131/272] factor to use FilePath more --- src/allmydata/scripts/tahoe_grid_manager.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f84f57766..42ded3344 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -330,22 +330,15 @@ def _save_gridmanager_config(file_path, grid_manager): f.write("{}\n".format(data)) -def _load_gridmanager_config(gm_config): +def _load_gridmanager_config(fp): """ Loads a Grid Manager configuration and returns it (a dict) after validating. Exceptions if the config can't be found, or has problems. - :param gm_config str: "-" (a single dash) for stdin or a filename + :param FilePath fp: None for stdin or a path to a Grid Manager + configuration directory """ - fp = None - if gm_config.strip() != '-': - fp = FilePath(gm_config.strip()) - if not fp.exists(): - raise RuntimeError( - "No such directory '{}'".format(gm_config) - ) - if fp is None: gm = json.load(sys.stdin) else: @@ -353,7 +346,7 @@ def _load_gridmanager_config(gm_config): gm = json.load(f) try: - return _GridManager.from_config(gm, gm_config) + return _GridManager.from_config(gm, fp or "") except ValueError as e: raise usage.UsageError(str(e)) From 2118a2446e4ae36fd282a60955e9e268a81392dd Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 22:23:53 -0700 Subject: [PATCH 132/272] grid-manager stand-alone, via Click --- setup.py | 7 +- src/allmydata/cli/__init__.py | 0 src/allmydata/cli/grid_manager.py | 207 ++++++++ src/allmydata/grid_manager.py | 217 +++++++++ src/allmydata/scripts/tahoe_grid_manager.py | 497 -------------------- 5 files changed, 430 insertions(+), 498 deletions(-) create mode 100644 src/allmydata/cli/__init__.py create mode 100644 src/allmydata/cli/grid_manager.py create mode 100644 src/allmydata/grid_manager.py delete mode 100644 src/allmydata/scripts/tahoe_grid_manager.py diff --git a/setup.py b/setup.py index 4151545f7..4c603147f 100644 --- a/setup.py +++ b/setup.py @@ -410,6 +410,11 @@ setup(name="tahoe-lafs", # also set in __init__.py }, include_package_data=True, setup_requires=setup_requires, - entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, + entry_points={ + 'console_scripts': [ + 'tahoe = allmydata.scripts.runner:run', + 'grid-manager = allmydata.cli.grid_manager:grid_manager', + ] + }, **setup_args ) diff --git a/src/allmydata/cli/__init__.py b/src/allmydata/cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py new file mode 100644 index 000000000..ce0197345 --- /dev/null +++ b/src/allmydata/cli/grid_manager.py @@ -0,0 +1,207 @@ +from datetime import ( + datetime, +) +import json + +import click + +from twisted.python.filepath import ( + FilePath, +) + +from allmydata.crypto import ( + ed25519, +) +from allmydata.util.abbreviate import ( + abbreviate_time, +) +from allmydata.grid_manager import ( + create_grid_manager, + save_grid_manager, + load_grid_manager, +) + + +@click.group() +@click.option( + '--config', '-c', + type=click.Path(), + help="Configuration directory (or - for stdin)", + required=True, +) +@click.pass_context +def grid_manager(ctx, config): + """ + A Tahoe Grid Manager issues certificates to storage-servers + + A Tahoe client with one or more Grid Manager public keys + configured will only upload to a Storage Server that presents a + valid certificate signed by one of the configured Grid + Manager keys. + + Grid Manager configuration can be in a local directory or given + via stdin. It contains long-term secret information (a private + signing key) and should be kept safe. + """ + + class Config(object): + """ + Availble to all sub-commands as Click's context.obj + """ + _grid_manager = None + + @property + def grid_manager(self): + if self._grid_manager is None: + config_path = _config_path_from_option(config) + self._grid_manager = load_grid_manager(config_path, config) + return self._grid_manager + + ctx.obj = Config() + + +@grid_manager.command() +@click.pass_context +def create(ctx): + """ + Make a new Grid Manager + """ + config_location = ctx.parent.params["config"] + fp = None + if config_location != '-': + fp = FilePath(config_location) + if fp.exists(): + raise click.ClickException( + "The directory '{}' already exists.".format(config_location) + ) + + gm = create_grid_manager() + save_grid_manager(fp, gm) + + +@grid_manager.command() +@click.pass_obj +def public_identity(config): + """ + Show the public identity key of a Grid Manager + + This is what you give to clients to add to their configuration so + they use announcements from this Grid Manager + """ + click.echo(config.grid_manager.public_identity()) + + +@grid_manager.command() +@click.argument("name") +@click.argument("public_key", type=click.UNPROCESSED) +@click.pass_context +def add(ctx, name, public_key): + """ + Add a new storage-server by name to a Grid Manager + + PUBLIC_KEY is the contents of a node.pubkey file from a Tahoe + node-directory. NAME is an arbitrary label. + """ + try: + ctx.obj.grid_manager.add_storage_server( + name, + ed25519.verifying_key_from_string(public_key), + ) + except KeyError: + raise click.ClickException( + "A storage-server called '{}' already exists".format(name) + ) + save_grid_manager( + _config_path_from_option(ctx.parent.params["config"]), + ctx.obj.grid_manager, + ) + return 0 + + +@grid_manager.command() +@click.argument("name") +@click.pass_context +def remove(ctx, name): + """ + Remove an existing storage-server by name from a Grid Manager + """ + fp = _config_path_from_option(ctx.parent.params["config"]) + try: + ctx.obj.grid_manager.remove_storage_server(name) + except KeyError: + raise click.ClickException( + "No storage-server called '{}' exists".format(name) + ) + cert_count = 0 + if fp is not None: + while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): + fp.child('{}.cert.{}'.format(name, cert_count)).remove() + cert_count += 1 + + save_grid_manager(fp, ctx.obj.grid_manager) + + +@grid_manager.command() +@click.pass_context +def list(ctx): + """ + List all storage-servers known to a Grid Manager + """ + fp = _config_path_from_option(ctx.parent.params["config"]) + for name in sorted(ctx.obj.grid_manager.storage_servers.keys()): + blank_name = " " * len(name) + click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key())) + if fp: + cert_count = 0 + while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): + container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) + cert_data = json.loads(container['certificate']) + expires = datetime.utcfromtimestamp(cert_data['expires']) + delta = datetime.utcnow() - expires + click.echo("{} cert {}: ".format(blank_name, cert_count), nl=False) + if delta.total_seconds() < 0: + click.echo("valid until {} ({})".format(expires, abbreviate_time(delta))) + else: + click.echo("expired {} ({})".format(expires, abbreviate_time(delta))) + cert_count += 1 + + +@grid_manager.command() +@click.argument("name") +@click.argument( + "expiry_days", + type=click.IntRange(1, 5*365), # XXX is 5 years a good maximum? +) +@click.pass_context +def sign(ctx, name, expiry_days): + """ + sign a new certificate + """ + fp = _config_path_from_option(ctx.parent.params["config"]) + expiry_seconds = int(expiry_days) * 86400 + + try: + certificate = ctx.obj.grid_manager.sign(name, expiry_seconds) + except KeyError: + raise click.ClickException( + "No storage-server called '{}' exists".format(name) + ) + + certificate_data = json.dumps(certificate, indent=4) + click.echo(certificate_data) + if fp is not None: + next_serial = 0 + while fp.child("{}.cert.{}".format(name, next_serial)).exists(): + next_serial += 1 + with fp.child('{}.cert.{}'.format(name, next_serial)).open('w') as f: + f.write(certificate_data) + + +def _config_path_from_option(config): + """ + :param string config: a path or - + :returns: a FilePath instance or None + """ + if config == "-": + return None + return FilePath(config) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py new file mode 100644 index 000000000..071e1233f --- /dev/null +++ b/src/allmydata/grid_manager.py @@ -0,0 +1,217 @@ +import sys +import json +from datetime import ( + datetime, + timedelta, +) + +from allmydata.crypto import ( + ed25519, +) +from allmydata.util import ( + fileutil, + base32, +) + + +class _GridManagerStorageServer(object): + """ + A Grid Manager's notion of a storage server + """ + + def __init__(self, name, public_key, certificates): + self.name = name + self._public_key = public_key + self._certificates = [] if certificates is None else certificates + + def add_certificate(self, certificate): + self._certificates.append(certificate) + + def public_key(self): + return ed25519.string_from_verifying_key(self._public_key) + + def marshal(self): + return { + u"public_key": self.public_key(), + } + + +def create_grid_manager(): + """ + Create a new Grid Manager with a fresh keypair + """ + private_key, public_key = ed25519.create_signing_keypair() + return _GridManager( + ed25519.string_from_signing_key(private_key), + {}, + ) + + +def load_grid_manager(config_path, config_location): + """ + Load a Grid Manager from existing configuration. + + :param FilePath config_path: the configuratino location (or None for + stdin) + + :param str config_location: a string describing the config's location + + :returns: a GridManager instance + """ + if config_path is None: + config_file = sys.stdin + else: + try: + config_file = config_path.child("config.json").open("r") + except IOError: + raise ValueError( + "'{}' is not a Grid Manager config-directory".format(config) + ) + with config_file: + config = json.load(config_file) + + if not config: + raise ValueError( + "Invalid Grid Manager config in '{}'".format(config_location) + ) + if 'private_key' not in config: + raise ValueError( + "Grid Manager config from '{}' requires a 'private_key'".format( + config_location, + ) + ) + + private_key_bytes = config['private_key'].encode('ascii') + try: + private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes) + except Exception as e: + raise ValueError( + "Invalid Grid Manager private_key: {}".format(e) + ) + + storage_servers = dict() + for name, srv_config in config.get(u'storage_servers', {}).items(): + if not 'public_key' in srv_config: + raise ValueError( + "No 'public_key' for storage server '{}'".format(name) + ) + storage_servers[name] = _GridManagerStorageServer( + name, + ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), + None, + ) + + gm_version = config.get(u'grid_manager_config_version', None) + if gm_version != 0: + raise ValueError( + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) + ) + return _GridManager(private_key_bytes, storage_servers) + + +class _GridManager(object): + """ + A Grid Manager's configuration. + """ + + def __init__(self, private_key_bytes, storage_servers): + self._storage_servers = dict() if storage_servers is None else storage_servers + self._private_key_bytes = private_key_bytes + self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) + self._version = 0 + + @property + def storage_servers(self): + return self._storage_servers + + def public_identity(self): + return ed25519.string_from_verifying_key(self._public_key) + + def sign(self, name, expiry_seconds): + try: + srv = self._storage_servers[name] + except KeyError: + raise KeyError( + u"No storage server named '{}'".format(name) + ) + expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) + epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() + cert_info = { + "expires": epoch_offset, + "public_key": srv.public_key(), + "version": 1, + } + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') + sig = ed25519.sign_data(self._private_key, cert_data) + certificate = { + u"certificate": cert_data, + u"signature": base32.b2a(sig), + } + + vk = ed25519.verifying_key_from_signing_key(self._private_key) + ed25519.verify_signature(vk, sig, cert_data) + + return certificate + + def add_storage_server(self, name, public_key): + """ + :param name: a user-meaningful name for the server + :param public_key: ed25519.VerifyingKey the public-key of the + storage provider (e.g. from the contents of node.pubkey + for the client) + """ + if name in self._storage_servers: + raise KeyError( + "Already have a storage server called '{}'".format(name) + ) + ss = _GridManagerStorageServer(name, public_key, None) + self._storage_servers[name] = ss + return ss + + def remove_storage_server(self, name): + """ + :param name: a user-meaningful name for the server + """ + try: + del self._storage_servers[name] + except KeyError: + raise KeyError( + "No storage server called '{}'".format(name) + ) + + def marshal(self): + data = { + u"grid_manager_config_version": self._version, + u"private_key": self._private_key_bytes.decode('ascii'), + } + if self._storage_servers: + data[u"storage_servers"] = { + name: srv.marshal() + for name, srv + in self._storage_servers.items() + } + return data + + +def save_grid_manager(file_path, grid_manager): + """ + Writes a Grid Manager configuration. + + :param file_path: a FilePath specifying where to write the config + (if None, stdout is used) + + :param grid_manager: a _GridManager instance + """ + data = json.dumps( + grid_manager.marshal(), + indent=4, + ) + + if file_path is None: + print("{}\n".format(data)) + else: + fileutil.make_dirs(file_path.path, mode=0o700) + with file_path.child("config.json").open("w") as f: + f.write("{}\n".format(data)) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py deleted file mode 100644 index 42ded3344..000000000 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ /dev/null @@ -1,497 +0,0 @@ -from __future__ import print_function - -import sys -import json -from datetime import datetime, timedelta - -from allmydata.scripts.common import BaseOptions -from allmydata.util.abbreviate import abbreviate_time -from twisted.python import usage -from twisted.python.filepath import FilePath -from allmydata.util import fileutil -from allmydata.util import base32 -from allmydata.crypto import ed25519 -from twisted.internet.defer import inlineCallbacks, returnValue - - -class CreateOptions(BaseOptions): - description = ( - "Create a new identity key and configuration of a Grid Manager" - ) - - -class ShowIdentityOptions(BaseOptions): - description = ( - "Show the public identity key of a Grid Manager\n" - "\n" - "This is what you give to clients to add to their configuration" - " so they use announcements from this Grid Manager" - ) - - -class AddOptions(BaseOptions): - description = ( - "Add a new storage-server's key to a Grid Manager configuration\n" - "using NAME and PUBIC_KEY (comes from a node.pubkey file)" - ) - - def getSynopsis(self): - return "{} add NAME PUBLIC_KEY".format(super(AddOptions, self).getSynopsis()) - - def parseArgs(self, *args, **kw): - BaseOptions.parseArgs(self, **kw) - if len(args) != 2: - raise usage.UsageError( - "Requires two arguments: name public_key" - ) - self['name'] = unicode(args[0]) - try: - # WTF?! why does it want 'str' and not six.text_type? - self['storage_public_key'] = ed25519.verifying_key_from_string(args[1]) - except Exception as e: - raise usage.UsageError( - "Invalid public_key argument: {}".format(e) - ) - - -class RemoveOptions(BaseOptions): - description = ( - "Remove a storage-server from a Grid Manager configuration" - ) - - def parseArgs(self, *args, **kw): - BaseOptions.parseArgs(self, **kw) - if len(args) != 1: - raise usage.UsageError( - "Requires one arguments: name" - ) - self['name'] = unicode(args[0]) - - -class ListOptions(BaseOptions): - description = ( - "List all storage servers in this Grid Manager" - ) - - -class SignOptions(BaseOptions): - description = ( - "Create and sign a new certificate for a storage-server" - ) - - def getSynopsis(self): - return "{} NAME EXPIRY_DAYS".format(super(SignOptions, self).getSynopsis()) - - def parseArgs(self, *args, **kw): - BaseOptions.parseArgs(self, **kw) - if len(args) != 2: - raise usage.UsageError( - "Requires two arguments: name expiry_days" - ) - self['name'] = unicode(args[0]) - self['expiry_days'] = int(args[1]) - if self['expiry_days'] < 1 or self['expiry_days'] > 20*365: - raise usage.UsageError( - "Certificate expires in an unreasonable number of days: {}".format( - self['expiry_days'], - ) - ) - - -class GridManagerOptions(BaseOptions): - subCommands = [ - ["create", None, CreateOptions, "Create a Grid Manager."], - ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], - ["add", None, AddOptions, "Add a storage server to this Grid Manager."], - ["remove", None, RemoveOptions, "Remove a storage server from this Grid Manager."], - ["list", None, ListOptions, "List all storage servers in this Grid Manager."], - ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], - ] - - optParameters = [ - ("config", "c", None, "How to find the Grid Manager's configuration") - ] - - def postOptions(self): - if not hasattr(self, 'subOptions'): - raise usage.UsageError("must specify a subcommand") - if self['config'] is None: - raise usage.UsageError("Must supply configuration with --config") - - description = ( - 'A "grid-manager" consists of some data defining a keypair (along with ' - 'some other details) and Tahoe sub-commands to manipulate the data and ' - 'produce certificates to give to storage-servers. Certificates assert ' - 'the statement: "Grid Manager X suggests you use storage-server Y to ' - 'upload shares to" (X and Y are public-keys).' - '\n\n' - 'Clients can use Grid Managers to decide which storage servers to ' - 'upload shares to. They do this by adding one or more Grid Manager ' - 'public keys to their config.' - ) - - -def _create_gridmanager(): - """ - :return: an object providing the GridManager interface initialized - with a new random keypair - """ - private_key, public_key = ed25519.create_signing_keypair() - return _GridManager( - ed25519.string_from_signing_key(private_key), - {}, - ) - -def _create(gridoptions, options): - """ - Create a new Grid Manager - """ - gm_config = gridoptions['config'] - - # pre-conditions check - fp = None - if gm_config.strip() != '-': - fp = FilePath(gm_config.strip()) - if fp.exists(): - raise usage.UsageError( - "The directory '{}' already exists.".format(gm_config) - ) - - gm = _create_gridmanager() - _save_gridmanager_config(fp, gm) - - -class _GridManagerStorageServer(object): - """ - A Grid Manager's notion of a storage server - """ - - def __init__(self, name, public_key, certificates): - self.name = name - self._public_key = public_key - self._certificates = [] if certificates is None else certificates - - def add_certificate(self, certificate): - self._certificates.append(certificate) - - def public_key(self): - return ed25519.string_from_verifying_key(self._public_key) - - def marshal(self): - return { - u"public_key": self.public_key(), - } - -class _GridManager(object): - """ - A Grid Manager's configuration. - """ - - @staticmethod - def from_config(config, config_location): - if not config: - raise ValueError( - "Invalid Grid Manager config in '{}'".format(config_location) - ) - if 'private_key' not in config: - raise ValueError( - "Grid Manager config from '{}' requires a 'private_key'".format( - config_location, - ) - ) - - private_key_bytes = config['private_key'].encode('ascii') - try: - private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes) - except Exception as e: - raise ValueError( - "Invalid Grid Manager private_key: {}".format(e) - ) - - storage_servers = dict() - for name, srv_config in config.get(u'storage_servers', {}).items(): - if not 'public_key' in srv_config: - raise ValueError( - "No 'public_key' for storage server '{}'".format(name) - ) - storage_servers[name] = _GridManagerStorageServer( - name, - ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), - None, - ) - - gm_version = config.get(u'grid_manager_config_version', None) - if gm_version != 0: - raise ValueError( - "Missing or unknown version '{}' of Grid Manager config".format( - gm_version - ) - ) - return _GridManager(private_key_bytes, storage_servers) - - def __init__(self, private_key_bytes, storage_servers): - self._storage_servers = dict() if storage_servers is None else storage_servers - self._private_key_bytes = private_key_bytes - self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) - self._version = 0 - - @property - def storage_servers(self): - return self._storage_servers - - def public_identity(self): - return ed25519.string_from_verifying_key(self._public_key) - - def sign(self, name, expiry_seconds): - try: - srv = self._storage_servers[name] - except KeyError: - raise KeyError( - u"No storage server named '{}'".format(name) - ) - expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) - epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() - cert_info = { - "expires": epoch_offset, - "public_key": srv.public_key(), - "version": 1, - } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') - sig = ed25519.sign_data(self._private_key, cert_data) - certificate = { - u"certificate": cert_data, - u"signature": base32.b2a(sig), - } - - vk = ed25519.verifying_key_from_signing_key(self._private_key) - ed25519.verify_signature(vk, sig, cert_data) - - return certificate - - def add_storage_server(self, name, public_key): - """ - :param name: a user-meaningful name for the server - :param public_key: ed25519.VerifyingKey the public-key of the - storage provider (e.g. from the contents of node.pubkey - for the client) - """ - if name in self._storage_servers: - raise KeyError( - "Already have a storage server called '{}'".format(name) - ) - ss = _GridManagerStorageServer(name, public_key, None) - self._storage_servers[name] = ss - return ss - - def remove_storage_server(self, name): - """ - :param name: a user-meaningful name for the server - """ - try: - del self._storage_servers[name] - except KeyError: - raise KeyError( - "No storage server called '{}'".format(name) - ) - - def marshal(self): - data = { - u"grid_manager_config_version": self._version, - u"private_key": self._private_key_bytes.decode('ascii'), - } - if self._storage_servers: - data[u"storage_servers"] = { - name: srv.marshal() - for name, srv - in self._storage_servers.items() - } - return data - - -def _save_gridmanager_config(file_path, grid_manager): - """ - Writes a Grid Manager configuration. - - :param file_path: a FilePath specifying where to write the config - (if None, stdout is used) - - :param grid_manager: a _GridManager instance - """ - data = json.dumps( - grid_manager.marshal(), - indent=4, - ) - - if file_path is None: - print("{}\n".format(data)) - else: - fileutil.make_dirs(file_path.path, mode=0o700) - with file_path.child("config.json").open("w") as f: - f.write("{}\n".format(data)) - - -def _load_gridmanager_config(fp): - """ - Loads a Grid Manager configuration and returns it (a dict) after - validating. Exceptions if the config can't be found, or has - problems. - - :param FilePath fp: None for stdin or a path to a Grid Manager - configuration directory - """ - if fp is None: - gm = json.load(sys.stdin) - else: - with fp.child("config.json").open("r") as f: - gm = json.load(f) - - try: - return _GridManager.from_config(gm, fp or "") - except ValueError as e: - raise usage.UsageError(str(e)) - - -def _show_identity(gridoptions, options): - """ - Output the public-key of a Grid Manager - """ - gm_config = gridoptions['config'].strip() - assert gm_config is not None - - gm = _load_gridmanager_config(gm_config) - print(gm.public_identity()) - - -def _add(gridoptions, options): - """ - Add a new storage-server by name to a Grid Manager - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - - gm = _load_gridmanager_config(gm_config) - try: - gm.add_storage_server( - options['name'], - options['storage_public_key'], - ) - except KeyError: - raise usage.UsageError( - "A storage-server called '{}' already exists".format(options['name']) - ) - - _save_gridmanager_config(fp, gm) - return 0 - - -def _remove(gridoptions, options): - """ - Remove an existing storage-server by name from a Grid Manager - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) - - try: - gm.remove_storage_server(options['name']) - except KeyError: - raise usage.UsageError( - "No storage-server called '{}' exists".format(options['name']) - ) - cert_count = 0 - if fp is not None: - while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): - fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() - cert_count += 1 - - _save_gridmanager_config(fp, gm) - return 0 - - -def _list(gridoptions, options): - """ - List all storage-servers known to a Grid Manager - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - - gm = _load_gridmanager_config(gm_config) - for name in sorted(gm.storage_servers.keys()): - print("{}: {}".format(name, gm.storage_servers[name].public_key())) - if fp: - cert_count = 0 - while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): - container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) - cert_data = json.loads(container['certificate']) - expires = datetime.utcfromtimestamp(cert_data['expires']) - delta = datetime.utcnow() - expires - if delta.total_seconds() < 0: - print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta))) - else: - print("{}: cert {}: expired ({})".format(name, cert_count, abbreviate_time(delta))) - cert_count += 1 - - -def _sign(gridoptions, options): - """ - sign a new certificate - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) - - expiry_seconds = int(options['expiry_days']) * 86400 - - try: - certificate = gm.sign(options['name'], expiry_seconds) - except KeyError: - raise usage.UsageError( - "No storage-server called '{}' exists".format(options['name']) - ) - - certificate_data = json.dumps(certificate, indent=4) - print(certificate_data) - if fp is not None: - next_serial = 0 - while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists(): - next_serial += 1 - with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f: - f.write(certificate_data) - - -grid_manager_commands = { - CreateOptions: _create, - ShowIdentityOptions: _show_identity, - AddOptions: _add, - RemoveOptions: _remove, - ListOptions: _list, - SignOptions: _sign, -} - -@inlineCallbacks -def gridmanager(config): - """ - Runs the 'tahoe grid-manager' command. - """ - if config.subCommand is None: - print(config) - returnValue(1) - - try: - f = grid_manager_commands[config.subOptions.__class__] - except KeyError: - print(config.subOptions, grid_manager_commands.keys()) - print("Unknown command 'tahoe grid-manager {}': no such grid-manager subcommand".format(config.subCommand)) - returnValue(2) - - x = yield f(config, config.subOptions) - returnValue(x) - -subCommands = [ - ["grid-manager", None, GridManagerOptions, - "Grid Manager subcommands: use 'tahoe grid-manager' for a list."], -] - -dispatch = { - "grid-manager": gridmanager, -} From d09690823df0b0bb7171044c37b7677947a532da Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Nov 2020 02:18:18 -0700 Subject: [PATCH 133/272] spelling --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 071e1233f..0982d0aa4 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -51,7 +51,7 @@ def load_grid_manager(config_path, config_location): """ Load a Grid Manager from existing configuration. - :param FilePath config_path: the configuratino location (or None for + :param FilePath config_path: the configuration location (or None for stdin) :param str config_location: a string describing the config's location From a8382a5356d6e8b43484ac794a95435efe1962f3 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Nov 2020 02:18:54 -0700 Subject: [PATCH 134/272] cleanup, more tests --- src/allmydata/grid_manager.py | 149 ++++++++++++++++-- src/allmydata/scripts/admin.py | 10 +- src/allmydata/scripts/runner.py | 5 +- src/allmydata/storage_client.py | 118 -------------- src/allmydata/test/test_grid_manager.py | 194 ++++++++++++++++++++++++ 5 files changed, 339 insertions(+), 137 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 0982d0aa4..dadf3b361 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -70,9 +70,12 @@ def load_grid_manager(config_path, config_location): with config_file: config = json.load(config_file) - if not config: + gm_version = config.get(u'grid_manager_config_version', None) + if gm_version != 0: raise ValueError( - "Invalid Grid Manager config in '{}'".format(config_location) + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) ) if 'private_key' not in config: raise ValueError( @@ -101,13 +104,6 @@ def load_grid_manager(config_path, config_location): None, ) - gm_version = config.get(u'grid_manager_config_version', None) - if gm_version != 0: - raise ValueError( - "Missing or unknown version '{}' of Grid Manager config".format( - gm_version - ) - ) return _GridManager(private_key_bytes, storage_servers) @@ -153,6 +149,7 @@ class _GridManager(object): vk = ed25519.verifying_key_from_signing_key(self._private_key) ed25519.verify_signature(vk, sig, cert_data) + srv.add_certificate(certificate) return certificate def add_storage_server(self, name, public_key): @@ -215,3 +212,137 @@ def save_grid_manager(file_path, grid_manager): fileutil.make_dirs(file_path.path, mode=0o700) with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) + + +def parse_grid_manager_certificate(gm_data): + """ + :param gm_data: some data that might be JSON that might be a valid + Grid Manager Certificate + + :returns: json data of a valid Grid Manager certificate, or an + exception if the data is not valid. + """ + + required_keys = { + 'certificate', + 'signature', + } + + js = json.loads(gm_data) + + if not isinstance(js, dict): + raise ValueError( + "Grid Manager certificate must be a dict" + ) + if set(js.keys()) != required_keys: + raise ValueError( + "Grid Manager certificate must contain: {}".format( + ", ".join("'{}'".format(k) for k in js.keys()), + ) + ) + return js + + +def validate_grid_manager_certificate(gm_key, alleged_cert): + """ + :param gm_key: a VerifyingKey instance, a Grid Manager's public + key. + + :param alleged_cert: dict with "certificate" and "signature" keys, where + "certificate" contains a JSON-serialized certificate for a Storage + Server (comes from a Grid Manager). + + :return: a dict consisting of the deserialized certificate data or + None if the signature is invalid. Note we do NOT check the + expiry time in this function. + """ + try: + ed25519.verify_signature( + gm_key, + base32.a2b(alleged_cert['signature'].encode('ascii')), + alleged_cert['certificate'].encode('ascii'), + ) + except ed25519.BadSignature: + return None + # signature is valid; now we can load the actual data + cert = json.loads(alleged_cert['certificate']) + return cert + + +def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): + """ + Creates a predicate for confirming some Grid Manager-issued + certificates against Grid Manager keys. A predicate is used + (instead of just returning True/False here) so that the + expiry-time can be tested on each call. + + :param list keys: 0 or more `VerifyingKey` instances + + :param list certs: 1 or more Grid Manager certificates each of + which is a `dict` containing 'signature' and 'certificate' keys. + + :param callable now_fn: a callable which returns the current UTC + timestamp (or datetime.utcnow if None). + + :param callable bad_cert: a two-argument callable which is invoked + when a certificate verification fails. The first argument is + the verifying key and the second is the certificate. If None + (the default) errors are print()-ed. Note that we may have + several certificates and only one must be valid, so this may + be called (multiple times) even if the function ultimately + returns successfully. + + :returns: a callable which will return True only-if there is at + least one valid certificate (that has not at this moment + expired) in `certs` signed by one of the keys in `keys`. + """ + + now_fn = datetime.utcnow if now_fn is None else now_fn + valid_certs = [] + + # if we have zero grid-manager keys then everything is valid + if not keys: + return lambda: True + + if bad_cert is None: + + def bad_cert(key, alleged_cert): + """ + We might want to let the user know about this failed-to-verify + certificate .. but also if you have multiple grid-managers + then a bunch of these messages would appear. Better would + be to bubble this up to some sort of status API (or maybe + on the Welcome page?) + + The only thing that might actually be interesting, though, + is whether this whole function returns false or not.. + """ + print( + "Grid Manager certificate signature failed. Certificate: " + "\"{cert}\" for key \"{key}\".".format( + cert=alleged_cert, + key=ed25519.string_from_verifying_key(key), + ) + ) + + # validate the signatures on any certificates we have (not yet the expiry dates) + for alleged_cert in certs: + for key in keys: + cert = _validate_grid_manager_certificate(key, alleged_cert) + if cert is not None: + valid_certs.append(cert) + else: + bad_cert(key, alleged_cert) + + def validate(): + now = now_fn() + # if *any* certificate is still valid then we consider the server valid + for cert in valid_certs: + expires = datetime.utcfromtimestamp(cert['expires']) + # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) + if expires > now: + # not-expired + return True + return False + + return validate diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e5b7dca2d..b13dfb07d 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -7,8 +7,8 @@ from os.path import exists, join from twisted.python import usage #from allmydata.node import read_config from allmydata.client import read_config -from allmydata.storage_client import ( - parse_grid_manager_data, +from allmydata.grid_manager import ( + parse_grid_manager_certificate, ) from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions @@ -90,7 +90,7 @@ class AddGridManagerCertOptions(BaseOptions): data = f.read() try: - self.certificate_data = parse_grid_manager_data(data) + self.certificate_data = parse_grid_manager_certificate(data) except ValueError as e: raise usage.UsageError( "Error parsing certificate: {}".format(e) @@ -142,11 +142,9 @@ def add_grid_manager_cert(options): # write all the data out fileutil.write(cert_path, cert_bytes) - # print("created {}: {} bytes".format(cert_fname, len(cert_bytes))) with open(config_path, "w") as f: - # XXX probably want a _Config.write_tahoe_cfg() or something? or just set_config() does that automagically + # XXX probably want a _Config.write_tahoe_cfg() or something? config.config.write(f) - # print("wrote {}".format(config_fname)) cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 209a379bc..24d027cc3 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -11,7 +11,7 @@ from allmydata.version_checks import get_package_versions_string from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ stats_gatherer, admin, tahoe_daemonize, tahoe_start, \ - tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite, tahoe_grid_manager + tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding from allmydata.util.eliotutil import ( opt_eliot_destination, @@ -63,7 +63,6 @@ class Options(usage.Options): + debug.subCommands + cli.subCommands + tahoe_invite.subCommands - + tahoe_grid_manager.subCommands ) optFlags = [ @@ -155,8 +154,6 @@ def dispatch(config, # these are blocking, and must be run in a thread f0 = cli.dispatch[command] f = lambda so: threads.deferToThread(f0, so) - elif command in tahoe_grid_manager.dispatch: - f = tahoe_grid_manager.dispatch[command] elif command in tahoe_invite.dispatch: f = tahoe_invite.dispatch[command] else: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5ffb44092..49c20866b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -431,124 +431,6 @@ class StubServer(object): return "?" -def parse_grid_manager_data(gm_data): - """ - :param gm_data: some data that might be JSON that might be a valid - Grid Manager Certificate - - :returns: json data of a valid Grid Manager certificate, or an - exception if the data is not valid. - """ - - required_keys = allowed_keys = [ - 'certificate', - 'signature', - ] - - js = json.loads(gm_data) - for k in js.keys(): - if k not in allowed_keys: - raise ValueError( - "Grid Manager certificate JSON may not contain '{}'".format( - k, - ) - ) - for k in required_keys: - if k not in js: - raise ValueError( - "Grid Manager certificate JSON must contain '{}'".format( - k, - ) - ) - return js - - -def _validate_grid_manager_certificate(gm_key, alleged_cert): - """ - :param gm_key: a VerifyingKey instance, a Grid Manager's public - key. - - :param alleged_cert: dict with "certificate" and "signature" keys, where - "certificate" contains a JSON-serialized certificate for a Storage - Server (comes from a Grid Manager). - - :return: a dict consisting of the deserialized certificate data or - None if the signature is invalid. Note we do NOT check the - expiry time in this function. - """ - try: - ed25519.verify_signature( - gm_key, - base32.a2b(alleged_cert['signature'].encode('ascii')), - alleged_cert['certificate'].encode('ascii'), - ) - except ed25519.BadSignature: - return None - # signature is valid; now we can load the actual data - cert = json.loads(alleged_cert['certificate']) - return cert - - -def create_grid_manager_verifier(keys, certs, now_fn=None): - """ - Creates a predicate for confirming some Grid Manager-issued - certificates against Grid Manager keys. A predicate is used - (instead of just returning True/False here) so that the - expiry-time can be tested on each call. - - :param list keys: 0 or more `VerifyingKey` instances - - :param list certs: 1 or more Grid Manager certificates each of - which is a `dict` containing 'signature' and 'certificate' keys. - - :param callable now_fn: a callable which returns the current UTC - timestamp (or datetime.utcnow if None). - - :returns: a callable which will return True only-if there is at - least one valid certificate in `certs` signed by one of the keys - in `keys`. - """ - - now_fn = datetime.utcnow if now_fn is None else now_fn - valid_certs = [] - - # if we have zero grid-manager keys then everything is valid - if not keys: - return lambda: True - - # validate the signatures on any certificates we have (not yet the expiry dates) - for alleged_cert in certs: - for key in keys: - cert = _validate_grid_manager_certificate(key, alleged_cert) - if cert is not None: - valid_certs.append(cert) - else: - # we might want to let the user know about this - # failed-to-verify certificate .. but also if you have - # multiple grid-managers then a bunch of these - # messages would appear - print( - "Grid Manager certificate signature failed. Certificate: " - "\"{cert}\" for key \"{key}\".".format( - key=key, - cert=alleged_cert, - ) - ) - - def validate(): - now = now_fn() - # if *any* certificate is still valid then we consider the server valid - for cert in valid_certs: - expires = datetime.utcfromtimestamp(cert['expires']) - # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) - if expires > now: - # not-expired - return True - return False - - return validate - - class IFoolscapStorageServer(Interface): """ An internal interface that mediates between ``NativeStorageServer`` and diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index a60739634..893941f7d 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,6 +1,10 @@ import json +from twisted.python.filepath import ( + FilePath, +) + from allmydata.client import ( create_storage_farm_broker, ) @@ -10,6 +14,17 @@ from allmydata.node import ( from allmydata.client import ( _valid_config as client_valid_config, ) +from allmydata.crypto import ( + ed25519, +) +from allmydata.util import ( + base32, +) +from allmydata.grid_manager import ( + load_grid_manager, + save_grid_manager, + create_grid_manager, +) from .common import SyncTestCase @@ -78,3 +93,182 @@ class GridManagerUtilities(SyncTestCase): 1, len(config.enumerate_section("grid_managers")) ) + + +class GridManagerVerifier(SyncTestCase): + """ + Tests related to rejecting or accepting Grid Manager certificates. + """ + + def setUp(self): + self.gm = create_grid_manager() + return super(GridManagerVerifier, self).setUp() + + def test_sign_cert(self): + """ + Add a storage-server and sign a certificate for it + """ + priv, pub = ed25519.create_signing_keypair() + self.gm.add_storage_server("test", pub) + cert = self.gm.sign("test", 86400) + + self.assertEqual( + set(cert.keys()), + {"certificate", "signature"}, + ) + gm_key = ed25519.verifying_key_from_string(self.gm.public_identity()) + self.assertEqual( + ed25519.verify_signature( + gm_key, + base32.a2b(cert["signature"]), + cert["certificate"], + ), + None + ) + + def test_sign_cert_wrong_name(self): + """ + Try to sign a storage-server that doesn't exist + """ + with self.assertRaises(KeyError): + self.gm.sign("doesn't exist", 86400) + + def test_add_cert(self): + """ + Add a storage-server and serialize it + """ + priv, pub = ed25519.create_signing_keypair() + self.gm.add_storage_server("test", pub) + + data = self.gm.marshal() + self.assertEqual( + data["storage_servers"], + { + "test": { + "public_key": ed25519.string_from_verifying_key(pub), + } + } + ) + + def test_remove(self): + """ + Add then remove a storage-server + """ + priv, pub = ed25519.create_signing_keypair() + self.gm.add_storage_server("test", pub) + self.gm.remove_storage_server("test") + self.assertEqual(len(self.gm.storage_servers), 0) + + def test_serialize(self): + """ + Write and then read a Grid Manager config + """ + priv0, pub0 = ed25519.create_signing_keypair() + priv1, pub1 = ed25519.create_signing_keypair() + self.gm.add_storage_server("test0", pub0) + self.gm.add_storage_server("test1", pub1) + + tempdir = self.mktemp() + fp = FilePath(tempdir) + + save_grid_manager(fp, self.gm) + gm2 = load_grid_manager(fp, tempdir) + self.assertEqual( + self.gm.public_identity(), + gm2.public_identity(), + ) + self.assertEqual( + len(self.gm.storage_servers), + len(gm2.storage_servers), + ) + for name, ss0 in self.gm.storage_servers.items(): + ss1 = gm2.storage_servers[name] + self.assertEqual(ss0.name, ss1.name) + self.assertEqual(ss0.public_key(), ss1.public_key()) + self.assertEqual(self.gm.marshal(), gm2.marshal()) + + def test_invalid_no_version(self): + """ + Invalid Grid Manager config with no version + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "private_key": "at least we have one", + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "unknown version", + str(ctx.exception), + ) + + def test_invalid_no_private_key(self): + """ + Invalid Grid Manager config with no private key + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "grid_manager_config_version": 0, + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "requires a 'private_key'", + str(ctx.exception), + ) + + def test_invalid_bad_private_key(self): + """ + Invalid Grid Manager config with bad private-key + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "grid_manager_config_version": 0, + "private_key": "not actually encoded key", + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "Invalid Grid Manager private_key", + str(ctx.exception), + ) + + def test_invalid_storage_server(self): + """ + Invalid Grid Manager config with missing public-key for + storage-server + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "grid_manager_config_version": 0, + "private_key": "priv-v0-ub7knkkmkptqbsax4tznymwzc4nk5lynskwjsiubmnhcpd7lvlqa", + "storage_servers": { + "bad": {} + } + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "No 'public_key' for storage server", + str(ctx.exception), + ) From 41fa8238d55b79be6f8ec013a17f5cf06d0d3912 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Nov 2020 03:26:05 -0700 Subject: [PATCH 135/272] more unittests --- src/allmydata/grid_manager.py | 2 +- src/allmydata/storage_client.py | 3 ++ src/allmydata/test/test_grid_manager.py | 51 +++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index dadf3b361..6c5230901 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -328,7 +328,7 @@ def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): # validate the signatures on any certificates we have (not yet the expiry dates) for alleged_cert in certs: for key in keys: - cert = _validate_grid_manager_certificate(key, alleged_cert) + cert = validate_grid_manager_certificate(key, alleged_cert) if cert is not None: valid_certs.append(cert) else: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 49c20866b..c17fbe4f7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -66,6 +66,9 @@ from allmydata.interfaces import ( IStorageServer, IFoolscapStoragePlugin, ) +from allmydata.grid_manager import ( + create_grid_manager_verifier, +) from allmydata.util import log, base32, connection_status from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 893941f7d..a420385f0 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -24,6 +24,8 @@ from allmydata.grid_manager import ( load_grid_manager, save_grid_manager, create_grid_manager, + parse_grid_manager_certificate, + create_grid_manager_verifier, ) from .common import SyncTestCase @@ -272,3 +274,52 @@ class GridManagerVerifier(SyncTestCase): "No 'public_key' for storage server", str(ctx.exception), ) + + def test_parse_cert(self): + """ + Parse an ostensibly valid storage certificate + """ + js = parse_grid_manager_certificate('{"certificate": "", "signature": ""}') + self.assertEqual( + set(js.keys()), + {"certificate", "signature"} + ) + # the signature isn't *valid*, but that's checked in a + # different function + + def test_parse_cert_not_dict(self): + """ + Certificate data not even a dict + """ + with self.assertRaises(ValueError) as ctx: + parse_grid_manager_certificate("[]") + self.assertIn( + "must be a dict", + str(ctx.exception), + ) + + def test_parse_cert_missing_signature(self): + """ + Missing the signature + """ + with self.assertRaises(ValueError) as ctx: + parse_grid_manager_certificate('{"certificate": ""}') + self.assertIn( + "must contain", + str(ctx.exception), + ) + + def test_validate_cert(self): + """ + Validate a correctly-signed certificate + """ + priv0, pub0 = ed25519.create_signing_keypair() + self.gm.add_storage_server("test0", pub0) + cert0 = self.gm.sign("test0", 86400) + + verify = create_grid_manager_verifier( + [self.gm._public_key], + [cert0], + ) + + self.assertTrue(verify()) From f3d530fc0bb673d9837bb287cc6b7bb0fc47e989 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 17:54:36 -0700 Subject: [PATCH 136/272] grid-manager CLI tests --- src/allmydata/test/cli/test_grid_manager.py | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/allmydata/test/cli/test_grid_manager.py diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py new file mode 100644 index 000000000..ebb963ab8 --- /dev/null +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -0,0 +1,56 @@ + +import os +import json + +from ..common import SyncTestCase +from allmydata.cli.grid_manager import ( + grid_manager, +) + +import click.testing + + +class GridManagerCommandLine(SyncTestCase): + """ + Test the mechanics of the `grid-manager` command + """ + + def setUp(self): + self.runner = click.testing.CliRunner() + super(GridManagerCommandLine, self).setUp() + + def test_create(self): + """ + Create a new grid-manager + """ + with self.runner.isolated_filesystem(): + result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.assertEqual(["foo"], os.listdir(".")) + self.assertEqual(["config.json"], os.listdir("./foo")) + + def test_create_stdout(self): + """ + Create a new grid-manager with no files + """ + with self.runner.isolated_filesystem(): + result = self.runner.invoke(grid_manager, ["--config", "-", "create"]) + self.assertEqual([], os.listdir(".")) + config = json.loads(result.output) + self.assertEqual( + {"private_key", "grid_manager_config_version"}, + set(config.keys()), + ) + + def test_add_and_sign(self): + """ + Add a new storage-server and sign a certificate for it + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + sigcert = json.loads(result.output) + self.assertEqual({"certificate", "signature"}, set(sigcert.keys())) + cert = json.loads(sigcert['certificate']) + self.assertEqual(cert["public_key"], pubkey) From 6e1bb1ecd54e6e47fb997e52b20e591be1ed7025 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 18:01:32 -0700 Subject: [PATCH 137/272] grid-manager CLI tests --- src/allmydata/test/cli/test_grid_manager.py | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index ebb963ab8..8383bb9cd 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -27,6 +27,8 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.assertEqual(["foo"], os.listdir(".")) self.assertEqual(["config.json"], os.listdir("./foo")) + result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) + self.assertTrue(result.output.startswith("pub-v0-")) def test_create_stdout(self): """ @@ -54,3 +56,24 @@ class GridManagerCommandLine(SyncTestCase): self.assertEqual({"certificate", "signature"}, set(sigcert.keys())) cert = json.loads(sigcert['certificate']) self.assertEqual(cert["public_key"], pubkey) + + def test_add_list_remove(self): + """ + Add a storage server, list it, remove it. + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + + result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + names = [ + line.split(':')[0] + for line in result.output.strip().split('\n') + ] + self.assertEqual(names, ["storage0"]) + + self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) + + result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + self.assertEqual(result.output.strip(), "") From b6fbfeee480331606484c7ed8f455f11f47846d7 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 18:23:21 -0700 Subject: [PATCH 138/272] more utests --- src/allmydata/client.py | 2 +- src/allmydata/test/test_grid_manager.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 019535a14..006b0eed3 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -565,7 +565,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] for name, gm_key in config.enumerate_section('grid_managers').items(): grid_manager_keys.append( - ed25519.verifying_key_from_string(gm_key) + ed25519.verifying_key_from_string(gm_key.encode("ascii")) ) # we don't actually use this keypair for anything (yet) as far diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index a420385f0..6ff4fb065 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -7,6 +7,7 @@ from twisted.python.filepath import ( from allmydata.client import ( create_storage_farm_broker, + _load_grid_manager_certificates, ) from allmydata.node import ( config_from_string, @@ -70,7 +71,7 @@ class GridManagerUtilities(SyncTestCase): } } sfb.set_static_servers(static_servers) - nss = sfb._make_storage_server(u"server0", {"ann": announcement}) + nss = sfb._make_storage_server(b"server0", {"ann": announcement}) # we have some grid-manager keys defined so the server should # only upload if there's a valid certificate -- but the only @@ -79,11 +80,12 @@ class GridManagerUtilities(SyncTestCase): def test_load_certificates(self): cert_path = self.mktemp() + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + } with open(cert_path, "w") as f: - f.write(json.dumps({ - "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", - "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" - })) + f.write(json.dumps(fake_cert)) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" @@ -95,6 +97,8 @@ class GridManagerUtilities(SyncTestCase): 1, len(config.enumerate_section("grid_managers")) ) + certs = _load_grid_manager_certificates(config) + self.assertEqual([fake_cert], certs) class GridManagerVerifier(SyncTestCase): From 32b19fa4d0868fd67bee9756d282e18be8cd3438 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 18:40:44 -0700 Subject: [PATCH 139/272] flake8 --- src/allmydata/grid_manager.py | 2 +- src/allmydata/storage_client.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 6c5230901..464e90b94 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -65,7 +65,7 @@ def load_grid_manager(config_path, config_location): config_file = config_path.child("config.json").open("r") except IOError: raise ValueError( - "'{}' is not a Grid Manager config-directory".format(config) + "'{}' is not a Grid Manager config-directory".format(config_path) ) with config_file: config = json.load(config_file) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7f24af410..bb5039f46 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -41,9 +41,7 @@ if PY2: import re import time -import json import hashlib -from datetime import datetime # On Python 2 this will be the backport. from configparser import NoSectionError From c7f4f36c8af02156505b4c15856da8f542ea7a5f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 21:30:23 -0700 Subject: [PATCH 140/272] merge config --- src/allmydata/client.py | 12 ------------ src/allmydata/storage_client.py | 21 ++++++++++++++++++--- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 006b0eed3..b2b819916 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -92,9 +92,6 @@ _client_config = configutil.ValidConfiguration( ), "grid_managers": None, # means "any options valid" "grid_manager_certificates": None, - "drop_upload": ( # deprecated already? - "enabled", - ), "ftpd": ( "accounts.file", "accounts.url", @@ -560,14 +557,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con **kwargs ) - # grid manager setup - - grid_manager_keys = [] - for name, gm_key in config.enumerate_section('grid_managers').items(): - grid_manager_keys.append( - ed25519.verifying_key_from_string(gm_key.encode("ascii")) - ) - # we don't actually use this keypair for anything (yet) as far # as I can see. # my_pubkey = keyutil.parse_pubkey( @@ -581,7 +570,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con tub_maker=tub_creator, node_config=config, storage_client_config=storage_client_config, - grid_manager_keys=grid_manager_keys, # XXX maybe roll into above storage_client_config? ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index bb5039f46..aa8431e05 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -74,6 +74,9 @@ from allmydata.interfaces import ( from allmydata.grid_manager import ( create_grid_manager_verifier, ) +from allmydata.crypto import ( + ed25519, +) from allmydata.util import log, base32, connection_status from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList @@ -111,9 +114,15 @@ class StorageClientConfig(object): :ivar dict[unicode, dict[unicode, unicode]] storage_plugins: A mapping from names of ``IFoolscapStoragePlugin`` configured in *tahoe.cfg* to the respective configuration. + + :ivar list[ed25519.VerifyKey] grid_manager_keys: with no keys in + this list, we'll upload to any storage server. Otherwise, we will + only upload to a storage-server that has a valid certificate + signed by at least one of these keys. """ preferred_peers = attr.ib(default=()) storage_plugins = attr.ib(default=attr.Factory(dict)) + grid_manager_keys = attr.ib(default=attr.Factory(list)) @classmethod def from_node_config(cls, config): @@ -145,9 +154,17 @@ class StorageClientConfig(object): plugin_config = [] storage_plugins[plugin_name] = dict(plugin_config) + grid_manager_keys = [] + for name, gm_key in config.enumerate_section('grid_managers').items(): + grid_manager_keys.append( + ed25519.verifying_key_from_string(gm_key.encode("ascii")) + ) + + return cls( preferred_peers, storage_plugins, + grid_manager_keys, ) @@ -173,13 +190,11 @@ class StorageFarmBroker(service.MultiService): tub_maker, node_config, storage_client_config=None, - grid_manager_keys=None, ): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker - self._grid_manager_keys = grid_manager_keys if grid_manager_keys else list() self.node_config = node_config @@ -268,7 +283,7 @@ class StorageFarmBroker(service.MultiService): assert isinstance(server_id, bytes) handler_overrides = server.get("connections", {}) gm_verifier = create_grid_manager_verifier( - self._grid_manager_keys, + self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), ) From 45128f49c5b77997f834461aed295cc2037d9c05 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:22:18 -0700 Subject: [PATCH 141/272] more works on config --- docs/proposed/grid-manager/managed-grid.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 01181d42a..289ecd004 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -1,7 +1,3 @@ -(This document is "in-progress", with feedback and input from two -devchats with Brain Warner and exarkun as well as other input, -discussion and edits from exarkun. It is NOT done). Search for -"DECIDE" for open questions. Managed Grid @@ -65,7 +61,16 @@ use ``--config -`` (the last character is a dash) and write a valid JSON configuration to stdin. All commands require the ``--config`` option and they all behave -similarly for "data from stdin" versus "data from disk". +similarly for "data from stdin" versus "data from disk". A directory +(and not a file) is used on disk because in that mode, each +certificate issued is also stored alongside the configuration +document; in "stdin / stdout" mode, an issued certificate is only +ever available on stdout. + +The configuration is a JSON document. It is subject to change as Grid +Manager evolves. It contains a version number in the +`grid_manager_config_version` key which should increment whenever the +document schema changes. tahoe grid-manager create From 5b4f5e89e14c2f736380db7c451cfcdea01e8892 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:22:27 -0700 Subject: [PATCH 142/272] explicit unicode --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b2b819916..4570e660d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -937,7 +937,7 @@ class _Client(node.Node, pollmixin.PollMixin): if self.config.get_config("storage", "grid_management", default=False, boolean=True): grid_manager_certificates = _load_grid_manager_certificates(self.config) - announcement["grid-manager-certificates"] = grid_manager_certificates + announcement[u"grid-manager-certificates"] = grid_manager_certificates # XXX we should probably verify that the certificates are # valid and not expired, as that could be confusing for the From c58dd36c8729c9ad0b4f22ff072b8e80ad89e9c4 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:28:06 -0700 Subject: [PATCH 143/272] eliot changes --- src/allmydata/mutable/publish.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index f8f486f51..a6e0d900f 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -35,8 +35,10 @@ from allmydata.mutable.layout import get_version_from_checkstring,\ MDMFSlotWriteProxy, \ SDMFSlotWriteProxy -import eliot - +from eliot import ( + Message, + start_action, +) KiB = 1024 DEFAULT_MAX_SEGMENT_SIZE = 128 * KiB @@ -959,7 +961,7 @@ class Publish(object): serverlist = [] - action = eliot.start_action( + action = start_action( action_type=u"mutable:upload:update_goal", homeless_shares=len(homeless_shares), ) @@ -967,17 +969,17 @@ class Publish(object): for i, server in enumerate(self.full_serverlist): serverid = server.get_serverid() if server in self.bad_servers: - action.log( + Message.log( + message_type=u"mutable:upload:bad-server", server_id=server.get_server_id(), - message="Server is bad", ) continue # if we have >= 1 grid-managers, this checks that we have # a valid certificate for this server if not server.upload_permitted(): - action.log( + Message.log( + message_type=u"mutable:upload:no-gm-certs", server_id=server.get_server_id(), - message="No valid grid-manager certificates", ) continue From bab77eded87505b0bb399a3076d7329159f78608 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:57:05 -0700 Subject: [PATCH 144/272] function not method --- src/allmydata/node.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 6333f8424..498b70a5d 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -298,19 +298,20 @@ class _Config(object): self.config = configparser if write_new_tahoecfg is None: - write_new_tahoecfg = self._default_write_new_tahoecfg + + def write_new_tahoecfg(config): + """ + Write to the default place, /tahoe.cfg + """ + fn = os.path.join(self._basedir, "tahoe.cfg") + with open(fn, "w") as f: + config.write(f) + self._write_config = write_new_tahoecfg self.nickname = self.get_config("node", "nickname", u"") assert isinstance(self.nickname, str) - def _default_write_new_tahoecfg(self, config): - """ - Write to the default place, /tahoe.cfg - """ - fn = os.path.join(self._basedir, "tahoe.cfg") - with open(fn, "w") as f: - config.write(f) def validate(self, valid_config_sections): configutil.validate_config(self._config_fname, self.config, valid_config_sections) From 0af033beb7453ef02c854e9a820e685c75ffe258 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:57:26 -0700 Subject: [PATCH 145/272] use filepath --- src/allmydata/scripts/admin.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index b13dfb07d..2b896fcfb 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -2,10 +2,10 @@ from __future__ import print_function import sys import json -from os.path import exists, join from twisted.python import usage -#from allmydata.node import read_config +from twisted.python.filepath import FilePath + from allmydata.client import read_config from allmydata.grid_manager import ( parse_grid_manager_certificate, @@ -16,6 +16,7 @@ from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import fileutil + class GenerateKeypairOptions(BaseOptions): def getUsage(self, width=None): @@ -80,7 +81,7 @@ class AddGridManagerCertOptions(BaseOptions): ) if self['filename'] == '-': print("reading certificate from stdin", file=self.parent.parent.stderr) - data = sys.stdin.read() + data = self.parent.parent.stdin.read() if len(data) == 0: raise usage.UsageError( "Reading certificate from stdin failed" @@ -124,11 +125,11 @@ def add_grid_manager_cert(options): config = read_config(nd, "portnum") config_path = join(nd, "tahoe.cfg") cert_fname = "{}.cert".format(options['name']) - cert_path = config.get_config_path(cert_fname) + cert_path = FilePath(config.get_config_path(cert_fname)) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' cert_name = options['name'] - if exists(cert_path): + if cert_path.exists(): msg = "Already have certificate for '{}' (at {})".format( options['name'], cert_path, @@ -140,11 +141,8 @@ def add_grid_manager_cert(options): config.set_config("grid_manager_certificates", cert_name, cert_fname) # write all the data out - - fileutil.write(cert_path, cert_bytes) - with open(config_path, "w") as f: - # XXX probably want a _Config.write_tahoe_cfg() or something? - config.config.write(f) + with cert_path.open("wb") as f: + f.wrwite(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) From 15e33f8b35a59b368f1d5923fde8921d67bf153a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:57:44 -0700 Subject: [PATCH 146/272] undo a wording change --- src/allmydata/test/test_abbreviate.py | 2 +- src/allmydata/util/abbreviate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 2fb3d6b95..3ef1e96a6 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -33,7 +33,7 @@ class Abbreviate(unittest.TestCase): def test_abbrev_time_future_5_minutes(self): diff = timedelta(minutes=-5) s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes from now', s) + self.assertEqual('5 minutes in the future', s) def test_abbrev_time_hours(self): diff = timedelta(hours=4) diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 5ca8aa3d1..f895c3727 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -40,7 +40,7 @@ def abbreviate_time(s): if s >= 0.0: postfix = ' ago' else: - postfix = ' from now' + postfix = ' in the future' s = -s def _plural(count, unit): count = int(count) From 1ccc074d04b42d7f5f509ca0c594046d1ce40fbf Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:59:11 -0700 Subject: [PATCH 147/272] docstring --- src/allmydata/test/test_grid_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 6ff4fb065..9188626c6 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -38,6 +38,10 @@ class GridManagerUtilities(SyncTestCase): """ def test_client_grid_manager(self): + """ + A client refuses to upload to a storage-server with an invalid + certificate when using Grid Manager. + """ config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" From 51b52174e19d292fc5f28152a31ffb1ae259884d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:59:43 -0700 Subject: [PATCH 148/272] docstring --- src/allmydata/test/test_grid_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 9188626c6..84ac280e1 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -83,6 +83,9 @@ class GridManagerUtilities(SyncTestCase): self.assertFalse(nss.upload_permitted()) def test_load_certificates(self): + """ + Grid Manager certificates are deserialized from config properly + """ cert_path = self.mktemp() fake_cert = { "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", From 3a552f68caa1cc470708417248a35463831d6c71 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:09:15 -0700 Subject: [PATCH 149/272] don't ned this test --- src/allmydata/test/test_grid_manager.py | 45 ------------------------- 1 file changed, 45 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 84ac280e1..946722540 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -37,51 +37,6 @@ class GridManagerUtilities(SyncTestCase): Confirm operation of utility functions used by GridManager """ - def test_client_grid_manager(self): - """ - A client refuses to upload to a storage-server with an invalid - certificate when using Grid Manager. - """ - config_data = ( - "[grid_managers]\n" - "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" - ) - config = config_from_string("/foo", "portnum", config_data, client_valid_config()) - sfb = create_storage_farm_broker(config, {}, {}, {}, []) - # could introspect sfb._grid_manager_certificates, but that's - # "cheating"? even though _make_storage_sever is also - # "private"? - - # ...but, okay, a "real" client will call set_static_servers() - # with any configured/cached servers (thus causing - # _make_storage_server to be called). The other way - # _make_storage_server is called is when _got_announcement - # runs, which is when an introducer client gets an - # announcement... - - invalid_cert = { - "certificate": "foo", - "signature": "43564356435643564356435643564356", - } - announcement = { - "anonymous-storage-FURL": b"pb://abcde@nowhere/fake", - "grid-manager-certificates": [ - invalid_cert, - ] - } - static_servers = { - "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia": { - "ann": announcement, - } - } - sfb.set_static_servers(static_servers) - nss = sfb._make_storage_server(b"server0", {"ann": announcement}) - - # we have some grid-manager keys defined so the server should - # only upload if there's a valid certificate -- but the only - # one we have is invalid - self.assertFalse(nss.upload_permitted()) - def test_load_certificates(self): """ Grid Manager certificates are deserialized from config properly From cb6ed4c47ffd8a11b411c157b0dd16883e2261d8 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:14:27 -0700 Subject: [PATCH 150/272] flake8 --- src/allmydata/scripts/admin.py | 3 --- src/allmydata/test/test_grid_manager.py | 1 - 2 files changed, 4 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 2b896fcfb..803f6561c 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,6 +1,5 @@ from __future__ import print_function -import sys import json from twisted.python import usage @@ -13,7 +12,6 @@ from allmydata.grid_manager import ( from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath -from allmydata.util import fileutil @@ -123,7 +121,6 @@ def add_grid_manager_cert(options): nd = _default_nodedir config = read_config(nd, "portnum") - config_path = join(nd, "tahoe.cfg") cert_fname = "{}.cert".format(options['name']) cert_path = FilePath(config.get_config_path(cert_fname)) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 946722540..f7c4d4c7f 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -6,7 +6,6 @@ from twisted.python.filepath import ( ) from allmydata.client import ( - create_storage_farm_broker, _load_grid_manager_certificates, ) from allmydata.node import ( From 7fa0f64f1ccd4bbef8aff4acc3fe13d8788e75ed Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:38:00 -0700 Subject: [PATCH 151/272] better assert --- src/allmydata/test/test_grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index f7c4d4c7f..45d2061b2 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -55,8 +55,8 @@ class GridManagerUtilities(SyncTestCase): ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) self.assertEqual( - 1, - len(config.enumerate_section("grid_managers")) + {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, + config.enumerate_section("grid_managers") ) certs = _load_grid_manager_certificates(config) self.assertEqual([fake_cert], certs) From f3b33412678bdd1466ae30150f4f1a2317f9dc77 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:40:17 -0700 Subject: [PATCH 152/272] better-ize test --- src/allmydata/test/test_node.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 5491ffc0e..c032d20d7 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -432,10 +432,17 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.failUnless(ns.called) def test_set_config_new_section(self): + """ + set_config() can create a new config section + """ basedir = "test_node/test_set_config_new_section" config = config_from_string(basedir, "", "") config.set_config("foo", "bar", "value1") config.set_config("foo", "bar", "value2") + self.assertEqual( + config.get_config("foo", "bar"), + "value2" + ) class TestMissingPorts(unittest.TestCase): From f6f53ad16514b379e99c02c7ad6eb4a0f5cf2896 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:41:08 -0700 Subject: [PATCH 153/272] proper spot --- docs/{proposed/grid-manager => }/managed-grid.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{proposed/grid-manager => }/managed-grid.rst (100%) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/managed-grid.rst similarity index 100% rename from docs/proposed/grid-manager/managed-grid.rst rename to docs/managed-grid.rst From 885f72ff2bebdca815fbc3a695db17dbe888c01c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 01:23:37 -0700 Subject: [PATCH 154/272] decisions --- docs/managed-grid.rst | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 289ecd004..1f062db19 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -212,7 +212,7 @@ Enrolling a Client: CLI tahoe add-grid-manager (PROPOSED) ````````````````````````````````` -DECIDE: this command hasn't actually been written yet. +(Note: this command hasn't actually been written yet). This takes two arguments: ``name`` and ``public-identity``. @@ -336,15 +336,6 @@ Put the key printed out above into Alice's ``tahoe.cfg`` in section example_name = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq -DECIDE: - - should the grid-manager be identified by a certificate? exarkun - points out: --name seems like the hint of the beginning of a - use-case for certificates rather than bare public keys?). - - (note the "--name" thing came from a former version of this - proposal that used CLI commands to add the public-keys -- but the - point remains, if there's to be metadata associated with "grid - managers" maybe they should be certificates..) - Now, re-start the "alice" client. Since we made Alice's parameters require 3 storage servers to be reachable (``--happy=3``), all their uploads should now fail (so ``tahoe put`` will fail) because they From 409e9bd65d2396cbf214d749a4816edf5b04ceca Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 01:27:16 -0700 Subject: [PATCH 155/272] file tickets for unimplemented commands --- docs/managed-grid.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 1f062db19..1fdafda7b 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -212,7 +212,8 @@ Enrolling a Client: CLI tahoe add-grid-manager (PROPOSED) ````````````````````````````````` -(Note: this command hasn't actually been written yet). +(Note: this command hasn't actually been written yet, see +https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3507). This takes two arguments: ``name`` and ``public-identity``. @@ -341,8 +342,9 @@ require 3 storage servers to be reachable (``--happy=3``), all their uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". -You can check Alice's "Welcome" page (where the list of connected servers -is) at http://localhost:6301/ and should be able to see details about -the "work-grid" Grid Manager that you added. When any Grid Managers -are enabled, each storage-server line will show whether it has a valid +(PROPOSED https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506) You can +check Alice's "Welcome" page (where the list of connected servers is) +at http://localhost:6301/ and should be able to see details about the +"work-grid" Grid Manager that you added. When any Grid Managers are +enabled, each storage-server line will show whether it has a valid certificate or not (and how much longer it's valid until). From 019772a2c28fc8389100691906572ca0b30fa19a Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 11:24:46 -0700 Subject: [PATCH 156/272] typo --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 803f6561c..28ab0f2ed 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -139,7 +139,7 @@ def add_grid_manager_cert(options): # write all the data out with cert_path.open("wb") as f: - f.wrwite(cert_bytes) + f.write(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) From a34093ed0e9c29d7e838dc65aa6c104bb8cc7195 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 11:58:27 -0700 Subject: [PATCH 157/272] fix some test-ability problems and add tests for 'tahoe admin add-grid-manager-cert' --- src/allmydata/scripts/runner.py | 6 +- src/allmydata/test/cli/test_grid_manager.py | 90 ++++++++++++++++++++- src/allmydata/test/common_util.py | 13 +-- 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 24d027cc3..2184569b6 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -115,8 +115,11 @@ def parse_options(argv, config=None): config.parseOptions(argv) # may raise usage.error return config -def parse_or_exit_with_explanation(argv, stdout=sys.stdout): +def parse_or_exit_with_explanation(argv, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin): config = Options() + config.stdout = stdout + config.stdin = stdin + config.stderr = stderr try: parse_options(argv, config=config) except usage.error as e: @@ -141,6 +144,7 @@ def dispatch(config, so.stdout = stdout so.stderr = stderr so.stdin = stdin + config.stdin = stdin if command in create_dispatch: f = create_dispatch[command] diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 8383bb9cd..6d71a14eb 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -2,13 +2,27 @@ import os import json -from ..common import SyncTestCase +from ..common import ( + SyncTestCase, + AsyncTestCase, +) from allmydata.cli.grid_manager import ( grid_manager, ) import click.testing +# these imports support the tests for `tahoe *` subcommands +from ..common_util import ( + run_cli, +) +from twisted.internet.defer import ( + inlineCallbacks, +) +from twisted.python.filepath import ( + FilePath, +) + class GridManagerCommandLine(SyncTestCase): """ @@ -77,3 +91,77 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) self.assertEqual(result.output.strip(), "") + + +# note: CLITestMixin can't function without also GridTestMixin ... :/ +class TahoeAddGridManagerCert(AsyncTestCase): + """ + Test `tahoe admin add-grid-manager-cert` subcommand + """ + + @inlineCallbacks + def test_help(self): + """ + some kind of help is printed + """ + code, out, err = yield run_cli("admin", "add-grid-manager-cert") + self.assertEqual(err, "") + self.assertNotEqual(0, code) + + @inlineCallbacks + def test_no_name(self): + """ + error to miss --name option + """ + code, out, err = yield run_cli( + "admin", "add-grid-manager-cert", "--filename", "-", + stdin="the cert", + ) + self.assertIn( + "Must provide --name", + out + ) + + @inlineCallbacks + def test_no_filename(self): + """ + error to miss --name option + """ + code, out, err = yield run_cli( + "admin", "add-grid-manager-cert", "--name", "foo", + stdin="the cert", + ) + self.assertIn( + "Must provide --filename", + out + ) + + @inlineCallbacks + def test_add_one(self): + """ + we can add a certificate + """ + nodedir = self.mktemp() + fake_cert = """{"certificate": "", "signature": ""}""" + + code, out, err = yield run_cli( + "--node-directory", nodedir, + "admin", "add-grid-manager-cert", "-f", "-", "--name", "foo", + stdin=fake_cert, + ignore_stderr=True, + ) + nodepath = FilePath(nodedir) + with nodepath.child("tahoe.cfg").open("r") as f: + config_data = f.read() + + self.assertIn("tahoe.cfg", nodepath.listdir()) + self.assertIn( + "foo = foo.cert", + config_data, + ) + self.assertIn("foo.cert", nodepath.listdir()) + with nodepath.child("foo.cert").open("r") as f: + self.assertEqual( + json.load(f), + json.loads(fake_cert) + ) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e3f5cf750..cbc00a135 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -40,14 +40,17 @@ def run_cli(verb, *args, **kwargs): "arguments to do_cli must be strs -- convert using unicode_to_argv", args=args) nodeargs = kwargs.get("nodeargs", []) argv = nodeargs + [verb] + list(args) - stdin = kwargs.get("stdin", "") + stdin = StringIO(kwargs.get("stdin", "")) stdout = StringIO() stderr = StringIO() d = defer.succeed(argv) - d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout) - d.addCallback(runner.dispatch, - stdin=StringIO(stdin), - stdout=stdout, stderr=stderr) + d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) + d.addCallback( + runner.dispatch, + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) def _done(rc): return 0, stdout.getvalue(), stderr.getvalue() def _err(f): From 6e77eba68837d3cc38f1576004e4218fa6922351 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 15:15:49 -0700 Subject: [PATCH 158/272] test another path --- src/allmydata/test/cli/test_grid_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 6d71a14eb..85dc757fd 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -79,11 +79,13 @@ class GridManagerCommandLine(SyncTestCase): with self.runner.isolated_filesystem(): self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "1"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) names = [ line.split(':')[0] for line in result.output.strip().split('\n') + if not line.startswith(" ") # "cert" lines start with whitespace ] self.assertEqual(names, ["storage0"]) @@ -93,7 +95,6 @@ class GridManagerCommandLine(SyncTestCase): self.assertEqual(result.output.strip(), "") -# note: CLITestMixin can't function without also GridTestMixin ... :/ class TahoeAddGridManagerCert(AsyncTestCase): """ Test `tahoe admin add-grid-manager-cert` subcommand From 3b3b95838e6d7f0e0e3efdacfc1dc50e1d0597de Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 16:55:06 -0700 Subject: [PATCH 159/272] tempdir should be native-string --- src/allmydata/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 498b70a5d..a72303e33 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -28,6 +28,7 @@ import configparser from twisted.python import log as twlog from twisted.application import service from twisted.python.failure import Failure +from twisted.python.compat import nativeString from foolscap.api import Tub, app_versions import foolscap.logging.log from allmydata.version_checks import get_package_versions, get_package_versions_string @@ -828,7 +829,7 @@ class Node(service.MultiService): tempdir = self.config.get_config_path(tempdir_config) if not os.path.exists(tempdir): fileutil.make_dirs(tempdir) - tempfile.tempdir = tempdir + tempfile.tempdir = nativeString(tempdir) # this should cause twisted.web.http (which uses # tempfile.TemporaryFile) to put large request bodies in the given # directory. Without this, the default temp dir is usually /tmp/, From d34c32cc503fc50438c1ead36be80d8a9a8342f1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 10:49:07 -0700 Subject: [PATCH 160/272] cleanup / review --- src/allmydata/grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 464e90b94..61b2c90c1 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -94,7 +94,7 @@ def load_grid_manager(config_path, config_location): storage_servers = dict() for name, srv_config in config.get(u'storage_servers', {}).items(): - if not 'public_key' in srv_config: + if 'public_key' not in srv_config: raise ValueError( "No 'public_key' for storage server '{}'".format(name) ) @@ -130,7 +130,7 @@ class _GridManager(object): srv = self._storage_servers[name] except KeyError: raise KeyError( - u"No storage server named '{}'".format(name) + "No storage server named '{}'".format(name) ) expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() From 800497a71977130272e1c26ea5d085a89510c7da Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 10:57:30 -0700 Subject: [PATCH 161/272] refactor load_grid_manager() to better present errors (review) --- src/allmydata/cli/grid_manager.py | 7 ++++++- src/allmydata/grid_manager.py | 10 ++++------ src/allmydata/test/test_grid_manager.py | 12 ++++++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index ce0197345..c839684ca 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -54,7 +54,12 @@ def grid_manager(ctx, config): def grid_manager(self): if self._grid_manager is None: config_path = _config_path_from_option(config) - self._grid_manager = load_grid_manager(config_path, config) + try: + self._grid_manager = load_grid_manager(config_path) + except ValueError as e: + raise click.ClickException( + "Error loading Grid Manager from '{}': {}".format(config, e) + ) return self._grid_manager ctx.obj = Config() diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 61b2c90c1..68465a6c1 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -47,16 +47,16 @@ def create_grid_manager(): ) -def load_grid_manager(config_path, config_location): +def load_grid_manager(config_path): """ Load a Grid Manager from existing configuration. :param FilePath config_path: the configuration location (or None for stdin) - :param str config_location: a string describing the config's location - :returns: a GridManager instance + + :raises: ValueError if the confguration is invalid """ if config_path is None: config_file = sys.stdin @@ -79,9 +79,7 @@ def load_grid_manager(config_path, config_location): ) if 'private_key' not in config: raise ValueError( - "Grid Manager config from '{}' requires a 'private_key'".format( - config_location, - ) + "'private_key' required in config" ) private_key_bytes = config['private_key'].encode('ascii') diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 45d2061b2..36d15bc46 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -139,7 +139,7 @@ class GridManagerVerifier(SyncTestCase): fp = FilePath(tempdir) save_grid_manager(fp, self.gm) - gm2 = load_grid_manager(fp, tempdir) + gm2 = load_grid_manager(fp) self.assertEqual( self.gm.public_identity(), gm2.public_identity(), @@ -168,7 +168,7 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( "unknown version", str(ctx.exception), @@ -188,9 +188,9 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( - "requires a 'private_key'", + "'private_key' required", str(ctx.exception), ) @@ -209,7 +209,7 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( "Invalid Grid Manager private_key", str(ctx.exception), @@ -234,7 +234,7 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( "No 'public_key' for storage server", str(ctx.exception), From 32625bf655c7b8fc7214ebcfe338c913e1247f8c Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 10:59:03 -0700 Subject: [PATCH 162/272] tahoe grid-manager -> grid-manager --- docs/managed-grid.rst | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 1fdafda7b..b045e9565 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -73,8 +73,8 @@ Manager evolves. It contains a version number in the document schema changes. -tahoe grid-manager create -````````````````````````` +grid-manager create +``````````````````` Create a new grid-manager. @@ -84,8 +84,8 @@ directory specified by the ``--config`` option. It is an error if the directory already exists. -tahoe grid-manager public-identity -`````````````````````````````````` +grid-manager public-identity +```````````````````````````` Print out a grid-manager's public key. This key is derived from the private-key of the grid-manager, so a valid grid-manager config must @@ -95,8 +95,8 @@ This public key is what is put in clients' configuration to actually validate and use grid-manager certificates. -tahoe grid-manager add -`````````````````````` +grid-manager add +```````````````` Takes two args: ``name pubkey``. The ``name`` is an arbitrary local identifier for the new storage node (also sometimes called "a petname" @@ -105,7 +105,7 @@ file in the storage-server's node directory (minus any whitespace). For example, if ``~/storage0`` contains a storage-node, you might do something like this: - tahoe grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) + grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) This adds a new storage-server to a Grid Manager's configuration. (Since it mutates the configuration, if you used @@ -114,15 +114,15 @@ usefulness of the ``name`` is solely for reference within this Grid Manager. -tahoe grid-manager list -``````````````````````` +grid-manager list +````````````````` Lists all storage-servers that have previously been added using -``tahoe grid-manager add``. +``grid-manager add``. -tahoe grid-manager sign -``````````````````````` +grid-manager sign +````````````````` Takes one arg: ``name``, the nickname used previously in a ``tahoe grid-manager add`` command. @@ -190,7 +190,7 @@ turn on grid-management with ``grid_management = true``. You then must also provide a ``[grid_management_certificates]`` section in the config-file which lists ``name = path/to/certificate`` pairs. -These certificate files are issued by the ``tahoe grid-manager sign`` +These certificate files are issued by the ``grid-manager sign`` command; these should be **securely transmitted** to the storage server. Relative paths are based from the node directory. Example:: @@ -223,7 +223,7 @@ client with zero Grid Managers will accept any announcement from an Introducer. The ``public-identity`` argument is the encoded public key of the Grid -Manager (that is, the output of ``tahoe grid-manager +Manager (that is, the output of ``grid-manager public-identity``). The client will have to be re-started once this change is made. @@ -262,7 +262,7 @@ own shell/terminal window or via something like ``systemd`` We'll store our Grid Manager configuration on disk, in ``./gm0``. To initialize this directory:: - tahoe grid-manager --config ./gm0 create + grid-manager --config ./gm0 create (If you already have a grid, you can :ref:`skip ahead `.) @@ -284,13 +284,13 @@ Next, we attach a couple of storage nodes:: We can now tell the Grid Manager about our new storage servers:: - tahoe grid-manager --config ./gm0 add storage0 $(cat storage0/node.pubkey) - tahoe grid-manager --config ./gm0 add storage1 $(cat storage1/node.pubkey) + grid-manager --config ./gm0 add storage0 $(cat storage0/node.pubkey) + grid-manager --config ./gm0 add storage1 $(cat storage1/node.pubkey) To produce a new certificate for each node, we do this:: - tahoe grid-manager --config ./gm0 sign storage0 > ./storage0/gridmanager.cert - tahoe grid-manager --config ./gm0 sign storage1 > ./storage1/gridmanager.cert + grid-manager --config ./gm0 sign storage0 > ./storage0/gridmanager.cert + grid-manager --config ./gm0 sign storage1 > ./storage1/gridmanager.cert Now, we want our storage servers to actually announce these certificates into the grid. We do this by adding some configuration @@ -328,7 +328,7 @@ grid-manager has given certificates to (``storage0`` and ``storage1``). We need the grid-manager's public key to put in Alice's configuration:: - tahoe grid-manager --config ./gm0 public-identity + grid-manager --config ./gm0 public-identity Put the key printed out above into Alice's ``tahoe.cfg`` in section ``client``:: From eb11809798f2d78ac6afa51a99879cdeebe0a700 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 11:02:33 -0700 Subject: [PATCH 163/272] introduction (review) --- docs/managed-grid.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index b045e9565..187651380 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -3,6 +3,15 @@ Managed Grid ============ +This document explains the "Grid Manager" concept and the +`grid-manager` command. Someone operating a grid may choose to use a +Grid Manager. Operators of storage-servers and clients will then be +given additional configuration in this case. + + +Overview and Motivation +----------------------- + In a grid using an Introducer, a client will use any storage-server the Introducer announces (and the Introducer will announce any storage-server that connects to it). This means that anyone with the From 61f348f7a5e853fe09ad6d520130a581f1df0809 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 11:05:40 -0700 Subject: [PATCH 164/272] clarify --- docs/managed-grid.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 187651380..eab72ca27 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -133,8 +133,8 @@ Lists all storage-servers that have previously been added using grid-manager sign ````````````````` -Takes one arg: ``name``, the nickname used previously in a ``tahoe -grid-manager add`` command. +Takes one arg: ``name``, the nickname used previously in a +``grid-manager add`` command. Note that this mutates the state of the grid-manager if it is on disk, by adding this certificate to our collection of issued @@ -200,8 +200,9 @@ also provide a ``[grid_management_certificates]`` section in the config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``grid-manager sign`` -command; these should be **securely transmitted** to the storage -server. Relative paths are based from the node directory. Example:: +command; these should be transmitted to the storage server operator +who includes them in the config for the storage sever. Relative paths +are based from the node directory. Example:: [storage] grid_management = true From 649cb9380e0e0fa771634c5dfbd6a0275c08556a Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 11:11:55 -0700 Subject: [PATCH 165/272] move proposals out --- docs/managed-grid.rst | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index eab72ca27..11c5035e8 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -215,29 +215,6 @@ connect to (and subsequently, the Introducer will give the certificate out to clients). -Enrolling a Client: CLI ------------------------ - - -tahoe add-grid-manager (PROPOSED) -````````````````````````````````` - -(Note: this command hasn't actually been written yet, see -https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3507). - -This takes two arguments: ``name`` and ``public-identity``. - -The ``name`` argument is a nickname to call this Grid Manager. A -client may have any number of grid-managers, so each one has a name. A -client with zero Grid Managers will accept any announcement from an -Introducer. - -The ``public-identity`` argument is the encoded public key of the Grid -Manager (that is, the output of ``grid-manager -public-identity``). The client will have to be re-started once this -change is made. - - Enrolling a Client: Config -------------------------- @@ -254,6 +231,8 @@ Manager. Example:: [grid_managers] example_grid = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq +See also https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3507 which +proposes a command to edit the config. Example Setup of a New Managed Grid @@ -352,9 +331,6 @@ require 3 storage servers to be reachable (``--happy=3``), all their uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". -(PROPOSED https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506) You can -check Alice's "Welcome" page (where the list of connected servers is) -at http://localhost:6301/ and should be able to see details about the -"work-grid" Grid Manager that you added. When any Grid Managers are -enabled, each storage-server line will show whether it has a valid -certificate or not (and how much longer it's valid until). +A proposal to expose more information about Grid Manager and +certifcate status in the Welcome page is discussed in +https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506 From 938cc5616e82f1dba8213777efc0c9d744dc3eac Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 17:46:26 -0700 Subject: [PATCH 166/272] refactor (review): move grid-manager certificate loading code out of cli --- src/allmydata/cli/grid_manager.py | 34 ++++----- src/allmydata/grid_manager.py | 98 ++++++++++++++++++++----- src/allmydata/test/test_grid_manager.py | 2 +- 3 files changed, 97 insertions(+), 37 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index c839684ca..d937c0f30 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -152,23 +152,16 @@ def list(ctx): """ List all storage-servers known to a Grid Manager """ - fp = _config_path_from_option(ctx.parent.params["config"]) for name in sorted(ctx.obj.grid_manager.storage_servers.keys()): blank_name = " " * len(name) - click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key())) - if fp: - cert_count = 0 - while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): - container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) - cert_data = json.loads(container['certificate']) - expires = datetime.utcfromtimestamp(cert_data['expires']) - delta = datetime.utcnow() - expires - click.echo("{} cert {}: ".format(blank_name, cert_count), nl=False) - if delta.total_seconds() < 0: - click.echo("valid until {} ({})".format(expires, abbreviate_time(delta))) - else: - click.echo("expired {} ({})".format(expires, abbreviate_time(delta))) - cert_count += 1 + click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key_string())) + for cert in ctx.obj.grid_manager.storage_servers[name].certificates: + delta = datetime.utcnow() - cert.expires + click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) + if delta.total_seconds() < 0: + click.echo("valid until {} ({})".format(cert.expires, abbreviate_time(delta))) + else: + click.echo("expired {} ({})".format(cert.expires, abbreviate_time(delta))) @grid_manager.command() @@ -196,15 +189,20 @@ def sign(ctx, name, expiry_days): click.echo(certificate_data) if fp is not None: next_serial = 0 - while fp.child("{}.cert.{}".format(name, next_serial)).exists(): + f = None + while f is None: + try: + f = fp.child("{}.cert.{}".format(name, next_serial)).create() + except Exception: + f = None next_serial += 1 - with fp.child('{}.cert.{}'.format(name, next_serial)).open('w') as f: + with f: f.write(certificate_data) def _config_path_from_option(config): """ - :param string config: a path or - + :param str config: a path or - :returns: a FilePath instance or None """ if config == "-": diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 68465a6c1..d753295ab 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,3 +1,4 @@ + import sys import json from datetime import ( @@ -13,29 +14,46 @@ from allmydata.util import ( base32, ) +import attr + +@attr.s class _GridManagerStorageServer(object): """ A Grid Manager's notion of a storage server """ - def __init__(self, name, public_key, certificates): - self.name = name - self._public_key = public_key - self._certificates = [] if certificates is None else certificates + name = attr.ib() + public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) + certificates = attr.ib( + default=attr.Factory(list), + validator=attr.validators.instance_of(list), + ) def add_certificate(self, certificate): - self._certificates.append(certificate) + self.certificates.append(certificate) - def public_key(self): - return ed25519.string_from_verifying_key(self._public_key) + def public_key_string(self): + return ed25519.string_from_verifying_key(self.public_key) def marshal(self): return { - u"public_key": self.public_key(), + u"public_key": self.public_key_string(), } +@attr.s +class _GridManagerCertificate(object): + """ + Represents a single certificate for a single storage-server + """ + + filename = attr.ib() + index = attr.ib(validator=attr.validators.instance_of(int)) + expires = attr.ib(validator=attr.validators.instance_of(datetime)) + public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) + + def create_grid_manager(): """ Create a new Grid Manager with a fresh keypair @@ -47,6 +65,52 @@ def create_grid_manager(): ) +def _load_certificates_for(config_path, name, gm_key=None): + """ + Load any existing certificates for the given storage-server. + + :param FilePath config_path: the configuration location (or None for + stdin) + + :param str name: the name of an existing storage-server + + :param ed25519.VerifyingKey gm_key: an optional Grid Manager + public key. If provided, certificates will be verified against it. + + :returns: list containing any known certificates (may be empty) + + :raises: ed25519.BadSignature if any certificate signature fails to verify + """ + if config_path is None: + return [] + cert_index = 0 + cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) + certificates = [] + while cert_path.exists(): + container = json.load(cert_path.open('r')) + if gm_key is not None: + validate_grid_manager_certificate(gm_key, container) + cert_data = json.loads(container['certificate']) + if cert_data['version'] != 1: + raise ValueError( + "Unknown certificate version '{}' in '{}'".format( + cert_data['version'], + cert_path.path, + ) + ) + certificates.append( + _GridManagerCertificate( + filename=cert_path.path, + index=cert_index, + expires=datetime.utcfromtimestamp(cert_data['expires']), + public_key=ed25519.verifying_key_from_string(cert_data['public_key'].encode('ascii')), + ) + ) + cert_index += 1 + cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) + return certificates + + def load_grid_manager(config_path): """ Load a Grid Manager from existing configuration. @@ -56,17 +120,15 @@ def load_grid_manager(config_path): :returns: a GridManager instance - :raises: ValueError if the confguration is invalid + :raises: ValueError if the confguration is invalid or IOError if + expected files can't be opened. """ if config_path is None: config_file = sys.stdin else: - try: - config_file = config_path.child("config.json").open("r") - except IOError: - raise ValueError( - "'{}' is not a Grid Manager config-directory".format(config_path) - ) + # this might raise IOError or similar but caller must handle it + config_file = config_path.child("config.json").open("r") + with config_file: config = json.load(config_file) @@ -99,7 +161,7 @@ def load_grid_manager(config_path): storage_servers[name] = _GridManagerStorageServer( name, ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), - None, + _load_certificates_for(config_path, name, public_key), ) return _GridManager(private_key_bytes, storage_servers) @@ -134,7 +196,7 @@ class _GridManager(object): epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() cert_info = { "expires": epoch_offset, - "public_key": srv.public_key(), + "public_key": srv.public_key_string(), "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') @@ -161,7 +223,7 @@ class _GridManager(object): raise KeyError( "Already have a storage server called '{}'".format(name) ) - ss = _GridManagerStorageServer(name, public_key, None) + ss = _GridManagerStorageServer(name, public_key, []) self._storage_servers[name] = ss return ss diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 36d15bc46..ec780918d 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -151,7 +151,7 @@ class GridManagerVerifier(SyncTestCase): for name, ss0 in self.gm.storage_servers.items(): ss1 = gm2.storage_servers[name] self.assertEqual(ss0.name, ss1.name) - self.assertEqual(ss0.public_key(), ss1.public_key()) + self.assertEqual(ss0.public_key_string(), ss1.public_key_string()) self.assertEqual(self.gm.marshal(), gm2.marshal()) def test_invalid_no_version(self): From 72f2e25f86022659632f873dd05b506a1b918bd9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:20:10 -0700 Subject: [PATCH 167/272] ask forgiveness not permission --- src/allmydata/cli/grid_manager.py | 11 ++++++----- src/allmydata/grid_manager.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index d937c0f30..d07e2d412 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -75,13 +75,14 @@ def create(ctx): fp = None if config_location != '-': fp = FilePath(config_location) - if fp.exists(): - raise click.ClickException( - "The directory '{}' already exists.".format(config_location) - ) gm = create_grid_manager() - save_grid_manager(fp, gm) + try: + save_grid_manager(fp, gm) + except OSError as e: + raise click.ClickException( + "Can't create '{}': {}".format(config_location, e) + ) @grid_manager.command() diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index d753295ab..df0a46e6d 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -269,7 +269,8 @@ def save_grid_manager(file_path, grid_manager): if file_path is None: print("{}\n".format(data)) else: - fileutil.make_dirs(file_path.path, mode=0o700) + file_path.makedirs() + file_path.chmod(0o700) with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) From 1e1aad8cc8586fc2e2681eedcc247663f9811d18 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:36:39 -0700 Subject: [PATCH 168/272] save only fails sometimes --- src/allmydata/cli/grid_manager.py | 6 +++--- src/allmydata/grid_manager.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index d07e2d412..a5e0ee09a 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -144,7 +144,7 @@ def remove(ctx, name): fp.child('{}.cert.{}'.format(name, cert_count)).remove() cert_count += 1 - save_grid_manager(fp, ctx.obj.grid_manager) + save_grid_manager(fp, ctx.obj.grid_manager, create=False) @grid_manager.command() @@ -177,10 +177,10 @@ def sign(ctx, name, expiry_days): sign a new certificate """ fp = _config_path_from_option(ctx.parent.params["config"]) - expiry_seconds = int(expiry_days) * 86400 + expiry = timedelta(days=expiry_days) try: - certificate = ctx.obj.grid_manager.sign(name, expiry_seconds) + certificate = ctx.obj.grid_manager.sign(name, expiry) except KeyError: raise click.ClickException( "No storage-server called '{}' exists".format(name) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index df0a46e6d..71395fd11 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -252,7 +252,7 @@ class _GridManager(object): return data -def save_grid_manager(file_path, grid_manager): +def save_grid_manager(file_path, grid_manager, create=True): """ Writes a Grid Manager configuration. @@ -260,6 +260,9 @@ def save_grid_manager(file_path, grid_manager): (if None, stdout is used) :param grid_manager: a _GridManager instance + + :param bool create: if True (the default) we are creating a new + grid-manager and will fail if the directory already exists. """ data = json.dumps( grid_manager.marshal(), @@ -269,8 +272,12 @@ def save_grid_manager(file_path, grid_manager): if file_path is None: print("{}\n".format(data)) else: - file_path.makedirs() - file_path.chmod(0o700) + try: + file_path.makedirs() + file_path.chmod(0o700) + except OSError: + if create: + raise with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) From a98d784ce44140e4b93ef9c627364bbbb5553578 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:36:50 -0700 Subject: [PATCH 169/272] timedelta, not seconds --- src/allmydata/cli/grid_manager.py | 2 ++ src/allmydata/grid_manager.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index a5e0ee09a..8e561d36a 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,5 +1,6 @@ from datetime import ( datetime, + timedelta, ) import json @@ -120,6 +121,7 @@ def add(ctx, name, public_key): save_grid_manager( _config_path_from_option(ctx.parent.params["config"]), ctx.obj.grid_manager, + create=False, ) return 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 71395fd11..d3256813d 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -185,14 +185,25 @@ class _GridManager(object): def public_identity(self): return ed25519.string_from_verifying_key(self._public_key) - def sign(self, name, expiry_seconds): + def sign(self, name, expiry): + """ + Create a new signed certificate for a particular server + + :param str name: the server to create a certificate for + + :param timedelta expiry: how far in the future the certificate + should expire. + + :returns: a dict defining the certificate (it has + "certificate" and "signature" keys). + """ try: srv = self._storage_servers[name] except KeyError: raise KeyError( "No storage server named '{}'".format(name) ) - expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) + expiration = datetime.utcnow() + expiry epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() cert_info = { "expires": epoch_offset, From d1adbe0f64c15dd137c71f808c37e3e574d6db8d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:37:37 -0700 Subject: [PATCH 170/272] unused --- src/allmydata/grid_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index d3256813d..4c16b572a 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -3,14 +3,12 @@ import sys import json from datetime import ( datetime, - timedelta, ) from allmydata.crypto import ( ed25519, ) from allmydata.util import ( - fileutil, base32, ) From a21381997e8962dd798531d6a49d5b726e07f313 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 23:23:20 -0700 Subject: [PATCH 171/272] sign uses timedelta --- src/allmydata/test/test_grid_manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index ec780918d..bcea741d7 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,4 +1,6 @@ - +from datetime import ( + timedelta, +) import json from twisted.python.filepath import ( @@ -77,7 +79,7 @@ class GridManagerVerifier(SyncTestCase): """ priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) - cert = self.gm.sign("test", 86400) + cert = self.gm.sign("test", timedelta(seconds=86400)) self.assertEqual( set(cert.keys()), @@ -98,7 +100,7 @@ class GridManagerVerifier(SyncTestCase): Try to sign a storage-server that doesn't exist """ with self.assertRaises(KeyError): - self.gm.sign("doesn't exist", 86400) + self.gm.sign("doesn't exist", timedelta(seconds=86400)) def test_add_cert(self): """ @@ -280,7 +282,7 @@ class GridManagerVerifier(SyncTestCase): """ priv0, pub0 = ed25519.create_signing_keypair() self.gm.add_storage_server("test0", pub0) - cert0 = self.gm.sign("test0", 86400) + cert0 = self.gm.sign("test0", timedelta(seconds=86400)) verify = create_grid_manager_verifier( [self.gm._public_key], From 1b531359d7df9038151a438db4379eca4d86e2b2 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 23:33:27 -0700 Subject: [PATCH 172/272] test create-twice --- src/allmydata/test/cli/test_grid_manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 85dc757fd..a8b0315e0 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -44,6 +44,20 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) self.assertTrue(result.output.startswith("pub-v0-")) + def test_create_already(self): + """ + It's an error to create a new grid-manager in an existing + directory. + """ + with self.runner.isolated_filesystem(): + result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.assertEqual(1, result.exit_code) + self.assertIn( + "Can't create", + result.stdout, + ) + def test_create_stdout(self): """ Create a new grid-manager with no files From 42b7d3974f5c70261132ba34fbf492b4aee8c16a Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 24 Nov 2020 01:19:10 -0700 Subject: [PATCH 173/272] check public key / server-id --- src/allmydata/grid_manager.py | 14 +++++++++----- src/allmydata/storage_client.py | 1 + src/allmydata/test/test_grid_manager.py | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 4c16b572a..fde88824e 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -346,7 +346,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): return cert -def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): +def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert=None): """ Creates a predicate for confirming some Grid Manager-issued certificates against Grid Manager keys. A predicate is used @@ -358,6 +358,9 @@ def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): :param list certs: 1 or more Grid Manager certificates each of which is a `dict` containing 'signature' and 'certificate' keys. + :param str public_key: the identifier of the server we expect + certificates for. + :param callable now_fn: a callable which returns the current UTC timestamp (or datetime.utcnow if None). @@ -416,10 +419,11 @@ def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): # if *any* certificate is still valid then we consider the server valid for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) - if expires > now: - # not-expired - return True + cert_pubkey = ed25519.verifying_key_from_string(cert['public_key'].encode('ascii')) + if cert['public_key'] == public_key: + if expires > now: + # not-expired + return True return False return validate diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index aa8431e05..293683f66 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -285,6 +285,7 @@ class StorageFarmBroker(service.MultiService): gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), + "pub-{}".format(server_id), # server_id is v0- not pub-v0-key .. for reasons? ) s = NativeStorageServer( diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index bcea741d7..5323a6548 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -287,6 +287,7 @@ class GridManagerVerifier(SyncTestCase): verify = create_grid_manager_verifier( [self.gm._public_key], [cert0], + ed25519.string_from_verifying_key(pub0), ) self.assertTrue(verify()) From da0fe23082060ccfc1f151f0a3c705c57c85b116 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 24 Nov 2020 14:08:41 -0700 Subject: [PATCH 174/272] _load_grid_manager_certificates -> method --- src/allmydata/client.py | 30 +----------------- src/allmydata/node.py | 41 +++++++++++++++++++++++++ src/allmydata/test/test_grid_manager.py | 5 +-- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 4570e660d..53047a81b 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -576,34 +576,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con return sb -def _load_grid_manager_certificates(config): - """ - Load all Grid Manager certificates in the config in a list. An - empty list is returned if there are none. - """ - grid_manager_certificates = [] - - cert_fnames = list(config.enumerate_section("grid_manager_certificates").values()) - for fname in cert_fnames: - fname = config.get_config_path(fname.decode('utf8')) - if not os.path.exists(fname): - raise ValueError( - "Grid Manager certificate file '{}' doesn't exist".format( - fname - ) - ) - with open(fname, 'r') as f: - cert = json.load(f) - if set(cert.keys()) != {"certificate", "signature"}: - raise ValueError( - "Unknown key in Grid Manager certificate '{}'".format( - fname - ) - ) - grid_manager_certificates.append(cert) - return grid_manager_certificates - - def _register_reference(key, config, tub, referenceable): """ Register a referenceable in a tub with a stable fURL. @@ -936,7 +908,7 @@ class _Client(node.Node, pollmixin.PollMixin): announcement.update(plugins_announcement) if self.config.get_config("storage", "grid_management", default=False, boolean=True): - grid_manager_certificates = _load_grid_manager_certificates(self.config) + grid_manager_certificates = self.config.get_grid_manager_certificates() announcement[u"grid-manager-certificates"] = grid_manager_certificates # XXX we should probably verify that the certificates are diff --git a/src/allmydata/node.py b/src/allmydata/node.py index a72303e33..b1c5b0eb6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -464,6 +464,12 @@ class _Config(object): """ returns an absolute path inside the 'private' directory with any extra args join()-ed + + This exists for historical reasons. New code should ideally + not call this because it makes it harder for e.g. a SQL-based + _Config object to exist. Code that needs to call this method + should probably be a _Config method itself. See + e.g. get_grid_manager_certificates() """ return os.path.join(self._basedir, "private", *args) @@ -471,6 +477,12 @@ class _Config(object): """ returns an absolute path inside the config directory with any extra args join()-ed + + This exists for historical reasons. New code should ideally + not call this because it makes it harder for e.g. a SQL-based + _Config object to exist. Code that needs to call this method + should probably be a _Config method itself. See + e.g. get_grid_manager_certificates() """ # note: we re-expand here (_basedir already went through this # expanduser function) in case the path we're being asked for @@ -479,6 +491,35 @@ class _Config(object): os.path.join(self._basedir, *args) ) + def get_grid_manager_certificates(self): + """ + Load all Grid Manager certificates in the config. + + :returns: A list of all certificates. An empty list is + returned if there are none. + """ + grid_manager_certificates = [] + + cert_fnames = list(self.enumerate_section("grid_manager_certificates").values()) + for fname in cert_fnames: + fname = self.get_config_path(fname.decode('utf8')) + if not os.path.exists(fname): + raise ValueError( + "Grid Manager certificate file '{}' doesn't exist".format( + fname + ) + ) + with open(fname, 'r') as f: + cert = json.load(f) + if set(cert.keys()) != {"certificate", "signature"}: + raise ValueError( + "Unknown key in Grid Manager certificate '{}'".format( + fname + ) + ) + grid_manager_certificates.append(cert) + return grid_manager_certificates + def create_tub_options(config): """ diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 5323a6548..8747afe08 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -7,9 +7,6 @@ from twisted.python.filepath import ( FilePath, ) -from allmydata.client import ( - _load_grid_manager_certificates, -) from allmydata.node import ( config_from_string, ) @@ -60,7 +57,7 @@ class GridManagerUtilities(SyncTestCase): {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, config.enumerate_section("grid_managers") ) - certs = _load_grid_manager_certificates(config) + certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) From 4450a7a4b23d84519b44292054e00b72b5cfd4ee Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 25 Nov 2020 19:33:32 -0700 Subject: [PATCH 175/272] better words --- src/allmydata/test/test_grid_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 8747afe08..c01266264 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -72,7 +72,11 @@ class GridManagerVerifier(SyncTestCase): def test_sign_cert(self): """ - Add a storage-server and sign a certificate for it + For a storage server previously added to a grid manager, + _GridManager.sign returns a dict with "certificate" and + "signature" properties where the value of "signature" gives + the ed25519 signature (using the grid manager's private key of + the value) of "certificate". """ priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) From 884c26495ffcb050309d6b01e594296b89aa5a8b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 25 Nov 2020 19:50:11 -0700 Subject: [PATCH 176/272] flake8 --- src/allmydata/client.py | 1 - src/allmydata/grid_manager.py | 1 - src/allmydata/node.py | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e2c0d354a..c8e03c16d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -2,7 +2,6 @@ import os import stat import time import weakref -import json from allmydata import node from base64 import urlsafe_b64encode from functools import partial diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index fde88824e..0a5753cc4 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -419,7 +419,6 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= # if *any* certificate is still valid then we consider the server valid for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - cert_pubkey = ed25519.verifying_key_from_string(cert['public_key'].encode('ascii')) if cert['public_key'] == public_key: if expires > now: # not-expired diff --git a/src/allmydata/node.py b/src/allmydata/node.py index a8ee973e2..d9d0a55e5 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -14,6 +14,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str, ensure_text +import json import datetime import os.path import re From 706d3085da42a17a1a5a796cb4614946ce6dfadb Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 13:42:51 -0700 Subject: [PATCH 177/272] create directories on save --- src/allmydata/util/configutil.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index dc7639991..4855241cb 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -66,13 +66,18 @@ def write_config(tahoe_cfg, config): """ Write a configuration to a file. - :param FilePath tahoe_cfg: The path to which to write the config. + :param FilePath tahoe_cfg: The path to which to write the + config. The directories are created if they do not already exist. :param ConfigParser config: The configuration to write. :return: ``None`` """ tmp = tahoe_cfg.temporarySibling() + try: + tahoe_cfg.parent().makedirs() + except OSError: + pass # FilePath.open can only open files in binary mode which does not work # with ConfigParser.write. with open(tmp.path, "wt") as fp: From 38968b40346f54a2f9edd1a92b7e3159cac73fdd Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 13:53:56 -0700 Subject: [PATCH 178/272] correct merge resolution --- src/allmydata/test/common_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index af484ef45..bf85e0580 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -82,7 +82,6 @@ def run_cli_bytes(verb, *args, **kwargs): nodeargs=nodeargs, ) argv = nodeargs + [verb] + list(args) - stdin = kwargs.get("stdin", "") if encoding is None: # The original behavior, the Python 2 behavior, is to accept either # bytes or unicode and try to automatically encode or decode as @@ -91,6 +90,7 @@ def run_cli_bytes(verb, *args, **kwargs): # away from this behavior. stdout = StringIO() stderr = StringIO() + stdin = StringIO(kwargs.get("stdin", "")) else: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python @@ -99,6 +99,7 @@ def run_cli_bytes(verb, *args, **kwargs): # encodings to exercise different behaviors. stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) + stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback( From 4884f81235c16185bbb34c7abac185489bcb5d40 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 15:49:42 -0700 Subject: [PATCH 179/272] brackets are important --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index bf85e0580..dcf81b267 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -99,7 +99,7 @@ def run_cli_bytes(verb, *args, **kwargs): # encodings to exercise different behaviors. stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) - stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding) + stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding)) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback( From 4fe65a8efde8be1a807ca92f8efc2e33f40029a9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 15:57:03 -0700 Subject: [PATCH 180/272] brackets are hard --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index dcf81b267..23bf15d9b 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -99,7 +99,7 @@ def run_cli_bytes(verb, *args, **kwargs): # encodings to exercise different behaviors. stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) - stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding)) + stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b"")), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback( From bd46ff2f67c2a3df98cc15346c669beb459c1657 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 15 Dec 2020 02:03:10 -0700 Subject: [PATCH 181/272] windows-only checks --- src/allmydata/util/configutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 4855241cb..b84d96319 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -85,7 +85,10 @@ def write_config(tahoe_cfg, config): # Windows doesn't have atomic overwrite semantics for moveTo. Thus we end # up slightly less than atomic. if platform.isWindows(): - tahoe_cfg.remove() + try: + tahoe_cfg.remove() + except OSError: + pass tmp.moveTo(tahoe_cfg) def validate_config(fname, cfg, valid_config): From fbddff39ea039b46f4c878a90a7e39becf21fdea Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:02:56 -0700 Subject: [PATCH 182/272] link grid-manager into the ToC --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 3d0a41302..e778319a4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,6 +27,7 @@ Contents: CODE_OF_CONDUCT servers + managed-grid helper convergence-secret garbage-collection From e38a1c3ee18fd3dd25431d82448fa05ccca7666c Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:03:24 -0700 Subject: [PATCH 183/272] two colons --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 11c5035e8..cc427d2c5 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -112,7 +112,7 @@ identifier for the new storage node (also sometimes called "a petname" or "nickname"). The pubkey is the tahoe-encoded key from a ``node.pubkey`` file in the storage-server's node directory (minus any whitespace). For example, if ``~/storage0`` contains a storage-node, -you might do something like this: +you might do something like this:: grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) From f4164aa1c9aa384094d0dfc60fbc0ac7205c0b3d Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:05:40 -0700 Subject: [PATCH 184/272] missed a new arg --- docs/managed-grid.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index cc427d2c5..1613260eb 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -133,8 +133,9 @@ Lists all storage-servers that have previously been added using grid-manager sign ````````````````` -Takes one arg: ``name``, the nickname used previously in a -``grid-manager add`` command. +Takes two args: ``name expiry_days``. The ``name`` is a nickname used +previously in a ``grid-manager add`` command and ``expiry_days`` is +the number of days in the future when the certificate should expire. Note that this mutates the state of the grid-manager if it is on disk, by adding this certificate to our collection of issued From 07180b295bb984822afec7098c88f13f8b3e3e93 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:12:11 -0700 Subject: [PATCH 185/272] spelling --- src/allmydata/mutable/publish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index ef93be721..1f2c04018 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -971,7 +971,7 @@ class Publish(object): if server in self.bad_servers: Message.log( message_type=u"mutable:upload:bad-server", - server_id=server.get_server_id(), + server_id=serverid, ) continue # if we have >= 1 grid-managers, this checks that we have @@ -979,7 +979,7 @@ class Publish(object): if not server.upload_permitted(): Message.log( message_type=u"mutable:upload:no-gm-certs", - server_id=server.get_server_id(), + server_id=serverid, ) continue From d267bb3ee1f87627a35719eaf0d860c147f6e1e4 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:15:23 -0700 Subject: [PATCH 186/272] leftover comment --- src/allmydata/storage_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6c6ea5eb4..04b87141d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -641,10 +641,6 @@ class _FoolscapStorage(object): short_description = server_id[:8] nickname = ann.get("nickname", "") - ## XXX FIXME post-merge, need to parse out - ## "grid-manager-certificates" and .. do something with them - ## (like store them on this object) - return cls( nickname=nickname, permutation_seed=permutation_seed, From 24ac2c69942efffe3202d9fe83fd5b132a64ffc6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:19:42 -0700 Subject: [PATCH 187/272] I guess only OSError we care about? --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 8e561d36a..70f1a7bed 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -196,7 +196,7 @@ def sign(ctx, name, expiry_days): while f is None: try: f = fp.child("{}.cert.{}".format(name, next_serial)).create() - except Exception: + except OSError: f = None next_serial += 1 with f: From 361acc206d62b608509aadae14ab35011be298ce Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:21:00 -0700 Subject: [PATCH 188/272] irrelevant comment --- src/allmydata/client.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b64feae46..22f5bd589 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -517,14 +517,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con **kwargs ) - # we don't actually use this keypair for anything (yet) as far - # as I can see. - # my_pubkey = keyutil.parse_pubkey( - # self.get_config_from_file("node.pubkey") - # ) - # create the actual storage-broker - sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, From bfd45d3a017a3efd7ceec51aa0ea33ea2285092a Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:21:31 -0700 Subject: [PATCH 189/272] typo --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 1613260eb..3630b956b 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -202,7 +202,7 @@ config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``grid-manager sign`` command; these should be transmitted to the storage server operator -who includes them in the config for the storage sever. Relative paths +who includes them in the config for the storage server. Relative paths are based from the node directory. Example:: [storage] From 5d4253be3e91ade95d5773b4a51c19e076de489a Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 23:03:06 -0700 Subject: [PATCH 190/272] reject a bunch of invalid signatures --- src/allmydata/test/strategies.py | 15 ++++++++++ src/allmydata/test/test_grid_manager.py | 37 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index 553b2c226..d8b758dab 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -15,6 +15,11 @@ from ..uri import ( MDMFDirectoryURI, ) +from allmydata.util.base32 import ( + b2a, +) + + def write_capabilities(): """ Build ``IURI`` providers representing all kinds of write capabilities. @@ -109,3 +114,13 @@ def dir2_mdmf_capabilities(): MDMFDirectoryURI, mdmf_capabilities(), ) + + +def base32text(): + """ + Build text()s that are valid base32 + """ + return builds( + b2a, + binary(), + ) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c01266264..734b54e4c 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -7,6 +7,8 @@ from twisted.python.filepath import ( FilePath, ) +from hypothesis import given + from allmydata.node import ( config_from_string, ) @@ -26,6 +28,9 @@ from allmydata.grid_manager import ( parse_grid_manager_certificate, create_grid_manager_verifier, ) +from allmydata.test.strategies import ( + base32text, +) from .common import SyncTestCase @@ -292,3 +297,35 @@ class GridManagerVerifier(SyncTestCase): ) self.assertTrue(verify()) + + +class GridManagerVerifier(SyncTestCase): + """ + Invalid certificate rejection tests + """ + + def setUp(self): + self.gm = create_grid_manager() + self.priv0, self.pub0 = ed25519.create_signing_keypair() + self.gm.add_storage_server("test0", self.pub0) + self.cert0 = self.gm.sign("test0", timedelta(seconds=86400)) + return super(GridManagerVerifier, self).setUp() + + @given( + base32text(), + ) + def test_validate_cert_invalid(self, invalid_signature): + """ + An incorrect signature is rejected + """ + # make signature invalid + self.cert0["signature"] = invalid_signature + + verify = create_grid_manager_verifier( + [self.gm._public_key], + [self.cert0], + ed25519.string_from_verifying_key(self.pub0), + bad_cert = lambda key, cert: None, + ) + + self.assertFalse(verify()) From 5eff6f9ddd109bca00ec56b5b5cffcb56206855e Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 23:15:14 -0700 Subject: [PATCH 191/272] add a few docstrings for accessors etc --- src/allmydata/grid_manager.py | 19 ++++++++++++++++++- src/allmydata/scripts/admin.py | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 0a5753cc4..120701255 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -29,12 +29,21 @@ class _GridManagerStorageServer(object): ) def add_certificate(self, certificate): + """ + Add ``certificate`` + """ self.certificates.append(certificate) def public_key_string(self): + """ + :returns: the public key as a string + """ return ed25519.string_from_verifying_key(self.public_key) def marshal(self): + """ + :returns: a dict suitable for JSON representing this object + """ return { u"public_key": self.public_key_string(), } @@ -181,6 +190,9 @@ class _GridManager(object): return self._storage_servers def public_identity(self): + """ + :returns: public key as a string + """ return ed25519.string_from_verifying_key(self._public_key) def sign(self, name, expiry): @@ -248,6 +260,9 @@ class _GridManager(object): ) def marshal(self): + """ + :returns: a dict suitable for JSON representing this object + """ data = { u"grid_manager_config_version": self._version, u"private_key": self._private_key_bytes.decode('ascii'), @@ -415,8 +430,10 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= bad_cert(key, alleged_cert) def validate(): + """ + :returns: True if if *any* certificate is still valid for a server + """ now = now_fn() - # if *any* certificate is still valid then we consider the server valid for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) if cert['public_key'] == public_key: diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 28ab0f2ed..8710641ec 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -59,6 +59,9 @@ def derive_pubkey(options): class AddGridManagerCertOptions(BaseOptions): + """ + Options for add-grid-manager-cert + """ optParameters = [ ['filename', 'f', None, "Filename of the certificate ('-', a dash, for stdin)"], From d31d8e19c5fde5a96eb44fa7f3053b79053b0997 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 23:17:21 -0700 Subject: [PATCH 192/272] module docstring --- src/allmydata/grid_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 120701255..39f258fd1 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,3 +1,6 @@ +""" +Functions and classes relating to the Grid Manager internal state +""" import sys import json From 80c68a41fd39b3620da8cdf35280b58ae7a280da Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 21 Dec 2020 00:49:56 -0700 Subject: [PATCH 193/272] cover more error-cases --- src/allmydata/test/cli/test_grid_manager.py | 56 ++++++++++++++++++ src/allmydata/test/test_grid_manager.py | 65 ++++++++++++++++++++- 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index a8b0315e0..a6f1f7ec2 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -44,6 +44,20 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) self.assertTrue(result.output.startswith("pub-v0-")) + def test_load_invalid(self): + """ + An invalid config is reported to the user + """ + with self.runner.isolated_filesystem(): + with open("config.json", "w") as f: + json.dump({"not": "valid"}, f) + result = self.runner.invoke(grid_manager, ["--config", ".", "public-identity"]) + self.assertNotEqual(result.exit_code, 0) + self.assertIn( + "Error loading Grid Manager", + result.output, + ) + def test_create_already(self): """ It's an error to create a new grid-manager in an existing @@ -85,6 +99,22 @@ class GridManagerCommandLine(SyncTestCase): cert = json.loads(sigcert['certificate']) self.assertEqual(cert["public_key"], pubkey) + def test_add_twice(self): + """ + An error is reported trying to add an existing server + """ + pubkey0 = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + pubkey1 = "pub-v0-5ysc55trfvfvg466v46j4zmfyltgus3y2gdejifctv7h4zkuyveq" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey0]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey1]) + self.assertNotEquals(result.exit_code, 0) + self.assertIn( + "A storage-server called 'storage0' already exists", + result.output, + ) + def test_add_list_remove(self): """ Add a storage server, list it, remove it. @@ -108,6 +138,32 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) self.assertEqual(result.output.strip(), "") + def test_remove_missing(self): + """ + Error reported when removing non-existant server + """ + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) + self.assertNotEquals(result.exit_code, 0) + self.assertIn( + "No storage-server called 'storage0' exists", + result.output, + ) + + def test_sign_missing(self): + """ + Error reported when signing non-existant server + """ + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) + self.assertNotEquals(result.exit_code, 0) + self.assertIn( + "No storage-server called 'storage0' exists", + result.output, + ) + class TahoeAddGridManagerCert(AsyncTestCase): """ diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 734b54e4c..4c899257d 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -65,6 +65,32 @@ class GridManagerUtilities(SyncTestCase): certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) + def test_load_certificates_invalid_version(self): + """ + An error is reported loading invalid certificate version + """ + cert_path = self.mktemp() + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + } + with open(cert_path, "w") as f: + f.write(json.dumps(fake_cert)) + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + self.assertEqual( + {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, + config.enumerate_section("grid_managers") + ) + certs = config.get_grid_manager_certificates() + self.assertEqual([fake_cert], certs) + print(certs) + class GridManagerVerifier(SyncTestCase): """ @@ -182,6 +208,41 @@ class GridManagerVerifier(SyncTestCase): str(ctx.exception), ) + def test_invalid_certificate_bad_version(self): + """ + Invalid Grid Manager config containing a certificate with an + illegal version + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + config = { + "grid_manager_config_version": 0, + "private_key": "priv-v0-ub7knkkmkptqbsax4tznymwzc4nk5lynskwjsiubmnhcpd7lvlqa", + "storage_servers": { + "alice": { + "public_key": "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + } + } + } + bad_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":0}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + } + + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(config, f) + with fp.child("alice.cert.0").open("w") as f: + json.dump(bad_cert, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp) + + self.assertIn( + "Unknown certificate version", + str(ctx.exception), + ) + def test_invalid_no_private_key(self): """ Invalid Grid Manager config with no private key @@ -299,7 +360,7 @@ class GridManagerVerifier(SyncTestCase): self.assertTrue(verify()) -class GridManagerVerifier(SyncTestCase): +class GridManagerInvalidVerifier(SyncTestCase): """ Invalid certificate rejection tests """ @@ -309,7 +370,7 @@ class GridManagerVerifier(SyncTestCase): self.priv0, self.pub0 = ed25519.create_signing_keypair() self.gm.add_storage_server("test0", self.pub0) self.cert0 = self.gm.sign("test0", timedelta(seconds=86400)) - return super(GridManagerVerifier, self).setUp() + return super(GridManagerInvalidVerifier, self).setUp() @given( base32text(), From 30e3cd23da641a6f3a9120e80c80154840aff44d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 15:39:08 -0700 Subject: [PATCH 194/272] un-writable directory test --- src/allmydata/cli/grid_manager.py | 12 +++++++++--- src/allmydata/test/cli/test_grid_manager.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 70f1a7bed..acf233b29 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -194,10 +194,16 @@ def sign(ctx, name, expiry_days): next_serial = 0 f = None while f is None: + fname = "{}.cert.{}".format(name, next_serial) try: - f = fp.child("{}.cert.{}".format(name, next_serial)).create() - except OSError: - f = None + f = fp.child(fname).create() + except OSError as e: + if e.errno == 17: # file exists + f = None + else: + raise click.ClickException( + "{}: {}".format(fname, e) + ) next_serial += 1 with f: f.write(certificate_data) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index a6f1f7ec2..8c8ab8140 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -164,6 +164,23 @@ class GridManagerCommandLine(SyncTestCase): result.output, ) + def test_sign_bad_perms(self): + """ + Error reported if we can't create certificate file + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + # make the directory un-writable (so we can't create a new cert) + os.chmod("foo", 0o550) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) + self.assertEquals(result.exit_code, 1) + self.assertIn( + "Permission denied", + result.output, + ) + class TahoeAddGridManagerCert(AsyncTestCase): """ From 66de141b8a8a7e0b667048cab672acf5b02c0187 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 16:10:20 -0700 Subject: [PATCH 195/272] test: list certs from stdin --- src/allmydata/test/cli/test_grid_manager.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 8c8ab8140..4bb82a433 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,6 +1,9 @@ import os import json +from io import ( + BytesIO, +) from ..common import ( SyncTestCase, @@ -85,6 +88,29 @@ class GridManagerCommandLine(SyncTestCase): set(config.keys()), ) + def test_list_stdout(self): + """ + Load Grid Manager without files (using 'list' subcommand, but any will do) + """ + config = { + "storage_servers": { + "storage0": { + "public_key": "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + } + }, + "private_key": "priv-v0-6uinzyaxy3zvscwgsps5pxcfezhrkfb43kvnrbrhhfzyduyqnniq", + "grid_manager_config_version": 0 + } + result = self.runner.invoke( + grid_manager, ["--config", "-", "list"], + input=BytesIO(json.dumps(config)), + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual( + "storage0: pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\n", + result.output, + ) + def test_add_and_sign(self): """ Add a new storage-server and sign a certificate for it From 6c9632b5ece73727a64fe74baa200c77ee1a3713 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 16:10:32 -0700 Subject: [PATCH 196/272] test: missing cert from config is reported --- src/allmydata/test/test_grid_manager.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 4c899257d..c3b85bbee 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -89,7 +89,25 @@ class GridManagerUtilities(SyncTestCase): ) certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) - print(certs) + + def test_load_certificates_missing(self): + """ + An error is reported for missing certificates + """ + cert_path = self.mktemp() + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + with self.assertRaises(ValueError) as ctx: + certs = config.get_grid_manager_certificates() + self.assertIn( + "Grid Manager certificate file '{}' doesn't exist".format(cert_path), + str(ctx.exception) + ) class GridManagerVerifier(SyncTestCase): From dbf385bd796997ab0e649cbb100cd25a78f7f4ec Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 19:40:23 -0700 Subject: [PATCH 197/272] test: missing config section --- src/allmydata/test/test_node.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..74ea6e5eb 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -245,6 +245,20 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): with self.assertRaises(MissingConfigEntry): config.get_config("node", "log_gatherer.furl") + def test_missing_config_section(self): + """ + Enumerating a missing section returns empty dict + """ + basedir = self.mktemp() + fileutil.make_dirs(basedir) + with open(os.path.join(basedir, 'tahoe.cfg'), 'w'): + pass + config = read_config(basedir, "") + self.assertEquals( + config.enumerate_section("not-a-section"), + {} + ) + def test_config_required(self): """ Asking for missing (but required) configuration is an error From 22239022c7ad4e3f8537794971a2d5f468c8e79d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 21:49:15 -0700 Subject: [PATCH 198/272] test: invalid cert key --- src/allmydata/grid_manager.py | 2 +- src/allmydata/test/test_grid_manager.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 39f258fd1..c288a34bb 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -332,7 +332,7 @@ def parse_grid_manager_certificate(gm_data): if set(js.keys()) != required_keys: raise ValueError( "Grid Manager certificate must contain: {}".format( - ", ".join("'{}'".format(k) for k in js.keys()), + ", ".join("'{}'".format(k) for k in required_keys), ) ) return js diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c3b85bbee..c28afd5fd 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -90,6 +90,31 @@ class GridManagerUtilities(SyncTestCase): certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) + def test_load_certificates_unknown_key(self): + """ + An error is reported loading certificates with invalid keys in them + """ + cert_path = self.mktemp() + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", + "something-else": "not valid in a v0 certificate" + } + with open(cert_path, "w") as f: + f.write(json.dumps(fake_cert)) + config_data = ( + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + with self.assertRaises(ValueError) as ctx: + certs = config.get_grid_manager_certificates() + + self.assertIn( + "Unknown key in Grid Manager certificate", + str(ctx.exception) + ) + def test_load_certificates_missing(self): """ An error is reported for missing certificates From 7783f31c1b0580211f7c05f8b8a08c40c4e4c976 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 21:49:40 -0700 Subject: [PATCH 199/272] tests for 'tahoe admin add-grid-manager-cert' --- src/allmydata/scripts/admin.py | 4 +- src/allmydata/test/cli/test_admin.py | 178 ++++++++++++++++++++++++ src/allmydata/test/test_grid_manager.py | 10 +- 3 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 src/allmydata/test/cli/test_admin.py diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 8710641ec..a0014a24e 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -115,8 +115,6 @@ def add_grid_manager_cert(options): """ Add a new Grid Manager certificate to our config """ - if options.certificate_data is None: - return 1 # XXX is there really not already a function for this? if options.parent.parent['node-directory']: nd = argv_to_abspath(options.parent.parent['node-directory']) @@ -132,7 +130,7 @@ def add_grid_manager_cert(options): if cert_path.exists(): msg = "Already have certificate for '{}' (at {})".format( options['name'], - cert_path, + cert_path.path, ) print(msg, file=options.parent.parent.stderr) return 1 diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py new file mode 100644 index 000000000..0d825945c --- /dev/null +++ b/src/allmydata/test/cli/test_admin.py @@ -0,0 +1,178 @@ +import json +from io import ( + BytesIO, + StringIO, +) + +from twisted.python.usage import ( + UsageError, +) +from twisted.python.filepath import ( + FilePath, +) + +from allmydata.scripts.admin import ( + AdminCommand, + AddGridManagerCertOptions, + add_grid_manager_cert, +) +from allmydata.scripts.runner import ( + Options, +) +from ..common import ( + SyncTestCase, +) + + +fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" +} + + +class AddCertificateOptions(SyncTestCase): + """ + Tests for 'tahoe admin add-grid-manager-cert' option validation + """ + + def setUp(self): + self.tahoe = Options() + return super(AddCertificateOptions, self).setUp() + + def test_parse_no_data(self): + """ + When no data is passed to stdin an error is produced + """ + self.tahoe.stdin = BytesIO(b"") + self.tahoe.stderr = BytesIO() # suppress message + + with self.assertRaises(UsageError) as ctx: + self.tahoe.parseOptions( + [ + "admin", "add-grid-manager-cert", + "--name", "random-name", + "--filename", "-", + ] + ) + + self.assertIn( + "Reading certificate from stdin failed", + str(ctx.exception) + ) + + def test_read_cert_file(self): + """ + A certificate can be read from a file + """ + tmp = self.mktemp() + with open(tmp, "w") as f: + json.dump(fake_cert, f) + + # certificate should be loaded + o = self.tahoe.parseOptions( + [ + "admin", "add-grid-manager-cert", + "--name", "random-name", + "--filename", tmp, + ] + ) + opts = self.tahoe.subOptions.subOptions + self.assertEqual( + fake_cert, + opts.certificate_data + ) + + def test_bad_certificate(self): + """ + Unparseable data produces an error + """ + self.tahoe.stdin = BytesIO(b"{}") + self.tahoe.stderr = BytesIO() # suppress message + + with self.assertRaises(UsageError) as ctx: + self.tahoe.parseOptions( + [ + "admin", "add-grid-manager-cert", + "--name", "random-name", + "--filename", "-", + ] + ) + + self.assertIn( + "Grid Manager certificate must contain", + str(ctx.exception) + ) + + +class AddCertificateCommand(SyncTestCase): + """ + Tests for 'tahoe admin add-grid-manager-cert' operation + """ + + def setUp(self): + self.tahoe = Options() + self.node_path = FilePath(self.mktemp()) + self.node_path.makedirs() + with self.node_path.child("tahoe.cfg").open("w") as f: + f.write("# minimal test config\n") + return super(AddCertificateCommand, self).setUp() + + def test_add_one(self): + """ + Adding a certificate succeeds + """ + self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.stderr = BytesIO() + self.tahoe.parseOptions( + [ + "--node-directory", self.node_path.path, + "admin", "add-grid-manager-cert", + "--name", "zero", + "--filename", "-", + ] + ) + rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) + + self.assertEqual(rc, 0) + self.assertEqual( + ["zero.cert", "tahoe.cfg"], + self.node_path.listdir() + ) + self.assertIn( + "There are now 1 certificates", + self.tahoe.stderr.getvalue() + ) + + def test_add_two(self): + """ + An error message is produced when adding a certificate with a + duplicate name. + """ + self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.stderr = BytesIO() + self.tahoe.parseOptions( + [ + "--node-directory", self.node_path.path, + "admin", "add-grid-manager-cert", + "--name", "zero", + "--filename", "-", + ] + ) + rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) + self.assertEqual(rc, 0) + + self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.parseOptions( + [ + "--node-directory", self.node_path.path, + "admin", "add-grid-manager-cert", + "--name", "zero", + "--filename", "-", + ] + ) + rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) + self.assertEqual(rc, 1) + self.assertIn( + "Already have certificate for 'zero'", + self.tahoe.stderr.getvalue() + ) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c28afd5fd..01cb50ccf 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -154,18 +154,20 @@ class GridManagerVerifier(SyncTestCase): """ priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) - cert = self.gm.sign("test", timedelta(seconds=86400)) + cert0 = self.gm.sign("test", timedelta(seconds=86400)) + cert1 = self.gm.sign("test", timedelta(seconds=86400)) + self.assertNotEqual(cert0, cert1) self.assertEqual( - set(cert.keys()), + set(cert0.keys()), {"certificate", "signature"}, ) gm_key = ed25519.verifying_key_from_string(self.gm.public_identity()) self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert["signature"]), - cert["certificate"], + base32.a2b(cert0["signature"]), + cert0["certificate"], ), None ) From c0f0076a3ea40be1619ae8b57f59637b763bbffc Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 Jan 2021 00:00:13 -0700 Subject: [PATCH 200/272] undo --- src/allmydata/scripts/runner.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 30b29f68e..f57397f74 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -162,13 +162,7 @@ def dispatch(config, # 4: return a Deferred that does 1 or 2 or 3 def _raise_sys_exit(rc): sys.exit(rc) - - def _error(f): - f.trap(usage.UsageError) - print("Error: {}".format(f.value.message)) - sys.exit(1) d.addCallback(_raise_sys_exit) - d.addErrback(_error) return d def _maybe_enable_eliot_logging(options, reactor): From 7d30bd53c7445c1bbb4d3709649b2ffe9796f4c4 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 Jan 2021 01:14:29 -0700 Subject: [PATCH 201/272] test: announcements contain grid-manager certs --- src/allmydata/test/test_client.py | 43 ++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 342fe4af1..2c4ccca25 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,4 +1,6 @@ -import os, sys +import os +import sys +import json from functools import ( partial, ) @@ -35,6 +37,7 @@ from testtools.matchers import ( AfterPreprocessing, MatchesListwise, MatchesDict, + ContainsDict, Always, Is, raises, @@ -1543,3 +1546,41 @@ enabled = {storage_enabled} ), ), ) + + def test_announcement_includes_grid_manager(self): + """ + When Grid Manager is enabled certificates are included in the + announcement + """ + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", + } + with self.basedir.child("zero.cert").open("w") as f: + json.dump(fake_cert, f) + + config = client.config_from_string( + self.basedir.path, + "tub.port", + self.get_config( + storage_enabled=True, + more_storage="grid_management = True", + more_sections=( + "[grid_manager_certificates]\n" + "foo = zero.cert\n" + ) + ), + ) + + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + succeeded(AfterPreprocessing( + lambda client: get_published_announcements(client)[0].ann, + ContainsDict({ + "grid-manager-certificates": Equals([fake_cert]), + }), + )), + ) From 2dc6c5f2e808b2208767f9f07e38ccf56912e3da Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Jan 2021 10:11:36 -0700 Subject: [PATCH 202/272] add a gm cert in the client test --- src/allmydata/test/test_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 2c4ccca25..8ce63dc8a 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1558,6 +1558,8 @@ enabled = {storage_enabled} } with self.basedir.child("zero.cert").open("w") as f: json.dump(fake_cert, f) + with self.basedir.child("gm0.cert").open("w") as f: + json.dump(fake_cert, f) config = client.config_from_string( self.basedir.path, @@ -1566,6 +1568,8 @@ enabled = {storage_enabled} storage_enabled=True, more_storage="grid_management = True", more_sections=( + "[grid_managers]\n" + "gm0 = pub-v0-ibpbsexcjfbv3ni7gwlclgn6mldaqnqd5mrtan2fnq2b27xnovca\n" "[grid_manager_certificates]\n" "foo = zero.cert\n" ) From 6560621e959ff269788e2fbcf6027dca1dc52e99 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Jan 2021 00:08:00 -0700 Subject: [PATCH 203/272] cover 'second certificate' codepath --- src/allmydata/test/cli/test_grid_manager.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 4bb82a433..6150c4710 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -125,6 +125,22 @@ class GridManagerCommandLine(SyncTestCase): cert = json.loads(sigcert['certificate']) self.assertEqual(cert["public_key"], pubkey) + def test_add_and_sign_second_cert(self): + """ + Add a new storage-server and sign two certificates. + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + # we should now have two certificates stored + self.assertEqual( + set(FilePath("foo").listdir()), + {'storage0.cert.1', 'storage0.cert.0', 'config.json'}, + ) + def test_add_twice(self): """ An error is reported trying to add an existing server From a2f0f6581ed0515360f4a65b8fbcb6554dcd6cd3 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Jan 2021 18:36:18 -0700 Subject: [PATCH 204/272] order not important --- src/allmydata/test/cli/test_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 0d825945c..6521272fa 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -135,8 +135,8 @@ class AddCertificateCommand(SyncTestCase): self.assertEqual(rc, 0) self.assertEqual( - ["zero.cert", "tahoe.cfg"], - self.node_path.listdir() + {"zero.cert", "tahoe.cfg"}, + set(self.node_path.listdir()) ) self.assertIn( "There are now 1 certificates", From 66d4118e5468f5a52714f522377bc3dcfff32145 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Jan 2021 18:42:02 -0700 Subject: [PATCH 205/272] I know know how permissions work on linux --- src/allmydata/test/cli/test_grid_manager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 6150c4710..80a39d540 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -4,6 +4,9 @@ import json from io import ( BytesIO, ) +from unittest import ( + skipIf, +) from ..common import ( SyncTestCase, @@ -25,6 +28,9 @@ from twisted.internet.defer import ( from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import ( + platform, +) class GridManagerCommandLine(SyncTestCase): @@ -206,6 +212,7 @@ class GridManagerCommandLine(SyncTestCase): result.output, ) + @skipIf(not platform.isLinux(), "I only know how permissions work on linux") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file From 75db6f3e622be1cc519c1a41a9093f60597c811a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 22 Jan 2021 16:01:29 -0700 Subject: [PATCH 206/272] relax test --- src/allmydata/test/test_grid_manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 01cb50ccf..56541aa16 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -129,8 +129,15 @@ class GridManagerUtilities(SyncTestCase): config = config_from_string("/foo", "portnum", config_data, client_valid_config()) with self.assertRaises(ValueError) as ctx: certs = config.get_grid_manager_certificates() + # we don't reliably know how Windows or MacOS will represent + # the path in the exception, so we don't check for the *exact* + # message with full-path here.. self.assertIn( - "Grid Manager certificate file '{}' doesn't exist".format(cert_path), + "Grid Manager certificate file", + str(ctx.exception) + ) + self.assertIn( + " doesn't exist", str(ctx.exception) ) From 7aaf86ac946791ee8b4bd39d59a56671110bbe1e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Jan 2021 15:23:47 -0700 Subject: [PATCH 207/272] flake8 cleanup --- src/allmydata/test/cli/test_admin.py | 5 +---- src/allmydata/test/cli/test_grid_manager.py | 4 ++-- src/allmydata/test/test_grid_manager.py | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 6521272fa..37f733f07 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,7 +1,6 @@ import json from io import ( BytesIO, - StringIO, ) from twisted.python.usage import ( @@ -12,8 +11,6 @@ from twisted.python.filepath import ( ) from allmydata.scripts.admin import ( - AdminCommand, - AddGridManagerCertOptions, add_grid_manager_cert, ) from allmydata.scripts.runner import ( @@ -69,7 +66,7 @@ class AddCertificateOptions(SyncTestCase): json.dump(fake_cert, f) # certificate should be loaded - o = self.tahoe.parseOptions( + self.tahoe.parseOptions( [ "admin", "add-grid-manager-cert", "--name", "random-name", diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 80a39d540..397824876 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -139,8 +139,8 @@ class GridManagerCommandLine(SyncTestCase): with self.runner.isolated_filesystem(): self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) # we should now have two certificates stored self.assertEqual( set(FilePath("foo").listdir()), diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 56541aa16..222631111 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -108,7 +108,7 @@ class GridManagerUtilities(SyncTestCase): ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) with self.assertRaises(ValueError) as ctx: - certs = config.get_grid_manager_certificates() + config.get_grid_manager_certificates() self.assertIn( "Unknown key in Grid Manager certificate", @@ -128,7 +128,7 @@ class GridManagerUtilities(SyncTestCase): ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) with self.assertRaises(ValueError) as ctx: - certs = config.get_grid_manager_certificates() + config.get_grid_manager_certificates() # we don't reliably know how Windows or MacOS will represent # the path in the exception, so we don't check for the *exact* # message with full-path here.. From 87c78fd8f341b4f2b8c3beb85794850e11dbb572 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 26 Jan 2021 13:32:23 -0700 Subject: [PATCH 208/272] use plain TestCase (fix eliot tests, maybe) --- src/allmydata/test/cli/test_grid_manager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 397824876..e63e6aaaa 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -8,9 +8,8 @@ from unittest import ( skipIf, ) -from ..common import ( - SyncTestCase, - AsyncTestCase, +from twisted.trial.unittest import ( + TestCase, ) from allmydata.cli.grid_manager import ( grid_manager, @@ -33,7 +32,7 @@ from twisted.python.runtime import ( ) -class GridManagerCommandLine(SyncTestCase): +class GridManagerCommandLine(TestCase): """ Test the mechanics of the `grid-manager` command """ @@ -231,7 +230,7 @@ class GridManagerCommandLine(SyncTestCase): ) -class TahoeAddGridManagerCert(AsyncTestCase): +class TahoeAddGridManagerCert(TestCase): """ Test `tahoe admin add-grid-manager-cert` subcommand """ From 8bb83e797eeac3659a133116ccc159dde0250ad6 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 26 Jan 2021 15:13:57 -0700 Subject: [PATCH 209/272] actually not-equal --- src/allmydata/test/test_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 222631111..e53984a05 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -162,7 +162,7 @@ class GridManagerVerifier(SyncTestCase): priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) cert0 = self.gm.sign("test", timedelta(seconds=86400)) - cert1 = self.gm.sign("test", timedelta(seconds=86400)) + cert1 = self.gm.sign("test", timedelta(seconds=3600)) self.assertNotEqual(cert0, cert1) self.assertEqual( From 7fa180176e5b656a789e83095875da5d481a26d2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:18:03 -0400 Subject: [PATCH 210/272] Pass tests on Python 2. --- src/allmydata/scripts/admin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 86209637e..178701f84 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -10,7 +10,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_binary +from six import ensure_binary, ensure_str import json try: @@ -100,7 +100,7 @@ class AddGridManagerCertOptions(BaseOptions): "Must provide --filename option" ) if self['filename'] == '-': - print("reading certificate from stdin", file=self.parent.parent.stderr) + print(ensure_str("reading certificate from stdin"), file=self.parent.parent.stderr) data = self.parent.parent.stdin.read() if len(data) == 0: raise usage.UsageError( @@ -151,7 +151,7 @@ def add_grid_manager_cert(options): options['name'], cert_path.path, ) - print(msg, file=options.parent.parent.stderr) + print(ensure_str(msg), file=options.parent.parent.stderr) return 1 config.set_config("storage", "grid_management", "True") @@ -162,7 +162,8 @@ def add_grid_manager_cert(options): f.write(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) - print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) + print(ensure_str("There are now {} certificates").format(cert_count), + file=options.parent.parent.stderr) return 0 From e23767db1b6921038ae8e46e60e954c0dcab23a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:39:41 -0400 Subject: [PATCH 211/272] Tests pass on Python 2 and Python 3. --- src/allmydata/scripts/admin.py | 5 ++-- src/allmydata/test/cli/test_admin.py | 36 +++++++++++++++------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 178701f84..7bce914cc 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -11,7 +11,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_binary, ensure_str -import json try: from allmydata.scripts.types_ import SubCommands @@ -28,7 +27,7 @@ from allmydata.grid_manager import ( from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath - +from allmydata.util import jsonbytes class GenerateKeypairOptions(BaseOptions): @@ -143,7 +142,7 @@ def add_grid_manager_cert(options): config = read_config(nd, "portnum") cert_fname = "{}.cert".format(options['name']) cert_path = FilePath(config.get_config_path(cert_fname)) - cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' + cert_bytes = jsonbytes.dumps_bytes(options.certificate_data, indent=4) + b'\n' cert_name = options['name'] if cert_path.exists(): diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 37f733f07..1ac047acb 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,7 +1,11 @@ -import json -from io import ( - BytesIO, -) +from future.utils import PY3 +from six import ensure_str + +# We're going to override stdin/stderr, so want to match their behavior on respective Python versions. +if PY3: + from io import StringIO +else: + from StringIO import StringIO from twisted.python.usage import ( UsageError, @@ -16,6 +20,7 @@ from allmydata.scripts.admin import ( from allmydata.scripts.runner import ( Options, ) +from allmydata.util import jsonbytes as json from ..common import ( SyncTestCase, ) @@ -31,7 +36,6 @@ class AddCertificateOptions(SyncTestCase): """ Tests for 'tahoe admin add-grid-manager-cert' option validation """ - def setUp(self): self.tahoe = Options() return super(AddCertificateOptions, self).setUp() @@ -40,8 +44,8 @@ class AddCertificateOptions(SyncTestCase): """ When no data is passed to stdin an error is produced """ - self.tahoe.stdin = BytesIO(b"") - self.tahoe.stderr = BytesIO() # suppress message + self.tahoe.stdin = StringIO(ensure_str("")) + self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: self.tahoe.parseOptions( @@ -63,7 +67,7 @@ class AddCertificateOptions(SyncTestCase): """ tmp = self.mktemp() with open(tmp, "w") as f: - json.dump(fake_cert, f) + f.write(json.dumps(fake_cert)) # certificate should be loaded self.tahoe.parseOptions( @@ -83,8 +87,8 @@ class AddCertificateOptions(SyncTestCase): """ Unparseable data produces an error """ - self.tahoe.stdin = BytesIO(b"{}") - self.tahoe.stderr = BytesIO() # suppress message + self.tahoe.stdin = StringIO(ensure_str("{}")) + self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: self.tahoe.parseOptions( @@ -111,15 +115,15 @@ class AddCertificateCommand(SyncTestCase): self.node_path = FilePath(self.mktemp()) self.node_path.makedirs() with self.node_path.child("tahoe.cfg").open("w") as f: - f.write("# minimal test config\n") + f.write(b"# minimal test config\n") return super(AddCertificateCommand, self).setUp() def test_add_one(self): """ Adding a certificate succeeds """ - self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) - self.tahoe.stderr = BytesIO() + self.tahoe.stdin = StringIO(json.dumps(fake_cert)) + self.tahoe.stderr = StringIO() self.tahoe.parseOptions( [ "--node-directory", self.node_path.path, @@ -145,8 +149,8 @@ class AddCertificateCommand(SyncTestCase): An error message is produced when adding a certificate with a duplicate name. """ - self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) - self.tahoe.stderr = BytesIO() + self.tahoe.stdin = StringIO(json.dumps(fake_cert)) + self.tahoe.stderr = StringIO() self.tahoe.parseOptions( [ "--node-directory", self.node_path.path, @@ -158,7 +162,7 @@ class AddCertificateCommand(SyncTestCase): rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 0) - self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.stdin = StringIO(json.dumps(fake_cert)) self.tahoe.parseOptions( [ "--node-directory", self.node_path.path, From 969f3fa9b3f87bd789a0da46dae8c0a8fbe1d4a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:46:28 -0400 Subject: [PATCH 212/272] Port to Python 3. --- src/allmydata/test/cli/test_admin.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 1ac047acb..fe7aa21a5 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,4 +1,15 @@ -from future.utils import PY3 +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str # We're going to override stdin/stderr, so want to match their behavior on respective Python versions. @@ -66,8 +77,8 @@ class AddCertificateOptions(SyncTestCase): A certificate can be read from a file """ tmp = self.mktemp() - with open(tmp, "w") as f: - f.write(json.dumps(fake_cert)) + with open(tmp, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) # certificate should be loaded self.tahoe.parseOptions( From 6df4fa315ecea6d8f06b991ba49ddf07a8721f42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:55:41 -0400 Subject: [PATCH 213/272] Add explicit dependency on Click. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 991f79e93..f79f1feed 100644 --- a/setup.py +++ b/setup.py @@ -137,6 +137,9 @@ install_requires = [ # Backported configparser for Python 2: "configparser ; python_version < '3.0'", + + # Command-line parsing + "click >= 7.0", ] setup_requires = [ From c7f0a099e1533f4b41563bf05d22cea3277fa21c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 15:44:54 -0400 Subject: [PATCH 214/272] More compatibility with stdlib json module. --- src/allmydata/util/jsonbytes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 08e0cb68e..8ddc4c453 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -116,6 +116,7 @@ def dumps_bytes(obj, *args, **kwargs): # To make this module drop-in compatible with json module: loads = json.loads +load = json.load __all__ = ["dumps", "loads"] From c3f6184960c88df8e54e0d5d9b45eac3c482600a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 16:55:03 -0400 Subject: [PATCH 215/272] Match documented behavior. --- src/allmydata/test/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index fea71799a..61d6d6e78 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -131,6 +131,6 @@ def base32text(): Build text()s that are valid base32 """ return builds( - b2a, + lambda b: str(b2a(b), "ascii"), binary(), ) From c88130d8a819d0600538182d9e1c6103d656d403 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 16:55:12 -0400 Subject: [PATCH 216/272] Make signed certificates an object instead of a dict. --- src/allmydata/cli/grid_manager.py | 4 +- src/allmydata/grid_manager.py | 51 ++++++++++++++++--------- src/allmydata/test/test_grid_manager.py | 12 +++--- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index acf233b29..eb09e787a 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,3 +1,5 @@ +from __future__ import print_function + from datetime import ( datetime, timedelta, @@ -188,7 +190,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(certificate, indent=4) + certificate_data = json.dumps(certificate.asdict(), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index c288a34bb..a5a120a67 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -2,6 +2,9 @@ Functions and classes relating to the Grid Manager internal state """ +from future.utils import PY2, PY3 +from past.builtins import unicode + import sys import json from datetime import ( @@ -18,6 +21,24 @@ from allmydata.util import ( import attr +@attr.s +class SignedCertificate(object): + """ + A signed certificate. + """ + # A JSON-encoded certificate. + certificate = attr.ib(type=unicode) + # The signature in base32. + signature = attr.ib(type=unicode) + + @classmethod + def load(cls, file_like): + return cls(**json.load(file_like)) + + def asdict(self): + return attr.asdict(self) + + @attr.s class _GridManagerStorageServer(object): """ @@ -97,10 +118,10 @@ def _load_certificates_for(config_path, name, gm_key=None): cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) certificates = [] while cert_path.exists(): - container = json.load(cert_path.open('r')) + container = SignedCertificate.load(cert_path.open('r')) if gm_key is not None: validate_grid_manager_certificate(gm_key, container) - cert_data = json.loads(container['certificate']) + cert_data = json.loads(container.certificate) if cert_data['version'] != 1: raise ValueError( "Unknown certificate version '{}' in '{}'".format( @@ -207,8 +228,7 @@ class _GridManager(object): :param timedelta expiry: how far in the future the certificate should expire. - :returns: a dict defining the certificate (it has - "certificate" and "signature" keys). + :returns SignedCertificate: the signed certificate. """ try: srv = self._storage_servers[name] @@ -225,11 +245,10 @@ class _GridManager(object): } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') sig = ed25519.sign_data(self._private_key, cert_data) - certificate = { - u"certificate": cert_data, - u"signature": base32.b2a(sig), - } - + certificate = SignedCertificate( + certificate=cert_data, + signature=base32.b2a(sig), + ) vk = ed25519.verifying_key_from_signing_key(self._private_key) ed25519.verify_signature(vk, sig, cert_data) @@ -343,9 +362,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): :param gm_key: a VerifyingKey instance, a Grid Manager's public key. - :param alleged_cert: dict with "certificate" and "signature" keys, where - "certificate" contains a JSON-serialized certificate for a Storage - Server (comes from a Grid Manager). + :param alleged_cert SignedCertificate: A signed certificate. :return: a dict consisting of the deserialized certificate data or None if the signature is invalid. Note we do NOT check the @@ -354,13 +371,13 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): try: ed25519.verify_signature( gm_key, - base32.a2b(alleged_cert['signature'].encode('ascii')), - alleged_cert['certificate'].encode('ascii'), + base32.a2b(alleged_cert.signature.encode('ascii')), + alleged_cert.certificate.encode('ascii'), ) except ed25519.BadSignature: return None # signature is valid; now we can load the actual data - cert = json.loads(alleged_cert['certificate']) + cert = json.loads(alleged_cert.certificate) return cert @@ -371,10 +388,10 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= (instead of just returning True/False here) so that the expiry-time can be tested on each call. - :param list keys: 0 or more `VerifyingKey` instances + :param list keys: 0 or more ``VerifyingKey`` instances :param list certs: 1 or more Grid Manager certificates each of - which is a `dict` containing 'signature' and 'certificate' keys. + which is a ``SignedCertificate``. :param str public_key: the identifier of the server we expect certificates for. diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index e53984a05..539ff5c2e 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -27,6 +27,7 @@ from allmydata.grid_manager import ( create_grid_manager, parse_grid_manager_certificate, create_grid_manager_verifier, + SignedCertificate, ) from allmydata.test.strategies import ( base32text, @@ -165,16 +166,13 @@ class GridManagerVerifier(SyncTestCase): cert1 = self.gm.sign("test", timedelta(seconds=3600)) self.assertNotEqual(cert0, cert1) - self.assertEqual( - set(cert0.keys()), - {"certificate", "signature"}, - ) + self.assertIsInstance(cert0, SignedCertificate) gm_key = ed25519.verifying_key_from_string(self.gm.public_identity()) self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert0["signature"]), - cert0["certificate"], + base32.a2b(cert0.signature.encode("ascii")), + cert0.certificate.encode("ascii"), ), None ) @@ -432,7 +430,7 @@ class GridManagerInvalidVerifier(SyncTestCase): An incorrect signature is rejected """ # make signature invalid - self.cert0["signature"] = invalid_signature + self.cert0.signature = invalid_signature verify = create_grid_manager_verifier( [self.gm._public_key], From e3a844e68444ca661342937b10c16fd31ed23ae2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 09:25:49 -0400 Subject: [PATCH 217/272] Maybe fields are better off as bytes in SignedCertificate. --- src/allmydata/grid_manager.py | 25 +++++++++++++++++-------- src/allmydata/test/test_grid_manager.py | 4 +++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index a5a120a67..e99dd1863 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -6,7 +6,6 @@ from future.utils import PY2, PY3 from past.builtins import unicode import sys -import json from datetime import ( datetime, ) @@ -16,6 +15,7 @@ from allmydata.crypto import ( ) from allmydata.util import ( base32, + jsonbytes as json, ) import attr @@ -26,14 +26,23 @@ class SignedCertificate(object): """ A signed certificate. """ - # A JSON-encoded certificate. - certificate = attr.ib(type=unicode) + # A JSON-encoded, UTF-8-encoded certificate. + certificate = attr.ib( + type=bytes, validator=attr.validators.instance_of(bytes) + ) # The signature in base32. - signature = attr.ib(type=unicode) + signature = attr.ib( + type=bytes, + validator=attr.validators.instance_of(bytes) + ) @classmethod def load(cls, file_like): - return cls(**json.load(file_like)) + data = json.load(file_like) + return cls( + certificate=data["certificate"].encode("ascii"), + signature=data["signature"].encode("ascii") + ) def asdict(self): return attr.asdict(self) @@ -243,7 +252,7 @@ class _GridManager(object): "public_key": srv.public_key_string(), "version": 1, } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') + cert_data = json.dumps_bytes(cert_info, separators=(',',':'), sort_keys=True) sig = ed25519.sign_data(self._private_key, cert_data) certificate = SignedCertificate( certificate=cert_data, @@ -371,8 +380,8 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): try: ed25519.verify_signature( gm_key, - base32.a2b(alleged_cert.signature.encode('ascii')), - alleged_cert.certificate.encode('ascii'), + base32.a2b(alleged_cert.signature), + alleged_cert.certificate, ) except ed25519.BadSignature: return None diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 539ff5c2e..5276df4f0 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + from datetime import ( timedelta, ) @@ -430,7 +432,7 @@ class GridManagerInvalidVerifier(SyncTestCase): An incorrect signature is rejected """ # make signature invalid - self.cert0.signature = invalid_signature + self.cert0.signature = invalid_signature.encode("ascii") verify = create_grid_manager_verifier( [self.gm._public_key], From f99f9cf7d1f1928d8d67adb32339565ffaf82196 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 09:47:42 -0400 Subject: [PATCH 218/272] Tests pass on Python 3. --- src/allmydata/grid_manager.py | 5 +++-- src/allmydata/node.py | 2 +- src/allmydata/test/test_grid_manager.py | 18 +++++++++--------- src/allmydata/test/test_util.py | 1 - 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index e99dd1863..018fcd461 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -334,7 +334,8 @@ def save_grid_manager(file_path, grid_manager, create=True): if create: raise with file_path.child("config.json").open("w") as f: - f.write("{}\n".format(data)) + f.write(data.encode("utf-8")) + f.write(b"\n") def parse_grid_manager_certificate(gm_data): @@ -465,7 +466,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= now = now_fn() for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - if cert['public_key'] == public_key: + if cert['public_key'].encode("ascii") == public_key: if expires > now: # not-expired return True diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0feaffb35..e6f3820d0 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -534,7 +534,7 @@ class _Config(object): cert_fnames = list(self.enumerate_section("grid_manager_certificates").values()) for fname in cert_fnames: - fname = self.get_config_path(fname.decode('utf8')) + fname = self.get_config_path(fname) if not os.path.exists(fname): raise ValueError( "Grid Manager certificate file '{}' doesn't exist".format( diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 5276df4f0..3ee4f8a88 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -3,7 +3,6 @@ from past.builtins import unicode from datetime import ( timedelta, ) -import json from twisted.python.filepath import ( FilePath, @@ -22,6 +21,7 @@ from allmydata.crypto import ( ) from allmydata.util import ( base32, + jsonbytes as json, ) from allmydata.grid_manager import ( load_grid_manager, @@ -173,8 +173,8 @@ class GridManagerVerifier(SyncTestCase): self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert0.signature.encode("ascii")), - cert0.certificate.encode("ascii"), + base32.a2b(cert0.signature), + cert0.certificate, ), None ) @@ -251,7 +251,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -283,9 +283,9 @@ class GridManagerVerifier(SyncTestCase): fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(config, f) + f.write(json.dumps_bytes(config)) with fp.child("alice.cert.0").open("w") as f: - json.dump(bad_cert, f) + f.write(json.dumps_bytes(bad_cert)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -306,7 +306,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -327,7 +327,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -352,7 +352,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a03845ed6..251a60d70 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -559,7 +559,6 @@ class JSONBytes(unittest.TestCase): ) - class FakeGetVersion(object): """Emulate an object with a get_version.""" From bb48974fd8645906431182f8fda21db0269fd88f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 09:50:03 -0400 Subject: [PATCH 219/272] Ported to Python 3. --- src/allmydata/test/test_grid_manager.py | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 3ee4f8a88..ded73e1e5 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import ( timedelta, @@ -52,8 +62,8 @@ class GridManagerUtilities(SyncTestCase): "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" } - with open(cert_path, "w") as f: - f.write(json.dumps(fake_cert)) + with open(cert_path, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" @@ -77,8 +87,8 @@ class GridManagerUtilities(SyncTestCase): "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" } - with open(cert_path, "w") as f: - f.write(json.dumps(fake_cert)) + with open(cert_path, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" @@ -103,8 +113,8 @@ class GridManagerUtilities(SyncTestCase): "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", "something-else": "not valid in a v0 certificate" } - with open(cert_path, "w") as f: - f.write(json.dumps(fake_cert)) + with open(cert_path, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) config_data = ( "[grid_manager_certificates]\n" "ding = {}\n".format(cert_path) @@ -234,7 +244,7 @@ class GridManagerVerifier(SyncTestCase): len(self.gm.storage_servers), len(gm2.storage_servers), ) - for name, ss0 in self.gm.storage_servers.items(): + for name, ss0 in list(self.gm.storage_servers.items()): ss1 = gm2.storage_servers[name] self.assertEqual(ss0.name, ss1.name) self.assertEqual(ss0.public_key_string(), ss1.public_key_string()) From d5b48e65c7dd1f8d6943c85602aed1c8114f9a42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 10:14:39 -0400 Subject: [PATCH 220/272] Fix bug in jsonbytes. --- src/allmydata/test/test_util.py | 6 ++++++ src/allmydata/util/jsonbytes.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 251a60d70..50532274a 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -558,6 +558,12 @@ class JSONBytes(unittest.TestCase): expected ) + def test_dumps_bytes_unicode_separators(self): + """Unicode separators don't prevent the result from being bytes.""" + result = jsonbytes.dumps_bytes([1, 2], separators=(u',', u':')) + self.assertIsInstance(result, bytes) + self.assertEqual(result, b"[1,2]") + class FakeGetVersion(object): """Emulate an object with a get_version.""" diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 8ddc4c453..159ae7990 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -109,7 +109,7 @@ def dumps_bytes(obj, *args, **kwargs): human consumption. """ result = dumps(obj, *args, **kwargs) - if PY3: + if PY3 or isinstance(result, str): result = result.encode("utf-8") return result From 4c750cf88dd73c4d90c0f5a503d24792a6aa9498 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 10:15:26 -0400 Subject: [PATCH 221/272] Port to Python 3. --- src/allmydata/grid_manager.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 018fcd461..e6fe29326 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,9 +1,16 @@ """ Functions and classes relating to the Grid Manager internal state -""" -from future.utils import PY2, PY3 -from past.builtins import unicode +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys from datetime import ( @@ -16,6 +23,7 @@ from allmydata.crypto import ( from allmydata.util import ( base32, jsonbytes as json, + dictutil, ) import attr @@ -193,7 +201,7 @@ def load_grid_manager(config_path): ) storage_servers = dict() - for name, srv_config in config.get(u'storage_servers', {}).items(): + for name, srv_config in list(config.get(u'storage_servers', {}).items()): if 'public_key' not in srv_config: raise ValueError( "No 'public_key' for storage server '{}'".format(name) @@ -213,7 +221,10 @@ class _GridManager(object): """ def __init__(self, private_key_bytes, storage_servers): - self._storage_servers = dict() if storage_servers is None else storage_servers + self._storage_servers = dictutil.UnicodeKeyDict( + {} if storage_servers is None else storage_servers + ) + assert isinstance(private_key_bytes, bytes) self._private_key_bytes = private_key_bytes self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) self._version = 0 @@ -239,6 +250,7 @@ class _GridManager(object): :returns SignedCertificate: the signed certificate. """ + assert isinstance(name, str) # must be unicode try: srv = self._storage_servers[name] except KeyError: From c0d07e1894d91fd1e37d5ff5f6d477c2c1290184 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:12:34 -0400 Subject: [PATCH 222/272] Bit more input validation. --- src/allmydata/grid_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index e6fe29326..8589e87c0 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -283,6 +283,7 @@ class _GridManager(object): storage provider (e.g. from the contents of node.pubkey for the client) """ + assert isinstance(name, str) # must be unicode if name in self._storage_servers: raise KeyError( "Already have a storage server called '{}'".format(name) @@ -295,6 +296,7 @@ class _GridManager(object): """ :param name: a user-meaningful name for the server """ + assert isinstance(name, str) # must be unicode try: del self._storage_servers[name] except KeyError: From 1acc80b56312f3533d098f76b923f18e27616ece Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:13:03 -0400 Subject: [PATCH 223/272] Validate commands actually succeeded! --- src/allmydata/test/cli/test_grid_manager.py | 54 ++++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index e63e6aaaa..8d2acc542 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -41,15 +41,21 @@ class GridManagerCommandLine(TestCase): self.runner = click.testing.CliRunner() super(GridManagerCommandLine, self).setUp() + def invoke_and_check(self, *args, **kwargs): + """Invoke a command with the runner and ensure it succeeded.""" + result = self.runner.invoke(*args, **kwargs) + self.assertEqual(result.exit_code, 0, result) + return result + def test_create(self): """ Create a new grid-manager """ with self.runner.isolated_filesystem(): - result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) self.assertEqual(["foo"], os.listdir(".")) self.assertEqual(["config.json"], os.listdir("./foo")) - result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "public-identity"]) self.assertTrue(result.output.startswith("pub-v0-")) def test_load_invalid(self): @@ -72,7 +78,7 @@ class GridManagerCommandLine(TestCase): directory. """ with self.runner.isolated_filesystem(): - result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.assertEqual(1, result.exit_code) self.assertIn( @@ -85,7 +91,7 @@ class GridManagerCommandLine(TestCase): Create a new grid-manager with no files """ with self.runner.isolated_filesystem(): - result = self.runner.invoke(grid_manager, ["--config", "-", "create"]) + result = self.invoke_and_check(grid_manager, ["--config", "-", "create"]) self.assertEqual([], os.listdir(".")) config = json.loads(result.output) self.assertEqual( @@ -106,7 +112,7 @@ class GridManagerCommandLine(TestCase): "private_key": "priv-v0-6uinzyaxy3zvscwgsps5pxcfezhrkfb43kvnrbrhhfzyduyqnniq", "grid_manager_config_version": 0 } - result = self.runner.invoke( + result = self.invoke_and_check( grid_manager, ["--config", "-", "list"], input=BytesIO(json.dumps(config)), ) @@ -122,9 +128,9 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) sigcert = json.loads(result.output) self.assertEqual({"certificate", "signature"}, set(sigcert.keys())) cert = json.loads(sigcert['certificate']) @@ -136,10 +142,10 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) - self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) # we should now have two certificates stored self.assertEqual( set(FilePath("foo").listdir()), @@ -153,8 +159,8 @@ class GridManagerCommandLine(TestCase): pubkey0 = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" pubkey1 = "pub-v0-5ysc55trfvfvg466v46j4zmfyltgus3y2gdejifctv7h4zkuyveq" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey0]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey0]) result = self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey1]) self.assertNotEquals(result.exit_code, 0) self.assertIn( @@ -168,11 +174,11 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "1"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "1"]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "list"]) names = [ line.split(':')[0] for line in result.output.strip().split('\n') @@ -180,9 +186,9 @@ class GridManagerCommandLine(TestCase): ] self.assertEqual(names, ["storage0"]) - self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "remove", "storage0"]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "list"]) self.assertEqual(result.output.strip(), "") def test_remove_missing(self): @@ -190,7 +196,7 @@ class GridManagerCommandLine(TestCase): Error reported when removing non-existant server """ with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) self.assertNotEquals(result.exit_code, 0) self.assertIn( @@ -203,7 +209,7 @@ class GridManagerCommandLine(TestCase): Error reported when signing non-existant server """ with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) self.assertNotEquals(result.exit_code, 0) self.assertIn( @@ -218,8 +224,8 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) # make the directory un-writable (so we can't create a new cert) os.chmod("foo", 0o550) result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) From aacdf9906daa47e072ac1fbcc18933442a4bc64f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:21:59 -0400 Subject: [PATCH 224/272] Make failures easier to debug. --- src/allmydata/cli/grid_manager.py | 1 + src/allmydata/test/cli/test_grid_manager.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index eb09e787a..44a524db9 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -111,6 +111,7 @@ def add(ctx, name, public_key): PUBLIC_KEY is the contents of a node.pubkey file from a Tahoe node-directory. NAME is an arbitrary label. """ + public_key = public_key.encode("ascii") try: ctx.obj.grid_manager.add_storage_server( name, diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 8d2acc542..f105dabeb 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,3 +1,4 @@ +from future.utils import raise_ import os import json @@ -44,6 +45,8 @@ class GridManagerCommandLine(TestCase): def invoke_and_check(self, *args, **kwargs): """Invoke a command with the runner and ensure it succeeded.""" result = self.runner.invoke(*args, **kwargs) + if result.exception is not None: + raise_(*result.exc_info) self.assertEqual(result.exit_code, 0, result) return result From 6bfcda2a3d82898d504db29f60bdd8aabc922029 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:26:51 -0400 Subject: [PATCH 225/272] Tests pass on Python 3. --- src/allmydata/cli/grid_manager.py | 11 +++++++---- src/allmydata/test/cli/test_grid_manager.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 44a524db9..78b037649 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,10 +1,10 @@ from __future__ import print_function +from past.builtins import unicode from datetime import ( datetime, timedelta, ) -import json import click @@ -23,6 +23,7 @@ from allmydata.grid_manager import ( save_grid_manager, load_grid_manager, ) +from allmydata.util import jsonbytes as json @click.group() @@ -102,7 +103,7 @@ def public_identity(config): @grid_manager.command() @click.argument("name") -@click.argument("public_key", type=click.UNPROCESSED) +@click.argument("public_key", type=click.STRING) @click.pass_context def add(ctx, name, public_key): """ @@ -160,7 +161,9 @@ def list(ctx): """ for name in sorted(ctx.obj.grid_manager.storage_servers.keys()): blank_name = " " * len(name) - click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key_string())) + click.echo("{}: {}".format( + name, + unicode(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) for cert in ctx.obj.grid_manager.storage_servers[name].certificates: delta = datetime.utcnow() - cert.expires click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) @@ -209,7 +212,7 @@ def sign(ctx, name, expiry_days): ) next_serial += 1 with f: - f.write(certificate_data) + f.write(certificate_data.encode("ascii")) def _config_path_from_option(config): diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index f105dabeb..7fbc6a563 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,7 +1,6 @@ from future.utils import raise_ import os -import json from io import ( BytesIO, ) @@ -31,6 +30,7 @@ from twisted.python.filepath import ( from twisted.python.runtime import ( platform, ) +from allmydata.util import jsonbytes as json class GridManagerCommandLine(TestCase): @@ -66,8 +66,8 @@ class GridManagerCommandLine(TestCase): An invalid config is reported to the user """ with self.runner.isolated_filesystem(): - with open("config.json", "w") as f: - json.dump({"not": "valid"}, f) + with open("config.json", "wb") as f: + f.write(json.dumps_bytes({"not": "valid"})) result = self.runner.invoke(grid_manager, ["--config", ".", "public-identity"]) self.assertNotEqual(result.exit_code, 0) self.assertIn( @@ -117,7 +117,7 @@ class GridManagerCommandLine(TestCase): } result = self.invoke_and_check( grid_manager, ["--config", "-", "list"], - input=BytesIO(json.dumps(config)), + input=BytesIO(json.dumps_bytes(config)), ) self.assertEqual(result.exit_code, 0) self.assertEqual( @@ -260,7 +260,7 @@ class TahoeAddGridManagerCert(TestCase): """ code, out, err = yield run_cli( "admin", "add-grid-manager-cert", "--filename", "-", - stdin="the cert", + stdin=b"the cert", ) self.assertIn( "Must provide --name", @@ -274,7 +274,7 @@ class TahoeAddGridManagerCert(TestCase): """ code, out, err = yield run_cli( "admin", "add-grid-manager-cert", "--name", "foo", - stdin="the cert", + stdin=b"the cert", ) self.assertIn( "Must provide --filename", @@ -287,7 +287,7 @@ class TahoeAddGridManagerCert(TestCase): we can add a certificate """ nodedir = self.mktemp() - fake_cert = """{"certificate": "", "signature": ""}""" + fake_cert = b"""{"certificate": "", "signature": ""}""" code, out, err = yield run_cli( "--node-directory", nodedir, @@ -301,7 +301,7 @@ class TahoeAddGridManagerCert(TestCase): self.assertIn("tahoe.cfg", nodepath.listdir()) self.assertIn( - "foo = foo.cert", + b"foo = foo.cert", config_data, ) self.assertIn("foo.cert", nodepath.listdir()) From e8e0ecd39e8f95659711305f381410d750ab9a7a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:38:15 -0400 Subject: [PATCH 226/272] Port to Python 3. --- src/allmydata/test/cli/test_grid_manager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 7fbc6a563..03e50efdd 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,4 +1,14 @@ -from future.utils import raise_ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, raise_ +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from io import ( From 2d2e8051f6a52bac860ed74858687743c842d4db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:39:32 -0400 Subject: [PATCH 227/272] Port to Python 3. --- src/allmydata/cli/grid_manager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 78b037649..24d12398c 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,5 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import ( datetime, @@ -163,7 +172,7 @@ def list(ctx): blank_name = " " * len(name) click.echo("{}: {}".format( name, - unicode(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) + str(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) for cert in ctx.obj.grid_manager.storage_servers[name].certificates: delta = datetime.utcnow() - cert.expires click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) From 75ba5c71fce340dcf6724bdb9ee015b51f55b6e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 12:04:42 -0400 Subject: [PATCH 228/272] More passing tests on Python 3. --- src/allmydata/test/common_util.py | 1 - src/allmydata/test/test_client.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index ec449c2af..b82a7f774 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -127,7 +127,6 @@ def run_cli_native(verb, *args, **kwargs): stdin = TextIOWrapper(BytesIO(stdin), encoding) stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) - stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b"")), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback(runner.dispatch, diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 5d91b5ea4..f870f9fa5 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -12,7 +12,6 @@ if PY2: import os import sys -import json from functools import ( partial, ) @@ -75,6 +74,7 @@ from allmydata.util import ( fileutil, encodingutil, configutil, + jsonbytes as json, ) from allmydata.util.eliotutil import capture_logging from allmydata.util.fileutil import abspath_expanduser_unicode @@ -1520,9 +1520,9 @@ enabled = {storage_enabled} "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", } with self.basedir.child("zero.cert").open("w") as f: - json.dump(fake_cert, f) + f.write(json.dumps_bytes(fake_cert)) with self.basedir.child("gm0.cert").open("w") as f: - json.dump(fake_cert, f) + f.write(json.dumps_bytes(fake_cert)) config = client.config_from_string( self.basedir.path, From b9a1cc3dde82662bdfe498c9c09ec96433c77235 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 10:27:15 -0400 Subject: [PATCH 229/272] Pacify flake8. --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 24d12398c..b12839eae 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -162,7 +162,7 @@ def remove(ctx, name): save_grid_manager(fp, ctx.obj.grid_manager, create=False) -@grid_manager.command() +@grid_manager.command() # noqa: F811 @click.pass_context def list(ctx): """ From 056f7748c5d38811caa8b14e1327d86e19f61098 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 10:29:49 -0400 Subject: [PATCH 230/272] Fix errant str()-of-bytes bug. --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f6f2919b4..6bd8b3ce5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -289,7 +289,7 @@ class StorageFarmBroker(service.MultiService): gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), - "pub-{}".format(server_id), # server_id is v0- not pub-v0-key .. for reasons? + "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) s = NativeStorageServer( From b3ab2fdb81f60d909572fc8e1be7defe3885e1ca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Jul 2022 15:12:31 -0400 Subject: [PATCH 231/272] Fix grid manager CLI tests. --- src/allmydata/test/common_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e63c3eef8..abf675644 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -134,6 +134,7 @@ def run_cli_native(verb, *args, **kwargs): stdin = TextIOWrapper(BytesIO(stdin), encoding) stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) + options.stdin = stdin d = defer.succeed(argv) d.addCallback( partial( From b4703ace930cadb276b627a3612816d4986f9231 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 10:35:28 -0400 Subject: [PATCH 232/272] Some tweaks for Python 3. --- src/allmydata/cli/grid_manager.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index b12839eae..4ef53887c 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,15 +1,8 @@ """ -Ported to Python 3. +A CLI for configuring a grid manager. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from typing import Optional from datetime import ( datetime, timedelta, @@ -224,7 +217,7 @@ def sign(ctx, name, expiry_days): f.write(certificate_data.encode("ascii")) -def _config_path_from_option(config): +def _config_path_from_option(config: str) -> Optional[FilePath]: """ :param str config: a path or - :returns: a FilePath instance or None From d84d366c72ddac5d15c27e739151fe8a8f91572d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 10:50:04 -0400 Subject: [PATCH 233/272] Some tweaks for Python 3 and modern attrs. --- src/allmydata/grid_manager.py | 62 ++++++++++++++--------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 8589e87c0..b6ae04136 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,21 +1,14 @@ """ Functions and classes relating to the Grid Manager internal state - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys from datetime import ( datetime, ) +from typing import Optional + +from twisted.python.filepath import FilePath from allmydata.crypto import ( ed25519, @@ -26,48 +19,41 @@ from allmydata.util import ( dictutil, ) -import attr +from attrs import define, asdict, Factory -@attr.s +@define class SignedCertificate(object): """ A signed certificate. """ # A JSON-encoded, UTF-8-encoded certificate. - certificate = attr.ib( - type=bytes, validator=attr.validators.instance_of(bytes) - ) + certificate : bytes + # The signature in base32. - signature = attr.ib( - type=bytes, - validator=attr.validators.instance_of(bytes) - ) + signature : bytes @classmethod def load(cls, file_like): data = json.load(file_like) return cls( - certificate=data["certificate"].encode("ascii"), + certificate=data["certificate"].encode("utf-8"), signature=data["signature"].encode("ascii") ) def asdict(self): - return attr.asdict(self) + return asdict(self) -@attr.s +@define class _GridManagerStorageServer(object): """ A Grid Manager's notion of a storage server """ - name = attr.ib() - public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) - certificates = attr.ib( - default=attr.Factory(list), - validator=attr.validators.instance_of(list), - ) + name : str + public_key : ed25519.Ed25519PublicKey + certificates : list = Factory(list) def add_certificate(self, certificate): """ @@ -75,9 +61,9 @@ class _GridManagerStorageServer(object): """ self.certificates.append(certificate) - def public_key_string(self): + def public_key_string(self) -> bytes: """ - :returns: the public key as a string + :returns: the public key as bytes. """ return ed25519.string_from_verifying_key(self.public_key) @@ -90,16 +76,16 @@ class _GridManagerStorageServer(object): } -@attr.s +@define class _GridManagerCertificate(object): """ Represents a single certificate for a single storage-server """ - filename = attr.ib() - index = attr.ib(validator=attr.validators.instance_of(int)) - expires = attr.ib(validator=attr.validators.instance_of(datetime)) - public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) + filename : str + index : int + expires : datetime + public_key : ed25519.Ed25519PublicKey def create_grid_manager(): @@ -113,7 +99,7 @@ def create_grid_manager(): ) -def _load_certificates_for(config_path, name, gm_key=None): +def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]): """ Load any existing certificates for the given storage-server. @@ -122,7 +108,7 @@ def _load_certificates_for(config_path, name, gm_key=None): :param str name: the name of an existing storage-server - :param ed25519.VerifyingKey gm_key: an optional Grid Manager + :param ed25519.Ed25519PublicKey gm_key: an optional Grid Manager public key. If provided, certificates will be verified against it. :returns: list containing any known certificates (may be empty) @@ -159,7 +145,7 @@ def _load_certificates_for(config_path, name, gm_key=None): return certificates -def load_grid_manager(config_path): +def load_grid_manager(config_path: Optional[FilePath]): """ Load a Grid Manager from existing configuration. From 90188ce4d14b5e19c600ebc6b5734cd2bc267c34 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:05:03 -0400 Subject: [PATCH 234/272] More Python 3 tweaks. --- src/allmydata/grid_manager.py | 4 ++-- src/allmydata/scripts/admin.py | 19 ++++++++----------- src/allmydata/scripts/types_.py | 2 -- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index b6ae04136..f502c413e 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -6,7 +6,7 @@ import sys from datetime import ( datetime, ) -from typing import Optional +from typing import Optional, Union from twisted.python.filepath import FilePath @@ -338,7 +338,7 @@ def save_grid_manager(file_path, grid_manager, create=True): f.write(b"\n") -def parse_grid_manager_certificate(gm_data): +def parse_grid_manager_certificate(gm_data: Union[str, bytes]): """ :param gm_data: some data that might be JSON that might be a valid Grid Manager Certificate diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 36266f3dd..6a24614c0 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -10,12 +10,9 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_binary, ensure_str +from six import ensure_binary -try: - from allmydata.scripts.types_ import SubCommands -except ImportError: - pass +from typing import Union from twisted.python import usage from twisted.python.filepath import ( @@ -29,14 +26,12 @@ from allmydata.storage import ( crawler, expirer, ) -from twisted.python.filepath import FilePath - +from allmydata.scripts.types_ import SubCommands from allmydata.client import read_config from allmydata.grid_manager import ( parse_grid_manager_certificate, ) from allmydata.scripts.cli import _default_nodedir -from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import jsonbytes @@ -127,8 +122,10 @@ class AddGridManagerCertOptions(BaseOptions): raise usage.UsageError( "Must provide --filename option" ) + + data : Union [bytes, str] if self['filename'] == '-': - print(ensure_str("reading certificate from stdin"), file=self.parent.parent.stderr) + print("reading certificate from stdin", file=self.parent.parent.stderr) data = self.parent.parent.stdin.read() if len(data) == 0: raise usage.UsageError( @@ -201,7 +198,7 @@ def add_grid_manager_cert(options): options['name'], cert_path.path, ) - print(ensure_str(msg), file=options.parent.parent.stderr) + print(msg, file=options.parent.parent.stderr) return 1 config.set_config("storage", "grid_management", "True") @@ -212,7 +209,7 @@ def add_grid_manager_cert(options): f.write(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) - print(ensure_str("There are now {} certificates").format(cert_count), + print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) return 0 diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py index 1bed6e11e..e2a5c2f1e 100644 --- a/src/allmydata/scripts/types_.py +++ b/src/allmydata/scripts/types_.py @@ -2,8 +2,6 @@ Type definitions used by modules in this package. """ -# Python 3 only - from typing import List, Tuple, Type, Sequence, Any from twisted.python.usage import Options From 6cf3bc75b65cbb90e3b4f7a577966df06f498fd0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:09:19 -0400 Subject: [PATCH 235/272] Some Python 3 cleanups. --- src/allmydata/crypto/ed25519.py | 35 +++++++++++---------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 098fa9758..e2d2ceb49 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -13,20 +13,7 @@ cut-and-pasteability. The base62 encoding is shorter than the base32 form, but the minor usability improvement is not worth the documentation and specification confusion of using a non-standard encoding. So we stick with base32. - -Ported to Python 3. ''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import six from cryptography.exceptions import ( InvalidSignature, @@ -72,7 +59,7 @@ def verifying_key_from_signing_key(private_key): return private_key.public_key() -def sign_data(private_key, data): +def sign_data(private_key, data: bytes) -> bytes: """ Sign the given data using the given private key @@ -86,7 +73,7 @@ def sign_data(private_key, data): """ _validate_private_key(private_key) - if not isinstance(data, six.binary_type): + if not isinstance(data, bytes): raise ValueError('data must be bytes') return private_key.sign(data) @@ -110,7 +97,7 @@ def string_from_signing_key(private_key): return PRIVATE_KEY_PREFIX + b2a(raw_key_bytes) -def signing_keypair_from_string(private_key_bytes): +def signing_keypair_from_string(private_key_bytes: bytes): """ Load a signing keypair from a string of bytes (which includes the PRIVATE_KEY_PREFIX) @@ -118,7 +105,7 @@ def signing_keypair_from_string(private_key_bytes): :returns: a 2-tuple of (private_key, public_key) """ - if not isinstance(private_key_bytes, six.binary_type): + if not isinstance(private_key_bytes, bytes): raise ValueError('private_key_bytes must be bytes') private_key = Ed25519PrivateKey.from_private_bytes( @@ -127,7 +114,7 @@ def signing_keypair_from_string(private_key_bytes): return private_key, private_key.public_key() -def verify_signature(public_key, alleged_signature, data): +def verify_signature(public_key, alleged_signature: bytes, data: bytes): """ :param public_key: a verifying key @@ -139,10 +126,10 @@ def verify_signature(public_key, alleged_signature, data): :returns: None (or raises an exception). """ - if not isinstance(alleged_signature, six.binary_type): + if not isinstance(alleged_signature, bytes): raise ValueError('alleged_signature must be bytes') - if not isinstance(data, six.binary_type): + if not isinstance(data, bytes): raise ValueError('data must be bytes') _validate_public_key(public_key) @@ -159,7 +146,7 @@ def verifying_key_from_string(public_key_bytes): :returns: a public_key """ - if not isinstance(public_key_bytes, six.binary_type): + if not isinstance(public_key_bytes, bytes): raise ValueError('public_key_bytes must be bytes') return Ed25519PublicKey.from_public_bytes( @@ -167,7 +154,7 @@ def verifying_key_from_string(public_key_bytes): ) -def string_from_verifying_key(public_key): +def string_from_verifying_key(public_key) -> bytes: """ Encode a public key to a string of bytes @@ -183,7 +170,7 @@ def string_from_verifying_key(public_key): return PUBLIC_KEY_PREFIX + b2a(raw_key_bytes) -def _validate_public_key(public_key): +def _validate_public_key(public_key: Ed25519PublicKey): """ Internal helper. Verify that `public_key` is an appropriate object """ @@ -192,7 +179,7 @@ def _validate_public_key(public_key): return None -def _validate_private_key(private_key): +def _validate_private_key(private_key: Ed25519PrivateKey): """ Internal helper. Verify that `private_key` is an appropriate object """ From 07a3d1ecd64661ae590405cd6a9917652a6a8e15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:15:03 -0400 Subject: [PATCH 236/272] Python 3 tweaks. --- src/allmydata/storage_client.py | 1 - src/allmydata/test/cli/test_admin.py | 12 ++++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f2d50a574..91073579d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -38,7 +38,6 @@ import re import time import hashlib -# On Python 2 this will be the backport. from configparser import NoSectionError import attr diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index ce59fa755..9bf12471f 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -6,16 +6,12 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_str # We're going to override stdin/stderr, so want to match their behavior on respective Python versions. -if PY3: - from io import StringIO -else: - from StringIO import StringIO +from io import StringIO from twisted.python.usage import ( UsageError, @@ -116,7 +112,7 @@ class AddCertificateOptions(SyncTestCase): """ When no data is passed to stdin an error is produced """ - self.tahoe.stdin = StringIO(ensure_str("")) + self.tahoe.stdin = StringIO("") self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: @@ -159,7 +155,7 @@ class AddCertificateOptions(SyncTestCase): """ Unparseable data produces an error """ - self.tahoe.stdin = StringIO(ensure_str("{}")) + self.tahoe.stdin = StringIO("{}") self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: From 8b0941acd26518e798130e70d6f4b05d3442c2d0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:20:46 -0400 Subject: [PATCH 237/272] Python 3 tweaks. --- src/allmydata/test/cli/test_grid_manager.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 03e50efdd..2ed738544 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,14 +1,6 @@ """ -Ported to Python 3. +Tests for the grid manager CLI. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2, raise_ -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from io import ( @@ -56,7 +48,7 @@ class GridManagerCommandLine(TestCase): """Invoke a command with the runner and ensure it succeeded.""" result = self.runner.invoke(*args, **kwargs) if result.exception is not None: - raise_(*result.exc_info) + raise result.exc_info[1].with_traceback(result.exc_info[2]) self.assertEqual(result.exit_code, 0, result) return result From 011b942372782a14a495b7372ccfed7ff3b903af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:26:33 -0400 Subject: [PATCH 238/272] Python 3 tweaks. --- src/allmydata/test/test_grid_manager.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index ded73e1e5..0f8df69d6 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,14 +1,6 @@ """ -Ported to Python 3. +Tests for the grid manager. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import ( timedelta, From 1bcca7f630884a66c7ae515496ca4765351a2a89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:29:20 -0400 Subject: [PATCH 239/272] Python 3 tweaks. --- src/allmydata/util/jsonbytes.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 159ae7990..1cc8c8ef0 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -9,7 +9,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -108,10 +108,7 @@ def dumps_bytes(obj, *args, **kwargs): UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for human consumption. """ - result = dumps(obj, *args, **kwargs) - if PY3 or isinstance(result, str): - result = result.encode("utf-8") - return result + return dumps(obj, *args, **kwargs).encode("utf-8") # To make this module drop-in compatible with json module: From 630ad1a60673ef794bb589925351d6e3d4560378 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:40:15 -0700 Subject: [PATCH 240/272] Update docs/managed-grid.rst Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 3630b956b..7b216c8bd 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -45,7 +45,7 @@ the statement: "Grid Manager X suggests you use storage-server Y to upload shares to" (X and Y are public-keys). Such a certificate consists of: - - a version + - the version of the format the certificate conforms to (`1`) - the public-key of a storage-server - an expiry timestamp - a signature of the above From 588f1fdb67d34cf78d3178a54b1abc862bd2306b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:40:35 -0700 Subject: [PATCH 241/272] Update docs/managed-grid.rst Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 7b216c8bd..17f7f370f 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -78,7 +78,7 @@ ever available on stdout. The configuration is a JSON document. It is subject to change as Grid Manager evolves. It contains a version number in the -`grid_manager_config_version` key which should increment whenever the +`grid_manager_config_version` key which will increment whenever the document schema changes. From 87c2f9b13e6465b0c7026b6d705d3df8903fdff3 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:44:04 -0700 Subject: [PATCH 242/272] Update docs/managed-grid.rst Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 17f7f370f..df12399d3 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -333,5 +333,5 @@ uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". A proposal to expose more information about Grid Manager and -certifcate status in the Welcome page is discussed in +certificate status in the Welcome page is discussed in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506 From 6da9e50a02b2ebb85fe5bf5e3871c1f1b9dc908c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:45:03 -0700 Subject: [PATCH 243/272] Update newsfragments/2916.feature Co-authored-by: Jean-Paul Calderone --- newsfragments/2916.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/2916.feature b/newsfragments/2916.feature index 292d429ea..c65f473a4 100644 --- a/newsfragments/2916.feature +++ b/newsfragments/2916.feature @@ -1 +1 @@ -A new 'Grid Manager' specification and implementation \ No newline at end of file +Tahoe-LAFS now includes a new "Grid Manager" specification and implementation adding more options to control which storage servers a client will use for uploads. \ No newline at end of file From b2431f3a89e28b76a4e98883c674e59b7ebc25fd Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:46:31 -0700 Subject: [PATCH 244/272] Update src/allmydata/cli/grid_manager.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 4ef53887c..220f091cd 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -52,7 +52,7 @@ def grid_manager(ctx, config): class Config(object): """ - Availble to all sub-commands as Click's context.obj + Available to all sub-commands as Click's context.obj """ _grid_manager = None From 039c1d8037c0a5266383449fa5753ab21705a47e Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:52:42 -0700 Subject: [PATCH 245/272] Update src/allmydata/grid_manager.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index f502c413e..ff5ce735d 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -461,7 +461,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= def validate(): """ - :returns: True if if *any* certificate is still valid for a server + :returns: True if *any* certificate is still valid for a server """ now = now_fn() for cert in valid_certs: From 5b14561ec0c1827744c6d4b5127de7bececabb4d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 12:02:34 -0700 Subject: [PATCH 246/272] use attrs directly --- src/allmydata/cli/grid_manager.py | 3 ++- src/allmydata/grid_manager.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 220f091cd..9e4e911bf 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -9,6 +9,7 @@ from datetime import ( ) import click +import attr from twisted.python.filepath import ( FilePath, @@ -196,7 +197,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(certificate.asdict(), indent=4) + certificate_data = json.dumps(attr.asdict(certificate), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index ff5ce735d..09b9ed001 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -19,7 +19,10 @@ from allmydata.util import ( dictutil, ) -from attrs import define, asdict, Factory +from attrs import ( + define, + Factory, +) @define @@ -41,9 +44,6 @@ class SignedCertificate(object): signature=data["signature"].encode("ascii") ) - def asdict(self): - return asdict(self) - @define class _GridManagerStorageServer(object): From a6cf06cc6d3b691d0971790582332cfbc3026139 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 12:09:51 -0700 Subject: [PATCH 247/272] http needs access to GridManager --- src/allmydata/grid_manager.py | 8 ++++++-- src/allmydata/storage_client.py | 33 +++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 09b9ed001..062a5d9d0 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -6,7 +6,11 @@ import sys from datetime import ( datetime, ) -from typing import Optional, Union +from typing import ( + Optional, + Union, + List, +) from twisted.python.filepath import FilePath @@ -99,7 +103,7 @@ def create_grid_manager(): ) -def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]): +def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: """ Load any existing certificates for the given storage-server. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8a8aa1f2e..6fab1707b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -293,17 +293,22 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) - if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: - s = HTTPNativeStorageServer(server_id, server["ann"]) - s.on_status_changed(lambda _: self._got_connection()) - return s - handler_overrides = server.get("connections", {}) gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) + if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: + s = HTTPNativeStorageServer( + server_id, + server["ann"], + grid_manager_verifier=gm_verifier, + ) + s.on_status_changed(lambda _: self._got_connection()) + return s + + handler_overrides = server.get("connections", {}) s = NativeStorageServer( server_id, server["ann"], @@ -1013,13 +1018,14 @@ class HTTPNativeStorageServer(service.MultiService): "connected". """ - def __init__(self, server_id: bytes, announcement, reactor=reactor): + def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id self.announcement = announcement self._on_status_changed = ObserverList() self._reactor = reactor + self._grid_manager_verifier = grid_manager_verifier furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( self._nickname, @@ -1069,6 +1075,21 @@ class HTTPNativeStorageServer(service.MultiService): """ return self._on_status_changed.subscribe(status_changed) + def upload_permitted(self): + """ + If our client is configured with Grid Manager public-keys, we will + only upload to storage servers that have a currently-valid + certificate signed by at least one of the Grid Managers we + accept. + + :return: True if we should use this server for uploads, False + otherwise. + """ + # if we have no Grid Manager keys configured, choice is easy + if self._grid_manager_verifier is None: + return True + return self._grid_manager_verifier() + # Special methods used by copy.copy() and copy.deepcopy(). When those are # used in allmydata.immutable.filenode to copy CheckResults during # repair, we want it to treat the IServer instances as singletons, and From bdf4c49a3467d84b076c415cde2f976819a74653 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 22:57:15 -0700 Subject: [PATCH 248/272] fine, move the if statement --- src/allmydata/grid_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 062a5d9d0..fb70a6d02 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -103,7 +103,7 @@ def create_grid_manager(): ) -def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: +def _load_certificates_for(config_path: FilePath, name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: """ Load any existing certificates for the given storage-server. @@ -119,8 +119,6 @@ def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Op :raises: ed25519.BadSignature if any certificate signature fails to verify """ - if config_path is None: - return [] cert_index = 0 cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) certificates = [] @@ -199,7 +197,7 @@ def load_grid_manager(config_path: Optional[FilePath]): storage_servers[name] = _GridManagerStorageServer( name, ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), - _load_certificates_for(config_path, name, public_key), + [] if config_path is None else _load_certificates_for(config_path, name, public_key), ) return _GridManager(private_key_bytes, storage_servers) From 7b98edac041312b6565a355d8cbc73c52c4a8f40 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 23:56:19 -0700 Subject: [PATCH 249/272] add more words --- docs/managed-grid.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index df12399d3..aac2bf20a 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -55,6 +55,10 @@ certificate, or no certificate) because clients check the ciphertext and re-assembled plaintext against the keys in the capability; "grid-manager" certificates only control uploads. +Clients make use of this functionality by configuring one or more Grid Manager public keys. +This tells the client to only upload to storage-servers that have a currently-valid certificate from any of the Grid Managers their client allows. +In case none are configured, the default behavior (of using any storage server) prevails. + Grid Manager Data Storage ------------------------- From 8e20fa0fbf033ac9559326e805d644293597e1e0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 00:01:13 -0700 Subject: [PATCH 250/272] whitespace --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a99447ad..98177bd41 100644 --- a/setup.py +++ b/setup.py @@ -146,7 +146,7 @@ install_requires = [ # Command-line parsing "click >= 7.0", - + # for pid-file support "psutil", "filelock", From affe0cb37b2c361c7e4a5fe8e9f98f5b366da02b Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 00:02:53 -0700 Subject: [PATCH 251/272] fine? --- src/allmydata/util/configutil.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index b84d96319..b48e2c034 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -74,10 +74,7 @@ def write_config(tahoe_cfg, config): :return: ``None`` """ tmp = tahoe_cfg.temporarySibling() - try: - tahoe_cfg.parent().makedirs() - except OSError: - pass + tahoe_cfg.parent().makedirs(ignoreExistingDirectory=True) # FilePath.open can only open files in binary mode which does not work # with ConfigParser.write. with open(tmp.path, "wt") as fp: From 43d29986a66b98d358cbb6ef3315ec1258b056c2 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 00:04:03 -0700 Subject: [PATCH 252/272] Update src/allmydata/cli/grid_manager.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/cli/grid_manager.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 220f091cd..af66fa1c6 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -205,13 +205,10 @@ def sign(ctx, name, expiry_days): fname = "{}.cert.{}".format(name, next_serial) try: f = fp.child(fname).create() + except FileExistsError: + f = None except OSError as e: - if e.errno == 17: # file exists - f = None - else: - raise click.ClickException( - "{}: {}".format(fname, e) - ) + raise click.ClickException(f"{fname}: {e}") next_serial += 1 with f: f.write(certificate_data.encode("ascii")) From 032b852bab1babd3cfd3cb9acc975560d74f7315 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 01:51:13 -0700 Subject: [PATCH 253/272] define -> frozen --- src/allmydata/grid_manager.py | 3 ++- src/allmydata/test/test_grid_manager.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index fb70a6d02..7f1c6eda5 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -25,11 +25,12 @@ from allmydata.util import ( from attrs import ( define, + frozen, Factory, ) -@define +@frozen class SignedCertificate(object): """ A signed certificate. diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 0f8df69d6..c282d1237 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -434,11 +434,14 @@ class GridManagerInvalidVerifier(SyncTestCase): An incorrect signature is rejected """ # make signature invalid - self.cert0.signature = invalid_signature.encode("ascii") + invalid_cert = SignedCertificate( + self.cert0.certificate, + invalid_signature.encode("ascii"), + ) verify = create_grid_manager_verifier( [self.gm._public_key], - [self.cert0], + [invalid_cert], ed25519.string_from_verifying_key(self.pub0), bad_cert = lambda key, cert: None, ) From 38669cc3cec58fa9d66b48f063d905382f11b4ec Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 02:15:20 -0700 Subject: [PATCH 254/272] define -> frozen --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 7f1c6eda5..6e71c9902 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -50,7 +50,7 @@ class SignedCertificate(object): ) -@define +@frozen class _GridManagerStorageServer(object): """ A Grid Manager's notion of a storage server From 1b6d5e1bda302ba464f20175896dcc2a12390e96 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 11:56:46 -0700 Subject: [PATCH 255/272] Revert "use attrs directly" This reverts commit 5b14561ec0c1827744c6d4b5127de7bececabb4d. --- src/allmydata/cli/grid_manager.py | 3 +-- src/allmydata/grid_manager.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 9e4e911bf..220f091cd 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -9,7 +9,6 @@ from datetime import ( ) import click -import attr from twisted.python.filepath import ( FilePath, @@ -197,7 +196,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(attr.asdict(certificate), indent=4) + certificate_data = json.dumps(certificate.asdict(), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 6e71c9902..31c342ed6 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -26,6 +26,7 @@ from allmydata.util import ( from attrs import ( define, frozen, + asdict, Factory, ) @@ -49,6 +50,9 @@ class SignedCertificate(object): signature=data["signature"].encode("ascii") ) + def asdict(self): + return asdict(self) + @frozen class _GridManagerStorageServer(object): From 82045b4298ea8eb4516b9e0639f6c856d21509f1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 14:19:44 -0700 Subject: [PATCH 256/272] store signature as raw bytes, not base32 --- src/allmydata/cli/grid_manager.py | 2 +- src/allmydata/grid_manager.py | 19 +++++++++++++------ src/allmydata/test/test_grid_manager.py | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 220f091cd..433e30434 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -196,7 +196,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(certificate.asdict(), indent=4) + certificate_data = json.dumps(certificate.marshal(), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 31c342ed6..a17a1fab5 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -39,7 +39,8 @@ class SignedCertificate(object): # A JSON-encoded, UTF-8-encoded certificate. certificate : bytes - # The signature in base32. + # The signature (although the signature is in base32 in "public", + # this contains the decoded raw bytes -- not base32) signature : bytes @classmethod @@ -47,11 +48,17 @@ class SignedCertificate(object): data = json.load(file_like) return cls( certificate=data["certificate"].encode("utf-8"), - signature=data["signature"].encode("ascii") + signature=base32.a2b(data["signature"].encode("ascii")), ) - def asdict(self): - return asdict(self) + def marshal(self): + """ + :return dict: a json-able dict + """ + return dict( + certificate=self.certificate, + signature=base32.b2a(self.signature), + ) @frozen @@ -261,7 +268,7 @@ class _GridManager(object): sig = ed25519.sign_data(self._private_key, cert_data) certificate = SignedCertificate( certificate=cert_data, - signature=base32.b2a(sig), + signature=sig, ) vk = ed25519.verifying_key_from_signing_key(self._private_key) ed25519.verify_signature(vk, sig, cert_data) @@ -388,7 +395,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): try: ed25519.verify_signature( gm_key, - base32.a2b(alleged_cert.signature), + alleged_cert.signature, alleged_cert.certificate, ) except ed25519.BadSignature: diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c282d1237..95395f12e 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -175,7 +175,7 @@ class GridManagerVerifier(SyncTestCase): self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert0.signature), + cert0.signature, cert0.certificate, ), None From d91bfcb1d2f95b8eb8714f679786df657d03426f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 14:38:43 -0700 Subject: [PATCH 257/272] clarify --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index a17a1fab5..319d87002 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -69,7 +69,7 @@ class _GridManagerStorageServer(object): name : str public_key : ed25519.Ed25519PublicKey - certificates : list = Factory(list) + certificates : list = Factory(list) # SignedCertificates def add_certificate(self, certificate): """ From 00ef4661a64062ee0e0cd5878c9201797bd0cc92 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 16:06:24 -0700 Subject: [PATCH 258/272] ISO dates, not seconds --- src/allmydata/cli/grid_manager.py | 3 ++- src/allmydata/grid_manager.py | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 433e30434..63094dfcd 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -24,6 +24,7 @@ from allmydata.grid_manager import ( create_grid_manager, save_grid_manager, load_grid_manager, + current_datetime_with_zone, ) from allmydata.util import jsonbytes as json @@ -167,7 +168,7 @@ def list(ctx): name, str(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) for cert in ctx.obj.grid_manager.storage_servers[name].certificates: - delta = datetime.utcnow() - cert.expires + delta = current_datetime_with_zone() - cert.expires click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) if delta.total_seconds() < 0: click.echo("valid until {} ({})".format(cert.expires, abbreviate_time(delta))) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 319d87002..7d1d12fcd 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -5,6 +5,7 @@ Functions and classes relating to the Grid Manager internal state import sys from datetime import ( datetime, + timezone, ) from typing import ( Optional, @@ -115,6 +116,14 @@ def create_grid_manager(): ) +def current_datetime_with_zone(): + """ + :returns: a timezone-aware datetime object representing the + current timestamp in UTC + """ + return datetime.now(timezone.utc) + + def _load_certificates_for(config_path: FilePath, name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: """ Load any existing certificates for the given storage-server. @@ -150,7 +159,7 @@ def _load_certificates_for(config_path: FilePath, name: str, gm_key=Optional[ed2 _GridManagerCertificate( filename=cert_path.path, index=cert_index, - expires=datetime.utcfromtimestamp(cert_data['expires']), + expires=datetime.fromisoformat(cert_data['expires']), public_key=ed25519.verifying_key_from_string(cert_data['public_key'].encode('ascii')), ) ) @@ -257,10 +266,9 @@ class _GridManager(object): raise KeyError( "No storage server named '{}'".format(name) ) - expiration = datetime.utcnow() + expiry - epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() + expiration = current_datetime_with_zone() + expiry cert_info = { - "expires": epoch_offset, + "expires": expiration.isoformat(), "public_key": srv.public_key_string(), "version": 1, } @@ -421,7 +429,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= certificates for. :param callable now_fn: a callable which returns the current UTC - timestamp (or datetime.utcnow if None). + timestamp (or current_datetime_with_zone() if None). :param callable bad_cert: a two-argument callable which is invoked when a certificate verification fails. The first argument is @@ -436,7 +444,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= expired) in `certs` signed by one of the keys in `keys`. """ - now_fn = datetime.utcnow if now_fn is None else now_fn + now_fn = current_datetime_with_zone if now_fn is None else now_fn valid_certs = [] # if we have zero grid-manager keys then everything is valid @@ -479,7 +487,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= """ now = now_fn() for cert in valid_certs: - expires = datetime.utcfromtimestamp(cert['expires']) + expires = datetime.fromisoformat(cert["expires"]) if cert['public_key'].encode("ascii") == public_key: if expires > now: # not-expired From 6ee5c7588059da0c7100c4e48b23858dfb58672b Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 16:13:11 -0700 Subject: [PATCH 259/272] comment -> ticket --- src/allmydata/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 2eee9ef9c..b98d7e0d8 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -859,11 +859,9 @@ class _Client(node.Node, pollmixin.PollMixin): grid_manager_certificates = self.config.get_grid_manager_certificates() announcement[u"grid-manager-certificates"] = grid_manager_certificates - # XXX we should probably verify that the certificates are - # valid and not expired, as that could be confusing for the - # storage-server operator -- but then we need the public key - # of the Grid Manager (should that go in the config too, - # then? How to handle multiple grid-managers?) + # Note: certificates are not verified for validity here, but + # that may be useful. See: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3977 for ic in self.introducer_clients: ic.publish("storage", announcement, self._node_private_key) From 70459cfbf1f3c6dd070d83f1d5cdb3e5ebfded44 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 16:47:58 -0700 Subject: [PATCH 260/272] use possibly-overriden stderr --- src/allmydata/scripts/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 6a24614c0..eceb9a55a 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -198,7 +198,7 @@ def add_grid_manager_cert(options): options['name'], cert_path.path, ) - print(msg, file=options.parent.parent.stderr) + print(msg, file=options.stderr) return 1 config.set_config("storage", "grid_management", "True") @@ -210,7 +210,7 @@ def add_grid_manager_cert(options): cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), - file=options.parent.parent.stderr) + file=options.stderr) return 0 From fb10e13c6842ea4e290a3b1c8d82a0b6d2a7e246 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 18:16:45 -0700 Subject: [PATCH 261/272] might want to use these at parse time --- src/allmydata/scripts/runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 973c18d95..d9fbc1b0a 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -165,6 +165,8 @@ def parse_or_exit(config, argv, stdout, stderr): :return: ``config``, after using it to parse the argument list. """ try: + config.stdout = stdout + config.stderr = stderr parse_options(argv[1:], config=config) except usage.error as e: # `parse_options` may have the side-effect of initializing a From 69a480dc082ec91dc0ab4590387fd720ab904267 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 18:44:09 -0700 Subject: [PATCH 262/272] rewrite test --- src/allmydata/test/test_grid_manager.py | 35 +++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 95395f12e..384c5636a 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -74,26 +74,33 @@ class GridManagerUtilities(SyncTestCase): """ An error is reported loading invalid certificate version """ - cert_path = self.mktemp() + gm_path = FilePath(self.mktemp()) + gm_path.makedirs() + config = { + "grid_manager_config_version": 0, + "private_key": "priv-v0-ub7knkkmkptqbsax4tznymwzc4nk5lynskwjsiubmnhcpd7lvlqa", + "storage_servers": { + "radia": { + "public_key": "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + } + } + } + with gm_path.child("config.json").open("wb") as f: + f.write(json.dumps_bytes(config)) + fake_cert = { "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" } - with open(cert_path, "wb") as f: + with gm_path.child("radia.cert.0").open("wb") as f: f.write(json.dumps_bytes(fake_cert)) - config_data = ( - "[grid_managers]\n" - "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" - "[grid_manager_certificates]\n" - "ding = {}\n".format(cert_path) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(gm_path) + self.assertIn( + "22", + str(ctx.exception), ) - config = config_from_string("/foo", "portnum", config_data, client_valid_config()) - self.assertEqual( - {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, - config.enumerate_section("grid_managers") - ) - certs = config.get_grid_manager_certificates() - self.assertEqual([fake_cert], certs) def test_load_certificates_unknown_key(self): """ From 0ae7da7352918101a1ee559dcc0ef72f57d4ecc8 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 23:20:28 -0700 Subject: [PATCH 263/272] prop up the fragile scaffolding --- src/allmydata/test/cli/test_admin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 9bf12471f..0570467a5 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -200,6 +200,8 @@ class AddCertificateCommand(SyncTestCase): "--filename", "-", ] ) + self.tahoe.subOptions.subOptions.stdin = self.tahoe.stdin + self.tahoe.subOptions.subOptions.stderr = self.tahoe.stderr rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 0) @@ -227,6 +229,8 @@ class AddCertificateCommand(SyncTestCase): "--filename", "-", ] ) + self.tahoe.subOptions.subOptions.stdin = self.tahoe.stdin + self.tahoe.subOptions.subOptions.stderr = self.tahoe.stderr rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 0) @@ -239,6 +243,8 @@ class AddCertificateCommand(SyncTestCase): "--filename", "-", ] ) + self.tahoe.subOptions.subOptions.stdin = self.tahoe.stdin + self.tahoe.subOptions.subOptions.stderr = self.tahoe.stderr rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 1) self.assertIn( From 6aff94dd8fd78e159d0d013674b9d73a3ee72aff Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Feb 2023 00:15:32 -0700 Subject: [PATCH 264/272] flake8, more frozen --- src/allmydata/cli/grid_manager.py | 1 - src/allmydata/grid_manager.py | 4 +--- src/allmydata/test/test_grid_manager.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 32234be0d..3110a072e 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -4,7 +4,6 @@ A CLI for configuring a grid manager. from typing import Optional from datetime import ( - datetime, timedelta, ) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 7d1d12fcd..e264734b2 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -25,9 +25,7 @@ from allmydata.util import ( ) from attrs import ( - define, frozen, - asdict, Factory, ) @@ -93,7 +91,7 @@ class _GridManagerStorageServer(object): } -@define +@frozen class _GridManagerCertificate(object): """ Represents a single certificate for a single storage-server diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 384c5636a..78280a168 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -22,7 +22,6 @@ from allmydata.crypto import ( ed25519, ) from allmydata.util import ( - base32, jsonbytes as json, ) from allmydata.grid_manager import ( From a6a2eb1c93d479bd936836a125a0d6ce5f4575a3 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 23 Feb 2023 15:37:46 -0700 Subject: [PATCH 265/272] export it too Co-authored-by: Jean-Paul Calderone --- src/allmydata/util/jsonbytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 1cc8c8ef0..4a1813275 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -116,4 +116,4 @@ loads = json.loads load = json.load -__all__ = ["dumps", "loads"] +__all__ = ["dumps", "loads", "load"] From 1587a71bbafcafdbbf83736f017e5f3ca8c84dd0 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:26:06 -0700 Subject: [PATCH 266/272] spelling Co-authored-by: Jean-Paul Calderone --- src/allmydata/util/configutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index b48e2c034..9e18e3716 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -167,7 +167,7 @@ class ValidConfiguration(object): def is_valid_item(self, section_name, item_name): """ - :return: True if the given section name, ite name pair is valid, False + :return: True if the given section name, item_name pair is valid, False otherwise. """ valid_items = self._static_valid_sections.get(section_name, ()) From 9f63441af65f030cabbd99c9b88eb65e1abcb990 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:31:39 -0700 Subject: [PATCH 267/272] types Co-authored-by: Jean-Paul Calderone --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index eceb9a55a..a59892617 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -113,7 +113,7 @@ class AddGridManagerCertOptions(BaseOptions): def getSynopsis(self): return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" - def postOptions(self): + def postOptions(self) -> None: if self['name'] is None: raise usage.UsageError( "Must provide --name option" From d55a4a1e651fea8867a0cf3955251d81ff226f3e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:32:16 -0700 Subject: [PATCH 268/272] whitespace Co-authored-by: Jean-Paul Calderone --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index a59892617..41921e2ee 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -123,7 +123,7 @@ class AddGridManagerCertOptions(BaseOptions): "Must provide --filename option" ) - data : Union [bytes, str] + data: str if self['filename'] == '-': print("reading certificate from stdin", file=self.parent.parent.stderr) data = self.parent.parent.stdin.read() From aed5061a96876bc2efc943c5f5ad60217e1ad9b1 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:33:53 -0700 Subject: [PATCH 269/272] wording Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index aac2bf20a..1c5acaa20 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -224,7 +224,8 @@ Enrolling a Client: Config -------------------------- You may instruct a Tahoe client to use only storage servers from given -Grid Managers. If there are no such keys, any servers are used. If +Grid Managers. If there are no such keys, any servers are used +(but see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3979). If there are one or more keys, the client will only upload to a storage server that has a valid certificate (from any of the keys). From b28ac6118b3e3329d627c23086db90a674803b3f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 28 Feb 2023 10:43:49 -0700 Subject: [PATCH 270/272] different way to say 'all items okay' --- src/allmydata/client.py | 5 ++--- src/allmydata/util/configutil.py | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b98d7e0d8..d9521d999 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -76,7 +76,8 @@ def _is_valid_section(section_name): """ return ( section_name.startswith("storageserver.plugins.") or - section_name.startswith("storageclient.plugins.") + section_name.startswith("storageclient.plugins.") or + section_name in ("grid_managers", "grid_manager_certificates") ) @@ -93,8 +94,6 @@ _client_config = configutil.ValidConfiguration( "shares.total", "storage.plugins", ), - "grid_managers": None, # means "any options valid" - "grid_manager_certificates": None, "storage": ( "debug_discard", "enabled", diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 9e18e3716..a82cb04a4 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -170,11 +170,8 @@ class ValidConfiguration(object): :return: True if the given section name, item_name pair is valid, False otherwise. """ - valid_items = self._static_valid_sections.get(section_name, ()) - if valid_items is None: - return True return ( - item_name in valid_items or + item_name in self._static_valid_sections.get(section_name, ()) or self._is_valid_item(section_name, item_name) ) From 5672a28350a453628c69b4d892e4079982756691 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 28 Feb 2023 10:43:55 -0700 Subject: [PATCH 271/272] more-specific error --- src/allmydata/util/configutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index a82cb04a4..74b0c714a 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -84,7 +84,7 @@ def write_config(tahoe_cfg, config): if platform.isWindows(): try: tahoe_cfg.remove() - except OSError: + except FileNotFoundError: pass tmp.moveTo(tahoe_cfg) From 8e7f2cd3ea79e0e59aa5c4b40e2ffa2745bd4433 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 28 Feb 2023 10:47:17 -0700 Subject: [PATCH 272/272] unused --- src/allmydata/scripts/admin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 41921e2ee..579505399 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -12,8 +12,6 @@ if PY2: from six import ensure_binary -from typing import Union - from twisted.python import usage from twisted.python.filepath import ( FilePath,