grid-manager stand-alone, via Click

This commit is contained in:
meejah 2020-11-06 22:23:53 -07:00
parent c7f4a1a157
commit 2118a2446e
5 changed files with 430 additions and 498 deletions

View File

@ -410,6 +410,11 @@ setup(name="tahoe-lafs", # also set in __init__.py
}, },
include_package_data=True, include_package_data=True,
setup_requires=setup_requires, setup_requires=setup_requires,
entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, entry_points={
'console_scripts': [
'tahoe = allmydata.scripts.runner:run',
'grid-manager = allmydata.cli.grid_manager:grid_manager',
]
},
**setup_args **setup_args
) )

View File

View File

@ -0,0 +1,207 @@
from datetime import (
datetime,
)
import json
import click
from twisted.python.filepath import (
FilePath,
)
from allmydata.crypto import (
ed25519,
)
from allmydata.util.abbreviate import (
abbreviate_time,
)
from allmydata.grid_manager import (
create_grid_manager,
save_grid_manager,
load_grid_manager,
)
@click.group()
@click.option(
'--config', '-c',
type=click.Path(),
help="Configuration directory (or - for stdin)",
required=True,
)
@click.pass_context
def grid_manager(ctx, config):
"""
A Tahoe Grid Manager issues certificates to storage-servers
A Tahoe client with one or more Grid Manager public keys
configured will only upload to a Storage Server that presents a
valid certificate signed by one of the configured Grid
Manager keys.
Grid Manager configuration can be in a local directory or given
via stdin. It contains long-term secret information (a private
signing key) and should be kept safe.
"""
class Config(object):
"""
Availble to all sub-commands as Click's context.obj
"""
_grid_manager = None
@property
def grid_manager(self):
if self._grid_manager is None:
config_path = _config_path_from_option(config)
self._grid_manager = load_grid_manager(config_path, config)
return self._grid_manager
ctx.obj = Config()
@grid_manager.command()
@click.pass_context
def create(ctx):
"""
Make a new Grid Manager
"""
config_location = ctx.parent.params["config"]
fp = None
if config_location != '-':
fp = FilePath(config_location)
if fp.exists():
raise click.ClickException(
"The directory '{}' already exists.".format(config_location)
)
gm = create_grid_manager()
save_grid_manager(fp, gm)
@grid_manager.command()
@click.pass_obj
def public_identity(config):
"""
Show the public identity key of a Grid Manager
This is what you give to clients to add to their configuration so
they use announcements from this Grid Manager
"""
click.echo(config.grid_manager.public_identity())
@grid_manager.command()
@click.argument("name")
@click.argument("public_key", type=click.UNPROCESSED)
@click.pass_context
def add(ctx, name, public_key):
"""
Add a new storage-server by name to a Grid Manager
PUBLIC_KEY is the contents of a node.pubkey file from a Tahoe
node-directory. NAME is an arbitrary label.
"""
try:
ctx.obj.grid_manager.add_storage_server(
name,
ed25519.verifying_key_from_string(public_key),
)
except KeyError:
raise click.ClickException(
"A storage-server called '{}' already exists".format(name)
)
save_grid_manager(
_config_path_from_option(ctx.parent.params["config"]),
ctx.obj.grid_manager,
)
return 0
@grid_manager.command()
@click.argument("name")
@click.pass_context
def remove(ctx, name):
"""
Remove an existing storage-server by name from a Grid Manager
"""
fp = _config_path_from_option(ctx.parent.params["config"])
try:
ctx.obj.grid_manager.remove_storage_server(name)
except KeyError:
raise click.ClickException(
"No storage-server called '{}' exists".format(name)
)
cert_count = 0
if fp is not None:
while fp.child('{}.cert.{}'.format(name, cert_count)).exists():
fp.child('{}.cert.{}'.format(name, cert_count)).remove()
cert_count += 1
save_grid_manager(fp, ctx.obj.grid_manager)
@grid_manager.command()
@click.pass_context
def list(ctx):
"""
List all storage-servers known to a Grid Manager
"""
fp = _config_path_from_option(ctx.parent.params["config"])
for name in sorted(ctx.obj.grid_manager.storage_servers.keys()):
blank_name = " " * len(name)
click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key()))
if fp:
cert_count = 0
while fp.child('{}.cert.{}'.format(name, cert_count)).exists():
container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r'))
cert_data = json.loads(container['certificate'])
expires = datetime.utcfromtimestamp(cert_data['expires'])
delta = datetime.utcnow() - expires
click.echo("{} cert {}: ".format(blank_name, cert_count), nl=False)
if delta.total_seconds() < 0:
click.echo("valid until {} ({})".format(expires, abbreviate_time(delta)))
else:
click.echo("expired {} ({})".format(expires, abbreviate_time(delta)))
cert_count += 1
@grid_manager.command()
@click.argument("name")
@click.argument(
"expiry_days",
type=click.IntRange(1, 5*365), # XXX is 5 years a good maximum?
)
@click.pass_context
def sign(ctx, name, expiry_days):
"""
sign a new certificate
"""
fp = _config_path_from_option(ctx.parent.params["config"])
expiry_seconds = int(expiry_days) * 86400
try:
certificate = ctx.obj.grid_manager.sign(name, expiry_seconds)
except KeyError:
raise click.ClickException(
"No storage-server called '{}' exists".format(name)
)
certificate_data = json.dumps(certificate, indent=4)
click.echo(certificate_data)
if fp is not None:
next_serial = 0
while fp.child("{}.cert.{}".format(name, next_serial)).exists():
next_serial += 1
with fp.child('{}.cert.{}'.format(name, next_serial)).open('w') as f:
f.write(certificate_data)
def _config_path_from_option(config):
"""
:param string config: a path or -
:returns: a FilePath instance or None
"""
if config == "-":
return None
return FilePath(config)

View File

@ -0,0 +1,217 @@
import sys
import json
from datetime import (
datetime,
timedelta,
)
from allmydata.crypto import (
ed25519,
)
from allmydata.util import (
fileutil,
base32,
)
class _GridManagerStorageServer(object):
"""
A Grid Manager's notion of a storage server
"""
def __init__(self, name, public_key, certificates):
self.name = name
self._public_key = public_key
self._certificates = [] if certificates is None else certificates
def add_certificate(self, certificate):
self._certificates.append(certificate)
def public_key(self):
return ed25519.string_from_verifying_key(self._public_key)
def marshal(self):
return {
u"public_key": self.public_key(),
}
def create_grid_manager():
"""
Create a new Grid Manager with a fresh keypair
"""
private_key, public_key = ed25519.create_signing_keypair()
return _GridManager(
ed25519.string_from_signing_key(private_key),
{},
)
def load_grid_manager(config_path, config_location):
"""
Load a Grid Manager from existing configuration.
:param FilePath config_path: the configuratino location (or None for
stdin)
:param str config_location: a string describing the config's location
:returns: a GridManager instance
"""
if config_path is None:
config_file = sys.stdin
else:
try:
config_file = config_path.child("config.json").open("r")
except IOError:
raise ValueError(
"'{}' is not a Grid Manager config-directory".format(config)
)
with config_file:
config = json.load(config_file)
if not config:
raise ValueError(
"Invalid Grid Manager config in '{}'".format(config_location)
)
if 'private_key' not in config:
raise ValueError(
"Grid Manager config from '{}' requires a 'private_key'".format(
config_location,
)
)
private_key_bytes = config['private_key'].encode('ascii')
try:
private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes)
except Exception as e:
raise ValueError(
"Invalid Grid Manager private_key: {}".format(e)
)
storage_servers = dict()
for name, srv_config in config.get(u'storage_servers', {}).items():
if not 'public_key' in srv_config:
raise ValueError(
"No 'public_key' for storage server '{}'".format(name)
)
storage_servers[name] = _GridManagerStorageServer(
name,
ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')),
None,
)
gm_version = config.get(u'grid_manager_config_version', None)
if gm_version != 0:
raise ValueError(
"Missing or unknown version '{}' of Grid Manager config".format(
gm_version
)
)
return _GridManager(private_key_bytes, storage_servers)
class _GridManager(object):
"""
A Grid Manager's configuration.
"""
def __init__(self, private_key_bytes, storage_servers):
self._storage_servers = dict() if storage_servers is None else storage_servers
self._private_key_bytes = private_key_bytes
self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes)
self._version = 0
@property
def storage_servers(self):
return self._storage_servers
def public_identity(self):
return ed25519.string_from_verifying_key(self._public_key)
def sign(self, name, expiry_seconds):
try:
srv = self._storage_servers[name]
except KeyError:
raise KeyError(
u"No storage server named '{}'".format(name)
)
expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds)
epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds()
cert_info = {
"expires": epoch_offset,
"public_key": srv.public_key(),
"version": 1,
}
cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8')
sig = ed25519.sign_data(self._private_key, cert_data)
certificate = {
u"certificate": cert_data,
u"signature": base32.b2a(sig),
}
vk = ed25519.verifying_key_from_signing_key(self._private_key)
ed25519.verify_signature(vk, sig, cert_data)
return certificate
def add_storage_server(self, name, public_key):
"""
:param name: a user-meaningful name for the server
:param public_key: ed25519.VerifyingKey the public-key of the
storage provider (e.g. from the contents of node.pubkey
for the client)
"""
if name in self._storage_servers:
raise KeyError(
"Already have a storage server called '{}'".format(name)
)
ss = _GridManagerStorageServer(name, public_key, None)
self._storage_servers[name] = ss
return ss
def remove_storage_server(self, name):
"""
:param name: a user-meaningful name for the server
"""
try:
del self._storage_servers[name]
except KeyError:
raise KeyError(
"No storage server called '{}'".format(name)
)
def marshal(self):
data = {
u"grid_manager_config_version": self._version,
u"private_key": self._private_key_bytes.decode('ascii'),
}
if self._storage_servers:
data[u"storage_servers"] = {
name: srv.marshal()
for name, srv
in self._storage_servers.items()
}
return data
def save_grid_manager(file_path, grid_manager):
"""
Writes a Grid Manager configuration.
:param file_path: a FilePath specifying where to write the config
(if None, stdout is used)
:param grid_manager: a _GridManager instance
"""
data = json.dumps(
grid_manager.marshal(),
indent=4,
)
if file_path is None:
print("{}\n".format(data))
else:
fileutil.make_dirs(file_path.path, mode=0o700)
with file_path.child("config.json").open("w") as f:
f.write("{}\n".format(data))

View File

@ -1,497 +0,0 @@
from __future__ import print_function
import sys
import json
from datetime import datetime, timedelta
from allmydata.scripts.common import BaseOptions
from allmydata.util.abbreviate import abbreviate_time
from twisted.python import usage
from twisted.python.filepath import FilePath
from allmydata.util import fileutil
from allmydata.util import base32
from allmydata.crypto import ed25519
from twisted.internet.defer import inlineCallbacks, returnValue
class CreateOptions(BaseOptions):
description = (
"Create a new identity key and configuration of a Grid Manager"
)
class ShowIdentityOptions(BaseOptions):
description = (
"Show the public identity key of a Grid Manager\n"
"\n"
"This is what you give to clients to add to their configuration"
" so they use announcements from this Grid Manager"
)
class AddOptions(BaseOptions):
description = (
"Add a new storage-server's key to a Grid Manager configuration\n"
"using NAME and PUBIC_KEY (comes from a node.pubkey file)"
)
def getSynopsis(self):
return "{} add NAME PUBLIC_KEY".format(super(AddOptions, self).getSynopsis())
def parseArgs(self, *args, **kw):
BaseOptions.parseArgs(self, **kw)
if len(args) != 2:
raise usage.UsageError(
"Requires two arguments: name public_key"
)
self['name'] = unicode(args[0])
try:
# WTF?! why does it want 'str' and not six.text_type?
self['storage_public_key'] = ed25519.verifying_key_from_string(args[1])
except Exception as e:
raise usage.UsageError(
"Invalid public_key argument: {}".format(e)
)
class RemoveOptions(BaseOptions):
description = (
"Remove a storage-server from a Grid Manager configuration"
)
def parseArgs(self, *args, **kw):
BaseOptions.parseArgs(self, **kw)
if len(args) != 1:
raise usage.UsageError(
"Requires one arguments: name"
)
self['name'] = unicode(args[0])
class ListOptions(BaseOptions):
description = (
"List all storage servers in this Grid Manager"
)
class SignOptions(BaseOptions):
description = (
"Create and sign a new certificate for a storage-server"
)
def getSynopsis(self):
return "{} NAME EXPIRY_DAYS".format(super(SignOptions, self).getSynopsis())
def parseArgs(self, *args, **kw):
BaseOptions.parseArgs(self, **kw)
if len(args) != 2:
raise usage.UsageError(
"Requires two arguments: name expiry_days"
)
self['name'] = unicode(args[0])
self['expiry_days'] = int(args[1])
if self['expiry_days'] < 1 or self['expiry_days'] > 20*365:
raise usage.UsageError(
"Certificate expires in an unreasonable number of days: {}".format(
self['expiry_days'],
)
)
class GridManagerOptions(BaseOptions):
subCommands = [
["create", None, CreateOptions, "Create a Grid Manager."],
["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."],
["add", None, AddOptions, "Add a storage server to this Grid Manager."],
["remove", None, RemoveOptions, "Remove a storage server from this Grid Manager."],
["list", None, ListOptions, "List all storage servers in this Grid Manager."],
["sign", None, SignOptions, "Create and sign a new Storage Certificate."],
]
optParameters = [
("config", "c", None, "How to find the Grid Manager's configuration")
]
def postOptions(self):
if not hasattr(self, 'subOptions'):
raise usage.UsageError("must specify a subcommand")
if self['config'] is None:
raise usage.UsageError("Must supply configuration with --config")
description = (
'A "grid-manager" consists of some data defining a keypair (along with '
'some other details) and Tahoe sub-commands to manipulate the data and '
'produce certificates to give to storage-servers. Certificates assert '
'the statement: "Grid Manager X suggests you use storage-server Y to '
'upload shares to" (X and Y are public-keys).'
'\n\n'
'Clients can use Grid Managers to decide which storage servers to '
'upload shares to. They do this by adding one or more Grid Manager '
'public keys to their config.'
)
def _create_gridmanager():
"""
:return: an object providing the GridManager interface initialized
with a new random keypair
"""
private_key, public_key = ed25519.create_signing_keypair()
return _GridManager(
ed25519.string_from_signing_key(private_key),
{},
)
def _create(gridoptions, options):
"""
Create a new Grid Manager
"""
gm_config = gridoptions['config']
# pre-conditions check
fp = None
if gm_config.strip() != '-':
fp = FilePath(gm_config.strip())
if fp.exists():
raise usage.UsageError(
"The directory '{}' already exists.".format(gm_config)
)
gm = _create_gridmanager()
_save_gridmanager_config(fp, gm)
class _GridManagerStorageServer(object):
"""
A Grid Manager's notion of a storage server
"""
def __init__(self, name, public_key, certificates):
self.name = name
self._public_key = public_key
self._certificates = [] if certificates is None else certificates
def add_certificate(self, certificate):
self._certificates.append(certificate)
def public_key(self):
return ed25519.string_from_verifying_key(self._public_key)
def marshal(self):
return {
u"public_key": self.public_key(),
}
class _GridManager(object):
"""
A Grid Manager's configuration.
"""
@staticmethod
def from_config(config, config_location):
if not config:
raise ValueError(
"Invalid Grid Manager config in '{}'".format(config_location)
)
if 'private_key' not in config:
raise ValueError(
"Grid Manager config from '{}' requires a 'private_key'".format(
config_location,
)
)
private_key_bytes = config['private_key'].encode('ascii')
try:
private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes)
except Exception as e:
raise ValueError(
"Invalid Grid Manager private_key: {}".format(e)
)
storage_servers = dict()
for name, srv_config in config.get(u'storage_servers', {}).items():
if not 'public_key' in srv_config:
raise ValueError(
"No 'public_key' for storage server '{}'".format(name)
)
storage_servers[name] = _GridManagerStorageServer(
name,
ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')),
None,
)
gm_version = config.get(u'grid_manager_config_version', None)
if gm_version != 0:
raise ValueError(
"Missing or unknown version '{}' of Grid Manager config".format(
gm_version
)
)
return _GridManager(private_key_bytes, storage_servers)
def __init__(self, private_key_bytes, storage_servers):
self._storage_servers = dict() if storage_servers is None else storage_servers
self._private_key_bytes = private_key_bytes
self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes)
self._version = 0
@property
def storage_servers(self):
return self._storage_servers
def public_identity(self):
return ed25519.string_from_verifying_key(self._public_key)
def sign(self, name, expiry_seconds):
try:
srv = self._storage_servers[name]
except KeyError:
raise KeyError(
u"No storage server named '{}'".format(name)
)
expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds)
epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds()
cert_info = {
"expires": epoch_offset,
"public_key": srv.public_key(),
"version": 1,
}
cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8')
sig = ed25519.sign_data(self._private_key, cert_data)
certificate = {
u"certificate": cert_data,
u"signature": base32.b2a(sig),
}
vk = ed25519.verifying_key_from_signing_key(self._private_key)
ed25519.verify_signature(vk, sig, cert_data)
return certificate
def add_storage_server(self, name, public_key):
"""
:param name: a user-meaningful name for the server
:param public_key: ed25519.VerifyingKey the public-key of the
storage provider (e.g. from the contents of node.pubkey
for the client)
"""
if name in self._storage_servers:
raise KeyError(
"Already have a storage server called '{}'".format(name)
)
ss = _GridManagerStorageServer(name, public_key, None)
self._storage_servers[name] = ss
return ss
def remove_storage_server(self, name):
"""
:param name: a user-meaningful name for the server
"""
try:
del self._storage_servers[name]
except KeyError:
raise KeyError(
"No storage server called '{}'".format(name)
)
def marshal(self):
data = {
u"grid_manager_config_version": self._version,
u"private_key": self._private_key_bytes.decode('ascii'),
}
if self._storage_servers:
data[u"storage_servers"] = {
name: srv.marshal()
for name, srv
in self._storage_servers.items()
}
return data
def _save_gridmanager_config(file_path, grid_manager):
"""
Writes a Grid Manager configuration.
:param file_path: a FilePath specifying where to write the config
(if None, stdout is used)
:param grid_manager: a _GridManager instance
"""
data = json.dumps(
grid_manager.marshal(),
indent=4,
)
if file_path is None:
print("{}\n".format(data))
else:
fileutil.make_dirs(file_path.path, mode=0o700)
with file_path.child("config.json").open("w") as f:
f.write("{}\n".format(data))
def _load_gridmanager_config(fp):
"""
Loads a Grid Manager configuration and returns it (a dict) after
validating. Exceptions if the config can't be found, or has
problems.
:param FilePath fp: None for stdin or a path to a Grid Manager
configuration directory
"""
if fp is None:
gm = json.load(sys.stdin)
else:
with fp.child("config.json").open("r") as f:
gm = json.load(f)
try:
return _GridManager.from_config(gm, fp or "<stdin>")
except ValueError as e:
raise usage.UsageError(str(e))
def _show_identity(gridoptions, options):
"""
Output the public-key of a Grid Manager
"""
gm_config = gridoptions['config'].strip()
assert gm_config is not None
gm = _load_gridmanager_config(gm_config)
print(gm.public_identity())
def _add(gridoptions, options):
"""
Add a new storage-server by name to a Grid Manager
"""
gm_config = gridoptions['config'].strip()
fp = FilePath(gm_config) if gm_config.strip() != '-' else None
gm = _load_gridmanager_config(gm_config)
try:
gm.add_storage_server(
options['name'],
options['storage_public_key'],
)
except KeyError:
raise usage.UsageError(
"A storage-server called '{}' already exists".format(options['name'])
)
_save_gridmanager_config(fp, gm)
return 0
def _remove(gridoptions, options):
"""
Remove an existing storage-server by name from a Grid Manager
"""
gm_config = gridoptions['config'].strip()
fp = FilePath(gm_config) if gm_config.strip() != '-' else None
gm = _load_gridmanager_config(gm_config)
try:
gm.remove_storage_server(options['name'])
except KeyError:
raise usage.UsageError(
"No storage-server called '{}' exists".format(options['name'])
)
cert_count = 0
if fp is not None:
while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists():
fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove()
cert_count += 1
_save_gridmanager_config(fp, gm)
return 0
def _list(gridoptions, options):
"""
List all storage-servers known to a Grid Manager
"""
gm_config = gridoptions['config'].strip()
fp = FilePath(gm_config) if gm_config.strip() != '-' else None
gm = _load_gridmanager_config(gm_config)
for name in sorted(gm.storage_servers.keys()):
print("{}: {}".format(name, gm.storage_servers[name].public_key()))
if fp:
cert_count = 0
while fp.child('{}.cert.{}'.format(name, cert_count)).exists():
container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r'))
cert_data = json.loads(container['certificate'])
expires = datetime.utcfromtimestamp(cert_data['expires'])
delta = datetime.utcnow() - expires
if delta.total_seconds() < 0:
print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta)))
else:
print("{}: cert {}: expired ({})".format(name, cert_count, abbreviate_time(delta)))
cert_count += 1
def _sign(gridoptions, options):
"""
sign a new certificate
"""
gm_config = gridoptions['config'].strip()
fp = FilePath(gm_config) if gm_config.strip() != '-' else None
gm = _load_gridmanager_config(gm_config)
expiry_seconds = int(options['expiry_days']) * 86400
try:
certificate = gm.sign(options['name'], expiry_seconds)
except KeyError:
raise usage.UsageError(
"No storage-server called '{}' exists".format(options['name'])
)
certificate_data = json.dumps(certificate, indent=4)
print(certificate_data)
if fp is not None:
next_serial = 0
while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists():
next_serial += 1
with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f:
f.write(certificate_data)
grid_manager_commands = {
CreateOptions: _create,
ShowIdentityOptions: _show_identity,
AddOptions: _add,
RemoveOptions: _remove,
ListOptions: _list,
SignOptions: _sign,
}
@inlineCallbacks
def gridmanager(config):
"""
Runs the 'tahoe grid-manager' command.
"""
if config.subCommand is None:
print(config)
returnValue(1)
try:
f = grid_manager_commands[config.subOptions.__class__]
except KeyError:
print(config.subOptions, grid_manager_commands.keys())
print("Unknown command 'tahoe grid-manager {}': no such grid-manager subcommand".format(config.subCommand))
returnValue(2)
x = yield f(config, config.subOptions)
returnValue(x)
subCommands = [
["grid-manager", None, GridManagerOptions,
"Grid Manager subcommands: use 'tahoe grid-manager' for a list."],
]
dispatch = {
"grid-manager": gridmanager,
}