tahoe-lafs/src/allmydata/scripts/magic_folder_cli.py

from __future__ import print_function

import os
import urllib
from types import NoneType
from six.moves import cStringIO as StringIO
from datetime import datetime
import json


from twisted.python import usage

from allmydata.util.assertutil import precondition

from .common import BaseOptions, BasedirOptions, get_aliases
from .cli import MakeDirectoryOptions, LnOptions, CreateAliasOptions
import tahoe_mv
from allmydata.util.encodingutil import argv_to_abspath, argv_to_unicode, to_str, \
    quote_local_unicode_path
from allmydata.scripts.common_http import do_http, BadResponse
from allmydata.util import fileutil
from allmydata import uri
from allmydata.util.abbreviate import abbreviate_space, abbreviate_time
from allmydata.frontends.magic_folder import load_magic_folders
from allmydata.frontends.magic_folder import save_magic_folders
from allmydata.frontends.magic_folder import maybe_upgrade_magic_folders


INVITE_SEPARATOR = "+"

class CreateOptions(BasedirOptions):
    nickname = None  # NOTE: *not* the "name of this magic-folder"
    local_dir = None
    synopsis = "MAGIC_ALIAS: [NICKNAME LOCAL_DIR]"
    optParameters = [
        ("poll-interval", "p", "60", "How often to ask for updates"),
        ("name", "n", "default", "The name of this magic-folder"),
    ]
    description = (
        "Create a new magic-folder. If you specify NICKNAME and "
        "LOCAL_DIR, this client will also be invited and join "
        "using the given nickname. A new alias (see 'tahoe list-aliases') "
        "will be added with the master folder's writecap."
    )

    def parseArgs(self, alias, nickname=None, local_dir=None):
        BasedirOptions.parseArgs(self)
        alias = argv_to_unicode(alias)
        if not alias.endswith(u':'):
            raise usage.UsageError("An alias must end with a ':' character.")
        self.alias = alias[:-1]
        self.nickname = None if nickname is None else argv_to_unicode(nickname)
        try:
            if int(self['poll-interval']) <= 0:
                raise ValueError("should be positive")
        except ValueError:
            raise usage.UsageError(
                "--poll-interval must be a positive integer"
            )

        # Expand the path relative to the current directory of the CLI command, not the node.
        self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False)

        if self.nickname and not self.local_dir:
            raise usage.UsageError("If NICKNAME is specified then LOCAL_DIR must also be specified.")
        node_url_file = os.path.join(self['node-directory'], u"node.url")
        self['node-url'] = fileutil.read(node_url_file).strip()

def _delegate_options(source_options, target_options):
    target_options.aliases = get_aliases(source_options['node-directory'])
    target_options["node-url"] = source_options["node-url"]
    target_options["node-directory"] = source_options["node-directory"]
    target_options["name"] = source_options["name"]
    target_options.stdin = StringIO("")
    target_options.stdout = StringIO()
    target_options.stderr = StringIO()
    return target_options

def create(options):
    precondition(isinstance(options.alias, unicode), alias=options.alias)
    precondition(isinstance(options.nickname, (unicode, NoneType)), nickname=options.nickname)
    precondition(isinstance(options.local_dir, (unicode, NoneType)), local_dir=options.local_dir)

    # make sure we don't already have a magic-folder with this name before we create the alias
    maybe_upgrade_magic_folders(options["node-directory"])
    folders = load_magic_folders(options["node-directory"])
    if options['name'] in folders:
        print("Already have a magic-folder named '{}'".format(options['name']), file=options.stderr)
        return 1

    # create an alias; this basically just remembers the cap for the
    # master directory
    from allmydata.scripts import tahoe_add_alias
    create_alias_options = _delegate_options(options, CreateAliasOptions())
    create_alias_options.alias = options.alias

    rc = tahoe_add_alias.create_alias(create_alias_options)
    if rc != 0:
        print(create_alias_options.stderr.getvalue(), file=options.stderr)
        return rc
    print(create_alias_options.stdout.getvalue(), file=options.stdout)

    if options.nickname is not None:
        print(u"Inviting myself as client '{}':".format(options.nickname), file=options.stdout)
        invite_options = _delegate_options(options, InviteOptions())
        invite_options.alias = options.alias
        invite_options.nickname = options.nickname
        invite_options['name'] = options['name']
        rc = invite(invite_options)
        if rc != 0:
            print(u"magic-folder: failed to invite after create\n", file=options.stderr)
            print(invite_options.stderr.getvalue(), file=options.stderr)
            return rc
        invite_code = invite_options.stdout.getvalue().strip()
        print(u"  created invite code", file=options.stdout)
        join_options = _delegate_options(options, JoinOptions())
        join_options['poll-interval'] = options['poll-interval']
        join_options.nickname = options.nickname
        join_options.local_dir = options.local_dir
        join_options.invite_code = invite_code
        rc = join(join_options)
        if rc != 0:
            print(u"magic-folder: failed to join after create\n", file=options.stderr)
            print(join_options.stderr.getvalue(), file=options.stderr)
            return rc
        print(u"  joined new magic-folder", file=options.stdout)
        print(
            u"Successfully created magic-folder '{}' with alias '{}:' "
            u"and client '{}'\nYou must re-start your node before the "
            u"magic-folder will be active."
        .format(options['name'], options.alias, options.nickname), file=options.stdout)
    return 0


class ListOptions(BasedirOptions):
    description = (
        "List all magic-folders this client has joined"
    )
    optFlags = [
        ("json", "", "Produce JSON output")
    ]


def list_(options):
    folders = load_magic_folders(options["node-directory"])
    if options["json"]:
        _list_json(options, folders)
        return 0
    _list_human(options, folders)
    return 0


def _list_json(options, folders):
    """
    List our magic-folders using JSON
    """
    info = dict()
    for name, details in folders.items():
        info[name] = {
            u"directory": details["directory"],
        }
    print(json.dumps(info), file=options.stdout)
    return 0


def _list_human(options, folders):
    """
    List our magic-folders for a human user
    """
    if folders:
        print("This client has the following magic-folders:", file=options.stdout)
        biggest = max([len(nm) for nm in folders.keys()])
        fmt = "  {:>%d}: {}" % (biggest, )
        for name, details in folders.items():
            print(fmt.format(name, details["directory"]), file=options.stdout)
    else:
        print("No magic-folders", file=options.stdout)


class InviteOptions(BasedirOptions):
    nickname = None
    synopsis = "MAGIC_ALIAS: NICKNAME"
    stdin = StringIO("")
    optParameters = [
        ("name", "n", "default", "The name of this magic-folder"),
    ]
    description = (
        "Invite a new participant to a given magic-folder. The resulting "
        "invite-code that is printed is secret information and MUST be "
        "transmitted securely to the invitee."
    )

    def parseArgs(self, alias, nickname=None):
        BasedirOptions.parseArgs(self)
        alias = argv_to_unicode(alias)
        if not alias.endswith(u':'):
            raise usage.UsageError("An alias must end with a ':' character.")
        self.alias = alias[:-1]
        self.nickname = argv_to_unicode(nickname)
        node_url_file = os.path.join(self['node-directory'], u"node.url")
        self['node-url'] = open(node_url_file, "r").read().strip()
        aliases = get_aliases(self['node-directory'])
        self.aliases = aliases


def invite(options):
    precondition(isinstance(options.alias, unicode), alias=options.alias)
    precondition(isinstance(options.nickname, unicode), nickname=options.nickname)

    from allmydata.scripts import tahoe_mkdir
    mkdir_options = _delegate_options(options, MakeDirectoryOptions())
    mkdir_options.where = None

    rc = tahoe_mkdir.mkdir(mkdir_options)
    if rc != 0:
        print("magic-folder: failed to mkdir\n", file=options.stderr)
        return rc

    # FIXME this assumes caps are ASCII.
    dmd_write_cap = mkdir_options.stdout.getvalue().strip()
    dmd_readonly_cap = uri.from_string(dmd_write_cap).get_readonly().to_string()
    if dmd_readonly_cap is None:
        print("magic-folder: failed to diminish dmd write cap\n", file=options.stderr)
        return 1

    magic_write_cap = get_aliases(options["node-directory"])[options.alias]
    magic_readonly_cap = uri.from_string(magic_write_cap).get_readonly().to_string()

    # tahoe ln CLIENT_READCAP COLLECTIVE_WRITECAP/NICKNAME
    ln_options = _delegate_options(options, LnOptions())
    ln_options.from_file = unicode(dmd_readonly_cap, 'utf-8')
    ln_options.to_file = u"%s/%s" % (unicode(magic_write_cap, 'utf-8'), options.nickname)
    rc = tahoe_mv.mv(ln_options, mode="link")
    if rc != 0:
        print("magic-folder: failed to create link\n", file=options.stderr)
        print(ln_options.stderr.getvalue(), file=options.stderr)
        return rc

    # FIXME: this assumes caps are ASCII.
    print("%s%s%s" % (magic_readonly_cap, INVITE_SEPARATOR, dmd_write_cap), file=options.stdout)
    return 0

class JoinOptions(BasedirOptions):
    synopsis = "INVITE_CODE LOCAL_DIR"
    dmd_write_cap = ""
    magic_readonly_cap = ""
    optParameters = [
        ("poll-interval", "p", "60", "How often to ask for updates"),
        ("name", "n", "default", "Name of the magic-folder"),
    ]

    def parseArgs(self, invite_code, local_dir):
        BasedirOptions.parseArgs(self)

        try:
            if int(self['poll-interval']) <= 0:
                raise ValueError("should be positive")
        except ValueError:
            raise usage.UsageError(
                "--poll-interval must be a positive integer"
            )
        # Expand the path relative to the current directory of the CLI command, not the node.
        self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False)
        self.invite_code = to_str(argv_to_unicode(invite_code))

def join(options):
    fields = options.invite_code.split(INVITE_SEPARATOR)
    if len(fields) != 2:
        raise usage.UsageError("Invalid invite code.")
    magic_readonly_cap, dmd_write_cap = fields

    maybe_upgrade_magic_folders(options["node-directory"])
    existing_folders = load_magic_folders(options["node-directory"])

    if options['name'] in existing_folders:
        print("This client already has a magic-folder named '{}'".format(options['name']), file=options.stderr)
        return 1

    db_fname = os.path.join(
        options["node-directory"],
        u"private",
        u"magicfolder_{}.sqlite".format(options['name']),
    )
    if os.path.exists(db_fname):
        print("Database '{}' already exists; not overwriting".format(db_fname), file=options.stderr)
        return 1

    folder = {
        u"directory": options.local_dir.encode('utf-8'),
        u"collective_dircap": magic_readonly_cap,
        u"upload_dircap": dmd_write_cap,
        u"poll_interval": options["poll-interval"],
    }
    existing_folders[options["name"]] = folder

    save_magic_folders(options["node-directory"], existing_folders)
    return 0


class LeaveOptions(BasedirOptions):
    synopsis = "Remove a magic-folder and forget all state"
    optParameters = [
        ("name", "n", "default", "Name of magic-folder to leave"),
    ]


def leave(options):
    from ConfigParser import SafeConfigParser

    existing_folders = load_magic_folders(options["node-directory"])

    if not existing_folders:
        print("No magic-folders at all", file=options.stderr)
        return 1

    if options["name"] not in existing_folders:
        print("No such magic-folder '{}'".format(options["name"]), file=options.stderr)
        return 1

    privdir = os.path.join(options["node-directory"], u"private")
    db_fname = os.path.join(privdir, u"magicfolder_{}.sqlite".format(options["name"]))

    # delete from YAML file and re-write it
    del existing_folders[options["name"]]
    save_magic_folders(options["node-directory"], existing_folders)

    # delete the database file
    try:
        fileutil.remove(db_fname)
    except Exception as e:
        print("Warning: unable to remove %s due to %s: %s"
            % (quote_local_unicode_path(db_fname), e.__class__.__name__, str(e)), file=options.stderr)

    # if this was the last magic-folder, disable them entirely
    if not existing_folders:
        parser = SafeConfigParser()
        parser.read(os.path.join(options["node-directory"], u"tahoe.cfg"))
        parser.remove_section("magic_folder")
        with open(os.path.join(options["node-directory"], u"tahoe.cfg"), "w") as f:
            parser.write(f)

    return 0


class StatusOptions(BasedirOptions):
    synopsis = ""
    stdin = StringIO("")
    optParameters = [
        ("name", "n", "default", "Name for the magic-folder to show status"),
    ]

    def parseArgs(self):
        BasedirOptions.parseArgs(self)
        node_url_file = os.path.join(self['node-directory'], u"node.url")
        with open(node_url_file, "r") as f:
            self['node-url'] = f.read().strip()


def _get_json_for_fragment(options, fragment, method='GET', post_args=None):
    nodeurl = options['node-url']
    if nodeurl.endswith('/'):
        nodeurl = nodeurl[:-1]

    url = u'%s/%s' % (nodeurl, fragment)
    if method == 'POST':
        if post_args is None:
            raise ValueError("Must pass post_args= for POST method")
        body = urllib.urlencode(post_args)
    else:
        body = ''
        if post_args is not None:
            raise ValueError("post_args= only valid for POST method")
    resp = do_http(method, url, body=body)
    if isinstance(resp, BadResponse):
        # specifically NOT using format_http_error() here because the
        # URL is pretty sensitive (we're doing /uri/<key>).
        raise RuntimeError(
            "Failed to get json from '%s': %s" % (nodeurl, resp.error)
        )

    data = resp.read()
    parsed = json.loads(data)
    if parsed is None:
        raise RuntimeError("No data from '%s'" % (nodeurl,))
    return parsed


def _get_json_for_cap(options, cap):
    return _get_json_for_fragment(
        options,
        'uri/%s?t=json' % urllib.quote(cap),
    )

def _print_item_status(item, now, longest):
    paddedname = (' ' * (longest - len(item['path']))) + item['path']
    if 'failure_at' in item:
        ts = datetime.fromtimestamp(item['started_at'])
        prog = 'Failed %s (%s)' % (abbreviate_time(now - ts), ts)
    elif item['percent_done'] < 100.0:
        if 'started_at' not in item:
            prog = 'not yet started'
        else:
            so_far = now - datetime.fromtimestamp(item['started_at'])
            if so_far.seconds > 0.0:
                rate = item['percent_done'] / so_far.seconds
                if rate != 0:
                    time_left = (100.0 - item['percent_done']) / rate
                    prog = '%2.1f%% done, around %s left' % (
                        item['percent_done'],
                        abbreviate_time(time_left),
                    )
                else:
                    time_left = None
                    prog = '%2.1f%% done' % (item['percent_done'],)
            else:
                prog = 'just started'
    else:
        prog = ''
        for verb in ['finished', 'started', 'queued']:
            keyname = verb + '_at'
            if keyname in item:
                when = datetime.fromtimestamp(item[keyname])
                prog = '%s %s' % (verb, abbreviate_time(now - when))
                break

    print("  %s: %s" % (paddedname, prog))


def status(options):
    nodedir = options["node-directory"]
    stdout, stderr = options.stdout, options.stderr
    magic_folders = load_magic_folders(os.path.join(options["node-directory"]))

    with open(os.path.join(nodedir, u'private', u'api_auth_token'), 'rb') as f:
        token = f.read()

    print("Magic-folder status for '{}':".format(options["name"]), file=stdout)

    if options["name"] not in magic_folders:
        raise Exception(
            "No such magic-folder '{}'".format(options["name"])
        )

    dmd_cap = magic_folders[options["name"]]["upload_dircap"]
    collective_readcap = magic_folders[options["name"]]["collective_dircap"]

    # do *all* our data-retrievals first in case there's an error
    try:
        dmd_data = _get_json_for_cap(options, dmd_cap)
        remote_data = _get_json_for_cap(options, collective_readcap)
        magic_data = _get_json_for_fragment(
            options,
            'magic_folder?t=json',
            method='POST',
            post_args=dict(
                t='json',
                name=options["name"],
                token=token,
            )
        )
    except Exception as e:
        print("failed to retrieve data: %s" % str(e), file=stderr)
        return 2

    for d in [dmd_data, remote_data, magic_data]:
        if isinstance(d, dict) and 'error' in d:
            print("Error from server: %s" % d['error'], file=stderr)
            print("This means we can't retrieve the remote shared directory.", file=stderr)
            return 3

    captype, dmd = dmd_data
    if captype != 'dirnode':
        print("magic_folder_dircap isn't a directory capability", file=stderr)
        return 2

    now = datetime.now()

    print("Local files:", file=stdout)
    for (name, child) in dmd['children'].items():
        captype, meta = child
        status = 'good'
        size = meta['size']
        created = datetime.fromtimestamp(meta['metadata']['tahoe']['linkcrtime'])
        version = meta['metadata']['version']
        nice_size = abbreviate_space(size)
        nice_created = abbreviate_time(now - created)
        if captype != 'filenode':
            print("%20s: error, should be a filecap" % name, file=stdout)
            continue
        print("  %s (%s): %s, version=%s, created %s" % (name, nice_size, status, version, nice_created), file=stdout)

    print(file=stdout)
    print("Remote files:", file=stdout)

    captype, collective = remote_data
    for (name, data) in collective['children'].items():
        if data[0] != 'dirnode':
            print("Error: '%s': expected a dirnode, not '%s'" % (name, data[0]), file=stdout)
        print("  %s's remote:" % name, file=stdout)
        dmd = _get_json_for_cap(options, data[1]['ro_uri'])
        if isinstance(dmd, dict) and 'error' in dmd:
            print("    Error: could not retrieve directory", file=stdout)
            continue
        if dmd[0] != 'dirnode':
            print("Error: should be a dirnode", file=stdout)
            continue
        for (n, d) in dmd[1]['children'].items():
            if d[0] != 'filenode':
                print("Error: expected '%s' to be a filenode." % (n,), file=stdout)

            meta = d[1]
            status = 'good'
            size = meta['size']
            created = datetime.fromtimestamp(meta['metadata']['tahoe']['linkcrtime'])
            version = meta['metadata']['version']
            nice_size = abbreviate_space(size)
            nice_created = abbreviate_time(now - created)
            print("    %s (%s): %s, version=%s, created %s" % (n, nice_size, status, version, nice_created), file=stdout)

    if len(magic_data):
        uploads = [item for item in magic_data if item['kind'] == 'upload']
        downloads = [item for item in magic_data if item['kind'] == 'download']
        longest = max([len(item['path']) for item in magic_data])

        # maybe gate this with --show-completed option or something?
        uploads = [item for item in uploads if item['status'] != 'success']
        downloads = [item for item in downloads if item['status'] != 'success']

        if len(uploads):
            print()
            print("Uploads:", file=stdout)
            for item in uploads:
                _print_item_status(item, now, longest)

        if len(downloads):
            print()
            print("Downloads:", file=stdout)
            for item in downloads:
                _print_item_status(item, now, longest)

        for item in magic_data:
            if item['status'] == 'failure':
                print("Failed:", item, file=stdout)

    return 0


class MagicFolderCommand(BaseOptions):
    subCommands = [
        ["create", None, CreateOptions, "Create a Magic Folder."],
        ["invite", None, InviteOptions, "Invite someone to a Magic Folder."],
        ["join", None, JoinOptions, "Join a Magic Folder."],
        ["leave", None, LeaveOptions, "Leave a Magic Folder."],
        ["status", None, StatusOptions, "Display status of uploads/downloads."],
        ["list", None, ListOptions, "List Magic Folders configured in this client."],
    ]
    optFlags = [
        ["debug", "d", "Print full stack-traces"],
    ]
    description = (
        "A magic-folder has an owner who controls the writecap "
        "containing a list of nicknames and readcaps. The owner can invite "
        "new participants. Every participant has the writecap for their "
        "own folder (the corresponding readcap is in the master folder). "
        "All clients download files from all other participants using the "
        "readcaps contained in the master magic-folder directory."
    )

    def postOptions(self):
        if not hasattr(self, 'subOptions'):
            raise usage.UsageError("must specify a subcommand")
    def getSynopsis(self):
        return "Usage: tahoe [global-options] magic-folder"
    def getUsage(self, width=None):
        t = BaseOptions.getUsage(self, width)
        t += (
            "Please run e.g. 'tahoe magic-folder create --help' for more "
            "details on each subcommand.\n"
        )
        return t

subDispatch = {
    "create": create,
    "invite": invite,
    "join": join,
    "leave": leave,
    "status": status,
    "list": list_,
}

def do_magic_folder(options):
    so = options.subOptions
    so.stdout = options.stdout
    so.stderr = options.stderr
    f = subDispatch[options.subCommand]
    try:
        return f(so)
    except Exception as e:
        print("Error: %s" % (e,), file=options.stderr)
        if options['debug']:
            raise

subCommands = [
    ["magic-folder", None, MagicFolderCommand,
     "Magic Folder subcommands: use 'tahoe magic-folder' for a list."],
]

dispatch = {
    "magic-folder": do_magic_folder,
}