tahoe-lafs/integration/test_magic_folder.py

import sys
import time
import shutil
from os import mkdir, unlink, utime
from os.path import join, exists, getmtime

import util

import pytest_twisted


# tests converted from check_magicfolder_smoke.py
# see "conftest.py" for the fixtures (e.g. "magic_folder")

def test_eliot_logs_are_written(alice, bob, temp_dir):
    # The integration test configuration arranges for this logging
    # configuration.  Verify it actually does what we want.
    #
    # The alice and bob arguments looks unused but they actually tell pytest
    # to set up all the magic-folder stuff.  The assertions here are about
    # side-effects of that setup.
    assert exists(join(temp_dir, "alice", "logs", "eliot.json"))
    assert exists(join(temp_dir, "bob", "logs", "eliot.json"))


def test_alice_writes_bob_receives(magic_folder):
    alice_dir, bob_dir = magic_folder

    with open(join(alice_dir, "first_file"), "w") as f:
        f.write("alice wrote this")

    util.await_file_contents(join(bob_dir, "first_file"), "alice wrote this")
    return


def test_alice_writes_bob_receives_multiple(magic_folder):
    """
    When Alice does a series of updates, Bob should just receive them
    with no .backup or .conflict files being produced.
    """
    alice_dir, bob_dir = magic_folder

    unwanted_files = [
        join(bob_dir, "multiple.backup"),
        join(bob_dir, "multiple.conflict")
    ]

    # first update
    with open(join(alice_dir, "multiple"), "w") as f:
        f.write("alice wrote this")

    util.await_file_contents(
        join(bob_dir, "multiple"), "alice wrote this",
        error_if=unwanted_files,
    )

    # second update
    with open(join(alice_dir, "multiple"), "w") as f:
        f.write("someone changed their mind")

    util.await_file_contents(
        join(bob_dir, "multiple"), "someone changed their mind",
        error_if=unwanted_files,
    )

    # third update
    with open(join(alice_dir, "multiple"), "w") as f:
        f.write("absolutely final version ship it")

    util.await_file_contents(
        join(bob_dir, "multiple"), "absolutely final version ship it",
        error_if=unwanted_files,
    )

    # forth update, but both "at once" so one should conflict
    time.sleep(2)
    with open(join(alice_dir, "multiple"), "w") as f:
        f.write("okay one more attempt")
    with open(join(bob_dir, "multiple"), "w") as f:
        f.write("...but just let me add")

    bob_conflict = join(bob_dir, "multiple.conflict")
    alice_conflict = join(alice_dir, "multiple.conflict")

    found = util.await_files_exist([
        bob_conflict,
        alice_conflict,
    ])

    assert len(found) > 0, "Should have found a conflict"
    print("conflict found (as expected)")


def test_alice_writes_bob_receives_old_timestamp(magic_folder):
    alice_dir, bob_dir = magic_folder
    fname = join(alice_dir, "ts_file")
    ts = time.time() - (60 * 60 * 36)  # 36 hours ago

    with open(fname, "w") as f:
        f.write("alice wrote this")
    utime(fname, (time.time(), ts))

    fname = join(bob_dir, "ts_file")
    util.await_file_contents(fname, "alice wrote this")
    # make sure the timestamp is correct
    assert int(getmtime(fname)) == int(ts)
    return


def test_bob_writes_alice_receives(magic_folder):
    alice_dir, bob_dir = magic_folder

    with open(join(bob_dir, "second_file"), "w") as f:
        f.write("bob wrote this")

    util.await_file_contents(join(alice_dir, "second_file"), "bob wrote this")
    return


def test_alice_deletes(magic_folder):
    # alice writes a file, waits for bob to get it and then deletes it.
    alice_dir, bob_dir = magic_folder

    with open(join(alice_dir, "delfile"), "w") as f:
        f.write("alice wrote this")

    util.await_file_contents(join(bob_dir, "delfile"), "alice wrote this")

    # bob has the file; now alices deletes it
    unlink(join(alice_dir, "delfile"))

    # bob should remove his copy, but preserve a backup
    util.await_file_vanishes(join(bob_dir, "delfile"))
    util.await_file_contents(join(bob_dir, "delfile.backup"), "alice wrote this")
    return


def test_alice_creates_bob_edits(magic_folder):
    alice_dir, bob_dir = magic_folder

    # alice writes a file
    with open(join(alice_dir, "editfile"), "w") as f:
        f.write("alice wrote this")

    util.await_file_contents(join(bob_dir, "editfile"), "alice wrote this")

    # now bob edits it
    with open(join(bob_dir, "editfile"), "w") as f:
        f.write("bob says foo")

    util.await_file_contents(join(alice_dir, "editfile"), "bob says foo")


def test_bob_creates_sub_directory(magic_folder):
    alice_dir, bob_dir = magic_folder

    # bob makes a sub-dir, with a file in it
    mkdir(join(bob_dir, "subdir"))
    with open(join(bob_dir, "subdir", "a_file"), "w") as f:
        f.write("bob wuz here")

    # alice gets it
    util.await_file_contents(join(alice_dir, "subdir", "a_file"), "bob wuz here")

    # now bob deletes it again
    shutil.rmtree(join(bob_dir, "subdir"))

    # alice should delete it as well
    util.await_file_vanishes(join(alice_dir, "subdir", "a_file"))
    # i *think* it's by design that the subdir won't disappear,
    # because a "a_file.backup" should appear...
    util.await_file_contents(join(alice_dir, "subdir", "a_file.backup"), "bob wuz here")


def test_bob_creates_alice_deletes_bob_restores(magic_folder):
    alice_dir, bob_dir = magic_folder

    # bob creates a file
    with open(join(bob_dir, "boom"), "w") as f:
        f.write("bob wrote this")

    util.await_file_contents(
        join(alice_dir, "boom"),
        "bob wrote this"
    )

    # alice deletes it (so bob should as well .. but keep a backup)
    unlink(join(alice_dir, "boom"))
    util.await_file_vanishes(join(bob_dir, "boom"))
    assert exists(join(bob_dir, "boom.backup"))

    # bob restore it, with new contents
    unlink(join(bob_dir, "boom.backup"))
    with open(join(bob_dir, "boom"), "w") as f:
        f.write("bob wrote this again, because reasons")

    # XXX double-check this behavior is correct!

    # alice sees bob's update, but marks it as a conflict (because
    # .. she previously deleted it? does that really make sense)

    util.await_file_contents(
        join(alice_dir, "boom"),
        "bob wrote this again, because reasons",
    )


def test_bob_creates_alice_deletes_alice_restores(magic_folder):
    alice_dir, bob_dir = magic_folder

    # bob creates a file
    with open(join(bob_dir, "boom2"), "w") as f:
        f.write("bob wrote this")

    util.await_file_contents(
        join(alice_dir, "boom2"),
        "bob wrote this"
    )

    # alice deletes it (so bob should as well)
    unlink(join(alice_dir, "boom2"))
    util.await_file_vanishes(join(bob_dir, "boom2"))

    # alice restore it, with new contents
    with open(join(alice_dir, "boom2"), "w") as f:
        f.write("alice re-wrote this again, because reasons")

    util.await_file_contents(
        join(bob_dir, "boom2"),
        "alice re-wrote this again, because reasons"
    )


def test_bob_conflicts_with_alice_fresh(magic_folder):
    # both alice and bob make a file at "the same time".
    alice_dir, bob_dir = magic_folder

    # either alice or bob will "win" by uploading to the DMD first.
    with open(join(bob_dir, 'alpha'), 'w') as f0, open(join(alice_dir, 'alpha'), 'w') as f1:
        f0.write("this is bob's alpha\n")
        f1.write("this is alice's alpha\n")

    # there should be conflicts
    _bob_conflicts_alice_await_conflicts('alpha', alice_dir, bob_dir)


def test_bob_conflicts_with_alice_preexisting(magic_folder):
    # both alice and bob edit a file at "the same time" (similar to
    # above, but the file already exists before the edits)
    alice_dir, bob_dir = magic_folder

    # have bob create the file
    with open(join(bob_dir, 'beta'), 'w') as f:
        f.write("original beta (from bob)\n")
    util.await_file_contents(join(alice_dir, 'beta'), "original beta (from bob)\n")

    # both alice and bob now have a "beta" file, at version 0

    # either alice or bob will "win" by uploading to the DMD first
    # (however, they should both detect a conflict)
    with open(join(bob_dir, 'beta'), 'w') as f:
        f.write("this is bob's beta\n")
    with open(join(alice_dir, 'beta'), 'w') as f:
        f.write("this is alice's beta\n")

    # both alice and bob should see a conflict
    _bob_conflicts_alice_await_conflicts("beta", alice_dir, bob_dir)


def _bob_conflicts_alice_await_conflicts(name, alice_dir, bob_dir):
    """
    shared code between _fresh and _preexisting conflict test
    """
    found = util.await_files_exist(
        [
            join(bob_dir, '{}.conflict'.format(name)),
            join(alice_dir, '{}.conflict'.format(name)),
        ],
    )

    assert len(found) >= 1, "should be at least one conflict"
    assert open(join(bob_dir, name), 'r').read() == "this is bob's {}\n".format(name)
    assert open(join(alice_dir, name), 'r').read() == "this is alice's {}\n".format(name)

    alice_conflict = join(alice_dir, '{}.conflict'.format(name))
    bob_conflict = join(bob_dir, '{}.conflict'.format(name))
    if exists(bob_conflict):
        assert open(bob_conflict, 'r').read() == "this is alice's {}\n".format(name)
    if exists(alice_conflict):
        assert open(alice_conflict, 'r').read() == "this is bob's {}\n".format(name)


@pytest_twisted.inlineCallbacks
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes):
    """
    ticket 2880: if a magic-folder client uploads something, then
    re-starts a spurious .backup file should not appear
    """

    edmond_dir = join(temp_dir, 'edmond')
    edmond = yield util._create_node(
        reactor, request, temp_dir, introducer_furl, flog_gatherer,
        "edmond", web_port="tcp:9985:interface=localhost",
        storage=False,
    )


    magic_folder = join(temp_dir, 'magic-edmond')
    mkdir(magic_folder)
    created = False
    # create a magic-folder
    # (how can we know that the grid is ready?)
    for _ in range(10):  # try 10 times
        try:
            proto = util._CollectOutputProtocol()
            transport = reactor.spawnProcess(
                proto,
                sys.executable,
                [
                    sys.executable, '-m', 'allmydata.scripts.runner',
                    'magic-folder', 'create',
                    '--poll-interval', '2',
                    '--basedir', edmond_dir,
                    'magik:',
                    'edmond_magic',
                    magic_folder,
                ]
            )
            yield proto.done
            created = True
            break
        except Exception as e:
            print("failed to create magic-folder: {}".format(e))
            time.sleep(1)

    assert created, "Didn't create a magic-folder"

    # to actually-start the magic-folder we have to re-start
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')

    # add a thing to the magic-folder
    with open(join(magic_folder, "its_a_file"), "w") as f:
        f.write("edmond wrote this")

    # fixme, do status-update attempts in a loop below
    time.sleep(5)

    # let it upload; poll the HTTP magic-folder status API until it is
    # uploaded
    from allmydata.scripts.magic_folder_cli import _get_json_for_fragment

    with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
        token = f.read()

    uploaded = False
    for _ in range(10):
        options = {
            "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(),
        }
        try:
            magic_data = _get_json_for_fragment(
                options,
                'magic_folder?t=json',
                method='POST',
                post_args=dict(
                    t='json',
                    name='default',
                    token=token,
                )
            )
            for mf in magic_data:
                if mf['status'] == u'success' and mf['path'] == u'its_a_file':
                    uploaded = True
                    break
        except Exception as e:
            time.sleep(1)

    assert uploaded, "expected to upload 'its_a_file'"

    # re-starting edmond right now would "normally" trigger the 2880 bug

    # kill edmond
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')

    # XXX how can we say for sure if we've waited long enough? look at
    # tail of logs for magic-folder ... somethingsomething?
    print("waiting 20 seconds to see if a .backup appears")
    for _ in range(20):
        assert exists(join(magic_folder, "its_a_file"))
        assert not exists(join(magic_folder, "its_a_file.backup"))
        time.sleep(1)


@pytest_twisted.inlineCallbacks
def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder):
    """
    Alice can add new files to a magic folder while Bob is offline.  When Bob
    comes back online his copy is updated to reflect the new files.
    """
    alice_magic_dir, bob_magic_dir = magic_folder
    alice_node_dir = join(temp_dir, "alice")
    bob_node_dir = join(temp_dir, "bob")

    # Take Bob offline.
    yield util.cli(reactor, bob_node_dir, "stop")

    # Create a couple files in Alice's local directory.
    some_files = list(
        (name * 3) + ".added-while-offline"
        for name
        in "xyz"
    )
    for name in some_files:
        with open(join(alice_magic_dir, name), "w") as f:
            f.write(name + " some content")

    good = False
    for i in range(15):
        status = yield util.magic_folder_cli(reactor, alice_node_dir, "status")
        good = status.count(".added-while-offline (36 B): good, version=0") == len(some_files) * 2
        if good:
            # We saw each file as having a local good state and a remote good
            # state.  That means we're ready to involve Bob.
            break
        else:
            time.sleep(1.0)

    assert good, (
        "Timed out waiting for good Alice state.  Last status:\n{}".format(status)
    )

    # Start Bob up again
    magic_text = 'Completed initial Magic Folder scan successfully'
    yield util._run_node(reactor, bob_node_dir, request, magic_text)

    yield util.await_files_exist(
        list(
            join(bob_magic_dir, name)
            for name
            in some_files
        ),
        await_all=True,
    )
    # Let it settle.  It would be nicer to have a readable status output we
    # could query.  Parsing the current text format is more than I want to
    # deal with right now.
    time.sleep(1.0)
    conflict_files = list(name + ".conflict" for name in some_files)
    assert all(
        list(
            not exists(join(bob_magic_dir, name))
            for name
            in conflict_files
        ),
    )