From 1fba2d6ea2d6ca59c427a58f034d8ad9c1081c6c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 13 Feb 2020 10:26:57 -0500 Subject: [PATCH] Remove the magic-folder integration tests --- integration/conftest.py | 91 ------ integration/test_aaa_aardvark.py | 4 - integration/test_magic_folder.py | 462 ------------------------------- integration/test_tor.py | 2 +- integration/util.py | 4 - 5 files changed, 1 insertion(+), 562 deletions(-) delete mode 100644 integration/test_magic_folder.py diff --git a/integration/conftest.py b/integration/conftest.py index 10927f358..a85bc15f6 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -367,97 +367,6 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques return process -@pytest.fixture(scope='session') -@log_call(action_type=u"integration:alice:invite", include_args=["temp_dir"]) -def alice_invite(reactor, alice, temp_dir, request): - node_dir = join(temp_dir, 'alice') - - with start_action(action_type=u"integration:alice:magic_folder:create"): - # FIXME XXX by the time we see "client running" in the logs, the - # storage servers aren't "really" ready to roll yet (uploads fairly - # consistently fail if we don't hack in this pause...) - proto = _CollectOutputProtocol() - _tahoe_runner_optional_coverage( - proto, - reactor, - request, - [ - 'magic-folder', 'create', - '--poll-interval', '2', - '--basedir', node_dir, 'magik:', 'alice', - join(temp_dir, 'magic-alice'), - ] - ) - pytest_twisted.blockon(proto.done) - - with start_action(action_type=u"integration:alice:magic_folder:invite") as a: - proto = _CollectOutputProtocol() - _tahoe_runner_optional_coverage( - proto, - reactor, - request, - [ - 'magic-folder', 'invite', - '--basedir', node_dir, 'magik:', 'bob', - ] - ) - pytest_twisted.blockon(proto.done) - invite = proto.output.getvalue() - a.add_success_fields(invite=invite) - - with start_action(action_type=u"integration:alice:magic_folder:restart"): - # before magic-folder works, we have to stop and restart (this is - # crappy for the tests -- can we fix it in magic-folder?) - try: - alice.transport.signalProcess('TERM') - pytest_twisted.blockon(alice.transport.exited) - except ProcessExitedAlready: - pass - with start_action(action_type=u"integration:alice:magic_folder:magic-text"): - magic_text = 'Completed initial Magic Folder scan successfully' - pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text)) - await_client_ready(alice) - return invite - - -@pytest.fixture(scope='session') -@log_call( - action_type=u"integration:magic_folder", - include_args=["alice_invite", "temp_dir"], -) -def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request): - print("pairing magic-folder") - bob_dir = join(temp_dir, 'bob') - proto = _CollectOutputProtocol() - _tahoe_runner_optional_coverage( - proto, - reactor, - request, - [ - 'magic-folder', 'join', - '--poll-interval', '1', - '--basedir', bob_dir, - alice_invite, - join(temp_dir, 'magic-bob'), - ] - ) - pytest_twisted.blockon(proto.done) - - # before magic-folder works, we have to stop and restart (this is - # crappy for the tests -- can we fix it in magic-folder?) - try: - print("Sending TERM to Bob") - bob.transport.signalProcess('TERM') - pytest_twisted.blockon(bob.transport.exited) - except ProcessExitedAlready: - pass - - magic_text = 'Completed initial Magic Folder scan successfully' - pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text)) - await_client_ready(bob) - return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob')) - - @pytest.fixture(scope='session') def chutney(reactor, temp_dir): chutney_dir = join(temp_dir, 'chutney') diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py index e4698a12a..4a2ef71a6 100644 --- a/integration/test_aaa_aardvark.py +++ b/integration/test_aaa_aardvark.py @@ -16,7 +16,3 @@ def test_create_introducer(introducer): def test_create_storage(storage_nodes): print("Created {} storage nodes".format(len(storage_nodes))) - - -def test_create_alice_bob_magicfolder(magic_folder): - print("Alice and Bob have paired magic-folders") diff --git a/integration/test_magic_folder.py b/integration/test_magic_folder.py deleted file mode 100644 index 32323cb64..000000000 --- a/integration/test_magic_folder.py +++ /dev/null @@ -1,462 +0,0 @@ -import sys -import time -import shutil -from os import mkdir, unlink, utime -from os.path import join, exists, getmtime - -import util - -import pytest_twisted - - -# tests converted from check_magicfolder_smoke.py -# see "conftest.py" for the fixtures (e.g. "magic_folder") - -def test_eliot_logs_are_written(alice, bob, temp_dir): - # The integration test configuration arranges for this logging - # configuration. Verify it actually does what we want. - # - # The alice and bob arguments looks unused but they actually tell pytest - # to set up all the magic-folder stuff. The assertions here are about - # side-effects of that setup. - assert exists(join(temp_dir, "alice", "logs", "eliot.json")) - assert exists(join(temp_dir, "bob", "logs", "eliot.json")) - - -def test_alice_writes_bob_receives(magic_folder): - alice_dir, bob_dir = magic_folder - - with open(join(alice_dir, "first_file"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents(join(bob_dir, "first_file"), "alice wrote this") - return - - -def test_alice_writes_bob_receives_multiple(magic_folder): - """ - When Alice does a series of updates, Bob should just receive them - with no .backup or .conflict files being produced. - """ - alice_dir, bob_dir = magic_folder - - unwanted_files = [ - join(bob_dir, "multiple.backup"), - join(bob_dir, "multiple.conflict") - ] - - # first update - with open(join(alice_dir, "multiple"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents( - join(bob_dir, "multiple"), "alice wrote this", - error_if=unwanted_files, - ) - - # second update - with open(join(alice_dir, "multiple"), "w") as f: - f.write("someone changed their mind") - - util.await_file_contents( - join(bob_dir, "multiple"), "someone changed their mind", - error_if=unwanted_files, - ) - - # third update - with open(join(alice_dir, "multiple"), "w") as f: - f.write("absolutely final version ship it") - - util.await_file_contents( - join(bob_dir, "multiple"), "absolutely final version ship it", - error_if=unwanted_files, - ) - - # forth update, but both "at once" so one should conflict - time.sleep(2) - with open(join(alice_dir, "multiple"), "w") as f: - f.write("okay one more attempt") - with open(join(bob_dir, "multiple"), "w") as f: - f.write("...but just let me add") - - bob_conflict = join(bob_dir, "multiple.conflict") - alice_conflict = join(alice_dir, "multiple.conflict") - - found = util.await_files_exist([ - bob_conflict, - alice_conflict, - ]) - - assert len(found) > 0, "Should have found a conflict" - print("conflict found (as expected)") - - -def test_alice_writes_bob_receives_old_timestamp(magic_folder): - alice_dir, bob_dir = magic_folder - fname = join(alice_dir, "ts_file") - ts = time.time() - (60 * 60 * 36) # 36 hours ago - - with open(fname, "w") as f: - f.write("alice wrote this") - utime(fname, (time.time(), ts)) - - fname = join(bob_dir, "ts_file") - util.await_file_contents(fname, "alice wrote this") - # make sure the timestamp is correct - assert int(getmtime(fname)) == int(ts) - return - - -def test_bob_writes_alice_receives(magic_folder): - alice_dir, bob_dir = magic_folder - - with open(join(bob_dir, "second_file"), "w") as f: - f.write("bob wrote this") - - util.await_file_contents(join(alice_dir, "second_file"), "bob wrote this") - return - - -def test_alice_deletes(magic_folder): - # alice writes a file, waits for bob to get it and then deletes it. - alice_dir, bob_dir = magic_folder - - with open(join(alice_dir, "delfile"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents(join(bob_dir, "delfile"), "alice wrote this") - - # bob has the file; now alices deletes it - unlink(join(alice_dir, "delfile")) - - # bob should remove his copy, but preserve a backup - util.await_file_vanishes(join(bob_dir, "delfile")) - util.await_file_contents(join(bob_dir, "delfile.backup"), "alice wrote this") - return - - -def test_alice_creates_bob_edits(magic_folder): - alice_dir, bob_dir = magic_folder - - # alice writes a file - with open(join(alice_dir, "editfile"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents(join(bob_dir, "editfile"), "alice wrote this") - - # now bob edits it - with open(join(bob_dir, "editfile"), "w") as f: - f.write("bob says foo") - - util.await_file_contents(join(alice_dir, "editfile"), "bob says foo") - - -def test_bob_creates_sub_directory(magic_folder): - alice_dir, bob_dir = magic_folder - - # bob makes a sub-dir, with a file in it - mkdir(join(bob_dir, "subdir")) - with open(join(bob_dir, "subdir", "a_file"), "w") as f: - f.write("bob wuz here") - - # alice gets it - util.await_file_contents(join(alice_dir, "subdir", "a_file"), "bob wuz here") - - # now bob deletes it again - shutil.rmtree(join(bob_dir, "subdir")) - - # alice should delete it as well - util.await_file_vanishes(join(alice_dir, "subdir", "a_file")) - # i *think* it's by design that the subdir won't disappear, - # because a "a_file.backup" should appear... - util.await_file_contents(join(alice_dir, "subdir", "a_file.backup"), "bob wuz here") - - -def test_bob_creates_alice_deletes_bob_restores(magic_folder): - alice_dir, bob_dir = magic_folder - - # bob creates a file - with open(join(bob_dir, "boom"), "w") as f: - f.write("bob wrote this") - - util.await_file_contents( - join(alice_dir, "boom"), - "bob wrote this" - ) - - # alice deletes it (so bob should as well .. but keep a backup) - unlink(join(alice_dir, "boom")) - util.await_file_vanishes(join(bob_dir, "boom")) - assert exists(join(bob_dir, "boom.backup")) - - # bob restore it, with new contents - unlink(join(bob_dir, "boom.backup")) - with open(join(bob_dir, "boom"), "w") as f: - f.write("bob wrote this again, because reasons") - - # XXX double-check this behavior is correct! - - # alice sees bob's update, but marks it as a conflict (because - # .. she previously deleted it? does that really make sense) - - util.await_file_contents( - join(alice_dir, "boom"), - "bob wrote this again, because reasons", - ) - - -def test_bob_creates_alice_deletes_alice_restores(magic_folder): - alice_dir, bob_dir = magic_folder - - # bob creates a file - with open(join(bob_dir, "boom2"), "w") as f: - f.write("bob wrote this") - - util.await_file_contents( - join(alice_dir, "boom2"), - "bob wrote this" - ) - - # alice deletes it (so bob should as well) - unlink(join(alice_dir, "boom2")) - util.await_file_vanishes(join(bob_dir, "boom2")) - - # alice restore it, with new contents - with open(join(alice_dir, "boom2"), "w") as f: - f.write("alice re-wrote this again, because reasons") - - util.await_file_contents( - join(bob_dir, "boom2"), - "alice re-wrote this again, because reasons" - ) - - -def test_bob_conflicts_with_alice_fresh(magic_folder): - # both alice and bob make a file at "the same time". - alice_dir, bob_dir = magic_folder - - # either alice or bob will "win" by uploading to the DMD first. - with open(join(bob_dir, 'alpha'), 'w') as f0, open(join(alice_dir, 'alpha'), 'w') as f1: - f0.write("this is bob's alpha\n") - f1.write("this is alice's alpha\n") - - # there should be conflicts - _bob_conflicts_alice_await_conflicts('alpha', alice_dir, bob_dir) - - -def test_bob_conflicts_with_alice_preexisting(magic_folder): - # both alice and bob edit a file at "the same time" (similar to - # above, but the file already exists before the edits) - alice_dir, bob_dir = magic_folder - - # have bob create the file - with open(join(bob_dir, 'beta'), 'w') as f: - f.write("original beta (from bob)\n") - util.await_file_contents(join(alice_dir, 'beta'), "original beta (from bob)\n") - - # both alice and bob now have a "beta" file, at version 0 - - # either alice or bob will "win" by uploading to the DMD first - # (however, they should both detect a conflict) - with open(join(bob_dir, 'beta'), 'w') as f: - f.write("this is bob's beta\n") - with open(join(alice_dir, 'beta'), 'w') as f: - f.write("this is alice's beta\n") - - # both alice and bob should see a conflict - _bob_conflicts_alice_await_conflicts("beta", alice_dir, bob_dir) - - -def _bob_conflicts_alice_await_conflicts(name, alice_dir, bob_dir): - """ - shared code between _fresh and _preexisting conflict test - """ - found = util.await_files_exist( - [ - join(bob_dir, '{}.conflict'.format(name)), - join(alice_dir, '{}.conflict'.format(name)), - ], - ) - - assert len(found) >= 1, "should be at least one conflict" - assert open(join(bob_dir, name), 'r').read() == "this is bob's {}\n".format(name) - assert open(join(alice_dir, name), 'r').read() == "this is alice's {}\n".format(name) - - alice_conflict = join(alice_dir, '{}.conflict'.format(name)) - bob_conflict = join(bob_dir, '{}.conflict'.format(name)) - if exists(bob_conflict): - assert open(bob_conflict, 'r').read() == "this is alice's {}\n".format(name) - if exists(alice_conflict): - assert open(alice_conflict, 'r').read() == "this is bob's {}\n".format(name) - - -@pytest_twisted.inlineCallbacks -def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes): - """ - ticket 2880: if a magic-folder client uploads something, then - re-starts a spurious .backup file should not appear - """ - - edmond_dir = join(temp_dir, 'edmond') - edmond = yield util._create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, - "edmond", web_port="tcp:9985:interface=localhost", - storage=False, - ) - - - magic_folder = join(temp_dir, 'magic-edmond') - mkdir(magic_folder) - created = False - # create a magic-folder - # (how can we know that the grid is ready?) - for _ in range(10): # try 10 times - try: - proto = util._CollectOutputProtocol() - transport = reactor.spawnProcess( - proto, - sys.executable, - [ - sys.executable, '-m', 'allmydata.scripts.runner', - 'magic-folder', 'create', - '--poll-interval', '2', - '--basedir', edmond_dir, - 'magik:', - 'edmond_magic', - magic_folder, - ] - ) - yield proto.done - created = True - break - except Exception as e: - print("failed to create magic-folder: {}".format(e)) - time.sleep(1) - - assert created, "Didn't create a magic-folder" - - # to actually-start the magic-folder we have to re-start - edmond.transport.signalProcess('TERM') - yield edmond.transport.exited - edmond = yield util._run_node(reactor, edmond.node_dir, request, 'Completed initial Magic Folder scan successfully') - util.await_client_ready(edmond) - - # add a thing to the magic-folder - with open(join(magic_folder, "its_a_file"), "w") as f: - f.write("edmond wrote this") - - # fixme, do status-update attempts in a loop below - time.sleep(5) - - # let it upload; poll the HTTP magic-folder status API until it is - # uploaded - from allmydata.scripts.magic_folder_cli import _get_json_for_fragment - - with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f: - token = f.read() - - uploaded = False - for _ in range(10): - options = { - "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(), - } - try: - magic_data = _get_json_for_fragment( - options, - 'magic_folder?t=json', - method='POST', - post_args=dict( - t='json', - name='default', - token=token, - ) - ) - for mf in magic_data: - if mf['status'] == u'success' and mf['path'] == u'its_a_file': - uploaded = True - break - except Exception as e: - time.sleep(1) - - assert uploaded, "expected to upload 'its_a_file'" - - # re-starting edmond right now would "normally" trigger the 2880 bug - - # kill edmond - edmond.transport.signalProcess('TERM') - yield edmond.transport.exited - time.sleep(1) - edmond = yield util._run_node(reactor, edmond.node_dir, request, 'Completed initial Magic Folder scan successfully') - util.await_client_ready(edmond) - - # XXX how can we say for sure if we've waited long enough? look at - # tail of logs for magic-folder ... somethingsomething? - print("waiting 20 seconds to see if a .backup appears") - for _ in range(20): - assert exists(join(magic_folder, "its_a_file")) - assert not exists(join(magic_folder, "its_a_file.backup")) - time.sleep(1) - - -@pytest_twisted.inlineCallbacks -def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder): - """ - Alice can add new files to a magic folder while Bob is offline. When Bob - comes back online his copy is updated to reflect the new files. - """ - alice_magic_dir, bob_magic_dir = magic_folder - alice_node_dir = join(temp_dir, "alice") - bob_node_dir = join(temp_dir, "bob") - - # Take Bob offline. - yield util.cli(request, reactor, bob_node_dir, "stop") - - # Create a couple files in Alice's local directory. - some_files = list( - (name * 3) + ".added-while-offline" - for name - in "xyz" - ) - for name in some_files: - with open(join(alice_magic_dir, name), "w") as f: - f.write(name + " some content") - - good = False - for i in range(15): - status = yield util.magic_folder_cli(request, reactor, alice_node_dir, "status") - good = status.count(".added-while-offline (36 B): good, version=0") == len(some_files) * 2 - if good: - # We saw each file as having a local good state and a remote good - # state. That means we're ready to involve Bob. - break - else: - time.sleep(1.0) - - assert good, ( - "Timed out waiting for good Alice state. Last status:\n{}".format(status) - ) - - # Start Bob up again - magic_text = 'Completed initial Magic Folder scan successfully' - yield util._run_node(reactor, bob_node_dir, request, magic_text) - - yield util.await_files_exist( - list( - join(bob_magic_dir, name) - for name - in some_files - ), - await_all=True, - ) - # Let it settle. It would be nicer to have a readable status output we - # could query. Parsing the current text format is more than I want to - # deal with right now. - time.sleep(1.0) - conflict_files = list(name + ".conflict" for name in some_files) - assert all( - list( - not exists(join(bob_magic_dir, name)) - for name - in conflict_files - ), - ) diff --git a/integration/test_tor.py b/integration/test_tor.py index 187754f08..633def8de 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -14,7 +14,7 @@ import pytest_twisted import util -# see "conftest.py" for the fixtures (e.g. "magic_folder") +# see "conftest.py" for the fixtures (e.g. "tor_network") @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): diff --git a/integration/util.py b/integration/util.py index aef9ba544..bbcf5efc6 100644 --- a/integration/util.py +++ b/integration/util.py @@ -498,7 +498,3 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2): tahoe, ) ) - - -def magic_folder_cli(request, reactor, node_dir, *argv): - return cli(request, reactor, node_dir, "magic-folder", *argv)