Merge PR349 (in rebased form)

closes tahoe-lafs/tahoe-lafs#349

.travis.yml

@@ -28,6 +28,7 @@ install:
 script:
   - tox -e codechecks
   - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then tox; else tox -e coverage; fi
+  - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then tox -e integration; fi
 after_success:
   coveralls
 notifications:

integration/conftest.py

@@ -65,14 +65,6 @@ def temp_dir(request):
     return tmp
 
 
-@pytest.fixture(scope='session')
-def tahoe_binary():
-    """
-    Finds the 'tahoe' binary, yields complete path
-    """
-    return which('tahoe')
-
-
 @pytest.fixture(scope='session')
 def flog_binary():
     return which('flogtool')
@@ -224,7 +216,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
 
 
 @pytest.fixture(scope='session')
-def introducer(reactor, temp_dir, tahoe_binary, flog_gatherer, request):
+def introducer(reactor, temp_dir, flog_gatherer, request):
     config = '''
 [node]
 nickname = introducer0
@@ -240,8 +232,14 @@ log_gatherer.furl = {log_furl}
         done_proto = _ProcessExitedProtocol()
         reactor.spawnProcess(
             done_proto,
-            tahoe_binary,
+            sys.executable,
-            ('tahoe', 'create-introducer', intro_dir),
+            (
+                sys.executable, '-m', 'allmydata.scripts.runner',
+                'create-introducer',
+                '--listen=tcp',
+                '--hostname=localhost',
+                intro_dir,
+            ),
         )
         pytest.blockon(done_proto.done)
 
@@ -255,8 +253,12 @@ log_gatherer.furl = {log_furl}
     protocol = _MagicTextProtocol('introducer running')
     process = reactor.spawnProcess(
         protocol,
-        tahoe_binary,
+        sys.executable,
-        ('tahoe', 'run', intro_dir),
+        (
+            sys.executable, '-m', 'allmydata.scripts.runner',
+            'run',
+            intro_dir,
+        ),
     )
 
     def cleanup():
@@ -281,7 +283,7 @@ def introducer_furl(introducer, temp_dir):
     return furl
 
 
-def _run_node(reactor, tahoe_binary, node_dir, request, magic_text):
+def _run_node(reactor, node_dir, request, magic_text):
     if magic_text is None:
         magic_text = "client running"
     protocol = _MagicTextProtocol(magic_text)
@@ -291,9 +293,14 @@ def _run_node(reactor, tahoe_binary, node_dir, request, magic_text):
     # between platforms.
     process = reactor.spawnProcess(
         protocol,
-        tahoe_binary,
+        sys.executable,
-        ('tahoe', 'run', node_dir),
+        (
+            sys.executable, '-m', 'allmydata.scripts.runner',
+            'run',
+            node_dir,
+        ),
     )
+    process.exited = protocol.exited
 
     def cleanup():
         try:
@@ -309,7 +316,7 @@ def _run_node(reactor, tahoe_binary, node_dir, request, magic_text):
     return protocol.magic_seen
 
 
-def _create_node(reactor, request, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, name, web_port, storage=True, magic_text=None):
+def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port, storage=True, magic_text=None):
     """
     Helper to create a single node, run it and return the instance
     spawnProcess returned (ITransport)
@@ -322,10 +329,12 @@ def _create_node(reactor, request, temp_dir, tahoe_binary, introducer_furl, flog
         mkdir(node_dir)
         done_proto = _ProcessExitedProtocol()
         args = [
-            'tahoe',
+            sys.executable, '-m', 'allmydata.scripts.runner',
             'create-node',
             '--nickname', name,
             '--introducer', introducer_furl,
+            '--hostname', 'localhost',
+            '--listen', 'tcp',
         ]
         if not storage:
             args.append('--no-storage')
@@ -333,7 +342,7 @@ def _create_node(reactor, request, temp_dir, tahoe_binary, introducer_furl, flog
 
         reactor.spawnProcess(
             done_proto,
-            tahoe_binary,
+            sys.executable,
             args,
         )
         pytest.blockon(done_proto.done)
@@ -360,11 +369,11 @@ shares.total = 4
     'log_furl': flog_gatherer,
 })
 
-    return _run_node(reactor, tahoe_binary, node_dir, request, magic_text)
+    return _run_node(reactor, node_dir, request, magic_text)
 
 
 @pytest.fixture(scope='session')
-def storage_nodes(reactor, temp_dir, tahoe_binary, introducer, introducer_furl, flog_gatherer, request):
+def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
     nodes = []
     # start all 5 nodes in parallel
     for x in range(5):
@@ -373,7 +382,7 @@ def storage_nodes(reactor, temp_dir, tahoe_binary, introducer, introducer_furl,
         nodes.append(
             pytest.blockon(
                 _create_node(
-                    reactor, request, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, name,
+                    reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
                     web_port=None, storage=True,
                 )
             )
@@ -383,7 +392,7 @@ def storage_nodes(reactor, temp_dir, tahoe_binary, introducer, introducer_furl,
 
 
 @pytest.fixture(scope='session')
-def alice(reactor, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, storage_nodes, request):
+def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
     try:
         mkdir(join(temp_dir, 'magic-alice'))
     except OSError:
@@ -391,7 +400,7 @@ def alice(reactor, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, stora
 
     process = pytest.blockon(
         _create_node(
-            reactor, request, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, "alice",
+            reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
             web_port="tcp:9980:interface=localhost",
             storage=False,
         )
@@ -400,7 +409,7 @@ def alice(reactor, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, stora
 
 
 @pytest.fixture(scope='session')
-def bob(reactor, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, storage_nodes, request):
+def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
     try:
         mkdir(join(temp_dir, 'magic-bob'))
     except OSError:
@@ -408,7 +417,7 @@ def bob(reactor, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, storage
 
     process = pytest.blockon(
         _create_node(
-            reactor, request, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, "bob",
+            reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
             web_port="tcp:9981:interface=localhost",
             storage=False,
         )
@@ -417,7 +426,7 @@ def bob(reactor, temp_dir, tahoe_binary, introducer_furl, flog_gatherer, storage
 
 
 @pytest.fixture(scope='session')
-def alice_invite(reactor, alice, tahoe_binary, temp_dir, request):
+def alice_invite(reactor, alice, temp_dir, request):
     node_dir = join(temp_dir, 'alice')
 
     # FIXME XXX by the time we see "client running" in the logs, the
@@ -427,9 +436,10 @@ def alice_invite(reactor, alice, tahoe_binary, temp_dir, request):
     proto = _CollectOutputProtocol()
     transport = reactor.spawnProcess(
         proto,
-        tahoe_binary,
+        sys.executable,
         [
-            'tahoe', 'magic-folder', 'create',
+            sys.executable, '-m', 'allmydata.scripts.runner',
+            'magic-folder', 'create',
             '--basedir', node_dir, 'magik:', 'alice',
             join(temp_dir, 'magic-alice'),
         ]
@@ -439,9 +449,10 @@ def alice_invite(reactor, alice, tahoe_binary, temp_dir, request):
     proto = _CollectOutputProtocol()
     transport = reactor.spawnProcess(
         proto,
-        tahoe_binary,
+        sys.executable,
         [
-            'tahoe', 'magic-folder', 'invite',
+            sys.executable, '-m', 'allmydata.scripts.runner',
+            'magic-folder', 'invite',
             '--basedir', node_dir, 'magik:', 'bob',
         ]
     )
@@ -451,31 +462,27 @@ def alice_invite(reactor, alice, tahoe_binary, temp_dir, request):
 
     # before magic-folder works, we have to stop and restart (this is
     # crappy for the tests -- can we fix it in magic-folder?)
-    proto = _CollectOutputProtocol()
+    try:
-    transport = reactor.spawnProcess(
+        alice.signalProcess('TERM')
-        proto,
+        pytest.blockon(alice.exited)
-        tahoe_binary,
+    except ProcessExitedAlready:
-        [
+        pass
-            'tahoe', 'stop', node_dir
-        ]
-    )
-    pytest.blockon(proto.done)
-
     magic_text = 'Completed initial Magic Folder scan successfully'
-    pytest.blockon(_run_node(reactor, tahoe_binary, node_dir, request, magic_text))
+    pytest.blockon(_run_node(reactor, node_dir, request, magic_text))
     return invite
 
 
 @pytest.fixture(scope='session')
-def magic_folder(reactor, alice_invite, alice, bob, tahoe_binary, temp_dir, request):
+def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request):
     print("pairing magic-folder")
     bob_dir = join(temp_dir, 'bob')
     proto = _CollectOutputProtocol()
     transport = reactor.spawnProcess(
         proto,
-        tahoe_binary,
+        sys.executable,
         [
-            'tahoe', 'magic-folder', 'join',
+            sys.executable, '-m', 'allmydata.scripts.runner',
+            'magic-folder', 'join',
             '--basedir', bob_dir,
             alice_invite,
             join(temp_dir, 'magic-bob'),
@@ -485,16 +492,13 @@ def magic_folder(reactor, alice_invite, alice, bob, tahoe_binary, temp_dir, requ
 
     # before magic-folder works, we have to stop and restart (this is
     # crappy for the tests -- can we fix it in magic-folder?)
-    proto = _CollectOutputProtocol()
+    try:
-    transport = reactor.spawnProcess(
+        print("Sending TERM to Bob")
-        proto,
+        bob.signalProcess('TERM')
-        tahoe_binary,
+        pytest.blockon(bob.exited)
-        [
+    except ProcessExitedAlready:
-            'tahoe', 'stop', bob_dir
+        pass
-        ]
-    )
-    pytest.blockon(proto.done)
 
     magic_text = 'Completed initial Magic Folder scan successfully'
-    pytest.blockon(_run_node(reactor, tahoe_binary, bob_dir, request, magic_text))
+    pytest.blockon(_run_node(reactor, bob_dir, request, magic_text))
     return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))

integration/test_smoke.py

@@ -114,9 +114,6 @@ def test_bob_creates_alice_deletes_bob_restores(magic_folder):
         "bob wrote this again, because reasons",
     )
 
-    # fix the conflict
-    shutil.move(join(alice_dir, "boom.conflict"), join(alice_dir, "boom"))
-
 
 def test_bob_creates_alice_deletes_alice_restores(magic_folder):
     alice_dir, bob_dir = magic_folder
@@ -177,7 +174,7 @@ def test_bob_conflicts_with_alice_preexisting(magic_folder):
     # this one by giving him a massive head start
     with open(join(bob_dir, 'beta'), 'w') as f:
         f.write("this is bob's beta\n")
-    time.sleep(0.2)
+    time.sleep(0.5)
     with open(join(alice_dir, 'beta'), 'w') as f:
         f.write("this is alice's beta\n")
 

integration/util.py

@@ -2,18 +2,22 @@ import time
 from os.path import exists
 
 
-def await_file_contents(path, contents, timeout=10):
+def await_file_contents(path, contents, timeout=15):
     start_time = time.time()
     while time.time() - start_time < timeout:
         print("  waiting for '{}'".format(path))
         if exists(path):
-            with open(path, 'r') as f:
+            try:
-                current = f.read()
+                with open(path, 'r') as f:
-            if current == contents:
+                    current = f.read()
-                return True
+            except IOError:
-            print("  file contents still mismatched")
+                print("IOError; trying again")
-            print("  wanted: {}".format(contents.replace('\n', ' ')))
+            else:
-            print("     got: {}".format(current.replace('\n', ' ')))
+                if current == contents:
+                    return True
+                print("  file contents still mismatched")
+                print("  wanted: {}".format(contents.replace('\n', ' ')))
+                print("     got: {}".format(current.replace('\n', ' ')))
         time.sleep(1)
     if exists(path):
         raise Exception("Contents of '{}' mismatched after {}s".format(path, timeout))

tox.ini

@@ -18,7 +18,7 @@ commands =
 
 [testenv:integration]
 commands =
-	 echo 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+	 # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
 	 py.test -v integration/
 
 [testenv:coverage]