Merge remote-tracking branch 'origin/master' into 3910-http-storage-server-tor-support

.github/workflows/ci.yml

@@ -166,7 +166,7 @@ jobs:
       matrix:
         include:
           - os: macos-12
-            python-version: "3.9"
+            python-version: "3.11"
             force-foolscap: false
           - os: windows-latest
             python-version: "3.11"

integration/test_tor.py

@@ -38,8 +38,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     The two nodes can talk to the introducer and each other: we upload to one
     node, read from the other.
     """
-    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
+    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
-    dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
+    dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
     yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
     yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
     yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave)
@@ -94,7 +94,7 @@ async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: ut
 
 
 @pytest_twisted.inlineCallbacks
-def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl) -> util.TahoeProcess:
+def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess:
     node_dir = FilePath(temp_dir).child(name)
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
@@ -116,7 +116,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 '--listen', 'tor',
                 '--shares-needed', '1',
                 '--shares-happy', '1',
-                '--shares-total', '2',
+                '--shares-total', str(shares_total),
                 node_dir.path,
             ),
             env=environ,
@@ -139,18 +139,25 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     print("okay, launched")
     return result
 
-
+@pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS')
 @pytest_twisted.inlineCallbacks
-def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
+def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
     """
-    A normal node (alice) and a normal introducer are configured, and one node
+    A normal node (normie) and a normal introducer are configured, and one node
     (anonymoose) which is configured to be anonymous by talking via Tor.
 
-    Anonymoose should be able to communicate with alice.
+    Anonymoose should be able to communicate with normie.
 
     TODO how to ensure that anonymoose is actually using Tor?
     """
-    anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl)
+    normie = yield util._create_node(
-    yield util.await_client_ready(anonymoose, minimum_number_of_servers=2, timeout=600)
+        reactor, request, temp_dir, introducer_furl, flog_gatherer, "normie",
+        web_port="tcp:9989:interface=localhost",
+        storage=True, needed=1, happy=1, total=1,
+    )
+    yield util.await_client_ready(normie)
 
-    yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, anonymoose)
+    anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1)
+    yield util.await_client_ready(anonymoose, minimum_number_of_servers=1, timeout=600)
+
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, normie, anonymoose)

integration/util.py

@@ -140,7 +140,8 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def outReceived(self, data):
         data = str(data, sys.stdout.encoding)
-        sys.stdout.write(self.name + data)
+        for line in data.splitlines():
+            sys.stdout.write(self.name + line + "\n")
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
             print("Saw '{}' in the logs".format(self._magic_text))
@@ -148,7 +149,8 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         data = str(data, sys.stderr.encoding)
-        sys.stdout.write(self.name + data)
+        for line in data.splitlines():
+            sys.stdout.write(self.name + line + "\n")
 
 
 def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
@@ -629,16 +631,9 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             server['last_received_data']
             for server in servers
         ]
-        # if any times are null/None that server has never been
+        # check that all times are 'recent enough' (it's OK if _some_ servers
-        # contacted (so it's down still, probably)
+        # are down, we just want to make sure a sufficient number are up)
-        never_received_data = server_times.count(None)
+        if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers:
-        if never_received_data > 0:
-            print(f"waiting because {never_received_data} server(s) not contacted")
-            time.sleep(1)
-            continue
-
-        # check that all times are 'recent enough'
-        if any([time.time() - t > liveness for t in server_times]):
             print("waiting because at least one server too old")
             time.sleep(1)
             continue