Merge remote-tracking branch 'origin/master' into 3795-istorageserver-tests-part-2

README.rst

@@ -1,5 +1,5 @@
 ======================================
-Free and Open decentralized data store
+Free and Open Decentralized Data Store
 ======================================
 
 |image0|
@@ -48,13 +48,17 @@ Please read more about Tahoe-LAFS architecture `here <docs/architecture.rst>`__.
 ✅ Installation
 ---------------
 
-For more detailed instructions, read `docs/INSTALL.rst <docs/INSTALL.rst>`__ .
+For more detailed instructions, read `Installing Tahoe-LAFS <docs/Installation/install-tahoe.rst>`__.
 
--  `Building Tahoe-LAFS on Windows <docs/windows.rst>`__
 
--  `OS-X Packaging <docs/OS-X.rst>`__
+Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS <docs/running.rst>`__ to learn how to set up your first Tahoe-LAFS node.
 
-Once tahoe --version works, see `docs/running.rst <docs/running.rst>`__ to learn how to set up your first Tahoe-LAFS node.
+🐍 Python 3 Support 
+--------------------
+
+Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2.
+System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version.
+Please, feel free to file issues if you run into bugs while running Tahoe on Python 3.
 
 
 🤖 Issues

newsfragments/3749.documentation

@@ -0,0 +1 @@
+Documentation and installation links in the README have been fixed.

newsfragments/3779.bugfix

@@ -0,0 +1 @@
+Fixed bug where share corruption events were not logged on storage servers running on Windows.

src/allmydata/immutable/downloader/share.py

@@ -475,7 +475,9 @@ class Share(object):
         # there was corruption somewhere in the given range
         reason = "corruption in share[%d-%d): %s" % (start, start+offset,
                                                      str(f.value))
-        self._rref.callRemoteOnly("advise_corrupt_share", reason.encode("utf-8"))
+        self._rref.callRemote(
+            "advise_corrupt_share", reason.encode("utf-8")
+        ).addErrback(log.err, "Error from remote call to advise_corrupt_share")
 
     def _satisfy_block_hash_tree(self, needed_hashes):
         o_bh = self.actual_offsets["block_hashes"]

src/allmydata/immutable/layout.py

@@ -15,7 +15,7 @@ from zope.interface import implementer
 from twisted.internet import defer
 from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \
      FileTooLargeError, HASH_SIZE
-from allmydata.util import mathutil, observer, pipeline
+from allmydata.util import mathutil, observer, pipeline, log
 from allmydata.util.assertutil import precondition
 from allmydata.storage.server import si_b2a
 
@@ -254,8 +254,7 @@ class WriteBucketProxy(object):
         return d
 
     def abort(self):
-        return self._rref.callRemoteOnly("abort")
-
+        self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket")
 
     def get_servername(self):
         return self._server.get_name()

src/allmydata/mutable/servermap.py

@@ -607,13 +607,14 @@ class ServermapUpdater(object):
         return d
 
     def _do_read(self, server, storage_index, shnums, readv):
+        """
+        If self._add_lease is true, a lease is added, and the result only fires
+        once the least has also been added.
+        """
         ss = server.get_storage_server()
         if self._add_lease:
             # send an add-lease message in parallel. The results are handled
-            # separately. This is sent before the slot_readv() so that we can
-            # be sure the add_lease is retired by the time slot_readv comes
-            # back (this relies upon our knowledge that the server code for
-            # add_lease is synchronous).
+            # separately.
             renew_secret = self._node.get_renewal_secret(server)
             cancel_secret = self._node.get_cancel_secret(server)
             d2 = ss.add_lease(
@@ -623,7 +624,16 @@ class ServermapUpdater(object):
             )
             # we ignore success
             d2.addErrback(self._add_lease_failed, server, storage_index)
+        else:
+            d2 = defer.succeed(None)
         d = ss.slot_readv(storage_index, shnums, readv)
+
+        def passthrough(result):
+            # Wait for d2, but fire with result of slot_readv() regardless of
+            # result of d2.
+            return d2.addBoth(lambda _: result)
+
+        d.addCallback(passthrough)
         return d
 
 

src/allmydata/scripts/runner.py

@@ -236,8 +236,6 @@ def _maybe_enable_eliot_logging(options, reactor):
     # Pass on the options so we can dispatch the subcommand.
     return options
 
-PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress."
-                    " Use at your own risk.")
 
 def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stderr):
     """
@@ -253,8 +251,6 @@ def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stde
 
     :raise SystemExit: Always raised after the run is complete.
     """
-    if six.PY3:
-        print(PYTHON_3_WARNING, file=stderr)
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize
         initialize()

src/allmydata/storage/server.py

@@ -708,8 +708,10 @@ class StorageServer(service.MultiService, Referenceable):
         now = time_format.iso_utc(sep="T")
         si_s = si_b2a(storage_index)
         # windows can't handle colons in the filename
-        fn = os.path.join(self.corruption_advisory_dir,
-                          "%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","")
+        fn = os.path.join(
+            self.corruption_advisory_dir,
+            ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","")
+        )
         with open(fn, "w") as f:
             f.write("report: Share Corruption\n")
             f.write("type: %s\n" % bytes_to_native_str(share_type))

src/allmydata/storage_client.py

@@ -1009,10 +1009,10 @@ class _StorageServer(object):
             shnum,
             reason,
     ):
-        return self._rref.callRemoteOnly(
+        self._rref.callRemote(
             "advise_corrupt_share",
             share_type,
             storage_index,
             shnum,
             reason,
-        )
+        ).addErrback(log.err, "Error from remote call to advise_corrupt_share")

src/allmydata/test/mutable/util.py

@@ -96,8 +96,14 @@ class FakeStorage(object):
         shares[shnum] = f.getvalue()
 
 
+# This doesn't actually implement the whole interface, but adding a commented
+# interface implementation annotation for grepping purposes.
+#@implementer(RIStorageServer)
 class FakeStorageServer(object):
-
+    """
+    A fake Foolscap remote object, implemented by overriding callRemote() to
+    call local methods.
+    """
     def __init__(self, peerid, storage):
         self.peerid = peerid
         self.storage = storage

src/allmydata/test/test_system.py

@@ -38,7 +38,6 @@ from allmydata.monitor import Monitor
 from allmydata.mutable.common import NotWriteableError
 from allmydata.mutable import layout as mutable_layout
 from allmydata.mutable.publish import MutableData
-from allmydata.scripts.runner import PYTHON_3_WARNING
 
 from foolscap.api import DeadReferenceError, fireEventually
 from twisted.python.failure import Failure
@@ -1749,18 +1748,16 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             newargs = ["--node-directory", self.getdir("client0"), verb] + list(args)
             return self.run_bintahoe(newargs, stdin=stdin, env=env)
 
-        def _check_succeeded(res, check_stderr=True):
+        def _check_succeeded(res):
             out, err, rc_or_sig = res
             self.failUnlessEqual(rc_or_sig, 0, str(res))
-            if check_stderr:
-                self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii")))
 
         d.addCallback(_run_in_subprocess, "create-alias", "newalias")
         d.addCallback(_check_succeeded)
 
         STDIN_DATA = b"This is the file to upload from stdin."
         d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA)
-        d.addCallback(_check_succeeded, check_stderr=False)
+        d.addCallback(_check_succeeded)
 
         def _mv_with_http_proxy(ign):
             env = os.environ
@@ -1773,7 +1770,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_ls(res):
             out, err, rc_or_sig = res
             self.failUnlessEqual(rc_or_sig, 0, str(res))
-            self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii")))
             self.failUnlessIn(b"tahoe-moved", out)
             self.failIfIn(b"tahoe-file", out)
         d.addCallback(_check_ls)

src/allmydata/test/test_upload.py

@@ -122,7 +122,15 @@ class SetDEPMixin(object):
              }
         self.node.encoding_params = p
 
+
+# This doesn't actually implement the whole interface, but adding a commented
+# interface implementation annotation for grepping purposes.
+#@implementer(RIStorageServer)
 class FakeStorageServer(object):
+    """
+    A fake Foolscap remote object, implemented by overriding callRemote() to
+    call local methods.
+    """
     def __init__(self, mode, reactor=None):
         self.mode = mode
         self.allocated = []