From cad7604ba30a6338a8a6385db64bef8df9e93418 Mon Sep 17 00:00:00 2001 From: YashNRam13 Date: Wed, 18 Aug 2021 14:27:03 +0530 Subject: [PATCH 01/16] docs: Update README.rst --- README.rst | 9 +++------ newsfragments/3749.documentation | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) create mode 100644 newsfragments/3749.documentation diff --git a/README.rst b/README.rst index 2cc6e38eb..99126c009 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ====================================== -Free and Open decentralized data store +Free and Open Decentralized Data Store ====================================== |image0| @@ -48,13 +48,10 @@ Please read more about Tahoe-LAFS architecture `here `__. ✅ Installation --------------- -For more detailed instructions, read `docs/INSTALL.rst `__ . +For more detailed instructions, read `Installing Tahoe-LAFS `__. -- `Building Tahoe-LAFS on Windows `__ -- `OS-X Packaging `__ - -Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. +Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS `__ to learn how to set up your first Tahoe-LAFS node. 🤖 Issues diff --git a/newsfragments/3749.documentation b/newsfragments/3749.documentation new file mode 100644 index 000000000..82e1e7856 --- /dev/null +++ b/newsfragments/3749.documentation @@ -0,0 +1 @@ +Fixes links to the installation and about Tahoe pages in the docs. From 3e0dc9449757054fb3a3fe3ffad8164c5fdb265d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 25 Aug 2021 13:06:10 -0400 Subject: [PATCH 02/16] Annotate the two fakes that (at least partially) implement RIStorageServer, so they're easier to find. --- src/allmydata/test/mutable/util.py | 8 +++++++- src/allmydata/test/test_upload.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 62e8d7295..7e3bd3ec7 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -96,8 +96,14 @@ class FakeStorage(object): shares[shnum] = f.getvalue() +# This doesn't actually implement the whole interface, but adding a commented +# interface implementation annotation for grepping purposes. +#@implementer(RIStorageServer) class FakeStorageServer(object): - + """ + A fake Foolscap remote object, implemented by overriding callRemote() to + call local methods. + """ def __init__(self, peerid, storage): self.peerid = peerid self.storage = storage diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index fc9bfd697..8d5435e88 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -122,7 +122,15 @@ class SetDEPMixin(object): } self.node.encoding_params = p + +# This doesn't actually implement the whole interface, but adding a commented +# interface implementation annotation for grepping purposes. +#@implementer(RIStorageServer) class FakeStorageServer(object): + """ + A fake Foolscap remote object, implemented by overriding callRemote() to + call local methods. + """ def __init__(self, mode, reactor=None): self.mode = mode self.allocated = [] From 342a1c2c31ead596cb705a30e69282ad28df9932 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sun, 29 Aug 2021 16:10:30 +0100 Subject: [PATCH 03/16] Python 3 support complete, so removed warning Signed-off-by: fenn-cs --- newsfragments/3781.minor | 0 src/allmydata/scripts/runner.py | 5 ----- src/allmydata/test/test_system.py | 8 ++------ 3 files changed, 2 insertions(+), 11 deletions(-) create mode 100644 newsfragments/3781.minor diff --git a/newsfragments/3781.minor b/newsfragments/3781.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 7f7f88bf6..a42b30624 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -183,13 +183,8 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options -PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.") def run(): - if six.PY3: - print(PYTHON_3_WARNING, file=sys.stderr) - if sys.platform == "win32": from allmydata.windows.fixups import initialize initialize() diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 627b6ef29..d61d6990b 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -43,7 +43,6 @@ from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from allmydata.mutable.publish import MutableData -from allmydata.scripts.runner import PYTHON_3_WARNING from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure @@ -2632,18 +2631,16 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): newargs = ["--node-directory", self.getdir("client0"), verb] + list(args) return self.run_bintahoe(newargs, stdin=stdin, env=env) - def _check_succeeded(res, check_stderr=True): + def _check_succeeded(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - if check_stderr: - self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) d.addCallback(_run_in_subprocess, "create-alias", "newalias") d.addCallback(_check_succeeded) STDIN_DATA = b"This is the file to upload from stdin." d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA) - d.addCallback(_check_succeeded, check_stderr=False) + d.addCallback(_check_succeeded) def _mv_with_http_proxy(ign): env = os.environ @@ -2656,7 +2653,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_ls(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) self.failUnlessIn(b"tahoe-moved", out) self.failIfIn(b"tahoe-file", out) d.addCallback(_check_ls) From d9561070ae6f7227dce4706f0ae02a77e963fb47 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 1 Sep 2021 23:07:19 +0100 Subject: [PATCH 04/16] Python3 support note in Readme Signed-off-by: fenn-cs --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 2cc6e38eb..ca491b353 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,11 @@ For more detailed instructions, read `docs/INSTALL.rst `__ . Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. +🐍 `Python3` Support +-------------------- + +`Python 3` support has been introduced as from ` Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start runing Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs running Tahoe on `Python 3`. + 🤖 Issues --------- From 07ac248f3b462d4eb57ece4a536267db19f9820a Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 1 Sep 2021 23:11:43 +0100 Subject: [PATCH 05/16] Python3 support note update Signed-off-by: fenn-cs --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ca491b353..bb8c5c252 100644 --- a/README.rst +++ b/README.rst @@ -56,10 +56,10 @@ For more detailed instructions, read `docs/INSTALL.rst `__ . Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. -🐍 `Python3` Support +🐍 `Python 3` Support -------------------- -`Python 3` support has been introduced as from ` Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start runing Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs running Tahoe on `Python 3`. +`Python 3` support has been introduced as from `Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start running Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs while running Tahoe on `Python 3`. 🤖 Issues From 6c679bd4e0292af572026353d0e7a1c74e01184f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 11:35:39 -0400 Subject: [PATCH 06/16] Stop using callRemoteOnly. --- src/allmydata/immutable/layout.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 3db9f096e..886a5db73 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -15,7 +15,7 @@ from zope.interface import implementer from twisted.internet import defer from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE -from allmydata.util import mathutil, observer, pipeline +from allmydata.util import mathutil, observer, pipeline, log from allmydata.util.assertutil import precondition from allmydata.storage.server import si_b2a @@ -254,8 +254,7 @@ class WriteBucketProxy(object): return d def abort(self): - return self._rref.callRemoteOnly("abort") - + self._rref.callRemote("abort").addErrback(log.err) def get_servername(self): return self._server.get_name() From 63bfff19e9be9665461b4355e56ced50a227d2a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 15:05:15 -0400 Subject: [PATCH 07/16] Don't rely on Foolscap's semantics. --- src/allmydata/mutable/servermap.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index 4f3226649..211b1fc16 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -607,13 +607,14 @@ class ServermapUpdater(object): return d def _do_read(self, server, storage_index, shnums, readv): + """ + If self._add_lease is true, a lease is added, and the result only fires + once the least has also been added. + """ ss = server.get_storage_server() if self._add_lease: # send an add-lease message in parallel. The results are handled - # separately. This is sent before the slot_readv() so that we can - # be sure the add_lease is retired by the time slot_readv comes - # back (this relies upon our knowledge that the server code for - # add_lease is synchronous). + # separately. renew_secret = self._node.get_renewal_secret(server) cancel_secret = self._node.get_cancel_secret(server) d2 = ss.add_lease( @@ -623,7 +624,16 @@ class ServermapUpdater(object): ) # we ignore success d2.addErrback(self._add_lease_failed, server, storage_index) + else: + d2 = defer.succeed(None) d = ss.slot_readv(storage_index, shnums, readv) + + def passthrough(result): + # Wait for d2, but fire with result of slot_readv() regardless of + # result of d2. + return d2.addBoth(lambda _: result) + + d.addCallback(passthrough) return d From 789a7edb56501e5e7d95c93e7570bb4fc1197507 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 15:21:42 -0400 Subject: [PATCH 08/16] Get rid of more callRemoteOnly usage. --- src/allmydata/immutable/downloader/share.py | 4 +++- src/allmydata/storage_client.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 86b99e99c..1e751500b 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -475,7 +475,9 @@ class Share(object): # there was corruption somewhere in the given range reason = "corruption in share[%d-%d): %s" % (start, start+offset, str(f.value)) - self._rref.callRemoteOnly("advise_corrupt_share", reason.encode("utf-8")) + self._rref.callRemote( + "advise_corrupt_share", reason.encode("utf-8") + ).addErrback(log.err) def _satisfy_block_hash_tree(self, needed_hashes): o_bh = self.actual_offsets["block_hashes"] diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 22dd09bcb..c278abce3 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1009,10 +1009,10 @@ class _StorageServer(object): shnum, reason, ): - return self._rref.callRemoteOnly( + self._rref.callRemote( "advise_corrupt_share", share_type, storage_index, shnum, reason, - ) + ).addErrback(log.err) From 04d2b27f46d9dee1122947e1f5bf50ac87a5b4aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 15:24:19 -0400 Subject: [PATCH 09/16] News file. --- newsfragments/3779.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3779.minor diff --git a/newsfragments/3779.minor b/newsfragments/3779.minor new file mode 100644 index 000000000..e69de29bb From 148a0573dea02d67e9b05e1156afe47807402a96 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Sep 2021 13:11:02 -0400 Subject: [PATCH 10/16] Replace colon on filename only, not on whole path. This would break Windows logging of corruption reports, since colon would be removed from e.g. "C:". --- src/allmydata/storage/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 56b47ae89..f4996756e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -708,8 +708,10 @@ class StorageServer(service.MultiService, Referenceable): now = time_format.iso_utc(sep="T") si_s = si_b2a(storage_index) # windows can't handle colons in the filename - fn = os.path.join(self.corruption_advisory_dir, - "%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + fn = os.path.join( + self.corruption_advisory_dir, + ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + ) with open(fn, "w") as f: f.write("report: Share Corruption\n") f.write("type: %s\n" % bytes_to_native_str(share_type)) From c7e82b1640238b9e8af65d3f05a7a21647cba183 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Sep 2021 13:12:57 -0400 Subject: [PATCH 11/16] Guess it's a bugfix. --- newsfragments/3779.bugfix | 1 + newsfragments/3779.minor | 0 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3779.bugfix delete mode 100644 newsfragments/3779.minor diff --git a/newsfragments/3779.bugfix b/newsfragments/3779.bugfix new file mode 100644 index 000000000..c6745f1af --- /dev/null +++ b/newsfragments/3779.bugfix @@ -0,0 +1 @@ +Fixed bug where share corruption events were not recorded on Windows. \ No newline at end of file diff --git a/newsfragments/3779.minor b/newsfragments/3779.minor deleted file mode 100644 index e69de29bb..000000000 From 93f2cf5ce1460192cae4c3ba9d42808c2caed107 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 09:41:32 -0400 Subject: [PATCH 12/16] Minor edits --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bb8c5c252..3686a58b8 100644 --- a/README.rst +++ b/README.rst @@ -56,10 +56,10 @@ For more detailed instructions, read `docs/INSTALL.rst `__ . Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. -🐍 `Python 3` Support +🐍 Python 3 Support -------------------- -`Python 3` support has been introduced as from `Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start running Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs while running Tahoe on `Python 3`. +Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. System admnistrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. 🤖 Issues From 02ad7b9709328319ef568b065de878d8bb9f6cc7 Mon Sep 17 00:00:00 2001 From: "F. E Noel Nfebe" Date: Fri, 10 Sep 2021 15:35:33 +0100 Subject: [PATCH 13/16] Improved python 3 support text format. Co-authored-by: Jean-Paul Calderone --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3686a58b8..df171b99f 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,9 @@ Once tahoe --version works, see `docs/running.rst `__ to learn 🐍 Python 3 Support -------------------- -Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. System admnistrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. +Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. +System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. +Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. 🤖 Issues From 38fcbd56e0c1fffe3c4baf0789ce5e855d200dba Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 10 Sep 2021 10:46:17 -0400 Subject: [PATCH 14/16] Adjust the news fragment to match the writing style guide --- newsfragments/3749.documentation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3749.documentation b/newsfragments/3749.documentation index 82e1e7856..554564a0b 100644 --- a/newsfragments/3749.documentation +++ b/newsfragments/3749.documentation @@ -1 +1 @@ -Fixes links to the installation and about Tahoe pages in the docs. +Documentation and installation links in the README have been fixed. From ac9875da75c1dd42080f1d18a506a17846671df6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 10 Sep 2021 11:39:48 -0400 Subject: [PATCH 15/16] Add explanation to new error logging. --- src/allmydata/immutable/downloader/share.py | 2 +- src/allmydata/immutable/layout.py | 2 +- src/allmydata/storage_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 1e751500b..41e11426f 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -477,7 +477,7 @@ class Share(object): str(f.value)) self._rref.callRemote( "advise_corrupt_share", reason.encode("utf-8") - ).addErrback(log.err) + ).addErrback(log.err, "Error from remote call to advise_corrupt_share") def _satisfy_block_hash_tree(self, needed_hashes): o_bh = self.actual_offsets["block_hashes"] diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 886a5db73..6c7362b8a 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -254,7 +254,7 @@ class WriteBucketProxy(object): return d def abort(self): - self._rref.callRemote("abort").addErrback(log.err) + self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket") def get_servername(self): return self._server.get_name() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c278abce3..e9dc8c84c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1015,4 +1015,4 @@ class _StorageServer(object): storage_index, shnum, reason, - ).addErrback(log.err) + ).addErrback(log.err, "Error from remote call to advise_corrupt_share") From fb61e29b5933fb9f84df4fc319e48a56f8e22659 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 10 Sep 2021 11:40:30 -0400 Subject: [PATCH 16/16] Better explanation. --- newsfragments/3779.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3779.bugfix b/newsfragments/3779.bugfix index c6745f1af..073046474 100644 --- a/newsfragments/3779.bugfix +++ b/newsfragments/3779.bugfix @@ -1 +1 @@ -Fixed bug where share corruption events were not recorded on Windows. \ No newline at end of file +Fixed bug where share corruption events were not logged on storage servers running on Windows. \ No newline at end of file