From add510701c0809cf89494434c1dccdfc3271df47 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 11:44:51 -0500 Subject: [PATCH 001/104] Run integration tests both with and without HTTP storage protocol. --- .github/workflows/ci.yml | 6 +++++- integration/util.py | 17 +++++++++-------- newsfragments/3937.minor | 0 src/allmydata/protocol_switch.py | 16 ++++++++++++++++ src/allmydata/testing/__init__.py | 18 ++++++++++++++++++ 5 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 newsfragments/3937.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327014ca..26574066c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,11 @@ jobs: # aren't too long. On Windows tox won't pass it through so it has no # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" - run: tox -e integration + run: | + # Run with Foolscap forced: + __TAHOE_INTEGRATION_FORCE_FOOLSCAP=1 tox -e integration + # Run with Foolscap not forced, which should result in HTTP being used. + __TAHOE_INTEGRATION_FORCE_FOOLSCAP=0 tox -e integration - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/integration/util.py b/integration/util.py index ad9249e45..cde837218 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,14 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import time @@ -38,6 +30,7 @@ from allmydata.util.configutil import ( write_config, ) from allmydata import client +from allmydata.testing import foolscap_only_for_integration_testing import pytest_twisted @@ -300,6 +293,14 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam u'log_gatherer.furl', flog_gatherer, ) + force_foolscap = foolscap_only_for_integration_testing() + if force_foolscap is not None: + set_config( + config, + 'storage', + 'force_foolscap', + str(force_foolscap), + ) write_config(FilePath(config_path), config) created_d.addCallback(created) diff --git a/newsfragments/3937.minor b/newsfragments/3937.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index b0af84c33..d88863fdb 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -30,6 +30,7 @@ from foolscap.api import Tub from .storage.http_server import HTTPServer, build_nurl from .storage.server import StorageServer +from .testing import foolscap_only_for_integration_testing class _PretendToBeNegotiation(type): @@ -170,6 +171,21 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # and later data, otherwise assume HTTPS. self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): + if foolscap_only_for_integration_testing() == False: + # Tahoe will prefer HTTP storage protocol over Foolscap when possible. + # + # If this is branch is taken, we are running a test that should + # be using HTTP for the storage protocol. As such, we + # aggressively disable Foolscap to ensure that HTTP is in fact + # going to be used. If we hit this branch that means our + # expectation that HTTP will be used was wrong, suggesting a + # bug in either the code of the integration testing setup. + # + # This branch should never be hit in production! + self.transport.loseConnection() + print("FOOLSCAP IS DISABLED, I PITY THE FOOLS WHO SEE THIS MESSAGE") + return + # We're a Foolscap Negotiation server protocol instance: transport = self.transport buf = self._buffer diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py index e69de29bb..119ae4101 100644 --- a/src/allmydata/testing/__init__.py +++ b/src/allmydata/testing/__init__.py @@ -0,0 +1,18 @@ +import os +from typing import Optional + + +def foolscap_only_for_integration_testing() -> Optional[bool]: + """ + Return whether HTTP storage protocol has been disabled / Foolscap + forced, for purposes of integration testing. + + This is determined by the __TAHOE_INTEGRATION_FORCE_FOOLSCAP environment + variable, which can be 1, 0, or not set, corresponding to results of + ``True``, ``False`` and ``None`` (i.e. default). + """ + force_foolscap = os.environ.get("__TAHOE_INTEGRATION_FORCE_FOOLSCAP") + if force_foolscap is None: + return None + + return bool(int(force_foolscap)) From 7afd821efc826f2ee644ed85369b0bc6b8dbb482 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 13:28:26 -0500 Subject: [PATCH 002/104] Sigh --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index bacb40290..a9421c3e5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -179,6 +179,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 4c8e8a74a4920359617bb4471f97eb3817eed37a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Nov 2022 12:25:37 -0500 Subject: [PATCH 003/104] Not needed. --- src/allmydata/test/test_storage_https.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index a9421c3e5..88435bf89 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -202,10 +202,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.001) - # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that # so I don't know how to test that! See From f5b24d51e909d4e5bc5a836fb2970ee025faf66f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:14:08 -0500 Subject: [PATCH 004/104] Add a test for missing Authorization --- src/allmydata/test/test_storage_http.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8dbe18545..de60812e3 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -37,6 +37,7 @@ from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound +from testtools.matchers import Equals from .common import SyncTestCase from ..storage.http_common import get_content_type, CBOR_MIME_TYPE @@ -555,6 +556,20 @@ class GenericHTTPAPITests(SyncTestCase): super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) + def test_missing_authentication(self) -> None: + """ + If nothing is given in the ``Authorization`` header at all an + ``Unauthorized`` response is returned. + """ + client = StubTreq(self.http.http_server.get_resource()) + response = self.http.result_of_with_flush( + client.request( + "GET", + "http://127.0.0.1/storage/v1/version", + ), + ) + self.assertThat(response.code, Equals(http.UNAUTHORIZED)) + def test_bad_authentication(self): """ If the wrong swissnum is used, an ``Unauthorized`` response code is From 920467dcea958dee101eec55e6cb67d7118e11ac Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:19:01 -0500 Subject: [PATCH 005/104] Treat missing Authorization as the same as empty Authorization --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3902976ba..96a491882 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -100,7 +100,7 @@ def _authorization_decorator(required_secrets): @wraps(f) def route(self, request, *args, **kwargs): if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [None])[0].encode( + request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( "utf-8" ), swissnum_auth_header(self._swissnum), From 57f13a2472c4fef1e99ffc4b8522a88d4be3c14c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:20:13 -0500 Subject: [PATCH 006/104] news fragment --- newsfragments/3942.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3942.minor diff --git a/newsfragments/3942.minor b/newsfragments/3942.minor new file mode 100644 index 000000000..e69de29bb From 4367e5a0fcfd5c905195b741eec727eb2416096d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:28:58 -0500 Subject: [PATCH 007/104] Bump the Twisted dependency so we can do this --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 768e44e29..a3b3d5b98 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,9 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - "Twisted[tls,conch] >= 19.10.0", + # * Twisted 22.8.0 added support for coroutine-returning functions in many + # places (mainly via `maybeDeferred`) + "Twisted[tls,conch] >= 22.8.0", "PyYAML >= 3.11", From 5cebe91406c5d9db2c4b5ce150f85a3fd50322e7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:29:57 -0500 Subject: [PATCH 008/104] update the module docstring --- src/allmydata/test/mutable/test_version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index d5c44f204..aa6fb539f 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -1,5 +1,6 @@ """ -Ported to Python 3. +Tests related to the way ``allmydata.mutable`` handles different versions +of data for an object. """ from __future__ import print_function from __future__ import absolute_import From 1acf8604eff5227ed372b81eac20bc08677a853a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:30:08 -0500 Subject: [PATCH 009/104] Remove the Py2/Py3 compatibility header --- src/allmydata/test/mutable/test_version.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index aa6fb539f..669baa8db 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -2,17 +2,9 @@ Tests related to the way ``allmydata.mutable`` handles different versions of data for an object. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from io import StringIO import os -from six.moves import cStringIO as StringIO from twisted.internet import defer from ..common import AsyncTestCase From a11eeaf240d1fde831e571ad5b5df3ebeed97168 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:30:37 -0500 Subject: [PATCH 010/104] Convert all of the asynchronous functions to use `async` and `await` --- src/allmydata/test/mutable/test_version.py | 546 +++++++++------------ 1 file changed, 228 insertions(+), 318 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 669baa8db..d14cc9295 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -5,8 +5,8 @@ of data for an object. from io import StringIO import os +from typing import Optional -from twisted.internet import defer from ..common import AsyncTestCase from testtools.matchers import ( Equals, @@ -40,343 +40,269 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ self.small_data = b"test data" * 10 # 90 B; SDMF - def do_upload_mdmf(self, data=None): + async def do_upload_mdmf(self, data: Optional[bytes] = None) -> MutableFileNode: if data is None: data = self.data - d = self.nm.create_mutable_file(MutableData(data), - version=MDMF_VERSION) - def _then(n): - self.assertThat(n, IsInstance(MutableFileNode)) - self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) - self.mdmf_node = n - return n - d.addCallback(_then) - return d + n = await self.nm.create_mutable_file(MutableData(data), + version=MDMF_VERSION) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) + self.mdmf_node = n + return n - def do_upload_sdmf(self, data=None): + async def do_upload_sdmf(self, data: Optional[bytes] = None) -> MutableFileNode: if data is None: data = self.small_data - d = self.nm.create_mutable_file(MutableData(data)) - def _then(n): - self.assertThat(n, IsInstance(MutableFileNode)) - self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) - self.sdmf_node = n - return n - d.addCallback(_then) - return d + n = await self.nm.create_mutable_file(MutableData(data)) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) + self.sdmf_node = n + return n - def do_upload_empty_sdmf(self): - d = self.nm.create_mutable_file(MutableData(b"")) - def _then(n): - self.assertThat(n, IsInstance(MutableFileNode)) - self.sdmf_zero_length_node = n - self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) - return n - d.addCallback(_then) - return d + async def do_upload_empty_sdmf(self) -> MutableFileNode: + n = await self.nm.create_mutable_file(MutableData(b"")) + self.assertThat(n, IsInstance(MutableFileNode)) + self.sdmf_zero_length_node = n + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) + return n - def do_upload(self): - d = self.do_upload_mdmf() - d.addCallback(lambda ign: self.do_upload_sdmf()) - return d + async def do_upload(self) -> MutableFileNode: + await self.do_upload_mdmf() + return await self.do_upload_sdmf() - def test_debug(self): - d = self.do_upload_mdmf() - def _debug(n): - fso = debug.FindSharesOptions() - storage_index = base32.b2a(n.get_storage_index()) - fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3 - fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir))) - for (i,ss,storedir) - in self.iterate_servers()] - fso.stdout = StringIO() - fso.stderr = StringIO() - debug.find_shares(fso) - sharefiles = fso.stdout.getvalue().splitlines() - expected = self.nm.default_encoding_parameters["n"] - self.assertThat(sharefiles, HasLength(expected)) + async def test_debug(self) -> None: + n = await self.do_upload_mdmf() + fso = debug.FindSharesOptions() + storage_index = base32.b2a(n.get_storage_index()) + fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3 + fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir))) + for (i,ss,storedir) + in self.iterate_servers()] + fso.stdout = StringIO() + fso.stderr = StringIO() + debug.find_shares(fso) + sharefiles = fso.stdout.getvalue().splitlines() + expected = self.nm.default_encoding_parameters["n"] + self.assertThat(sharefiles, HasLength(expected)) - do = debug.DumpOptions() - do["filename"] = sharefiles[0] - do.stdout = StringIO() - debug.dump_share(do) - output = do.stdout.getvalue() - lines = set(output.splitlines()) - self.assertTrue("Mutable slot found:" in lines, output) - self.assertTrue(" share_type: MDMF" in lines, output) - self.assertTrue(" num_extra_leases: 0" in lines, output) - self.assertTrue(" MDMF contents:" in lines, output) - self.assertTrue(" seqnum: 1" in lines, output) - self.assertTrue(" required_shares: 3" in lines, output) - self.assertTrue(" total_shares: 10" in lines, output) - self.assertTrue(" segsize: 131073" in lines, output) - self.assertTrue(" datalen: %d" % len(self.data) in lines, output) - vcap = str(n.get_verify_cap().to_string(), "utf-8") - self.assertTrue(" verify-cap: %s" % vcap in lines, output) - cso = debug.CatalogSharesOptions() - cso.nodedirs = fso.nodedirs - cso.stdout = StringIO() - cso.stderr = StringIO() - debug.catalog_shares(cso) - shares = cso.stdout.getvalue().splitlines() - oneshare = shares[0] # all shares should be MDMF - self.failIf(oneshare.startswith("UNKNOWN"), oneshare) - self.assertTrue(oneshare.startswith("MDMF"), oneshare) - fields = oneshare.split() - self.assertThat(fields[0], Equals("MDMF")) - self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) - self.assertThat(fields[2], Equals("3/10")) - self.assertThat(fields[3], Equals("%d" % len(self.data))) - self.assertTrue(fields[4].startswith("#1:"), fields[3]) - # the rest of fields[4] is the roothash, which depends upon - # encryption salts and is not constant. fields[5] is the - # remaining time on the longest lease, which is timing dependent. - # The rest of the line is the quoted pathname to the share. - d.addCallback(_debug) - return d + do = debug.DumpOptions() + do["filename"] = sharefiles[0] + do.stdout = StringIO() + debug.dump_share(do) + output = do.stdout.getvalue() + lines = set(output.splitlines()) + self.assertTrue("Mutable slot found:" in lines, output) + self.assertTrue(" share_type: MDMF" in lines, output) + self.assertTrue(" num_extra_leases: 0" in lines, output) + self.assertTrue(" MDMF contents:" in lines, output) + self.assertTrue(" seqnum: 1" in lines, output) + self.assertTrue(" required_shares: 3" in lines, output) + self.assertTrue(" total_shares: 10" in lines, output) + self.assertTrue(" segsize: 131073" in lines, output) + self.assertTrue(" datalen: %d" % len(self.data) in lines, output) + vcap = str(n.get_verify_cap().to_string(), "utf-8") + self.assertTrue(" verify-cap: %s" % vcap in lines, output) + cso = debug.CatalogSharesOptions() + cso.nodedirs = fso.nodedirs + cso.stdout = StringIO() + cso.stderr = StringIO() + debug.catalog_shares(cso) + shares = cso.stdout.getvalue().splitlines() + oneshare = shares[0] # all shares should be MDMF + self.failIf(oneshare.startswith("UNKNOWN"), oneshare) + self.assertTrue(oneshare.startswith("MDMF"), oneshare) + fields = oneshare.split() + self.assertThat(fields[0], Equals("MDMF")) + self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) + self.assertThat(fields[2], Equals("3/10")) + self.assertThat(fields[3], Equals("%d" % len(self.data))) + self.assertTrue(fields[4].startswith("#1:"), fields[3]) + # the rest of fields[4] is the roothash, which depends upon + # encryption salts and is not constant. fields[5] is the + # remaining time on the longest lease, which is timing dependent. + # The rest of the line is the quoted pathname to the share. + + async def test_get_sequence_number(self) -> None: + await self.do_upload() + bv = await self.mdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(1)) + bv = await self.sdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(1)) - def test_get_sequence_number(self): - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(1))) - d.addCallback(lambda ignored: - self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(1))) # Now update. The sequence number in both cases should be 1 in # both cases. - def _do_update(ignored): - new_data = MutableData(b"foo bar baz" * 100000) - new_small_data = MutableData(b"foo bar baz" * 10) - d1 = self.mdmf_node.overwrite(new_data) - d2 = self.sdmf_node.overwrite(new_small_data) - dl = gatherResults([d1, d2]) - return dl - d.addCallback(_do_update) - d.addCallback(lambda ignored: - self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(2))) - d.addCallback(lambda ignored: - self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(2))) - return d + new_data = MutableData(b"foo bar baz" * 100000) + new_small_data = MutableData(b"foo bar baz" * 10) + d1 = self.mdmf_node.overwrite(new_data) + d2 = self.sdmf_node.overwrite(new_small_data) + await gatherResults([d1, d2]) + bv = await self.mdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(2)) + bv = await self.sdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(2)) - - def test_cap_after_upload(self): + async def test_cap_after_upload(self) -> None: # If we create a new mutable file and upload things to it, and # it's an MDMF file, we should get an MDMF cap back from that # file and should be able to use that. # That's essentially what MDMF node is, so just check that. - d = self.do_upload_mdmf() - def _then(ign): - mdmf_uri = self.mdmf_node.get_uri() - cap = uri.from_string(mdmf_uri) - self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) - readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() - cap = uri.from_string(readonly_mdmf_uri) - self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) - d.addCallback(_then) - return d + await self.do_upload_mdmf() + mdmf_uri = self.mdmf_node.get_uri() + cap = uri.from_string(mdmf_uri) + self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) + readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() + cap = uri.from_string(readonly_mdmf_uri) + self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) - def test_mutable_version(self): + async def test_mutable_version(self) -> None: # assert that getting parameters from the IMutableVersion object # gives us the same data as getting them from the filenode itself - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version()) - def _check_mdmf(bv): - n = self.mdmf_node - self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) - self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) - self.assertFalse(bv.is_readonly()) - d.addCallback(_check_mdmf) - d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version()) - def _check_sdmf(bv): - n = self.sdmf_node - self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) - self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) - self.assertFalse(bv.is_readonly()) - d.addCallback(_check_sdmf) - return d + await self.do_upload() + bv = await self.mdmf_node.get_best_mutable_version() + n = self.mdmf_node + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) + + bv = await self.sdmf_node.get_best_mutable_version() + n = self.sdmf_node + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) - def test_get_readonly_version(self): - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) + async def test_get_readonly_version(self) -> None: + await self.do_upload() + bv = await self.mdmf_node.get_best_readable_version() + self.assertTrue(bv.is_readonly()) # Attempting to get a mutable version of a mutable file from a # filenode initialized with a readcap should return a readonly # version of that same node. - d.addCallback(lambda ign: self.mdmf_node.get_readonly()) - d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.assertTrue(v.is_readonly())) + ro = self.mdmf_node.get_readonly() + v = await ro.get_best_mutable_version() + self.assertTrue(v.is_readonly()) - d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) + bv = await self.sdmf_node.get_best_readable_version() + self.assertTrue(bv.is_readonly()) - d.addCallback(lambda ign: self.sdmf_node.get_readonly()) - d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.assertTrue(v.is_readonly())) - return d + ro = self.sdmf_node.get_readonly() + v = await ro.get_best_mutable_version() + self.assertTrue(v.is_readonly()) - def test_toplevel_overwrite(self): + async def test_toplevel_overwrite(self) -> None: new_data = MutableData(b"foo bar baz" * 100000) new_small_data = MutableData(b"foo bar baz" * 10) - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.overwrite(new_data)) - d.addCallback(lambda ignored: - self.mdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Equals(b"foo bar baz" * 100000))) - d.addCallback(lambda ignored: - self.sdmf_node.overwrite(new_small_data)) - d.addCallback(lambda ignored: - self.sdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Equals(b"foo bar baz" * 10))) - return d + await self.do_upload() + await self.mdmf_node.overwrite(new_data) + data = await self.mdmf_node.download_best_version() + self.assertThat(data, Equals(b"foo bar baz" * 100000)) + await self.sdmf_node.overwrite(new_small_data) + data = await self.sdmf_node.download_best_version() + self.assertThat(data, Equals(b"foo bar baz" * 10)) - def test_toplevel_modify(self): - d = self.do_upload() + async def test_toplevel_modify(self) -> None: + await self.do_upload() def modifier(old_contents, servermap, first_time): return old_contents + b"modified" - d.addCallback(lambda ign: self.mdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.mdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - d.addCallback(lambda ignored: - self.sdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.sdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - return d + await self.mdmf_node.modify(modifier) + data = await self.mdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) + await self.sdmf_node.modify(modifier) + data = await self.sdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) - def test_version_modify(self): + async def test_version_modify(self) -> None: # TODO: When we can publish multiple versions, alter this test # to modify a version other than the best usable version, then # test to see that the best recoverable version is that. - d = self.do_upload() + await self.do_upload() def modifier(old_contents, servermap, first_time): return old_contents + b"modified" - d.addCallback(lambda ign: self.mdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.mdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - d.addCallback(lambda ignored: - self.sdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.sdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - return d + await self.mdmf_node.modify(modifier) + data = await self.mdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) + await self.sdmf_node.modify(modifier) + data = await self.sdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) - def test_download_version(self): - d = self.publish_multiple() + async def test_download_version(self) -> None: + await self.publish_multiple() # We want to have two recoverable versions on the grid. - d.addCallback(lambda res: - self._set_versions({0:0,2:0,4:0,6:0,8:0, - 1:1,3:1,5:1,7:1,9:1})) + self._set_versions({0:0,2:0,4:0,6:0,8:0, + 1:1,3:1,5:1,7:1,9:1}) # Now try to download each version. We should get the plaintext # associated with that version. - d.addCallback(lambda ignored: - self._fn.get_servermap(mode=MODE_READ)) - def _got_servermap(smap): - versions = smap.recoverable_versions() - assert len(versions) == 2 + smap = await self._fn.get_servermap(mode=MODE_READ) + versions = smap.recoverable_versions() + assert len(versions) == 2 - self.servermap = smap - self.version1, self.version2 = versions - assert self.version1 != self.version2 + self.servermap = smap + self.version1, self.version2 = versions + assert self.version1 != self.version2 - self.version1_seqnum = self.version1[0] - self.version2_seqnum = self.version2[0] - self.version1_index = self.version1_seqnum - 1 - self.version2_index = self.version2_seqnum - 1 + self.version1_seqnum = self.version1[0] + self.version2_seqnum = self.version2[0] + self.version1_index = self.version1_seqnum - 1 + self.version2_index = self.version2_seqnum - 1 - d.addCallback(_got_servermap) - d.addCallback(lambda ignored: - self._fn.download_version(self.servermap, self.version1)) - d.addCallback(lambda results: - self.assertThat(self.CONTENTS[self.version1_index], - Equals(results))) - d.addCallback(lambda ignored: - self._fn.download_version(self.servermap, self.version2)) - d.addCallback(lambda results: - self.assertThat(self.CONTENTS[self.version2_index], - Equals(results))) - return d + results = await self._fn.download_version(self.servermap, self.version1) + self.assertThat(self.CONTENTS[self.version1_index], + Equals(results)) + results = await self._fn.download_version(self.servermap, self.version2) + self.assertThat(self.CONTENTS[self.version2_index], + Equals(results)) - def test_download_nonexistent_version(self): - d = self.do_upload_mdmf() - d.addCallback(lambda ign: self.mdmf_node.get_servermap(mode=MODE_WRITE)) - def _set_servermap(servermap): - self.servermap = servermap - d.addCallback(_set_servermap) - d.addCallback(lambda ignored: - self.shouldFail(UnrecoverableFileError, "nonexistent version", - None, - self.mdmf_node.download_version, self.servermap, - "not a version")) - return d + async def test_download_nonexistent_version(self) -> None: + await self.do_upload_mdmf() + servermap = await self.mdmf_node.get_servermap(mode=MODE_WRITE) + await self.shouldFail(UnrecoverableFileError, "nonexistent version", + None, + self.mdmf_node.download_version, servermap, + "not a version") - def _test_partial_read(self, node, expected, modes, step): - d = node.get_best_readable_version() + async def _test_partial_read(self, node, expected, modes, step) -> None: + version = await node.get_best_readable_version() for (name, offset, length) in modes: - d.addCallback(self._do_partial_read, name, expected, offset, length) + version = await self._do_partial_read(version, name, expected, offset, length) # then read the whole thing, but only a few bytes at a time, and see # that the results are what we expect. - def _read_data(version): - c = consumer.MemoryConsumer() - d2 = defer.succeed(None) - for i in range(0, len(expected), step): - d2.addCallback(lambda ignored, i=i: version.read(c, i, step)) - d2.addCallback(lambda ignored: - self.assertThat(expected, Equals(b"".join(c.chunks)))) - return d2 - d.addCallback(_read_data) - return d - - def _do_partial_read(self, version, name, expected, offset, length): c = consumer.MemoryConsumer() - d = version.read(c, offset, length) + for i in range(0, len(expected), step): + await version.read(c, i, step) + self.assertThat(expected, Equals(b"".join(c.chunks))) + + async def _do_partial_read(self, version, name, expected, offset, length) -> None: + c = consumer.MemoryConsumer() + await version.read(c, offset, length) if length is None: expected_range = expected[offset:] else: expected_range = expected[offset:offset+length] - d.addCallback(lambda ignored: b"".join(c.chunks)) - def _check(results): - if results != expected_range: - print("read([%d]+%s) got %d bytes, not %d" % \ - (offset, length, len(results), len(expected_range))) - print("got: %s ... %s" % (results[:20], results[-20:])) - print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) - self.fail("results[%s] != expected_range" % name) - return version # daisy-chained to next call - d.addCallback(_check) - return d + results = b"".join(c.chunks) + if results != expected_range: + print("read([%d]+%s) got %d bytes, not %d" % \ + (offset, length, len(results), len(expected_range))) + print("got: %s ... %s" % (results[:20], results[-20:])) + print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) + self.fail("results[%s] != expected_range" % name) + return version # daisy-chained to next call - def test_partial_read_mdmf_0(self): + async def test_partial_read_mdmf_0(self) -> None: data = b"" - d = self.do_upload_mdmf(data=data) + result = await self.do_upload_mdmf(data=data) modes = [("all1", 0,0), ("all2", 0,None), ] - d.addCallback(self._test_partial_read, data, modes, 1) - return d + await self._test_partial_read(result, data, modes, 1) - def test_partial_read_mdmf_large(self): + async def test_partial_read_mdmf_large(self) -> None: segment_boundary = mathutil.next_multiple(128 * 1024, 3) modes = [("start_on_segment_boundary", segment_boundary, 50), ("ending_one_byte_after_segment_boundary", segment_boundary-50, 51), @@ -386,20 +312,18 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file1", 0, len(self.data)), ("complete_file2", 0, None), ] - d = self.do_upload_mdmf() - d.addCallback(self._test_partial_read, self.data, modes, 10000) - return d + result = await self.do_upload_mdmf() + await self._test_partial_read(result, self.data, modes, 10000) - def test_partial_read_sdmf_0(self): + async def test_partial_read_sdmf_0(self) -> None: data = b"" modes = [("all1", 0,0), ("all2", 0,None), ] - d = self.do_upload_sdmf(data=data) - d.addCallback(self._test_partial_read, data, modes, 1) - return d + result = await self.do_upload_sdmf(data=data) + await self._test_partial_read(result, data, modes, 1) - def test_partial_read_sdmf_2(self): + async def test_partial_read_sdmf_2(self) -> None: data = b"hi" modes = [("one_byte", 0, 1), ("last_byte", 1, 1), @@ -407,11 +331,10 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file", 0, 2), ("complete_file2", 0, None), ] - d = self.do_upload_sdmf(data=data) - d.addCallback(self._test_partial_read, data, modes, 1) - return d + result = await self.do_upload_sdmf(data=data) + await self._test_partial_read(result, data, modes, 1) - def test_partial_read_sdmf_90(self): + async def test_partial_read_sdmf_90(self) -> None: modes = [("start_at_middle", 50, 40), ("start_at_middle2", 50, None), ("zero_length_at_start", 0, 0), @@ -420,11 +343,10 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file1", 0, None), ("complete_file2", 0, 90), ] - d = self.do_upload_sdmf() - d.addCallback(self._test_partial_read, self.small_data, modes, 10) - return d + result = await self.do_upload_sdmf() + await self._test_partial_read(result, self.small_data, modes, 10) - def test_partial_read_sdmf_100(self): + async def test_partial_read_sdmf_100(self) -> None: data = b"test data "*10 modes = [("start_at_middle", 50, 50), ("start_at_middle2", 50, None), @@ -433,42 +355,30 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file1", 0, 100), ("complete_file2", 0, None), ] - d = self.do_upload_sdmf(data=data) - d.addCallback(self._test_partial_read, data, modes, 10) - return d + result = await self.do_upload_sdmf(data=data) + await self._test_partial_read(result, data, modes, 10) + async def _test_read_and_download(self, node, expected) -> None: + version = await node.get_best_readable_version() + c = consumer.MemoryConsumer() + await version.read(c) + self.assertThat(expected, Equals(b"".join(c.chunks))) - def _test_read_and_download(self, node, expected): - d = node.get_best_readable_version() - def _read_data(version): - c = consumer.MemoryConsumer() - c2 = consumer.MemoryConsumer() - d2 = defer.succeed(None) - d2.addCallback(lambda ignored: version.read(c)) - d2.addCallback(lambda ignored: - self.assertThat(expected, Equals(b"".join(c.chunks)))) + c2 = consumer.MemoryConsumer() + await version.read(c2, offset=0, size=len(expected)) + self.assertThat(expected, Equals(b"".join(c2.chunks))) - d2.addCallback(lambda ignored: version.read(c2, offset=0, - size=len(expected))) - d2.addCallback(lambda ignored: - self.assertThat(expected, Equals(b"".join(c2.chunks)))) - return d2 - d.addCallback(_read_data) - d.addCallback(lambda ignored: node.download_best_version()) - d.addCallback(lambda data: self.assertThat(expected, Equals(data))) - return d + data = await node.download_best_version() + self.assertThat(expected, Equals(data)) - def test_read_and_download_mdmf(self): - d = self.do_upload_mdmf() - d.addCallback(self._test_read_and_download, self.data) - return d + async def test_read_and_download_mdmf(self) -> None: + result = await self.do_upload_mdmf() + await self._test_read_and_download(result, self.data) - def test_read_and_download_sdmf(self): - d = self.do_upload_sdmf() - d.addCallback(self._test_read_and_download, self.small_data) - return d + async def test_read_and_download_sdmf(self) -> None: + result = await self.do_upload_sdmf() + await self._test_read_and_download(result, self.small_data) - def test_read_and_download_sdmf_zero_length(self): - d = self.do_upload_empty_sdmf() - d.addCallback(self._test_read_and_download, b"") - return d + async def test_read_and_download_sdmf_zero_length(self) -> None: + result = await self.do_upload_empty_sdmf() + await self._test_read_and_download(result, b"") From e72847115be571559167181d5209fa3dccfbd458 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:37:26 -0500 Subject: [PATCH 011/104] news fragment --- newsfragments/3947.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3947.minor diff --git a/newsfragments/3947.minor b/newsfragments/3947.minor new file mode 100644 index 000000000..e69de29bb From 156954c621f7b39406831ca18bed00a2dedf8b70 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:43:01 -0500 Subject: [PATCH 012/104] no longer any need to "daisy chain" this value --- src/allmydata/test/mutable/test_version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index d14cc9295..1d9467694 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -270,7 +270,7 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ async def _test_partial_read(self, node, expected, modes, step) -> None: version = await node.get_best_readable_version() for (name, offset, length) in modes: - version = await self._do_partial_read(version, name, expected, offset, length) + await self._do_partial_read(version, name, expected, offset, length) # then read the whole thing, but only a few bytes at a time, and see # that the results are what we expect. c = consumer.MemoryConsumer() @@ -292,7 +292,6 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ print("got: %s ... %s" % (results[:20], results[-20:])) print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) self.fail("results[%s] != expected_range" % name) - return version # daisy-chained to next call async def test_partial_read_mdmf_0(self) -> None: data = b"" From 05dfa875a771e6ff27006b8fc13aad3dc1709b67 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:46:13 -0500 Subject: [PATCH 013/104] Quite a mypy warning about formatting bytes into a string --- src/allmydata/test/mutable/test_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 1d9467694..87050424b 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -289,8 +289,8 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ if results != expected_range: print("read([%d]+%s) got %d bytes, not %d" % \ (offset, length, len(results), len(expected_range))) - print("got: %s ... %s" % (results[:20], results[-20:])) - print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) + print("got: %r ... %r" % (results[:20], results[-20:])) + print("exp: %r ... %r" % (expected_range[:20], expected_range[-20:])) self.fail("results[%s] != expected_range" % name) async def test_partial_read_mdmf_0(self) -> None: From 11fb194d74599dc3f27f31b14ba340acbd3a2615 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:00:41 -0500 Subject: [PATCH 014/104] kick ci --- newsfragments/3942.minor | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/3942.minor b/newsfragments/3942.minor index e69de29bb..8b1378917 100644 --- a/newsfragments/3942.minor +++ b/newsfragments/3942.minor @@ -0,0 +1 @@ + From 88ee978d98aea740ddf357acf08c85768bd0e950 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:06:24 -0500 Subject: [PATCH 015/104] Some features we depend on are broken in tox 4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 768e44e29..8558abd02 100644 --- a/setup.py +++ b/setup.py @@ -396,7 +396,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pyflakes == 2.2.0", "coverage ~= 5.0", "mock", - "tox", + "tox ~= 3.0", "pytest", "pytest-twisted", "hypothesis >= 3.6.1", From 6485eb5186190a5e73eb55f05b66a42ffb6655ff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:07:38 -0500 Subject: [PATCH 016/104] Also constrain tox here --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 051e690b7..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,10 +133,10 @@ jobs: steps: - "checkout" - - run: + - run: &INSTALL_TOX name: "Install tox" command: | - pip install --user tox + pip install --user 'tox~=3.0' - run: name: "Static-ish code checks" @@ -152,9 +152,7 @@ jobs: - "checkout" - run: - name: "Install tox" - command: | - pip install --user tox + <<: *INSTALL_TOX - run: name: "Make PyInstaller executable" From f6a46c86d24b59110d148b2879c2d7e6647d0501 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:11:59 -0500 Subject: [PATCH 017/104] Populate the wheelhouse with a working version of tox --- .circleci/populate-wheelhouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 519a80cac..39bf4ae4c 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel" # Python packages we need to support the test infrastructure. *Not* packages # Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="tox codecov" +TEST_DEPS="'tox~=3.0' codecov" # Python packages we need to generate test reports for CI infrastructure. # *Not* packages Tahoe-LAFS itself (implement or test suite) need. From 13aa000d0b4c42c550e42a7d85ad2a0035b7b56d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:06:24 -0500 Subject: [PATCH 018/104] Some features we depend on are broken in tox 4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 768e44e29..8558abd02 100644 --- a/setup.py +++ b/setup.py @@ -396,7 +396,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pyflakes == 2.2.0", "coverage ~= 5.0", "mock", - "tox", + "tox ~= 3.0", "pytest", "pytest-twisted", "hypothesis >= 3.6.1", From 666cd24c2b07e5a4ea70100ec3a1554296c47507 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:07:38 -0500 Subject: [PATCH 019/104] Also constrain tox here --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 051e690b7..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,10 +133,10 @@ jobs: steps: - "checkout" - - run: + - run: &INSTALL_TOX name: "Install tox" command: | - pip install --user tox + pip install --user 'tox~=3.0' - run: name: "Static-ish code checks" @@ -152,9 +152,7 @@ jobs: - "checkout" - run: - name: "Install tox" - command: | - pip install --user tox + <<: *INSTALL_TOX - run: name: "Make PyInstaller executable" From 43c044a11b8c98565dcc035608ec82a18affcf08 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:13:29 -0500 Subject: [PATCH 020/104] build me the images --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..89748c5aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # Start with jobs testing various platforms. - - "debian-10": - {} - - "debian-11": - {} + # # Start with jobs testing various platforms. + # - "debian-10": + # {} + # - "debian-11": + # {} - - "ubuntu-20-04": - {} - - "ubuntu-18-04": - requires: - - "ubuntu-20-04" + # - "ubuntu-20-04": + # {} + # - "ubuntu-18-04": + # requires: + # - "ubuntu-20-04" - # Equivalent to RHEL 8; CentOS 8 is dead. - - "oraclelinux-8": - {} + # # Equivalent to RHEL 8; CentOS 8 is dead. + # - "oraclelinux-8": + # {} - - "nixos": - name: "NixOS 21.05" - nixpkgs: "21.05" + # - "nixos": + # name: "NixOS 21.05" + # nixpkgs: "21.05" - - "nixos": - name: "NixOS 21.11" - nixpkgs: "21.11" + # - "nixos": + # name: "NixOS 21.11" + # nixpkgs: "21.11" - # Eventually, test against PyPy 3.8 - #- "pypy27-buster": - # {} + # # Eventually, test against PyPy 3.8 + # #- "pypy27-buster": + # # {} - # Other assorted tasks and configurations - - "codechecks": - {} - - "pyinstaller": - {} - - "c-locale": - {} - # Any locale other than C or UTF-8. - - "another-locale": - {} + # # Other assorted tasks and configurations + # - "codechecks": + # {} + # - "pyinstaller": + # {} + # - "c-locale": + # {} + # # Any locale other than C or UTF-8. + # - "another-locale": + # {} - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-11" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-11" - - "typechecks": - {} - - "docs": - {} + # - "typechecks": + # {} + # - "docs": + # {} - images: - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # images: + # # Build the Docker images used by the ci jobs. This makes the ci jobs + # # faster and takes various spurious failures out of the critical path. + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" - jobs: + # jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From e835ed538fe81876898b36cbccd5fd32bac75554 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:18:40 -0500 Subject: [PATCH 021/104] Okay don't quote it then --- .circleci/populate-wheelhouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 39bf4ae4c..857171979 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel" # Python packages we need to support the test infrastructure. *Not* packages # Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="'tox~=3.0' codecov" +TEST_DEPS="tox~=3.0 codecov" # Python packages we need to generate test reports for CI infrastructure. # *Not* packages Tahoe-LAFS itself (implement or test suite) need. From d5380fe1569f6548df37002949b0169a55ca4151 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:27:37 -0500 Subject: [PATCH 022/104] regular ci config --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89748c5aa..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # # Start with jobs testing various platforms. - # - "debian-10": - # {} - # - "debian-11": - # {} + # Start with jobs testing various platforms. + - "debian-10": + {} + - "debian-11": + {} - # - "ubuntu-20-04": - # {} - # - "ubuntu-18-04": - # requires: - # - "ubuntu-20-04" + - "ubuntu-20-04": + {} + - "ubuntu-18-04": + requires: + - "ubuntu-20-04" - # # Equivalent to RHEL 8; CentOS 8 is dead. - # - "oraclelinux-8": - # {} + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": + {} - # - "nixos": - # name: "NixOS 21.05" - # nixpkgs: "21.05" + - "nixos": + name: "NixOS 21.05" + nixpkgs: "21.05" - # - "nixos": - # name: "NixOS 21.11" - # nixpkgs: "21.11" + - "nixos": + name: "NixOS 21.11" + nixpkgs: "21.11" - # # Eventually, test against PyPy 3.8 - # #- "pypy27-buster": - # # {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} - # # Other assorted tasks and configurations - # - "codechecks": - # {} - # - "pyinstaller": - # {} - # - "c-locale": - # {} - # # Any locale other than C or UTF-8. - # - "another-locale": - # {} + # Other assorted tasks and configurations + - "codechecks": + {} + - "pyinstaller": + {} + - "c-locale": + {} + # Any locale other than C or UTF-8. + - "another-locale": + {} - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-11" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-11" - # - "typechecks": - # {} - # - "docs": - # {} + - "typechecks": + {} + - "docs": + {} - # images: - # # Build the Docker images used by the ci jobs. This makes the ci jobs - # # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + images: + # Build the Docker images used by the ci jobs. This makes the ci jobs + # faster and takes various spurious failures out of the critical path. + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" - # jobs: + jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From ea0426318ea695def6a66593ae44d372b17d194e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 12 Dec 2022 10:02:43 -0500 Subject: [PATCH 023/104] news fragment --- newsfragments/3950.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3950.minor diff --git a/newsfragments/3950.minor b/newsfragments/3950.minor new file mode 100644 index 000000000..e69de29bb From 98e25507df5fdde29b5047c7d325607cb3906b5a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:43:36 -0500 Subject: [PATCH 024/104] A different approach to forcing foolscap in integration tests. --- .github/workflows/ci.yml | 27 +++++++++++++-------------- integration/conftest.py | 16 +++++++--------- integration/util.py | 17 ++++++++--------- src/allmydata/protocol_switch.py | 16 ---------------- src/allmydata/testing/__init__.py | 18 ------------------ 5 files changed, 28 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41de7baed..afbe5c7a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,19 +161,21 @@ jobs: strategy: fail-fast: false matrix: - os: - - windows-latest - # 22.04 has some issue with Tor at the moment: - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - - ubuntu-20.04 - python-version: - - 3.7 - - 3.9 include: - # On macOS don't bother with 3.7, just to get faster builds. - os: macos-latest python-version: 3.9 - + extra-tox-options: "" + - os: windows-latest + python-version: 3.10 + extra-tox-options: "" + # 22.04 has some issue with Tor at the moment: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 + - os: ubuntu-20.04 + python-version: 3.8 + extra-tox-options: "--force-foolscap integration/" + - os: ubuntu-20.04 + python-version: 3.10 + extra-tox-options: "" steps: - name: Install Tor [Ubuntu] @@ -232,10 +234,7 @@ jobs: # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - # Run with Foolscap forced: - __TAHOE_INTEGRATION_FORCE_FOOLSCAP=1 tox -e integration - # Run with Foolscap not forced, which should result in HTTP being used. - __TAHOE_INTEGRATION_FORCE_FOOLSCAP=0 tox -e integration + tox -e integration ${{ matrix.extra-tox-options }} - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/integration/conftest.py b/integration/conftest.py index e284b5cba..5cbe9ad6b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,15 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import sys import shutil from time import sleep @@ -66,6 +57,13 @@ def pytest_addoption(parser): "--coverage", action="store_true", dest="coverage", help="Collect coverage statistics", ) + parser.addoption( + "--force-foolscap", action="store_true", default=False, + dest="force_foolscap", + help=("If set, force Foolscap only for the storage protocol. " + + "Otherwise HTTP will be used.") + ) + @pytest.fixture(autouse=True, scope='session') def eliot_logging(): diff --git a/integration/util.py b/integration/util.py index cde837218..7d885ee6c 100644 --- a/integration/util.py +++ b/integration/util.py @@ -30,7 +30,6 @@ from allmydata.util.configutil import ( write_config, ) from allmydata import client -from allmydata.testing import foolscap_only_for_integration_testing import pytest_twisted @@ -293,14 +292,14 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam u'log_gatherer.furl', flog_gatherer, ) - force_foolscap = foolscap_only_for_integration_testing() - if force_foolscap is not None: - set_config( - config, - 'storage', - 'force_foolscap', - str(force_foolscap), - ) + force_foolscap = request.config.getoption("force_foolscap") + assert force_foolscap in (True, False) + set_config( + config, + 'storage', + 'force_foolscap', + str(force_foolscap), + ) write_config(FilePath(config_path), config) created_d.addCallback(created) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index d88863fdb..b0af84c33 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -30,7 +30,6 @@ from foolscap.api import Tub from .storage.http_server import HTTPServer, build_nurl from .storage.server import StorageServer -from .testing import foolscap_only_for_integration_testing class _PretendToBeNegotiation(type): @@ -171,21 +170,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # and later data, otherwise assume HTTPS. self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): - if foolscap_only_for_integration_testing() == False: - # Tahoe will prefer HTTP storage protocol over Foolscap when possible. - # - # If this is branch is taken, we are running a test that should - # be using HTTP for the storage protocol. As such, we - # aggressively disable Foolscap to ensure that HTTP is in fact - # going to be used. If we hit this branch that means our - # expectation that HTTP will be used was wrong, suggesting a - # bug in either the code of the integration testing setup. - # - # This branch should never be hit in production! - self.transport.loseConnection() - print("FOOLSCAP IS DISABLED, I PITY THE FOOLS WHO SEE THIS MESSAGE") - return - # We're a Foolscap Negotiation server protocol instance: transport = self.transport buf = self._buffer diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py index 119ae4101..e69de29bb 100644 --- a/src/allmydata/testing/__init__.py +++ b/src/allmydata/testing/__init__.py @@ -1,18 +0,0 @@ -import os -from typing import Optional - - -def foolscap_only_for_integration_testing() -> Optional[bool]: - """ - Return whether HTTP storage protocol has been disabled / Foolscap - forced, for purposes of integration testing. - - This is determined by the __TAHOE_INTEGRATION_FORCE_FOOLSCAP environment - variable, which can be 1, 0, or not set, corresponding to results of - ``True``, ``False`` and ``None`` (i.e. default). - """ - force_foolscap = os.environ.get("__TAHOE_INTEGRATION_FORCE_FOOLSCAP") - if force_foolscap is None: - return None - - return bool(int(force_foolscap)) From c5c616afd5146f8cde9dddead3bdbeb092890992 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:44:49 -0500 Subject: [PATCH 025/104] Garbage. --- src/allmydata/test/test_storage_https.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 3d2a31143..a11b0eed5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -181,10 +181,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 106df423be0e864100ba2d96cdb91558d206160a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:52:01 -0500 Subject: [PATCH 026/104] Another approach. --- .github/workflows/ci.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ffc260df..8c3eaf29e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,18 +156,18 @@ jobs: include: - os: macos-latest python-version: 3.9 - extra-tox-options: "" + force-foolscap: false - os: windows-latest python-version: 3.10 - extra-tox-options: "" + force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 python-version: 3.8 - extra-tox-options: "--force-foolscap integration/" + force-foolscap: true - os: ubuntu-20.04 python-version: 3.10 - extra-tox-options: "" + force-foolscap: false steps: - name: Install Tor [Ubuntu] @@ -208,14 +208,24 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" + if: "${{ !matrix.force-foolscap }}" env: # On macOS this is necessary to ensure unix socket paths for tor # aren't too long. On Windows tox won't pass it through so it has no # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - tox -e integration ${{ matrix.extra-tox-options }} + tox -e integration + - name: Run "Python 3 integration tests (force Foolscap)" + if: "${{ matrix.force-foolscap }}" + env: + # On macOS this is necessary to ensure unix socket paths for tor + # aren't too long. On Windows tox won't pass it through so it has no + # effect. On Linux it doesn't make a difference one way or another. + TMPDIR: "/tmp" + run: | + tox -e integration -- --force-foolscap integration/ - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 if: failure() From 742b352861629a0d1f1b900c4c71d6e1ba22a0f2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:52:17 -0500 Subject: [PATCH 027/104] Whitespace. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c3eaf29e..e87337a1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,7 @@ jobs: TMPDIR: "/tmp" run: | tox -e integration -- --force-foolscap integration/ + - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 if: failure() From d05a1313d1773d8f4bf7041d51f4bca1ab0de4b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:54:23 -0500 Subject: [PATCH 028/104] Don't change versions for now, use strings so it'll be future compatible with 3.10. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e87337a1b..01f0890da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,18 +155,18 @@ jobs: matrix: include: - os: macos-latest - python-version: 3.9 + python-version: "3.9" force-foolscap: false - os: windows-latest - python-version: 3.10 + python-version: "3.9" force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 - python-version: 3.8 + python-version: "3.7" force-foolscap: true - os: ubuntu-20.04 - python-version: 3.10 + python-version: "3.9" force-foolscap: false steps: From 366cbf90017874ec24d60bd0df3b3d9f75d79182 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:55:07 -0500 Subject: [PATCH 029/104] Tox is bad? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01f0890da..37f41d06b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,7 @@ jobs: # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - tox -e integration -- --force-foolscap integration/ + tox -e integration -- --force-foolscap,integration/ - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 From 6a1f49551b6683f64b110c2060dcd309c21bdd8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 11:05:09 -0500 Subject: [PATCH 030/104] No, that's not it. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f41d06b..01f0890da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,7 @@ jobs: # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - tox -e integration -- --force-foolscap,integration/ + tox -e integration -- --force-foolscap integration/ - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 From b8680750daa6af924d4da4549780f9cdbc224605 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 12 Dec 2022 11:47:32 -0500 Subject: [PATCH 031/104] pin it in more places --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15e7d8fa4..c1e0c9391 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox tox-gh-actions setuptools + pip install --upgrade codecov "tox<4" tox-gh-actions setuptools pip list - name: Display tool versions @@ -199,7 +199,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox + pip install --upgrade "tox<4" pip list - name: Display tool versions @@ -247,7 +247,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox + pip install --upgrade "tox<4" pip list - name: Display tool versions From 8282fce4cdd574ec4b8cc849070ed4ac2ee03cc8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Dec 2022 08:57:21 -0500 Subject: [PATCH 032/104] build the images again --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..89748c5aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # Start with jobs testing various platforms. - - "debian-10": - {} - - "debian-11": - {} + # # Start with jobs testing various platforms. + # - "debian-10": + # {} + # - "debian-11": + # {} - - "ubuntu-20-04": - {} - - "ubuntu-18-04": - requires: - - "ubuntu-20-04" + # - "ubuntu-20-04": + # {} + # - "ubuntu-18-04": + # requires: + # - "ubuntu-20-04" - # Equivalent to RHEL 8; CentOS 8 is dead. - - "oraclelinux-8": - {} + # # Equivalent to RHEL 8; CentOS 8 is dead. + # - "oraclelinux-8": + # {} - - "nixos": - name: "NixOS 21.05" - nixpkgs: "21.05" + # - "nixos": + # name: "NixOS 21.05" + # nixpkgs: "21.05" - - "nixos": - name: "NixOS 21.11" - nixpkgs: "21.11" + # - "nixos": + # name: "NixOS 21.11" + # nixpkgs: "21.11" - # Eventually, test against PyPy 3.8 - #- "pypy27-buster": - # {} + # # Eventually, test against PyPy 3.8 + # #- "pypy27-buster": + # # {} - # Other assorted tasks and configurations - - "codechecks": - {} - - "pyinstaller": - {} - - "c-locale": - {} - # Any locale other than C or UTF-8. - - "another-locale": - {} + # # Other assorted tasks and configurations + # - "codechecks": + # {} + # - "pyinstaller": + # {} + # - "c-locale": + # {} + # # Any locale other than C or UTF-8. + # - "another-locale": + # {} - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-11" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-11" - - "typechecks": - {} - - "docs": - {} + # - "typechecks": + # {} + # - "docs": + # {} - images: - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # images: + # # Build the Docker images used by the ci jobs. This makes the ci jobs + # # faster and takes various spurious failures out of the critical path. + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" - jobs: + # jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From 815c998c3323829f066752be0f3f707e2716a490 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Dec 2022 09:09:02 -0500 Subject: [PATCH 033/104] regular ci --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89748c5aa..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # # Start with jobs testing various platforms. - # - "debian-10": - # {} - # - "debian-11": - # {} + # Start with jobs testing various platforms. + - "debian-10": + {} + - "debian-11": + {} - # - "ubuntu-20-04": - # {} - # - "ubuntu-18-04": - # requires: - # - "ubuntu-20-04" + - "ubuntu-20-04": + {} + - "ubuntu-18-04": + requires: + - "ubuntu-20-04" - # # Equivalent to RHEL 8; CentOS 8 is dead. - # - "oraclelinux-8": - # {} + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": + {} - # - "nixos": - # name: "NixOS 21.05" - # nixpkgs: "21.05" + - "nixos": + name: "NixOS 21.05" + nixpkgs: "21.05" - # - "nixos": - # name: "NixOS 21.11" - # nixpkgs: "21.11" + - "nixos": + name: "NixOS 21.11" + nixpkgs: "21.11" - # # Eventually, test against PyPy 3.8 - # #- "pypy27-buster": - # # {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} - # # Other assorted tasks and configurations - # - "codechecks": - # {} - # - "pyinstaller": - # {} - # - "c-locale": - # {} - # # Any locale other than C or UTF-8. - # - "another-locale": - # {} + # Other assorted tasks and configurations + - "codechecks": + {} + - "pyinstaller": + {} + - "c-locale": + {} + # Any locale other than C or UTF-8. + - "another-locale": + {} - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-11" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-11" - # - "typechecks": - # {} - # - "docs": - # {} + - "typechecks": + {} + - "docs": + {} - # images: - # # Build the Docker images used by the ci jobs. This makes the ci jobs - # # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + images: + # Build the Docker images used by the ci jobs. This makes the ci jobs + # faster and takes various spurious failures out of the critical path. + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" - # jobs: + jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From be3ace7adebffaa5d410e3f8e248b6db08bd7d50 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Dec 2022 15:39:04 -0500 Subject: [PATCH 034/104] News file. --- newsfragments/3954.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3954.minor diff --git a/newsfragments/3954.minor b/newsfragments/3954.minor new file mode 100644 index 000000000..e69de29bb From 6ae40a932d5504edc66176fc0fbdef45998dec77 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Dec 2022 15:54:19 -0500 Subject: [PATCH 035/104] A much more reasonable number of HTTP connections. --- src/allmydata/storage/http_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 79bf061c9..90bda7fc0 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -323,6 +323,7 @@ class StorageClient(object): swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") pool = HTTPConnectionPool(reactor) + pool.maxPersistentPerHost = 20 if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: cls.TEST_MODE_REGISTER_HTTP_POOL(pool) From 2057f59950fcbd6576d530526d41f9835e42ec7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:35:06 -0500 Subject: [PATCH 036/104] news fragment --- newsfragments/3953.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3953.minor diff --git a/newsfragments/3953.minor b/newsfragments/3953.minor new file mode 100644 index 000000000..e69de29bb From a1cb8893083d06da0c7f1bca760e3333334acac3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:35:10 -0500 Subject: [PATCH 037/104] Take typechecks and codechecks out of the GitHub Actions config There's a dedicated job on CircleCI. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index db4748033..96eed4e40 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage,typechecks,codechecks + 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage 3.10: py310-coverage From 2677f26455f2b91f13e8c453b91f43d9c08f0527 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:46:39 -0500 Subject: [PATCH 038/104] news fragment --- newsfragments/3914.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3914.minor diff --git a/newsfragments/3914.minor b/newsfragments/3914.minor new file mode 100644 index 000000000..e69de29bb From 05c7450376bbbfc4cfe0fb977265b9a1365cf588 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:47:05 -0500 Subject: [PATCH 039/104] Try to use an upcoming python-cryptography feature to avoid some costs If the key is the wrong number of bits then we don't care about any other validation results because we're just going to reject it. So, check that before applying other validation, if possible. This is untested since the version of python-cryptography that supports it is not released yet and I don't feel like setting up a Rust build tool chain at the moment. --- src/allmydata/crypto/rsa.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 95cf01413..96885cfa1 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -72,20 +72,39 @@ def create_signing_keypair_from_string(private_key_der): :returns: 2-tuple of (private_key, public_key) """ - priv_key = load_der_private_key( + load = partial( + load_der_private_key, private_key_der, password=None, backend=default_backend(), ) - if not isinstance(priv_key, rsa.RSAPrivateKey): + + try: + # Load it once without the potentially expensive OpenSSL validation + # checks. These have superlinear complexity. We *will* run them just + # below - but first we'll apply our own constant-time checks. + unsafe_priv_key = load(unsafe_skip_rsa_key_validation=True) + except TypeError: + # cryptography<39 does not support this parameter, so just load the + # key with validation... + unsafe_priv_key = load() + # But avoid *reloading* it since that will run the expensive + # validation *again*. + load = lambda: unsafe_priv_key + + if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey): raise ValueError( "Private Key did not decode to an RSA key" ) - if priv_key.key_size != 2048: + if unsafe_priv_key.key_size != 2048: raise ValueError( "Private Key must be 2048 bits" ) - return priv_key, priv_key.public_key() + + # Now re-load it with OpenSSL's validation applied. + safe_priv_key = load() + + return safe_priv_key, safe_priv_key.public_key() def der_string_from_signing_key(private_key): From c014ad55b1aa42295db7295f7a7d99092fee39fd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:48:02 -0500 Subject: [PATCH 040/104] remove Python 2 boilerplate --- src/allmydata/crypto/rsa.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 96885cfa1..cdd9a6035 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -12,14 +12,9 @@ on any of their methods. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from functools import partial from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend From 761bf9cb9c03313c2d378c7e08cda44468c05ac3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 09:49:50 -0500 Subject: [PATCH 041/104] See if we can get a triggered image build too --- .circleci/config.yml | 72 ++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..446c6b3a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,45 @@ # version: 2.1 +# A template that can be shared between the two different image-building +# workflows. +.images: &IMAGES + jobs: + # Every job that pushes a Docker image from Docker Hub needs to provide + # credentials. Use this first job to define a yaml anchor that can be + # used to supply a CircleCI job context which makes Docker Hub credentials + # available in the environment. + # + # Contexts are managed in the CircleCI web interface: + # + # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" + - "build-image-debian-11": + <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-18-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-20-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-fedora-35": + <<: *DOCKERHUB_CONTEXT + - "build-image-oraclelinux-8": + <<: *DOCKERHUB_CONTEXT + # Restore later as PyPy38 + #- "build-image-pypy27-buster": + # <<: *DOCKERHUB_CONTEXT + +parameters: + build-images: + default: false + type: "boolean" + run-tests: + default: true + type: "boolean" + workflows: ci: + when: "<< pipeline.parameters.run-tests >>" jobs: # Start with jobs testing various platforms. - "debian-10": @@ -64,7 +101,15 @@ workflows: - "docs": {} - images: + triggered-images: + <<: *IMAGES + + # Build as part of the workflow but only if requested. + when: "<< pipeline.parameters.build-images >>" + + scheduled-images: + <<: *IMAGES + # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: @@ -76,31 +121,6 @@ workflows: only: - "master" - jobs: - # Every job that pushes a Docker image from Docker Hub needs to provide - # credentials. Use this first job to define a yaml anchor that can be - # used to supply a CircleCI job context which makes Docker Hub - # credentials available in the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-10": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - - "build-image-debian-11": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-35": - <<: *DOCKERHUB_CONTEXT - - "build-image-oraclelinux-8": - <<: *DOCKERHUB_CONTEXT - # Restore later as PyPy38 - #- "build-image-pypy27-buster": - # <<: *DOCKERHUB_CONTEXT - jobs: dockerhub-auth-template: From 1d248f4bd2d9fd9313cf419f94ec8ebbed31fdd0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 09:56:16 -0500 Subject: [PATCH 042/104] comments --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 446c6b3a9..b7c6cdbee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,17 @@ version: 2.1 # <<: *DOCKERHUB_CONTEXT parameters: + # Control whether the image-building workflow runs as part of this pipeline. + # Generally we do not want this to run because we don't need our + # dependencies to move around all the time and because building the image + # takes a couple minutes. build-images: default: false type: "boolean" + + # Control whether the test-running workflow runs as part of this pipeline. + # Generally we do want this to run because running the tests is the primary + # purpose of this pipeline. run-tests: default: true type: "boolean" From d66d928fb4abf7a81dce91fdca2583c32cd64cf9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 10:07:07 -0500 Subject: [PATCH 043/104] Provide a helper for rebuilding the images --- .circleci/rebuild-images.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .circleci/rebuild-images.sh diff --git a/.circleci/rebuild-images.sh b/.circleci/rebuild-images.sh new file mode 100644 index 000000000..7ee17b8ff --- /dev/null +++ b/.circleci/rebuild-images.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Get your API token here: +# https://app.circleci.com/settings/user/tokens +API_TOKEN=$1 +shift + +# Name the branch you want to trigger the build for +BRANCH=$1 +shift + +curl \ + --verbose \ + --request POST \ + --url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \ + --header 'Circle-Token: $API_TOKEN' \ + --header 'content-type: application/json' \ + --data '{"branch":"$BRANCH","parameters":{"build-images":true,"run-tests":false}}' From 793033caa8004851c7d35f9378101972a84f849f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 12:44:11 -0500 Subject: [PATCH 044/104] Fix quoting on the helper --- .circleci/rebuild-images.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 .circleci/rebuild-images.sh diff --git a/.circleci/rebuild-images.sh b/.circleci/rebuild-images.sh old mode 100644 new mode 100755 index 7ee17b8ff..901651905 --- a/.circleci/rebuild-images.sh +++ b/.circleci/rebuild-images.sh @@ -15,6 +15,6 @@ curl \ --verbose \ --request POST \ --url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \ - --header 'Circle-Token: $API_TOKEN' \ - --header 'content-type: application/json' \ - --data '{"branch":"$BRANCH","parameters":{"build-images":true,"run-tests":false}}' + --header "Circle-Token: $API_TOKEN" \ + --header "content-type: application/json" \ + --data '{"branch":"'"$BRANCH"'","parameters":{"build-images":true,"run-tests":false}}' From f053ef371a2162a9ac9c833694e534c8a0cdfad4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 12:47:50 -0500 Subject: [PATCH 045/104] Get rid of the scheduled image building workflow. --- .circleci/config.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7c6cdbee..722ad390f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,26 +109,12 @@ workflows: - "docs": {} - triggered-images: + images: <<: *IMAGES # Build as part of the workflow but only if requested. when: "<< pipeline.parameters.build-images >>" - scheduled-images: - <<: *IMAGES - - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" - jobs: dockerhub-auth-template: From 63fd24d0607cf4a9440f4837f58347e1caab6300 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 12:48:53 -0500 Subject: [PATCH 046/104] Note how you can get this parameter set --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 722ad390f..480926825 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,10 @@ parameters: # Generally we do not want this to run because we don't need our # dependencies to move around all the time and because building the image # takes a couple minutes. + # + # An easy way to trigger a pipeline with this set to true is with the + # rebuild-images.sh tool in this directory. You can also do so via the + # CircleCI web UI. build-images: default: false type: "boolean" From 8c8a04fa1bbc0e95b78e487d3734504f50be8120 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 13:24:36 -0500 Subject: [PATCH 047/104] news fragment --- newsfragments/3958.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3958.minor diff --git a/newsfragments/3958.minor b/newsfragments/3958.minor new file mode 100644 index 000000000..e69de29bb From 96347e22e27b0614ba5e9797a401129c5bdb8101 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Dec 2022 13:14:49 -0500 Subject: [PATCH 048/104] Make a test demonstrating the problem. --- src/allmydata/test/test_system.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 670ac5868..33b0284da 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -477,9 +477,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _corrupt_mutable_share(self, filename, which): msf = MutableShareFile(filename) - datav = msf.readv([ (0, 1000000) ]) + # Read more than share length: + datav = msf.readv([ (0, 10_000_000) ]) final_share = datav[0] - assert len(final_share) < 1000000 # ought to be truncated + assert len(final_share) < 10_000_000 # ought to be truncated pieces = mutable_layout.unpack_share(final_share) (seqnum, root_hash, IV, k, N, segsize, datalen, verification_key, signature, share_hash_chain, block_hash_tree, @@ -524,7 +525,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): DATA_uploadable = MutableData(DATA) NEWDATA = b"new contents yay" NEWDATA_uploadable = MutableData(NEWDATA) - NEWERDATA = b"this is getting old" + NEWERDATA = b"this is getting old" * 1_000_000 NEWERDATA_uploadable = MutableData(NEWERDATA) d = self.set_up_nodes() From 1d3464a430a465b4b5ef568b33a6138c0a7a495c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 19 Dec 2022 13:37:20 -0500 Subject: [PATCH 049/104] Add end-to-end MDMF test. --- src/allmydata/test/test_system.py | 37 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 33b0284da..55bf0ed8d 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -34,7 +34,7 @@ from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.consumer import MemoryConsumer, download_to_data from allmydata.interfaces import IDirectoryNode, IFileNode, \ - NoSuchChildError, NoSharesError + NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout @@ -520,7 +520,13 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): msf.writev( [(0, final_share)], None) - def test_mutable(self): + def test_mutable_sdmf(self): + return self._test_mutable(SDMF_VERSION) + + def test_mutable_mdmf(self): + return self._test_mutable(MDMF_VERSION) + + def _test_mutable(self, mutable_version): DATA = b"initial contents go here." # 25 bytes % 3 != 0 DATA_uploadable = MutableData(DATA) NEWDATA = b"new contents yay" @@ -533,7 +539,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _create_mutable(res): c = self.clients[0] log.msg("starting create_mutable_file") - d1 = c.create_mutable_file(DATA_uploadable) + d1 = c.create_mutable_file(DATA_uploadable, mutable_version) def _done(res): log.msg("DONE: %s" % (res,)) self._mutable_node_1 = res @@ -555,27 +561,33 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): filename) self.failUnlessEqual(rc, 0) try: + share_type = 'SDMF' if mutable_version == SDMF_VERSION else 'MDMF' self.failUnless("Mutable slot found:\n" in output) - self.failUnless("share_type: SDMF\n" in output) + self.assertIn(f"share_type: {share_type}\n", output) peerid = idlib.nodeid_b2a(self.clients[client_num].nodeid) self.failUnless(" WE for nodeid: %s\n" % peerid in output) self.failUnless(" num_extra_leases: 0\n" in output) self.failUnless(" secrets are for nodeid: %s\n" % peerid in output) - self.failUnless(" SDMF contents:\n" in output) + self.failUnless(f" {share_type} contents:\n" in output) self.failUnless(" seqnum: 1\n" in output) self.failUnless(" required_shares: 3\n" in output) self.failUnless(" total_shares: 10\n" in output) - self.failUnless(" segsize: 27\n" in output, (output, filename)) + if mutable_version == SDMF_VERSION: + self.failUnless(" segsize: 27\n" in output, (output, filename)) self.failUnless(" datalen: 25\n" in output) # the exact share_hash_chain nodes depends upon the sharenum, # and is more of a hassle to compute than I want to deal with # now self.failUnless(" share_hash_chain: " in output) self.failUnless(" block_hash_tree: 1 nodes\n" in output) - expected = (" verify-cap: URI:SSK-Verifier:%s:" % - str(base32.b2a(storage_index), "ascii")) - self.failUnless(expected in output) + if mutable_version == SDMF_VERSION: + expected = (" verify-cap: URI:SSK-Verifier:%s:" % + str(base32.b2a(storage_index), "ascii")) + else: + expected = (" verify-cap: URI:MDMF-Verifier:%s" % + str(base32.b2a(storage_index), "ascii")) + self.assertIn(expected, output) except unittest.FailTest: print() print("dump-share output was:") @@ -695,7 +707,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # when we retrieve this, we should get three signature # failures (where we've mangled seqnum, R, and segsize). The # pubkey mangling - d.addCallback(_corrupt_shares) + + if mutable_version == SDMF_VERSION: + # TODO Corrupting shares in test_systm doesn't work for MDMF right now + d.addCallback(_corrupt_shares) d.addCallback(lambda res: self._newnode3.download_best_version()) d.addCallback(_check_download_5) @@ -703,7 +718,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_empty_file(res): # make sure we can create empty files, this usually screws up the # segsize math - d1 = self.clients[2].create_mutable_file(MutableData(b"")) + d1 = self.clients[2].create_mutable_file(MutableData(b""), mutable_version) d1.addCallback(lambda newnode: newnode.download_best_version()) d1.addCallback(lambda res: self.failUnlessEqual(b"", res)) return d1 From 78e04cc82170f8139b67b419f6cc72e3e75bc477 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:25:22 -0500 Subject: [PATCH 050/104] Modernize cachix usage; attempt to fix CircleCI conditional CIRCLE_PR_NUMBER documentation may just be wrong. It seems like maybe it is never set? Try inspecting the source repo value instead. --- .circleci/config.yml | 73 ++++++++++++-------------------------------- .circleci/lib.sh | 25 +++++++++++++++ 2 files changed, 44 insertions(+), 54 deletions(-) create mode 100644 .circleci/lib.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..4dcf2a2db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -380,7 +380,7 @@ jobs: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.3.16" + image: "nixos/nix:2.10.3" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and @@ -390,27 +390,21 @@ jobs: steps: - "run": - # The nixos/nix image does not include ssh. Install it so the - # `checkout` step will succeed. We also want cachix for - # Nix-friendly caching. + # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" nix-env \ - --file https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz \ + --file $NIXPKGS \ --install \ - -A openssh cachix bash + -A cachix bash + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" - "checkout" - - run: - name: "Cachix setup" - # Record the store paths that exist before we did much. There's no - # reason to cache these, they're either in the image or have to be - # retrieved before we can use cachix to restore from cache. - command: | - cachix use "${CACHIX_NAME}" - nix path-info --all > /tmp/store-path-pre-build - - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" @@ -432,50 +426,21 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<>" + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 3 \ + --max-jobs 2 \ + --argstr pkgsVersion "nixpkgs-<>" - "run": name: "Test" command: | # Let it go somewhat wild for the test suite itself - nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<>" tests.nix - - - run: - # Send any new store objects to cachix. - name: "Push to Cachix" - when: "always" - command: | - # Cribbed from - # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/ - if [ -n "$CIRCLE_PR_NUMBER" ]; then - # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the - # number of the PR being built". Sorry, dear reader, you have - # guessed poorly. It is also conditionally set based on whether - # this is a PR from a fork or not. - # - # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables - echo "Skipping Cachix push for forked PR." - else - # If this *isn't* a build from a fork then we have the Cachix - # write key in our environment and we can push any new objects - # to Cachix. - # - # To decide what to push, we inspect the list of store objects - # that existed before and after we did most of our work. Any - # that are new after the work is probably a useful thing to have - # around so push it to the cache. We exclude all derivation - # objects (.drv files) because they're cheap to reconstruct and - # by the time you know their cache key you've already done all - # the work anyway. - # - # This shell expression for finding the objects and pushing them - # was from the Cachix docs: - # - # https://docs.cachix.org/continuous-integration-setup/circleci.html - # - # but they seem to have removed it now. - bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" - fi + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 8 \ + --argstr pkgsVersion "nixpkgs-<>" \ + tests.nix typechecks: docker: diff --git a/.circleci/lib.sh b/.circleci/lib.sh new file mode 100644 index 000000000..f3fe07bae --- /dev/null +++ b/.circleci/lib.sh @@ -0,0 +1,25 @@ +# Run a command, enabling cache writes to cachix if possible. The command is +# accepted as a variable number of positional arguments (like argv). +function cache_if_able() { + # The `cachix watch-exec ...` does our cache population. When it sees + # something added to the store (I guess) it pushes it to the named cache. + # + # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. + # in-repo jobs will get this from CircleCI configuration but jobs from + # forks may not. + if [ -v CACHIX_AUTH_TOKEN ]; then + echo "Cachix credentials present; will attempt to write to cache." + cachix watch-exec "${CACHIX_NAME}" -- "$@" + else + # If we're building a from a forked repository then we're allowed to + # not have the credentials (but it's also fine if the owner of the + # fork supplied their own). + if [ "${CIRCLE_PR_REPONAME}" == "https://github.com/tahoe-lafs/tahoe-lafs" ]; then + echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." + return 1 + else + echo "Cachix credentials missing; will not attempt cache writes." + "$@" + fi + fi +} From 21af00bf83ff8b1f684d965d772c564d7af92e2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:27:41 -0500 Subject: [PATCH 051/104] Report the CIRCLE_PR_REPONAME too, because who knows --- .circleci/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index f3fe07bae..cc7ce5e97 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -7,6 +7,7 @@ function cache_if_able() { # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. # in-repo jobs will get this from CircleCI configuration but jobs from # forks may not. + echo "Building PR from repo: ${CIRCLE_PR_REPONAME}" if [ -v CACHIX_AUTH_TOKEN ]; then echo "Cachix credentials present; will attempt to write to cache." cachix watch-exec "${CACHIX_NAME}" -- "$@" From 25eb3ca262e0a2bff842e8eff78284f0723faa42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:47:21 -0500 Subject: [PATCH 052/104] Switch to a variable observed in practice There is apparently no CIRCLE_PR_REPONAME set in the runtime environment, either, despite what the docs say. --- .circleci/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index cc7ce5e97..7717cdb18 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -7,7 +7,7 @@ function cache_if_able() { # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. # in-repo jobs will get this from CircleCI configuration but jobs from # forks may not. - echo "Building PR from repo: ${CIRCLE_PR_REPONAME}" + echo "Building PR from user/org: ${CIRCLE_PROJECT_USERNAME}" if [ -v CACHIX_AUTH_TOKEN ]; then echo "Cachix credentials present; will attempt to write to cache." cachix watch-exec "${CACHIX_NAME}" -- "$@" @@ -15,7 +15,7 @@ function cache_if_able() { # If we're building a from a forked repository then we're allowed to # not have the credentials (but it's also fine if the owner of the # fork supplied their own). - if [ "${CIRCLE_PR_REPONAME}" == "https://github.com/tahoe-lafs/tahoe-lafs" ]; then + if [ "${CIRCLE_PROJECT_USERNAME}" == "tahoe-lafs" ]; then echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." return 1 else From 2da3d43b2e4e7a0b6dff7f2efd7a8bb675a00ced Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 07:22:37 -0500 Subject: [PATCH 053/104] news fragment --- newsfragments/3870.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3870.minor diff --git a/newsfragments/3870.minor b/newsfragments/3870.minor new file mode 100644 index 000000000..e69de29bb From a71e873c21836898318512c61c45696a847f4134 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 20 Dec 2022 14:07:12 -0500 Subject: [PATCH 054/104] pycddl 0.2 is broken, 0.3 is missing mmap() support. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8558abd02..dd50e0fcf 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,8 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - "pycddl >= 0.2", + # Need 0.4 to be able to pass in mmap() + "pycddl >= 0.4", # for pid-file support "psutil", From 6d2e797581ed214488da9562e09b78c7dd7299a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:16:18 -0500 Subject: [PATCH 055/104] News file. --- newsfragments/3956.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3956.minor diff --git a/newsfragments/3956.minor b/newsfragments/3956.minor new file mode 100644 index 000000000..e69de29bb From 1a4dcc70e26ce3e720180fb2572e02def7fca351 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:24:31 -0500 Subject: [PATCH 056/104] Support large mutable uploads in a memory-efficient manner. --- src/allmydata/storage/http_server.py | 60 ++++++++++++++++++------- src/allmydata/test/test_storage_http.py | 47 ++++++++++++------- 2 files changed, 74 insertions(+), 33 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3902976ba..d76948d93 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -9,6 +9,10 @@ from functools import wraps from base64 import b64decode import binascii from tempfile import TemporaryFile +from os import SEEK_END, SEEK_SET +from io import BytesIO +import mmap +import sys from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -39,7 +43,7 @@ from cryptography.x509 import load_pem_x509_certificate # TODO Make sure to use pure Python versions? -from cbor2 import dump, loads +import cbor2 from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import ( @@ -515,7 +519,7 @@ class HTTPServer(object): if accept.best == CBOR_MIME_TYPE: request.setHeader("Content-Type", CBOR_MIME_TYPE) f = TemporaryFile() - dump(data, f) + cbor2.dump(data, f) def read_data(offset: int, length: int) -> bytes: f.seek(offset) @@ -527,27 +531,47 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - def _read_encoded(self, request, schema: Schema) -> Any: + def _read_encoded( + self, request, schema: Schema, max_size: int = 1024 * 1024 + ) -> Any: """ Read encoded request body data, decoding it with CBOR by default. - Somewhat arbitrarily, limit body size to 1MB; this may be too low, we - may want to customize per query type, but this is the starting point - for now. + Somewhat arbitrarily, limit body size to 1MiB by default. """ content_type = get_content_type(request.requestHeaders) - if content_type == CBOR_MIME_TYPE: - # Read 1 byte more than 1MB. We expect length to be 1MB or - # less; if it's more assume it's not a legitimate message. - message = request.content.read(1024 * 1024 + 1) - if len(message) > 1024 * 1024: - raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) - schema.validate_cbor(message) - result = loads(message) - return result - else: + if content_type != CBOR_MIME_TYPE: raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) + # Make sure it's not too large: + request.content.seek(SEEK_END, 0) + if request.content.tell() > max_size: + raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) + request.content.seek(SEEK_SET, 0) + + # We don't want to load the whole message into memory, cause it might + # be quite large. The CDDL validator takes a read-only bytes-like + # thing. Luckily, for large request bodies twisted.web will buffer the + # data in a file, so we can use mmap() to get a memory view. The CDDL + # validator will not make a copy, so it won't increase memory usage + # beyond that. + try: + fd = request.content.fileno() + except (ValueError, OSError): + fd = -1 + if fd > 0: + # It's a file, so we can use mmap() to save memory. + message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + else: + message = request.content.read() + schema.validate_cbor(message) + + # The CBOR parser will allocate more memory, but at least we can feed + # it the file-like object, so that if it's large it won't be make two + # copies. + request.content.seek(SEEK_SET, 0) + return cbor2.load(request.content) + ##### Generic APIs ##### @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) @@ -746,7 +770,9 @@ class HTTPServer(object): ) def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) + rtw_request = self._read_encoded( + request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 + ) secrets = ( authorization[Secrets.WRITE_ENABLER], authorization[Secrets.LEASE_RENEW], diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8dbe18545..bc2e10eb6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1186,18 +1186,42 @@ class MutableHTTPAPIsTests(SyncTestCase): ) return storage_index, write_secret, lease_secret - def test_write_can_be_read(self): + def test_write_can_be_read_small_data(self): + """ + Small written data can be read using ``read_share_chunk``. + """ + self.write_can_be_read(b"abcdef") + + def test_write_can_be_read_large_data(self): + """ + Large written data (50MB) can be read using ``read_share_chunk``. + """ + self.write_can_be_read(b"abcdefghij" * 5 * 1024 * 1024) + + def write_can_be_read(self, data): """ Written data can be read using ``read_share_chunk``. """ - storage_index, _, _ = self.create_upload() - data0 = self.http.result_of_with_flush( - self.mut_client.read_share_chunk(storage_index, 0, 1, 7) + lease_secret = urandom(32) + storage_index = urandom(16) + self.http.result_of_with_flush( + self.mut_client.read_test_write_chunks( + storage_index, + urandom(32), + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=0, data=data)] + ), + }, + [], + ) ) - data1 = self.http.result_of_with_flush( - self.mut_client.read_share_chunk(storage_index, 1, 0, 8) + read_data = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, len(data)) ) - self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1")) + self.assertEqual(read_data, data) def test_read_before_write(self): """In combo read/test/write operation, reads happen before writes.""" @@ -1276,15 +1300,6 @@ class MutableHTTPAPIsTests(SyncTestCase): b"aXYZef-0", ) - def test_too_large_write(self): - """ - Writing too large of a chunk results in a REQUEST ENTITY TOO LARGE http - error. - """ - with self.assertRaises(ClientException) as e: - self.create_upload(b"0123456789" * 1024 * 1024) - self.assertEqual(e.exception.code, http.REQUEST_ENTITY_TOO_LARGE) - def test_list_shares(self): """``list_shares()`` returns the shares for a given storage index.""" storage_index, _, _ = self.create_upload() From 54da6eb60a35a53dc981eca3f5172c5fae6faf38 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:34:25 -0500 Subject: [PATCH 057/104] Remove unneeded imports. --- src/allmydata/storage/http_server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d76948d93..47fac879f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -10,9 +10,7 @@ from base64 import b64decode import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET -from io import BytesIO import mmap -import sys from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer From d1b464d0d871b9c09b0f0312136a33bbc08239df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:35:10 -0500 Subject: [PATCH 058/104] Writing large files can involve many writes. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 47fac879f..6d22c92df 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -280,7 +280,7 @@ _SCHEMAS = { "test-write-vectors": { 0*256 share_number : { "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}] - "write": [0*30 {"offset": uint, "data": bstr}] + "write": [* {"offset": uint, "data": bstr}] "new-length": uint / null } } From 15e22dcc52c73721f05a2bc48f84c9b5d21b005e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 2 Jan 2023 19:29:13 -0500 Subject: [PATCH 059/104] Add `keypair` to `NodeMaker.create_mutable_file` Previously `NodeMaker` always took responsibility for generating a keypair to use. Now the caller may supply one. --- src/allmydata/crypto/rsa.py | 14 +++++--------- src/allmydata/nodemaker.py | 19 +++++++++---------- src/allmydata/test/mutable/test_filenode.py | 11 +++++++++++ src/allmydata/test/test_dirnode.py | 3 ++- src/allmydata/test/web/test_web.py | 5 ++++- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 95cf01413..7ea4e6c13 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -9,17 +9,11 @@ features of any objects that `cryptography` documents. That is, the public and private keys are opaque objects; DO NOT depend on any of their methods. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations + +from typing import TypeVar from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend @@ -30,6 +24,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l from allmydata.crypto.error import BadSignature +PublicKey = TypeVar("PublicKey", bound=rsa.RSAPublicKey) +PrivateKey = TypeVar("PrivateKey", bound=rsa.RSAPrivateKey) # This is the value that was used by `pycryptopp`, and we must continue to use it for # both backwards compatibility and interoperability. diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 23ba4b451..1b7ea5f45 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -1,17 +1,12 @@ """ -Ported to Python 3. +Create file nodes of various types. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import weakref from zope.interface import implementer +from twisted.internet.defer import succeed from allmydata.util.assertutil import precondition from allmydata.interfaces import INodeMaker from allmydata.immutable.literal import LiteralFileNode @@ -22,6 +17,7 @@ from allmydata.mutable.publish import MutableData from allmydata.dirnode import DirectoryNode, pack_children from allmydata.unknown import UnknownNode from allmydata.blacklist import ProhibitedNode +from allmydata.crypto.rsa import PublicKey, PrivateKey from allmydata import uri @@ -126,12 +122,15 @@ class NodeMaker(object): return self._create_dirnode(filenode) return None - def create_mutable_file(self, contents=None, version=None): + def create_mutable_file(self, contents=None, version=None, keypair: tuple[PublicKey, PrivateKey] | None = None): if version is None: version = self.mutable_file_default n = MutableFileNode(self.storage_broker, self.secret_holder, self.default_encoding_parameters, self.history) - d = self.key_generator.generate() + if keypair is None: + d = self.key_generator.generate() + else: + d = succeed(keypair) d.addCallback(n.create_with_keys, contents, version=version) d.addCallback(lambda res: n) return d diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 579734433..6c00e4420 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -30,6 +30,7 @@ from allmydata.mutable.publish import MutableData from ..test_download import PausingConsumer, PausingAndStoppingConsumer, \ StoppingConsumer, ImmediatelyStoppingConsumer from .. import common_util as testutil +from ...crypto.rsa import create_signing_keypair from .util import ( FakeStorage, make_nodemaker_with_peers, @@ -65,6 +66,16 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(_created) return d + async def test_create_with_keypair(self): + """ + An SDMF can be created using a given keypair. + """ + (priv, pub) = create_signing_keypair(2048) + node = await self.nodemaker.create_mutable_file(keypair=(pub, priv)) + self.assertThat( + (node.get_privkey(), node.get_pubkey()), + Equals((priv, pub)), + ) def test_create_mdmf(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 67d331430..2319e3ce1 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1619,7 +1619,8 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation return defer.succeed(None) class FakeNodeMaker(NodeMaker): - def create_mutable_file(self, contents=b"", keysize=None, version=None): + def create_mutable_file(self, contents=b"", keysize=None, version=None, keypair=None): + assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs" return defer.succeed(FakeMutableFile(contents)) class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 03cd6e560..4fc9b37e5 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -102,7 +102,10 @@ class FakeNodeMaker(NodeMaker): self.encoding_params, None, self.all_contents).init_from_cap(cap) def create_mutable_file(self, contents=b"", keysize=None, - version=SDMF_VERSION): + version=SDMF_VERSION, + keypair=None, + ): + assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs" n = FakeMutableFileNode(None, None, self.encoding_params, None, self.all_contents) return n.create(contents, version=version) From 23f2d8b019578b4ae9706b2e143ca1bfe44f57b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:28:32 -0500 Subject: [PATCH 060/104] add some type annotations to allmydata.crypto.rsa --- src/allmydata/crypto/rsa.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 7ea4e6c13..a4b2090a0 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -42,12 +42,12 @@ RSA_PADDING = padding.PSS( -def create_signing_keypair(key_size): +def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]: """ Create a new RSA signing (private) keypair from scratch. Can be used with `sign_data` function. - :param int key_size: length of key in bits + :param key_size: length of key in bits :returns: 2-tuple of (private_key, public_key) """ @@ -59,12 +59,12 @@ def create_signing_keypair(key_size): return priv_key, priv_key.public_key() -def create_signing_keypair_from_string(private_key_der): +def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]: """ Create an RSA signing (private) key from previously serialized private key bytes. - :param bytes private_key_der: blob as returned from `der_string_from_signing_keypair` + :param private_key_der: blob as returned from `der_string_from_signing_keypair` :returns: 2-tuple of (private_key, public_key) """ @@ -84,7 +84,7 @@ def create_signing_keypair_from_string(private_key_der): return priv_key, priv_key.public_key() -def der_string_from_signing_key(private_key): +def der_string_from_signing_key(private_key: PrivateKey) -> bytes: """ Serializes a given RSA private key to a DER string @@ -101,7 +101,7 @@ def der_string_from_signing_key(private_key): ) -def der_string_from_verifying_key(public_key): +def der_string_from_verifying_key(public_key: PublicKey) -> bytes: """ Serializes a given RSA public key to a DER string. @@ -117,7 +117,7 @@ def der_string_from_verifying_key(public_key): ) -def create_verifying_key_from_string(public_key_der): +def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey: """ Create an RSA verifying key from a previously serialized public key @@ -133,12 +133,12 @@ def create_verifying_key_from_string(public_key_der): return pub_key -def sign_data(private_key, data): +def sign_data(private_key: PrivateKey, data: bytes) -> bytes: """ :param private_key: the private part of a keypair returned from `create_signing_keypair_from_string` or `create_signing_keypair` - :param bytes data: the bytes to sign + :param data: the bytes to sign :returns: bytes which are a signature of the bytes given as `data`. """ @@ -149,7 +149,7 @@ def sign_data(private_key, data): hashes.SHA256(), ) -def verify_signature(public_key, alleged_signature, data): +def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None: """ :param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key` @@ -169,7 +169,7 @@ def verify_signature(public_key, alleged_signature, data): raise BadSignature() -def _validate_public_key(public_key): +def _validate_public_key(public_key: PublicKey) -> None: """ Internal helper. Checks that `public_key` is a valid cryptography object @@ -180,7 +180,7 @@ def _validate_public_key(public_key): ) -def _validate_private_key(private_key): +def _validate_private_key(private_key: PrivateKey) -> None: """ Internal helper. Checks that `public_key` is a valid cryptography object From f6d9c335261c3f7f9b1ef61fa26e7559a16af141 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:28:59 -0500 Subject: [PATCH 061/104] Give slightly better error messages from rsa key validation failure --- src/allmydata/crypto/rsa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index a4b2090a0..5acc59ab2 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -176,7 +176,7 @@ def _validate_public_key(public_key: PublicKey) -> None: """ if not isinstance(public_key, rsa.RSAPublicKey): raise ValueError( - "public_key must be an RSAPublicKey" + f"public_key must be an RSAPublicKey not {type(public_key)}" ) @@ -187,5 +187,5 @@ def _validate_private_key(private_key: PrivateKey) -> None: """ if not isinstance(private_key, rsa.RSAPrivateKey): raise ValueError( - "private_key must be an RSAPrivateKey" + f"private_key must be an RSAPrivateKey not {type(private_key)}" ) From 6b58b6667786ec712c48486d2711d70868c1fa2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:32:03 -0500 Subject: [PATCH 062/104] Clean up some Python 2 remnants --- src/allmydata/web/filenode.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index dd793888e..678078ba3 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -1,23 +1,13 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - # Use native unicode() as str() to prevent leaking futurebytes in ways that - # break string formattin. - from past.builtins import unicode as str -from past.builtins import long from twisted.web import http, static from twisted.internet import defer from twisted.web.resource import ( - Resource, # note: Resource is an old-style class + Resource, ErrorPage, ) @@ -395,7 +385,7 @@ class FileDownloader(Resource, object): # list of (first,last) inclusive range tuples. filesize = self.filenode.get_size() - assert isinstance(filesize, (int,long)), filesize + assert isinstance(filesize, int), filesize try: # byte-ranges-specifier @@ -408,19 +398,19 @@ class FileDownloader(Resource, object): if first == '': # suffix-byte-range-spec - first = filesize - long(last) + first = filesize - int(last) last = filesize - 1 else: # byte-range-spec # first-byte-pos - first = long(first) + first = int(first) # last-byte-pos if last == '': last = filesize - 1 else: - last = long(last) + last = int(last) if last < first: raise ValueError @@ -456,7 +446,7 @@ class FileDownloader(Resource, object): b'attachment; filename="%s"' % self.filename) filesize = self.filenode.get_size() - assert isinstance(filesize, (int,long)), filesize + assert isinstance(filesize, int), filesize first, size = 0, None contentsize = filesize req.setHeader("accept-ranges", "bytes") From a58d8a567af39f0ab86a2d28794999a84c9b0cf0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:33:37 -0500 Subject: [PATCH 063/104] Clean up some more Python 2 remnants --- src/allmydata/mutable/common.py | 9 +- src/allmydata/mutable/filenode.py | 9 +- src/allmydata/mutable/retrieve.py | 10 +- src/allmydata/mutable/servermap.py | 20 +-- src/allmydata/test/_win_subprocess.py | 210 -------------------------- src/allmydata/test/common.py | 25 +-- 6 files changed, 13 insertions(+), 270 deletions(-) delete mode 100644 src/allmydata/test/_win_subprocess.py diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index 87951c7b2..a2e482d3c 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations MODE_CHECK = "MODE_CHECK" # query all peers MODE_ANYTHING = "MODE_ANYTHING" # one recoverable version diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index cd8cb0dc7..99fdcc085 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import random diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 32aaa72e5..efb2c0f85 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -1,15 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes and str, to prevent API leakage - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401 +from __future__ import annotations import time diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index 211b1fc16..cd220ce0f 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -1,16 +1,8 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - # Doesn't import str to prevent API leakage on Python 2 - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 -from past.builtins import unicode from six import ensure_str import sys, time, copy @@ -203,8 +195,8 @@ class ServerMap(object): (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = verinfo print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" % - (unicode(server.get_name(), "utf-8"), shnum, - seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), k, N, + (str(server.get_name(), "utf-8"), shnum, + seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), k, N, datalength), file=out) if self._problems: print("%d PROBLEMS" % len(self._problems), file=out) @@ -276,7 +268,7 @@ class ServerMap(object): """Take a versionid, return a string that describes it.""" (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = verinfo - return "seq%d-%s" % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8")) + return "seq%d-%s" % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8")) def summarize_versions(self): """Return a string describing which versions we know about.""" @@ -824,7 +816,7 @@ class ServermapUpdater(object): def notify_server_corruption(self, server, shnum, reason): - if isinstance(reason, unicode): + if isinstance(reason, str): reason = reason.encode("utf-8") ss = server.get_storage_server() ss.advise_corrupt_share( @@ -879,7 +871,7 @@ class ServermapUpdater(object): # ok, it's a valid verinfo. Add it to the list of validated # versions. self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d" - % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), + % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), ensure_str(server.get_name()), shnum, k, n, segsize, datalen), parent=lp) diff --git a/src/allmydata/test/_win_subprocess.py b/src/allmydata/test/_win_subprocess.py deleted file mode 100644 index bf9767e73..000000000 --- a/src/allmydata/test/_win_subprocess.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -This module is only necessary on Python 2. Once Python 2 code is dropped, it -can be deleted. -""" - -from future.utils import PY3 -if PY3: - raise RuntimeError("Just use subprocess.Popen") - -# This is necessary to pacify flake8 on Python 3, while we're still supporting -# Python 2. -from past.builtins import unicode - -# -*- coding: utf-8 -*- - -## Copyright (C) 2021 Valentin Lab -## -## Redistribution and use in source and binary forms, with or without -## modification, are permitted provided that the following conditions -## are met: -## -## 1. Redistributions of source code must retain the above copyright -## notice, this list of conditions and the following disclaimer. -## -## 2. Redistributions in binary form must reproduce the above -## copyright notice, this list of conditions and the following -## disclaimer in the documentation and/or other materials provided -## with the distribution. -## -## 3. Neither the name of the copyright holder nor the names of its -## contributors may be used to endorse or promote products derived -## from this software without specific prior written permission. -## -## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -## FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -## COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -## INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -## (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -## HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -## STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -## OF THE POSSIBILITY OF SUCH DAMAGE. -## - -## issue: https://bugs.python.org/issue19264 - -# See allmydata/windows/fixups.py -import sys -assert sys.platform == "win32" - -import os -import ctypes -import subprocess -import _subprocess -from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \ - Structure, sizeof, c_wchar, WinError -from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \ - HANDLE - - -## -## Types -## - -CREATE_UNICODE_ENVIRONMENT = 0x00000400 -LPCTSTR = c_char_p -LPTSTR = c_wchar_p -LPSECURITY_ATTRIBUTES = c_void_p -LPBYTE = ctypes.POINTER(BYTE) - -class STARTUPINFOW(Structure): - _fields_ = [ - ("cb", DWORD), ("lpReserved", LPWSTR), - ("lpDesktop", LPWSTR), ("lpTitle", LPWSTR), - ("dwX", DWORD), ("dwY", DWORD), - ("dwXSize", DWORD), ("dwYSize", DWORD), - ("dwXCountChars", DWORD), ("dwYCountChars", DWORD), - ("dwFillAtrribute", DWORD), ("dwFlags", DWORD), - ("wShowWindow", WORD), ("cbReserved2", WORD), - ("lpReserved2", LPBYTE), ("hStdInput", HANDLE), - ("hStdOutput", HANDLE), ("hStdError", HANDLE), - ] - -LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW) - - -class PROCESS_INFORMATION(Structure): - _fields_ = [ - ("hProcess", HANDLE), ("hThread", HANDLE), - ("dwProcessId", DWORD), ("dwThreadId", DWORD), - ] - -LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION) - - -class DUMMY_HANDLE(ctypes.c_void_p): - - def __init__(self, *a, **kw): - super(DUMMY_HANDLE, self).__init__(*a, **kw) - self.closed = False - - def Close(self): - if not self.closed: - windll.kernel32.CloseHandle(self) - self.closed = True - - def __int__(self): - return self.value - - -CreateProcessW = windll.kernel32.CreateProcessW -CreateProcessW.argtypes = [ - LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES, - LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR, - LPSTARTUPINFOW, LPPROCESS_INFORMATION, -] -CreateProcessW.restype = BOOL - - -## -## Patched functions/classes -## - -def CreateProcess(executable, args, _p_attr, _t_attr, - inherit_handles, creation_flags, env, cwd, - startup_info): - """Create a process supporting unicode executable and args for win32 - - Python implementation of CreateProcess using CreateProcessW for Win32 - - """ - - si = STARTUPINFOW( - dwFlags=startup_info.dwFlags, - wShowWindow=startup_info.wShowWindow, - cb=sizeof(STARTUPINFOW), - ## XXXvlab: not sure of the casting here to ints. - hStdInput=int(startup_info.hStdInput), - hStdOutput=int(startup_info.hStdOutput), - hStdError=int(startup_info.hStdError), - ) - - wenv = None - if env is not None: - ## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar - env = (unicode("").join([ - unicode("%s=%s\0") % (k, v) - for k, v in env.items()])) + unicode("\0") - wenv = (c_wchar * len(env))() - wenv.value = env - - pi = PROCESS_INFORMATION() - creation_flags |= CREATE_UNICODE_ENVIRONMENT - - if CreateProcessW(executable, args, None, None, - inherit_handles, creation_flags, - wenv, cwd, byref(si), byref(pi)): - return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread), - pi.dwProcessId, pi.dwThreadId) - raise WinError() - - -class Popen(subprocess.Popen): - """This superseeds Popen and corrects a bug in cPython 2.7 implem""" - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, to_close, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Code from part of _execute_child from Python 2.7 (9fbb65e) - - There are only 2 little changes concerning the construction of - the the final string in shell mode: we preempt the creation of - the command string when shell is True, because original function - will try to encode unicode args which we want to avoid to be able to - sending it as-is to ``CreateProcess``. - - """ - if not isinstance(args, subprocess.types.StringTypes): - args = subprocess.list2cmdline(args) - - if startupinfo is None: - startupinfo = subprocess.STARTUPINFO() - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", unicode("cmd.exe")) - args = unicode('{} /c "{}"').format(comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - w9xpopen = self._find_w9xpopen() - args = unicode('"%s" %s') % (w9xpopen, args) - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - cp = _subprocess.CreateProcess - _subprocess.CreateProcess = CreateProcess - try: - super(Popen, self)._execute_child( - args, executable, - preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, False, to_close, p2cread, - p2cwrite, c2pread, c2pwrite, errread, errwrite, - ) - finally: - _subprocess.CreateProcess = cp diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index b652b2e48..5728b84ad 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,14 +1,8 @@ """ -Ported to Python 3. +Functionality related to a lot of the test suite. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2, native_str -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr __all__ = [ @@ -117,19 +111,8 @@ from .eliotutil import ( ) from .common_util import ShouldFailMixin # noqa: F401 -if sys.platform == "win32" and PY2: - # Python 2.7 doesn't have good options for launching a process with - # non-ASCII in its command line. So use this alternative that does a - # better job. However, only use it on Windows because it doesn't work - # anywhere else. - from ._win_subprocess import ( - Popen, - ) -else: - from subprocess import ( - Popen, - ) from subprocess import ( + Popen, PIPE, ) @@ -298,7 +281,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str), + introducer_furl = attr.ib(validator=attr.validators.instance_of(str), converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) From 5bad92cfc533a4a236ffb1bec21acfd611aef40d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:34:39 -0500 Subject: [PATCH 064/104] Another Python 2 remnant cleanup --- src/allmydata/test/web/test_web.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 4fc9b37e5..c220b0a0b 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1,14 +1,8 @@ """ -Ported to Python 3. +Tests for a bunch of web-related APIs. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_binary import os.path, re, time From c7bb190290ce2b85cb78605599daeb8aefbc072b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:38:18 -0500 Subject: [PATCH 065/104] Factor some SSK "signature" key handling code into a more reusable shape This gives the test suite access to the derivation function so it can re-derive certain values to use as expected results to compare against actual results. --- src/allmydata/mutable/common.py | 33 ++++++++++++++++++++++++++++++ src/allmydata/mutable/filenode.py | 33 +++++++++++------------------- src/allmydata/mutable/retrieve.py | 7 ++++--- src/allmydata/mutable/servermap.py | 7 ++++--- 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index a2e482d3c..33a1c2731 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -10,6 +10,9 @@ MODE_WRITE = "MODE_WRITE" # replace all shares, probably.. not for initial MODE_READ = "MODE_READ" MODE_REPAIR = "MODE_REPAIR" # query all peers, get the privkey +from allmydata.crypto import aes, rsa +from allmydata.util import hashutil + class NotWriteableError(Exception): pass @@ -61,3 +64,33 @@ class CorruptShareError(BadShareError): class UnknownVersionError(BadShareError): """The share we received was of a version we don't recognize.""" + + +def encrypt_privkey(writekey: bytes, privkey: rsa.PrivateKey) -> bytes: + """ + For SSK, encrypt a private ("signature") key using the writekey. + """ + encryptor = aes.create_encryptor(writekey) + crypttext = aes.encrypt_data(encryptor, privkey) + return crypttext + +def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> rsa.PrivateKey: + """ + The inverse of ``encrypt_privkey``. + """ + decryptor = aes.create_decryptor(writekey) + privkey = aes.decrypt_data(decryptor, enc_privkey) + return privkey + +def derive_mutable_keys(keypair: tuple[rsa.PublicKey, rsa.PrivateKey]) -> tuple[bytes, bytes, bytes]: + """ + Derive the SSK writekey, encrypted writekey, and fingerprint from the + public/private ("verification" / "signature") keypair. + """ + pubkey, privkey = keypair + pubkey_s = rsa.der_string_from_verifying_key(pubkey) + privkey_s = rsa.der_string_from_signing_key(privkey) + writekey = hashutil.ssk_writekey_hash(privkey_s) + encprivkey = encrypt_privkey(writekey, privkey_s) + fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) + return writekey, encprivkey, fingerprint diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 99fdcc085..00b31c52b 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -9,8 +9,6 @@ from zope.interface import implementer from twisted.internet import defer, reactor from foolscap.api import eventually -from allmydata.crypto import aes -from allmydata.crypto import rsa from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \ NotEnoughSharesError, MDMF_VERSION, SDMF_VERSION, IMutableUploadable, \ IMutableFileVersion, IWriteable @@ -21,8 +19,14 @@ from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \ from allmydata.monitor import Monitor from allmydata.mutable.publish import Publish, MutableData,\ TransformingUploadable -from allmydata.mutable.common import MODE_READ, MODE_WRITE, MODE_CHECK, UnrecoverableFileError, \ - UncoordinatedWriteError +from allmydata.mutable.common import ( + MODE_READ, + MODE_WRITE, + MODE_CHECK, + UnrecoverableFileError, + UncoordinatedWriteError, + derive_mutable_keys, +) from allmydata.mutable.servermap import ServerMap, ServermapUpdater from allmydata.mutable.retrieve import Retrieve from allmydata.mutable.checker import MutableChecker, MutableCheckAndRepairer @@ -132,13 +136,10 @@ class MutableFileNode(object): Deferred that fires (with the MutableFileNode instance you should use) when it completes. """ - (pubkey, privkey) = keypair - self._pubkey, self._privkey = pubkey, privkey - pubkey_s = rsa.der_string_from_verifying_key(self._pubkey) - privkey_s = rsa.der_string_from_signing_key(self._privkey) - self._writekey = hashutil.ssk_writekey_hash(privkey_s) - self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s) - self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) + self._pubkey, self._privkey = keypair + self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys( + keypair, + ) if version == MDMF_VERSION: self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint) self._protocol_version = version @@ -164,16 +165,6 @@ class MutableFileNode(object): (contents, type(contents)) return contents(self) - def _encrypt_privkey(self, writekey, privkey): - encryptor = aes.create_encryptor(writekey) - crypttext = aes.encrypt_data(encryptor, privkey) - return crypttext - - def _decrypt_privkey(self, enc_privkey): - decryptor = aes.create_decryptor(self._writekey) - privkey = aes.decrypt_data(decryptor, enc_privkey) - return privkey - def _populate_pubkey(self, pubkey): self._pubkey = pubkey def _populate_required_shares(self, required_shares): diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index efb2c0f85..64573a49a 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -24,7 +24,7 @@ from allmydata import hashtree, codec from allmydata.storage.server import si_b2a from allmydata.mutable.common import CorruptShareError, BadShareError, \ - UncoordinatedWriteError + UncoordinatedWriteError, decrypt_privkey from allmydata.mutable.layout import MDMFSlotReadProxy @implementer(IRetrieveStatus) @@ -923,9 +923,10 @@ class Retrieve(object): def _try_to_validate_privkey(self, enc_privkey, reader, server): - alleged_privkey_s = self._node._decrypt_privkey(enc_privkey) + node_writekey = self._node.get_writekey() + alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) - if alleged_writekey != self._node.get_writekey(): + if alleged_writekey != node_writekey: self.log("invalid privkey from %s shnum %d" % (reader, reader.shnum), level=log.WEIRD, umid="YIw4tA") diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index cd220ce0f..99aa85d24 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -21,7 +21,7 @@ from allmydata.storage.server import si_b2a from allmydata.interfaces import IServermapUpdaterStatus from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \ - MODE_READ, MODE_REPAIR, CorruptShareError + MODE_READ, MODE_REPAIR, CorruptShareError, decrypt_privkey from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MDMFSlotReadProxy @implementer(IServermapUpdaterStatus) @@ -943,9 +943,10 @@ class ServermapUpdater(object): writekey stored in my node. If it is valid, then I set the privkey and encprivkey properties of the node. """ - alleged_privkey_s = self._node._decrypt_privkey(enc_privkey) + node_writekey = self._node.get_writekey() + alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) - if alleged_writekey != self._node.get_writekey(): + if alleged_writekey != node_writekey: self.log("invalid privkey from %r shnum %d" % (server.get_name(), shnum), parent=lp, level=log.WEIRD, umid="aJVccw") From 3423bfb351b717281a9088404f2a090534179fc4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 11:31:29 -0500 Subject: [PATCH 066/104] Expose the pre-constructed keypair functionality to the HTTP API --- src/allmydata/client.py | 34 ++++++++++++++++++-- src/allmydata/test/common.py | 51 ++++++++++++++++++++++++------ src/allmydata/test/web/test_web.py | 51 ++++++++++++++++++++++++++---- src/allmydata/web/filenode.py | 21 ++++++++++-- 4 files changed, 138 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 1a158a1aa..a8238e4ee 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -32,6 +32,7 @@ from allmydata.storage.server import StorageServer, FoolscapStorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper +from allmydata.mutable.filenode import MutableFileNode from allmydata.introducer.client import IntroducerClient from allmydata.util import ( hashutil, base32, pollmixin, log, idlib, @@ -1086,9 +1087,38 @@ class _Client(node.Node, pollmixin.PollMixin): def create_immutable_dirnode(self, children, convergence=None): return self.nodemaker.create_immutable_directory(children, convergence) - def create_mutable_file(self, contents=None, version=None): + def create_mutable_file( + self, + contents: bytes | None = None, + version: int | None = None, + *, + unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None, + ) -> MutableFileNode: + """ + Create *and upload* a new mutable object. + + :param contents: If given, the initial contents for the new object. + + :param version: If given, the mutable file format for the new object + (otherwise a format will be chosen automatically). + + :param unique_keypair: **Warning** This valuely independently + determines the identity of the mutable object to create. There + cannot be two different mutable objects that share a keypair. + They will merge into one object (with undefined contents). + + It is not common to pass a non-None value for this parameter. If + None is given then a new random keypair will be generated. + + If non-None, the given public/private keypair will be used for the + new object. + + :return: A Deferred which will fire with a representation of the new + mutable object after it has been uploaded. + """ return self.nodemaker.create_mutable_file(contents, - version=version) + version=version, + keypair=unique_keypair) def upload(self, uploadable, reactor=None): uploader = self.getServiceNamed("uploader") diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 5728b84ad..37d390da5 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -105,6 +105,7 @@ from allmydata.scripts.common import ( from ..crypto import ( ed25519, + rsa, ) from .eliotutil import ( EliotLoggedRunTest, @@ -622,15 +623,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation MUTABLE_SIZELIMIT = 10000 - def __init__(self, storage_broker, secret_holder, - default_encoding_parameters, history, all_contents): + _public_key: rsa.PublicKey | None + _private_key: rsa.PrivateKey | None + + def __init__(self, + storage_broker, + secret_holder, + default_encoding_parameters, + history, + all_contents, + keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None + ): self.all_contents = all_contents self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION - self.init_from_cap(make_mutable_file_cap()) + self.init_from_cap(make_mutable_file_cap(keypair)) self._k = default_encoding_parameters['k'] self._segsize = default_encoding_parameters['max_segment_size'] - def create(self, contents, key_generator=None, keysize=None, - version=SDMF_VERSION): + if keypair is None: + self._public_key = self._private_key = None + else: + self._public_key, self._private_key = keypair + + def create(self, contents, version=SDMF_VERSION): if version == MDMF_VERSION and \ isinstance(self.my_uri, (uri.ReadonlySSKFileURI, uri.WriteableSSKFileURI)): @@ -826,9 +840,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation return defer.succeed(consumer) -def make_mutable_file_cap(): - return uri.WriteableSSKFileURI(writekey=os.urandom(16), - fingerprint=os.urandom(32)) +def make_mutable_file_cap( + keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None, +) -> uri.WriteableSSKFileURI: + """ + Create a local representation of a mutable object. + + :param keypair: If None, a random keypair will be generated for the new + object. Otherwise, this is the keypair for that object. + """ + if keypair is None: + writekey = os.urandom(16) + fingerprint = os.urandom(32) + else: + pubkey, privkey = keypair + pubkey_s = rsa.der_string_from_verifying_key(pubkey) + privkey_s = rsa.der_string_from_signing_key(privkey) + writekey = hashutil.ssk_writekey_hash(privkey_s) + fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) + + return uri.WriteableSSKFileURI( + writekey=writekey, fingerprint=fingerprint, + ) def make_mdmf_mutable_file_cap(): return uri.WriteableMDMFFileURI(writekey=os.urandom(16), @@ -858,7 +891,7 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None): encoding_params['max_segment_size'] = 128*1024 filenode = FakeMutableFileNode(None, None, encoding_params, None, - all_contents) + all_contents, None) filenode.init_from_cap(cap) if mdmf: filenode.create(MutableData(contents), version=MDMF_VERSION) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index c220b0a0b..bb1f27322 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -8,6 +8,7 @@ from six import ensure_binary import os.path, re, time import treq from urllib.parse import quote as urlquote, unquote as urlunquote +from base64 import urlsafe_b64encode from bs4 import BeautifulSoup @@ -32,6 +33,7 @@ from allmydata.util import fileutil, base32, hashutil, jsonbytes as json from allmydata.util.consumer import download_to_data from allmydata.util.encodingutil import to_bytes from ...util.connection_status import ConnectionStatus +from ...crypto.rsa import PublicKey, PrivateKey, create_signing_keypair, der_string_from_signing_key from ..common import ( EMPTY_CLIENT_CONFIG, FakeCHKFileNode, @@ -59,6 +61,7 @@ from allmydata.interfaces import ( MustBeReadonlyError, ) from allmydata.mutable import servermap, publish, retrieve +from allmydata.mutable.common import derive_mutable_keys from .. import common_util as testutil from ..common_util import TimezoneMixin from ..common_web import ( @@ -94,14 +97,19 @@ class FakeNodeMaker(NodeMaker): def _create_mutable(self, cap): return FakeMutableFileNode(None, None, self.encoding_params, None, - self.all_contents).init_from_cap(cap) - def create_mutable_file(self, contents=b"", keysize=None, - version=SDMF_VERSION, - keypair=None, + self.all_contents, None).init_from_cap(cap) + def create_mutable_file(self, + contents=None, + version=None, + keypair: tuple[PublicKey, PrivateKey] | None=None, ): - assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs" + if contents is None: + contents = b"" + if version is None: + version = SDMF_VERSION + n = FakeMutableFileNode(None, None, self.encoding_params, None, - self.all_contents) + self.all_contents, keypair) return n.create(contents, version=version) class FakeUploader(service.Service): @@ -2865,6 +2873,37 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi "Unknown format: foo", method="post", data=body, headers=headers) + async def test_POST_upload_keypair(self) -> None: + """ + A *POST* creating a new mutable object may include a *private-key* + query argument giving a urlsafe-base64-encoded RSA private key to use + as the "signature key". The given signature key is used, rather than + a new one being generated. + """ + format = "sdmf" + priv, pub = create_signing_keypair(2048) + encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii") + filename = "predetermined-sdmf" + actual_cap = uri.from_string(await self.POST( + self.public_url + + f"/foo?t=upload&format={format}&private-key={encoded_privkey}", + file=(filename, self.NEWFILE_CONTENTS * 100), + )) + # Ideally we would inspect the private ("signature") and public + # ("verification") keys but they are not made easily accessible here + # (ostensibly because we have a FakeMutableFileNode instead of a real + # one). + # + # So, instead, re-compute the writekey and fingerprint and compare + # those against the capability string. + expected_writekey, _, expected_fingerprint = derive_mutable_keys((pub, priv)) + self.assertEqual( + (expected_writekey, expected_fingerprint), + (actual_cap.writekey, actual_cap.fingerprint), + ) + + + def test_POST_upload_format(self): def _check_upload(ign, format, uri_prefix, fn=None): filename = format + ".txt" diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 678078ba3..1b0db5045 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -3,8 +3,10 @@ Ported to Python 3. """ from __future__ import annotations +from base64 import urlsafe_b64decode from twisted.web import http, static +from twisted.web.iweb import IRequest from twisted.internet import defer from twisted.web.resource import ( Resource, @@ -45,6 +47,19 @@ from allmydata.web.check_results import ( ) from allmydata.web.info import MoreInfo from allmydata.util import jsonbytes as json +from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string + + +def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: + """ + Load a keypair from a urlsafe-base64-encoded RSA private key in the + **private-key** argument of the given request, if there is one. + """ + privkey_der = get_arg(request, "private-key", None) + if privkey_der is None: + return None + privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) + return pubkey, privkey class ReplaceMeMixin(object): @@ -54,7 +69,8 @@ class ReplaceMeMixin(object): mutable_type = get_mutable_type(file_format) if mutable_type is not None: data = MutableFileHandle(req.content) - d = client.create_mutable_file(data, version=mutable_type) + keypair = get_keypair(req) + d = client.create_mutable_file(data, version=mutable_type, unique_keypair=keypair) def _uploaded(newnode): d2 = self.parentnode.set_node(self.name, newnode, overwrite=replace) @@ -96,7 +112,8 @@ class ReplaceMeMixin(object): if file_format in ("SDMF", "MDMF"): mutable_type = get_mutable_type(file_format) uploadable = MutableFileHandle(contents.file) - d = client.create_mutable_file(uploadable, version=mutable_type) + keypair = get_keypair(req) + d = client.create_mutable_file(uploadable, version=mutable_type, unique_keypair=keypair) def _uploaded(newnode): d2 = self.parentnode.set_node(self.name, newnode, overwrite=replace) From e236cc95a56ed01dafdc874515ee6ec2727eca83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:36:08 -0500 Subject: [PATCH 067/104] Move get_keypair to a shared location --- src/allmydata/web/common.py | 24 ++++++++++++++---------- src/allmydata/web/filenode.py | 17 +---------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index bf89044a3..2d5fe6297 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -1,15 +1,7 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - from past.builtins import unicode as str # prevent leaking newbytes/newstr into code that can't handle it +from __future__ import annotations from six import ensure_str @@ -21,6 +13,7 @@ except ImportError: import time import json from functools import wraps +from base64 import urlsafe_b64decode from hyperlink import ( DecodedURL, @@ -94,7 +87,7 @@ from allmydata.util.encodingutil import ( to_bytes, ) from allmydata.util import abbreviate - +from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string class WebError(Exception): def __init__(self, text, code=http.BAD_REQUEST): @@ -833,3 +826,14 @@ def abbreviate_time(data): if s >= 0.001: return u"%.1fms" % (1000*s) return u"%.0fus" % (1000000*s) + +def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: + """ + Load a keypair from a urlsafe-base64-encoded RSA private key in the + **private-key** argument of the given request, if there is one. + """ + privkey_der = get_arg(request, "private-key", None) + if privkey_der is None: + return None + privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) + return pubkey, privkey diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 1b0db5045..52ef48e1e 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from __future__ import annotations -from base64 import urlsafe_b64decode - from twisted.web import http, static from twisted.web.iweb import IRequest from twisted.internet import defer @@ -26,6 +24,7 @@ from allmydata.blacklist import ( ) from allmydata.web.common import ( + get_keypair, boolean_of_arg, exception_to_child, get_arg, @@ -47,20 +46,6 @@ from allmydata.web.check_results import ( ) from allmydata.web.info import MoreInfo from allmydata.util import jsonbytes as json -from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string - - -def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: - """ - Load a keypair from a urlsafe-base64-encoded RSA private key in the - **private-key** argument of the given request, if there is one. - """ - privkey_der = get_arg(request, "private-key", None) - if privkey_der is None: - return None - privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) - return pubkey, privkey - class ReplaceMeMixin(object): def replace_me_with_a_child(self, req, client, replace): From 3ff9c45e95be2a8144ff70e35a1e7e41ea67ded6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:40:48 -0500 Subject: [PATCH 068/104] expose the private-key feature in the `tahoe put` cli --- src/allmydata/scripts/cli.py | 15 ++++++- src/allmydata/scripts/tahoe_put.py | 41 +++++++++++------ src/allmydata/test/cli/test_put.py | 72 ++++++++++++++++++++++++++---- src/allmydata/web/filenode.py | 1 - src/allmydata/web/unlinked.py | 13 ++---- 5 files changed, 108 insertions(+), 34 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 55975b8c5..aa7644e18 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -180,10 +180,21 @@ class GetOptions(FileStoreOptions): class PutOptions(FileStoreOptions): optFlags = [ ("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"), - ] + ] + optParameters = [ ("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"), - ] + + ("private-key-path", None, None, + "***Warning*** " + "It is possible to use this option to spoil the normal security properties of mutable objects. " + "It is also possible to corrupt or destroy data with this option. " + "For mutables only, " + "this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. " + "The private key must be handled at least as strictly as the resulting capability string. " + "A single private key must not be used for more than one mutable." + ), + ] def parseArgs(self, arg1=None, arg2=None): # see Examples below diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 1ea45e8ea..fd746a43d 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,23 +1,31 @@ """ -Ported to Python 3. +Implement the ``tahoe put`` command. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations from io import BytesIO from urllib.parse import quote as url_quote +from base64 import urlsafe_b64encode +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +from twisted.python.filepath import FilePath + +from allmydata.crypto.rsa import der_string_from_signing_key from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.util.encodingutil import quote_output +def load_private_key(path: str) -> str: + """ + Load a private key from a file and return it in a format appropriate + to include in the HTTP request. + """ + privkey = load_pem_private_key(FilePath(path).getContent(), password=None) + derbytes = der_string_from_signing_key(privkey) + return urlsafe_b64encode(derbytes).decode("ascii") + def put(options): """ @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose @@ -29,6 +37,10 @@ def put(options): from_file = options.from_file to_file = options.to_file mutable = options['mutable'] + if options["private-key-path"] is None: + private_key = None + else: + private_key = load_private_key(options["private-key-path"]) format = options['format'] if options['quiet']: verbosity = 0 @@ -79,6 +91,12 @@ def put(options): queryargs = [] if mutable: queryargs.append("mutable=true") + if private_key is not None: + queryargs.append(f"private-key={private_key}") + else: + if private_key is not None: + raise Exception("Can only supply a private key for mutables.") + if format: queryargs.append("format=%s" % format) if queryargs: @@ -92,10 +110,7 @@ def put(options): if verbosity > 0: print("waiting for file data on stdin..", file=stderr) # We're uploading arbitrary files, so this had better be bytes: - if PY2: - stdinb = stdin - else: - stdinb = stdin.buffer + stdinb = stdin.buffer data = stdinb.read() infileobj = BytesIO(data) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 03306ab71..6f33a14fd 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -1,19 +1,17 @@ """ -Ported to Python 3. +Tests for the ``tahoe put`` CLI tool. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations +from typing import Callable, Awaitable, TypeVar import os.path from twisted.trial import unittest from twisted.python import usage +from twisted.python.filepath import FilePath +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +from allmydata.uri import from_string from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli @@ -22,6 +20,9 @@ from ..common_util import skip_if_cannot_represent_filename from allmydata.util.encodingutil import get_io_encoding from allmydata.util.fileutil import abspath_expanduser_unicode from .common import CLITestMixin +from allmydata.mutable.common import derive_mutable_keys + +T = TypeVar("T") class Put(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -215,6 +216,59 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): return d + async def test_unlinked_mutable_specified_private_key(self) -> None: + """ + A new unlinked mutable can be created using a specified private + key. + """ + self.basedir = "cli/Put/unlinked-mutable-with-key" + await self._test_mutable_specified_key( + lambda do_cli, pempath, datapath: do_cli( + "put", "--mutable", "--private-key-path", pempath.path, + stdin=datapath.getContent(), + ), + ) + + async def test_linked_mutable_specified_private_key(self) -> None: + """ + A new linked mutable can be created using a specified private key. + """ + self.basedir = "cli/Put/linked-mutable-with-key" + await self._test_mutable_specified_key( + lambda do_cli, pempath, datapath: do_cli( + "put", "--mutable", "--private-key-path", pempath.path, datapath.path, + ), + ) + + async def _test_mutable_specified_key( + self, + run: Callable[[Callable[..., T], FilePath, FilePath], Awaitable[T]], + ) -> None: + """ + A helper for testing mutable creation. + + :param run: A function to do the creation. It is called with + ``self.do_cli`` and the path to a private key PEM file and a data + file. It returns whatever ``do_cli`` returns. + """ + self.set_up_grid(oneshare=True) + + pempath = FilePath(__file__).parent().sibling("data").child("openssl-rsa-2048.txt") + datapath = FilePath(self.basedir).child("data") + datapath.setContent(b"Hello world" * 1024) + + (rc, out, err) = await run(self.do_cli, pempath, datapath) + self.assertEqual(rc, 0, (out, err)) + cap = from_string(out.strip()) + # The capability is derived from the key we specified. + privkey = load_pem_private_key(pempath.getContent(), password=None) + pubkey = privkey.public_key() + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + self.assertEqual( + (writekey, fingerprint), + (cap.writekey, cap.fingerprint), + ) + def test_mutable(self): # echo DATA1 | tahoe put --mutable - uploaded.txt # echo DATA2 | tahoe put - uploaded.txt # should modify-in-place diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 52ef48e1e..680ca3331 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -4,7 +4,6 @@ Ported to Python 3. from __future__ import annotations from twisted.web import http, static -from twisted.web.iweb import IRequest from twisted.internet import defer from twisted.web.resource import ( Resource, diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 425622496..2c7be6f30 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations from urllib.parse import quote as urlquote @@ -25,6 +18,7 @@ from twisted.web.template import ( from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle from allmydata.web.common import ( + get_keypair, get_arg, boolean_of_arg, convert_children_json, @@ -48,7 +42,8 @@ def PUTUnlinkedSSK(req, client, version): # SDMF: files are small, and we can only upload data req.content.seek(0) data = MutableFileHandle(req.content) - d = client.create_mutable_file(data, version=version) + keypair = get_keypair(req) + d = client.create_mutable_file(data, version=version, unique_keypair=keypair) d.addCallback(lambda n: n.get_uri()) return d From 1d125b7be80f4ac3300fffd6e343d41df6debde5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:51:36 -0500 Subject: [PATCH 069/104] news fragment --- newsfragments/3962.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3962.feature diff --git a/newsfragments/3962.feature b/newsfragments/3962.feature new file mode 100644 index 000000000..86cf62781 --- /dev/null +++ b/newsfragments/3962.feature @@ -0,0 +1 @@ +Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties. \ No newline at end of file From e829b891b300dba118a18490cfc74e356b36915c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:51:59 -0500 Subject: [PATCH 070/104] important data file ... --- src/allmydata/test/data/openssl-rsa-2048.txt | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/allmydata/test/data/openssl-rsa-2048.txt diff --git a/src/allmydata/test/data/openssl-rsa-2048.txt b/src/allmydata/test/data/openssl-rsa-2048.txt new file mode 100644 index 000000000..8f989f42c --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048.txt @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDF1MeXulDWFO05 +YXCh8aqNc1dS1ddJRzsti4BOWuDOepUc0oCaSIcC5aR7XJ+vhX7a02mTIwvLcuEH +8sxx0BJU4jCDpRI6aAqaKJxwZx1e6AcVFJDl7vzymhvWhqHuKh0jTvwM2zONWTwV +V8m2PbDdxu0Prwdx+Mt2sDT6xHEhJj5fI/GUDUEdkhLJF6DQSulFRqqd0qP7qcI9 +fSHZbM7MywfzqFUe8J1+tk4fBh2v7gNzN1INpzh2mDtLPAtxr4ZPtEb/0D0U4PsP +CniOHP0U8sF3VY0+K5qoCQr92cLRJvT/vLpQGVNUTFdFrtbqDoFxUCyEH4FUqRDX +2mVrPo2xAgMBAAECggEAA0Ev1y5/1NTPbgytBeIIH3d+v9hwKDbHecVoMwnOVeFJ +BZpONrOToovhAc1NXH2wj4SvwYWfpJ1HR9piDAuLeKlnuUu4ffzfE0gQok4E+v4r +2yg9ZcYBs/NOetAYVwbq960tiv/adFRr71E0WqbfS3fBx8q2L3Ujkkhd98PudUhQ +izbrTvkT7q00OPCWGwgWepMlLEowUWwZehGI0MlbONg7SbRraZZmG586Iy0tpC3e +AM7wC1/ORzFqcRgTIxXizQ5RHL7S0OQPLhbEJbuwPonNjze3p0EP4wNBELZTaVOd +xeA22Py4Bh/d1q3aEgbwR7tLyA8YfEzshTaY6oV8AQKBgQD0uFo8pyWk0AWXfjzn +jV4yYyPWy8pJA6YfAJAST8m7B/JeYgGlfHxTlNZiB40DsJq08tOZv3HAubgMpFIa +reuDxPqo6/Quwdy4Syu+AFhY48KIuwuoegG/L+5qcQLE69r1w71ZV6wUvLmXYX2I +Y6nYz+OdpD1JrMIr6Js60XURsQKBgQDO8yWl7ufIDKMbQpbs0PgUQsH4FtzGcP4J +j/7/8GfhKYt6rPsrojPHUbAi1+25xBVOuhm0Zx2ku2t+xPIMJoS+15EcER1Z2iHZ +Zci9UGpJpUxGcUhG7ETF1HZv0xKHcEOl9eIIOcAP9Vd9DqnGk85gy6ti6MHe/5Tn +IMD36OQ8AQKBgQDwqE7NMM67KnslRNaeG47T3F0FQbm3XehCuqnz6BUJYcI+gQD/ +fdFB3K+LDcPmKgmqAtaGbxdtoPXXMM0xQXHHTrH15rxmMu1dK0dj/TDkkW7gSZko +YHtRSdCbSnGfuBXG9GxD7QzkA8g7j3sE4oXIGoDLqRVAW61DwubMy+jlsQKBgGNB ++Zepi1/Gt+BWQt8YpzPIhRIBnShMf3uEphCJdLlo3K4dE2btKBp8UpeTq0CDDJky +5ytAndYp0jf+K/2p59dEuyOUDdjPp5aGnA446JGkB35tzPW/Uoj0C049FVEChl+u +HBhH4peE285uXv2QXNbOOMh6zKmxOfDVI9iDyhwBAoGBAIXq2Ar0zDXXaL3ncEKo +pXt9BZ8OpJo2pvB1t2VPePOwEQ0wdT+H62fKNY47NiF9+LyS541/ps5Qhv6AmiKJ +Z7I0Vb6+sxQljYH/LNW+wc2T/pIAi/7sNcmnlBtZfoVwt99bk2CyoRALPLWHYCkh +c7Tty2bZzDZy6aCX+FGRt5N/ +-----END PRIVATE KEY----- From 2dc6466ef5c260e7cb5d46cfcbfc456399f29b71 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 17:12:59 -0500 Subject: [PATCH 071/104] fix some errors reported by mypy --- src/allmydata/crypto/rsa.py | 6 +++--- src/allmydata/mutable/common.py | 2 +- src/allmydata/test/cli/test_put.py | 4 ++-- src/allmydata/test/common.py | 2 +- src/allmydata/test/web/test_web.py | 1 + src/allmydata/web/common.py | 19 ++++++++++++------- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 5acc59ab2..f16b1b95f 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -13,7 +13,7 @@ on any of their methods. from __future__ import annotations -from typing import TypeVar +from typing_extensions import TypeAlias from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend @@ -24,8 +24,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l from allmydata.crypto.error import BadSignature -PublicKey = TypeVar("PublicKey", bound=rsa.RSAPublicKey) -PrivateKey = TypeVar("PrivateKey", bound=rsa.RSAPrivateKey) +PublicKey: TypeAlias = rsa.RSAPublicKey +PrivateKey: TypeAlias = rsa.RSAPrivateKey # This is the value that was used by `pycryptopp`, and we must continue to use it for # both backwards compatibility and interoperability. diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index 33a1c2731..a498ab02a 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -66,7 +66,7 @@ class UnknownVersionError(BadShareError): """The share we received was of a version we don't recognize.""" -def encrypt_privkey(writekey: bytes, privkey: rsa.PrivateKey) -> bytes: +def encrypt_privkey(writekey: bytes, privkey: bytes) -> bytes: """ For SSK, encrypt a private ("signature") key using the writekey. """ diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 6f33a14fd..98407bb7e 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -3,7 +3,7 @@ Tests for the ``tahoe put`` CLI tool. """ from __future__ import annotations -from typing import Callable, Awaitable, TypeVar +from typing import Callable, Awaitable, TypeVar, Any import os.path from twisted.trial import unittest from twisted.python import usage @@ -242,7 +242,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): async def _test_mutable_specified_key( self, - run: Callable[[Callable[..., T], FilePath, FilePath], Awaitable[T]], + run: Callable[[Any, FilePath, FilePath], Awaitable[tuple[int, bytes, bytes]]], ) -> None: """ A helper for testing mutable creation. diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 37d390da5..db2921e86 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -635,7 +635,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None ): self.all_contents = all_contents - self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION + self.file_types: dict[bytes, int] = {} # storage index => MDMF_VERSION or SDMF_VERSION self.init_from_cap(make_mutable_file_cap(keypair)) self._k = default_encoding_parameters['k'] self._segsize = default_encoding_parameters['max_segment_size'] diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index bb1f27322..7793c023c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -90,6 +90,7 @@ class FakeNodeMaker(NodeMaker): 'happy': 7, 'max_segment_size':128*1024 # 1024=KiB } + all_contents: dict[bytes, object] def _create_lit(self, cap): return FakeCHKFileNode(cap, self.all_contents) def _create_immutable(self, cap): diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 2d5fe6297..25e9e51f3 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -6,7 +6,7 @@ from __future__ import annotations from six import ensure_str try: - from typing import Optional, Union, Tuple, Any + from typing import Optional, Union, Tuple, Any, TypeVar except ImportError: pass @@ -706,8 +706,9 @@ def url_for_string(req, url_string): ) return url +T = TypeVar("T") -def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any] +def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bool = False) -> Union[bytes, tuple[bytes, ...], T]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take @@ -719,13 +720,17 @@ def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Uni :return: Either bytes or tuple of bytes. """ if isinstance(argname, str): - argname = argname.encode("utf-8") + argname_bytes = argname.encode("utf-8") + else: + argname_bytes = argname + if isinstance(default, str): default = default.encode("utf-8") + results = [] - if argname in req.args: - results.extend(req.args[argname]) - argname_unicode = str(argname, "utf-8") + if argname_bytes in req.args: + results.extend(req.args[argname_bytes]) + argname_unicode = str(argname_bytes, "utf-8") if req.fields and argname_unicode in req.fields: value = req.fields[argname_unicode].value if isinstance(value, str): @@ -832,7 +837,7 @@ def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: Load a keypair from a urlsafe-base64-encoded RSA private key in the **private-key** argument of the given request, if there is one. """ - privkey_der = get_arg(request, "private-key", None) + privkey_der = get_arg(request, "private-key", default=None, multiple=False) if privkey_der is None: return None privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) From a806b2fabaf985a28d9872ea2db292a30c91810f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 18:11:47 -0500 Subject: [PATCH 072/104] Fix some more mypy errors --- src/allmydata/crypto/rsa.py | 22 +++++++++++++++------- src/allmydata/web/common.py | 16 +++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 3554ec557..f1ba0d10a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -14,6 +14,7 @@ on any of their methods. from __future__ import annotations from typing_extensions import TypeAlias +from typing import Callable from functools import partial @@ -70,22 +71,29 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK :returns: 2-tuple of (private_key, public_key) """ - load = partial( - load_der_private_key, + _load = partial( + load_der_public_key, private_key_der, password=None, backend=default_backend(), ) + def load_with_validation() -> PrivateKey: + return _load() + + def load_without_validation() -> PrivateKey: + return _load(unsafe_skip_rsa_key_validation=True) + + # Load it once without the potentially expensive OpenSSL validation + # checks. These have superlinear complexity. We *will* run them just + # below - but first we'll apply our own constant-time checks. + load: Callable[[], PrivateKey] = load_without_validation try: - # Load it once without the potentially expensive OpenSSL validation - # checks. These have superlinear complexity. We *will* run them just - # below - but first we'll apply our own constant-time checks. - unsafe_priv_key = load(unsafe_skip_rsa_key_validation=True) + unsafe_priv_key = load() except TypeError: # cryptography<39 does not support this parameter, so just load the # key with validation... - unsafe_priv_key = load() + unsafe_priv_key = load_without_validation() # But avoid *reloading* it since that will run the expensive # validation *again*. load = lambda: unsafe_priv_key diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 25e9e51f3..8f81aec94 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -6,7 +6,7 @@ from __future__ import annotations from six import ensure_str try: - from typing import Optional, Union, Tuple, Any, TypeVar + from typing import Optional, Union, Tuple, Any, TypeVar, Literal, overload except ImportError: pass @@ -708,7 +708,13 @@ def url_for_string(req, url_string): T = TypeVar("T") -def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bool = False) -> Union[bytes, tuple[bytes, ...], T]: +@overload +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> bytes: ... + +@overload +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> tuple[bytes, ...]: ... + +def get_arg(req: IRequest, argname: str | bytes, default: T | None = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take @@ -724,9 +730,6 @@ def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bo else: argname_bytes = argname - if isinstance(default, str): - default = default.encode("utf-8") - results = [] if argname_bytes in req.args: results.extend(req.args[argname_bytes]) @@ -740,6 +743,9 @@ def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bo return tuple(results) if results: return results[0] + + if isinstance(default, str): + return default.encode("utf-8") return default From c9e23dea13a51114f47c8c64ed1caf34d57cfcdc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 20:59:48 -0500 Subject: [PATCH 073/104] we should always be able to get these and we always need overload now --- src/allmydata/web/common.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 8f81aec94..b55a49d4d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -5,10 +5,7 @@ from __future__ import annotations from six import ensure_str -try: - from typing import Optional, Union, Tuple, Any, TypeVar, Literal, overload -except ImportError: - pass +from typing import Optional, Union, TypeVar, Literal, overload import time import json From 85234b07a0ce4f12be3676562a0ed7a24f650d27 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 21:00:04 -0500 Subject: [PATCH 074/104] load the right kind of key! --- src/allmydata/crypto/rsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index f1ba0d10a..b8de52c4f 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -72,7 +72,7 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK :returns: 2-tuple of (private_key, public_key) """ _load = partial( - load_der_public_key, + load_der_private_key, private_key_der, password=None, backend=default_backend(), From 8c56ccad725ebb6c9ae745a60d6258ed0f8b5b2f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 21:00:10 -0500 Subject: [PATCH 075/104] fall back to *with* validation, not without --- src/allmydata/crypto/rsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index b8de52c4f..c21b522cd 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -93,7 +93,7 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK except TypeError: # cryptography<39 does not support this parameter, so just load the # key with validation... - unsafe_priv_key = load_without_validation() + unsafe_priv_key = load_with_validation() # But avoid *reloading* it since that will run the expensive # validation *again*. load = lambda: unsafe_priv_key From e893d06cb35fd7c32a8682b3f538086c663f30de Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 21:00:21 -0500 Subject: [PATCH 076/104] RSAPrivateKey certainly does have this method I don't know why mypy fails to see it. --- src/allmydata/crypto/rsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index c21b522cd..3ad893dbf 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -123,7 +123,7 @@ def der_string_from_signing_key(private_key: PrivateKey) -> bytes: :returns: bytes representing `private_key` """ _validate_private_key(private_key) - return private_key.private_bytes( + return private_key.private_bytes( # type: ignore[attr-defined] encoding=Encoding.DER, format=PrivateFormat.PKCS8, encryption_algorithm=NoEncryption(), From 3ce5ee6f0304131a5cc4ebbbed2237842084f3d2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 7 Jan 2023 07:17:40 -0500 Subject: [PATCH 077/104] get Literal from somewhere it is more likely to be --- src/allmydata/web/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index b55a49d4d..470170e7d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -5,7 +5,8 @@ from __future__ import annotations from six import ensure_str -from typing import Optional, Union, TypeVar, Literal, overload +from typing import Optional, Union, TypeVar, overload +from typing_extensions import Literal import time import json From 22227c70948e6c057287b50ca05224f14227bb09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 10:31:48 -0500 Subject: [PATCH 078/104] Support old pycddl too so nix can keep working. --- setup.py | 7 +++++-- src/allmydata/storage/http_server.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index dd50e0fcf..e4045d76b 100644 --- a/setup.py +++ b/setup.py @@ -137,8 +137,11 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - # Need 0.4 to be able to pass in mmap() - "pycddl >= 0.4", + # Ideally we want 0.4+ to be able to pass in mmap(), but it's not strictly + # necessary yet until we fix the workaround to + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963 in + # allmydata.storage.http_server. + "pycddl", # for pid-file support "psutil", diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6d22c92df..aa2e532cb 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,6 +11,7 @@ import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET import mmap +from importlib.metadata import version as get_package_version from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -59,6 +60,12 @@ from ..util.base32 import rfc3548_alphabet from allmydata.interfaces import BadWriteEnablerError +# Until we figure out Nix (https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963), +# need to support old pycddl which can only take bytes: +from distutils.version import LooseVersion +PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion("0.4") + + class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" @@ -557,7 +564,7 @@ class HTTPServer(object): fd = request.content.fileno() except (ValueError, OSError): fd = -1 - if fd > 0: + if fd > 0 and not PYCDDL_BYTES_ONLY: # It's a file, so we can use mmap() to save memory. message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) else: From f6d9c5a1b237c4819635be52eba22d2563d02260 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 10:46:09 -0500 Subject: [PATCH 079/104] Fix PyInstaller. --- src/allmydata/storage/http_server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index aa2e532cb..306d54ea0 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,7 +11,7 @@ import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET import mmap -from importlib.metadata import version as get_package_version +from importlib.metadata import version as get_package_version, PackageNotFoundError from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -63,7 +63,15 @@ from allmydata.interfaces import BadWriteEnablerError # Until we figure out Nix (https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963), # need to support old pycddl which can only take bytes: from distutils.version import LooseVersion -PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion("0.4") + +try: + PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion( + "0.4" + ) +except PackageNotFoundError: + # This can happen when building PyInstaller distribution. We'll just assume + # you installed a modern pycddl, cause why wouldn't you? + PYCDDL_BYTES_ONLY = False class ClientSecretsException(Exception): From 825fd64dddc860e24fd85dcc891c728ef35779e6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 10:52:24 -0500 Subject: [PATCH 080/104] News file. --- newsfragments/3964.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3964.removed diff --git a/newsfragments/3964.removed b/newsfragments/3964.removed new file mode 100644 index 000000000..1c2c3e544 --- /dev/null +++ b/newsfragments/3964.removed @@ -0,0 +1 @@ +Python 3.7 is no longer supported. \ No newline at end of file From 1482d419181c76d760359b09867c6d667f0753c9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:01:45 -0500 Subject: [PATCH 081/104] Drop 3.7. --- .circleci/config.yml | 52 ++++------------------------------------ .github/workflows/ci.yml | 10 +++----- README.rst | 2 +- setup.py | 9 +++---- tox.ini | 6 +---- 5 files changed, 13 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4dcf2a2db..21f60368c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -167,12 +167,7 @@ jobs: command: | dist/Tahoe-LAFS/tahoe --version - debian-10: &DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py3.7" - user: "nobody" - + debian-11: &DEBIAN environment: &UTF_8_ENVIRONMENT # In general, the test suite is not allowed to fail while the job # succeeds. But you can set this to "yes" if you want it to be @@ -184,7 +179,7 @@ jobs: # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "py37" + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -252,15 +247,11 @@ jobs: /tmp/venv/bin/codecov fi - debian-11: - <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/debian:11-py3.9" user: "nobody" - environment: - <<: *UTF_8_ENVIRONMENT - TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + # Restore later using PyPy3.8 # pypy27-buster: @@ -312,22 +303,6 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - ubuntu-18-04: &UBUNTU_18_04 - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3.7" - user: "nobody" - - environment: - <<: *UTF_8_ENVIRONMENT - # The default trial args include --rterrors which is incompatible with - # this reporter on Python 3. So drop that and just specify the - # reporter. - TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py37" - - ubuntu-20-04: <<: *DEBIAN docker: @@ -445,7 +420,7 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3.7" + image: "tahoelafsci/ubuntu:20.04-py3.9" steps: - "checkout" @@ -457,7 +432,7 @@ jobs: docs: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3.7" + image: "tahoelafsci/ubuntu:20.04-py3.9" steps: - "checkout" @@ -508,15 +483,6 @@ jobs: docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION} - build-image-debian-10: - <<: *BUILD_IMAGE - - environment: - DISTRO: "debian" - TAG: "10" - PYTHON_VERSION: "3.7" - - build-image-debian-11: <<: *BUILD_IMAGE @@ -525,14 +491,6 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-ubuntu-18-04: - <<: *BUILD_IMAGE - - environment: - DISTRO: "ubuntu" - TAG: "18.04" - PYTHON_VERSION: "3.7" - build-image-ubuntu-20-04: <<: *BUILD_IMAGE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7fa3244b..80b312008 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,21 +48,20 @@ jobs: - windows-latest - ubuntu-latest python-version: - - "3.7" - "3.8" - "3.9" - "3.10" include: - # On macOS don't bother with 3.7-3.8, just to get faster builds. + # On macOS don't bother with 3.8, just to get faster builds. - os: macos-latest python-version: "3.9" - os: macos-latest python-version: "3.10" # We only support PyPy on Linux at the moment. - - os: ubuntu-latest - python-version: "pypy-3.7" - os: ubuntu-latest python-version: "pypy-3.8" + - os: ubuntu-latest + python-version: "pypy-3.9" steps: # See https://github.com/actions/checkout. A fetch-depth of 0 @@ -162,9 +161,6 @@ jobs: force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - - os: ubuntu-20.04 - python-version: "3.7" - force-foolscap: true - os: ubuntu-20.04 python-version: "3.9" force-foolscap: false diff --git a/README.rst b/README.rst index 317378fae..bbf88610d 100644 --- a/README.rst +++ b/README.rst @@ -56,7 +56,7 @@ Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS `__ 🐍 Python 2 ----------- -Python 3.7 or later is now required. +Python 3.8 or later is required. If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1. diff --git a/setup.py b/setup.py index 1075e2129..edef7a4c3 100644 --- a/setup.py +++ b/setup.py @@ -223,7 +223,7 @@ def run_command(args, cwd=None): use_shell = sys.platform == "win32" try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+ + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.8+ print("Warning: unable to run %r." % (" ".join(args),)) print(e) return None @@ -374,8 +374,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. 3.11 is not supported yet. - python_requires=">=3.7, <3.11", + # We support Python 3.8 or later. 3.11 is not supported yet. + python_requires=">=3.8, <3.11", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See @@ -388,9 +388,6 @@ setup(name="tahoe-lafs", # also set in __init__.py ], "test": [ "flake8", - # On Python 3.7, importlib_metadata v5 breaks flake8. - # https://github.com/python/importlib_metadata/issues/407 - "importlib_metadata<5; python_version < '3.8'", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it diff --git a/tox.ini b/tox.ini index 96eed4e40..3e2dacbb2 100644 --- a/tox.ini +++ b/tox.ini @@ -7,11 +7,9 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage 3.10: py310-coverage - pypy-3.7: pypy37 pypy-3.8: pypy38 pypy-3.9: pypy39 @@ -19,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39,310}-{coverage},pypy27,pypy37,pypy38,pypy39,integration +envlist = typechecks,codechecks,py{38,39,310}-{coverage},pypy27,pypy38,pypy39,integration minversion = 2.4 [testenv] @@ -49,8 +47,6 @@ deps = # regressions in new releases of this package that cause us the kind of # suffering we're trying to avoid with the above pins. certifi - # VCS hooks support - py37,!coverage: pre-commit # We add usedevelop=False because testing against a true installation gives # more useful results. From c4153d54055e0c2e33343de812a77b14f7069439 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:03:15 -0500 Subject: [PATCH 082/104] Additional changes. --- newsfragments/3964.removed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3964.removed b/newsfragments/3964.removed index 1c2c3e544..d022f94af 100644 --- a/newsfragments/3964.removed +++ b/newsfragments/3964.removed @@ -1 +1 @@ -Python 3.7 is no longer supported. \ No newline at end of file +Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested. \ No newline at end of file From 8c418832bb6ba62fefb1c740bea973cbaf5c07f9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:06:57 -0500 Subject: [PATCH 083/104] Remove references to missing jobs. --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21f60368c..834c5f9d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,16 +15,11 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - "debian-10": - {} - "debian-11": {} - "ubuntu-20-04": {} - - "ubuntu-18-04": - requires: - - "ubuntu-20-04" # Equivalent to RHEL 8; CentOS 8 is dead. - "oraclelinux-8": @@ -85,12 +80,8 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-10": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT - "build-image-fedora-35": From 34f5da7246b725589d5dc1ec4febb1bce8c4029e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:08:31 -0500 Subject: [PATCH 084/104] And add back necessary anchor. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 834c5f9d2..9080c43ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-11": + - "build-image-debian-11": &DOCKERHUB_CONTEXT <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT From 6bb57e248de0410faf1d7b6ab43efa99a3b1e5eb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 13:09:59 -0500 Subject: [PATCH 085/104] Try to switch Nix off 3.7. --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 5f4db2c78..e4f2dd4d4 100644 --- a/default.nix +++ b/default.nix @@ -29,7 +29,7 @@ in , pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use # for dependency resolution -, pythonVersion ? "python37" # a string choosing the python derivation from +, pythonVersion ? "python39" # a string choosing the python derivation from # nixpkgs to target , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, From 7b2f19b0fa0ada0f1d5a05056927fc84fa3fa327 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 13:16:04 -0500 Subject: [PATCH 086/104] Switch Nix off 3.7 some more. --- tests.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.nix b/tests.nix index dd477c273..f8ed678f3 100644 --- a/tests.nix +++ b/tests.nix @@ -5,7 +5,7 @@ in { pkgsVersion ? "nixpkgs-21.11" , pkgs ? import sources.${pkgsVersion} { } , pypiData ? sources.pypi-deps-db -, pythonVersion ? "python37" +, pythonVersion ? "python39" , mach-nix ? import sources.mach-nix { inherit pkgs pypiData; python = pythonVersion; @@ -21,7 +21,7 @@ let inherit pkgs; lib = pkgs.lib; }; - tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test; + tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test; # Get the Tahoe-LAFS package itself. This does not include test # requirements and we don't ask for test requirements so that we can just From b05793e56b3bfcedd75c633fc9186adf02aad48a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 13:45:22 -0500 Subject: [PATCH 087/104] Meaningless tweak to rerun CI. --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index bbf88610d..56451701a 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,7 @@ Tahoe-LAFS was first designed in 2007, following the "principle of least authori Please read more about Tahoe-LAFS architecture `here `__. + ✅ Installation --------------- From 046d9cf802e541d77f203ad1d29edbff6f628790 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 14:25:47 -0500 Subject: [PATCH 088/104] Another meaningless tweak. --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 56451701a..bbf88610d 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,6 @@ Tahoe-LAFS was first designed in 2007, following the "principle of least authori Please read more about Tahoe-LAFS architecture `here `__. - ✅ Installation --------------- From ccb5956645e4763999b35e165095681fe8025b40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 10 Jan 2023 11:04:22 -0500 Subject: [PATCH 089/104] 0 is also valid FD. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8e987b4d1..387353d24 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -572,7 +572,7 @@ class HTTPServer(object): fd = request.content.fileno() except (ValueError, OSError): fd = -1 - if fd > 0 and not PYCDDL_BYTES_ONLY: + if fd >= 0 and not PYCDDL_BYTES_ONLY: # It's a file, so we can use mmap() to save memory. message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) else: From 828fc588c506586e56c610bdfdc1a170eb73acdc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 10 Jan 2023 11:10:31 -0500 Subject: [PATCH 090/104] Add minimal docstrings. --- src/allmydata/test/test_system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 55bf0ed8d..10a64c1fe 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -521,9 +521,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_mutable_sdmf(self): + """SDMF mutables can be uploaded, downloaded, and many other things.""" return self._test_mutable(SDMF_VERSION) def test_mutable_mdmf(self): + """MDMF mutables can be uploaded, downloaded, and many other things.""" return self._test_mutable(MDMF_VERSION) def _test_mutable(self, mutable_version): From 98624f3d6a32882d5fb2283fa61690226e81299e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 09:53:07 -0500 Subject: [PATCH 091/104] Attempt to workaround for 3960. --- .github/workflows/ci.yml | 9 +++++++++ newsfragments/3960.minor | 0 2 files changed, 9 insertions(+) create mode 100644 newsfragments/3960.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80b312008..2a8f5246e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,17 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run tox for corresponding Python version + if: ${{ !contains(matrix.os, 'windows') }} run: python -m tox + # On Windows, a non-blocking pipe might respond (when emulating Unix-y + # API) with ENOSPC to indicate buffer full. Trial doesn't handle this + # well. So, we pipe the output through Get-Content (similar to piping + # through cat) to make buffer handling someone else's problem. + - name: Run tox for corresponding Python version + if: ${{ contains(matrix.os, 'windows') }} + run: python -m tox | Get-Content + - name: Upload eliot.log uses: actions/upload-artifact@v3 with: diff --git a/newsfragments/3960.minor b/newsfragments/3960.minor new file mode 100644 index 000000000..e69de29bb From e142f051b497124cb8828106242d14cb46643431 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 10:01:09 -0500 Subject: [PATCH 092/104] Cats solve problems, right? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a8f5246e..1b8ab0d57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,11 +91,11 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this - # well. So, we pipe the output through Get-Content (similar to piping - # through cat) to make buffer handling someone else's problem. + # well. So, we pipe the output through cat to make buffer handling someone + # else's problem. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} - run: python -m tox | Get-Content + run: python -m tox | cat - name: Upload eliot.log uses: actions/upload-artifact@v3 From dd89ca6d4f2738f92534f0b33da2a365b9f30016 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 10:36:39 -0500 Subject: [PATCH 093/104] Another approach. --- .github/workflows/ci.yml | 8 ++++--- misc/windows-enospc/passthrough.py | 38 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 misc/windows-enospc/passthrough.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b8ab0d57..1eb12bdd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,11 +91,13 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this - # well. So, we pipe the output through cat to make buffer handling someone - # else's problem. + # well. So, we pipe the output through pipethrough that will hopefully be + # able to do the right thing by using Windows APIs. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} - run: python -m tox | cat + run: | + pip install twisted + python -m tox | python misc/windows-enospc/passthrough.py - name: Upload eliot.log uses: actions/upload-artifact@v3 diff --git a/misc/windows-enospc/passthrough.py b/misc/windows-enospc/passthrough.py new file mode 100644 index 000000000..6be3d7dbe --- /dev/null +++ b/misc/windows-enospc/passthrough.py @@ -0,0 +1,38 @@ +""" +Writing to non-blocking pipe can result in ENOSPC when using Unix APIs on +Windows. So, this program passes through data from stdin to stdout, using +Windows APIs instead of Unix-y APIs. +""" + +import sys + +from twisted.internet.stdio import StandardIO +from twisted.internet import reactor +from twisted.internet.protocol import Protocol +from twisted.internet.interfaces import IHalfCloseableProtocol +from twisted.internet.error import ReactorNotRunning +from zope.interface import implementer + +@implementer(IHalfCloseableProtocol) +class Passthrough(Protocol): + def readConnectionLost(self): + self.transport.loseConnection() + + def writeConnectionLost(self): + try: + reactor.stop() + except ReactorNotRunning: + pass + + def dataReceived(self, data): + self.transport.write(data) + + def connectionLost(self, reason): + try: + reactor.stop() + except ReactorNotRunning: + pass + + +std = StandardIO(Passthrough()) +reactor.run() From e997bb9c398d191c28c7a82840a2df3bbbb657e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 10:39:50 -0500 Subject: [PATCH 094/104] Also need pywin32. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eb12bdd5..0f8040e96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} run: | - pip install twisted + pip install twisted pywin32 python -m tox | python misc/windows-enospc/passthrough.py - name: Upload eliot.log From ee5ad549fed7150d2a6782ca4409b27c5821d546 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 11:12:00 -0500 Subject: [PATCH 095/104] Clarify. --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f8040e96..011bfc1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,9 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this - # well. So, we pipe the output through pipethrough that will hopefully be - # able to do the right thing by using Windows APIs. + # well, so it breaks test runs. To attempt to solve this, we pipe the + # output through pipethrough that will hopefully be able to do the right + # thing by using Windows APIs. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} run: | From a765d8a35b8d0bdbe6163585dcdd96f4d6f7eddf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 11:18:05 -0500 Subject: [PATCH 096/104] Unused. --- misc/windows-enospc/passthrough.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misc/windows-enospc/passthrough.py b/misc/windows-enospc/passthrough.py index 6be3d7dbe..1d4cd48bb 100644 --- a/misc/windows-enospc/passthrough.py +++ b/misc/windows-enospc/passthrough.py @@ -4,8 +4,6 @@ Windows. So, this program passes through data from stdin to stdout, using Windows APIs instead of Unix-y APIs. """ -import sys - from twisted.internet.stdio import StandardIO from twisted.internet import reactor from twisted.internet.protocol import Protocol From 2490f0f58ac3b170cf768d7e40cabebbb699c4db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:33:37 -0500 Subject: [PATCH 097/104] some minor rationalization of the return type --- src/allmydata/web/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 470170e7d..c49354217 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -707,12 +707,12 @@ def url_for_string(req, url_string): T = TypeVar("T") @overload -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> bytes: ... +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> T | bytes: ... @overload -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> tuple[bytes, ...]: ... +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> T | tuple[bytes, ...]: ... -def get_arg(req: IRequest, argname: str | bytes, default: T | None = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take From 2d23e2e6401e5c696002e9eefadd30193a39201f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:37:07 -0500 Subject: [PATCH 098/104] some doc improvements --- src/allmydata/client.py | 16 +++++++++------- src/allmydata/scripts/cli.py | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a8238e4ee..73672f30a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1102,16 +1102,18 @@ class _Client(node.Node, pollmixin.PollMixin): :param version: If given, the mutable file format for the new object (otherwise a format will be chosen automatically). - :param unique_keypair: **Warning** This valuely independently - determines the identity of the mutable object to create. There - cannot be two different mutable objects that share a keypair. - They will merge into one object (with undefined contents). + :param unique_keypair: **Warning** This value independently determines + the identity of the mutable object to create. There cannot be two + different mutable objects that share a keypair. They will merge + into one object (with undefined contents). - It is not common to pass a non-None value for this parameter. If - None is given then a new random keypair will be generated. + It is common to pass a None value (or not pass a valuye) for this + parameter. In these cases, a new random keypair will be + generated. If non-None, the given public/private keypair will be used for the - new object. + new object. The expected use-case is for implementing compliance + tests. :return: A Deferred which will fire with a representation of the new mutable object after it has been uploaded. diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index aa7644e18..579b37906 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -189,6 +189,7 @@ class PutOptions(FileStoreOptions): "***Warning*** " "It is possible to use this option to spoil the normal security properties of mutable objects. " "It is also possible to corrupt or destroy data with this option. " + "Most users will not need this option and can ignore it. " "For mutables only, " "this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. " "The private key must be handled at least as strictly as the resulting capability string. " From e6ef45d3371321e5a4667e43a3d53223bdc153c1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:37:12 -0500 Subject: [PATCH 099/104] test that we can also download the mutable --- src/allmydata/test/cli/test_put.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 98407bb7e..e8f83188a 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -268,6 +268,11 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): (writekey, fingerprint), (cap.writekey, cap.fingerprint), ) + # Also the capability we were given actually refers to the data we + # uploaded. + (rc, out, err) = await self.do_cli("get", out.strip()) + self.assertEqual(rc, 0, (out, err)) + self.assertEqual(out, datapath.getContent().decode("ascii")) def test_mutable(self): # echo DATA1 | tahoe put --mutable - uploaded.txt From 47ec418f7afcc745e7b0376e9485812351efab1a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:43:54 -0500 Subject: [PATCH 100/104] Test that we can also download the mutable data via the web interface --- src/allmydata/test/web/test_web.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 7793c023c..4c828817a 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -2885,10 +2885,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi priv, pub = create_signing_keypair(2048) encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii") filename = "predetermined-sdmf" + expected_content = self.NEWFILE_CONTENTS * 100 actual_cap = uri.from_string(await self.POST( self.public_url + f"/foo?t=upload&format={format}&private-key={encoded_privkey}", - file=(filename, self.NEWFILE_CONTENTS * 100), + file=(filename, expected_content), )) # Ideally we would inspect the private ("signature") and public # ("verification") keys but they are not made easily accessible here @@ -2903,7 +2904,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi (actual_cap.writekey, actual_cap.fingerprint), ) - + # And the capability we got can be used to download the data we + # uploaded. + downloaded_content = await self.GET(f"/uri/{actual_cap.to_string().decode('ascii')}") + self.assertEqual(expected_content, downloaded_content) def test_POST_upload_format(self): def _check_upload(ign, format, uri_prefix, fn=None): From c856f1aa298a10ea91707fcce91899931a8924c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 16:16:55 -0500 Subject: [PATCH 101/104] Censor private key values in the HTTP log, too. --- src/allmydata/test/web/test_webish.py | 10 +++ src/allmydata/webish.py | 98 +++++++++++++++------------ 2 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 4a77d21ae..050f77d1c 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -202,6 +202,16 @@ class TahoeLAFSSiteTests(SyncTestCase): ), ) + def test_private_key_censoring(self): + """ + The log event for a request including a **private-key** query + argument has the private key value censored. + """ + self._test_censoring( + b"/uri?uri=URI:CHK:aaa:bbb&private-key=AAAAaaaabbbb==", + b"/uri?uri=[CENSORED]&private-key=[CENSORED]", + ) + def test_uri_censoring(self): """ The log event for a request for **/uri/** has the capability value diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 519b3e1f0..1b2b8192a 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1,18 +1,12 @@ """ -Ported to Python 3. +General web server-related utilities. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations from six import ensure_str import re, time, tempfile +from urllib.parse import parse_qsl, urlencode from cgi import ( FieldStorage, @@ -45,40 +39,37 @@ from .web.storage_plugins import ( ) -if PY2: - FileUploadFieldStorage = FieldStorage -else: - class FileUploadFieldStorage(FieldStorage): - """ - Do terrible things to ensure files are still bytes. +class FileUploadFieldStorage(FieldStorage): + """ + Do terrible things to ensure files are still bytes. - On Python 2, uploaded files were always bytes. On Python 3, there's a - heuristic: if the filename is set on a field, it's assumed to be a file - upload and therefore bytes. If no filename is set, it's Unicode. + On Python 2, uploaded files were always bytes. On Python 3, there's a + heuristic: if the filename is set on a field, it's assumed to be a file + upload and therefore bytes. If no filename is set, it's Unicode. - Unfortunately, we always want it to be bytes, and Tahoe-LAFS also - enables setting the filename not via the MIME filename, but via a - separate field called "name". + Unfortunately, we always want it to be bytes, and Tahoe-LAFS also + enables setting the filename not via the MIME filename, but via a + separate field called "name". - Thus we need to do this ridiculous workaround. Mypy doesn't like it - either, thus the ``# type: ignore`` below. + Thus we need to do this ridiculous workaround. Mypy doesn't like it + either, thus the ``# type: ignore`` below. - Source for idea: - https://mail.python.org/pipermail/python-dev/2017-February/147402.html - """ - @property # type: ignore - def filename(self): - if self.name == "file" and not self._mime_filename: - # We use the file field to upload files, see directory.py's - # _POST_upload. Lack of _mime_filename means we need to trick - # FieldStorage into thinking there is a filename so it'll - # return bytes. - return "unknown-filename" - return self._mime_filename + Source for idea: + https://mail.python.org/pipermail/python-dev/2017-February/147402.html + """ + @property # type: ignore + def filename(self): + if self.name == "file" and not self._mime_filename: + # We use the file field to upload files, see directory.py's + # _POST_upload. Lack of _mime_filename means we need to trick + # FieldStorage into thinking there is a filename so it'll + # return bytes. + return "unknown-filename" + return self._mime_filename - @filename.setter - def filename(self, value): - self._mime_filename = value + @filename.setter + def filename(self, value): + self._mime_filename = value class TahoeLAFSRequest(Request, object): @@ -180,12 +171,7 @@ def _logFormatter(logDateTime, request): queryargs = b"" else: path, queryargs = x - # there is a form handler which redirects POST /uri?uri=FOO into - # GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make - # sure we censor these too. - if queryargs.startswith(b"uri="): - queryargs = b"uri=[CENSORED]" - queryargs = b"?" + queryargs + queryargs = b"?" + censor(queryargs) if path.startswith(b"/uri/"): path = b"/uri/[CENSORED]" elif path.startswith(b"/file/"): @@ -207,6 +193,30 @@ def _logFormatter(logDateTime, request): ) +def censor(queryargs: bytes) -> bytes: + """ + Replace potentially sensitive values in query arguments with a + constant string. + """ + args = parse_qsl(queryargs.decode("ascii"), keep_blank_values=True, encoding="utf8") + result = [] + for k, v in args: + if k == "uri": + # there is a form handler which redirects POST /uri?uri=FOO into + # GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make + # sure we censor these. + v = "[CENSORED]" + elif k == "private-key": + # Likewise, sometimes a private key is supplied with mutable + # creation. + v = "[CENSORED]" + + result.append((k, v)) + + # Customize safe to try to leave our markers intact. + return urlencode(result, safe="[]").encode("ascii") + + class TahoeLAFSSite(Site, object): """ The HTTP protocol factory used by Tahoe-LAFS. From 1a807a0232996447c47dd095a13498488e16d541 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 16:32:32 -0500 Subject: [PATCH 102/104] mollify the type checker --- src/allmydata/crypto/rsa.py | 9 +++++++-- src/allmydata/scripts/tahoe_put.py | 3 ++- src/allmydata/test/cli/test_put.py | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 3ad893dbf..e579a3d2a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -79,10 +79,14 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK ) def load_with_validation() -> PrivateKey: - return _load() + k = _load() + assert isinstance(k, PrivateKey) + return k def load_without_validation() -> PrivateKey: - return _load(unsafe_skip_rsa_key_validation=True) + k = _load(unsafe_skip_rsa_key_validation=True) + assert isinstance(k, PrivateKey) + return k # Load it once without the potentially expensive OpenSSL validation # checks. These have superlinear complexity. We *will* run them just @@ -159,6 +163,7 @@ def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey: public_key_der, backend=default_backend(), ) + assert isinstance(pub_key, PublicKey) return pub_key diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index fd746a43d..c04b6b4bc 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -11,7 +11,7 @@ from cryptography.hazmat.primitives.serialization import load_pem_private_key from twisted.python.filepath import FilePath -from allmydata.crypto.rsa import der_string_from_signing_key +from allmydata.crypto.rsa import PrivateKey, der_string_from_signing_key from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -23,6 +23,7 @@ def load_private_key(path: str) -> str: to include in the HTTP request. """ privkey = load_pem_private_key(FilePath(path).getContent(), password=None) + assert isinstance(privkey, PrivateKey) derbytes = der_string_from_signing_key(privkey) return urlsafe_b64encode(derbytes).decode("ascii") diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index e8f83188a..c5f32a553 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -11,6 +11,7 @@ from twisted.python.filepath import FilePath from cryptography.hazmat.primitives.serialization import load_pem_private_key +from allmydata.crypto.rsa import PrivateKey from allmydata.uri import from_string from allmydata.util import fileutil from allmydata.scripts.common import get_aliases @@ -262,6 +263,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): cap = from_string(out.strip()) # The capability is derived from the key we specified. privkey = load_pem_private_key(pempath.getContent(), password=None) + assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) self.assertEqual( From 0eee22cccf8af1a95b176c2a91b4de91539f9411 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 Jan 2023 09:53:38 -0500 Subject: [PATCH 103/104] Pin older charset_normalizer. --- newsfragments/3966.bugfix | 1 + setup.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 newsfragments/3966.bugfix diff --git a/newsfragments/3966.bugfix b/newsfragments/3966.bugfix new file mode 100644 index 000000000..ead94c47c --- /dev/null +++ b/newsfragments/3966.bugfix @@ -0,0 +1 @@ +Fix incompatibility with newer versions of the transitive charset_normalizer dependency when using PyInstaller. \ No newline at end of file diff --git a/setup.py b/setup.py index 3681dc441..a9b42d522 100644 --- a/setup.py +++ b/setup.py @@ -148,6 +148,13 @@ install_requires = [ # for pid-file support "psutil", "filelock", + + # treq needs requests, requests needs charset_normalizer, + # charset_normalizer breaks PyInstaller + # (https://github.com/Ousret/charset_normalizer/issues/253). So work around + # this by using a lower version number. Once upstream issue is fixed, or + # requests drops charset_normalizer, this can go away. + "charset_normalizer < 3", ] setup_requires = [ From e64c6b02e6370290a9c6d9ca17257b2e736f693a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Jan 2023 10:29:22 -0500 Subject: [PATCH 104/104] Fix a typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 011bfc1ea..4d67f09bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this # well, so it breaks test runs. To attempt to solve this, we pipe the - # output through pipethrough that will hopefully be able to do the right + # output through passthrough.py that will hopefully be able to do the right # thing by using Windows APIs. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }}