From e9aaaaccc4c9783a9d1eb74ca241d72797ceceec Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:31:09 -0700 Subject: [PATCH 01/39] test for json welcome page --- src/allmydata/test/web/test_root.py | 88 ++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 1d5e45ba4..b0789b1d2 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -11,6 +11,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time +import json from urllib.parse import ( quote, @@ -24,14 +25,23 @@ from twisted.web.template import Tag from twisted.web.test.requesthelper import DummyRequest from twisted.application import service from testtools.twistedsupport import succeeded -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import ( + inlineCallbacks, + succeed, +) from ...storage_client import ( NativeStorageServer, StorageFarmBroker, ) -from ...web.root import RootElement +from ...web.root import ( + RootElement, + Root, +) from ...util.connection_status import ConnectionStatus +from ...crypto.ed25519 import ( + create_signing_keypair, +) from allmydata.web.root import URIHandler from allmydata.client import _Client @@ -47,6 +57,7 @@ from ..common import ( from ..common import ( SyncTestCase, + AsyncTestCase, ) from testtools.matchers import ( @@ -138,3 +149,76 @@ class RenderServiceRow(SyncTestCase): self.assertThat(item.slotData.get("version"), Equals("")) self.assertThat(item.slotData.get("nickname"), Equals("")) + + +class RenderRoot(AsyncTestCase): + + @inlineCallbacks + def test_root_json(self): + """ + """ + ann = { + "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) + + class FakeClient(_Client): + history = [] + stats_provider = object() + nickname = "" + nodeid = b"asdf" + _node_public_key = create_signing_keypair()[1] + introducer_clients = [] + helper = None + + def __init__(self): + service.MultiService.__init__(self) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) + self.storage_broker.test_add_server(b"test-srv", srv) + + root = Root(FakeClient(), now_fn=time.time) + + lines = [] + + req = DummyRequest(b"") + req.fields = {} + req.args = { + "t": ["json"], + } + + # for some reason, DummyRequest is already finished when we + # try to add a notifyFinish handler, so override that + # behavior. + + def nop(): + return succeed(None) + req.notifyFinish = nop + req.write = lines.append + + yield root.render(req) + + raw_js = b"".join(lines).decode("utf8") + self.assertThat( + json.loads(raw_js), + Equals({ + "introducers": { + "statuses": [] + }, + "servers": [ + { + "connection_status": "summary", + "nodeid": "server_id", + "last_received_data": 0, + "version": None, + "available_space": None, + "nickname": "" + } + ] + }) + ) From 94b540215f6c32db026cbcaf588f22a9ebdfa866 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:30 -0700 Subject: [PATCH 02/39] args are bytes --- src/allmydata/test/web/test_root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index b0789b1d2..44b91fa48 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -189,7 +189,7 @@ class RenderRoot(AsyncTestCase): req = DummyRequest(b"") req.fields = {} req.args = { - "t": ["json"], + b"t": [b"json"], } # for some reason, DummyRequest is already finished when we From 5be5714bb378a9ad7180f7878ce75b96120afc5c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:40 -0700 Subject: [PATCH 03/39] fix; get rid of sorting --- src/allmydata/web/root.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 1debc1d10..f1a8569d6 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -297,14 +297,12 @@ class Root(MultiFormatResource): } return json.dumps(result, indent=1) + "\n" - def _describe_known_servers(self, broker): - return sorted(list( + return list( self._describe_server(server) for server in broker.get_known_servers() - ), key=lambda o: sorted(o.items())) - + ) def _describe_server(self, server): status = server.get_connection_status() From 872ce021c85b48321fe389200661cf3f087e959f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:59 -0700 Subject: [PATCH 04/39] news --- newsfragments/3852.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3852.minor diff --git a/newsfragments/3852.minor b/newsfragments/3852.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3852.minor @@ -0,0 +1 @@ + From 2f94fdf372116f18fde2d764b0331edb995303fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 12:47:44 -0500 Subject: [PATCH 05/39] Extra testing coverage, including reproducer for #3854. --- src/allmydata/test/web/test_webish.py | 48 +++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 12a04a6eb..4a77d21ae 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -90,10 +90,11 @@ class TahoeLAFSRequestTests(SyncTestCase): """ self._fields_test(b"GET", {}, b"", Equals(None)) - def test_form_fields(self): + def test_form_fields_if_filename_set(self): """ When a ``POST`` request is received, form fields are parsed into - ``TahoeLAFSRequest.fields``. + ``TahoeLAFSRequest.fields`` and the body is bytes (presuming ``filename`` + is set). """ form_data, boundary = multipart_formdata([ [param(u"name", u"foo"), @@ -121,6 +122,49 @@ class TahoeLAFSRequestTests(SyncTestCase): ), ) + def test_form_fields_if_name_is_file(self): + """ + When a ``POST`` request is received, form fields are parsed into + ``TahoeLAFSRequest.fields`` and the body is bytes when ``name`` + is set to ``"file"``. + """ + form_data, boundary = multipart_formdata([ + [param(u"name", u"foo"), + body(u"bar"), + ], + [param(u"name", u"file"), + body(u"some file contents"), + ], + ]) + self._fields_test( + b"POST", + {b"content-type": b"multipart/form-data; boundary=" + bytes(boundary, 'ascii')}, + form_data.encode("ascii"), + AfterPreprocessing( + lambda fs: { + k: fs.getvalue(k) + for k + in fs.keys() + }, + Equals({ + "foo": "bar", + "file": b"some file contents", + }), + ), + ) + + def test_form_fields_require_correct_mime_type(self): + """ + The body of a ``POST`` is not parsed into fields if its mime type is + not ``multipart/form-data``. + + Reproducer for https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3854 + """ + data = u'{"lalala": "lolo"}' + data = data.encode("utf-8") + self._fields_test(b"POST", {"content-type": "application/json"}, + data, Equals(None)) + class TahoeLAFSSiteTests(SyncTestCase): """ From 9f5d7c6d22d40183aaab480990d83c122049495d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:09:25 -0500 Subject: [PATCH 06/39] Fix a bug where we did unnecessary parsing. --- src/allmydata/webish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 9b63a220c..559b475cb 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -114,7 +114,8 @@ class TahoeLAFSRequest(Request, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - if self.method == b'POST': + content_type = (self.requestHeaders.getRawHeaders("content-type") or [""])[0] + if self.method == b'POST' and content_type.split(";")[0] == "multipart/form-data": # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. From 310b77aef0765bf26a28ac4ed8d03f10c05dbb49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:10:13 -0500 Subject: [PATCH 07/39] News file. --- newsfragments/3854.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3854.bugfix diff --git a/newsfragments/3854.bugfix b/newsfragments/3854.bugfix new file mode 100644 index 000000000..d12e174f9 --- /dev/null +++ b/newsfragments/3854.bugfix @@ -0,0 +1 @@ +Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. \ No newline at end of file From 2864ff872d4ddb1f4a16f1669e597e6e7ab3565a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:34:56 -0500 Subject: [PATCH 08/39] Another MIME type that needs to be handled by FieldStorage. --- src/allmydata/webish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 559b475cb..519b3e1f0 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -115,7 +115,7 @@ class TahoeLAFSRequest(Request, object): self.args = parse_qs(argstring, 1) content_type = (self.requestHeaders.getRawHeaders("content-type") or [""])[0] - if self.method == b'POST' and content_type.split(";")[0] == "multipart/form-data": + if self.method == b'POST' and content_type.split(";")[0] in ("multipart/form-data", "application/x-www-form-urlencoded"): # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. From 983f90116b7b120d30a01b336f59bf1c0a62b9f2 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 6 Jan 2022 13:15:31 -0700 Subject: [PATCH 09/39] check differently, don't depend on order --- src/allmydata/test/web/test_web.py | 50 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1c9d6b65c..03cd6e560 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -820,29 +820,37 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ d = self.GET("/?t=json") def _check(res): + """ + Check that the results are correct. + We can't depend on the order of servers in the output + """ decoded = json.loads(res) - expected = { - u'introducers': { - u'statuses': [], + self.assertEqual(decoded['introducers'], {u'statuses': []}) + actual_servers = decoded[u"servers"] + self.assertEquals(len(actual_servers), 2) + self.assertIn( + { + u"nodeid": u'other_nodeid', + u'available_space': 123456, + u'connection_status': u'summary', + u'last_received_data': 30, + u'nickname': u'other_nickname \u263b', + u'version': u'1.0', }, - u'servers': sorted([ - {u"nodeid": u'other_nodeid', - u'available_space': 123456, - u'connection_status': u'summary', - u'last_received_data': 30, - u'nickname': u'other_nickname \u263b', - u'version': u'1.0', - }, - {u"nodeid": u'disconnected_nodeid', - u'available_space': 123456, - u'connection_status': u'summary', - u'last_received_data': 35, - u'nickname': u'disconnected_nickname \u263b', - u'version': u'1.0', - }, - ], key=lambda o: sorted(o.items())), - } - self.assertEqual(expected, decoded) + actual_servers + ) + self.assertIn( + { + u"nodeid": u'disconnected_nodeid', + u'available_space': 123456, + u'connection_status': u'summary', + u'last_received_data': 35, + u'nickname': u'disconnected_nickname \u263b', + u'version': u'1.0', + }, + actual_servers + ) + d.addCallback(_check) return d From 0bf713c38ab36651e841ca8c84e23ecf104aea55 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 10:12:21 -0500 Subject: [PATCH 10/39] News fragment. --- newsfragments/3856.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3856.minor diff --git a/newsfragments/3856.minor b/newsfragments/3856.minor new file mode 100644 index 000000000..e69de29bb From 7e3cb44ede60de3bed90470bf7f7803abac607b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 10:13:29 -0500 Subject: [PATCH 11/39] Pin non-broken version of Paramiko. --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e7a955c6..53057b808 100644 --- a/setup.py +++ b/setup.py @@ -409,7 +409,9 @@ setup(name="tahoe-lafs", # also set in __init__.py "html5lib", "junitxml", "tenacity", - "paramiko", + # Pin old version until + # https://github.com/paramiko/paramiko/issues/1961 is fixed. + "paramiko < 2.9", "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: "prometheus-client == 0.11.0", From 11f2097591e8161416237ecb4676d1843478eb5d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:58:58 -0700 Subject: [PATCH 12/39] docstring --- src/allmydata/test/web/test_root.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 44b91fa48..199c8e545 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -156,6 +156,11 @@ class RenderRoot(AsyncTestCase): @inlineCallbacks def test_root_json(self): """ + The 'welcome' / root page renders properly with ?t=json when some + servers show None for available_space while others show a + valid int + + See also https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3852 """ ann = { "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", From a49baf44b68eac81bb1538000c042ed537e57ef0 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:59:13 -0700 Subject: [PATCH 13/39] actually-reproduce 3852 --- src/allmydata/test/web/test_root.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 199c8e545..8c46b809a 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -166,8 +166,13 @@ class RenderRoot(AsyncTestCase): "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) - srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) + srv0 = NativeStorageServer(b"server_id0", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv0.get_connection_status = lambda: ConnectionStatus(False, "summary0", {}, 0, 0) + + srv1 = NativeStorageServer(b"server_id1", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv1.get_connection_status = lambda: ConnectionStatus(False, "summary1", {}, 0, 0) + # arrange for this server to have some valid available space + srv1.get_available_space = lambda: 12345 class FakeClient(_Client): history = [] @@ -185,7 +190,8 @@ class RenderRoot(AsyncTestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server(b"test-srv", srv) + self.storage_broker.test_add_server(b"test-srv0", srv0) + self.storage_broker.test_add_server(b"test-srv1", srv1) root = Root(FakeClient(), now_fn=time.time) @@ -217,12 +223,20 @@ class RenderRoot(AsyncTestCase): }, "servers": [ { - "connection_status": "summary", - "nodeid": "server_id", + "connection_status": "summary0", + "nodeid": "server_id0", "last_received_data": 0, "version": None, "available_space": None, "nickname": "" + }, + { + "connection_status": "summary1", + "nodeid": "server_id1", + "last_received_data": 0, + "version": None, + "available_space": 12345, + "nickname": "" } ] }) From e8f5023ae2e8b6404f3d7ad37db34fb28a3c4333 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:59:34 -0700 Subject: [PATCH 14/39] its a bugfix --- newsfragments/3852.minor => 3852.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/3852.minor => 3852.bugfix (100%) diff --git a/newsfragments/3852.minor b/3852.bugfix similarity index 100% rename from newsfragments/3852.minor rename to 3852.bugfix From 9d823aef67d7328c9b8ee2d2ae75703b5cd3b26a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 11:05:35 -0700 Subject: [PATCH 15/39] newsfragment to correct spot --- 3852.bugfix => newsfragments/3852.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 3852.bugfix => newsfragments/3852.bugfix (100%) diff --git a/3852.bugfix b/newsfragments/3852.bugfix similarity index 100% rename from 3852.bugfix rename to newsfragments/3852.bugfix From 9644532916de535b738f1e85f5ce060c5e77c604 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 11:28:55 -0700 Subject: [PATCH 16/39] don't depend on order --- src/allmydata/test/web/test_root.py | 49 ++++++++++++++--------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 8c46b809a..228b8e449 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -215,29 +215,28 @@ class RenderRoot(AsyncTestCase): yield root.render(req) raw_js = b"".join(lines).decode("utf8") - self.assertThat( - json.loads(raw_js), - Equals({ - "introducers": { - "statuses": [] - }, - "servers": [ - { - "connection_status": "summary0", - "nodeid": "server_id0", - "last_received_data": 0, - "version": None, - "available_space": None, - "nickname": "" - }, - { - "connection_status": "summary1", - "nodeid": "server_id1", - "last_received_data": 0, - "version": None, - "available_space": 12345, - "nickname": "" - } - ] - }) + js = json.loads(raw_js) + servers = js["servers"] + self.assertEquals(len(servers), 2) + self.assertIn( + { + "connection_status": "summary0", + "nodeid": "server_id0", + "last_received_data": 0, + "version": None, + "available_space": None, + "nickname": "" + }, + servers + ) + self.assertIn( + { + "connection_status": "summary1", + "nodeid": "server_id1", + "last_received_data": 0, + "version": None, + "available_space": 12345, + "nickname": "" + }, + servers ) From b91835a2007764fc924d05f484563b303100f8b5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:06:26 -0700 Subject: [PATCH 17/39] update NEWS.txt for release --- NEWS.rst | 14 ++++++++++++++ newsfragments/3848.minor | 0 newsfragments/3849.minor | 0 newsfragments/3850.minor | 0 newsfragments/3852.bugfix | 1 - newsfragments/3854.bugfix | 1 - newsfragments/3856.minor | 0 7 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 newsfragments/3848.minor delete mode 100644 newsfragments/3849.minor delete mode 100644 newsfragments/3850.minor delete mode 100644 newsfragments/3852.bugfix delete mode 100644 newsfragments/3854.bugfix delete mode 100644 newsfragments/3856.minor diff --git a/NEWS.rst b/NEWS.rst index 15cb9459d..62d1587dd 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,20 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) +'''''''''''''''''''''''''''''''''' + +Bug Fixes +--------- + +- (`#3852 `_) +- Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. (`#3854 `_) + + +Misc/Other +---------- + +- `#3848 `_, `#3849 `_, `#3850 `_, `#3856 `_ Release 1.17.0 (2021-12-06) diff --git a/newsfragments/3848.minor b/newsfragments/3848.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3849.minor b/newsfragments/3849.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3850.minor b/newsfragments/3850.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3852.bugfix b/newsfragments/3852.bugfix deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3852.bugfix +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3854.bugfix b/newsfragments/3854.bugfix deleted file mode 100644 index d12e174f9..000000000 --- a/newsfragments/3854.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. \ No newline at end of file diff --git a/newsfragments/3856.minor b/newsfragments/3856.minor deleted file mode 100644 index e69de29bb..000000000 From 22734dccba2cc95752de1c360830b728d4ac83b2 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:13:44 -0700 Subject: [PATCH 18/39] fix text for 3852 --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 62d1587dd..c2d405f8f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -11,7 +11,7 @@ Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) Bug Fixes --------- -- (`#3852 `_) +- Fixed regression on Python 3 causing the JSON version of the Welcome page to sometimes produce a 500 error (`#3852 `_) - Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. (`#3854 `_) From e9ece061f4a1a6373f39fe37291e1775df3a0391 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:18:03 -0700 Subject: [PATCH 19/39] news --- newsfragments/3858.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3858.minor diff --git a/newsfragments/3858.minor b/newsfragments/3858.minor new file mode 100644 index 000000000..e69de29bb From f9ddd3b3bedf692ffdf598d9def96b3c79097602 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:21:44 -0700 Subject: [PATCH 20/39] fix NEWS title --- NEWS.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index c2d405f8f..0f9194cc4 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,8 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) -'''''''''''''''''''''''''''''''''' +Release 1.17.1 (2022-01-07) +''''''''''''''''''''''''''' Bug Fixes --------- From b5251eb0a12eaa07411473583facd5c21cee729f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:27:53 -0700 Subject: [PATCH 21/39] update relnotes --- relnotes.txt | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index dff4f192e..e9b298771 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1 -The Tahoe-LAFS team is pleased to announce version 1.17.0 of +The Tahoe-LAFS team is pleased to announce version 1.17.1 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,19 +15,12 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.16.0, released on -October 19, 2021. +The previous stable release of Tahoe-LAFS was v1.17.0, released on +December 6, 2021. -This release fixes several security issues raised as part of an audit -by Cure53. We developed fixes for these issues in a private -repository. Shortly after this release, public tickets will be updated -with further information (along with, of course, all the code). +This release fixes two Python3-releated regressions and 4 minor bugs. -There is also OpenMetrics support now and several bug fixes. - -In all, 46 issues have been fixed since the last release. - -Please see ``NEWS.rst`` for a more complete list of changes. +Please see ``NEWS.rst`` [1] for a complete list of changes. WHAT IS IT GOOD FOR? @@ -66,12 +59,12 @@ to v1.0 (which was released March 25, 2008). Clients from this release can read files and directories produced by clients of all versions since v1.0. -Network connections are limited by the Introducer protocol in -use. If the Introducer is running v1.10 or v1.11, then servers -from this release (v1.12) can serve clients of all versions -back to v1.0 . If it is running v1.12, then they can only -serve clients back to v1.10. Clients from this release can use -servers back to v1.10, but not older servers. +Network connections are limited by the Introducer protocol in use. If +the Introducer is running v1.10 or v1.11, then servers from this +release can serve clients of all versions back to v1.0 . If it is +running v1.12 or higher, then they can only serve clients back to +v1.10. Clients from this release can use servers back to v1.10, but +not older servers. Except for the new optional MDMF format, we have not made any intentional compatibility changes. However we do not yet have @@ -79,7 +72,7 @@ the test infrastructure to continuously verify that all new versions are interoperable with previous versions. We intend to build such an infrastructure in the future. -This is the twenty-first release in the version 1 series. This +This is the twenty-second release in the version 1 series. This series of Tahoe-LAFS will be actively supported and maintained for the foreseeable future, and future versions of Tahoe-LAFS will retain the ability to read and write files compatible @@ -139,7 +132,7 @@ Of Fame" [13]. ACKNOWLEDGEMENTS -This is the eighteenth release of Tahoe-LAFS to be created +This is the nineteenth release of Tahoe-LAFS to be created solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. @@ -147,16 +140,16 @@ Tahoe-LAFS possible. meejah on behalf of the Tahoe-LAFS team -December 6, 2021 +January 7, 2022 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From c7664762365e1e14dd2c52374e3206ecd9b077a5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:28:27 -0700 Subject: [PATCH 22/39] nix --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 04d6c4163..2b41e676e 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.17.0). + # Most of the time this is not exactly the release version (eg 1.17.1). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.17.0.post1"; + version = "1.17.1.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From aa81bfc937b1ee7bbe2b43e814cc7683eed1d29e Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:29:45 -0700 Subject: [PATCH 23/39] cleanup whitespace --- docs/Installation/install-tahoe.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Installation/install-tahoe.rst b/docs/Installation/install-tahoe.rst index 2fe47f4a8..8ceca2e01 100644 --- a/docs/Installation/install-tahoe.rst +++ b/docs/Installation/install-tahoe.rst @@ -28,15 +28,15 @@ To install Tahoe-LAFS on Windows: 3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. 4. Start PowerShell and enter the following command to verify python installation:: - + python --version 5. Enter the following command to install Tahoe-LAFS:: - + pip install tahoe-lafs 6. Verify installation by checking for the version:: - + tahoe --version If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`. @@ -56,13 +56,13 @@ If you are working on MacOS or a Linux distribution which does not have Tahoe-LA * **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation `_. 2. Install Tahoe-LAFS using pip:: - + pip install tahoe-lafs 3. Verify installation by checking for the version:: - + tahoe --version -If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. From f7477043c5025642ef0fbeb042310decb774bd01 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:29:52 -0700 Subject: [PATCH 24/39] unnecessary step --- docs/release-checklist.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index f943abb5d..2b954449e 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -70,7 +70,6 @@ Create Branch and Apply Updates - commit it - update "docs/known_issues.rst" if appropriate -- update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github - Create a (draft) PR; this should trigger CI (note that github doesn't let you create a PR without some changes on the branch so From 852ebe90e5bd5b04b5a75d1850df87673a78955f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:48:55 -0700 Subject: [PATCH 25/39] clean clone --- docs/release-checklist.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 2b954449e..edfe9e20f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -106,6 +106,11 @@ they will need to evaluate which contributors' signatures they trust. - tox -e deprecations,upcoming-deprecations +- clone to a clean, local checkout (to avoid extra files being included in the release) + + - cd /tmp + - git clone /home/meejah/src/tahoe-lafs + - build tarballs - tox -e tarballs From d2ff2a7376f99f08fba22ee3c3b28cba535a0117 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:02 -0700 Subject: [PATCH 26/39] whitespace --- docs/release-checklist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index edfe9e20f..3c984d122 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -158,7 +158,8 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - secure-copy all release artifacts to the download area on the tahoe-lafs.org host machine. `~source/downloads` on there maps to https://tahoe-lafs.org/downloads/ on the Web. -- scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads + - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads + - the following developers have access to do this: - exarkun From 1446c9c4adebda255276659dfef883f17770ca7f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:15 -0700 Subject: [PATCH 27/39] add 'push the tags' step --- docs/release-checklist.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3c984d122..7acca6bb3 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -166,6 +166,10 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - meejah - warner +Push the signed tag to the main repository: + +- git push origin_push tahoe-lafs-1.17.1 + For the actual release, the tarball and signature files need to be uploaded to PyPI as well. From 8cd4e7a4b5069c3fb30c195934974755e8f0c53c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:31 -0700 Subject: [PATCH 28/39] news --- newsfragments/3859.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3859.minor diff --git a/newsfragments/3859.minor b/newsfragments/3859.minor new file mode 100644 index 000000000..e69de29bb From ea83b16d1171b789c2041ed1e67e2ffa6dec3ff4 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 14:17:50 -0700 Subject: [PATCH 29/39] most people say 'origin' --- docs/release-checklist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 7acca6bb3..165aa8826 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -168,7 +168,7 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` Push the signed tag to the main repository: -- git push origin_push tahoe-lafs-1.17.1 +- git push origin tahoe-lafs-1.17.1 For the actual release, the tarball and signature files need to be uploaded to PyPI as well. From a753a71105a8865cb27c9a59258fe349d55ba06a Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 14:22:57 -0700 Subject: [PATCH 30/39] please the Sphinx --- docs/release-checklist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 165aa8826..d2f1b3eb8 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -157,7 +157,8 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - secure-copy all release artifacts to the download area on the tahoe-lafs.org host machine. `~source/downloads` on there maps to - https://tahoe-lafs.org/downloads/ on the Web. + https://tahoe-lafs.org/downloads/ on the Web: + - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads - the following developers have access to do this: From 764e493c98798f2b4248189e1c0faa3a9df27ffb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:32:27 -0500 Subject: [PATCH 31/39] News. --- newsfragments/3865.incompat | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3865.incompat diff --git a/newsfragments/3865.incompat b/newsfragments/3865.incompat new file mode 100644 index 000000000..59381b269 --- /dev/null +++ b/newsfragments/3865.incompat @@ -0,0 +1 @@ +Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. \ No newline at end of file From 8eb6ab47653f7e8be52c310332f8f6d9686cd2ae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:40:26 -0500 Subject: [PATCH 32/39] Switch to Python 3.7 as minimal version. --- .circleci/config.yml | 12 ++++++------ .github/workflows/ci.yml | 7 +++---- Makefile | 4 ++-- misc/python3/Makefile | 4 ++-- setup.py | 5 ++--- tox.ini | 5 ++--- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fc8e88e7..d55b80469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,8 +49,8 @@ workflows: - "pypy27-buster": {} - # Just one Python 3.6 configuration while the port is in-progress. - - "python36": + # Test against Python 3: + - "python37": {} # Other assorted tasks and configurations @@ -118,7 +118,7 @@ workflows: <<: *DOCKERHUB_CONTEXT - "build-image-pypy27-buster": <<: *DOCKERHUB_CONTEXT - - "build-image-python36-ubuntu": + - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -379,7 +379,7 @@ jobs: user: "nobody" - python36: + python37: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH @@ -392,7 +392,7 @@ jobs: # this reporter on Python 3. So drop that and just specify the # reporter. TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" ubuntu-20-04: @@ -577,7 +577,7 @@ jobs: PYTHON_VERSION: "2.7" - build-image-python36-ubuntu: + build-image-python37-ubuntu: <<: *BUILD_IMAGE environment: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8209108bf..5ae70a3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,12 +39,11 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 - 3.7 - 3.8 - 3.9 include: - # On macOS don't bother with 3.6-3.8, just to get faster builds. + # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest @@ -181,10 +180,10 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 + - 3.7 - 3.9 include: - # On macOS don't bother with 3.6, just to get faster builds. + # On macOS don't bother with 3.7, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest diff --git a/Makefile b/Makefile index 5d8bf18ba..33a40df02 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py36,pypy27' + tox --develop -p auto -e 'py27,py37,pypy27' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -51,7 +51,7 @@ test-venv-coverage: .PHONY: test-py3-all ## Run all tests under Python 3 test-py3-all: .tox/create-venvs.log - tox --develop -e py36 allmydata + tox --develop -e py37 allmydata # This is necessary only if you want to automatically produce a new # _version.py file from the current git history (without doing a build). diff --git a/misc/python3/Makefile b/misc/python3/Makefile index f0ef8b12a..43cb3e3ce 100644 --- a/misc/python3/Makefile +++ b/misc/python3/Makefile @@ -37,8 +37,8 @@ test-py3-all-diff: ../../.tox/make-test-py3-all.diff # `$ make .tox/make-test-py3-all.diff` $(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ + tox --develop --notest -e py37-coverage + (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ test-venv-coverage || true) | \ sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ tee "./misc/python3/$(@)" diff --git a/setup.py b/setup.py index 53057b808..9a1c76bd8 100644 --- a/setup.py +++ b/setup.py @@ -376,9 +376,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and we're working on support for 3.6 (the - # highest version that PyPy currently supports). - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + # We support Python 2.7, and Python 3.7 or later. + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See diff --git a/tox.ini b/tox.ini index 38cee1f9f..34d555aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,6 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -18,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -51,7 +50,7 @@ deps = # suffering we're trying to avoid with the above pins. certifi # VCS hooks support - py36,!coverage: pre-commit + py37,!coverage: pre-commit # We add usedevelop=False because testing against a true installation gives # more useful results. From fa2b4a11c76a1b55100ea51fa25f9d0dcda9ff6d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:50:40 -0500 Subject: [PATCH 33/39] Welcome to the WORLD OF TOMORROW --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d55b80469..5ef9c81a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,7 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:17.05.0-ce-git" + image: "docker:20.10" environment: DISTRO: "tahoelafsci/:foo-py2" From f04e121a7d31d1b18bdd5c5d40618f98ce3bb2ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:51:55 -0500 Subject: [PATCH 34/39] Try to use correct Docker image. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ef9c81a8..05686fcf8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -383,7 +383,7 @@ jobs: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" user: "nobody" environment: @@ -583,7 +583,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "18.04" - PYTHON_VERSION: "3" + PYTHON_VERSION: "3.7" build-image-ubuntu-20-04: From 02740f075bb69f74ae3ed198a426a0fb3750a5e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:56:11 -0500 Subject: [PATCH 35/39] Temporarily enable image builds on every push. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05686fcf8..6578f1d1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 31e4556bd1910c19a74e30b793d5098ca5f27b8c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:01:47 -0500 Subject: [PATCH 36/39] Need image with Docker _and_ git+ssh. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6578f1d1a..d28196097 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,9 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:20.10" + # CircleCI build images; https://github.com/CircleCI-Public/cimg-base + # for details. + image: "cimg/base:2022.01" environment: DISTRO: "tahoelafsci/:foo-py2" From 04cf206e0d78a174c0f80a80180340838f332b67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:06:58 -0500 Subject: [PATCH 37/39] Switch back to running image building on schedule. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d28196097..a650313ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 54996185dec11b7b2c78a1f90e6a17bee9ff3ed6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:05 -0500 Subject: [PATCH 38/39] No longer used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index 43cb3e3ce..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py37-coverage - (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From e1f9f7de94c68d8c4584ea650e6d9584614b3eb7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:18 -0500 Subject: [PATCH 39/39] Note for future improvement. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result