From 9b6067dcd1d05f8e4296c83157bd9447df1c0699 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:39:32 -0400 Subject: [PATCH 01/15] Make test match reality, and fix corresponding bug. --- src/allmydata/test/test_tor_provider.py | 4 +++- src/allmydata/util/tor_provider.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index 148d813f5..86d54803a 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -14,6 +14,7 @@ import os from twisted.trial import unittest from twisted.internet import defer, error from six.moves import StringIO +from six import ensure_str import mock from ..util import tor_provider from ..scripts import create_node, runner @@ -185,7 +186,8 @@ class CreateOnion(unittest.TestCase): protocol))) txtorcon = mock.Mock() ehs = mock.Mock() - ehs.private_key = b"privkey" + # This appears to be a native string in the real txtorcon object... + ehs.private_key = ensure_str("privkey") ehs.hostname = "ONION.onion" txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs) ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index c4c63f61a..4ca19c01c 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -211,6 +211,8 @@ def create_config(reactor, cli_config): "tor_onion.privkey") privkeyfile = os.path.join(private_dir, "tor_onion.privkey") with open(privkeyfile, "wb") as f: + if isinstance(privkey, str): + privkey = privkey.encode("ascii") f.write(privkey) # tahoe_config_tor: this is a dictionary of keys/values to add to the From 070691caa2a8f204eeda04f815162f2f9baa4c79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:54:30 -0400 Subject: [PATCH 02/15] Make sure it always goes to stderr. --- src/allmydata/scripts/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 7122e499e..f63b9b330 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -185,8 +185,8 @@ def _maybe_enable_eliot_logging(options, reactor): def run(): if six.PY3: - warnings.warn("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.") + print("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.", file=sys.stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize From 4043b2fe1fffdef95fb6efa449cd1b5f20dc0dce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:56:21 -0400 Subject: [PATCH 03/15] Tests pass on Python 3. --- integration/test_tor.py | 12 +++++++----- integration/util.py | 6 ++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 3b374f669..bea48ba5d 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -6,7 +6,7 @@ from os.path import join import pytest import pytest_twisted -import util +from . import util from twisted.python.filepath import ( FilePath, @@ -55,7 +55,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) - proto = util._CollectOutputProtocol() + proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, sys.executable, @@ -68,7 +68,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne yield proto.done dave_got = proto.output.getvalue().strip() - assert dave_got == open(gold_path, 'r').read().strip() + assert dave_got == open(gold_path, 'rb').read().strip() @pytest_twisted.inlineCallbacks @@ -100,7 +100,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) with node_dir.child('tahoe.cfg').open('w') as f: - f.write(''' + node_config = ''' [node] nickname = %(name)s web.port = %(web_port)s @@ -125,7 +125,9 @@ shares.total = 2 'log_furl': flog_gatherer, 'control_port': control_port, 'local_port': control_port + 1000, -}) +} + node_config = node_config.encode("utf-8") + f.write(node_config) print("running") yield util._run_node(reactor, node_dir.path, request, None) diff --git a/integration/util.py b/integration/util.py index 4f3c40666..a233471c7 100644 --- a/integration/util.py +++ b/integration/util.py @@ -57,9 +57,10 @@ class _CollectOutputProtocol(ProcessProtocol): self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self): + def __init__(self, capture_stderr=True): self.done = Deferred() self.output = BytesIO() + self.capture_stderr = capture_stderr def processEnded(self, reason): if not self.done.called: @@ -74,7 +75,8 @@ class _CollectOutputProtocol(ProcessProtocol): def errReceived(self, data): print("ERR: {!r}".format(data)) - self.output.write(data) + if self.capture_err: + self.output.write(data) class _DumpOutputProtocol(ProcessProtocol): From 02897b9968ef5b1e341c8d0600b2b7dd8fde1be6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 13:21:47 -0400 Subject: [PATCH 04/15] Fix typo. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index a233471c7..c8ba5d16b 100644 --- a/integration/util.py +++ b/integration/util.py @@ -75,7 +75,7 @@ class _CollectOutputProtocol(ProcessProtocol): def errReceived(self, data): print("ERR: {!r}".format(data)) - if self.capture_err: + if self.capture_stderr: self.output.write(data) From fba23ef91cc95fdc0da8429fd38fed802c96c9fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 13:34:35 -0400 Subject: [PATCH 05/15] Port to Python 3. --- integration/test_tor.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + tox.ini | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index bea48ba5d..15d888e36 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4bbfc0e68..2bde5941e 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -25,6 +25,7 @@ PORTED_INTEGRATION_TESTS = [ "integration.test_servers_of_happiness", "integration.test_sftp", "integration.test_streaming_logs", + "integration.test_tor", ] diff --git a/tox.ini b/tox.ini index 7d8e90040..04eae7833 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py} + python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py integration/test_tor.py} coverage combine coverage report From d517304a7914ee3a73ced2462d2743b70b6e617b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 13:47:49 -0400 Subject: [PATCH 06/15] A lot closer to passing on Python 3. --- integration/test_web.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index d48ad3aa6..8fac6909d 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -12,12 +12,12 @@ exists anywhere, however. from past.builtins import unicode import time -import json -import urllib2 +from urllib.parse import unquote as url_unquote, quote as url_quote import allmydata.uri +from allmydata.util import jsonbytes as json -import util +from . import util import requests import html5lib @@ -66,7 +66,7 @@ def test_upload_download(alice): u"filename": u"boom", } ) - assert data == FILE_CONTENTS + assert unicode(data, "utf-8") == FILE_CONTENTS def test_put(alice): @@ -117,10 +117,10 @@ def test_deep_stats(alice): # when creating a directory, we'll be re-directed to a URL # containing our writecap.. - uri = urllib2.unquote(resp.url) + uri = url_unquote(resp.url) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') - dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap))) + dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap))) # POST a file into this directory FILE_CONTENTS = u"a file in a directory" @@ -147,7 +147,7 @@ def test_deep_stats(alice): k, data = d assert k == u"dirnode" assert len(data['children']) == 1 - k, child = data['children'].values()[0] + k, child = list(data['children'].values())[0] assert k == u"filenode" assert child['size'] == len(FILE_CONTENTS) @@ -198,11 +198,11 @@ def test_status(alice): print("Uploaded data, cap={}".format(cap)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))), + util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))), ) print("Downloaded {} bytes of data".format(len(resp.content))) - assert resp.content == FILE_CONTENTS + assert unicode(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( util.node_url(alice.node_dir, "status"), @@ -221,12 +221,12 @@ def test_status(alice): continue resp = requests.get(util.node_url(alice.node_dir, href)) if href.startswith(u"/status/up"): - assert "File Upload Status" in resp.content - if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + assert b"File Upload Status" in resp.content + if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_upload = True elif href.startswith(u"/status/down"): - assert "File Download Status" in resp.content - if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + assert b"File Download Status" in resp.content + if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_download = True # download the specialized event information @@ -299,7 +299,7 @@ def test_directory_deep_check(alice): print("Uploaded data1, cap={}".format(cap1)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))), + util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))), params={u"t": u"info"}, ) @@ -440,7 +440,7 @@ def test_introducer_info(introducer): resp = requests.get( util.node_url(introducer.node_dir, u""), ) - assert "Introducer" in resp.content + assert b"Introducer" in resp.content resp = requests.get( util.node_url(introducer.node_dir, u""), @@ -513,6 +513,6 @@ def test_mkdir_with_children(alice): params={u"t": "mkdir-with-children"}, data=json.dumps(meta), ) - assert resp.startswith("URI:DIR2") + assert resp.startswith(b"URI:DIR2") cap = allmydata.uri.from_string(resp) assert isinstance(cap, allmydata.uri.DirectoryURI) From e2ce207fdd4d94de3b785ca22080c87ecaf6646e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:13:39 -0400 Subject: [PATCH 07/15] Don't convert bytes to unicode implicitly. --- src/allmydata/web/status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0401fb586..158d897f9 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1173,7 +1173,8 @@ class MapupdateStatusElement(Element): def privkey_from(self, req, tag): server = self._update_status.get_privkey_from() if server: - return tag(tags.li("Got privkey from: [%s]" % server.get_name())) + return tag(tags.li("Got privkey from: [%s]" % str( + server.get_name(), "utf-8"))) else: return tag From 079041fc1eb6d284382f39bd307ad8fba81b487e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:20:49 -0400 Subject: [PATCH 08/15] Port to Python 3. --- integration/test_web.py | 21 +++++++++++++++------ src/allmydata/util/_python3.py | 4 +--- tox.ini | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index 8fac6909d..22f08da82 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -7,9 +7,18 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did at the time of testing' -- not necessarily a cohesive idea of what the WebAPI *should* do in every situation. It's not clear the latter exists anywhere, however. + +Ported to Python 3. """ -from past.builtins import unicode +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time from urllib.parse import unquote as url_unquote, quote as url_quote @@ -66,7 +75,7 @@ def test_upload_download(alice): u"filename": u"boom", } ) - assert unicode(data, "utf-8") == FILE_CONTENTS + assert str(data, "utf-8") == FILE_CONTENTS def test_put(alice): @@ -97,7 +106,7 @@ def test_helper_status(storage_nodes): resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") - assert unicode(dom.h1.string) == u"Helper Status" + assert str(dom.h1.string) == u"Helper Status" def test_deep_stats(alice): @@ -202,7 +211,7 @@ def test_status(alice): ) print("Downloaded {} bytes of data".format(len(resp.content))) - assert unicode(resp.content, "ascii") == FILE_CONTENTS + assert str(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( util.node_url(alice.node_dir, "status"), @@ -400,9 +409,9 @@ def test_directory_deep_check(alice): for _ in range(5): resp = requests.get(deepcheck_uri) dom = BeautifulSoup(resp.content, "html5lib") - if dom.h1 and u'Results' in unicode(dom.h1.string): + if dom.h1 and u'Results' in str(dom.h1.string): break - if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string): + if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string): dom = None time.sleep(1) assert dom is not None, "Operation never completed" diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2bde5941e..c15301c3c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -17,15 +17,13 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -# Every time a module is added here, also add it to tox.ini environment -# integrations3. Bit of duplication, but it's only a handful of files and quite -# temporary, just until we've ported them all. PORTED_INTEGRATION_TESTS = [ "integration.test_aaa_aardvark", "integration.test_servers_of_happiness", "integration.test_sftp", "integration.test_streaming_logs", "integration.test_tor", + "integration.test_web", ] diff --git a/tox.ini b/tox.ini index 04eae7833..365c5d0e9 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py integration/test_tor.py} + python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration} coverage combine coverage report From 22f8b9b4281e15d3851eb7097e2c076907c621be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:25:52 -0400 Subject: [PATCH 09/15] Port to Python 3. --- integration/conftest.py | 10 ++++++++++ integration/util.py | 26 ++++++++++++++++++-------- src/allmydata/util/_python3.py | 4 ++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 918f2d4c9..39ff3b42b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys import shutil from time import sleep diff --git a/integration/util.py b/integration/util.py index c8ba5d16b..7c7a1efd2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import time @@ -96,11 +106,11 @@ class _DumpOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): - data = unicode(data, sys.stdout.encoding) + data = str(data, sys.stdout.encoding) self._out.write(data) def errReceived(self, data): - data = unicode(data, sys.stdout.encoding) + data = str(data, sys.stdout.encoding) self._out.write(data) @@ -120,7 +130,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): - data = unicode(data, sys.stdout.encoding) + data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): @@ -128,7 +138,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): - data = unicode(data, sys.stderr.encoding) + data = str(data, sys.stderr.encoding) sys.stdout.write(data) @@ -269,9 +279,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam '--hostname', 'localhost', '--listen', 'tcp', '--webport', web_port, - '--shares-needed', unicode(needed), - '--shares-happy', unicode(happy), - '--shares-total', unicode(total), + '--shares-needed', str(needed), + '--shares-happy', str(happy), + '--shares-total', str(total), '--helper', ] if not storage: diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index c15301c3c..f6cd9752c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -279,4 +279,8 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_web", "allmydata.test.web.test_webish", "allmydata.test.test_windows", + + "integration", + "integration.conftest", + "integration.util", ] From ba03a7b436a6f1f9b26c11ff2209f3e71953bc03 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:26:14 -0400 Subject: [PATCH 10/15] News file. --- newsfragments/3709.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3709.minor diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor new file mode 100644 index 000000000..e69de29bb From d7ad72f651cd16158364921a573170771142d36c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:21:22 -0400 Subject: [PATCH 11/15] Just do integration tests as part of normal CI runners. --- .github/workflows/ci.yml | 89 ++++++---------------------------------- tox.ini | 4 +- 2 files changed, 15 insertions(+), 78 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f3d57310..6fd542e0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,19 @@ jobs: python-version: 2.7 steps: + - name: Install Tor [Ubuntu] + if: matrix.os == 'ubuntu-latest' + run: sudo apt install tor + + - name: Install Tor [macOS] + if: matrix.os == 'macos-latest' + run: brew install tor + + - name: Install Tor [Windows] + if: matrix.os == 'windows-latest' + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install tor # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. @@ -172,82 +185,6 @@ jobs: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - integration: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - windows-latest - python-version: - - 2.7 - - steps: - - - name: Install Tor [Ubuntu] - if: matrix.os == 'ubuntu-latest' - run: sudo apt install tor - - - name: Install Tor [macOS] - if: matrix.os == 'macos-latest' - run: brew install tor - - - name: Install Tor [Windows] - if: matrix.os == 'windows-latest' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install tor - - - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - # See this step under coverage job. - - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x86' - - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python packages - run: | - pip install --upgrade tox - pip list - - - name: Display tool versions - run: python misc/build_helpers/show-tool-versions.py - - - name: Run "tox -e integration" - run: tox -e integration - - - name: Upload eliot.log in case of failure - uses: actions/upload-artifact@v1 - if: failure() - with: - name: integration.eliot.json - path: integration.eliot.json packaging: runs-on: ${{ matrix.os }} diff --git a/tox.ini b/tox.ini index 365c5d0e9..2eddfd766 100644 --- a/tox.ini +++ b/tox.ini @@ -6,11 +6,11 @@ # Map Python versions in GitHub Actions to tox environments to run. [gh-actions] python = - 2.7: py27-coverage,codechecks + 2.7: py27-coverage,codechecks,integration 3.6: py36-coverage,integration3 3.7: py37-coverage 3.8: py38-coverage - 3.9: py39-coverage,typechecks,codechecks3 + 3.9: py39-coverage,typechecks,codechecks3,integration3 pypy-3.7: pypy3 [pytest] From c4dd8dde27237592c87a90d6502bc8fd4a9153b1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:51:47 -0400 Subject: [PATCH 12/15] Flake. --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index f63b9b330..a17ce3ef4 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -import warnings + import os, sys from six.moves import StringIO import six From 8741c33b3562391e9035ff469621d74d8282fb1d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:58:02 -0400 Subject: [PATCH 13/15] Needs to be in separate list since it's not checked. --- src/allmydata/util/_python3.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f6cd9752c..53e5998fb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -26,6 +26,11 @@ PORTED_INTEGRATION_TESTS = [ "integration.test_web", ] +PORTED_INTEGRATION_MODULES = [ + "integration", + "integration.conftest", + "integration.util", +] # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ @@ -279,8 +284,4 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_web", "allmydata.test.web.test_webish", "allmydata.test.test_windows", - - "integration", - "integration.conftest", - "integration.util", ] From c4c053a37d5da1398ab18884436f6856098c9685 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 12:02:22 -0400 Subject: [PATCH 14/15] Fix failing tests on Python 3. --- src/allmydata/scripts/runner.py | 6 ++++-- src/allmydata/test/test_system.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index a17ce3ef4..43a3b8807 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -183,10 +183,12 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options +PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.") + def run(): if six.PY3: - print("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.", file=sys.stderr) + print(PYTHON_3_WARNING, file=sys.stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 12ae846eb..627b6ef29 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -43,6 +43,7 @@ from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from allmydata.mutable.publish import MutableData +from allmydata.scripts.runner import PYTHON_3_WARNING from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure @@ -2635,7 +2636,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) if check_stderr: - self.failUnlessEqual(err, b"") + self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) d.addCallback(_run_in_subprocess, "create-alias", "newalias") d.addCallback(_check_succeeded) @@ -2655,7 +2656,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_ls(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.failUnlessEqual(err, b"", str(res)) + self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) self.failUnlessIn(b"tahoe-moved", out) self.failIfIn(b"tahoe-file", out) d.addCallback(_check_ls) From 73a3e82856047954103890c6e30e429051591473 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 May 2021 11:29:11 -0400 Subject: [PATCH 15/15] Don't run integration tests here either. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cb134ffd2..964403a40 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ python = 3.6: py36-coverage 3.7: py37-coverage 3.8: py38-coverage - 3.9: py39-coverage,typechecks,codechecks3,integration3 + 3.9: py39-coverage,typechecks,codechecks3 pypy-3.7: pypy3 [pytest]