From 0d093c45df583bed9400004bebd2f3becaa7523d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 10:06:12 -0400 Subject: [PATCH 1/7] Test fails on Python 3, instead of spinning until timeout. --- integration/conftest.py | 2 +- integration/test_servers_of_happiness.py | 2 +- integration/util.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 533cbdb67..918f2d4c9 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -28,7 +28,7 @@ from twisted.internet.error import ( import pytest import pytest_twisted -from util import ( +from .util import ( _CollectOutputProtocol, _MagicTextProtocol, _DumpOutputProtocol, diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 1f350eb8e..f8f69b1cc 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -3,7 +3,7 @@ from os.path import join from twisted.internet.error import ProcessTerminated -import util +from . import util import pytest_twisted diff --git a/integration/util.py b/integration/util.py index cd7f15e84..2052d2ae6 100644 --- a/integration/util.py +++ b/integration/util.py @@ -116,6 +116,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): + data = unicode(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): @@ -123,6 +124,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): + data = unicode(data, sys.stderr.encoding) sys.stdout.write(data) From 6df076dc10b22e4b42905875df33a929b95937e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 10:16:46 -0400 Subject: [PATCH 2/7] Bit more progress. --- integration/util.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integration/util.py b/integration/util.py index 2052d2ae6..bb2f9e8e4 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,7 @@ import time import json from os import mkdir, environ from os.path import exists, join -from six.moves import StringIO +from io import StringIO, BytesIO from functools import partial from subprocess import check_output @@ -59,7 +59,7 @@ class _CollectOutputProtocol(ProcessProtocol): """ def __init__(self): self.done = Deferred() - self.output = StringIO() + self.output = BytesIO() def processEnded(self, reason): if not self.done.called: @@ -73,7 +73,7 @@ class _CollectOutputProtocol(ProcessProtocol): self.output.write(data) def errReceived(self, data): - print("ERR: {}".format(data)) + print("ERR: {!r}".format(data)) self.output.write(data) @@ -94,9 +94,11 @@ class _DumpOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): + data = unicode(data, sys.stdout.encoding) self._out.write(data) def errReceived(self, data): + data = unicode(data, sys.stdout.encoding) self._out.write(data) @@ -284,7 +286,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam config, u'node', u'log_gatherer.furl', - flog_gatherer.decode("utf-8"), + flog_gatherer, ) write_config(FilePath(config_path), config) created_d.addCallback(created) From b11cc9137b87c6ca443b7244f4de4353be5bd250 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 10:48:50 -0400 Subject: [PATCH 3/7] Integration test passed on Python 3. --- integration/test_servers_of_happiness.py | 2 +- src/allmydata/web/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index f8f69b1cc..594397999 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -42,4 +42,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto assert isinstance(e, ProcessTerminated) output = proto.output.getvalue() - assert "shares could be placed on only" in output + assert b"shares could be placed on only" in output diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index b7dc8b5f4..1debc1d10 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -318,7 +318,7 @@ class Root(MultiFormatResource): } version = server.get_version() if version is not None: - description[u"version"] = version["application-version"] + description[u"version"] = version[b"application-version"] return description From 04fc8e704650e1758a2eac6e5f146092aa04f310 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:21:23 -0400 Subject: [PATCH 4/7] Port to Python 3. --- integration/test_servers_of_happiness.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 594397999..b9de0c075 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5f78af626..41270b430 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,12 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +PORTED_INTEGRATION_TESTS = [ + "integration.test_servers_of_happiness", +] + + # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata", From add20d802430113c949c89d6bdccac8d463b653f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:42:38 -0400 Subject: [PATCH 5/7] First pass at integration support on Python 3. --- src/allmydata/util/_python3.py | 3 +++ tox.ini | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 41270b430..c265424c6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -23,6 +23,9 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +# Every time a module is added here, also add it to tox.ini environment +# integrations3. Bit of duplication, but it's only a handful of files so quite +# temporary. PORTED_INTEGRATION_TESTS = [ "integration.test_servers_of_happiness", ] diff --git a/tox.ini b/tox.ini index a6e5e0cc3..6a0c628d7 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage + 3.6: py36-coverage,integration3 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage,typechecks,codechecks3 @@ -97,6 +97,17 @@ commands = coverage report +[testenv:integration3] +basepython = python3 +setenv = + COVERAGE_PROCESS_START=.coveragerc +commands = + # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' + py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py} + coverage combine + coverage report + + [testenv:codechecks] basepython = python2.7 # On macOS, git inside of towncrier needs $HOME. From 01baa13fca6f71931682210dfa1b7170207244c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:10:02 -0400 Subject: [PATCH 6/7] News file --- newsfragments/3703.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3703.minor diff --git a/newsfragments/3703.minor b/newsfragments/3703.minor new file mode 100644 index 000000000..e69de29bb From 7c2f9e7d8c1e5e02d5ee46ded0971b891d7bb26d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:19:21 -0400 Subject: [PATCH 7/7] Fix test to match real-world types. --- src/allmydata/test/web/test_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e73cc12f8..1c9d6b65c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -218,7 +218,7 @@ class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/35 return self.connected def get_version(self): return { - "application-version": "1.0" + b"application-version": b"1.0" } def get_permutation_seed(self): return b""