Merge pull request #1056 from tahoe-lafs/3703.integration-tests-python-3

First integration tests running on Python 3

Fixes ticket:3703

.github/workflows/ci.yml

@@ -182,6 +182,9 @@ jobs:
           - windows-latest
         python-version:
           - 2.7
+        include:
+          - os: ubuntu-latest
+            python-version: 3.6
 
     steps:
 
@@ -239,9 +242,14 @@ jobs:
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e integration"
+      - name: Run "Python 2 integration tests"
+        if: ${{ matrix.python-version == '2.7' }}
         run: tox -e integration
 
+      - name: Run "Python 3 integration tests"
+        if: ${{ matrix.python-version != '2.7' }}
+        run: tox -e integration3
+
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1
         if: failure()

integration/conftest.py

@@ -28,7 +28,7 @@ from twisted.internet.error import (
 import pytest
 import pytest_twisted
 
-from util import (
+from .util import (
     _CollectOutputProtocol,
     _MagicTextProtocol,
     _DumpOutputProtocol,

integration/test_servers_of_happiness.py

@@ -1,9 +1,21 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import sys
 from os.path import join
 
 from twisted.internet.error import ProcessTerminated
 
-import util
+from . import util
 
 import pytest_twisted
 
@@ -42,4 +54,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
         assert isinstance(e, ProcessTerminated)
 
     output = proto.output.getvalue()
-    assert "shares could be placed on only" in output
+    assert b"shares could be placed on only" in output

integration/util.py

@@ -5,7 +5,7 @@ import time
 import json
 from os import mkdir, environ
 from os.path import exists, join
-from six.moves import StringIO
+from io import StringIO, BytesIO
 from functools import partial
 from subprocess import check_output
 
@@ -59,7 +59,7 @@ class _CollectOutputProtocol(ProcessProtocol):
     """
     def __init__(self):
         self.done = Deferred()
-        self.output = StringIO()
+        self.output = BytesIO()
 
     def processEnded(self, reason):
         if not self.done.called:
@@ -73,7 +73,7 @@ class _CollectOutputProtocol(ProcessProtocol):
         self.output.write(data)
 
     def errReceived(self, data):
-        print("ERR: {}".format(data))
+        print("ERR: {!r}".format(data))
         self.output.write(data)
 
 
@@ -94,9 +94,11 @@ class _DumpOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         self._out.write(data)
 
     def errReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         self._out.write(data)
 
 
@@ -116,6 +118,7 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
@@ -123,6 +126,7 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
+        data = unicode(data, sys.stderr.encoding)
         sys.stdout.write(data)
 
 
@@ -282,7 +286,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
                 config,
                 u'node',
                 u'log_gatherer.furl',
-                flog_gatherer.decode("utf-8"),
+                flog_gatherer,
             )
             write_config(FilePath(config_path), config)
         created_d.addCallback(created)

src/allmydata/test/web/test_web.py

@@ -218,7 +218,7 @@ class FakeDisplayableServer(StubServer):  # type: ignore  # tahoe-lafs/ticket/35
         return self.connected
     def get_version(self):
         return {
-            "application-version": "1.0"
+            b"application-version": b"1.0"
         }
     def get_permutation_seed(self):
         return b""

src/allmydata/util/_python3.py

@@ -16,6 +16,15 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+
+# Every time a module is added here, also add it to tox.ini environment
+# integrations3. Bit of duplication, but it's only a handful of files so quite
+# temporary.
+PORTED_INTEGRATION_TESTS = [
+    "integration.test_servers_of_happiness",
+]
+
+
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
     "allmydata",

src/allmydata/web/root.py

@@ -318,7 +318,7 @@ class Root(MultiFormatResource):
         }
         version = server.get_version()
         if version is not None:
-            description[u"version"] = version["application-version"]
+            description[u"version"] = version[b"application-version"]
 
         return description
 

tox.ini

@@ -18,7 +18,7 @@ python =
 twisted = 1
 
 [tox]
-envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3
+envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3
 minversion = 2.4
 
 [testenv]
@@ -97,6 +97,18 @@ commands =
          coverage report
 
 
+[testenv:integration3]
+basepython = python3
+setenv =
+         COVERAGE_PROCESS_START=.coveragerc
+commands =
+         python --version
+         # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+         py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py}
+         coverage combine
+         coverage report
+
+
 [testenv:codechecks]
 basepython = python2.7
 # On macOS, git inside of towncrier needs $HOME.