diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index f6efe785a..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,95 +0,0 @@ -# adapted from https://packaging.python.org/en/latest/appveyor/ - -environment: - - matrix: - - # For Python versions available on Appveyor, see - # http://www.appveyor.com/docs/installed-software#python - - PYTHON: "C:\\Python27" - - PYTHON: "C:\\Python27-x64" - # DISTUTILS_USE_SDK: "1" - # TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB" - -install: - - | - %PYTHON%\python.exe -m pip install -U pip - %PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv - -# note: -# %PYTHON% has: python.exe -# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip) - -# We have a custom "build" system. We don't need MSBuild or whatever. -build: off - -# Do not build feature branch with open pull requests. This is documented but -# it's not clear it does anything. -skip_branch_with_pr: true - -# This, perhaps, is effective. -branches: - # whitelist - only: - - 'master' - -skip_commits: - files: - # The Windows builds are unaffected by news fragments. - - 'newsfragments/*' - # Also, all this build junk. - - '.circleci/*' - - '.lgtm.yml' - - '.travis.yml' - -# we run from C:\projects\tahoe-lafs - -test_script: - # Put your test command here. - # Note that you must use the environment variable %PYTHON% to refer to - # the interpreter you're using - Appveyor does not do anything special - # to put the Python version you want to use on PATH. - - | - %PYTHON%\Scripts\tox.exe -e coverage - %PYTHON%\Scripts\tox.exe -e pyinstaller - # To verify that the resultant PyInstaller-generated binary executes - # cleanly (i.e., that it terminates with an exit code of 0 and isn't - # failing due to import/packaging-related errors, etc.). - - dist\Tahoe-LAFS\tahoe.exe --version - -after_test: - # This builds the main tahoe wheel, and wheels for all dependencies. - # Again, you only need build.cmd if you're building C extensions for - # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct - # interpreter. If _trial_temp still exists, the "pip wheel" fails on - # _trial_temp\local_dir (not sure why). - - | - copy _trial_temp\test.log trial_test_log.txt - rd /s /q _trial_temp - %PYTHON%\python.exe setup.py bdist_wheel - %PYTHON%\python.exe -m pip wheel -w dist . - - | - %PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5" - %PYTHON%\python.exe -m coverage xml -o coverage.xml -i - %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml - -artifacts: - # bdist_wheel puts your built wheel in the dist directory - # "pip wheel -w dist ." puts all the dependency wheels there too - # this gives us a zipfile with everything - - path: 'dist\*' - - path: trial_test_log.txt - name: Trial test.log - - path: eliot.log - name: Eliot test log - -on_failure: - # Artifacts are not normally uploaded when the job fails. To get the test - # logs, we have to push them ourselves. - - ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log - - ps: Push-AppveyorArtifact eliot.log -Filename eliot.log - -#on_success: -# You can use this step to upload your artifacts to a public website. -# See Appveyor's documentation for more details. Or you can simply -# access your wheels from the Appveyor "artifacts" tab for your build. diff --git a/.circleci/config.yml b/.circleci/config.yml index c15eb1746..df181f058 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -285,7 +285,7 @@ jobs: # this reporter on Python 3. So drop that and just specify the # reporter. TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage" ubuntu-20.04: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cd97dcca..34a4e0875 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,8 @@ jobs: - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "tox -e coverage" - run: tox -e coverage + - name: Run "tox -e py27-coverage" + run: tox -e py27-coverage - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/docs/how_to_make_a_tahoe-lafs_release.org b/docs/how_to_make_a_tahoe-lafs_release.org index 44b9e3dd1..b3f2a84d7 100644 --- a/docs/how_to_make_a_tahoe-lafs_release.org +++ b/docs/how_to_make_a_tahoe-lafs_release.org @@ -36,7 +36,7 @@ people are Release Maintainers: - [ ] documentation is ready (see above) - [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z - [ ] build code locally: - tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations + tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations - [ ] created tarballs (they'll be in dist/ for later comparison) tox -e tarballs - [ ] release version is reporting itself as intended version diff --git a/newsfragments/3355.other b/newsfragments/3355.other new file mode 100644 index 000000000..4e854e4dd --- /dev/null +++ b/newsfragments/3355.other @@ -0,0 +1 @@ +The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments. diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3395.minor b/newsfragments/3395.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 081d9a33d..c93c1d81d 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,4 +1,3 @@ - from past.builtins import long from zope.interface import Interface, Attribute diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 75162c39e..f013c0205 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -6,6 +6,8 @@ from twisted.python import usage from allmydata.util import configutil from ..common_util import run_cli, parse_cli from ...scripts import create_node +from ... import client + def read_config(basedir): tahoe_cfg = os.path.join(basedir, "tahoe.cfg") @@ -33,6 +35,31 @@ class Config(unittest.TestCase): e = self.assertRaises(usage.UsageError, parse_cli, verb, *args) self.assertIn("option %s not recognized" % (option,), str(e)) + def test_create_client_config(self): + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + + with open(fname, 'w') as f: + opts = {"nickname": "nick", + "webport": "tcp:3456", + "hide-ip": False, + "listen": "none", + "shares-needed": "1", + "shares-happy": "1", + "shares-total": "1", + } + create_node.write_node_config(f, opts) + create_node.write_client_config(f, opts) + + config = configutil.get_config(fname) + # should succeed, no exceptions + configutil.validate_config( + fname, + config, + client._valid_config(), + ) + @defer.inlineCallbacks def test_client(self): basedir = self.mktemp() diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index 45eb6ac25..c57381289 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -1,14 +1,26 @@ +""" +Tests for allmydata.util.configutil. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # Omitted dict, cause worried about interactions. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest from allmydata.util import configutil -from allmydata.test.no_network import GridTestMixin -from ..scripts import create_node -from .. import client -class ConfigUtilTests(GridTestMixin, unittest.TestCase): +class ConfigUtilTests(unittest.TestCase): def setUp(self): super(ConfigUtilTests, self).setUp() self.static_valid_config = configutil.ValidConfiguration( @@ -20,10 +32,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): lambda section_name, item_name: (section_name, item_name) == ("node", "valid"), ) + def create_tahoe_cfg(self, cfg): + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + with open(fname, "w") as f: + f.write(cfg) + return fname + def test_config_utils(self): - self.basedir = "cli/ConfigUtilTests/test-config-utils" - self.set_up_grid(oneshare=True) - tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg") + tahoe_cfg = self.create_tahoe_cfg("""\ +[node] +nickname = client-0 +web.port = adopt-socket:fd=5 +[storage] +enabled = false +""") # test that at least one option was read correctly config = configutil.get_config(tahoe_cfg) @@ -45,12 +69,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): self.failUnlessEqual(config.get("node", "descriptor"), descriptor) def test_config_validation_success(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n') config = configutil.get_config(fname) # should succeed, no exceptions @@ -66,12 +85,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): validation but are matched by the dynamic validation is considered valid. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n') config = configutil.get_config(fname) # should succeed, no exceptions @@ -82,12 +96,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): ) def test_config_validation_invalid_item(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\ninvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -103,12 +112,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n[invalid]\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -124,12 +128,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n[invalid]\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -145,12 +144,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section, item pair that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\ninvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -160,28 +154,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): self.dynamic_valid_config, ) self.assertIn("section [node] contains unknown option 'invalid'", str(e)) - - def test_create_client_config(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - opts = {"nickname": "nick", - "webport": "tcp:3456", - "hide-ip": False, - "listen": "none", - "shares-needed": "1", - "shares-happy": "1", - "shares-total": "1", - } - create_node.write_node_config(f, opts) - create_node.write_client_config(f, opts) - - config = configutil.get_config(fname) - # should succeed, no exceptions - configutil.validate_config( - fname, - config, - client._valid_config(), - ) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py new file mode 100644 index 000000000..2bd8bf6ab --- /dev/null +++ b/src/allmydata/test/test_connection_status.py @@ -0,0 +1,122 @@ +""" +Tests for allmydata.util.connection_status. + +Port to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import mock + +from twisted.trial import unittest + +from ..util import connection_status + +class Status(unittest.TestCase): + def test_hint_statuses(self): + ncs = connection_status._hint_statuses(["h2","h1"], + {"h1": "hand1", "h4": "hand4"}, + {"h1": "st1", "h2": "st2", + "h3": "st3"}) + self.assertEqual(ncs, {"h1 via hand1": "st1", + "h2": "st2"}) + + def test_reconnector_connected(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1"} + ci.connectionHandlers = {"h1": "hand1"} + ci.winningHint = "h1" + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected to h1 via hand1") + self.assertEqual(cs.non_connected_statuses, {}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connected_others(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ci.winningHint = "h1" + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected to h1 via hand1") + self.assertEqual(cs.non_connected_statuses, {"h2": "st2"}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connected_listener(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ci.listenerStatus = ("listener1", "successful") + ci.winningHint = None + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected via listener (listener1)") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connecting(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ri = mock.Mock() + ri.state = "connecting" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, False) + self.assertEqual(cs.summary, "Trying to connect") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, None) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_waiting(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ri = mock.Mock() + ri.state = "waiting" + ri.lastAttempt = 10 + ri.nextAttempt = 20 + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + with mock.patch("time.time", return_value=12): + cs = connection_status.from_foolscap_reconnector(rc, 5) + self.assertEqual(cs.connected, False) + self.assertEqual(cs.summary, + "Reconnecting in 8 seconds (last attempt 2s ago)") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, None) + self.assertEqual(cs.last_received_time, 5) diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 3e2806dd0..9b5bd7f30 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -7,7 +7,6 @@ from foolscap.connections import tcp from ..node import PrivacyError, config_from_string from ..node import create_connection_handlers from ..node import create_main_tub, _tub_portlocation -from ..util import connection_status from ..util.i2p_provider import create as create_i2p_provider from ..util.tor_provider import create as create_tor_provider @@ -463,106 +462,3 @@ class Privacy(unittest.TestCase): str(ctx.exception), "tub.location includes tcp: hint", ) - -class Status(unittest.TestCase): - def test_hint_statuses(self): - ncs = connection_status._hint_statuses(["h2","h1"], - {"h1": "hand1", "h4": "hand4"}, - {"h1": "st1", "h2": "st2", - "h3": "st3"}) - self.assertEqual(ncs, {"h1 via hand1": "st1", - "h2": "st2"}) - - def test_reconnector_connected(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected to h1 via hand1") - self.assertEqual(cs.non_connected_statuses, {}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connected_others(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected to h1 via hand1") - self.assertEqual(cs.non_connected_statuses, {"h2": "st2"}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connected_listener(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.listenerStatus = ("listener1", "successful") - ci.winningHint = None - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected via listener (listener1)") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connecting(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() - ri.state = "connecting" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, False) - self.assertEqual(cs.summary, "Trying to connect") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, None) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_waiting(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() - ri.state = "waiting" - ri.lastAttempt = 10 - ri.nextAttempt = 20 - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - with mock.patch("time.time", return_value=12): - cs = connection_status.from_foolscap_reconnector(rc, 5) - self.assertEqual(cs.connected, False) - self.assertEqual(cs.summary, - "Reconnecting in 8 seconds (last attempt 2s ago)") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, None) - self.assertEqual(cs.last_received_time, 5) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 9c64c2f45..19f98851f 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -9,7 +9,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: # Omitted list sinc it broke a test on Python 2. Shouldn't require further # work, when we switch to Python 3 we'll be dropping this, anyway. @@ -19,7 +19,6 @@ import time import os.path import re import json -from unittest import skipIf from twisted.trial import unittest @@ -107,7 +106,6 @@ class MyStorageServer(StorageServer): class BucketCounter(unittest.TestCase, pollmixin.PollMixin): - @skipIf(PY3, "Not ported yet.") def setUp(self): self.s = service.MultiService() self.s.startService() @@ -130,12 +128,12 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): # this sample is before the crawler has started doing anything html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Accepting new shares: Yes", s) - self.failUnlessIn("Reserved space: - 0 B (0)", s) - self.failUnlessIn("Total buckets: Not computed yet", s) - self.failUnlessIn("Next crawl in", s) + self.failUnlessIn(b"Accepting new shares: Yes", s) + self.failUnlessIn(b"Reserved space: - 0 B (0)", s) + self.failUnlessIn(b"Total buckets: Not computed yet", s) + self.failUnlessIn(b"Next crawl in", s) # give the bucket-counting-crawler one tick to get started. The # cpu_slice=0 will force it to yield right after it processes the @@ -154,8 +152,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible html = renderSynchronously(w) s = remove_tags(html) - self.failUnlessIn(" Current crawl ", s) - self.failUnlessIn(" (next work in ", s) + self.failUnlessIn(b" Current crawl ", s) + self.failUnlessIn(b" (next work in ", s) d.addCallback(_check) # now give it enough time to complete a full cycle @@ -166,8 +164,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): ss.bucket_counter.cpu_slice = orig_cpu_slice html = renderSynchronously(w) s = remove_tags(html) - self.failUnlessIn("Total buckets: 0 (the number of", s) - self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) + self.failUnlessIn(b"Total buckets: 0 (the number of", s) + self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) d.addCallback(_check2) return d @@ -228,20 +226,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): # no ETA is available yet html = renderSynchronously(w) s = remove_tags(html) - self.failUnlessIn("complete (next work", s) + self.failUnlessIn(b"complete (next work", s) def _check_2(ignored): # one prefix has finished, so an ETA based upon that elapsed time # should be available. html = renderSynchronously(w) s = remove_tags(html) - self.failUnlessIn("complete (ETA ", s) + self.failUnlessIn(b"complete (ETA ", s) def _check_3(ignored): # two prefixes have finished html = renderSynchronously(w) s = remove_tags(html) - self.failUnlessIn("complete (ETA ", s) + self.failUnlessIn(b"complete (ETA ", s) d.callback("done") hooks[0].addCallback(_check_1).addErrback(d.errback) @@ -1172,7 +1170,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): class WebStatus(unittest.TestCase, pollmixin.PollMixin): - @skipIf(PY3, "Not ported yet.") def setUp(self): self.s = service.MultiService() self.s.startService() @@ -1182,7 +1179,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): def test_no_server(self): w = StorageStatus(None) html = renderSynchronously(w) - self.failUnlessIn("

No Storage Server Running

", html) + self.failUnlessIn(b"

No Storage Server Running

", html) def test_status(self): basedir = "storage/WebStatus/status" @@ -1193,12 +1190,12 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): w = StorageStatus(ss, "nickname") d = renderDeferred(w) def _check_html(html): - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Server Nickname: nickname", s) - self.failUnlessIn("Server Nodeid: %s" % base32.b2a(nodeid), s) - self.failUnlessIn("Accepting new shares: Yes", s) - self.failUnlessIn("Reserved space: - 0 B (0)", s) + self.failUnlessIn(b"Server Nickname: nickname", s) + self.failUnlessIn(b"Server Nodeid: %s" % base32.b2a(nodeid), s) + self.failUnlessIn(b"Accepting new shares: Yes", s) + self.failUnlessIn(b"Reserved space: - 0 B (0)", s) d.addCallback(_check_html) d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): @@ -1225,11 +1222,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Accepting new shares: Yes", s) - self.failUnlessIn("Total disk space: ?", s) - self.failUnlessIn("Space Available to Tahoe: ?", s) + self.failUnlessIn(b"Accepting new shares: Yes", s) + self.failUnlessIn(b"Total disk space: ?", s) + self.failUnlessIn(b"Space Available to Tahoe: ?", s) self.failUnless(ss.get_available_space() is None) def test_status_bad_disk_stats(self): @@ -1245,11 +1242,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Accepting new shares: No", s) - self.failUnlessIn("Total disk space: ?", s) - self.failUnlessIn("Space Available to Tahoe: ?", s) + self.failUnlessIn(b"Accepting new shares: No", s) + self.failUnlessIn(b"Total disk space: ?", s) + self.failUnlessIn(b"Space Available to Tahoe: ?", s) self.failUnlessEqual(ss.get_available_space(), 0) def test_status_right_disk_stats(self): @@ -1282,14 +1279,14 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): w = StorageStatus(ss) html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Total disk space: 5.00 GB", s) - self.failUnlessIn("Disk space used: - 1.00 GB", s) - self.failUnlessIn("Disk space free (root): 4.00 GB", s) - self.failUnlessIn("Disk space free (non-root): 3.00 GB", s) - self.failUnlessIn("Reserved space: - 1.00 GB", s) - self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s) + self.failUnlessIn(b"Total disk space: 5.00 GB", s) + self.failUnlessIn(b"Disk space used: - 1.00 GB", s) + self.failUnlessIn(b"Disk space free (root): 4.00 GB", s) + self.failUnlessIn(b"Disk space free (non-root): 3.00 GB", s) + self.failUnlessIn(b"Reserved space: - 1.00 GB", s) + self.failUnlessIn(b"Space Available to Tahoe: 2.00 GB", s) self.failUnlessEqual(ss.get_available_space(), 2*GB) def test_readonly(self): @@ -1299,9 +1296,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Accepting new shares: No", s) + self.failUnlessIn(b"Accepting new shares: No", s) def test_reserved(self): basedir = "storage/WebStatus/reserved" @@ -1310,9 +1307,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) + self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s) def test_huge_reserved(self): basedir = "storage/WebStatus/reserved" @@ -1321,9 +1318,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) html = renderSynchronously(w) - self.failUnlessIn("

Storage Server Status

", html) + self.failUnlessIn(b"

Storage Server Status

", html) s = remove_tags(html) - self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) + self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s) def test_util(self): w = StorageStatusElement(None, None) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6aa1010bc..2ec91ed96 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -41,6 +41,8 @@ PORTED_MODULES = [ "allmydata.util.assertutil", "allmydata.util.base32", "allmydata.util.base62", + "allmydata.util.configutil", + "allmydata.util.connection_status", "allmydata.util.deferredutil", "allmydata.util.fileutil", "allmydata.util.dictutil", @@ -66,6 +68,8 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_configutil", + "allmydata.test.test_connection_status", "allmydata.test.test_crawler", "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", @@ -83,7 +87,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_python3", "allmydata.test.test_spans", "allmydata.test.test_statistics", - "allmydata.test.test_storage_web", # partial, WIP + "allmydata.test.test_storage_web", "allmydata.test.test_time_format", "allmydata.test.test_uri", "allmydata.test.test_util", diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 3699db35d..1a1a93f18 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -1,8 +1,32 @@ +""" +Read/write config files. -from ConfigParser import SafeConfigParser +Configuration is returned as native strings. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We don't do open(), because we want files to read/write native strs when + # we do "r" or "w". + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +if PY2: + # In theory on Python 2 configparser also works, but then code gets the + # wrong exceptions and they don't get handled. So just use native parser + # for now. + from ConfigParser import SafeConfigParser +else: + from configparser import SafeConfigParser import attr + class UnknownConfigError(Exception): """ An unknown config item was found. @@ -12,11 +36,16 @@ class UnknownConfigError(Exception): def get_config(tahoe_cfg): + """Load the config, returning a SafeConfigParser. + + Configuration is returned as native strings. + """ config = SafeConfigParser() - with open(tahoe_cfg, "rb") as f: - # Skip any initial Byte Order Mark. Since this is an ordinary file, we - # don't need to handle incomplete reads, and can assume seekability. - if f.read(3) != '\xEF\xBB\xBF': + with open(tahoe_cfg, "r") as f: + # On Python 2, where we read in bytes, skip any initial Byte Order + # Mark. Since this is an ordinary file, we don't need to handle + # incomplete reads, and can assume seekability. + if PY2 and f.read(3) != b'\xEF\xBB\xBF': f.seek(0) config.readfp(f) return config @@ -28,7 +57,7 @@ def set_config(config, section, option, value): assert config.get(section, option) == value def write_config(tahoe_cfg, config): - with open(tahoe_cfg, "wb") as f: + with open(tahoe_cfg, "w") as f: config.write(f) def validate_config(fname, cfg, valid_config): diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index 44c12f220..0e8595e81 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -1,3 +1,18 @@ +""" +Parse connection status from Foolscap. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time from zope.interface import implementer from ..interfaces import IConnectionStatus @@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses): def from_foolscap_reconnector(rc, last_received): ri = rc.getReconnectionInfo() - # See foolscap/reconnector.py, ReconnectionInfo, for details about - # possible states. + # See foolscap/reconnector.py, ReconnectionInfo, for details about possible + # states. The returned result is a native string, it seems, so convert to + # unicode. state = ri.state + if isinstance(state, bytes): # Python 2 + state = str(state, "ascii") if state == "unstarted": return ConnectionStatus.unstarted() diff --git a/tox.ini b/tox.ini index 98ca90c39..2cb59388f 100644 --- a/tox.ini +++ b/tox.ini @@ -44,36 +44,42 @@ usedevelop = False # We use extras=test to get things like "mock" that are required for our unit # tests. extras = test -commands = - trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} - tahoe --version -[testenv:py36] +setenv = + # Define TEST_SUITE in the environment as an aid to constructing the + # correct test command below. + !py36: TEST_SUITE = allmydata + py36: TEST_SUITE = allmydata.test.python3_tests + commands = - trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests} + # As an aid to debugging, dump all of the Python packages and their + # versions that are installed in the test environment. This is + # particularly useful to get from CI runs - though hopefully the + # version pinning we do limits the variability of this output + pip freeze + + # The tahoe script isn't sufficiently ported for this to succeed on + # Python 3.x yet. + !py36: tahoe --version + + !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}} + + # measuring coverage is somewhat slower than not measuring coverage + # so only do it on request. + coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}} + coverage: coverage combine + coverage: coverage xml + [testenv:integration] setenv = COVERAGE_PROCESS_START=.coveragerc commands = - # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --coverage -v {posargs:integration} + # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' + py.test --coverage -v {posargs:integration} coverage combine coverage report -[testenv:coverage] -# coverage (with --branch) takes about 65% longer to run -commands = - # As an aid to debugging, dump all of the Python packages and their - # versions that are installed in the test environment. This is - # particularly useful to get from CI runs - though hopefully the - # version pinning we do limits the variability of this output - # somewhat. - pip freeze - tahoe --version - coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata} - coverage combine - coverage xml [testenv:codechecks] # On macOS, git inside of towncrier needs $HOME. @@ -87,11 +93,11 @@ commands = python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py python misc/coding_tools/check-miscaptures.py - # If towncrier.check fails, you forgot to add a towncrier news - # fragment explaining the change in this branch. Create one at - # `newsfragments/.` with some text for the news - # file. See pyproject.toml for legal values. - python -m towncrier.check --pyproject towncrier.pyproject.toml + # If towncrier.check fails, you forgot to add a towncrier news + # fragment explaining the change in this branch. Create one at + # `newsfragments/.` with some text for the news + # file. See pyproject.toml for legal values. + python -m towncrier.check --pyproject towncrier.pyproject.toml [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH @@ -110,9 +116,9 @@ commands = # # Some discussion is available at # https://github.com/pypa/pip/issues/5696 - # - # towncrier post 19.2 (unreleased as of this writing) adds a --config - # option that can be used instead of this file shuffling. + # + # towncrier post 19.2 (unreleased as of this writing) adds a --config + # option that can be used instead of this file shuffling. mv towncrier.pyproject.toml pyproject.toml # towncrier 19.2 + works with python2.7 @@ -138,9 +144,9 @@ commands = # # Some discussion is available at # https://github.com/pypa/pip/issues/5696 - # - # towncrier post 19.2 (unreleased as of this writing) adds a --config - # option that can be used instead of this file shuffling. + # + # towncrier post 19.2 (unreleased as of this writing) adds a --config + # option that can be used instead of this file shuffling. mv towncrier.pyproject.toml pyproject.toml # towncrier 19.2 + works with python2.7