Merge remote-tracking branch 'origin/master' into 3396.storage-tests-python-3

This commit is contained in:
Itamar Turner-Trauring 2020-08-27 15:15:46 -04:00
commit f998e0e752
17 changed files with 328 additions and 355 deletions

View File

@ -1,95 +0,0 @@
# adapted from https://packaging.python.org/en/latest/appveyor/
environment:
matrix:
# For Python versions available on Appveyor, see
# http://www.appveyor.com/docs/installed-software#python
- PYTHON: "C:\\Python27"
- PYTHON: "C:\\Python27-x64"
# DISTUTILS_USE_SDK: "1"
# TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB"
install:
- |
%PYTHON%\python.exe -m pip install -U pip
%PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv
# note:
# %PYTHON% has: python.exe
# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip)
# We have a custom "build" system. We don't need MSBuild or whatever.
build: off
# Do not build feature branch with open pull requests. This is documented but
# it's not clear it does anything.
skip_branch_with_pr: true
# This, perhaps, is effective.
branches:
# whitelist
only:
- 'master'
skip_commits:
files:
# The Windows builds are unaffected by news fragments.
- 'newsfragments/*'
# Also, all this build junk.
- '.circleci/*'
- '.lgtm.yml'
- '.travis.yml'
# we run from C:\projects\tahoe-lafs
test_script:
# Put your test command here.
# Note that you must use the environment variable %PYTHON% to refer to
# the interpreter you're using - Appveyor does not do anything special
# to put the Python version you want to use on PATH.
- |
%PYTHON%\Scripts\tox.exe -e coverage
%PYTHON%\Scripts\tox.exe -e pyinstaller
# To verify that the resultant PyInstaller-generated binary executes
# cleanly (i.e., that it terminates with an exit code of 0 and isn't
# failing due to import/packaging-related errors, etc.).
- dist\Tahoe-LAFS\tahoe.exe --version
after_test:
# This builds the main tahoe wheel, and wheels for all dependencies.
# Again, you only need build.cmd if you're building C extensions for
# 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
# interpreter. If _trial_temp still exists, the "pip wheel" fails on
# _trial_temp\local_dir (not sure why).
- |
copy _trial_temp\test.log trial_test_log.txt
rd /s /q _trial_temp
%PYTHON%\python.exe setup.py bdist_wheel
%PYTHON%\python.exe -m pip wheel -w dist .
- |
%PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5"
%PYTHON%\python.exe -m coverage xml -o coverage.xml -i
%PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
artifacts:
# bdist_wheel puts your built wheel in the dist directory
# "pip wheel -w dist ." puts all the dependency wheels there too
# this gives us a zipfile with everything
- path: 'dist\*'
- path: trial_test_log.txt
name: Trial test.log
- path: eliot.log
name: Eliot test log
on_failure:
# Artifacts are not normally uploaded when the job fails. To get the test
# logs, we have to push them ourselves.
- ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log
- ps: Push-AppveyorArtifact eliot.log -Filename eliot.log
#on_success:
# You can use this step to upload your artifacts to a public website.
# See Appveyor's documentation for more details. Or you can simply
# access your wheels from the Appveyor "artifacts" tab for your build.

View File

@ -285,7 +285,7 @@ jobs:
# this reporter on Python 3. So drop that and just specify the
# reporter.
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
ubuntu-20.04:

View File

@ -49,8 +49,8 @@ jobs:
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run "tox -e coverage"
run: tox -e coverage
- name: Run "tox -e py27-coverage"
run: tox -e py27-coverage
- name: Upload eliot.log in case of failure
uses: actions/upload-artifact@v1

View File

@ -36,7 +36,7 @@ people are Release Maintainers:
- [ ] documentation is ready (see above)
- [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
- [ ] build code locally:
tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations
tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
- [ ] created tarballs (they'll be in dist/ for later comparison)
tox -e tarballs
- [ ] release version is reporting itself as intended version

1
newsfragments/3355.other Normal file
View File

@ -0,0 +1 @@
The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.

0
newsfragments/3377.minor Normal file
View File

0
newsfragments/3395.minor Normal file
View File

View File

@ -1,4 +1,3 @@
from past.builtins import long
from zope.interface import Interface, Attribute

View File

@ -6,6 +6,8 @@ from twisted.python import usage
from allmydata.util import configutil
from ..common_util import run_cli, parse_cli
from ...scripts import create_node
from ... import client
def read_config(basedir):
tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
@ -33,6 +35,31 @@ class Config(unittest.TestCase):
e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
self.assertIn("option %s not recognized" % (option,), str(e))
def test_create_client_config(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
opts = {"nickname": "nick",
"webport": "tcp:3456",
"hide-ip": False,
"listen": "none",
"shares-needed": "1",
"shares-happy": "1",
"shares-total": "1",
}
create_node.write_node_config(f, opts)
create_node.write_client_config(f, opts)
config = configutil.get_config(fname)
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
client._valid_config(),
)
@defer.inlineCallbacks
def test_client(self):
basedir = self.mktemp()

View File

@ -1,14 +1,26 @@
"""
Tests for allmydata.util.configutil.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Omitted dict, cause worried about interactions.
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
import os.path
from twisted.trial import unittest
from allmydata.util import configutil
from allmydata.test.no_network import GridTestMixin
from ..scripts import create_node
from .. import client
class ConfigUtilTests(GridTestMixin, unittest.TestCase):
class ConfigUtilTests(unittest.TestCase):
def setUp(self):
super(ConfigUtilTests, self).setUp()
self.static_valid_config = configutil.ValidConfiguration(
@ -20,10 +32,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
)
def create_tahoe_cfg(self, cfg):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, "w") as f:
f.write(cfg)
return fname
def test_config_utils(self):
self.basedir = "cli/ConfigUtilTests/test-config-utils"
self.set_up_grid(oneshare=True)
tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
tahoe_cfg = self.create_tahoe_cfg("""\
[node]
nickname = client-0
web.port = adopt-socket:fd=5
[storage]
enabled = false
""")
# test that at least one option was read correctly
config = configutil.get_config(tahoe_cfg)
@ -45,12 +69,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
def test_config_validation_success(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
config = configutil.get_config(fname)
# should succeed, no exceptions
@ -66,12 +85,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
validation but are matched by the dynamic validation is considered
valid.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
config = configutil.get_config(fname)
# should succeed, no exceptions
@ -82,12 +96,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
)
def test_config_validation_invalid_item(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\ninvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -103,12 +112,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
A configuration with a section that is matched by neither the static nor
dynamic validators is rejected.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n[invalid]\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -124,12 +128,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
A configuration with a section that is matched by neither the static nor
dynamic validators is rejected.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n[invalid]\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -145,12 +144,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
A configuration with a section, item pair that is matched by neither the
static nor dynamic validators is rejected.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\ninvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -160,28 +154,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
self.dynamic_valid_config,
)
self.assertIn("section [node] contains unknown option 'invalid'", str(e))
def test_create_client_config(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
opts = {"nickname": "nick",
"webport": "tcp:3456",
"hide-ip": False,
"listen": "none",
"shares-needed": "1",
"shares-happy": "1",
"shares-total": "1",
}
create_node.write_node_config(f, opts)
create_node.write_client_config(f, opts)
config = configutil.get_config(fname)
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
client._valid_config(),
)

View File

@ -0,0 +1,122 @@
"""
Tests for allmydata.util.connection_status.
Port to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import mock
from twisted.trial import unittest
from ..util import connection_status
class Status(unittest.TestCase):
def test_hint_statuses(self):
ncs = connection_status._hint_statuses(["h2","h1"],
{"h1": "hand1", "h4": "hand4"},
{"h1": "st1", "h2": "st2",
"h3": "st3"})
self.assertEqual(ncs, {"h1 via hand1": "st1",
"h2": "st2"})
def test_reconnector_connected(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_others(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_listener(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.listenerStatus = ("listener1", "successful")
ci.winningHint = None
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected via listener (listener1)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connecting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "connecting"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary, "Trying to connect")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_waiting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "waiting"
ri.lastAttempt = 10
ri.nextAttempt = 20
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
with mock.patch("time.time", return_value=12):
cs = connection_status.from_foolscap_reconnector(rc, 5)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary,
"Reconnecting in 8 seconds (last attempt 2s ago)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 5)

View File

@ -7,7 +7,6 @@ from foolscap.connections import tcp
from ..node import PrivacyError, config_from_string
from ..node import create_connection_handlers
from ..node import create_main_tub, _tub_portlocation
from ..util import connection_status
from ..util.i2p_provider import create as create_i2p_provider
from ..util.tor_provider import create as create_tor_provider
@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
str(ctx.exception),
"tub.location includes tcp: hint",
)
class Status(unittest.TestCase):
def test_hint_statuses(self):
ncs = connection_status._hint_statuses(["h2","h1"],
{"h1": "hand1", "h4": "hand4"},
{"h1": "st1", "h2": "st2",
"h3": "st3"})
self.assertEqual(ncs, {"h1 via hand1": "st1",
"h2": "st2"})
def test_reconnector_connected(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_others(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_listener(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.listenerStatus = ("listener1", "successful")
ci.winningHint = None
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected via listener (listener1)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connecting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "connecting"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary, "Trying to connect")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_waiting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "waiting"
ri.lastAttempt = 10
ri.nextAttempt = 20
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
with mock.patch("time.time", return_value=12):
cs = connection_status.from_foolscap_reconnector(rc, 5)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary,
"Reconnecting in 8 seconds (last attempt 2s ago)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 5)

View File

@ -9,7 +9,7 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, PY3
from future.utils import PY2
if PY2:
# Omitted list sinc it broke a test on Python 2. Shouldn't require further
# work, when we switch to Python 3 we'll be dropping this, anyway.
@ -19,7 +19,6 @@ import time
import os.path
import re
import json
from unittest import skipIf
from twisted.trial import unittest
@ -107,7 +106,6 @@ class MyStorageServer(StorageServer):
class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
@skipIf(PY3, "Not ported yet.")
def setUp(self):
self.s = service.MultiService()
self.s.startService()
@ -130,12 +128,12 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
# this sample is before the crawler has started doing anything
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Accepting new shares: Yes", s)
self.failUnlessIn("Reserved space: - 0 B (0)", s)
self.failUnlessIn("Total buckets: Not computed yet", s)
self.failUnlessIn("Next crawl in", s)
self.failUnlessIn(b"Accepting new shares: Yes", s)
self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
self.failUnlessIn(b"Total buckets: Not computed yet", s)
self.failUnlessIn(b"Next crawl in", s)
# give the bucket-counting-crawler one tick to get started. The
# cpu_slice=0 will force it to yield right after it processes the
@ -154,8 +152,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible
html = renderSynchronously(w)
s = remove_tags(html)
self.failUnlessIn(" Current crawl ", s)
self.failUnlessIn(" (next work in ", s)
self.failUnlessIn(b" Current crawl ", s)
self.failUnlessIn(b" (next work in ", s)
d.addCallback(_check)
# now give it enough time to complete a full cycle
@ -166,8 +164,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
ss.bucket_counter.cpu_slice = orig_cpu_slice
html = renderSynchronously(w)
s = remove_tags(html)
self.failUnlessIn("Total buckets: 0 (the number of", s)
self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
self.failUnlessIn(b"Total buckets: 0 (the number of", s)
self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
d.addCallback(_check2)
return d
@ -228,20 +226,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
# no ETA is available yet
html = renderSynchronously(w)
s = remove_tags(html)
self.failUnlessIn("complete (next work", s)
self.failUnlessIn(b"complete (next work", s)
def _check_2(ignored):
# one prefix has finished, so an ETA based upon that elapsed time
# should be available.
html = renderSynchronously(w)
s = remove_tags(html)
self.failUnlessIn("complete (ETA ", s)
self.failUnlessIn(b"complete (ETA ", s)
def _check_3(ignored):
# two prefixes have finished
html = renderSynchronously(w)
s = remove_tags(html)
self.failUnlessIn("complete (ETA ", s)
self.failUnlessIn(b"complete (ETA ", s)
d.callback("done")
hooks[0].addCallback(_check_1).addErrback(d.errback)
@ -1172,7 +1170,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
class WebStatus(unittest.TestCase, pollmixin.PollMixin):
@skipIf(PY3, "Not ported yet.")
def setUp(self):
self.s = service.MultiService()
self.s.startService()
@ -1182,7 +1179,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
def test_no_server(self):
w = StorageStatus(None)
html = renderSynchronously(w)
self.failUnlessIn("<h1>No Storage Server Running</h1>", html)
self.failUnlessIn(b"<h1>No Storage Server Running</h1>", html)
def test_status(self):
basedir = "storage/WebStatus/status"
@ -1193,12 +1190,12 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
w = StorageStatus(ss, "nickname")
d = renderDeferred(w)
def _check_html(html):
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Server Nickname: nickname", s)
self.failUnlessIn("Server Nodeid: %s" % base32.b2a(nodeid), s)
self.failUnlessIn("Accepting new shares: Yes", s)
self.failUnlessIn("Reserved space: - 0 B (0)", s)
self.failUnlessIn(b"Server Nickname: nickname", s)
self.failUnlessIn(b"Server Nodeid: %s" % base32.b2a(nodeid), s)
self.failUnlessIn(b"Accepting new shares: Yes", s)
self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
d.addCallback(_check_html)
d.addCallback(lambda ign: renderJSON(w))
def _check_json(raw):
@ -1225,11 +1222,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Accepting new shares: Yes", s)
self.failUnlessIn("Total disk space: ?", s)
self.failUnlessIn("Space Available to Tahoe: ?", s)
self.failUnlessIn(b"Accepting new shares: Yes", s)
self.failUnlessIn(b"Total disk space: ?", s)
self.failUnlessIn(b"Space Available to Tahoe: ?", s)
self.failUnless(ss.get_available_space() is None)
def test_status_bad_disk_stats(self):
@ -1245,11 +1242,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Accepting new shares: No", s)
self.failUnlessIn("Total disk space: ?", s)
self.failUnlessIn("Space Available to Tahoe: ?", s)
self.failUnlessIn(b"Accepting new shares: No", s)
self.failUnlessIn(b"Total disk space: ?", s)
self.failUnlessIn(b"Space Available to Tahoe: ?", s)
self.failUnlessEqual(ss.get_available_space(), 0)
def test_status_right_disk_stats(self):
@ -1282,14 +1279,14 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
w = StorageStatus(ss)
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Total disk space: 5.00 GB", s)
self.failUnlessIn("Disk space used: - 1.00 GB", s)
self.failUnlessIn("Disk space free (root): 4.00 GB", s)
self.failUnlessIn("Disk space free (non-root): 3.00 GB", s)
self.failUnlessIn("Reserved space: - 1.00 GB", s)
self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s)
self.failUnlessIn(b"Total disk space: 5.00 GB", s)
self.failUnlessIn(b"Disk space used: - 1.00 GB", s)
self.failUnlessIn(b"Disk space free (root): 4.00 GB", s)
self.failUnlessIn(b"Disk space free (non-root): 3.00 GB", s)
self.failUnlessIn(b"Reserved space: - 1.00 GB", s)
self.failUnlessIn(b"Space Available to Tahoe: 2.00 GB", s)
self.failUnlessEqual(ss.get_available_space(), 2*GB)
def test_readonly(self):
@ -1299,9 +1296,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Accepting new shares: No", s)
self.failUnlessIn(b"Accepting new shares: No", s)
def test_reserved(self):
basedir = "storage/WebStatus/reserved"
@ -1310,9 +1307,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
def test_huge_reserved(self):
basedir = "storage/WebStatus/reserved"
@ -1321,9 +1318,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = renderSynchronously(w)
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
s = remove_tags(html)
self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
def test_util(self):
w = StorageStatusElement(None, None)

View File

@ -41,6 +41,8 @@ PORTED_MODULES = [
"allmydata.util.assertutil",
"allmydata.util.base32",
"allmydata.util.base62",
"allmydata.util.configutil",
"allmydata.util.connection_status",
"allmydata.util.deferredutil",
"allmydata.util.fileutil",
"allmydata.util.dictutil",
@ -66,6 +68,8 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_abbreviate",
"allmydata.test.test_base32",
"allmydata.test.test_base62",
"allmydata.test.test_configutil",
"allmydata.test.test_connection_status",
"allmydata.test.test_crawler",
"allmydata.test.test_crypto",
"allmydata.test.test_deferredutil",
@ -83,7 +87,7 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_python3",
"allmydata.test.test_spans",
"allmydata.test.test_statistics",
"allmydata.test.test_storage_web", # partial, WIP
"allmydata.test.test_storage_web",
"allmydata.test.test_time_format",
"allmydata.test.test_uri",
"allmydata.test.test_util",

View File

@ -1,8 +1,32 @@
"""
Read/write config files.
from ConfigParser import SafeConfigParser
Configuration is returned as native strings.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# We don't do open(), because we want files to read/write native strs when
# we do "r" or "w".
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
if PY2:
# In theory on Python 2 configparser also works, but then code gets the
# wrong exceptions and they don't get handled. So just use native parser
# for now.
from ConfigParser import SafeConfigParser
else:
from configparser import SafeConfigParser
import attr
class UnknownConfigError(Exception):
"""
An unknown config item was found.
@ -12,11 +36,16 @@ class UnknownConfigError(Exception):
def get_config(tahoe_cfg):
"""Load the config, returning a SafeConfigParser.
Configuration is returned as native strings.
"""
config = SafeConfigParser()
with open(tahoe_cfg, "rb") as f:
# Skip any initial Byte Order Mark. Since this is an ordinary file, we
# don't need to handle incomplete reads, and can assume seekability.
if f.read(3) != '\xEF\xBB\xBF':
with open(tahoe_cfg, "r") as f:
# On Python 2, where we read in bytes, skip any initial Byte Order
# Mark. Since this is an ordinary file, we don't need to handle
# incomplete reads, and can assume seekability.
if PY2 and f.read(3) != b'\xEF\xBB\xBF':
f.seek(0)
config.readfp(f)
return config
@ -28,7 +57,7 @@ def set_config(config, section, option, value):
assert config.get(section, option) == value
def write_config(tahoe_cfg, config):
with open(tahoe_cfg, "wb") as f:
with open(tahoe_cfg, "w") as f:
config.write(f)
def validate_config(fname, cfg, valid_config):

View File

@ -1,3 +1,18 @@
"""
Parse connection status from Foolscap.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
from zope.interface import implementer
from ..interfaces import IConnectionStatus
@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
def from_foolscap_reconnector(rc, last_received):
ri = rc.getReconnectionInfo()
# See foolscap/reconnector.py, ReconnectionInfo, for details about
# possible states.
# See foolscap/reconnector.py, ReconnectionInfo, for details about possible
# states. The returned result is a native string, it seems, so convert to
# unicode.
state = ri.state
if isinstance(state, bytes): # Python 2
state = str(state, "ascii")
if state == "unstarted":
return ConnectionStatus.unstarted()

68
tox.ini
View File

@ -44,36 +44,42 @@ usedevelop = False
# We use extras=test to get things like "mock" that are required for our unit
# tests.
extras = test
commands =
trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
tahoe --version
[testenv:py36]
setenv =
# Define TEST_SUITE in the environment as an aid to constructing the
# correct test command below.
!py36: TEST_SUITE = allmydata
py36: TEST_SUITE = allmydata.test.python3_tests
commands =
trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
# As an aid to debugging, dump all of the Python packages and their
# versions that are installed in the test environment. This is
# particularly useful to get from CI runs - though hopefully the
# version pinning we do limits the variability of this output
pip freeze
# The tahoe script isn't sufficiently ported for this to succeed on
# Python 3.x yet.
!py36: tahoe --version
!coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
# measuring coverage is somewhat slower than not measuring coverage
# so only do it on request.
coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
coverage: coverage combine
coverage: coverage xml
[testenv:integration]
setenv =
COVERAGE_PROCESS_START=.coveragerc
commands =
# NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
py.test --coverage -v {posargs:integration}
# NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
py.test --coverage -v {posargs:integration}
coverage combine
coverage report
[testenv:coverage]
# coverage (with --branch) takes about 65% longer to run
commands =
# As an aid to debugging, dump all of the Python packages and their
# versions that are installed in the test environment. This is
# particularly useful to get from CI runs - though hopefully the
# version pinning we do limits the variability of this output
# somewhat.
pip freeze
tahoe --version
coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
coverage combine
coverage xml
[testenv:codechecks]
# On macOS, git inside of towncrier needs $HOME.
@ -87,11 +93,11 @@ commands =
python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
python misc/coding_tools/check-miscaptures.py
# If towncrier.check fails, you forgot to add a towncrier news
# fragment explaining the change in this branch. Create one at
# `newsfragments/<ticket>.<change type>` with some text for the news
# file. See pyproject.toml for legal <change type> values.
python -m towncrier.check --pyproject towncrier.pyproject.toml
# If towncrier.check fails, you forgot to add a towncrier news
# fragment explaining the change in this branch. Create one at
# `newsfragments/<ticket>.<change type>` with some text for the news
# file. See pyproject.toml for legal <change type> values.
python -m towncrier.check --pyproject towncrier.pyproject.toml
[testenv:draftnews]
passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
@ -110,9 +116,9 @@ commands =
#
# Some discussion is available at
# https://github.com/pypa/pip/issues/5696
#
# towncrier post 19.2 (unreleased as of this writing) adds a --config
# option that can be used instead of this file shuffling.
#
# towncrier post 19.2 (unreleased as of this writing) adds a --config
# option that can be used instead of this file shuffling.
mv towncrier.pyproject.toml pyproject.toml
# towncrier 19.2 + works with python2.7
@ -138,9 +144,9 @@ commands =
#
# Some discussion is available at
# https://github.com/pypa/pip/issues/5696
#
# towncrier post 19.2 (unreleased as of this writing) adds a --config
# option that can be used instead of this file shuffling.
#
# towncrier post 19.2 (unreleased as of this writing) adds a --config
# option that can be used instead of this file shuffling.
mv towncrier.pyproject.toml pyproject.toml
# towncrier 19.2 + works with python2.7