mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-04-08 19:34:18 +00:00
Merge 'origin/master' into 3382.remove-multi-format-page
This commit is contained in:
Sajith Sasidharan
commit
ebd20b7f22
@ -1,95 +0,0 @@
|
||||
# adapted from https://packaging.python.org/en/latest/appveyor/
|
||||
|
||||
environment:
|
||||
|
||||
matrix:
|
||||
|
||||
# For Python versions available on Appveyor, see
|
||||
# http://www.appveyor.com/docs/installed-software#python
|
||||
- PYTHON: "C:\\Python27"
|
||||
- PYTHON: "C:\\Python27-x64"
|
||||
# DISTUTILS_USE_SDK: "1"
|
||||
# TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB"
|
||||
|
||||
install:
|
||||
- |
|
||||
%PYTHON%\python.exe -m pip install -U pip
|
||||
%PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv
|
||||
|
||||
# note:
|
||||
# %PYTHON% has: python.exe
|
||||
# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip)
|
||||
|
||||
# We have a custom "build" system. We don't need MSBuild or whatever.
|
||||
build: off
|
||||
|
||||
# Do not build feature branch with open pull requests. This is documented but
|
||||
# it's not clear it does anything.
|
||||
skip_branch_with_pr: true
|
||||
|
||||
# This, perhaps, is effective.
|
||||
branches:
|
||||
# whitelist
|
||||
only:
|
||||
- 'master'
|
||||
|
||||
skip_commits:
|
||||
files:
|
||||
# The Windows builds are unaffected by news fragments.
|
||||
- 'newsfragments/*'
|
||||
# Also, all this build junk.
|
||||
- '.circleci/*'
|
||||
- '.lgtm.yml'
|
||||
- '.travis.yml'
|
||||
|
||||
# we run from C:\projects\tahoe-lafs
|
||||
|
||||
test_script:
|
||||
# Put your test command here.
|
||||
# Note that you must use the environment variable %PYTHON% to refer to
|
||||
# the interpreter you're using - Appveyor does not do anything special
|
||||
# to put the Python version you want to use on PATH.
|
||||
- |
|
||||
%PYTHON%\Scripts\tox.exe -e coverage
|
||||
%PYTHON%\Scripts\tox.exe -e pyinstaller
|
||||
# To verify that the resultant PyInstaller-generated binary executes
|
||||
# cleanly (i.e., that it terminates with an exit code of 0 and isn't
|
||||
# failing due to import/packaging-related errors, etc.).
|
||||
- dist\Tahoe-LAFS\tahoe.exe --version
|
||||
|
||||
after_test:
|
||||
# This builds the main tahoe wheel, and wheels for all dependencies.
|
||||
# Again, you only need build.cmd if you're building C extensions for
|
||||
# 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
|
||||
# interpreter. If _trial_temp still exists, the "pip wheel" fails on
|
||||
# _trial_temp\local_dir (not sure why).
|
||||
- |
|
||||
copy _trial_temp\test.log trial_test_log.txt
|
||||
rd /s /q _trial_temp
|
||||
%PYTHON%\python.exe setup.py bdist_wheel
|
||||
%PYTHON%\python.exe -m pip wheel -w dist .
|
||||
- |
|
||||
%PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5"
|
||||
%PYTHON%\python.exe -m coverage xml -o coverage.xml -i
|
||||
%PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
|
||||
|
||||
artifacts:
|
||||
# bdist_wheel puts your built wheel in the dist directory
|
||||
# "pip wheel -w dist ." puts all the dependency wheels there too
|
||||
# this gives us a zipfile with everything
|
||||
- path: 'dist\*'
|
||||
- path: trial_test_log.txt
|
||||
name: Trial test.log
|
||||
- path: eliot.log
|
||||
name: Eliot test log
|
||||
|
||||
on_failure:
|
||||
# Artifacts are not normally uploaded when the job fails. To get the test
|
||||
# logs, we have to push them ourselves.
|
||||
- ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log
|
||||
- ps: Push-AppveyorArtifact eliot.log -Filename eliot.log
|
||||
|
||||
#on_success:
|
||||
# You can use this step to upload your artifacts to a public website.
|
||||
# See Appveyor's documentation for more details. Or you can simply
|
||||
# access your wheels from the Appveyor "artifacts" tab for your build.
|
||||
@ -211,7 +211,8 @@ jobs:
|
||||
|
||||
environment:
|
||||
<<: *UTF_8_ENVIRONMENT
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27-coverage"
|
||||
# We don't do coverage since it makes PyPy far too slow:
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27"
|
||||
|
||||
|
||||
c-locale:
|
||||
@ -285,7 +286,7 @@ jobs:
|
||||
# this reporter on Python 3. So drop that and just specify the
|
||||
# reporter.
|
||||
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
|
||||
|
||||
|
||||
ubuntu-20.04:
|
||||
|
||||
@ -68,6 +68,10 @@ export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}"
|
||||
export TAHOE_LAFS_TRIAL_ARGS="${TAHOE_LAFS_TRIAL_ARGS:---reporter=subunitv2-file --rterrors}"
|
||||
export PIP_NO_INDEX="1"
|
||||
|
||||
# Make output unbuffered, so progress reports from subunitv2-file get streamed
|
||||
# and notify CircleCI we're still alive.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
if [ "${ALLOWED_FAILURE}" = "yes" ]; then
|
||||
alternative="true"
|
||||
else
|
||||
|
||||
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
@ -49,8 +49,8 @@ jobs:
|
||||
- name: Display tool versions
|
||||
run: python misc/build_helpers/show-tool-versions.py
|
||||
|
||||
- name: Run "tox -e coverage"
|
||||
run: tox -e coverage
|
||||
- name: Run "tox -e py27-coverage"
|
||||
run: tox -e py27-coverage
|
||||
|
||||
- name: Upload eliot.log in case of failure
|
||||
uses: actions/upload-artifact@v1
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -9,6 +9,7 @@ venv*
|
||||
*~
|
||||
*.DS_Store
|
||||
.*.kate-swp
|
||||
*.bak
|
||||
|
||||
/build/
|
||||
/support/
|
||||
|
||||
@ -36,7 +36,7 @@ people are Release Maintainers:
|
||||
- [ ] documentation is ready (see above)
|
||||
- [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
|
||||
- [ ] build code locally:
|
||||
tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations
|
||||
tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
|
||||
- [ ] created tarballs (they'll be in dist/ for later comparison)
|
||||
tox -e tarballs
|
||||
- [ ] release version is reporting itself as intended version
|
||||
|
||||
1
newsfragments/3355.other
Normal file
1
newsfragments/3355.other
Normal file
@ -0,0 +1 @@
|
||||
The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.
|
||||
0
newsfragments/3374.minor
Normal file
0
newsfragments/3374.minor
Normal file
0
newsfragments/3377.minor
Normal file
0
newsfragments/3377.minor
Normal file
0
newsfragments/3392.minor
Normal file
0
newsfragments/3392.minor
Normal file
0
newsfragments/3393.minor
Normal file
0
newsfragments/3393.minor
Normal file
0
newsfragments/3395.minor
Normal file
0
newsfragments/3395.minor
Normal file
0
newsfragments/3396.minor
Normal file
0
newsfragments/3396.minor
Normal file
0
newsfragments/3397.minor
Normal file
0
newsfragments/3397.minor
Normal file
0
newsfragments/3401.minor
Normal file
0
newsfragments/3401.minor
Normal file
0
newsfragments/3403.minor
Normal file
0
newsfragments/3403.minor
Normal file
5
setup.py
5
setup.py
@ -116,6 +116,11 @@ install_requires = [
|
||||
# know works on Python 2.7.
|
||||
"eliot ~= 1.7",
|
||||
|
||||
# Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped
|
||||
# Python 2 entirely; stick to the version known to work for us.
|
||||
# XXX: drop this bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3404
|
||||
"pyrsistent < 0.17.0",
|
||||
|
||||
# A great way to define types of values.
|
||||
# XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390
|
||||
"attrs >= 18.2.0, < 20",
|
||||
|
||||
@ -2,7 +2,10 @@ import os, stat, time, weakref
|
||||
from base64 import urlsafe_b64encode
|
||||
from functools import partial
|
||||
from errno import ENOENT, EPERM
|
||||
from ConfigParser import NoSectionError
|
||||
try:
|
||||
from ConfigParser import NoSectionError
|
||||
except ImportError:
|
||||
from configparser import NoSectionError
|
||||
|
||||
from foolscap.furl import (
|
||||
decode_furl,
|
||||
|
||||
@ -1,4 +1,16 @@
|
||||
# -*- test-case-name: allmydata.test.test_encode_share -*-
|
||||
"""
|
||||
CRS encoding and decoding.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
from zope.interface import implementer
|
||||
from twisted.internet import defer
|
||||
@ -9,7 +21,7 @@ import zfec
|
||||
|
||||
@implementer(ICodecEncoder)
|
||||
class CRSEncoder(object):
|
||||
ENCODER_TYPE = "crs"
|
||||
ENCODER_TYPE = b"crs"
|
||||
|
||||
def set_params(self, data_size, required_shares, max_shares):
|
||||
assert required_shares <= max_shares
|
||||
@ -27,8 +39,8 @@ class CRSEncoder(object):
|
||||
return (self.data_size, self.required_shares, self.max_shares)
|
||||
|
||||
def get_serialized_params(self):
|
||||
return "%d-%d-%d" % (self.data_size, self.required_shares,
|
||||
self.max_shares)
|
||||
return b"%d-%d-%d" % (self.data_size, self.required_shares,
|
||||
self.max_shares)
|
||||
|
||||
def get_block_size(self):
|
||||
return self.share_size
|
||||
@ -37,7 +49,7 @@ class CRSEncoder(object):
|
||||
precondition(desired_share_ids is None or len(desired_share_ids) <= self.max_shares, desired_share_ids, self.max_shares)
|
||||
|
||||
if desired_share_ids is None:
|
||||
desired_share_ids = range(self.max_shares)
|
||||
desired_share_ids = list(range(self.max_shares))
|
||||
|
||||
for inshare in inshares:
|
||||
assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares)
|
||||
@ -71,5 +83,5 @@ class CRSDecoder(object):
|
||||
return defer.succeed(data)
|
||||
|
||||
def parse_params(serializedparams):
|
||||
pieces = serializedparams.split("-")
|
||||
pieces = serializedparams.split(b"-")
|
||||
return int(pieces[0]), int(pieces[1]), int(pieces[2])
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
"""Directory Node implementation."""
|
||||
from past.builtins import unicode
|
||||
|
||||
import time
|
||||
|
||||
from zope.interface import implementer
|
||||
@ -227,7 +229,7 @@ def pack_children(childrenx, writekey, deep_immutable=False):
|
||||
return _pack_normalized_children(children, writekey=writekey, deep_immutable=deep_immutable)
|
||||
|
||||
|
||||
ZERO_LEN_NETSTR=netstring('')
|
||||
ZERO_LEN_NETSTR=netstring(b'')
|
||||
def _pack_normalized_children(children, writekey, deep_immutable=False):
|
||||
"""Take a dict that maps:
|
||||
children[unicode_nfc_name] = (IFileSystemNode, metadata_dict)
|
||||
|
||||
@ -4,7 +4,7 @@ from foolscap.api import eventually
|
||||
from allmydata.interfaces import NotEnoughSharesError, NoSharesError
|
||||
from allmydata.util import log
|
||||
from allmydata.util.dictutil import DictOfSets
|
||||
from common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \
|
||||
from .common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \
|
||||
BadSegmentNumberError
|
||||
|
||||
class SegmentFetcher(object):
|
||||
|
||||
@ -5,7 +5,7 @@ from foolscap.api import eventually
|
||||
from allmydata.util import base32, log
|
||||
from twisted.internet import reactor
|
||||
|
||||
from share import Share, CommonShare
|
||||
from .share import Share, CommonShare
|
||||
|
||||
def incidentally(res, f, *args, **kwargs):
|
||||
"""Add me to a Deferred chain like this:
|
||||
|
||||
@ -13,10 +13,10 @@ from allmydata.hashtree import IncompleteHashTree, BadHashError, \
|
||||
NotEnoughHashesError
|
||||
|
||||
# local imports
|
||||
from finder import ShareFinder
|
||||
from fetcher import SegmentFetcher
|
||||
from segmentation import Segmentation
|
||||
from common import BadCiphertextHashError
|
||||
from .finder import ShareFinder
|
||||
from .fetcher import SegmentFetcher
|
||||
from .segmentation import Segmentation
|
||||
from .common import BadCiphertextHashError
|
||||
|
||||
class IDownloadStatusHandlingConsumer(Interface):
|
||||
def set_download_status_read_event(read_ev):
|
||||
|
||||
@ -9,7 +9,7 @@ from allmydata.util import log
|
||||
from allmydata.util.spans import overlap
|
||||
from allmydata.interfaces import DownloadStopped
|
||||
|
||||
from common import BadSegmentNumberError, WrongSegmentError
|
||||
from .common import BadSegmentNumberError, WrongSegmentError
|
||||
|
||||
@implementer(IPushProducer)
|
||||
class Segmentation(object):
|
||||
|
||||
@ -13,7 +13,7 @@ from allmydata.hashtree import IncompleteHashTree, BadHashError, \
|
||||
|
||||
from allmydata.immutable.layout import make_write_bucket_proxy
|
||||
from allmydata.util.observer import EventStreamObserver
|
||||
from common import COMPLETE, CORRUPT, DEAD, BADSEGNUM
|
||||
from .common import COMPLETE, CORRUPT, DEAD, BADSEGNUM
|
||||
|
||||
|
||||
class LayoutInvalid(Exception):
|
||||
|
||||
@ -171,7 +171,7 @@ class WriteBucketProxy(object):
|
||||
def put_block(self, segmentnum, data):
|
||||
offset = self._offsets['data'] + segmentnum * self._block_size
|
||||
assert offset + len(data) <= self._offsets['uri_extension']
|
||||
assert isinstance(data, str)
|
||||
assert isinstance(data, bytes)
|
||||
if segmentnum < self._num_segments-1:
|
||||
precondition(len(data) == self._block_size,
|
||||
len(data), self._block_size)
|
||||
@ -185,7 +185,7 @@ class WriteBucketProxy(object):
|
||||
def put_crypttext_hashes(self, hashes):
|
||||
offset = self._offsets['crypttext_hash_tree']
|
||||
assert isinstance(hashes, list)
|
||||
data = "".join(hashes)
|
||||
data = b"".join(hashes)
|
||||
precondition(len(data) == self._segment_hash_size,
|
||||
len(data), self._segment_hash_size)
|
||||
precondition(offset + len(data) <= self._offsets['block_hashes'],
|
||||
@ -196,7 +196,7 @@ class WriteBucketProxy(object):
|
||||
def put_block_hashes(self, blockhashes):
|
||||
offset = self._offsets['block_hashes']
|
||||
assert isinstance(blockhashes, list)
|
||||
data = "".join(blockhashes)
|
||||
data = b"".join(blockhashes)
|
||||
precondition(len(data) == self._segment_hash_size,
|
||||
len(data), self._segment_hash_size)
|
||||
precondition(offset + len(data) <= self._offsets['share_hashes'],
|
||||
@ -209,7 +209,7 @@ class WriteBucketProxy(object):
|
||||
# as 2+32=34 bytes each
|
||||
offset = self._offsets['share_hashes']
|
||||
assert isinstance(sharehashes, list)
|
||||
data = "".join([struct.pack(">H", hashnum) + hashvalue
|
||||
data = b"".join([struct.pack(">H", hashnum) + hashvalue
|
||||
for hashnum,hashvalue in sharehashes])
|
||||
precondition(len(data) == self._share_hashtree_size,
|
||||
len(data), self._share_hashtree_size)
|
||||
@ -220,7 +220,7 @@ class WriteBucketProxy(object):
|
||||
|
||||
def put_uri_extension(self, data):
|
||||
offset = self._offsets['uri_extension']
|
||||
assert isinstance(data, str)
|
||||
assert isinstance(data, bytes)
|
||||
precondition(len(data) <= self._uri_extension_size_max,
|
||||
len(data), self._uri_extension_size_max)
|
||||
length = struct.pack(self.fieldstruct, len(data))
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
from past.builtins import long
|
||||
|
||||
import os, time, weakref, itertools
|
||||
from zope.interface import implementer
|
||||
from twisted.python import failure
|
||||
@ -26,7 +28,7 @@ from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \
|
||||
from allmydata.immutable import layout
|
||||
|
||||
from six.moves import cStringIO as StringIO
|
||||
from happiness_upload import share_placement, calculate_happiness
|
||||
from .happiness_upload import share_placement, calculate_happiness
|
||||
|
||||
from ..util.eliotutil import (
|
||||
log_call_deferred,
|
||||
|
||||
@ -1,3 +1,18 @@
|
||||
"""
|
||||
Interfaces for Tahoe-LAFS.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
# Don't import object/str/dict/etc. types, so we don't break any
|
||||
# interfaces. Not importing open() because it triggers bogus flake8 error.
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min # noqa: F401
|
||||
|
||||
from past.builtins import long
|
||||
|
||||
@ -59,7 +74,7 @@ class RIBucketReader(RemoteInterface):
|
||||
def read(offset=Offset, length=ReadSize):
|
||||
return ShareData
|
||||
|
||||
def advise_corrupt_share(reason=str):
|
||||
def advise_corrupt_share(reason=bytes):
|
||||
"""Clients who discover hash failures in shares that they have
|
||||
downloaded from me will use this method to inform me about the
|
||||
failures. I will record their concern so that my operator can
|
||||
@ -72,7 +87,7 @@ class RIBucketReader(RemoteInterface):
|
||||
"""
|
||||
|
||||
|
||||
TestVector = ListOf(TupleOf(Offset, ReadSize, str, str))
|
||||
TestVector = ListOf(TupleOf(Offset, ReadSize, bytes, bytes))
|
||||
# elements are (offset, length, operator, specimen)
|
||||
# operator is one of "lt, le, eq, ne, ge, gt"
|
||||
# nop always passes and is used to fetch data while writing.
|
||||
@ -90,13 +105,13 @@ ReadData = ListOf(ShareData)
|
||||
|
||||
|
||||
class RIStorageServer(RemoteInterface):
|
||||
__remote_name__ = "RIStorageServer.tahoe.allmydata.com"
|
||||
__remote_name__ = b"RIStorageServer.tahoe.allmydata.com"
|
||||
|
||||
def get_version():
|
||||
"""
|
||||
Return a dictionary of version information.
|
||||
"""
|
||||
return DictOf(str, Any())
|
||||
return DictOf(bytes, Any())
|
||||
|
||||
def allocate_buckets(storage_index=StorageIndex,
|
||||
renew_secret=LeaseRenewSecret,
|
||||
@ -278,8 +293,8 @@ class RIStorageServer(RemoteInterface):
|
||||
"""
|
||||
return TupleOf(bool, DictOf(int, ReadData))
|
||||
|
||||
def advise_corrupt_share(share_type=str, storage_index=StorageIndex,
|
||||
shnum=int, reason=str):
|
||||
def advise_corrupt_share(share_type=bytes, storage_index=StorageIndex,
|
||||
shnum=int, reason=bytes):
|
||||
"""Clients who discover hash failures in shares that they have
|
||||
downloaded from me will use this method to inform me about the
|
||||
failures. I will record their concern so that my operator can
|
||||
@ -2860,7 +2875,7 @@ UploadResults = Any() #DictOf(str, str)
|
||||
|
||||
|
||||
class RIEncryptedUploadable(RemoteInterface):
|
||||
__remote_name__ = "RIEncryptedUploadable.tahoe.allmydata.com"
|
||||
__remote_name__ = b"RIEncryptedUploadable.tahoe.allmydata.com"
|
||||
|
||||
def get_size():
|
||||
return Offset
|
||||
@ -2876,7 +2891,7 @@ class RIEncryptedUploadable(RemoteInterface):
|
||||
|
||||
|
||||
class RICHKUploadHelper(RemoteInterface):
|
||||
__remote_name__ = "RIUploadHelper.tahoe.allmydata.com"
|
||||
__remote_name__ = b"RIUploadHelper.tahoe.allmydata.com"
|
||||
|
||||
def get_version():
|
||||
"""
|
||||
@ -2889,7 +2904,7 @@ class RICHKUploadHelper(RemoteInterface):
|
||||
|
||||
|
||||
class RIHelper(RemoteInterface):
|
||||
__remote_name__ = "RIHelper.tahoe.allmydata.com"
|
||||
__remote_name__ = b"RIHelper.tahoe.allmydata.com"
|
||||
|
||||
def get_version():
|
||||
"""
|
||||
@ -2916,7 +2931,7 @@ class RIHelper(RemoteInterface):
|
||||
|
||||
|
||||
class RIStatsProvider(RemoteInterface):
|
||||
__remote_name__ = "RIStatsProvider.tahoe.allmydata.com"
|
||||
__remote_name__ = b"RIStatsProvider.tahoe.allmydata.com"
|
||||
"""
|
||||
Provides access to statistics and monitoring information.
|
||||
"""
|
||||
@ -2933,7 +2948,7 @@ class RIStatsProvider(RemoteInterface):
|
||||
|
||||
|
||||
class RIStatsGatherer(RemoteInterface):
|
||||
__remote_name__ = "RIStatsGatherer.tahoe.allmydata.com"
|
||||
__remote_name__ = b"RIStatsGatherer.tahoe.allmydata.com"
|
||||
"""
|
||||
Provides a monitoring service for centralised collection of stats
|
||||
"""
|
||||
|
||||
@ -1,7 +1,21 @@
|
||||
"""
|
||||
Manage status of long-running operations.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
from zope.interface import Interface, implementer
|
||||
from allmydata.util import observer
|
||||
|
||||
|
||||
class IMonitor(Interface):
|
||||
"""I manage status, progress, and cancellation for long-running operations.
|
||||
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
from past.utils import old_div
|
||||
|
||||
import struct
|
||||
from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError, \
|
||||
@ -180,11 +181,11 @@ def pack_offsets(verification_key_length, signature_length,
|
||||
def pack_share(prefix, verification_key, signature,
|
||||
share_hash_chain, block_hash_tree,
|
||||
share_data, encprivkey):
|
||||
share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
|
||||
for i in sorted(share_hash_chain.keys())])
|
||||
share_hash_chain_s = b"".join([struct.pack(">H32s", i, share_hash_chain[i])
|
||||
for i in sorted(share_hash_chain.keys())])
|
||||
for h in block_hash_tree:
|
||||
assert len(h) == 32
|
||||
block_hash_tree_s = "".join(block_hash_tree)
|
||||
block_hash_tree_s = b"".join(block_hash_tree)
|
||||
|
||||
offsets = pack_offsets(len(verification_key),
|
||||
len(signature),
|
||||
@ -192,14 +193,14 @@ def pack_share(prefix, verification_key, signature,
|
||||
len(block_hash_tree_s),
|
||||
len(share_data),
|
||||
len(encprivkey))
|
||||
final_share = "".join([prefix,
|
||||
offsets,
|
||||
verification_key,
|
||||
signature,
|
||||
share_hash_chain_s,
|
||||
block_hash_tree_s,
|
||||
share_data,
|
||||
encprivkey])
|
||||
final_share = b"".join([prefix,
|
||||
offsets,
|
||||
verification_key,
|
||||
signature,
|
||||
share_hash_chain_s,
|
||||
block_hash_tree_s,
|
||||
share_data,
|
||||
encprivkey])
|
||||
return final_share
|
||||
|
||||
def pack_prefix(seqnum, root_hash, IV,
|
||||
@ -255,7 +256,7 @@ class SDMFSlotWriteProxy(object):
|
||||
self._required_shares)
|
||||
assert expected_segment_size == segment_size
|
||||
|
||||
self._block_size = self._segment_size / self._required_shares
|
||||
self._block_size = old_div(self._segment_size, self._required_shares)
|
||||
|
||||
# This is meant to mimic how SDMF files were built before MDMF
|
||||
# entered the picture: we generate each share in its entirety,
|
||||
@ -296,7 +297,7 @@ class SDMFSlotWriteProxy(object):
|
||||
salt)
|
||||
else:
|
||||
checkstring = checkstring_or_seqnum
|
||||
self._testvs = [(0, len(checkstring), "eq", checkstring)]
|
||||
self._testvs = [(0, len(checkstring), b"eq", checkstring)]
|
||||
|
||||
|
||||
def get_checkstring(self):
|
||||
@ -306,7 +307,7 @@ class SDMFSlotWriteProxy(object):
|
||||
"""
|
||||
if self._testvs:
|
||||
return self._testvs[0][3]
|
||||
return ""
|
||||
return b""
|
||||
|
||||
|
||||
def put_block(self, data, segnum, salt):
|
||||
@ -343,7 +344,7 @@ class SDMFSlotWriteProxy(object):
|
||||
assert len(h) == HASH_SIZE
|
||||
|
||||
# serialize the blockhashes, then set them.
|
||||
blockhashes_s = "".join(blockhashes)
|
||||
blockhashes_s = b"".join(blockhashes)
|
||||
self._share_pieces['block_hash_tree'] = blockhashes_s
|
||||
|
||||
return defer.succeed(None)
|
||||
@ -354,12 +355,12 @@ class SDMFSlotWriteProxy(object):
|
||||
Add the share hash chain to the share.
|
||||
"""
|
||||
assert isinstance(sharehashes, dict)
|
||||
for h in sharehashes.itervalues():
|
||||
for h in sharehashes.values():
|
||||
assert len(h) == HASH_SIZE
|
||||
|
||||
# serialize the sharehashes, then set them.
|
||||
sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
|
||||
for i in sorted(sharehashes.keys())])
|
||||
sharehashes_s = b"".join([struct.pack(">H32s", i, sharehashes[i])
|
||||
for i in sorted(sharehashes.keys())])
|
||||
self._share_pieces['share_hash_chain'] = sharehashes_s
|
||||
|
||||
return defer.succeed(None)
|
||||
@ -383,7 +384,7 @@ class SDMFSlotWriteProxy(object):
|
||||
assert len(salt) == SALT_SIZE
|
||||
|
||||
self._share_pieces['salt'] = salt
|
||||
self._share_pieces['sharedata'] = ""
|
||||
self._share_pieces['sharedata'] = b""
|
||||
|
||||
|
||||
def get_signable(self):
|
||||
@ -519,14 +520,14 @@ class SDMFSlotWriteProxy(object):
|
||||
# to the remote server in one write.
|
||||
offsets = self._pack_offsets()
|
||||
prefix = self.get_signable()
|
||||
final_share = "".join([prefix,
|
||||
offsets,
|
||||
self._share_pieces['verification_key'],
|
||||
self._share_pieces['signature'],
|
||||
self._share_pieces['share_hash_chain'],
|
||||
self._share_pieces['block_hash_tree'],
|
||||
self._share_pieces['sharedata'],
|
||||
self._share_pieces['encprivkey']])
|
||||
final_share = b"".join([prefix,
|
||||
offsets,
|
||||
self._share_pieces['verification_key'],
|
||||
self._share_pieces['signature'],
|
||||
self._share_pieces['share_hash_chain'],
|
||||
self._share_pieces['block_hash_tree'],
|
||||
self._share_pieces['sharedata'],
|
||||
self._share_pieces['encprivkey']])
|
||||
|
||||
# Our only data vector is going to be writing the final share,
|
||||
# in its entirely.
|
||||
@ -537,7 +538,7 @@ class SDMFSlotWriteProxy(object):
|
||||
# yet, so we assume that we are writing a new share, and set
|
||||
# a test vector that will allow a new share to be written.
|
||||
self._testvs = []
|
||||
self._testvs.append(tuple([0, 1, "eq", ""]))
|
||||
self._testvs.append(tuple([0, 1, b"eq", b""]))
|
||||
|
||||
tw_vectors = {}
|
||||
tw_vectors[self.shnum] = (self._testvs, datavs, None)
|
||||
@ -788,7 +789,7 @@ class MDMFSlotWriteProxy(object):
|
||||
# and also because it provides a useful amount of bounds checking.
|
||||
self._num_segments = mathutil.div_ceil(self._data_length,
|
||||
self._segment_size)
|
||||
self._block_size = self._segment_size / self._required_shares
|
||||
self._block_size = old_div(self._segment_size, self._required_shares)
|
||||
# We also calculate the share size, to help us with block
|
||||
# constraints later.
|
||||
tail_size = self._data_length % self._segment_size
|
||||
@ -797,7 +798,7 @@ class MDMFSlotWriteProxy(object):
|
||||
else:
|
||||
self._tail_block_size = mathutil.next_multiple(tail_size,
|
||||
self._required_shares)
|
||||
self._tail_block_size /= self._required_shares
|
||||
self._tail_block_size = old_div(self._tail_block_size, self._required_shares)
|
||||
|
||||
# We already know where the sharedata starts; right after the end
|
||||
# of the header (which is defined as the signable part + the offsets)
|
||||
@ -868,7 +869,7 @@ class MDMFSlotWriteProxy(object):
|
||||
else:
|
||||
checkstring = seqnum_or_checkstring
|
||||
|
||||
if checkstring == "":
|
||||
if checkstring == b"":
|
||||
# We special-case this, since len("") = 0, but we need
|
||||
# length of 1 for the case of an empty share to work on the
|
||||
# storage server, which is what a checkstring that is the
|
||||
@ -876,7 +877,7 @@ class MDMFSlotWriteProxy(object):
|
||||
self._testvs = []
|
||||
else:
|
||||
self._testvs = []
|
||||
self._testvs.append((0, len(checkstring), "eq", checkstring))
|
||||
self._testvs.append((0, len(checkstring), b"eq", checkstring))
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
@ -893,7 +894,7 @@ class MDMFSlotWriteProxy(object):
|
||||
if self._root_hash:
|
||||
roothash = self._root_hash
|
||||
else:
|
||||
roothash = "\x00" * 32
|
||||
roothash = b"\x00" * 32
|
||||
return struct.pack(MDMFCHECKSTRING,
|
||||
1,
|
||||
self._seqnum,
|
||||
@ -964,7 +965,7 @@ class MDMFSlotWriteProxy(object):
|
||||
|
||||
assert isinstance(blockhashes, list)
|
||||
|
||||
blockhashes_s = "".join(blockhashes)
|
||||
blockhashes_s = b"".join(blockhashes)
|
||||
self._offsets['EOF'] = self._offsets['block_hash_tree'] + len(blockhashes_s)
|
||||
|
||||
self._writevs.append(tuple([self._offsets['block_hash_tree'],
|
||||
@ -998,7 +999,7 @@ class MDMFSlotWriteProxy(object):
|
||||
if "verification_key" in self._offsets:
|
||||
raise LayoutInvalid("You must write the share hash chain "
|
||||
"before you write the signature")
|
||||
sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
|
||||
sharehashes_s = b"".join([struct.pack(">H32s", i, sharehashes[i])
|
||||
for i in sorted(sharehashes.keys())])
|
||||
self._offsets['signature'] = self._offsets['share_hash_chain'] + \
|
||||
len(sharehashes_s)
|
||||
@ -1149,7 +1150,7 @@ class MDMFSlotWriteProxy(object):
|
||||
tw_vectors = {}
|
||||
if not self._testvs:
|
||||
self._testvs = []
|
||||
self._testvs.append(tuple([0, 1, "eq", ""]))
|
||||
self._testvs.append(tuple([0, 1, b"eq", b""]))
|
||||
if not self._written:
|
||||
# Write a new checkstring to the share when we write it, so
|
||||
# that we have something to check later.
|
||||
@ -1157,7 +1158,7 @@ class MDMFSlotWriteProxy(object):
|
||||
datavs.append((0, new_checkstring))
|
||||
def _first_write():
|
||||
self._written = True
|
||||
self._testvs = [(0, len(new_checkstring), "eq", new_checkstring)]
|
||||
self._testvs = [(0, len(new_checkstring), b"eq", new_checkstring)]
|
||||
on_success = _first_write
|
||||
tw_vectors[self.shnum] = (self._testvs, datavs, None)
|
||||
d = self._storage_server.slot_testv_and_readv_and_writev(
|
||||
@ -1194,7 +1195,7 @@ class MDMFSlotReadProxy(object):
|
||||
storage_server,
|
||||
storage_index,
|
||||
shnum,
|
||||
data="",
|
||||
data=b"",
|
||||
data_is_everything=False):
|
||||
# Start the initialization process.
|
||||
self._storage_server = storage_server
|
||||
@ -1238,7 +1239,7 @@ class MDMFSlotReadProxy(object):
|
||||
# None if there isn't any cached data, but the way we index the
|
||||
# cached data requires a string, so convert None to "".
|
||||
if self._data == None:
|
||||
self._data = ""
|
||||
self._data = b""
|
||||
|
||||
|
||||
def _maybe_fetch_offsets_and_header(self, force_remote=False):
|
||||
@ -1317,7 +1318,7 @@ class MDMFSlotReadProxy(object):
|
||||
self._segment_size = segsize
|
||||
self._data_length = datalen
|
||||
|
||||
self._block_size = self._segment_size / self._required_shares
|
||||
self._block_size = old_div(self._segment_size, self._required_shares)
|
||||
# We can upload empty files, and need to account for this fact
|
||||
# so as to avoid zero-division and zero-modulo errors.
|
||||
if datalen > 0:
|
||||
@ -1329,7 +1330,7 @@ class MDMFSlotReadProxy(object):
|
||||
else:
|
||||
self._tail_block_size = mathutil.next_multiple(tail_size,
|
||||
self._required_shares)
|
||||
self._tail_block_size /= self._required_shares
|
||||
self._tail_block_size = old_div(self._tail_block_size, self._required_shares)
|
||||
|
||||
return encoding_parameters
|
||||
|
||||
@ -1416,7 +1417,7 @@ class MDMFSlotReadProxy(object):
|
||||
# when we fetched the header
|
||||
data = results[self.shnum]
|
||||
if not data:
|
||||
data = ""
|
||||
data = b""
|
||||
else:
|
||||
if len(data) != 1:
|
||||
raise BadShareError("got %d vectors, not 1" % len(data))
|
||||
@ -1425,7 +1426,7 @@ class MDMFSlotReadProxy(object):
|
||||
else:
|
||||
data = results[self.shnum]
|
||||
if not data:
|
||||
salt = data = ""
|
||||
salt = data = b""
|
||||
else:
|
||||
salt_and_data = results[self.shnum][0]
|
||||
salt = salt_and_data[:SALT_SIZE]
|
||||
@ -1743,7 +1744,7 @@ class MDMFSlotReadProxy(object):
|
||||
|
||||
|
||||
def _read(self, readvs, force_remote=False):
|
||||
unsatisfiable = filter(lambda x: x[0] + x[1] > len(self._data), readvs)
|
||||
unsatisfiable = list(filter(lambda x: x[0] + x[1] > len(self._data), readvs))
|
||||
# TODO: It's entirely possible to tweak this so that it just
|
||||
# fulfills the requests that it can, and not demand that all
|
||||
# requests are satisfiable before running it.
|
||||
|
||||
@ -2,12 +2,17 @@
|
||||
This module contains classes and functions to implement and manage
|
||||
a node for Tahoe-LAFS.
|
||||
"""
|
||||
from past.builtins import unicode
|
||||
|
||||
import datetime
|
||||
import os.path
|
||||
import re
|
||||
import types
|
||||
import errno
|
||||
import ConfigParser
|
||||
try:
|
||||
import ConfigParser
|
||||
except ImportError:
|
||||
import configparser as ConfigParser
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
from base64 import b32decode, b32encode
|
||||
@ -67,7 +72,7 @@ def _common_valid_config():
|
||||
|
||||
# Add our application versions to the data that Foolscap's LogPublisher
|
||||
# reports.
|
||||
for thing, things_version in get_package_versions().iteritems():
|
||||
for thing, things_version in get_package_versions().items():
|
||||
app_versions.add_version(thing, str(things_version))
|
||||
|
||||
# group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
|
||||
@ -272,7 +277,10 @@ class _Config(object):
|
||||
self.config = configparser
|
||||
|
||||
nickname_utf8 = self.get_config("node", "nickname", "<unspecified>")
|
||||
self.nickname = nickname_utf8.decode("utf-8")
|
||||
if isinstance(nickname_utf8, bytes): # Python 2
|
||||
self.nickname = nickname_utf8.decode("utf-8")
|
||||
else:
|
||||
self.nickname = nickname_utf8
|
||||
assert type(self.nickname) is unicode
|
||||
|
||||
def validate(self, valid_config_sections):
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
from future.utils import bytes_to_native_str
|
||||
|
||||
import os, stat, struct, time
|
||||
|
||||
from foolscap.api import Referenceable
|
||||
@ -85,7 +87,7 @@ class ShareFile(object):
|
||||
seekpos = self._data_offset+offset
|
||||
actuallength = max(0, min(length, self._lease_offset-seekpos))
|
||||
if actuallength == 0:
|
||||
return ""
|
||||
return b""
|
||||
with open(self.home, 'rb') as f:
|
||||
f.seek(seekpos)
|
||||
return f.read(actuallength)
|
||||
@ -298,7 +300,9 @@ class BucketReader(Referenceable):
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s %s %s>" % (self.__class__.__name__,
|
||||
base32.b2a(self.storage_index[:8])[:12],
|
||||
bytes_to_native_str(
|
||||
base32.b2a(self.storage_index[:8])[:12]
|
||||
),
|
||||
self.shnum)
|
||||
|
||||
def remote_read(self, offset, length):
|
||||
@ -309,7 +313,7 @@ class BucketReader(Referenceable):
|
||||
return data
|
||||
|
||||
def remote_advise_corrupt_share(self, reason):
|
||||
return self.ss.remote_advise_corrupt_share("immutable",
|
||||
return self.ss.remote_advise_corrupt_share(b"immutable",
|
||||
self.storage_index,
|
||||
self.shnum,
|
||||
reason)
|
||||
|
||||
@ -113,7 +113,7 @@ class MutableShareFile(object):
|
||||
# start beyond the end of the data return an empty string.
|
||||
length = max(0, data_length-offset)
|
||||
if length == 0:
|
||||
return ""
|
||||
return b""
|
||||
precondition(offset+length <= data_length)
|
||||
f.seek(self.DATA_OFFSET+offset)
|
||||
data = f.read(length)
|
||||
@ -421,18 +421,18 @@ class MutableShareFile(object):
|
||||
# self._change_container_size() here.
|
||||
|
||||
def testv_compare(a, op, b):
|
||||
assert op in ("lt", "le", "eq", "ne", "ge", "gt")
|
||||
if op == "lt":
|
||||
assert op in (b"lt", b"le", b"eq", b"ne", b"ge", b"gt")
|
||||
if op == b"lt":
|
||||
return a < b
|
||||
if op == "le":
|
||||
if op == b"le":
|
||||
return a <= b
|
||||
if op == "eq":
|
||||
if op == b"eq":
|
||||
return a == b
|
||||
if op == "ne":
|
||||
if op == b"ne":
|
||||
return a != b
|
||||
if op == "ge":
|
||||
if op == b"ge":
|
||||
return a >= b
|
||||
if op == "gt":
|
||||
if op == b"gt":
|
||||
return a > b
|
||||
# never reached
|
||||
|
||||
@ -441,7 +441,7 @@ class EmptyShare(object):
|
||||
def check_testv(self, testv):
|
||||
test_good = True
|
||||
for (offset, length, operator, specimen) in testv:
|
||||
data = ""
|
||||
data = b""
|
||||
if not testv_compare(data, operator, specimen):
|
||||
test_good = False
|
||||
break
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
from future.utils import bytes_to_native_str
|
||||
import os, re, struct, time
|
||||
import weakref
|
||||
import six
|
||||
@ -51,6 +52,7 @@ class StorageServer(service.MultiService, Referenceable):
|
||||
service.MultiService.__init__(self)
|
||||
assert isinstance(nodeid, bytes)
|
||||
assert len(nodeid) == 20
|
||||
assert isinstance(nodeid, bytes)
|
||||
self.my_nodeid = nodeid
|
||||
self.storedir = storedir
|
||||
sharedir = os.path.join(storedir, "shares")
|
||||
@ -398,7 +400,7 @@ class StorageServer(service.MultiService, Referenceable):
|
||||
# since all shares get the same lease data, we just grab the leases
|
||||
# from the first share
|
||||
try:
|
||||
shnum, filename = self._get_bucket_shares(storage_index).next()
|
||||
shnum, filename = next(self._get_bucket_shares(storage_index))
|
||||
sf = ShareFile(filename)
|
||||
return sf.get_leases()
|
||||
except StopIteration:
|
||||
@ -676,6 +678,10 @@ class StorageServer(service.MultiService, Referenceable):
|
||||
|
||||
def remote_advise_corrupt_share(self, share_type, storage_index, shnum,
|
||||
reason):
|
||||
# This is a remote API, I believe, so this has to be bytes for legacy
|
||||
# protocol backwards compatibility reasons.
|
||||
assert isinstance(share_type, bytes)
|
||||
assert isinstance(reason, bytes)
|
||||
fileutil.make_dirs(self.corruption_advisory_dir)
|
||||
now = time_format.iso_utc(sep="T")
|
||||
si_s = si_b2a(storage_index)
|
||||
@ -684,11 +690,11 @@ class StorageServer(service.MultiService, Referenceable):
|
||||
"%s--%s-%d" % (now, si_s, shnum)).replace(":","")
|
||||
with open(fn, "w") as f:
|
||||
f.write("report: Share Corruption\n")
|
||||
f.write("type: %s\n" % share_type)
|
||||
f.write("storage_index: %s\n" % si_s)
|
||||
f.write("type: %s\n" % bytes_to_native_str(share_type))
|
||||
f.write("storage_index: %s\n" % bytes_to_native_str(si_s))
|
||||
f.write("share_number: %d\n" % shnum)
|
||||
f.write("\n")
|
||||
f.write(reason)
|
||||
f.write(bytes_to_native_str(reason))
|
||||
f.write("\n")
|
||||
log.msg(format=("client claims corruption in (%(share_type)s) " +
|
||||
"%(si)s-%(shnum)d: %(reason)s"),
|
||||
|
||||
@ -30,9 +30,12 @@ the foolscap-based server implemented in src/allmydata/storage/*.py .
|
||||
|
||||
|
||||
import re, time, hashlib
|
||||
from ConfigParser import (
|
||||
NoSectionError,
|
||||
)
|
||||
try:
|
||||
from ConfigParser import (
|
||||
NoSectionError,
|
||||
)
|
||||
except ImportError:
|
||||
from configparser import NoSectionError
|
||||
import attr
|
||||
from zope.interface import (
|
||||
Attribute,
|
||||
@ -534,11 +537,11 @@ class _NullStorage(object):
|
||||
which we can't communicate.
|
||||
"""
|
||||
nickname = ""
|
||||
permutation_seed = hashlib.sha256("").digest()
|
||||
tubid = hashlib.sha256("").digest()
|
||||
permutation_seed = hashlib.sha256(b"").digest()
|
||||
tubid = hashlib.sha256(b"").digest()
|
||||
storage_server = None
|
||||
|
||||
lease_seed = hashlib.sha256("").digest()
|
||||
lease_seed = hashlib.sha256(b"").digest()
|
||||
|
||||
name = "<unsupported>"
|
||||
longname = "<storage with unsupported protocol>"
|
||||
|
||||
@ -6,6 +6,8 @@ from twisted.python import usage
|
||||
from allmydata.util import configutil
|
||||
from ..common_util import run_cli, parse_cli
|
||||
from ...scripts import create_node
|
||||
from ... import client
|
||||
|
||||
|
||||
def read_config(basedir):
|
||||
tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
|
||||
@ -33,6 +35,31 @@ class Config(unittest.TestCase):
|
||||
e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
|
||||
self.assertIn("option %s not recognized" % (option,), str(e))
|
||||
|
||||
def test_create_client_config(self):
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
opts = {"nickname": "nick",
|
||||
"webport": "tcp:3456",
|
||||
"hide-ip": False,
|
||||
"listen": "none",
|
||||
"shares-needed": "1",
|
||||
"shares-happy": "1",
|
||||
"shares-total": "1",
|
||||
}
|
||||
create_node.write_node_config(f, opts)
|
||||
create_node.write_client_config(f, opts)
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
# should succeed, no exceptions
|
||||
configutil.validate_config(
|
||||
fname,
|
||||
config,
|
||||
client._valid_config(),
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_client(self):
|
||||
basedir = self.mktemp()
|
||||
|
||||
@ -52,7 +52,6 @@ from twisted.internet.defer import inlineCallbacks, returnValue
|
||||
from twisted.internet.interfaces import IPullProducer
|
||||
from twisted.python import failure
|
||||
from twisted.python.filepath import FilePath
|
||||
from twisted.application import service
|
||||
from twisted.web.error import Error as WebError
|
||||
from twisted.internet.interfaces import (
|
||||
IStreamServerEndpointStringParser,
|
||||
@ -88,6 +87,8 @@ from ..crypto import (
|
||||
from .eliotutil import (
|
||||
EliotLoggedRunTest,
|
||||
)
|
||||
# Backwards compatibility imports:
|
||||
from .common_py3 import LoggingServiceParent, ShouldFailMixin # noqa: F401
|
||||
|
||||
|
||||
TEST_RSA_KEY_SIZE = 522
|
||||
@ -780,53 +781,8 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
|
||||
return filenode
|
||||
|
||||
|
||||
class LoggingServiceParent(service.MultiService):
|
||||
def log(self, *args, **kwargs):
|
||||
return log.msg(*args, **kwargs)
|
||||
|
||||
|
||||
TEST_DATA="\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1)
|
||||
|
||||
class ShouldFailMixin(object):
|
||||
def shouldFail(self, expected_failure, which, substring,
|
||||
callable, *args, **kwargs):
|
||||
"""Assert that a function call raises some exception. This is a
|
||||
Deferred-friendly version of TestCase.assertRaises() .
|
||||
|
||||
Suppose you want to verify the following function:
|
||||
|
||||
def broken(a, b, c):
|
||||
if a < 0:
|
||||
raise TypeError('a must not be negative')
|
||||
return defer.succeed(b+c)
|
||||
|
||||
You can use:
|
||||
d = self.shouldFail(TypeError, 'test name',
|
||||
'a must not be negative',
|
||||
broken, -4, 5, c=12)
|
||||
in your test method. The 'test name' string will be included in the
|
||||
error message, if any, because Deferred chains frequently make it
|
||||
difficult to tell which assertion was tripped.
|
||||
|
||||
The substring= argument, if not None, must appear in the 'repr'
|
||||
of the message wrapped by this Failure, or the test will fail.
|
||||
"""
|
||||
|
||||
assert substring is None or isinstance(substring, str)
|
||||
d = defer.maybeDeferred(callable, *args, **kwargs)
|
||||
def done(res):
|
||||
if isinstance(res, failure.Failure):
|
||||
res.trap(expected_failure)
|
||||
if substring:
|
||||
message = repr(res.value.args[0])
|
||||
self.failUnless(substring in message,
|
||||
"%s: substring '%s' not in '%s'"
|
||||
% (which, substring, message))
|
||||
else:
|
||||
self.fail("%s was supposed to raise %s, not get '%s'" %
|
||||
(which, expected_failure, res))
|
||||
d.addBoth(done)
|
||||
return d
|
||||
|
||||
class WebErrorMixin(object):
|
||||
def explain_web_error(self, f):
|
||||
|
||||
@ -19,11 +19,13 @@ import time
|
||||
import signal
|
||||
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.application import service
|
||||
from twisted.python import failure
|
||||
from twisted.trial import unittest
|
||||
|
||||
from ..util.assertutil import precondition
|
||||
from ..util.encodingutil import unicode_platform, get_filesystem_encoding
|
||||
from ..util import log
|
||||
|
||||
|
||||
class TimezoneMixin(object):
|
||||
@ -77,6 +79,28 @@ class ShouldFailMixin(object):
|
||||
|
||||
def shouldFail(self, expected_failure, which, substring,
|
||||
callable, *args, **kwargs):
|
||||
"""Assert that a function call raises some exception. This is a
|
||||
Deferred-friendly version of TestCase.assertRaises() .
|
||||
|
||||
Suppose you want to verify the following function:
|
||||
|
||||
def broken(a, b, c):
|
||||
if a < 0:
|
||||
raise TypeError('a must not be negative')
|
||||
return defer.succeed(b+c)
|
||||
|
||||
You can use:
|
||||
d = self.shouldFail(TypeError, 'test name',
|
||||
'a must not be negative',
|
||||
broken, -4, 5, c=12)
|
||||
in your test method. The 'test name' string will be included in the
|
||||
error message, if any, because Deferred chains frequently make it
|
||||
difficult to tell which assertion was tripped.
|
||||
|
||||
The substring= argument, if not None, must appear in the 'repr'
|
||||
of the message wrapped by this Failure, or the test will fail.
|
||||
"""
|
||||
|
||||
assert substring is None or isinstance(substring, (bytes, unicode))
|
||||
d = defer.maybeDeferred(callable, *args, **kwargs)
|
||||
def done(res):
|
||||
@ -135,3 +159,9 @@ class FakeCanary(object):
|
||||
if self.ignore:
|
||||
return
|
||||
del self.disconnectors[marker]
|
||||
|
||||
|
||||
class LoggingServiceParent(service.MultiService):
|
||||
def log(self, *args, **kwargs):
|
||||
return log.msg(*args, **kwargs)
|
||||
|
||||
|
||||
@ -1,64 +1,7 @@
|
||||
|
||||
import re
|
||||
import treq
|
||||
from twisted.internet import defer
|
||||
from twisted.web.error import Error
|
||||
from nevow.testutil import FakeRequest
|
||||
from nevow import inevow, context
|
||||
|
||||
class WebRenderingMixin(object):
|
||||
# d=page.renderString() or s=page.renderSynchronously() will exercise
|
||||
# docFactory, render_*/data_* . It won't exercise want_json(), or my
|
||||
# renderHTTP() override which tests want_json(). To exercise args=, we
|
||||
# must build a context. Pages which use a return_to= argument need a
|
||||
# context.
|
||||
|
||||
# d=page.renderHTTP(ctx) will exercise my renderHTTP, want_json, and
|
||||
# docFactory/render_*/data_*, but it requires building a context. Since
|
||||
# we're already building a context, it is easy to exercise args= .
|
||||
|
||||
# so, use at least two d=page.renderHTTP(ctx) per page (one for json, one
|
||||
# for html), then use lots of simple s=page.renderSynchronously() to
|
||||
# exercise the fine details (the ones that don't require args=).
|
||||
|
||||
def make_context(self, req):
|
||||
ctx = context.RequestContext(tag=req)
|
||||
ctx.remember(req, inevow.IRequest)
|
||||
ctx.remember(None, inevow.IData)
|
||||
ctx = context.WovenContext(parent=ctx, precompile=False)
|
||||
return ctx
|
||||
|
||||
def render1(self, page, **kwargs):
|
||||
# use this to exercise an overridden renderHTTP, usually for
|
||||
# output=json or render_GET. It always returns a Deferred.
|
||||
req = FakeRequest(**kwargs)
|
||||
req.fields = None
|
||||
ctx = self.make_context(req)
|
||||
d = defer.maybeDeferred(page.renderHTTP, ctx)
|
||||
def _done(res):
|
||||
if isinstance(res, str):
|
||||
return res + req.v
|
||||
return req.v
|
||||
d.addCallback(_done)
|
||||
return d
|
||||
|
||||
def render2(self, page, **kwargs):
|
||||
# use this to exercise the normal Nevow docFactory rendering. It
|
||||
# returns a string. If one of the render_* methods returns a
|
||||
# Deferred, this will throw an exception. (note that
|
||||
# page.renderString is the Deferred-returning equivalent)
|
||||
req = FakeRequest(**kwargs)
|
||||
req.fields = None
|
||||
ctx = self.make_context(req)
|
||||
return page.renderSynchronously(ctx)
|
||||
|
||||
def failUnlessIn(self, substring, s):
|
||||
self.failUnless(substring in s, s)
|
||||
|
||||
def remove_tags(self, s):
|
||||
s = re.sub(r'<[^>]*>', ' ', s)
|
||||
s = re.sub(r'\s+', ' ', s)
|
||||
return s
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def do_http(method, url, **kwargs):
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
Tools aimed at the interaction between tests and Eliot.
|
||||
"""
|
||||
|
||||
from past.builtins import unicode
|
||||
|
||||
__all__ = [
|
||||
"RUN_TEST",
|
||||
"EliotLoggedRunTest",
|
||||
|
||||
@ -1,8 +1,21 @@
|
||||
"""
|
||||
Tests for allmydata.codec.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
import os
|
||||
from twisted.trial import unittest
|
||||
from twisted.python import log
|
||||
from allmydata.codec import CRSEncoder, CRSDecoder
|
||||
from allmydata.codec import CRSEncoder, CRSDecoder, parse_params
|
||||
import random
|
||||
from allmydata.util import mathutil
|
||||
|
||||
@ -13,6 +26,8 @@ class T(unittest.TestCase):
|
||||
enc.set_params(size, required_shares, max_shares)
|
||||
params = enc.get_params()
|
||||
assert params == (size, required_shares, max_shares)
|
||||
serialized_params = enc.get_serialized_params()
|
||||
self.assertEqual(parse_params(serialized_params), params)
|
||||
log.msg("params: %s" % (params,))
|
||||
d = enc.encode(data0s)
|
||||
def _done_encoding_all(shares_and_shareids):
|
||||
@ -23,7 +38,7 @@ class T(unittest.TestCase):
|
||||
d.addCallback(_done_encoding_all)
|
||||
if fewer_shares is not None:
|
||||
# also validate that the desired_shareids= parameter works
|
||||
desired_shareids = random.sample(range(max_shares), fewer_shares)
|
||||
desired_shareids = random.sample(list(range(max_shares)), fewer_shares)
|
||||
d.addCallback(lambda res: enc.encode(data0s, desired_shareids))
|
||||
def _check_fewer_shares(some_shares_and_their_shareids):
|
||||
(some_shares, their_shareids) = some_shares_and_their_shareids
|
||||
@ -38,11 +53,11 @@ class T(unittest.TestCase):
|
||||
return d1
|
||||
|
||||
def _check_data(decoded_shares):
|
||||
self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s)))
|
||||
self.failUnlessEqual(len(b''.join(decoded_shares)), len(b''.join(data0s)))
|
||||
self.failUnlessEqual(len(decoded_shares), len(data0s))
|
||||
for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)):
|
||||
self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],))
|
||||
self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",))
|
||||
self.failUnless(b''.join(decoded_shares) == b''.join(data0s), "%s" % ("???",))
|
||||
# 0data0sclipped = tuple(data0s)
|
||||
# data0sclipped[-1] =
|
||||
# self.failUnless(tuple(decoded_shares) == tuple(data0s))
|
||||
@ -59,7 +74,7 @@ class T(unittest.TestCase):
|
||||
def _decode_some_random(res):
|
||||
log.msg("_decode_some_random")
|
||||
# use a randomly-selected minimal subset
|
||||
l = random.sample(zip(self.shares, self.shareids), required_shares)
|
||||
l = random.sample(list(zip(self.shares, self.shareids)), required_shares)
|
||||
some_shares = [ x[0] for x in l ]
|
||||
some_shareids = [ x[1] for x in l ]
|
||||
return _decode((some_shares, some_shareids))
|
||||
@ -70,10 +85,10 @@ class T(unittest.TestCase):
|
||||
log.msg("_decode_multiple")
|
||||
# make sure we can re-use the decoder object
|
||||
shares1 = random.sample(self.shares, required_shares)
|
||||
sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares)
|
||||
sharesl1 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
|
||||
shares1 = [ x[0] for x in sharesl1 ]
|
||||
shareids1 = [ x[1] for x in sharesl1 ]
|
||||
sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares)
|
||||
sharesl2 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
|
||||
shares2 = [ x[0] for x in sharesl2 ]
|
||||
shareids2 = [ x[1] for x in sharesl2 ]
|
||||
dec = CRSDecoder()
|
||||
|
||||
@ -1,14 +1,26 @@
|
||||
"""
|
||||
Tests for allmydata.util.configutil.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
# Omitted dict, cause worried about interactions.
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
import os.path
|
||||
|
||||
from twisted.trial import unittest
|
||||
|
||||
from allmydata.util import configutil
|
||||
from allmydata.test.no_network import GridTestMixin
|
||||
from ..scripts import create_node
|
||||
from .. import client
|
||||
|
||||
|
||||
class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
class ConfigUtilTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
super(ConfigUtilTests, self).setUp()
|
||||
self.static_valid_config = configutil.ValidConfiguration(
|
||||
@ -20,10 +32,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
|
||||
)
|
||||
|
||||
def create_tahoe_cfg(self, cfg):
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
with open(fname, "w") as f:
|
||||
f.write(cfg)
|
||||
return fname
|
||||
|
||||
def test_config_utils(self):
|
||||
self.basedir = "cli/ConfigUtilTests/test-config-utils"
|
||||
self.set_up_grid(oneshare=True)
|
||||
tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
|
||||
tahoe_cfg = self.create_tahoe_cfg("""\
|
||||
[node]
|
||||
nickname = client-0
|
||||
web.port = adopt-socket:fd=5
|
||||
[storage]
|
||||
enabled = false
|
||||
""")
|
||||
|
||||
# test that at least one option was read correctly
|
||||
config = configutil.get_config(tahoe_cfg)
|
||||
@ -45,12 +69,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
|
||||
|
||||
def test_config_validation_success(self):
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
f.write('[node]\nvalid = foo\n')
|
||||
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
# should succeed, no exceptions
|
||||
@ -66,12 +85,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
validation but are matched by the dynamic validation is considered
|
||||
valid.
|
||||
"""
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
f.write('[node]\nvalid = foo\n')
|
||||
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
# should succeed, no exceptions
|
||||
@ -82,12 +96,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_config_validation_invalid_item(self):
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
f.write('[node]\nvalid = foo\ninvalid = foo\n')
|
||||
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
e = self.assertRaises(
|
||||
@ -103,12 +112,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
A configuration with a section that is matched by neither the static nor
|
||||
dynamic validators is rejected.
|
||||
"""
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
f.write('[node]\nvalid = foo\n[invalid]\n')
|
||||
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
e = self.assertRaises(
|
||||
@ -124,12 +128,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
A configuration with a section that is matched by neither the static nor
|
||||
dynamic validators is rejected.
|
||||
"""
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
f.write('[node]\nvalid = foo\n[invalid]\n')
|
||||
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
e = self.assertRaises(
|
||||
@ -145,12 +144,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
A configuration with a section, item pair that is matched by neither the
|
||||
static nor dynamic validators is rejected.
|
||||
"""
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
f.write('[node]\nvalid = foo\ninvalid = foo\n')
|
||||
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
e = self.assertRaises(
|
||||
@ -160,28 +154,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
|
||||
self.dynamic_valid_config,
|
||||
)
|
||||
self.assertIn("section [node] contains unknown option 'invalid'", str(e))
|
||||
|
||||
def test_create_client_config(self):
|
||||
d = self.mktemp()
|
||||
os.mkdir(d)
|
||||
fname = os.path.join(d, 'tahoe.cfg')
|
||||
|
||||
with open(fname, 'w') as f:
|
||||
opts = {"nickname": "nick",
|
||||
"webport": "tcp:3456",
|
||||
"hide-ip": False,
|
||||
"listen": "none",
|
||||
"shares-needed": "1",
|
||||
"shares-happy": "1",
|
||||
"shares-total": "1",
|
||||
}
|
||||
create_node.write_node_config(f, opts)
|
||||
create_node.write_client_config(f, opts)
|
||||
|
||||
config = configutil.get_config(fname)
|
||||
# should succeed, no exceptions
|
||||
configutil.validate_config(
|
||||
fname,
|
||||
config,
|
||||
client._valid_config(),
|
||||
)
|
||||
|
||||
122
src/allmydata/test/test_connection_status.py
Normal file
122
src/allmydata/test/test_connection_status.py
Normal file
@ -0,0 +1,122 @@
|
||||
"""
|
||||
Tests for allmydata.util.connection_status.
|
||||
|
||||
Port to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
import mock
|
||||
|
||||
from twisted.trial import unittest
|
||||
|
||||
from ..util import connection_status
|
||||
|
||||
class Status(unittest.TestCase):
|
||||
def test_hint_statuses(self):
|
||||
ncs = connection_status._hint_statuses(["h2","h1"],
|
||||
{"h1": "hand1", "h4": "hand4"},
|
||||
{"h1": "st1", "h2": "st2",
|
||||
"h3": "st3"})
|
||||
self.assertEqual(ncs, {"h1 via hand1": "st1",
|
||||
"h2": "st2"})
|
||||
|
||||
def test_reconnector_connected(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ci.winningHint = "h1"
|
||||
ci.establishedAt = 120
|
||||
ri = mock.Mock()
|
||||
ri.state = "connected"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, True)
|
||||
self.assertEqual(cs.summary, "Connected to h1 via hand1")
|
||||
self.assertEqual(cs.non_connected_statuses, {})
|
||||
self.assertEqual(cs.last_connection_time, 120)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_connected_others(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ci.winningHint = "h1"
|
||||
ci.establishedAt = 120
|
||||
ri = mock.Mock()
|
||||
ri.state = "connected"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, True)
|
||||
self.assertEqual(cs.summary, "Connected to h1 via hand1")
|
||||
self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, 120)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_connected_listener(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ci.listenerStatus = ("listener1", "successful")
|
||||
ci.winningHint = None
|
||||
ci.establishedAt = 120
|
||||
ri = mock.Mock()
|
||||
ri.state = "connected"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, True)
|
||||
self.assertEqual(cs.summary, "Connected via listener (listener1)")
|
||||
self.assertEqual(cs.non_connected_statuses,
|
||||
{"h1 via hand1": "st1", "h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, 120)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_connecting(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ri = mock.Mock()
|
||||
ri.state = "connecting"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, False)
|
||||
self.assertEqual(cs.summary, "Trying to connect")
|
||||
self.assertEqual(cs.non_connected_statuses,
|
||||
{"h1 via hand1": "st1", "h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, None)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_waiting(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ri = mock.Mock()
|
||||
ri.state = "waiting"
|
||||
ri.lastAttempt = 10
|
||||
ri.nextAttempt = 20
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
with mock.patch("time.time", return_value=12):
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 5)
|
||||
self.assertEqual(cs.connected, False)
|
||||
self.assertEqual(cs.summary,
|
||||
"Reconnecting in 8 seconds (last attempt 2s ago)")
|
||||
self.assertEqual(cs.non_connected_statuses,
|
||||
{"h1 via hand1": "st1", "h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, None)
|
||||
self.assertEqual(cs.last_received_time, 5)
|
||||
@ -7,7 +7,6 @@ from foolscap.connections import tcp
|
||||
from ..node import PrivacyError, config_from_string
|
||||
from ..node import create_connection_handlers
|
||||
from ..node import create_main_tub, _tub_portlocation
|
||||
from ..util import connection_status
|
||||
from ..util.i2p_provider import create as create_i2p_provider
|
||||
from ..util.tor_provider import create as create_tor_provider
|
||||
|
||||
@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
|
||||
str(ctx.exception),
|
||||
"tub.location includes tcp: hint",
|
||||
)
|
||||
|
||||
class Status(unittest.TestCase):
|
||||
def test_hint_statuses(self):
|
||||
ncs = connection_status._hint_statuses(["h2","h1"],
|
||||
{"h1": "hand1", "h4": "hand4"},
|
||||
{"h1": "st1", "h2": "st2",
|
||||
"h3": "st3"})
|
||||
self.assertEqual(ncs, {"h1 via hand1": "st1",
|
||||
"h2": "st2"})
|
||||
|
||||
def test_reconnector_connected(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ci.winningHint = "h1"
|
||||
ci.establishedAt = 120
|
||||
ri = mock.Mock()
|
||||
ri.state = "connected"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, True)
|
||||
self.assertEqual(cs.summary, "Connected to h1 via hand1")
|
||||
self.assertEqual(cs.non_connected_statuses, {})
|
||||
self.assertEqual(cs.last_connection_time, 120)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_connected_others(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ci.winningHint = "h1"
|
||||
ci.establishedAt = 120
|
||||
ri = mock.Mock()
|
||||
ri.state = "connected"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, True)
|
||||
self.assertEqual(cs.summary, "Connected to h1 via hand1")
|
||||
self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, 120)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_connected_listener(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ci.listenerStatus = ("listener1", "successful")
|
||||
ci.winningHint = None
|
||||
ci.establishedAt = 120
|
||||
ri = mock.Mock()
|
||||
ri.state = "connected"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, True)
|
||||
self.assertEqual(cs.summary, "Connected via listener (listener1)")
|
||||
self.assertEqual(cs.non_connected_statuses,
|
||||
{"h1 via hand1": "st1", "h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, 120)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_connecting(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ri = mock.Mock()
|
||||
ri.state = "connecting"
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 123)
|
||||
self.assertEqual(cs.connected, False)
|
||||
self.assertEqual(cs.summary, "Trying to connect")
|
||||
self.assertEqual(cs.non_connected_statuses,
|
||||
{"h1 via hand1": "st1", "h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, None)
|
||||
self.assertEqual(cs.last_received_time, 123)
|
||||
|
||||
def test_reconnector_waiting(self):
|
||||
ci = mock.Mock()
|
||||
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
|
||||
ci.connectionHandlers = {"h1": "hand1"}
|
||||
ri = mock.Mock()
|
||||
ri.state = "waiting"
|
||||
ri.lastAttempt = 10
|
||||
ri.nextAttempt = 20
|
||||
ri.connectionInfo = ci
|
||||
rc = mock.Mock
|
||||
rc.getReconnectionInfo = mock.Mock(return_value=ri)
|
||||
with mock.patch("time.time", return_value=12):
|
||||
cs = connection_status.from_foolscap_reconnector(rc, 5)
|
||||
self.assertEqual(cs.connected, False)
|
||||
self.assertEqual(cs.summary,
|
||||
"Reconnecting in 8 seconds (last attempt 2s ago)")
|
||||
self.assertEqual(cs.non_connected_statuses,
|
||||
{"h1 via hand1": "st1", "h2": "st2"})
|
||||
self.assertEqual(cs.last_connection_time, None)
|
||||
self.assertEqual(cs.last_received_time, 5)
|
||||
|
||||
52
src/allmydata/test/test_monitor.py
Normal file
52
src/allmydata/test/test_monitor.py
Normal file
@ -0,0 +1,52 @@
|
||||
"""
|
||||
Tests for allmydata.monitor.
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
from twisted.trial import unittest
|
||||
|
||||
from allmydata.monitor import Monitor, OperationCancelledError
|
||||
|
||||
|
||||
class MonitorTests(unittest.TestCase):
|
||||
"""Tests for the Monitor class."""
|
||||
|
||||
def test_cancellation(self):
|
||||
"""The monitor can be cancelled."""
|
||||
m = Monitor()
|
||||
self.assertFalse(m.is_cancelled())
|
||||
m.raise_if_cancelled()
|
||||
m.cancel()
|
||||
self.assertTrue(m.is_cancelled())
|
||||
with self.assertRaises(OperationCancelledError):
|
||||
m.raise_if_cancelled()
|
||||
|
||||
def test_status(self):
|
||||
"""The monitor can have its status set."""
|
||||
m = Monitor()
|
||||
self.assertEqual(m.get_status(), None)
|
||||
m.set_status("discombobulated")
|
||||
self.assertEqual(m.get_status(), "discombobulated")
|
||||
|
||||
def test_finish(self):
|
||||
"""The monitor can finish."""
|
||||
m = Monitor()
|
||||
self.assertFalse(m.is_finished())
|
||||
d = m.when_done()
|
||||
self.assertNoResult(d)
|
||||
|
||||
result = m.finish(300)
|
||||
self.assertEqual(result, 300)
|
||||
self.assertEqual(m.get_status(), 300)
|
||||
self.assertTrue(m.is_finished())
|
||||
|
||||
d.addBoth(self.assertEqual, 300)
|
||||
return d
|
||||
File diff suppressed because it is too large
Load Diff
@ -9,7 +9,7 @@ from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2, PY3
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
# Omitted list sinc it broke a test on Python 2. Shouldn't require further
|
||||
# work, when we switch to Python 3 we'll be dropping this, anyway.
|
||||
@ -19,7 +19,6 @@ import time
|
||||
import os.path
|
||||
import re
|
||||
import json
|
||||
from unittest import skipIf
|
||||
|
||||
from twisted.trial import unittest
|
||||
|
||||
@ -107,7 +106,6 @@ class MyStorageServer(StorageServer):
|
||||
|
||||
class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
@skipIf(PY3, "Not ported yet.")
|
||||
def setUp(self):
|
||||
self.s = service.MultiService()
|
||||
self.s.startService()
|
||||
@ -130,12 +128,12 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
# this sample is before the crawler has started doing anything
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: Yes", s)
|
||||
self.failUnlessIn("Reserved space: - 0 B (0)", s)
|
||||
self.failUnlessIn("Total buckets: Not computed yet", s)
|
||||
self.failUnlessIn("Next crawl in", s)
|
||||
self.failUnlessIn(b"Accepting new shares: Yes", s)
|
||||
self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
|
||||
self.failUnlessIn(b"Total buckets: Not computed yet", s)
|
||||
self.failUnlessIn(b"Next crawl in", s)
|
||||
|
||||
# give the bucket-counting-crawler one tick to get started. The
|
||||
# cpu_slice=0 will force it to yield right after it processes the
|
||||
@ -154,8 +152,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn(" Current crawl ", s)
|
||||
self.failUnlessIn(" (next work in ", s)
|
||||
self.failUnlessIn(b" Current crawl ", s)
|
||||
self.failUnlessIn(b" (next work in ", s)
|
||||
d.addCallback(_check)
|
||||
|
||||
# now give it enough time to complete a full cycle
|
||||
@ -166,8 +164,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.bucket_counter.cpu_slice = orig_cpu_slice
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Total buckets: 0 (the number of", s)
|
||||
self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
|
||||
self.failUnlessIn(b"Total buckets: 0 (the number of", s)
|
||||
self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
|
||||
d.addCallback(_check2)
|
||||
return d
|
||||
|
||||
@ -228,20 +226,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
# no ETA is available yet
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("complete (next work", s)
|
||||
self.failUnlessIn(b"complete (next work", s)
|
||||
|
||||
def _check_2(ignored):
|
||||
# one prefix has finished, so an ETA based upon that elapsed time
|
||||
# should be available.
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("complete (ETA ", s)
|
||||
self.failUnlessIn(b"complete (ETA ", s)
|
||||
|
||||
def _check_3(ignored):
|
||||
# two prefixes have finished
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("complete (ETA ", s)
|
||||
self.failUnlessIn(b"complete (ETA ", s)
|
||||
d.callback("done")
|
||||
|
||||
hooks[0].addCallback(_check_1).addErrback(d.errback)
|
||||
@ -1172,7 +1170,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
@skipIf(PY3, "Not ported yet.")
|
||||
def setUp(self):
|
||||
self.s = service.MultiService()
|
||||
self.s.startService()
|
||||
@ -1182,7 +1179,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
def test_no_server(self):
|
||||
w = StorageStatus(None)
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>No Storage Server Running</h1>", html)
|
||||
self.failUnlessIn(b"<h1>No Storage Server Running</h1>", html)
|
||||
|
||||
def test_status(self):
|
||||
basedir = "storage/WebStatus/status"
|
||||
@ -1193,12 +1190,12 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
w = StorageStatus(ss, "nickname")
|
||||
d = renderDeferred(w)
|
||||
def _check_html(html):
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Server Nickname: nickname", s)
|
||||
self.failUnlessIn("Server Nodeid: %s" % base32.b2a(nodeid), s)
|
||||
self.failUnlessIn("Accepting new shares: Yes", s)
|
||||
self.failUnlessIn("Reserved space: - 0 B (0)", s)
|
||||
self.failUnlessIn(b"Server Nickname: nickname", s)
|
||||
self.failUnlessIn(b"Server Nodeid: %s" % base32.b2a(nodeid), s)
|
||||
self.failUnlessIn(b"Accepting new shares: Yes", s)
|
||||
self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
|
||||
d.addCallback(_check_html)
|
||||
d.addCallback(lambda ign: renderJSON(w))
|
||||
def _check_json(raw):
|
||||
@ -1225,11 +1222,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: Yes", s)
|
||||
self.failUnlessIn("Total disk space: ?", s)
|
||||
self.failUnlessIn("Space Available to Tahoe: ?", s)
|
||||
self.failUnlessIn(b"Accepting new shares: Yes", s)
|
||||
self.failUnlessIn(b"Total disk space: ?", s)
|
||||
self.failUnlessIn(b"Space Available to Tahoe: ?", s)
|
||||
self.failUnless(ss.get_available_space() is None)
|
||||
|
||||
def test_status_bad_disk_stats(self):
|
||||
@ -1245,11 +1242,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: No", s)
|
||||
self.failUnlessIn("Total disk space: ?", s)
|
||||
self.failUnlessIn("Space Available to Tahoe: ?", s)
|
||||
self.failUnlessIn(b"Accepting new shares: No", s)
|
||||
self.failUnlessIn(b"Total disk space: ?", s)
|
||||
self.failUnlessIn(b"Space Available to Tahoe: ?", s)
|
||||
self.failUnlessEqual(ss.get_available_space(), 0)
|
||||
|
||||
def test_status_right_disk_stats(self):
|
||||
@ -1282,14 +1279,14 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
w = StorageStatus(ss)
|
||||
html = renderSynchronously(w)
|
||||
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Total disk space: 5.00 GB", s)
|
||||
self.failUnlessIn("Disk space used: - 1.00 GB", s)
|
||||
self.failUnlessIn("Disk space free (root): 4.00 GB", s)
|
||||
self.failUnlessIn("Disk space free (non-root): 3.00 GB", s)
|
||||
self.failUnlessIn("Reserved space: - 1.00 GB", s)
|
||||
self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s)
|
||||
self.failUnlessIn(b"Total disk space: 5.00 GB", s)
|
||||
self.failUnlessIn(b"Disk space used: - 1.00 GB", s)
|
||||
self.failUnlessIn(b"Disk space free (root): 4.00 GB", s)
|
||||
self.failUnlessIn(b"Disk space free (non-root): 3.00 GB", s)
|
||||
self.failUnlessIn(b"Reserved space: - 1.00 GB", s)
|
||||
self.failUnlessIn(b"Space Available to Tahoe: 2.00 GB", s)
|
||||
self.failUnlessEqual(ss.get_available_space(), 2*GB)
|
||||
|
||||
def test_readonly(self):
|
||||
@ -1299,9 +1296,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: No", s)
|
||||
self.failUnlessIn(b"Accepting new shares: No", s)
|
||||
|
||||
def test_reserved(self):
|
||||
basedir = "storage/WebStatus/reserved"
|
||||
@ -1310,9 +1307,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
|
||||
self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
|
||||
|
||||
def test_huge_reserved(self):
|
||||
basedir = "storage/WebStatus/reserved"
|
||||
@ -1321,9 +1318,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
|
||||
self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
|
||||
|
||||
def test_util(self):
|
||||
w = StorageStatusElement(None, None)
|
||||
|
||||
@ -980,6 +980,8 @@ class CountingDataUploadable(upload.Data):
|
||||
|
||||
class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
|
||||
timeout = 180
|
||||
|
||||
def test_connections(self):
|
||||
self.basedir = "system/SystemTest/test_connections"
|
||||
d = self.set_up_nodes()
|
||||
|
||||
@ -24,6 +24,7 @@ if PY2:
|
||||
|
||||
# Keep these sorted alphabetically, to reduce merge conflicts:
|
||||
PORTED_MODULES = [
|
||||
"allmydata.codec",
|
||||
"allmydata.crypto",
|
||||
"allmydata.crypto.aes",
|
||||
"allmydata.crypto.ed25519",
|
||||
@ -32,6 +33,8 @@ PORTED_MODULES = [
|
||||
"allmydata.crypto.util",
|
||||
"allmydata.hashtree",
|
||||
"allmydata.immutable.happiness_upload",
|
||||
"allmydata.interfaces",
|
||||
"allmydata.monitor",
|
||||
"allmydata.storage.crawler",
|
||||
"allmydata.storage.expirer",
|
||||
"allmydata.test.common_py3",
|
||||
@ -41,6 +44,8 @@ PORTED_MODULES = [
|
||||
"allmydata.util.assertutil",
|
||||
"allmydata.util.base32",
|
||||
"allmydata.util.base62",
|
||||
"allmydata.util.configutil",
|
||||
"allmydata.util.connection_status",
|
||||
"allmydata.util.deferredutil",
|
||||
"allmydata.util.fileutil",
|
||||
"allmydata.util.dictutil",
|
||||
@ -66,6 +71,9 @@ PORTED_TEST_MODULES = [
|
||||
"allmydata.test.test_abbreviate",
|
||||
"allmydata.test.test_base32",
|
||||
"allmydata.test.test_base62",
|
||||
"allmydata.test.test_codec",
|
||||
"allmydata.test.test_configutil",
|
||||
"allmydata.test.test_connection_status",
|
||||
"allmydata.test.test_crawler",
|
||||
"allmydata.test.test_crypto",
|
||||
"allmydata.test.test_deferredutil",
|
||||
@ -77,13 +85,15 @@ PORTED_TEST_MODULES = [
|
||||
"allmydata.test.test_humanreadable",
|
||||
"allmydata.test.test_iputil",
|
||||
"allmydata.test.test_log",
|
||||
"allmydata.test.test_monitor",
|
||||
"allmydata.test.test_netstring",
|
||||
"allmydata.test.test_observer",
|
||||
"allmydata.test.test_pipeline",
|
||||
"allmydata.test.test_python3",
|
||||
"allmydata.test.test_spans",
|
||||
"allmydata.test.test_statistics",
|
||||
"allmydata.test.test_storage_web", # partial, WIP
|
||||
"allmydata.test.test_storage",
|
||||
"allmydata.test.test_storage_web",
|
||||
"allmydata.test.test_time_format",
|
||||
"allmydata.test.test_uri",
|
||||
"allmydata.test.test_util",
|
||||
|
||||
@ -1,8 +1,32 @@
|
||||
"""
|
||||
Read/write config files.
|
||||
|
||||
from ConfigParser import SafeConfigParser
|
||||
Configuration is returned as native strings.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
# We don't do open(), because we want files to read/write native strs when
|
||||
# we do "r" or "w".
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
if PY2:
|
||||
# In theory on Python 2 configparser also works, but then code gets the
|
||||
# wrong exceptions and they don't get handled. So just use native parser
|
||||
# for now.
|
||||
from ConfigParser import SafeConfigParser
|
||||
else:
|
||||
from configparser import SafeConfigParser
|
||||
|
||||
import attr
|
||||
|
||||
|
||||
class UnknownConfigError(Exception):
|
||||
"""
|
||||
An unknown config item was found.
|
||||
@ -12,11 +36,16 @@ class UnknownConfigError(Exception):
|
||||
|
||||
|
||||
def get_config(tahoe_cfg):
|
||||
"""Load the config, returning a SafeConfigParser.
|
||||
|
||||
Configuration is returned as native strings.
|
||||
"""
|
||||
config = SafeConfigParser()
|
||||
with open(tahoe_cfg, "rb") as f:
|
||||
# Skip any initial Byte Order Mark. Since this is an ordinary file, we
|
||||
# don't need to handle incomplete reads, and can assume seekability.
|
||||
if f.read(3) != '\xEF\xBB\xBF':
|
||||
with open(tahoe_cfg, "r") as f:
|
||||
# On Python 2, where we read in bytes, skip any initial Byte Order
|
||||
# Mark. Since this is an ordinary file, we don't need to handle
|
||||
# incomplete reads, and can assume seekability.
|
||||
if PY2 and f.read(3) != b'\xEF\xBB\xBF':
|
||||
f.seek(0)
|
||||
config.readfp(f)
|
||||
return config
|
||||
@ -28,7 +57,7 @@ def set_config(config, section, option, value):
|
||||
assert config.get(section, option) == value
|
||||
|
||||
def write_config(tahoe_cfg, config):
|
||||
with open(tahoe_cfg, "wb") as f:
|
||||
with open(tahoe_cfg, "w") as f:
|
||||
config.write(f)
|
||||
|
||||
def validate_config(fname, cfg, valid_config):
|
||||
|
||||
@ -1,3 +1,18 @@
|
||||
"""
|
||||
Parse connection status from Foolscap.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
import time
|
||||
from zope.interface import implementer
|
||||
from ..interfaces import IConnectionStatus
|
||||
@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
|
||||
|
||||
def from_foolscap_reconnector(rc, last_received):
|
||||
ri = rc.getReconnectionInfo()
|
||||
# See foolscap/reconnector.py, ReconnectionInfo, for details about
|
||||
# possible states.
|
||||
# See foolscap/reconnector.py, ReconnectionInfo, for details about possible
|
||||
# states. The returned result is a native string, it seems, so convert to
|
||||
# unicode.
|
||||
state = ri.state
|
||||
if isinstance(state, bytes): # Python 2
|
||||
state = str(state, "ascii")
|
||||
if state == "unstarted":
|
||||
return ConnectionStatus.unstarted()
|
||||
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
"""
|
||||
Implement a work-around for <https://github.com/twisted/nevow/issues/106>.
|
||||
"""
|
||||
|
||||
from __future__ import (
|
||||
print_function,
|
||||
unicode_literals,
|
||||
absolute_import,
|
||||
division,
|
||||
)
|
||||
|
||||
from nevow import inevow
|
||||
from twisted.internet import defer
|
||||
|
||||
def renderHTTP(self, ctx):
|
||||
request = inevow.IRequest(ctx)
|
||||
if self.real_prepath_len is not None:
|
||||
request.postpath = request.prepath + request.postpath
|
||||
request.prepath = request.postpath[:self.real_prepath_len]
|
||||
del request.postpath[:self.real_prepath_len]
|
||||
result = defer.maybeDeferred(self.original.render, request).addCallback(
|
||||
self._handle_NOT_DONE_YET, request)
|
||||
return result
|
||||
|
||||
|
||||
def patch():
|
||||
"""
|
||||
Monkey-patch the proposed fix into place.
|
||||
"""
|
||||
from nevow.appserver import OldResourceAdapter
|
||||
OldResourceAdapter.renderHTTP = renderHTTP
|
||||
@ -54,11 +54,6 @@ from .logs import (
|
||||
create_log_resources,
|
||||
)
|
||||
|
||||
# Hotfix work-around https://github.com/twisted/nevow/issues/106
|
||||
from . import _nevow_106
|
||||
_nevow_106.patch()
|
||||
del _nevow_106
|
||||
|
||||
SCHEME = b"tahoe-lafs"
|
||||
|
||||
class IToken(ICredentials):
|
||||
|
||||
68
tox.ini
68
tox.ini
@ -44,36 +44,42 @@ usedevelop = False
|
||||
# We use extras=test to get things like "mock" that are required for our unit
|
||||
# tests.
|
||||
extras = test
|
||||
commands =
|
||||
trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
|
||||
tahoe --version
|
||||
|
||||
[testenv:py36]
|
||||
setenv =
|
||||
# Define TEST_SUITE in the environment as an aid to constructing the
|
||||
# correct test command below.
|
||||
!py36: TEST_SUITE = allmydata
|
||||
py36: TEST_SUITE = allmydata.test.python3_tests
|
||||
|
||||
commands =
|
||||
trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
|
||||
# As an aid to debugging, dump all of the Python packages and their
|
||||
# versions that are installed in the test environment. This is
|
||||
# particularly useful to get from CI runs - though hopefully the
|
||||
# version pinning we do limits the variability of this output
|
||||
pip freeze
|
||||
|
||||
# The tahoe script isn't sufficiently ported for this to succeed on
|
||||
# Python 3.x yet.
|
||||
!py36: tahoe --version
|
||||
|
||||
!coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
|
||||
|
||||
# measuring coverage is somewhat slower than not measuring coverage
|
||||
# so only do it on request.
|
||||
coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
|
||||
coverage: coverage combine
|
||||
coverage: coverage xml
|
||||
|
||||
|
||||
[testenv:integration]
|
||||
setenv =
|
||||
COVERAGE_PROCESS_START=.coveragerc
|
||||
commands =
|
||||
# NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
|
||||
py.test --coverage -v {posargs:integration}
|
||||
# NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
|
||||
py.test --coverage -v {posargs:integration}
|
||||
coverage combine
|
||||
coverage report
|
||||
|
||||
[testenv:coverage]
|
||||
# coverage (with --branch) takes about 65% longer to run
|
||||
commands =
|
||||
# As an aid to debugging, dump all of the Python packages and their
|
||||
# versions that are installed in the test environment. This is
|
||||
# particularly useful to get from CI runs - though hopefully the
|
||||
# version pinning we do limits the variability of this output
|
||||
# somewhat.
|
||||
pip freeze
|
||||
tahoe --version
|
||||
coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
|
||||
coverage combine
|
||||
coverage xml
|
||||
|
||||
[testenv:codechecks]
|
||||
# On macOS, git inside of towncrier needs $HOME.
|
||||
@ -87,11 +93,11 @@ commands =
|
||||
python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
|
||||
python misc/coding_tools/check-miscaptures.py
|
||||
|
||||
# If towncrier.check fails, you forgot to add a towncrier news
|
||||
# fragment explaining the change in this branch. Create one at
|
||||
# `newsfragments/<ticket>.<change type>` with some text for the news
|
||||
# file. See pyproject.toml for legal <change type> values.
|
||||
python -m towncrier.check --pyproject towncrier.pyproject.toml
|
||||
# If towncrier.check fails, you forgot to add a towncrier news
|
||||
# fragment explaining the change in this branch. Create one at
|
||||
# `newsfragments/<ticket>.<change type>` with some text for the news
|
||||
# file. See pyproject.toml for legal <change type> values.
|
||||
python -m towncrier.check --pyproject towncrier.pyproject.toml
|
||||
|
||||
[testenv:draftnews]
|
||||
passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
|
||||
@ -110,9 +116,9 @@ commands =
|
||||
#
|
||||
# Some discussion is available at
|
||||
# https://github.com/pypa/pip/issues/5696
|
||||
#
|
||||
# towncrier post 19.2 (unreleased as of this writing) adds a --config
|
||||
# option that can be used instead of this file shuffling.
|
||||
#
|
||||
# towncrier post 19.2 (unreleased as of this writing) adds a --config
|
||||
# option that can be used instead of this file shuffling.
|
||||
mv towncrier.pyproject.toml pyproject.toml
|
||||
|
||||
# towncrier 19.2 + works with python2.7
|
||||
@ -138,9 +144,9 @@ commands =
|
||||
#
|
||||
# Some discussion is available at
|
||||
# https://github.com/pypa/pip/issues/5696
|
||||
#
|
||||
# towncrier post 19.2 (unreleased as of this writing) adds a --config
|
||||
# option that can be used instead of this file shuffling.
|
||||
#
|
||||
# towncrier post 19.2 (unreleased as of this writing) adds a --config
|
||||
# option that can be used instead of this file shuffling.
|
||||
mv towncrier.pyproject.toml pyproject.toml
|
||||
|
||||
# towncrier 19.2 + works with python2.7
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user