mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2024-12-24 15:16:41 +00:00
Merge remote-tracking branch 'origin/master' into 3606.test_system-web-python3
This commit is contained in:
Itamar Turner-Trauring
commit
06cd015647
@ -29,7 +29,7 @@ workflows:
|
||||
- "debian-9": &DOCKERHUB_CONTEXT
|
||||
context: "dockerhub-auth"
|
||||
|
||||
- "debian-8":
|
||||
- "debian-10":
|
||||
<<: *DOCKERHUB_CONTEXT
|
||||
requires:
|
||||
- "debian-9"
|
||||
@ -86,11 +86,6 @@ workflows:
|
||||
# integration tests.
|
||||
- "debian-9"
|
||||
|
||||
# Generate the underlying data for a visualization to aid with Python 3
|
||||
# porting.
|
||||
- "build-porting-depgraph":
|
||||
<<: *DOCKERHUB_CONTEXT
|
||||
|
||||
- "typechecks":
|
||||
<<: *DOCKERHUB_CONTEXT
|
||||
|
||||
@ -107,7 +102,7 @@ workflows:
|
||||
- "master"
|
||||
|
||||
jobs:
|
||||
- "build-image-debian-8":
|
||||
- "build-image-debian-10":
|
||||
<<: *DOCKERHUB_CONTEXT
|
||||
- "build-image-debian-9":
|
||||
<<: *DOCKERHUB_CONTEXT
|
||||
@ -213,7 +208,7 @@ jobs:
|
||||
# filenames and argv).
|
||||
LANG: "en_US.UTF-8"
|
||||
# Select a tox environment to run for this job.
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py27"
|
||||
# Additional arguments to pass to tox.
|
||||
TAHOE_LAFS_TOX_ARGS: ""
|
||||
# The path in which test artifacts will be placed.
|
||||
@ -223,7 +218,7 @@ jobs:
|
||||
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
|
||||
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
|
||||
# Upload the coverage report.
|
||||
UPLOAD_COVERAGE: "yes"
|
||||
UPLOAD_COVERAGE: ""
|
||||
|
||||
# pip cannot install packages if the working directory is not readable.
|
||||
# We want to run a lot of steps as nobody instead of as root.
|
||||
@ -277,11 +272,11 @@ jobs:
|
||||
fi
|
||||
|
||||
|
||||
debian-8:
|
||||
debian-10:
|
||||
<<: *DEBIAN
|
||||
docker:
|
||||
- <<: *DOCKERHUB_AUTH
|
||||
image: "tahoelafsci/debian:8-py2.7"
|
||||
image: "tahoelafsci/debian:10-py2.7"
|
||||
user: "nobody"
|
||||
|
||||
|
||||
@ -376,7 +371,7 @@ jobs:
|
||||
# this reporter on Python 3. So drop that and just specify the
|
||||
# reporter.
|
||||
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
|
||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
|
||||
|
||||
|
||||
ubuntu-20-04:
|
||||
@ -451,33 +446,6 @@ jobs:
|
||||
# them in parallel.
|
||||
nix-build --cores 3 --max-jobs 2 nix/
|
||||
|
||||
# Generate up-to-date data for the dependency graph visualizer.
|
||||
build-porting-depgraph:
|
||||
# Get a system in which we can easily install Tahoe-LAFS and all its
|
||||
# dependencies. The dependency graph analyzer works by executing the code.
|
||||
# It's Python, what do you expect?
|
||||
<<: *DEBIAN
|
||||
|
||||
steps:
|
||||
- "checkout"
|
||||
|
||||
- add_ssh_keys:
|
||||
fingerprints:
|
||||
# Jean-Paul Calderone <exarkun@twistedmatrix.com> (CircleCI depgraph key)
|
||||
# This lets us push to tahoe-lafs/tahoe-depgraph in the next step.
|
||||
- "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4"
|
||||
|
||||
- run:
|
||||
name: "Setup Python Environment"
|
||||
command: |
|
||||
/tmp/venv/bin/pip install -e /tmp/project
|
||||
|
||||
- run:
|
||||
name: "Generate dependency graph data"
|
||||
command: |
|
||||
. /tmp/venv/bin/activate
|
||||
./misc/python3/depgraph.sh
|
||||
|
||||
typechecks:
|
||||
docker:
|
||||
- <<: *DOCKERHUB_AUTH
|
||||
@ -529,12 +497,12 @@ jobs:
|
||||
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
|
||||
|
||||
|
||||
build-image-debian-8:
|
||||
build-image-debian-10:
|
||||
<<: *BUILD_IMAGE
|
||||
|
||||
environment:
|
||||
DISTRO: "debian"
|
||||
TAG: "8"
|
||||
TAG: "10"
|
||||
PYTHON_VERSION: "2.7"
|
||||
|
||||
|
||||
|
||||
48
.codecov.yml
48
.codecov.yml
@ -1,48 +0,0 @@
|
||||
# Override defaults for codecov.io checks.
|
||||
#
|
||||
# Documentation is at https://docs.codecov.io/docs/codecov-yaml;
|
||||
# reference is at https://docs.codecov.io/docs/codecovyml-reference.
|
||||
#
|
||||
# To validate this file, use:
|
||||
#
|
||||
# curl --data-binary @.codecov.yml https://codecov.io/validate
|
||||
#
|
||||
# Codecov's defaults seem to leave red marks in GitHub CI checks in a
|
||||
# rather arbitrary manner, probably because of non-determinism in
|
||||
# coverage (see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2891)
|
||||
# and maybe because computers are bad with floating point numbers.
|
||||
|
||||
# Allow coverage percentage a precision of zero decimals, and round to
|
||||
# the nearest number (for example, 89.957 to to 90; 89.497 to 89%).
|
||||
# Coverage above 90% is good, below 80% is bad.
|
||||
coverage:
|
||||
round: nearest
|
||||
range: 80..90
|
||||
precision: 0
|
||||
|
||||
# Aim for a target test coverage of 90% in codecov/project check (do
|
||||
# not allow project coverage to drop below that), and allow
|
||||
# codecov/patch a threshold of 1% (allow coverage in changes to drop
|
||||
# by that much, and no less). That should be good enough for us.
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
target: 90%
|
||||
threshold: 1%
|
||||
patch:
|
||||
default:
|
||||
threshold: 1%
|
||||
|
||||
|
||||
codecov:
|
||||
# This is a public repository so supposedly we don't "need" to use an upload
|
||||
# token. However, using one makes sure that CI jobs running against forked
|
||||
# repositories have coverage uploaded to the right place in codecov so
|
||||
# their reports aren't incomplete.
|
||||
token: "abf679b6-e2e6-4b33-b7b5-6cfbd41ee691"
|
||||
|
||||
notify:
|
||||
# The reference documentation suggests that this is the default setting:
|
||||
# https://docs.codecov.io/docs/codecovyml-reference#codecovnotifywait_for_ci
|
||||
# However observation suggests otherwise.
|
||||
wait_for_ci: true
|
||||
109
.github/workflows/ci.yml
vendored
109
.github/workflows/ci.yml
vendored
@ -79,11 +79,110 @@ jobs:
|
||||
name: eliot.log
|
||||
path: eliot.log
|
||||
|
||||
- name: Upload coverage report
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
token: abf679b6-e2e6-4b33-b7b5-6cfbd41ee691
|
||||
file: coverage.xml
|
||||
# Upload this job's coverage data to Coveralls. While there is a GitHub
|
||||
# Action for this, as of Jan 2021 it does not support Python coverage
|
||||
# files - only lcov files. Therefore, we use coveralls-python, the
|
||||
# coveralls.io-supplied Python reporter, for this.
|
||||
- name: "Report Coverage to Coveralls"
|
||||
run: |
|
||||
pip install coveralls
|
||||
python -m coveralls
|
||||
env:
|
||||
# Some magic value required for some magic reason.
|
||||
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||
# Help coveralls identify our project.
|
||||
COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
|
||||
# Every source of coverage reports needs a unique "flag name".
|
||||
# Construct one by smashing a few variables from the matrix together
|
||||
# here.
|
||||
COVERALLS_FLAG_NAME: "run-${{ matrix.os }}-${{ matrix.python-version }}"
|
||||
# Mark the data as just one piece of many because we have more than
|
||||
# one instance of this job (Windows, macOS) which collects and
|
||||
# reports coverage. This is necessary to cause Coveralls to merge
|
||||
# multiple coverage results into a single report. Note the merge
|
||||
# only happens when we "finish" a particular build, as identified by
|
||||
# its "build_num" (aka "service_number").
|
||||
COVERALLS_PARALLEL: true
|
||||
|
||||
# Tell Coveralls that we're done reporting coverage data. Since we're using
|
||||
# the "parallel" mode where more than one coverage data file is merged into
|
||||
# a single report, we have to tell Coveralls when we've uploaded all of the
|
||||
# data files. This does it. We make sure it runs last by making it depend
|
||||
# on *all* of the coverage-collecting jobs.
|
||||
finish-coverage-report:
|
||||
# There happens to just be one coverage-collecting job at the moment. If
|
||||
# the coverage reports are broken and someone added more
|
||||
# coverage-collecting jobs to this workflow but didn't update this, that's
|
||||
# why.
|
||||
needs:
|
||||
- "coverage"
|
||||
runs-on: "ubuntu-latest"
|
||||
steps:
|
||||
- name: "Check out Tahoe-LAFS sources"
|
||||
uses: "actions/checkout@v2"
|
||||
|
||||
- name: "Finish Coveralls Reporting"
|
||||
run: |
|
||||
# coveralls-python does have a `--finish` option but it doesn't seem
|
||||
# to work, at least for us.
|
||||
# https://github.com/coveralls-clients/coveralls-python/issues/248
|
||||
#
|
||||
# But all it does is this simple POST so we can just send it
|
||||
# ourselves. The only hard part is guessing what the POST
|
||||
# parameters mean. And I've done that for you already.
|
||||
#
|
||||
# Since the build is done I'm going to guess that "done" is a fine
|
||||
# value for status.
|
||||
#
|
||||
# That leaves "build_num". The coveralls documentation gives some
|
||||
# hints about it. It suggests using $CIRCLE_WORKFLOW_ID if your job
|
||||
# is on CircleCI. CircleCI documentation says this about
|
||||
# CIRCLE_WORKFLOW_ID:
|
||||
#
|
||||
# Observation of the coveralls.io web interface, logs from the
|
||||
# coveralls command in action, and experimentation suggests the
|
||||
# value for PRs is something more like:
|
||||
#
|
||||
# <GIT MERGE COMMIT HASH>-PR-<PR NUM>
|
||||
#
|
||||
# For branches, it's just the git branch tip hash.
|
||||
|
||||
# For pull requests, refs/pull/<PR NUM>/merge was just checked out
|
||||
# by so HEAD will refer to the right revision. For branches, HEAD
|
||||
# is also the tip of the branch.
|
||||
REV=$(git rev-parse HEAD)
|
||||
|
||||
# We can get the PR number from the "context".
|
||||
#
|
||||
# https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request
|
||||
#
|
||||
# (via <https://github.community/t/github-ref-is-inconsistent/17728/3>).
|
||||
#
|
||||
# If this is a pull request, `github.event` is a `pull_request`
|
||||
# structure which has `number` right in it.
|
||||
#
|
||||
# If this is a push, `github.event` is a `push` instead but we only
|
||||
# need the revision to construct the build_num.
|
||||
|
||||
PR=${{ github.event.number }}
|
||||
|
||||
if [ "${PR}" = "" ]; then
|
||||
BUILD_NUM=$REV
|
||||
else
|
||||
BUILD_NUM=$REV-PR-$PR
|
||||
fi
|
||||
REPO_NAME=$GITHUB_REPOSITORY
|
||||
|
||||
curl \
|
||||
-k \
|
||||
https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \
|
||||
-d \
|
||||
"payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME"
|
||||
env:
|
||||
# Some magic value required for some magic reason.
|
||||
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||
# Help coveralls identify our project.
|
||||
COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
|
||||
|
||||
integration:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@ -6,7 +6,7 @@ Free and Open decentralized data store
|
||||
|
||||
`Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.
|
||||
|
||||
|Contributor Covenant| |readthedocs| |travis| |circleci| |codecov|
|
||||
|Contributor Covenant| |readthedocs| |travis| |circleci| |coveralls|
|
||||
|
||||
|
||||
Table of contents
|
||||
@ -125,9 +125,9 @@ See `TGPPL.PDF <https://tahoe-lafs.org/~zooko/tgppl.pdf>`__ for why the TGPPL ex
|
||||
.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
|
||||
:target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
|
||||
|
||||
.. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master
|
||||
:alt: test coverage percentage
|
||||
:target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master
|
||||
.. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg
|
||||
:alt: code coverage
|
||||
:target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs
|
||||
|
||||
.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg
|
||||
:alt: code of conduct
|
||||
|
||||
1
newsfragments/3326.installation
Normal file
1
newsfragments/3326.installation
Normal file
@ -0,0 +1 @@
|
||||
Debian 8 support has been replaced with Debian 10 support.
|
||||
0
newsfragments/3385.minor
Normal file
0
newsfragments/3385.minor
Normal file
1
newsfragments/3588.incompat
Normal file
1
newsfragments/3588.incompat
Normal file
@ -0,0 +1 @@
|
||||
The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale.
|
||||
0
newsfragments/3592.minor
Normal file
0
newsfragments/3592.minor
Normal file
0
newsfragments/3600.minor
Normal file
0
newsfragments/3600.minor
Normal file
0
newsfragments/3607.minor
Normal file
0
newsfragments/3607.minor
Normal file
0
newsfragments/3612.minor
Normal file
0
newsfragments/3612.minor
Normal file
@ -74,6 +74,13 @@ ADD_FILE = ActionType(
|
||||
u"Add a new file as a child of a directory.",
|
||||
)
|
||||
|
||||
|
||||
class _OnlyFiles(object):
|
||||
"""Marker for replacement option of only replacing files."""
|
||||
|
||||
ONLY_FILES = _OnlyFiles()
|
||||
|
||||
|
||||
def update_metadata(metadata, new_metadata, now):
|
||||
"""Updates 'metadata' in-place with the information in 'new_metadata'.
|
||||
|
||||
@ -175,11 +182,16 @@ class MetadataSetter(object):
|
||||
|
||||
class Adder(object):
|
||||
def __init__(self, node, entries=None, overwrite=True, create_readonly_node=None):
|
||||
"""
|
||||
:param overwrite: Either True (allow overwriting anything existing),
|
||||
False (don't allow overwriting), or ONLY_FILES (only files can be
|
||||
overwritten).
|
||||
"""
|
||||
self.node = node
|
||||
if entries is None:
|
||||
entries = {}
|
||||
precondition(isinstance(entries, dict), entries)
|
||||
precondition(overwrite in (True, False, "only-files"), overwrite)
|
||||
precondition(overwrite in (True, False, ONLY_FILES), overwrite)
|
||||
# keys of 'entries' may not be normalized.
|
||||
self.entries = entries
|
||||
self.overwrite = overwrite
|
||||
@ -205,7 +217,7 @@ class Adder(object):
|
||||
if not self.overwrite:
|
||||
raise ExistingChildError("child %s already exists" % quote_output(name, encoding='utf-8'))
|
||||
|
||||
if self.overwrite == "only-files" and IDirectoryNode.providedBy(children[name][0]):
|
||||
if self.overwrite == ONLY_FILES and IDirectoryNode.providedBy(children[name][0]):
|
||||
raise ExistingChildError("child %s already exists as a directory" % quote_output(name, encoding='utf-8'))
|
||||
metadata = children[name][1].copy()
|
||||
|
||||
@ -701,7 +713,7 @@ class DirectoryNode(object):
|
||||
'new_child_namex' and 'current_child_namex' need not be normalized.
|
||||
|
||||
The overwrite parameter may be True (overwrite any existing child),
|
||||
False (error if the new child link already exists), or "only-files"
|
||||
False (error if the new child link already exists), or ONLY_FILES
|
||||
(error if the new child link exists and points to a directory).
|
||||
"""
|
||||
if self.is_readonly() or new_parent.is_readonly():
|
||||
|
||||
@ -501,7 +501,7 @@ def list_aliases(options):
|
||||
rc = tahoe_add_alias.list_aliases(options)
|
||||
return rc
|
||||
|
||||
def list(options):
|
||||
def list_(options):
|
||||
from allmydata.scripts import tahoe_ls
|
||||
rc = tahoe_ls.list(options)
|
||||
return rc
|
||||
@ -587,7 +587,7 @@ dispatch = {
|
||||
"add-alias": add_alias,
|
||||
"create-alias": create_alias,
|
||||
"list-aliases": list_aliases,
|
||||
"ls": list,
|
||||
"ls": list_,
|
||||
"get": get,
|
||||
"put": put,
|
||||
"cp": cp,
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
from ...util.encodingutil import unicode_to_argv
|
||||
from six import ensure_str
|
||||
|
||||
from ...scripts import runner
|
||||
from ..common_util import ReallyEqualMixin, run_cli, run_cli_unicode
|
||||
|
||||
@ -45,6 +46,12 @@ class CLITestMixin(ReallyEqualMixin):
|
||||
# client_num is used to execute client CLI commands on a specific
|
||||
# client.
|
||||
client_num = kwargs.pop("client_num", 0)
|
||||
client_dir = unicode_to_argv(self.get_clientdir(i=client_num))
|
||||
# If we were really going to launch a child process then
|
||||
# `unicode_to_argv` would be the right thing to do here. However,
|
||||
# we're just going to call some Python functions directly and those
|
||||
# Python functions want native strings. So ignore the requirements
|
||||
# for passing arguments to another process and make sure this argument
|
||||
# is a native string.
|
||||
client_dir = ensure_str(self.get_clientdir(i=client_num))
|
||||
nodeargs = [ b"--node-directory", client_dir ]
|
||||
return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
|
||||
|
||||
@ -99,22 +99,6 @@ class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
def test_list_latin_1(self):
|
||||
"""
|
||||
An alias composed of all Latin-1-encodeable code points can be created
|
||||
when the active encoding is Latin-1.
|
||||
|
||||
This is very similar to ``test_list_utf_8`` but the assumption of
|
||||
UTF-8 is nearly ubiquitous and explicitly exercising the codepaths
|
||||
with a UTF-8-incompatible encoding helps flush out unintentional UTF-8
|
||||
assumptions.
|
||||
"""
|
||||
return self._check_create_alias(
|
||||
u"taho\N{LATIN SMALL LETTER E WITH ACUTE}",
|
||||
encoding="latin-1",
|
||||
)
|
||||
|
||||
|
||||
def test_list_utf_8(self):
|
||||
"""
|
||||
An alias composed of all UTF-8-encodeable code points can be created when
|
||||
|
||||
@ -7,7 +7,7 @@ from allmydata.scripts.common import get_aliases
|
||||
from allmydata.scripts import cli
|
||||
from ..no_network import GridTestMixin
|
||||
from ..common_util import skip_if_cannot_represent_filename
|
||||
from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv
|
||||
from allmydata.util.encodingutil import get_io_encoding
|
||||
from allmydata.util.fileutil import abspath_expanduser_unicode
|
||||
from .common import CLITestMixin
|
||||
|
||||
@ -46,21 +46,21 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||
self.basedir = "cli/Put/unlinked_immutable_from_file"
|
||||
self.set_up_grid(oneshare=True)
|
||||
|
||||
rel_fn = os.path.join(self.basedir, "DATAFILE")
|
||||
abs_fn = unicode_to_argv(abspath_expanduser_unicode(unicode(rel_fn)))
|
||||
rel_fn = unicode(os.path.join(self.basedir, "DATAFILE"))
|
||||
abs_fn = abspath_expanduser_unicode(rel_fn)
|
||||
# we make the file small enough to fit in a LIT file, for speed
|
||||
fileutil.write(rel_fn, "short file")
|
||||
d = self.do_cli("put", rel_fn)
|
||||
d = self.do_cli_unicode(u"put", [rel_fn])
|
||||
def _uploaded(args):
|
||||
(rc, out, err) = args
|
||||
readcap = out
|
||||
self.failUnless(readcap.startswith("URI:LIT:"), readcap)
|
||||
self.readcap = readcap
|
||||
d.addCallback(_uploaded)
|
||||
d.addCallback(lambda res: self.do_cli("put", "./" + rel_fn))
|
||||
d.addCallback(lambda res: self.do_cli_unicode(u"put", [u"./" + rel_fn]))
|
||||
d.addCallback(lambda rc_stdout_stderr:
|
||||
self.failUnlessReallyEqual(rc_stdout_stderr[1], self.readcap))
|
||||
d.addCallback(lambda res: self.do_cli("put", abs_fn))
|
||||
d.addCallback(lambda res: self.do_cli_unicode(u"put", [abs_fn]))
|
||||
d.addCallback(lambda rc_stdout_stderr:
|
||||
self.failUnlessReallyEqual(rc_stdout_stderr[1], self.readcap))
|
||||
# we just have to assume that ~ is handled properly
|
||||
|
||||
@ -9,10 +9,15 @@ __all__ = [
|
||||
"flush_logged_errors",
|
||||
"skip",
|
||||
"skipIf",
|
||||
|
||||
# Selected based on platform and re-exported for convenience.
|
||||
"Popen",
|
||||
"PIPE",
|
||||
]
|
||||
|
||||
from past.builtins import chr as byteschr, unicode
|
||||
|
||||
import sys
|
||||
import os, random, struct
|
||||
import six
|
||||
import tempfile
|
||||
@ -101,6 +106,21 @@ from .eliotutil import (
|
||||
)
|
||||
from .common_util import ShouldFailMixin # noqa: F401
|
||||
|
||||
if sys.platform == "win32":
|
||||
# Python 2.7 doesn't have good options for launching a process with
|
||||
# non-ASCII in its command line. So use this alternative that does a
|
||||
# better job. However, only use it on Windows because it doesn't work
|
||||
# anywhere else.
|
||||
from ._win_subprocess import (
|
||||
Popen,
|
||||
)
|
||||
else:
|
||||
from subprocess import (
|
||||
Popen,
|
||||
)
|
||||
from subprocess import (
|
||||
PIPE,
|
||||
)
|
||||
|
||||
TEST_RSA_KEY_SIZE = 522
|
||||
|
||||
|
||||
@ -6,29 +6,43 @@ Tools aimed at the interaction between tests and Eliot.
|
||||
# Can't use `builtins.str` because it's not JSON encodable:
|
||||
# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
|
||||
from past.builtins import unicode as str
|
||||
from future.utils import PY3
|
||||
from future.utils import PY2
|
||||
from six import ensure_text
|
||||
|
||||
__all__ = [
|
||||
"RUN_TEST",
|
||||
"EliotLoggedRunTest",
|
||||
"eliot_logged_test",
|
||||
]
|
||||
|
||||
try:
|
||||
from typing import Callable
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from functools import (
|
||||
wraps,
|
||||
partial,
|
||||
wraps,
|
||||
)
|
||||
|
||||
import attr
|
||||
|
||||
from zope.interface import (
|
||||
implementer,
|
||||
)
|
||||
|
||||
from eliot import (
|
||||
ActionType,
|
||||
Field,
|
||||
MemoryLogger,
|
||||
ILogger,
|
||||
)
|
||||
from eliot.testing import (
|
||||
swap_logger,
|
||||
check_for_errors,
|
||||
)
|
||||
from eliot.testing import capture_logging
|
||||
|
||||
from twisted.internet.defer import (
|
||||
maybeDeferred,
|
||||
from twisted.python.monkey import (
|
||||
MonkeyPatcher,
|
||||
)
|
||||
|
||||
from ..util.jsonbytes import BytesJSONEncoder
|
||||
@ -48,92 +62,12 @@ RUN_TEST = ActionType(
|
||||
)
|
||||
|
||||
|
||||
def eliot_logged_test(f):
|
||||
"""
|
||||
Decorate a test method to run in a dedicated Eliot action context.
|
||||
|
||||
The action will finish after the test is done (after the returned Deferred
|
||||
fires, if a Deferred is returned). It will note the name of the test
|
||||
being run.
|
||||
|
||||
All messages emitted by the test will be validated. They will still be
|
||||
delivered to the global logger.
|
||||
"""
|
||||
# A convenient, mutable container into which nested functions can write
|
||||
# state to be shared among them.
|
||||
class storage(object):
|
||||
pass
|
||||
|
||||
|
||||
# On Python 3, we want to use our custom JSON encoder when validating
|
||||
# messages can be encoded to JSON:
|
||||
if PY3:
|
||||
capture = lambda f : capture_logging(None, encoder_=BytesJSONEncoder)(f)
|
||||
# On Python 3, we want to use our custom JSON encoder when validating messages
|
||||
# can be encoded to JSON:
|
||||
if PY2:
|
||||
_memory_logger = MemoryLogger
|
||||
else:
|
||||
capture = lambda f : capture_logging(None)(f)
|
||||
|
||||
@wraps(f)
|
||||
def run_and_republish(self, *a, **kw):
|
||||
# Unfortunately the only way to get at the global/default logger...
|
||||
# This import is delayed here so that we get the *current* default
|
||||
# logger at the time the decorated function is run.
|
||||
from eliot._output import _DEFAULT_LOGGER as default_logger
|
||||
|
||||
def republish():
|
||||
# This is called as a cleanup function after capture_logging has
|
||||
# restored the global/default logger to its original state. We
|
||||
# can now emit messages that go to whatever global destinations
|
||||
# are installed.
|
||||
|
||||
# storage.logger.serialize() seems like it would make more sense
|
||||
# than storage.logger.messages here. However, serialize()
|
||||
# explodes, seemingly as a result of double-serializing the logged
|
||||
# messages. I don't understand this.
|
||||
for msg in storage.logger.messages:
|
||||
default_logger.write(msg)
|
||||
|
||||
# And now that we've re-published all of the test's messages, we
|
||||
# can finish the test's action.
|
||||
storage.action.finish()
|
||||
|
||||
@capture
|
||||
def run(self, logger):
|
||||
# Record the MemoryLogger for later message extraction.
|
||||
storage.logger = logger
|
||||
# Give the test access to the logger as well. It would be just
|
||||
# fine to pass this as a keyword argument to `f` but implementing
|
||||
# that now will give me conflict headaches so I'm not doing it.
|
||||
self.eliot_logger = logger
|
||||
return f(self, *a, **kw)
|
||||
|
||||
# Arrange for all messages written to the memory logger that
|
||||
# `capture_logging` installs to be re-written to the global/default
|
||||
# logger so they might end up in a log file somewhere, if someone
|
||||
# wants. This has to be done in a cleanup function (or later) because
|
||||
# capture_logging restores the original logger in a cleanup function.
|
||||
# We install our cleanup function here, before we call run, so that it
|
||||
# runs *after* the cleanup function capture_logging installs (cleanup
|
||||
# functions are a stack).
|
||||
self.addCleanup(republish)
|
||||
|
||||
# Begin an action that should comprise all messages from the decorated
|
||||
# test method.
|
||||
with RUN_TEST(name=self.id()).context() as action:
|
||||
# When the test method Deferred fires, the RUN_TEST action is
|
||||
# done. However, we won't have re-published the MemoryLogger
|
||||
# messages into the global/default logger when this Deferred
|
||||
# fires. So we need to delay finishing the action until that has
|
||||
# happened. Record the action so we can do that.
|
||||
storage.action = action
|
||||
|
||||
# Support both Deferred-returning and non-Deferred-returning
|
||||
# tests.
|
||||
d = maybeDeferred(run, self)
|
||||
|
||||
# Let the test runner do its thing.
|
||||
return d
|
||||
|
||||
return run_and_republish
|
||||
_memory_logger = lambda: MemoryLogger(encoder=BytesJSONEncoder)
|
||||
|
||||
|
||||
@attr.s
|
||||
@ -174,10 +108,91 @@ class EliotLoggedRunTest(object):
|
||||
def id(self):
|
||||
return self.case.id()
|
||||
|
||||
@eliot_logged_test
|
||||
def run(self, result=None):
|
||||
def run(self, result):
|
||||
"""
|
||||
Run the test case in the context of a distinct Eliot action.
|
||||
|
||||
The action will finish after the test is done. It will note the name of
|
||||
the test being run.
|
||||
|
||||
All messages emitted by the test will be validated. They will still be
|
||||
delivered to the global logger.
|
||||
"""
|
||||
# The idea here is to decorate the test method itself so that all of
|
||||
# the extra logic happens at the point where test/application logic is
|
||||
# expected to be. This `run` method is more like test infrastructure
|
||||
# and things do not go well when we add too much extra behavior here.
|
||||
# For example, exceptions raised here often just kill the whole
|
||||
# runner.
|
||||
patcher = MonkeyPatcher()
|
||||
|
||||
# So, grab the test method.
|
||||
name = self.case._testMethodName
|
||||
original = getattr(self.case, name)
|
||||
decorated = with_logging(ensure_text(self.case.id()), original)
|
||||
patcher.addPatch(self.case, name, decorated)
|
||||
try:
|
||||
# Patch it in
|
||||
patcher.patch()
|
||||
# Then use the rest of the machinery to run it.
|
||||
return self._run_tests_with_factory(
|
||||
self.case,
|
||||
self.handlers,
|
||||
self.last_resort,
|
||||
).run(result)
|
||||
finally:
|
||||
# Clean up the patching for idempotency or something.
|
||||
patcher.restore()
|
||||
|
||||
|
||||
def with_logging(
|
||||
test_id, # type: str
|
||||
test_method, # type: Callable
|
||||
):
|
||||
"""
|
||||
Decorate a test method with additional log-related behaviors.
|
||||
|
||||
1. The test method will run in a distinct Eliot action.
|
||||
2. Typed log messages will be validated.
|
||||
3. Logged tracebacks will be added as errors.
|
||||
|
||||
:param test_id: The full identifier of the test being decorated.
|
||||
:param test_method: The method itself.
|
||||
"""
|
||||
@wraps(test_method)
|
||||
def run_with_logging(*args, **kwargs):
|
||||
validating_logger = _memory_logger()
|
||||
original = swap_logger(None)
|
||||
try:
|
||||
swap_logger(_TwoLoggers(original, validating_logger))
|
||||
with RUN_TEST(name=test_id):
|
||||
try:
|
||||
return test_method(*args, **kwargs)
|
||||
finally:
|
||||
check_for_errors(validating_logger)
|
||||
finally:
|
||||
swap_logger(original)
|
||||
return run_with_logging
|
||||
|
||||
|
||||
@implementer(ILogger)
|
||||
class _TwoLoggers(object):
|
||||
"""
|
||||
Log to two loggers.
|
||||
|
||||
A single logger can have multiple destinations so this isn't typically a
|
||||
useful thing to do. However, MemoryLogger has inline validation instead
|
||||
of destinations. That means this *is* useful to simultaneously write to
|
||||
the normal places and validate all written log messages.
|
||||
"""
|
||||
def __init__(self, a, b):
|
||||
"""
|
||||
:param ILogger a: One logger
|
||||
:param ILogger b: Another logger
|
||||
"""
|
||||
self._a = a # type: ILogger
|
||||
self._b = b # type: ILogger
|
||||
|
||||
def write(self, dictionary, serializer=None):
|
||||
self._a.write(dictionary, serializer)
|
||||
self._b.write(dictionary, serializer)
|
||||
|
||||
@ -1978,12 +1978,12 @@ class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
|
||||
overwrite=False))
|
||||
d.addCallback(lambda res:
|
||||
root_node.set_node(u'file1', filenode,
|
||||
overwrite="only-files"))
|
||||
overwrite=dirnode.ONLY_FILES))
|
||||
d.addCallback(lambda res:
|
||||
self.shouldFail(ExistingChildError, "set_node",
|
||||
"child 'dir1' already exists",
|
||||
root_node.set_node, u'dir1', filenode,
|
||||
overwrite="only-files"))
|
||||
overwrite=dirnode.ONLY_FILES))
|
||||
return d
|
||||
|
||||
d.addCallback(_test_adder)
|
||||
|
||||
@ -18,17 +18,25 @@ if PY2:
|
||||
from sys import stdout
|
||||
import logging
|
||||
|
||||
from unittest import (
|
||||
skip,
|
||||
)
|
||||
|
||||
from fixtures import (
|
||||
TempDir,
|
||||
)
|
||||
from testtools import (
|
||||
TestCase,
|
||||
)
|
||||
from testtools import (
|
||||
TestResult,
|
||||
)
|
||||
from testtools.matchers import (
|
||||
Is,
|
||||
IsInstance,
|
||||
MatchesStructure,
|
||||
Equals,
|
||||
HasLength,
|
||||
AfterPreprocessing,
|
||||
)
|
||||
from testtools.twistedsupport import (
|
||||
@ -38,12 +46,16 @@ from testtools.twistedsupport import (
|
||||
|
||||
from eliot import (
|
||||
Message,
|
||||
MessageType,
|
||||
fields,
|
||||
FileDestination,
|
||||
MemoryLogger,
|
||||
)
|
||||
from eliot.twisted import DeferredContext
|
||||
from eliot.testing import (
|
||||
capture_logging,
|
||||
assertHasAction,
|
||||
swap_logger,
|
||||
)
|
||||
|
||||
from twisted.internet.defer import (
|
||||
@ -173,6 +185,62 @@ class EliotLoggingTests(TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_validation_failure(self):
|
||||
"""
|
||||
If a test emits a log message that fails validation then an error is added
|
||||
to the result.
|
||||
"""
|
||||
# Make sure we preserve the original global Eliot state.
|
||||
original = swap_logger(MemoryLogger())
|
||||
self.addCleanup(lambda: swap_logger(original))
|
||||
|
||||
class ValidationFailureProbe(SyncTestCase):
|
||||
def test_bad_message(self):
|
||||
# This message does not validate because "Hello" is not an
|
||||
# int.
|
||||
MSG = MessageType("test:eliotutil", fields(foo=int))
|
||||
MSG(foo="Hello").write()
|
||||
|
||||
result = TestResult()
|
||||
case = ValidationFailureProbe("test_bad_message")
|
||||
case.run(result)
|
||||
|
||||
self.assertThat(
|
||||
result.errors,
|
||||
HasLength(1),
|
||||
)
|
||||
|
||||
def test_skip_cleans_up(self):
|
||||
"""
|
||||
After a skipped test the global Eliot logging state is restored.
|
||||
"""
|
||||
# Save the logger that's active before we do anything so that we can
|
||||
# restore it later. Also install another logger so we can compare it
|
||||
# to the active logger later.
|
||||
expected = MemoryLogger()
|
||||
original = swap_logger(expected)
|
||||
|
||||
# Restore it, whatever else happens.
|
||||
self.addCleanup(lambda: swap_logger(original))
|
||||
|
||||
class SkipProbe(SyncTestCase):
|
||||
@skip("It's a skip test.")
|
||||
def test_skipped(self):
|
||||
pass
|
||||
|
||||
case = SkipProbe("test_skipped")
|
||||
case.run()
|
||||
|
||||
# Retrieve the logger that's active now that the skipped test is done
|
||||
# so we can check it against the expected value.
|
||||
actual = swap_logger(MemoryLogger())
|
||||
self.assertThat(
|
||||
actual,
|
||||
Is(expected),
|
||||
)
|
||||
|
||||
|
||||
|
||||
class LogCallDeferredTests(TestCase):
|
||||
"""
|
||||
Tests for ``log_call_deferred``.
|
||||
|
||||
@ -70,7 +70,7 @@ if __name__ == "__main__":
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
import os, sys, locale
|
||||
import os, sys
|
||||
from unittest import skipIf
|
||||
|
||||
from twisted.trial import unittest
|
||||
@ -81,99 +81,28 @@ from allmydata.test.common_util import (
|
||||
ReallyEqualMixin, skip_if_cannot_represent_filename,
|
||||
)
|
||||
from allmydata.util import encodingutil, fileutil
|
||||
from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \
|
||||
from allmydata.util.encodingutil import unicode_to_url, \
|
||||
unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \
|
||||
quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \
|
||||
get_io_encoding, get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \
|
||||
get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \
|
||||
to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \
|
||||
unicode_to_argv
|
||||
from twisted.python import usage
|
||||
|
||||
|
||||
class MockStdout(object):
|
||||
pass
|
||||
|
||||
class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
|
||||
def test_get_io_encoding(self):
|
||||
mock_stdout = MockStdout()
|
||||
self.patch(sys, 'stdout', mock_stdout)
|
||||
|
||||
mock_stdout.encoding = 'UTF-8'
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(get_io_encoding(), 'utf-8')
|
||||
|
||||
mock_stdout.encoding = 'cp65001'
|
||||
_reload()
|
||||
self.assertEqual(get_io_encoding(), 'utf-8')
|
||||
|
||||
mock_stdout.encoding = 'koi8-r'
|
||||
expected = sys.platform == "win32" and 'utf-8' or 'koi8-r'
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(get_io_encoding(), expected)
|
||||
|
||||
mock_stdout.encoding = 'nonexistent_encoding'
|
||||
if sys.platform == "win32":
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(get_io_encoding(), 'utf-8')
|
||||
else:
|
||||
self.failUnlessRaises(AssertionError, _reload)
|
||||
|
||||
def test_get_io_encoding_not_from_stdout(self):
|
||||
preferredencoding = 'koi8-r'
|
||||
def call_locale_getpreferredencoding():
|
||||
return preferredencoding
|
||||
self.patch(locale, 'getpreferredencoding', call_locale_getpreferredencoding)
|
||||
mock_stdout = MockStdout()
|
||||
self.patch(sys, 'stdout', mock_stdout)
|
||||
|
||||
expected = sys.platform == "win32" and 'utf-8' or 'koi8-r'
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(get_io_encoding(), expected)
|
||||
|
||||
mock_stdout.encoding = None
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(get_io_encoding(), expected)
|
||||
|
||||
preferredencoding = None
|
||||
_reload()
|
||||
self.assertEqual(get_io_encoding(), 'utf-8')
|
||||
|
||||
def test_argv_to_unicode(self):
|
||||
encodingutil.io_encoding = 'utf-8'
|
||||
self.failUnlessRaises(usage.UsageError,
|
||||
argv_to_unicode,
|
||||
lumiere_nfc.encode('latin1'))
|
||||
|
||||
@skipIf(PY3, "Python 2 only.")
|
||||
def test_unicode_to_output(self):
|
||||
encodingutil.io_encoding = 'koi8-r'
|
||||
self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, lumiere_nfc)
|
||||
|
||||
def test_no_unicode_normalization(self):
|
||||
# Pretend to run on a Unicode platform.
|
||||
# listdir_unicode normalized to NFC in 1.7beta, but now doesn't.
|
||||
|
||||
def call_os_listdir(path):
|
||||
return [Artonwall_nfd]
|
||||
self.patch(os, 'listdir', call_os_listdir)
|
||||
self.patch(sys, 'platform', 'darwin')
|
||||
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
|
||||
|
||||
|
||||
# The following tests apply only to platforms that don't store filenames as
|
||||
# Unicode entities on the filesystem.
|
||||
class EncodingUtilNonUnicodePlatform(unittest.TestCase):
|
||||
@skipIf(PY3, "Python 3 is always Unicode, regardless of OS.")
|
||||
def setUp(self):
|
||||
# Mock sys.platform because unicode_platform() uses it
|
||||
self.original_platform = sys.platform
|
||||
sys.platform = 'linux'
|
||||
# Make sure everything goes back to the way it was at the end of the
|
||||
# test.
|
||||
self.addCleanup(_reload)
|
||||
|
||||
def tearDown(self):
|
||||
sys.platform = self.original_platform
|
||||
_reload()
|
||||
# Mock sys.platform because unicode_platform() uses it. Cleanups run
|
||||
# in reverse order so we do this second so it gets undone first.
|
||||
self.patch(sys, "platform", "linux")
|
||||
|
||||
def test_listdir_unicode(self):
|
||||
# What happens if latin1-encoded filenames are encountered on an UTF-8
|
||||
@ -206,25 +135,8 @@ class EncodingUtilNonUnicodePlatform(unittest.TestCase):
|
||||
|
||||
class EncodingUtil(ReallyEqualMixin):
|
||||
def setUp(self):
|
||||
self.original_platform = sys.platform
|
||||
sys.platform = self.platform
|
||||
|
||||
def tearDown(self):
|
||||
sys.platform = self.original_platform
|
||||
_reload()
|
||||
|
||||
def test_argv_to_unicode(self):
|
||||
if 'argv' not in dir(self):
|
||||
return
|
||||
|
||||
mock_stdout = MockStdout()
|
||||
mock_stdout.encoding = self.io_encoding
|
||||
self.patch(sys, 'stdout', mock_stdout)
|
||||
|
||||
argu = lumiere_nfc
|
||||
argv = self.argv
|
||||
_reload()
|
||||
self.failUnlessReallyEqual(argv_to_unicode(argv), argu)
|
||||
self.addCleanup(_reload)
|
||||
self.patch(sys, "platform", self.platform)
|
||||
|
||||
def test_unicode_to_url(self):
|
||||
self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re")
|
||||
@ -245,15 +157,19 @@ class EncodingUtil(ReallyEqualMixin):
|
||||
def test_unicode_to_output_py3(self):
|
||||
self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), lumiere_nfc)
|
||||
|
||||
@skipIf(PY3, "Python 2 only.")
|
||||
def test_unicode_to_argv_py2(self):
|
||||
"""unicode_to_argv() converts to bytes on Python 2."""
|
||||
self.assertEqual(unicode_to_argv("abc"), u"abc".encode(self.io_encoding))
|
||||
def test_unicode_to_argv(self):
|
||||
"""
|
||||
unicode_to_argv() returns its unicode argument on Windows and Python 2 and
|
||||
converts to bytes using UTF-8 elsewhere.
|
||||
"""
|
||||
result = unicode_to_argv(lumiere_nfc)
|
||||
if PY3 or self.platform == "win32":
|
||||
expected_value = lumiere_nfc
|
||||
else:
|
||||
expected_value = lumiere_nfc.encode(self.io_encoding)
|
||||
|
||||
@skipIf(PY2, "Python 3 only.")
|
||||
def test_unicode_to_argv_py3(self):
|
||||
"""unicode_to_argv() is noop on Python 3."""
|
||||
self.assertEqual(unicode_to_argv("abc"), "abc")
|
||||
self.assertIsInstance(result, type(expected_value))
|
||||
self.assertEqual(result, expected_value)
|
||||
|
||||
@skipIf(PY3, "Python 3 only.")
|
||||
def test_unicode_platform_py2(self):
|
||||
@ -463,13 +379,6 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase):
|
||||
check(u"\n", u"\"\\x0a\"", quote_newlines=True)
|
||||
|
||||
def test_quote_output_default(self):
|
||||
self.patch(encodingutil, 'io_encoding', 'ascii')
|
||||
self.test_quote_output_ascii(None)
|
||||
|
||||
self.patch(encodingutil, 'io_encoding', 'latin1')
|
||||
self.test_quote_output_latin1(None)
|
||||
|
||||
self.patch(encodingutil, 'io_encoding', 'utf-8')
|
||||
self.test_quote_output_utf8(None)
|
||||
|
||||
|
||||
@ -581,14 +490,6 @@ class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase):
|
||||
io_encoding = 'UTF-8'
|
||||
dirlist = [b'test_file', b'\xc3\x84rtonwall.mp3', b'Blah blah.txt']
|
||||
|
||||
class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase):
|
||||
uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
|
||||
argv = b'lumi\xe8re'
|
||||
platform = 'linux2'
|
||||
filesystem_encoding = 'ISO-8859-1'
|
||||
io_encoding = 'ISO-8859-1'
|
||||
dirlist = [b'test_file', b'Blah blah.txt', b'\xc4rtonwall.mp3']
|
||||
|
||||
class Windows(EncodingUtil, unittest.TestCase):
|
||||
uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
|
||||
argv = b'lumi\xc3\xa8re'
|
||||
@ -605,20 +506,6 @@ class MacOSXLeopard(EncodingUtil, unittest.TestCase):
|
||||
io_encoding = 'UTF-8'
|
||||
dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
|
||||
|
||||
class MacOSXLeopard7bit(EncodingUtil, unittest.TestCase):
|
||||
uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
|
||||
platform = 'darwin'
|
||||
filesystem_encoding = 'utf-8'
|
||||
io_encoding = 'US-ASCII'
|
||||
dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
|
||||
|
||||
class OpenBSD(EncodingUtil, unittest.TestCase):
|
||||
uname = 'OpenBSD 4.1 GENERIC#187 i386 Intel(R) Celeron(R) CPU 2.80GHz ("GenuineIntel" 686-class)'
|
||||
platform = 'openbsd4'
|
||||
filesystem_encoding = '646'
|
||||
io_encoding = '646'
|
||||
# Oops, I cannot write filenames containing non-ascii characters
|
||||
|
||||
|
||||
class TestToFromStr(ReallyEqualMixin, unittest.TestCase):
|
||||
def test_to_bytes(self):
|
||||
|
||||
@ -126,6 +126,42 @@ class HashUtilTests(unittest.TestCase):
|
||||
base32.a2b(b"2ckv3dfzh6rgjis6ogfqhyxnzy"),
|
||||
)
|
||||
|
||||
def test_convergence_hasher_tag(self):
|
||||
"""
|
||||
``_convergence_hasher_tag`` constructs the convergence hasher tag from a
|
||||
unique prefix, the required, total, and segment size parameters, and a
|
||||
convergence secret.
|
||||
"""
|
||||
self.assertEqual(
|
||||
b"allmydata_immutable_content_to_key_with_added_secret_v1+"
|
||||
b"16:\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42,"
|
||||
b"9:3,10,1024,",
|
||||
hashutil._convergence_hasher_tag(
|
||||
k=3,
|
||||
n=10,
|
||||
segsize=1024,
|
||||
convergence=b"\x42" * 16,
|
||||
),
|
||||
)
|
||||
|
||||
def test_convergence_hasher_out_of_bounds(self):
|
||||
"""
|
||||
``_convergence_hasher_tag`` raises ``ValueError`` if k or n is not between
|
||||
1 and 256 inclusive or if k is greater than n.
|
||||
"""
|
||||
segsize = 1024
|
||||
secret = b"\x42" * 16
|
||||
for bad_k in (0, 2, 257):
|
||||
with self.assertRaises(ValueError):
|
||||
hashutil._convergence_hasher_tag(
|
||||
k=bad_k, n=1, segsize=segsize, convergence=secret,
|
||||
)
|
||||
for bad_n in (0, 1, 257):
|
||||
with self.assertRaises(ValueError):
|
||||
hashutil._convergence_hasher_tag(
|
||||
k=2, n=bad_n, segsize=segsize, convergence=secret,
|
||||
)
|
||||
|
||||
def test_known_answers(self):
|
||||
"""
|
||||
Verify backwards compatibility by comparing hash outputs for some
|
||||
|
||||
@ -6,6 +6,10 @@ from __future__ import (
|
||||
import os.path, re, sys
|
||||
from os import linesep
|
||||
|
||||
from eliot import (
|
||||
log_call,
|
||||
)
|
||||
|
||||
from twisted.trial import unittest
|
||||
|
||||
from twisted.internet import reactor
|
||||
@ -19,22 +23,25 @@ from twisted.python.runtime import (
|
||||
platform,
|
||||
)
|
||||
from allmydata.util import fileutil, pollmixin
|
||||
from allmydata.util.encodingutil import unicode_to_argv, unicode_to_output
|
||||
from allmydata.util.encodingutil import unicode_to_argv, get_filesystem_encoding
|
||||
from allmydata.test import common_util
|
||||
import allmydata
|
||||
from .common_util import parse_cli, run_cli
|
||||
from .common import (
|
||||
PIPE,
|
||||
Popen,
|
||||
)
|
||||
from .common_util import (
|
||||
parse_cli,
|
||||
run_cli,
|
||||
)
|
||||
from .cli_node_api import (
|
||||
CLINodeAPI,
|
||||
Expect,
|
||||
on_stdout,
|
||||
on_stdout_and_stderr,
|
||||
)
|
||||
from ._twisted_9607 import (
|
||||
getProcessOutputAndValue,
|
||||
)
|
||||
from ..util.eliotutil import (
|
||||
inline_callbacks,
|
||||
log_call_deferred,
|
||||
)
|
||||
|
||||
def get_root_from_file(src):
|
||||
@ -54,93 +61,92 @@ srcfile = allmydata.__file__
|
||||
rootdir = get_root_from_file(srcfile)
|
||||
|
||||
|
||||
class RunBinTahoeMixin(object):
|
||||
@log_call_deferred(action_type="run-bin-tahoe")
|
||||
def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
|
||||
command = sys.executable
|
||||
argv = python_options + ["-m", "allmydata.scripts.runner"] + args
|
||||
@log_call(action_type="run-bin-tahoe")
|
||||
def run_bintahoe(extra_argv, python_options=None):
|
||||
"""
|
||||
Run the main Tahoe entrypoint in a child process with the given additional
|
||||
arguments.
|
||||
|
||||
if env is None:
|
||||
env = os.environ
|
||||
:param [unicode] extra_argv: More arguments for the child process argv.
|
||||
|
||||
d = getProcessOutputAndValue(command, argv, env, stdinBytes=stdin)
|
||||
def fix_signal(result):
|
||||
# Mirror subprocess.Popen.returncode structure
|
||||
(out, err, signal) = result
|
||||
return (out, err, -signal)
|
||||
d.addErrback(fix_signal)
|
||||
return d
|
||||
:return: A three-tuple of stdout (unicode), stderr (unicode), and the
|
||||
child process "returncode" (int).
|
||||
"""
|
||||
argv = [sys.executable.decode(get_filesystem_encoding())]
|
||||
if python_options is not None:
|
||||
argv.extend(python_options)
|
||||
argv.extend([u"-m", u"allmydata.scripts.runner"])
|
||||
argv.extend(extra_argv)
|
||||
argv = list(unicode_to_argv(arg) for arg in argv)
|
||||
p = Popen(argv, stdout=PIPE, stderr=PIPE)
|
||||
out = p.stdout.read().decode("utf-8")
|
||||
err = p.stderr.read().decode("utf-8")
|
||||
returncode = p.wait()
|
||||
return (out, err, returncode)
|
||||
|
||||
|
||||
class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin):
|
||||
class BinTahoe(common_util.SignalMixin, unittest.TestCase):
|
||||
def test_unicode_arguments_and_output(self):
|
||||
"""
|
||||
The runner script receives unmangled non-ASCII values in argv.
|
||||
"""
|
||||
tricky = u"\u2621"
|
||||
try:
|
||||
tricky_arg = unicode_to_argv(tricky, mangle=True)
|
||||
tricky_out = unicode_to_output(tricky)
|
||||
except UnicodeEncodeError:
|
||||
raise unittest.SkipTest("A non-ASCII argument/output could not be encoded on this platform.")
|
||||
out, err, returncode = run_bintahoe([tricky])
|
||||
self.assertEqual(returncode, 1)
|
||||
self.assertIn(u"Unknown command: " + tricky, out)
|
||||
|
||||
d = self.run_bintahoe([tricky_arg])
|
||||
def _cb(res):
|
||||
out, err, rc_or_sig = res
|
||||
self.failUnlessEqual(rc_or_sig, 1, str(res))
|
||||
self.failUnlessIn("Unknown command: "+tricky_out, out)
|
||||
d.addCallback(_cb)
|
||||
return d
|
||||
def test_with_python_options(self):
|
||||
"""
|
||||
Additional options for the Python interpreter don't prevent the runner
|
||||
script from receiving the arguments meant for it.
|
||||
"""
|
||||
# This seems like a redundant test for someone else's functionality
|
||||
# but on Windows we parse the whole command line string ourselves so
|
||||
# we have to have our own implementation of skipping these options.
|
||||
|
||||
def test_run_with_python_options(self):
|
||||
# -t is a harmless option that warns about tabs.
|
||||
d = self.run_bintahoe(["--version"], python_options=["-t"])
|
||||
def _cb(res):
|
||||
out, err, rc_or_sig = res
|
||||
self.assertEqual(rc_or_sig, 0, str(res))
|
||||
self.assertTrue(out.startswith(allmydata.__appname__ + '/'), str(res))
|
||||
d.addCallback(_cb)
|
||||
return d
|
||||
# -t is a harmless option that warns about tabs so we can add it
|
||||
# without impacting other behavior noticably.
|
||||
out, err, returncode = run_bintahoe([u"--version"], python_options=[u"-t"])
|
||||
self.assertEqual(returncode, 0)
|
||||
self.assertTrue(out.startswith(allmydata.__appname__ + '/'))
|
||||
|
||||
@inlineCallbacks
|
||||
def test_help_eliot_destinations(self):
|
||||
out, err, rc_or_sig = yield self.run_bintahoe(["--help-eliot-destinations"])
|
||||
self.assertIn("\tfile:<path>", out)
|
||||
self.assertEqual(rc_or_sig, 0)
|
||||
out, err, returncode = run_bintahoe([u"--help-eliot-destinations"])
|
||||
self.assertIn(u"\tfile:<path>", out)
|
||||
self.assertEqual(returncode, 0)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_eliot_destination(self):
|
||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
||||
out, err, returncode = run_bintahoe([
|
||||
# Proves little but maybe more than nothing.
|
||||
"--eliot-destination=file:-",
|
||||
u"--eliot-destination=file:-",
|
||||
# Throw in *some* command or the process exits with error, making
|
||||
# it difficult for us to see if the previous arg was accepted or
|
||||
# not.
|
||||
"--help",
|
||||
u"--help",
|
||||
])
|
||||
self.assertEqual(rc_or_sig, 0)
|
||||
self.assertEqual(returncode, 0)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_unknown_eliot_destination(self):
|
||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
||||
"--eliot-destination=invalid:more",
|
||||
out, err, returncode = run_bintahoe([
|
||||
u"--eliot-destination=invalid:more",
|
||||
])
|
||||
self.assertEqual(1, rc_or_sig)
|
||||
self.assertIn("Unknown destination description", out)
|
||||
self.assertIn("invalid:more", out)
|
||||
self.assertEqual(1, returncode)
|
||||
self.assertIn(u"Unknown destination description", out)
|
||||
self.assertIn(u"invalid:more", out)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_malformed_eliot_destination(self):
|
||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
||||
"--eliot-destination=invalid",
|
||||
out, err, returncode = run_bintahoe([
|
||||
u"--eliot-destination=invalid",
|
||||
])
|
||||
self.assertEqual(1, rc_or_sig)
|
||||
self.assertIn("must be formatted like", out)
|
||||
self.assertEqual(1, returncode)
|
||||
self.assertIn(u"must be formatted like", out)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_escape_in_eliot_destination(self):
|
||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
||||
"--eliot-destination=file:@foo",
|
||||
out, err, returncode = run_bintahoe([
|
||||
u"--eliot-destination=file:@foo",
|
||||
])
|
||||
self.assertEqual(1, rc_or_sig)
|
||||
self.assertIn("Unsupported escape character", out)
|
||||
self.assertEqual(1, returncode)
|
||||
self.assertIn(u"Unsupported escape character", out)
|
||||
|
||||
|
||||
class CreateNode(unittest.TestCase):
|
||||
@ -250,8 +256,7 @@ class CreateNode(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
|
||||
RunBinTahoeMixin):
|
||||
class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
|
||||
"""
|
||||
exercise "tahoe run" for both introducer and client node, by spawning
|
||||
"tahoe run" as a subprocess. This doesn't get us line-level coverage, but
|
||||
@ -271,18 +276,18 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
|
||||
The introducer furl is stable across restarts.
|
||||
"""
|
||||
basedir = self.workdir("test_introducer")
|
||||
c1 = os.path.join(basedir, "c1")
|
||||
c1 = os.path.join(basedir, u"c1")
|
||||
tahoe = CLINodeAPI(reactor, FilePath(c1))
|
||||
self.addCleanup(tahoe.stop_and_wait)
|
||||
|
||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
||||
"--quiet",
|
||||
"create-introducer",
|
||||
"--basedir", c1,
|
||||
"--hostname", "127.0.0.1",
|
||||
out, err, returncode = run_bintahoe([
|
||||
u"--quiet",
|
||||
u"create-introducer",
|
||||
u"--basedir", c1,
|
||||
u"--hostname", u"127.0.0.1",
|
||||
])
|
||||
|
||||
self.assertEqual(rc_or_sig, 0)
|
||||
self.assertEqual(returncode, 0)
|
||||
|
||||
# This makes sure that node.url is written, which allows us to
|
||||
# detect when the introducer restarts in _node_has_restarted below.
|
||||
@ -350,18 +355,18 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
|
||||
3) Verify that the pid file is removed after SIGTERM (on POSIX).
|
||||
"""
|
||||
basedir = self.workdir("test_client")
|
||||
c1 = os.path.join(basedir, "c1")
|
||||
c1 = os.path.join(basedir, u"c1")
|
||||
|
||||
tahoe = CLINodeAPI(reactor, FilePath(c1))
|
||||
# Set this up right now so we don't forget later.
|
||||
self.addCleanup(tahoe.cleanup)
|
||||
|
||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
||||
"--quiet", "create-node", "--basedir", c1,
|
||||
"--webport", "0",
|
||||
"--hostname", "localhost",
|
||||
out, err, returncode = run_bintahoe([
|
||||
u"--quiet", u"create-node", u"--basedir", c1,
|
||||
u"--webport", u"0",
|
||||
u"--hostname", u"localhost",
|
||||
])
|
||||
self.failUnlessEqual(rc_or_sig, 0)
|
||||
self.failUnlessEqual(returncode, 0)
|
||||
|
||||
# Check that the --webport option worked.
|
||||
config = fileutil.read(tahoe.config_file.path)
|
||||
|
||||
@ -51,6 +51,10 @@ from twisted.python.filepath import (
|
||||
FilePath,
|
||||
)
|
||||
|
||||
from ._twisted_9607 import (
|
||||
getProcessOutputAndValue,
|
||||
)
|
||||
|
||||
from .common import (
|
||||
TEST_RSA_KEY_SIZE,
|
||||
SameProcessStreamEndpointAssigner,
|
||||
@ -61,13 +65,32 @@ from .web.common import (
|
||||
)
|
||||
|
||||
# TODO: move this to common or common_util
|
||||
from allmydata.test.test_runner import RunBinTahoeMixin
|
||||
from . import common_util as testutil
|
||||
from .common_util import run_cli_unicode
|
||||
from ..scripts.common import (
|
||||
write_introducer,
|
||||
)
|
||||
|
||||
class RunBinTahoeMixin(object):
|
||||
def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
|
||||
# test_runner.run_bintahoe has better unicode support but doesn't
|
||||
# support env yet and is also synchronous. If we could get rid of
|
||||
# this in favor of that, though, it would probably be an improvement.
|
||||
command = sys.executable
|
||||
argv = python_options + ["-m", "allmydata.scripts.runner"] + args
|
||||
|
||||
if env is None:
|
||||
env = os.environ
|
||||
|
||||
d = getProcessOutputAndValue(command, argv, env, stdinBytes=stdin)
|
||||
def fix_signal(result):
|
||||
# Mirror subprocess.Popen.returncode structure
|
||||
(out, err, signal) = result
|
||||
return (out, err, -signal)
|
||||
d.addErrback(fix_signal)
|
||||
return d
|
||||
|
||||
|
||||
def run_cli(*args, **kwargs):
|
||||
"""
|
||||
Run a Tahoe-LAFS CLI utility, but inline.
|
||||
|
||||
@ -29,11 +29,6 @@ from json import (
|
||||
from textwrap import (
|
||||
dedent,
|
||||
)
|
||||
from subprocess import (
|
||||
PIPE,
|
||||
Popen,
|
||||
)
|
||||
|
||||
from twisted.python.filepath import (
|
||||
FilePath,
|
||||
)
|
||||
@ -66,6 +61,8 @@ from hypothesis.strategies import (
|
||||
)
|
||||
|
||||
from .common import (
|
||||
PIPE,
|
||||
Popen,
|
||||
SyncTestCase,
|
||||
)
|
||||
|
||||
@ -132,13 +129,6 @@ class GetArgvTests(SyncTestCase):
|
||||
``get_argv`` returns a list representing the result of tokenizing the
|
||||
"command line" argument string provided to Windows processes.
|
||||
"""
|
||||
# Python 2.7 doesn't have good options for launching a process with
|
||||
# non-ASCII in its command line. So use this alternative that does a
|
||||
# better job. Bury the import here because it only works on Windows.
|
||||
from ._win_subprocess import (
|
||||
Popen
|
||||
)
|
||||
|
||||
working_path = FilePath(self.mktemp())
|
||||
working_path.makedirs()
|
||||
save_argv_path = working_path.child("script.py")
|
||||
|
||||
@ -12,17 +12,18 @@ if PY2:
|
||||
|
||||
from twisted.trial import unittest
|
||||
from allmydata.web import status, common
|
||||
from allmydata.dirnode import ONLY_FILES
|
||||
from ..common import ShouldFailMixin
|
||||
from .. import common_util as testutil
|
||||
|
||||
class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
|
||||
|
||||
def test_parse_replace_arg(self):
|
||||
self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
|
||||
self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
|
||||
self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
|
||||
"only-files")
|
||||
self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
|
||||
self.failUnlessReallyEqual(common.parse_replace_arg(b"true"), True)
|
||||
self.failUnlessReallyEqual(common.parse_replace_arg(b"false"), False)
|
||||
self.failUnlessReallyEqual(common.parse_replace_arg(b"only-files"),
|
||||
ONLY_FILES)
|
||||
self.failUnlessRaises(common.WebError, common.parse_replace_arg, b"only_fles")
|
||||
|
||||
def test_abbreviate_time(self):
|
||||
self.failUnlessReallyEqual(common.abbreviate_time(None), "")
|
||||
|
||||
@ -115,6 +115,7 @@ PORTED_MODULES = [
|
||||
"allmydata.util.spans",
|
||||
"allmydata.util.statistics",
|
||||
"allmydata.util.time_format",
|
||||
"allmydata.web.common",
|
||||
"allmydata.web.logs",
|
||||
"allmydata.webish",
|
||||
]
|
||||
|
||||
@ -18,8 +18,9 @@ if PY2:
|
||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
||||
|
||||
from past.builtins import unicode
|
||||
from six import ensure_str
|
||||
|
||||
import sys, os, re, locale
|
||||
import sys, os, re
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
@ -50,36 +51,25 @@ def check_encoding(encoding):
|
||||
try:
|
||||
u"test".encode(encoding)
|
||||
except (LookupError, AttributeError):
|
||||
raise AssertionError("The character encoding '%s' is not supported for conversion." % (encoding,))
|
||||
raise AssertionError(
|
||||
"The character encoding '%s' is not supported for conversion." % (encoding,),
|
||||
)
|
||||
|
||||
# On Windows we install UTF-8 stream wrappers for sys.stdout and
|
||||
# sys.stderr, and reencode the arguments as UTF-8 (see scripts/runner.py).
|
||||
#
|
||||
# On POSIX, we are moving towards a UTF-8-everything and ignore the locale.
|
||||
io_encoding = "utf-8"
|
||||
|
||||
filesystem_encoding = None
|
||||
io_encoding = None
|
||||
is_unicode_platform = False
|
||||
use_unicode_filepath = False
|
||||
|
||||
def _reload():
|
||||
global filesystem_encoding, io_encoding, is_unicode_platform, use_unicode_filepath
|
||||
global filesystem_encoding, is_unicode_platform, use_unicode_filepath
|
||||
|
||||
filesystem_encoding = canonical_encoding(sys.getfilesystemencoding())
|
||||
check_encoding(filesystem_encoding)
|
||||
|
||||
if sys.platform == 'win32':
|
||||
# On Windows we install UTF-8 stream wrappers for sys.stdout and
|
||||
# sys.stderr, and reencode the arguments as UTF-8 (see scripts/runner.py).
|
||||
io_encoding = 'utf-8'
|
||||
else:
|
||||
ioenc = None
|
||||
if hasattr(sys.stdout, 'encoding'):
|
||||
ioenc = sys.stdout.encoding
|
||||
if ioenc is None:
|
||||
try:
|
||||
ioenc = locale.getpreferredencoding()
|
||||
except Exception:
|
||||
pass # work around <http://bugs.python.org/issue1443504>
|
||||
io_encoding = canonical_encoding(ioenc)
|
||||
|
||||
check_encoding(io_encoding)
|
||||
|
||||
is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"]
|
||||
|
||||
# Despite the Unicode-mode FilePath support added to Twisted in
|
||||
@ -110,6 +100,8 @@ def get_io_encoding():
|
||||
def argv_to_unicode(s):
|
||||
"""
|
||||
Decode given argv element to unicode. If this fails, raise a UsageError.
|
||||
|
||||
This is the inverse of ``unicode_to_argv``.
|
||||
"""
|
||||
if isinstance(s, unicode):
|
||||
return s
|
||||
@ -133,26 +125,22 @@ def argv_to_abspath(s, **kwargs):
|
||||
% (quote_output(s), quote_output(os.path.join('.', s))))
|
||||
return abspath_expanduser_unicode(decoded, **kwargs)
|
||||
|
||||
|
||||
def unicode_to_argv(s, mangle=False):
|
||||
"""
|
||||
Encode the given Unicode argument as a bytestring.
|
||||
If the argument is to be passed to a different process, then the 'mangle' argument
|
||||
should be true; on Windows, this uses a mangled encoding that will be reversed by
|
||||
code in runner.py.
|
||||
Make the given unicode string suitable for use in an argv list.
|
||||
|
||||
On Python 3, just return the string unchanged, since argv is unicode.
|
||||
On Python 2 on POSIX, this encodes using UTF-8. On Python 3 and on
|
||||
Windows, this returns the input unmodified.
|
||||
"""
|
||||
precondition(isinstance(s, unicode), s)
|
||||
if PY3:
|
||||
warnings.warn("This will be unnecessary once Python 2 is dropped.",
|
||||
DeprecationWarning)
|
||||
if sys.platform == "win32":
|
||||
return s
|
||||
return ensure_str(s)
|
||||
|
||||
if mangle and sys.platform == "win32":
|
||||
# This must be the same as 'mangle' in bin/tahoe-script.template.
|
||||
return bytes(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s), io_encoding)
|
||||
else:
|
||||
return s.encode(io_encoding)
|
||||
|
||||
def unicode_to_url(s):
|
||||
"""
|
||||
|
||||
@ -176,10 +176,44 @@ def convergence_hash(k, n, segsize, data, convergence):
|
||||
return h.digest()
|
||||
|
||||
|
||||
def convergence_hasher(k, n, segsize, convergence):
|
||||
def _convergence_hasher_tag(k, n, segsize, convergence):
|
||||
"""
|
||||
Create the convergence hashing tag.
|
||||
|
||||
:param int k: Required shares (in [1..256]).
|
||||
:param int n: Total shares (in [1..256]).
|
||||
:param int segsize: Maximum segment size.
|
||||
:param bytes convergence: The convergence secret.
|
||||
|
||||
:return bytes: The bytestring to use as a tag in the convergence hash.
|
||||
"""
|
||||
assert isinstance(convergence, bytes)
|
||||
if k > n:
|
||||
raise ValueError(
|
||||
"k > n not allowed; k = {}, n = {}".format(k, n),
|
||||
)
|
||||
if k < 1 or n < 1:
|
||||
# It doesn't make sense to have zero shares. Zero shares carry no
|
||||
# information, cannot encode any part of the application data.
|
||||
raise ValueError(
|
||||
"k, n < 1 not allowed; k = {}, n = {}".format(k, n),
|
||||
)
|
||||
if k > 256 or n > 256:
|
||||
# ZFEC supports encoding application data into a maximum of 256
|
||||
# shares. If we ignore the limitations of ZFEC, it may be fine to use
|
||||
# a configuration with more shares than that and it may be fine to
|
||||
# construct a convergence tag from such a configuration. Since ZFEC
|
||||
# is the only supported encoder, though, this is moot for now.
|
||||
raise ValueError(
|
||||
"k, n > 256 not allowed; k = {}, n = {}".format(k, n),
|
||||
)
|
||||
param_tag = netstring(b"%d,%d,%d" % (k, n, segsize))
|
||||
tag = CONVERGENT_ENCRYPTION_TAG + netstring(convergence) + param_tag
|
||||
return tag
|
||||
|
||||
|
||||
def convergence_hasher(k, n, segsize, convergence):
|
||||
tag = _convergence_hasher_tag(k, n, segsize, convergence)
|
||||
return tagged_hasher(tag, KEYLEN)
|
||||
|
||||
|
||||
|
||||
@ -1,5 +1,22 @@
|
||||
from past.builtins import unicode
|
||||
from six import ensure_text, ensure_str
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
||||
from past.builtins import unicode as str # prevent leaking newbytes/newstr into code that can't handle it
|
||||
|
||||
from six import ensure_str
|
||||
|
||||
try:
|
||||
from typing import Optional, Union, Tuple, Any
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import time
|
||||
import json
|
||||
@ -51,6 +68,7 @@ from twisted.web.resource import (
|
||||
IResource,
|
||||
)
|
||||
|
||||
from allmydata.dirnode import ONLY_FILES, _OnlyFiles
|
||||
from allmydata import blacklist
|
||||
from allmydata.interfaces import (
|
||||
EmptyPathnameComponentError,
|
||||
@ -74,11 +92,13 @@ from allmydata.util.encodingutil import (
|
||||
quote_output,
|
||||
to_bytes,
|
||||
)
|
||||
from allmydata.util import abbreviate
|
||||
|
||||
# Originally part of this module, so still part of its API:
|
||||
from .common_py3 import ( # noqa: F401
|
||||
get_arg, abbreviate_time, MultiFormatResource, WebError,
|
||||
)
|
||||
|
||||
class WebError(Exception):
|
||||
def __init__(self, text, code=http.BAD_REQUEST):
|
||||
self.text = text
|
||||
self.code = code
|
||||
|
||||
|
||||
def get_filenode_metadata(filenode):
|
||||
@ -98,17 +118,17 @@ def get_filenode_metadata(filenode):
|
||||
metadata['size'] = size
|
||||
return metadata
|
||||
|
||||
def boolean_of_arg(arg):
|
||||
# TODO: ""
|
||||
arg = ensure_text(arg)
|
||||
if arg.lower() not in ("true", "t", "1", "false", "f", "0", "on", "off"):
|
||||
def boolean_of_arg(arg): # type: (bytes) -> bool
|
||||
assert isinstance(arg, bytes)
|
||||
if arg.lower() not in (b"true", b"t", b"1", b"false", b"f", b"0", b"on", b"off"):
|
||||
raise WebError("invalid boolean argument: %r" % (arg,), http.BAD_REQUEST)
|
||||
return arg.lower() in ("true", "t", "1", "on")
|
||||
return arg.lower() in (b"true", b"t", b"1", b"on")
|
||||
|
||||
def parse_replace_arg(replace):
|
||||
replace = ensure_text(replace)
|
||||
if replace.lower() == "only-files":
|
||||
return replace
|
||||
|
||||
def parse_replace_arg(replace): # type: (bytes) -> Union[bool,_OnlyFiles]
|
||||
assert isinstance(replace, bytes)
|
||||
if replace.lower() == b"only-files":
|
||||
return ONLY_FILES
|
||||
try:
|
||||
return boolean_of_arg(replace)
|
||||
except WebError:
|
||||
@ -145,19 +165,19 @@ def get_mutable_type(file_format): # accepts result of get_format()
|
||||
return None
|
||||
|
||||
|
||||
def parse_offset_arg(offset):
|
||||
def parse_offset_arg(offset): # type: (bytes) -> Union[int,None]
|
||||
# XXX: This will raise a ValueError when invoked on something that
|
||||
# is not an integer. Is that okay? Or do we want a better error
|
||||
# message? Since this call is going to be used by programmers and
|
||||
# their tools rather than users (through the wui), it is not
|
||||
# inconsistent to return that, I guess.
|
||||
if offset is not None:
|
||||
offset = int(offset)
|
||||
return int(offset)
|
||||
|
||||
return offset
|
||||
|
||||
|
||||
def get_root(req):
|
||||
def get_root(req): # type: (IRequest) -> str
|
||||
"""
|
||||
Get a relative path with parent directory segments that refers to the root
|
||||
location known to the given request. This seems a lot like the constant
|
||||
@ -186,8 +206,8 @@ def convert_children_json(nodemaker, children_json):
|
||||
children = {}
|
||||
if children_json:
|
||||
data = json.loads(children_json)
|
||||
for (namex, (ctype, propdict)) in data.items():
|
||||
namex = unicode(namex)
|
||||
for (namex, (ctype, propdict)) in list(data.items()):
|
||||
namex = str(namex)
|
||||
writecap = to_bytes(propdict.get("rw_uri"))
|
||||
readcap = to_bytes(propdict.get("ro_uri"))
|
||||
metadata = propdict.get("metadata", {})
|
||||
@ -208,7 +228,8 @@ def compute_rate(bytes, seconds):
|
||||
assert bytes > -1
|
||||
assert seconds > 0
|
||||
|
||||
return 1.0 * bytes / seconds
|
||||
return bytes / seconds
|
||||
|
||||
|
||||
def abbreviate_rate(data):
|
||||
"""
|
||||
@ -229,6 +250,7 @@ def abbreviate_rate(data):
|
||||
return u"%.1fkBps" % (r/1000)
|
||||
return u"%.0fBps" % r
|
||||
|
||||
|
||||
def abbreviate_size(data):
|
||||
"""
|
||||
Convert number of bytes into human readable strings (unicode).
|
||||
@ -265,7 +287,7 @@ def text_plain(text, req):
|
||||
return text
|
||||
|
||||
def spaces_to_nbsp(text):
|
||||
return unicode(text).replace(u' ', u'\u00A0')
|
||||
return str(text).replace(u' ', u'\u00A0')
|
||||
|
||||
def render_time_delta(time_1, time_2):
|
||||
return spaces_to_nbsp(format_delta(time_1, time_2))
|
||||
@ -283,7 +305,7 @@ def render_time_attr(t):
|
||||
# actual exception). The latter is growing increasingly annoying.
|
||||
|
||||
def should_create_intermediate_directories(req):
|
||||
t = unicode(get_arg(req, "t", "").strip(), "ascii")
|
||||
t = str(get_arg(req, "t", "").strip(), "ascii")
|
||||
return bool(req.method in (b"PUT", b"POST") and
|
||||
t not in ("delete", "rename", "rename-form", "check"))
|
||||
|
||||
@ -565,7 +587,7 @@ def _finish(result, render, request):
|
||||
resource=fullyQualifiedName(type(result)),
|
||||
)
|
||||
result.render(request)
|
||||
elif isinstance(result, unicode):
|
||||
elif isinstance(result, str):
|
||||
Message.log(
|
||||
message_type=u"allmydata:web:common-render:unicode",
|
||||
)
|
||||
@ -647,7 +669,7 @@ def _renderHTTP_exception(request, failure):
|
||||
def _renderHTTP_exception_simple(request, text, code):
|
||||
request.setResponseCode(code)
|
||||
request.setHeader("content-type", "text/plain;charset=utf-8")
|
||||
if isinstance(text, unicode):
|
||||
if isinstance(text, str):
|
||||
text = text.encode("utf-8")
|
||||
request.setHeader("content-length", b"%d" % len(text))
|
||||
return text
|
||||
@ -689,3 +711,124 @@ def url_for_string(req, url_string):
|
||||
port=port,
|
||||
)
|
||||
return url
|
||||
|
||||
|
||||
def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any]
|
||||
"""Extract an argument from either the query args (req.args) or the form
|
||||
body fields (req.fields). If multiple=False, this returns a single value
|
||||
(or the default, which defaults to None), and the query args take
|
||||
precedence. If multiple=True, this returns a tuple of arguments (possibly
|
||||
empty), starting with all those in the query args.
|
||||
|
||||
:param TahoeLAFSRequest req: The request to consider.
|
||||
|
||||
:return: Either bytes or tuple of bytes.
|
||||
"""
|
||||
if isinstance(argname, str):
|
||||
argname = argname.encode("utf-8")
|
||||
if isinstance(default, str):
|
||||
default = default.encode("utf-8")
|
||||
results = []
|
||||
if argname in req.args:
|
||||
results.extend(req.args[argname])
|
||||
argname_unicode = str(argname, "utf-8")
|
||||
if req.fields and argname_unicode in req.fields:
|
||||
value = req.fields[argname_unicode].value
|
||||
if isinstance(value, str):
|
||||
value = value.encode("utf-8")
|
||||
results.append(value)
|
||||
if multiple:
|
||||
return tuple(results)
|
||||
if results:
|
||||
return results[0]
|
||||
return default
|
||||
|
||||
|
||||
class MultiFormatResource(resource.Resource, object):
|
||||
"""
|
||||
``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
|
||||
a number of different formats.
|
||||
|
||||
Rendered format is controlled by a query argument (given by
|
||||
``self.formatArgument``). Different resources may support different
|
||||
formats but ``json`` is a pretty common one. ``html`` is the default
|
||||
format if nothing else is given as the ``formatDefault``.
|
||||
"""
|
||||
formatArgument = "t"
|
||||
formatDefault = None # type: Optional[str]
|
||||
|
||||
def render(self, req):
|
||||
"""
|
||||
Dispatch to a renderer for a particular format, as selected by a query
|
||||
argument.
|
||||
|
||||
A renderer for the format given by the query argument matching
|
||||
``formatArgument`` will be selected and invoked. render_HTML will be
|
||||
used as a default if no format is selected (either by query arguments
|
||||
or by ``formatDefault``).
|
||||
|
||||
:return: The result of the selected renderer.
|
||||
"""
|
||||
t = get_arg(req, self.formatArgument, self.formatDefault)
|
||||
# It's either bytes or None.
|
||||
if isinstance(t, bytes):
|
||||
t = str(t, "ascii")
|
||||
renderer = self._get_renderer(t)
|
||||
result = renderer(req)
|
||||
# On Python 3, json.dumps() returns Unicode for example, but
|
||||
# twisted.web expects bytes. Instead of updating every single render
|
||||
# method, just handle Unicode one time here.
|
||||
if isinstance(result, str):
|
||||
result = result.encode("utf-8")
|
||||
return result
|
||||
|
||||
def _get_renderer(self, fmt):
|
||||
"""
|
||||
Get the renderer for the indicated format.
|
||||
|
||||
:param str fmt: The format. If a method with a prefix of ``render_``
|
||||
and a suffix of this format (upper-cased) is found, it will be
|
||||
used.
|
||||
|
||||
:return: A callable which takes a twisted.web Request and renders a
|
||||
response.
|
||||
"""
|
||||
renderer = None
|
||||
|
||||
if fmt is not None:
|
||||
try:
|
||||
renderer = getattr(self, "render_{}".format(fmt.upper()))
|
||||
except AttributeError:
|
||||
return resource.ErrorPage(
|
||||
http.BAD_REQUEST,
|
||||
"Bad Format",
|
||||
"Unknown {} value: {!r}".format(self.formatArgument, fmt),
|
||||
).render
|
||||
|
||||
if renderer is None:
|
||||
renderer = self.render_HTML
|
||||
|
||||
return renderer
|
||||
|
||||
|
||||
def abbreviate_time(data):
|
||||
"""
|
||||
Convert number of seconds into human readable string.
|
||||
|
||||
:param data: Either ``None`` or integer or float, seconds.
|
||||
|
||||
:return: Unicode string.
|
||||
"""
|
||||
# 1.23s, 790ms, 132us
|
||||
if data is None:
|
||||
return u""
|
||||
s = float(data)
|
||||
if s >= 10:
|
||||
return abbreviate.abbreviate_time(data)
|
||||
if s >= 1.0:
|
||||
return u"%.2fs" % s
|
||||
if s >= 0.01:
|
||||
return u"%.0fms" % (1000*s)
|
||||
if s >= 0.001:
|
||||
return u"%.1fms" % (1000*s)
|
||||
return u"%.0fus" % (1000000*s)
|
||||
|
||||
@ -1,143 +0,0 @@
|
||||
"""
|
||||
Common utilities that are available from Python 3.
|
||||
|
||||
Can eventually be merged back into allmydata.web.common.
|
||||
"""
|
||||
|
||||
from past.builtins import unicode
|
||||
|
||||
try:
|
||||
from typing import Optional
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from twisted.web import resource, http
|
||||
|
||||
from allmydata.util import abbreviate
|
||||
|
||||
|
||||
class WebError(Exception):
|
||||
def __init__(self, text, code=http.BAD_REQUEST):
|
||||
self.text = text
|
||||
self.code = code
|
||||
|
||||
|
||||
def get_arg(req, argname, default=None, multiple=False):
|
||||
"""Extract an argument from either the query args (req.args) or the form
|
||||
body fields (req.fields). If multiple=False, this returns a single value
|
||||
(or the default, which defaults to None), and the query args take
|
||||
precedence. If multiple=True, this returns a tuple of arguments (possibly
|
||||
empty), starting with all those in the query args.
|
||||
|
||||
:param TahoeLAFSRequest req: The request to consider.
|
||||
|
||||
:return: Either bytes or tuple of bytes.
|
||||
"""
|
||||
if isinstance(argname, unicode):
|
||||
argname = argname.encode("utf-8")
|
||||
if isinstance(default, unicode):
|
||||
default = default.encode("utf-8")
|
||||
results = []
|
||||
if argname in req.args:
|
||||
results.extend(req.args[argname])
|
||||
argname_unicode = unicode(argname, "utf-8")
|
||||
if req.fields and argname_unicode in req.fields:
|
||||
value = req.fields[argname_unicode].value
|
||||
if isinstance(value, unicode):
|
||||
value = value.encode("utf-8")
|
||||
results.append(value)
|
||||
if multiple:
|
||||
return tuple(results)
|
||||
if results:
|
||||
return results[0]
|
||||
return default
|
||||
|
||||
|
||||
class MultiFormatResource(resource.Resource, object):
|
||||
"""
|
||||
``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
|
||||
a number of different formats.
|
||||
|
||||
Rendered format is controlled by a query argument (given by
|
||||
``self.formatArgument``). Different resources may support different
|
||||
formats but ``json`` is a pretty common one. ``html`` is the default
|
||||
format if nothing else is given as the ``formatDefault``.
|
||||
"""
|
||||
formatArgument = "t"
|
||||
formatDefault = None # type: Optional[str]
|
||||
|
||||
def render(self, req):
|
||||
"""
|
||||
Dispatch to a renderer for a particular format, as selected by a query
|
||||
argument.
|
||||
|
||||
A renderer for the format given by the query argument matching
|
||||
``formatArgument`` will be selected and invoked. render_HTML will be
|
||||
used as a default if no format is selected (either by query arguments
|
||||
or by ``formatDefault``).
|
||||
|
||||
:return: The result of the selected renderer.
|
||||
"""
|
||||
t = get_arg(req, self.formatArgument, self.formatDefault)
|
||||
# It's either bytes or None.
|
||||
if isinstance(t, bytes):
|
||||
t = unicode(t, "ascii")
|
||||
renderer = self._get_renderer(t)
|
||||
result = renderer(req)
|
||||
# On Python 3, json.dumps() returns Unicode for example, but
|
||||
# twisted.web expects bytes. Instead of updating every single render
|
||||
# method, just handle Unicode one time here.
|
||||
if isinstance(result, unicode):
|
||||
result = result.encode("utf-8")
|
||||
return result
|
||||
|
||||
def _get_renderer(self, fmt):
|
||||
"""
|
||||
Get the renderer for the indicated format.
|
||||
|
||||
:param str fmt: The format. If a method with a prefix of ``render_``
|
||||
and a suffix of this format (upper-cased) is found, it will be
|
||||
used.
|
||||
|
||||
:return: A callable which takes a twisted.web Request and renders a
|
||||
response.
|
||||
"""
|
||||
renderer = None
|
||||
|
||||
if fmt is not None:
|
||||
try:
|
||||
renderer = getattr(self, "render_{}".format(fmt.upper()))
|
||||
except AttributeError:
|
||||
return resource.ErrorPage(
|
||||
http.BAD_REQUEST,
|
||||
"Bad Format",
|
||||
"Unknown {} value: {!r}".format(self.formatArgument, fmt),
|
||||
).render
|
||||
|
||||
if renderer is None:
|
||||
renderer = self.render_HTML
|
||||
|
||||
return renderer
|
||||
|
||||
|
||||
def abbreviate_time(data):
|
||||
"""
|
||||
Convert number of seconds into human readable string.
|
||||
|
||||
:param data: Either ``None`` or integer or float, seconds.
|
||||
|
||||
:return: Unicode string.
|
||||
"""
|
||||
# 1.23s, 790ms, 132us
|
||||
if data is None:
|
||||
return u""
|
||||
s = float(data)
|
||||
if s >= 10:
|
||||
return abbreviate.abbreviate_time(data)
|
||||
if s >= 1.0:
|
||||
return u"%.2fs" % s
|
||||
if s >= 0.01:
|
||||
return u"%.0fms" % (1000*s)
|
||||
if s >= 0.001:
|
||||
return u"%.1fms" % (1000*s)
|
||||
return u"%.0fus" % (1000000*s)
|
||||
@ -9,7 +9,7 @@ from twisted.web.template import (
|
||||
renderer,
|
||||
renderElement
|
||||
)
|
||||
from allmydata.web.common_py3 import (
|
||||
from allmydata.web.common import (
|
||||
abbreviate_time,
|
||||
MultiFormatResource
|
||||
)
|
||||
|
||||
@ -13,7 +13,7 @@ from __future__ import print_function
|
||||
import sys
|
||||
assert sys.platform == "win32"
|
||||
|
||||
import codecs, re
|
||||
import codecs
|
||||
from functools import partial
|
||||
|
||||
from ctypes import WINFUNCTYPE, windll, POINTER, c_int, WinError, byref, get_last_error
|
||||
@ -174,37 +174,14 @@ def initialize():
|
||||
except Exception as e:
|
||||
_complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,))
|
||||
|
||||
# This works around <http://bugs.python.org/issue2128>.
|
||||
|
||||
# Because of <http://bugs.python.org/issue8775> (and similar limitations in
|
||||
# twisted), the 'bin/tahoe' script cannot invoke us with the actual Unicode arguments.
|
||||
# Instead it "mangles" or escapes them using \x7F as an escape character, which we
|
||||
# unescape here.
|
||||
def unmangle(s):
|
||||
return re.sub(
|
||||
u'\\x7F[0-9a-fA-F]*\\;',
|
||||
# type ignored for 'unichr' (Python 2 only)
|
||||
lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore
|
||||
s,
|
||||
)
|
||||
|
||||
argv_unicode = get_argv()
|
||||
try:
|
||||
argv = [unmangle(argv_u).encode('utf-8') for argv_u in argv_unicode]
|
||||
except Exception as e:
|
||||
_complain("%s: could not unmangle Unicode arguments.\n%r"
|
||||
% (sys.argv[0], argv_unicode))
|
||||
raise
|
||||
argv = list(arg.encode("utf-8") for arg in get_argv())
|
||||
|
||||
# Take only the suffix with the same number of arguments as sys.argv.
|
||||
# This accounts for anything that can cause initial arguments to be stripped,
|
||||
# for example, the Python interpreter or any options passed to it, or runner
|
||||
# scripts such as 'coverage run'. It works even if there are no such arguments,
|
||||
# as in the case of a frozen executable created by bb-freeze or similar.
|
||||
|
||||
sys.argv = argv[-len(sys.argv):]
|
||||
if sys.argv[0].endswith('.pyscript'):
|
||||
sys.argv[0] = sys.argv[0][:-9]
|
||||
|
||||
|
||||
def a_console(handle):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user