diff --git a/.appveyor.yml b/.appveyor.yml index 640701b77..f6efe785a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -51,6 +51,11 @@ test_script: # to put the Python version you want to use on PATH. - | %PYTHON%\Scripts\tox.exe -e coverage + %PYTHON%\Scripts\tox.exe -e pyinstaller + # To verify that the resultant PyInstaller-generated binary executes + # cleanly (i.e., that it terminates with an exit code of 0 and isn't + # failing due to import/packaging-related errors, etc.). + - dist\Tahoe-LAFS\tahoe.exe --version after_test: # This builds the main tahoe wheel, and wheels for all dependencies. @@ -64,7 +69,7 @@ after_test: %PYTHON%\python.exe setup.py bdist_wheel %PYTHON%\python.exe -m pip wheel -w dist . - | - %PYTHON%\python.exe -m pip install codecov coverage + %PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5" %PYTHON%\python.exe -m coverage xml -o coverage.xml -i %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.centos index 3b8827a35..febb61545 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.centos @@ -11,11 +11,11 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python \ - python-devel \ + python2 \ + python2-devel \ libffi-devel \ openssl-devel \ - libyaml-devel \ + libyaml \ /usr/bin/virtualenv \ net-tools @@ -23,4 +23,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 94d2a609c..de16148e0 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -23,7 +23,7 @@ RUN apt-get --quiet update && \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" # Only the integration tests currently need this but it doesn't hurt to always # have it present and it's simpler than building a whole extra image just for diff --git a/.circleci/Dockerfile.fedora b/.circleci/Dockerfile.fedora index 72e65a9f1..6ad22d676 100644 --- a/.circleci/Dockerfile.fedora +++ b/.circleci/Dockerfile.fedora @@ -23,4 +23,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" diff --git a/.circleci/Dockerfile.pypy b/.circleci/Dockerfile.pypy new file mode 100644 index 000000000..471611ecc --- /dev/null +++ b/.circleci/Dockerfile.pypy @@ -0,0 +1,23 @@ +FROM pypy:2.7-buster + +ENV WHEELHOUSE_PATH /tmp/wheelhouse +ENV VIRTUALENV_PATH /tmp/venv +# This will get updated by the CircleCI checkout step. +ENV BUILD_SRC_ROOT /tmp/project + +RUN apt-get --quiet update && \ + apt-get --quiet --yes install \ + git \ + lsb-release \ + sudo \ + build-essential \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + virtualenv + +# Get the project source. This is better than it seems. CircleCI will +# *update* this checkout on each job run, saving us more time per-job. +COPY . ${BUILD_SRC_ROOT} + +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "pypy" diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware index 495af9360..73ba6b32d 100644 --- a/.circleci/Dockerfile.slackware +++ b/.circleci/Dockerfile.slackware @@ -46,4 +46,4 @@ RUN slackpkg install \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index d1c2c26df..882dfe308 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -26,4 +26,4 @@ RUN apt-get --quiet update && \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" diff --git a/.circleci/config.yml b/.circleci/config.yml index 1173d0fc0..61ed12a5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,12 +21,18 @@ workflows: requires: - "fedora-29" - - "centos-7" + - "centos-8" - "slackware-14.2" + - "nixos-19.09" + + # Test against PyPy 2.7 + - "pypy2.7-buster" + # Other assorted tasks and configurations - "lint" + - "pyinstaller" - "deprecations" - "c-locale" # Any locale other than C or UTF-8. @@ -38,6 +44,10 @@ workflows: # integration tests. - "debian-9" + # Generate the underlying data for a visualization to aid with Python 3 + # porting. + - "build-porting-depgraph" + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -57,8 +67,9 @@ workflows: - "build-image-ubuntu-18.04" - "build-image-fedora-28" - "build-image-fedora-29" - - "build-image-centos-7" + - "build-image-centos-8" - "build-image-slackware-14.2" + - "build-image-pypy-2.7-buster" jobs: @@ -79,19 +90,48 @@ jobs: command: | ~/.local/bin/tox -e codechecks + pyinstaller: + docker: + - image: "circleci/python:2" + + steps: + - "checkout" + + - run: + name: "Install tox" + command: | + pip install --user tox + + - run: + name: "Make PyInstaller executable" + command: | + ~/.local/bin/tox -e pyinstaller + + - run: + # To verify that the resultant PyInstaller-generated binary executes + # cleanly (i.e., that it terminates with an exit code of 0 and isn't + # failing due to import/packaging-related errors, etc.). + name: "Test PyInstaller executable" + command: | + dist/Tahoe-LAFS/tahoe --version + debian-9: &DEBIAN docker: - image: "tahoelafsci/debian:9" user: "nobody" environment: &UTF_8_ENVIRONMENT + # In general, the test suite is not allowed to fail while the job + # succeeds. But you can set this to "yes" if you want it to be + # otherwise. + ALLOWED_FAILURE: "no" # Tell Hypothesis which configuration we want it to use. TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" # Tell the C runtime things about character encoding (mainly to do with # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "coverage" + TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -123,6 +163,7 @@ jobs: /tmp/project/.circleci/run-tests.sh \ "/tmp/venv" \ "/tmp/project" \ + "${ALLOWED_FAILURE}" \ "${ARTIFACTS_OUTPUT_PATH}" \ "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ "${TAHOE_LAFS_TOX_ARGS}" @@ -157,6 +198,18 @@ jobs: user: "nobody" + pypy2.7-buster: + <<: *DEBIAN + docker: + - image: "tahoelafsci/pypy:2.7-buster" + user: "nobody" + + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27-coverage" + ALLOWED_FAILURE: "yes" + + c-locale: <<: *DEBIAN @@ -216,9 +269,9 @@ jobs: user: "nobody" - centos-7: &RHEL_DERIV + centos-8: &RHEL_DERIV docker: - - image: "tahoelafsci/centos:7" + - image: "tahoelafsci/centos:8" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -271,6 +324,58 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE + nixos-19.09: + docker: + # Run in a highly Nix-capable environment. + - image: "nixorg/nix:circleci" + + environment: + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" + + steps: + - "checkout" + - "run": + name: "Build and Test" + command: | + # CircleCI build environment looks like it has a zillion and a + # half cores. Don't let Nix autodetect this high core count + # because it blows up memory usage and fails the test run. Pick a + # number of cores that suites the build environment we're paying + # for (the free one!). + # + # Also, let it run more than one job at a time because we have to + # build a couple simple little dependencies that don't take + # advantage of multiple cores and we get a little speedup by doing + # them in parallel. + nix-build --cores 3 --max-jobs 2 nix/ + + # Generate up-to-date data for the dependency graph visualizer. + build-porting-depgraph: + # Get a system in which we can easily install Tahoe-LAFS and all its + # dependencies. The dependency graph analyzer works by executing the code. + # It's Python, what do you expect? + <<: *DEBIAN + + steps: + - "checkout" + + - add_ssh_keys: + fingerprints: + # Jean-Paul Calderone (CircleCI depgraph key) + # This lets us push to tahoe-lafs/tahoe-depgraph in the next step. + - "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4" + + - run: + name: "Setup Python Environment" + command: | + /tmp/venv/bin/pip install -e /tmp/project + + - run: + name: "Generate dependency graph data" + command: | + . /tmp/venv/bin/activate + ./misc/python3/depgraph.sh + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on @@ -376,12 +481,12 @@ jobs: TAG: "18.04" - build-image-centos-7: + build-image-centos-8: <<: *BUILD_IMAGE environment: DISTRO: "centos" - TAG: "7" + TAG: "8" build-image-fedora-28: @@ -406,3 +511,11 @@ jobs: environment: DISTRO: "slackware" TAG: "14.2" + + + build-image-pypy-2.7-buster: + <<: *BUILD_IMAGE + + environment: + DISTRO: "pypy" + TAG: "2.7-buster" diff --git a/.circleci/create-virtualenv.sh b/.circleci/create-virtualenv.sh index a18fdf614..810ce5ae2 100755 --- a/.circleci/create-virtualenv.sh +++ b/.circleci/create-virtualenv.sh @@ -13,9 +13,14 @@ shift BOOTSTRAP_VENV="$1" shift +# The basename of the Python executable (found on PATH) that will be used with +# this image. This lets us create a virtualenv that uses the correct Python. +PYTHON="$1" +shift + # Set up the virtualenv as a non-root user so we can run the test suite as a # non-root user. See below. -virtualenv --python python2.7 "${BOOTSTRAP_VENV}" +virtualenv --python "${PYTHON}" "${BOOTSTRAP_VENV}" # For convenience. PIP="${BOOTSTRAP_VENV}/bin/pip" @@ -33,5 +38,12 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" "${PIP}" install certifi # Get a new, awesome version of pip and setuptools. For example, the -# distro-packaged virtualenv's pip may not know about wheels. -"${PIP}" install --upgrade pip setuptools wheel +# distro-packaged virtualenv's pip may not know about wheels. Get the newer +# version of pip *first* in case we have a really old one now which can't even +# install setuptools properly. +"${PIP}" install --upgrade pip + +# setuptools 45 requires Python 3.5 or newer. Even though we upgraded pip +# above, it may still not be able to get us a compatible version unless we +# explicitly ask for one. +"${PIP}" install --upgrade setuptools==44.0.0 wheel diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 80b684eba..75afb6f6f 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -40,7 +40,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ - "${PROJECT_ROOT}"[test,tor,i2p] \ + "${PROJECT_ROOT}"[test] \ ${BASIC_DEPS} \ ${TEST_DEPS} \ ${REPORTING_DEPS} diff --git a/.circleci/prepare-image.sh b/.circleci/prepare-image.sh index 271a7004d..842e5bbe2 100755 --- a/.circleci/prepare-image.sh +++ b/.circleci/prepare-image.sh @@ -18,6 +18,11 @@ shift PROJECT_ROOT="$1" shift +# The basename of the Python executable (found on PATH) that will be used with +# this image. This lets us create a virtualenv that uses the correct Python. +PYTHON="$1" +shift + "${PROJECT_ROOT}"/.circleci/fix-permissions.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}" -sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" +sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PYTHON}" sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/populate-wheelhouse.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}" diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index ba552dc5a..c26acdcbc 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -13,6 +13,9 @@ shift PROJECT_ROOT="$1" shift +ALLOWED_FAILURE="$1" +shift + ARTIFACTS=$1 shift @@ -40,6 +43,17 @@ else JUNITXML="" fi +# A prefix for the test command that ensure it will exit after no more than a +# certain amount of time. Ideally, we would only enforce a "silent" period +# timeout but there isn't obviously a ready-made tool for that. The test +# suite only takes about 5 - 6 minutes on CircleCI right now. 15 minutes +# seems like a moderately safe window. +# +# This is primarily aimed at catching hangs on the PyPy job which runs for +# about 21 minutes and then gets killed by CircleCI in a way that fails the +# job and bypasses our "allowed failure" logic. +TIMEOUT="timeout --kill-after 1m 15m" + # Run the test suite as a non-root user. This is the expected usage some # small areas of the test suite assume non-root privileges (such as unreadable # files being unreadable). @@ -54,14 +68,20 @@ export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors" export PIP_NO_INDEX="1" -${BOOTSTRAP_VENV}/bin/tox \ +if [ "${ALLOWED_FAILURE}" = "yes" ]; then + alternative="true" +else + alternative="false" +fi + +${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \ -c ${PROJECT_ROOT}/tox.ini \ --workdir /tmp/tahoe-lafs.tox \ -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ - ${TAHOE_LAFS_TOX_ARGS} + ${TAHOE_LAFS_TOX_ARGS} || "${alternative}" if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" + ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" fi diff --git a/.coveragerc b/.coveragerc index 4028f8ea0..eaac16d82 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,3 +8,5 @@ source = omit = */allmydata/test/* */allmydata/_version.py +parallel = True +branch = True diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..5d41a78b4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,172 @@ +name: CI + +on: + push: + branches: + - "master" + pull_request: + +jobs: + + coverage: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-latest + python-version: + - 2.7 + + steps: + + # Get vcpython27 on Windows + Python 2.7, to build zfec + # extension. See https://chocolatey.org/packages/vcpython27 and + # https://github.com/crazy-max/ghaction-chocolatey + - name: Install MSVC 9.0 for Python 2.7 [Windows] + if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install vcpython27 + + - name: Check out Tahoe-LAFS sources + uses: actions/checkout@v2 + + - name: Fetch all history for all tags and branches + run: git fetch --prune --unshallow + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python packages + run: | + pip install --upgrade codecov tox setuptools + pip list + + - name: Display tool versions + run: python misc/build_helpers/show-tool-versions.py + + - name: Run "tox -e coverage" + run: tox -e coverage + + - name: Upload eliot.log in case of failure + uses: actions/upload-artifact@v1 + if: failure() + with: + name: eliot.log + path: eliot.log + + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: abf679b6-e2e6-4b33-b7b5-6cfbd41ee691 + file: coverage.xml + + integration: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + python-version: + - 2.7 + + steps: + + - name: Install Tor [Ubuntu] + if: matrix.os == 'ubuntu-latest' + run: sudo apt install tor + + - name: Install Tor [macOS] + if: matrix.os == 'macos-latest' + run: brew install tor + + - name: Install Tor [Windows] + if: matrix.os == 'windows-latest' + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install tor + + - name: Install MSVC 9.0 for Python 2.7 [Windows] + if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install vcpython27 + + - name: Check out Tahoe-LAFS sources + uses: actions/checkout@v2 + + - name: Fetch all history for all tags and branches + run: git fetch --prune --unshallow + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python packages + run: | + pip install --upgrade tox + pip list + + - name: Display tool versions + run: python misc/build_helpers/show-tool-versions.py + + - name: Run "tox -e integration" + run: tox -e integration + + - name: Upload eliot.log in case of failure + uses: actions/upload-artifact@v1 + if: failure() + with: + name: integration.eliot.json + path: integration.eliot.json + + packaging: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-latest + - ubuntu-latest + python-version: + - 2.7 + + steps: + + # Get vcpython27 on Windows + Python 2.7, to build zfec + # extension. See https://chocolatey.org/packages/vcpython27 and + # https://github.com/crazy-max/ghaction-chocolatey + - name: Install MSVC 9.0 for Python 2.7 [Windows] + if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install vcpython27 + + - name: Check out Tahoe-LAFS sources + uses: actions/checkout@v2 + + - name: Fetch all history for all tags and branches + run: git fetch --prune --unshallow + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python packages + run: | + pip install --upgrade codecov tox setuptools + pip list + + - name: Display tool versions + run: python misc/build_helpers/show-tool-versions.py + + - name: Run "tox -e pyinstaller" + run: tox -e pyinstaller diff --git a/.gitignore b/.gitignore index bd76fc2ce..ee3b02b8f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,6 @@ zope.interface-*.egg /.tox/ /docs/_build/ /coverage.xml -/smoke_magicfolder/ /.hypothesis/ # This is the plaintext of the private environment needed for some CircleCI diff --git a/.travis.yml b/.travis.yml index 1fdcac4f9..490e37b7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,10 +25,14 @@ script: - | set -eo pipefail if [ "${T}" = "py35" ]; then - python3 -m compileall -f . + python3 -m compileall -f -x tahoe-depgraph.py . else tox -e ${T} fi + # To verify that the resultant PyInstaller-generated binary executes + # cleanly (i.e., that it terminates with an exit code of 0 and isn't + # failing due to import/packaging-related errors, etc.). + if [ "${T}" = "pyinstaller" ]; then dist/Tahoe-LAFS/tahoe --version; fi after_success: - if [ "${T}" = "coverage" ]; then codecov; fi diff --git a/Makefile b/Makefile index 63d9db980..6e0009ddf 100644 --- a/Makefile +++ b/Makefile @@ -42,12 +42,6 @@ upload-osx-pkg: # echo not uploading tahoe-lafs-osx-pkg because this is not trunk but is branch \"${BB_BRANCH}\" ; \ # fi -.PHONY: smoketest -smoketest: - -python ./src/allmydata/test/check_magicfolder_smoke.py kill - -rm -rf smoke_magicfolder/ - python ./src/allmydata/test/check_magicfolder_smoke.py - # code coverage-based testing is disabled temporarily, as we switch to tox. # This will eventually be added to a tox environment. The following comments # and variable settings are retained as notes for that future effort. diff --git a/NEWS.rst b/NEWS.rst index 7525a4285..2ca67a1f6 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,115 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.14.0 (2020-03-11) +''''''''''''''''''''''''''' + +Features +-------- + +- Magic-Folders are now supported on macOS. (`#1432 `_) +- Add a "tox -e draftnews" which runs towncrier in draft mode (`#2942 `_) +- Fedora 29 is now tested as part of the project's continuous integration system. (`#2955 `_) +- The Magic-Folder frontend now emits structured, causal logs. This makes it easier for developers to make sense of its behavior and for users to submit useful debugging information alongside problem reports. (`#2972 `_) +- The `tahoe` CLI now accepts arguments for configuring structured logging messages which Tahoe-LAFS is being converted to emit. This change does not introduce any new defaults for on-filesystem logging. (`#2975 `_) +- The web API now publishes streaming Eliot logs via a token-protected WebSocket at /private/logs/v1. (`#3006 `_) +- End-to-end in-memory tests for websocket features (`#3041 `_) +- allmydata.interfaces.IFoolscapStoragePlugin has been introduced, an extension point for customizing the storage protocol. (`#3049 `_) +- Static storage server "announcements" in ``private/servers.yaml`` are now individually logged and ignored if they cannot be interpreted. (`#3051 `_) +- Storage servers can now be configured to load plugins for allmydata.interfaces.IFoolscapStoragePlugin and offer them to clients. (`#3053 `_) +- Storage clients can now be configured to load plugins for allmydata.interfaces.IFoolscapStoragePlugin and use them to negotiate with servers. (`#3054 `_) +- The [storage] configuration section now accepts a boolean *anonymous* item to enable or disable anonymous storage access. The default behavior remains unchanged. (`#3184 `_) +- Enable the helper when creating a node with `tahoe create-node --helper` (`#3235 `_) + + +Bug Fixes +--------- + +- refactor initialization code to be more async-friendly (`#2870 `_) +- Configuration-checking code wasn't being called due to indenting (`#2935 `_) +- refactor configuration handling out of Node into _Config (`#2936 `_) +- "tox -e codechecks" no longer dirties the working tree. (`#2941 `_) +- Updated the Tor release key, used by the integration tests. (`#2944 `_) +- `tahoe backup` no longer fails with an unhandled exception when it encounters a special file (device, fifo) in the backup source. (`#2950 `_) +- Magic-Folders now creates spurious conflict files in fewer cases. In particular, if files are added to the folder while a client is offline, that client will not create conflict files for all those new files when it starts up. (`#2965 `_) +- The confusing and misplaced sub-command group headings in `tahoe --help` output have been removed. (`#2976 `_) +- The Magic-Folder frontend is now more responsive to subtree changes on Windows. (`#2997 `_) +- remove ancient bundled jquery and d3, and the "dowload timeline" feature they support (`#3228 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS no longer makes start-up time assertions about the versions of its dependencies. It is the responsibility of the administrator of the installation to ensure the correct version of dependencies are supplied. (`#2749 `_) +- Tahoe-LAFS now depends on Twisted 16.6 or newer. (`#2957 `_) + + +Removed Features +---------------- + +- "tahoe rm", an old alias for "tahoe unlink", has been removed. (`#1827 `_) +- The direct dependencies on pyutil and zbase32 have been removed. (`#2098 `_) +- Untested and unmaintained code for running Tahoe-LAFS as a Windows service has been removed. (`#2239 `_) +- The redundant "pypywin32" dependency has been removed. (`#2392 `_) +- Fedora 27 is no longer tested as part of the project's continuous integration system. (`#2955 `_) +- "tahoe start", "tahoe daemonize", "tahoe restart", and "tahoe stop" are now deprecated in favor of using "tahoe run", possibly with a third-party process manager. (`#3273 `_) + + +Other Changes +------------- + +- Tahoe-LAFS now tests for PyPy compatibility on CI. (`#2479 `_) +- Tahoe-LAFS now requires Twisted 18.4.0 or newer. (`#2771 `_) +- Tahoe-LAFS now uses towncrier to maintain the NEWS file. (`#2908 `_) +- The release process document has been updated. (`#2920 `_) +- allmydata.test.test_system.SystemTest is now more reliable with respect to bound address collisions. (`#2933 `_) +- The Tox configuration has been fixed to work around a problem on Windows CI. (`#2956 `_) +- The PyInstaller CI job now works around a pip/pyinstaller incompatibility. (`#2958 `_) +- Some CI jobs for integration tests have been moved from TravisCI to CircleCI. (`#2959 `_) +- Several warnings from a new release of pyflakes have been fixed. (`#2960 `_) +- Some Slackware 14.2 continuous integration problems have been resolved. (`#2961 `_) +- Some macOS continuous integration failures have been fixed. (`#2962 `_) +- The NoNetworkGrid implementation has been somewhat improved. (`#2966 `_) +- A bug in the test suite for the create-alias command has been fixed. (`#2967 `_) +- The integration test suite has been updated to use pytest-twisted instead of deprecated pytest APIs. (`#2968 `_) +- The magic-folder integration test suite now performs more aggressive cleanup of the processes it launches. (`#2969 `_) +- The integration tests now correctly document the `--keep-tempdir` option. (`#2970 `_) +- A misuse of super() in the integration tests has been fixed. (`#2971 `_) +- Several utilities to facilitate the use of the Eliot causal logging library have been introduced. (`#2973 `_) +- The Windows CI configuration has been tweaked. (`#2974 `_) +- The Magic-Folder frontend has had additional logging improvements. (`#2977 `_) +- (`#2981 `_, `#2982 `_) +- Added a simple sytax checker so that once a file has reached python3 compatibility, it will not regress. (`#3001 `_) +- Converted all uses of the print statement to the print function in the ./misc/ directory. (`#3002 `_) +- The contributor guidelines are now linked from the GitHub pull request creation page. (`#3003 `_) +- Updated the testing code to use the print function instead of the print statement. (`#3008 `_) +- Replaced print statement with print fuction for all tahoe_* scripts. (`#3009 `_) +- Replaced all remaining instances of the print statement with the print function. (`#3010 `_) +- Replace StringIO imports with six.moves. (`#3011 `_) +- Updated all Python files to use PEP-3110 exception syntax for Python3 compatibility. (`#3013 `_) +- Update raise syntax for Python3 compatibility. (`#3014 `_) +- Updated instances of octal literals to use the format 0o123 for Python3 compatibility. (`#3015 `_) +- allmydata.test.no_network, allmydata.test.test_system, and allmydata.test.web.test_introducer are now more reliable with respect to bound address collisions. (`#3016 `_) +- Removed tuple unpacking from function and lambda definitions for Python3 compatibility. (`#3019 `_) +- Updated Python2 long numeric literals for Python3 compatibility. (`#3020 `_) +- CircleCI jobs are now faster as a result of pre-building configured Docker images for the CI jobs. (`#3024 `_) +- Removed used of backticks for "repr" for Python3 compatibility. (`#3027 `_) +- Updated string literal syntax for Python3 compatibility. (`#3028 `_) +- Updated CI to enforce Python3 syntax for entire repo. (`#3030 `_) +- Replaced pycryptopp with cryptography. (`#3031 `_) +- All old-style classes ported to new-style. (`#3042 `_) +- Whitelisted "/bin/mv" as command for codechecks performed by tox. This fixes a current warning and prevents future errors (for tox 4). (`#3043 `_) +- Progress towards Python 3 compatibility is now visible at . (`#3152 `_) +- Collect coverage information from integration tests (`#3234 `_) +- NixOS is now a supported Tahoe-LAFS platform. (`#3266 `_) + + +Misc/Other +---------- + +- `#1893 `_, `#2266 `_, `#2283 `_, `#2766 `_, `#2980 `_, `#2985 `_, `#2986 `_, `#2987 `_, `#2988 `_, `#2989 `_, `#2990 `_, `#2991 `_, `#2992 `_, `#2995 `_, `#3000 `_, `#3004 `_, `#3005 `_, `#3007 `_, `#3012 `_, `#3017 `_, `#3021 `_, `#3023 `_, `#3025 `_, `#3026 `_, `#3029 `_, `#3036 `_, `#3038 `_, `#3048 `_, `#3086 `_, `#3097 `_, `#3111 `_, `#3118 `_, `#3119 `_, `#3227 `_, `#3229 `_, `#3232 `_, `#3233 `_, `#3237 `_, `#3238 `_, `#3239 `_, `#3240 `_, `#3242 `_, `#3243 `_, `#3245 `_, `#3246 `_, `#3248 `_, `#3250 `_, `#3252 `_, `#3255 `_, `#3256 `_, `#3259 `_, `#3261 `_, `#3262 `_, `#3263 `_, `#3264 `_, `#3265 `_, `#3267 `_, `#3268 `_, `#3271 `_, `#3272 `_, `#3274 `_, `#3275 `_, `#3276 `_, `#3279 `_, `#3281 `_, `#3282 `_, `#3285 `_ + + Release 1.13.0 (05-August-2018) ''''''''''''''''''''''''''''''' diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index d3e0ae8b7..51c7f6da7 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -68,6 +68,8 @@ compile the dependencies yourself (instead of using ``--find-links`` to take advantage of the pre-compiled ones we host), you'll also need to install Xcode and its command-line tools. +**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. + Python 2.7 ---------- @@ -121,6 +123,9 @@ On Debian/Ubuntu-derived systems, the necessary packages are ``python-dev``, RPM-based system (like Fedora) these may be named ``python-devel``, etc, instead, and cam be installed with ``yum`` or ``rpm``. +**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. + + Install the Latest Tahoe-LAFS Release ===================================== @@ -158,7 +163,7 @@ from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.13.0 + tahoe-lafs: 1.14.0 foolscap: ... % @@ -178,16 +183,27 @@ You can also install directly from the source tarball URL:: New python executable in ~/venv/bin/python2.7 Installing setuptools, pip, wheel...done. - % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.13.0.tar.bz2 - Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.13.0.tar.bz2 + % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2 + Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2 ... Installing collected packages: ... Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.13.0 + tahoe-lafs: 1.14.0 ... +Extras +------ + +Tahoe-LAFS provides some functionality only when explicitly requested at installation time. +It does this using the "extras" feature of setuptools. +You can request these extra features when running the ``pip install`` command like this:: + + % venv/bin/pip install tahoe-lafs[tor] + +This example enables support for listening and connecting using Tor. +The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested. Hacking On Tahoe-LAFS --------------------- @@ -208,7 +224,7 @@ the additional libraries needed to run the unit tests:: Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.13.0.post34.dev0 + tahoe-lafs: 1.14.0.post34.dev0 ... This way, you won't have to re-run the ``pip install`` step each time you @@ -257,7 +273,7 @@ result in a "all tests passed" mesage:: % tox GLOB sdist-make: ~/tahoe-lafs/setup.py py27 recreate: ~/tahoe-lafs/.tox/py27 - py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.13.0.post8.dev0.zip + py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.14.0.post8.dev0.zip py27 runtests: commands[0] | tahoe --version py27 runtests: commands[1] | trial --rterrors allmydata allmydata.test.test_auth @@ -284,6 +300,8 @@ Similar errors about ``openssl/crypto.h`` indicate that you are missing the OpenSSL development headers (``libssl-dev``). Likewise ``ffi.h`` means you need ``libffi-dev``. +**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. + Using Tahoe-LAFS ================ diff --git a/docs/architecture.rst b/docs/architecture.rst index f3759a517..4cfabd844 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -98,7 +98,7 @@ subset are needed to reconstruct the segment (3 out of 10, with the default settings). It sends one block from each segment to a given server. The set of blocks on -a given server constitutes a "share". Therefore a subset f the shares (3 out +a given server constitutes a "share". Therefore a subset of the shares (3 out of 10, by default) are needed to reconstruct the file. A hash of the encryption key is used to form the "storage index", which is diff --git a/docs/configuration.rst b/docs/configuration.rst index 1f5a862b0..25175afe5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -9,6 +9,7 @@ Configuring a Tahoe-LAFS node #. `Connection Management`_ #. `Client Configuration`_ #. `Storage Server Configuration`_ +#. `Storage Server Plugin Configuration`_ #. `Frontend Configuration`_ #. `Running A Helper`_ #. `Running An Introducer`_ @@ -81,7 +82,6 @@ Client/server nodes provide one or more of the following services: * web-API service * SFTP service * FTP service -* Magic Folder service * helper service * storage service. @@ -718,12 +718,6 @@ SFTP, FTP for instructions on configuring these services, and the ``[sftpd]`` and ``[ftpd]`` sections of ``tahoe.cfg``. -Magic Folder - - A node running on Linux or Windows can be configured to automatically - upload files that are created or changed in a specified local directory. - See :doc:`frontends/magic-folder` for details. - Storage Server Configuration ============================ @@ -738,6 +732,17 @@ Storage Server Configuration for clients who do not wish to provide storage service. The default value is ``True``. +``anonymous = (boolean, optional)`` + + If this is ``True``, the node will expose the storage server via Foolscap + without any additional authentication or authorization. The capability to + use all storage services is conferred by knowledge of the Foolscap fURL + for the storage server which will be included in the storage server's + announcement. If it is ``False``, the node will not expose this and + storage must be exposed using the storage server plugin system (see + `Storage Server Plugin Configuration`_ for details). The default value is + ``True``. + ``readonly = (boolean, optional)`` If ``True``, the node will run a storage server but will not accept any @@ -798,6 +803,33 @@ Storage Server Configuration In addition, see :doc:`accepting-donations` for a convention encouraging donations to storage server operators. + +Storage Server Plugin Configuration +=================================== + +In addition to the built-in storage server, +it is also possible to load and configure storage server plugins into Tahoe-LAFS. + +Plugins to load are specified in the ``[storage]`` section. + +``plugins = (string, optional)`` + + This gives a comma-separated list of plugin names. + Plugins named here will be loaded and offered to clients. + The default is for no such plugins to be loaded. + +Each plugin can also be configured in a dedicated section. +The section for each plugin is named after the plugin itself:: + + [storageserver.plugins.] + +For example, +the configuration section for a plugin named ``acme-foo-v1`` is ``[storageserver.plugins.acme-foo-v1]``. + +The contents of such sections are defined by the plugins themselves. +Refer to the documentation provided with those plugins. + + Running A Helper ================ diff --git a/docs/debian.rst b/docs/debian.rst index 91b0697af..6b62a7b5e 100644 --- a/docs/debian.rst +++ b/docs/debian.rst @@ -39,16 +39,16 @@ virtualenv. The ``.deb`` packages, of course, rely solely upon other ``.deb`` packages. For reference, here is a list of the debian package names that provide Tahoe's -dependencies as of the 1.9 release: +dependencies as of the 1.14.0 release: * python * python-zfec -* python-pycryptopp * python-foolscap * python-openssl (needed by foolscap) * python-twisted * python-nevow * python-mock +* python-cryptography * python-simplejson * python-setuptools * python-support (for Debian-specific install-time tools) diff --git a/docs/frontends/CLI.rst b/docs/frontends/CLI.rst index 9b6343c58..e46936bad 100644 --- a/docs/frontends/CLI.rst +++ b/docs/frontends/CLI.rst @@ -44,7 +44,7 @@ arguments. "``tahoe --help``" might also provide something useful. Running "``tahoe --version``" will display a list of version strings, starting with the "allmydata" module (which contains the majority of the Tahoe-LAFS functionality) and including versions for a number of dependent libraries, -like Twisted, Foolscap, pycryptopp, and zfec. "``tahoe --version-and-path``" +like Twisted, Foolscap, cryptography, and zfec. "``tahoe --version-and-path``" will also show the path from which each library was imported. On Unix systems, the shell expands filename wildcards (``'*'`` and ``'?'``) diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index f4863af4c..dc348af34 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -211,14 +211,7 @@ Dependencies The Tahoe-LAFS SFTP server requires the Twisted "Conch" component (a "conch" is a twisted shell, get it?). Many Linux distributions package the Conch code -separately: debian puts it in the "python-twisted-conch" package. Conch -requires the "pycrypto" package, which is a Python+C implementation of many -cryptographic functions (the debian package is named "python-crypto"). - -Note that "pycrypto" is different than the "pycryptopp" package that -Tahoe-LAFS uses (which is a Python wrapper around the C++ -based Crypto++ -library, a library that is frequently installed as /usr/lib/libcryptopp.a, to -avoid problems with non-alphanumerics in filenames). +separately: debian puts it in the "python-twisted-conch" package. Immutable and Mutable Files =========================== diff --git a/docs/frontends/magic-folder.rst b/docs/frontends/magic-folder.rst deleted file mode 100644 index 694958563..000000000 --- a/docs/frontends/magic-folder.rst +++ /dev/null @@ -1,148 +0,0 @@ -.. -*- coding: utf-8-with-signature -*- - -================================ -Tahoe-LAFS Magic Folder Frontend -================================ - -1. `Introduction`_ -2. `Configuration`_ -3. `Known Issues and Limitations With Magic-Folder`_ - - -Introduction -============ - -The Magic Folder frontend synchronizes local directories on two or more -clients, using a Tahoe-LAFS grid for storage. Whenever a file is created -or changed under the local directory of one of the clients, the change is -propagated to the grid and then to the other clients. - -The implementation of the "drop-upload" frontend, on which Magic Folder is -based, was written as a prototype at the First International Tahoe-LAFS -Summit in June 2011. In 2015, with the support of a grant from the -`Open Technology Fund`_, it was redesigned and extended to support -synchronization between clients. It currently works on Linux and Windows. - -Magic Folder is not currently in as mature a state as the other frontends -(web, CLI, SFTP and FTP). This means that you probably should not rely on -all changes to files in the local directory to result in successful uploads. -There might be (and have been) incompatible changes to how the feature is -configured. - -We are very interested in feedback on how well this feature works for you, and -suggestions to improve its usability, functionality, and reliability. - -.. _`Open Technology Fund`: https://www.opentech.fund/ - - -Configuration -============= - -The Magic Folder frontend runs as part of a gateway node. To set it up, you -must use the tahoe magic-folder CLI. For detailed information see our -:doc:`Magic-Folder CLI design -documentation<../proposed/magic-folder/user-interface-design>`. For a -given Magic-Folder collective directory you need to run the ``tahoe -magic-folder create`` command. After that the ``tahoe magic-folder invite`` -command must used to generate an *invite code* for each member of the -magic-folder collective. A confidential, authenticated communications channel -should be used to transmit the invite code to each member, who will be -joining using the ``tahoe magic-folder join`` command. - -These settings are persisted in the ``[magic_folder]`` section of the -gateway's ``tahoe.cfg`` file. - -``[magic_folder]`` - -``enabled = (boolean, optional)`` - - If this is ``True``, Magic Folder will be enabled. The default value is - ``False``. - -``local.directory = (UTF-8 path)`` - - This specifies the local directory to be monitored for new or changed - files. If the path contains non-ASCII characters, it should be encoded - in UTF-8 regardless of the system's filesystem encoding. Relative paths - will be interpreted starting from the node's base directory. - -You should not normally need to set these fields manually because they are -set by the ``tahoe magic-folder create`` and/or ``tahoe magic-folder join`` -commands. Use the ``--help`` option to these commands for more information. - -After setting up a Magic Folder collective and starting or restarting each -gateway, you can confirm that the feature is working by copying a file into -any local directory, and checking that it appears on other clients. -Large files may take some time to appear. - -The 'Operational Statistics' page linked from the Welcome page shows counts -of the number of files uploaded, the number of change events currently -queued, and the number of failed uploads. The 'Recent Uploads and Downloads' -page and the node :doc:`log<../logging>` may be helpful to determine the -cause of any failures. - - -.. _Known Issues in Magic-Folder: - -Known Issues and Limitations With Magic-Folder -============================================== - -This feature only works on Linux and Windows. There is a ticket to add -support for Mac OS X and BSD-based systems (`#1432`_). - -The only way to determine whether uploads have failed is to look at the -'Operational Statistics' page linked from the Welcome page. This only shows -a count of failures, not the names of files. Uploads are never retried. - -The Magic Folder frontend performs its uploads sequentially (i.e. it waits -until each upload is finished before starting the next), even when there -would be enough memory and bandwidth to efficiently perform them in parallel. -A Magic Folder upload can occur in parallel with an upload by a different -frontend, though. (`#1459`_) - -On Linux, if there are a large number of near-simultaneous file creation or -change events (greater than the number specified in the file -``/proc/sys/fs/inotify/max_queued_events``), it is possible that some events -could be missed. This is fairly unlikely under normal circumstances, because -the default value of ``max_queued_events`` in most Linux distributions is -16384, and events are removed from this queue immediately without waiting for -the corresponding upload to complete. (`#1430`_) - -The Windows implementation might also occasionally miss file creation or -change events, due to limitations of the underlying Windows API -(ReadDirectoryChangesW). We do not know how likely or unlikely this is. -(`#1431`_) - -Some filesystems may not support the necessary change notifications. -So, it is recommended for the local directory to be on a directly attached -disk-based filesystem, not a network filesystem or one provided by a virtual -machine. - -The ``private/magic_folder_dircap`` and ``private/collective_dircap`` files -cannot use an alias or path to specify the upload directory. (`#1711`_) - -If a file in the upload directory is changed (actually relinked to a new -file), then the old file is still present on the grid, and any other caps -to it will remain valid. Eventually it will be possible to use -:doc:`../garbage-collection` to reclaim the space used by these files; however -currently they are retained indefinitely. (`#2440`_) - -Unicode filenames are supported on both Linux and Windows, but on Linux, the -local name of a file must be encoded correctly in order for it to be uploaded. -The expected encoding is that printed by -``python -c "import sys; print sys.getfilesystemencoding()"``. - -On Windows, local directories with non-ASCII names are not currently working. -(`#2219`_) - -On Windows, when a node has Magic Folder enabled, it is unresponsive to Ctrl-C -(it can only be killed using Task Manager or similar). (`#2218`_) - -.. _`#1430`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1430 -.. _`#1431`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1431 -.. _`#1432`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1432 -.. _`#1459`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1459 -.. _`#1711`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1711 -.. _`#2218`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2218 -.. _`#2219`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2219 -.. _`#2440`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2440 diff --git a/docs/historical/historical_known_issues.txt b/docs/historical/historical_known_issues.txt index d43ba2439..9d4e1d427 100644 --- a/docs/historical/historical_known_issues.txt +++ b/docs/historical/historical_known_issues.txt @@ -272,22 +272,3 @@ that size, assume that they have been corrupted and are not retrievable from the Tahoe storage grid. Tahoe v1.1 clients will refuse to upload files larger than 12 GiB with a clean failure. A future release of Tahoe will remove this limitation so that larger files can be uploaded. - - -=== pycryptopp defect resulting in data corruption === - -Versions of pycryptopp earlier than pycryptopp-0.5.0 had a defect -which, when compiled with some compilers, would cause AES-256 -encryption and decryption to be computed incorrectly. This could -cause data corruption. Tahoe v1.0 required, and came with a bundled -copy of, pycryptopp v0.3. - -==== how to manage it ==== - -You can detect whether pycryptopp-0.3 has this failure when it is -compiled by your compiler. Run the unit tests that come with -pycryptopp-0.3: unpack the "pycryptopp-0.3.tar" file that comes in the -Tahoe v1.0 {{{misc/dependencies}}} directory, cd into the resulting -{{{pycryptopp-0.3.0}}} directory, and execute {{{python ./setup.py -test}}}. If the tests pass, then your compiler does not trigger this -failure. diff --git a/docs/how_to_make_a_tahoe-lafs_release.org b/docs/how_to_make_a_tahoe-lafs_release.org index 79438c85d..44b9e3dd1 100644 --- a/docs/how_to_make_a_tahoe-lafs_release.org +++ b/docs/how_to_make_a_tahoe-lafs_release.org @@ -17,13 +17,14 @@ people are Release Maintainers: - [ ] all appveyor checks pass - [ ] all buildbot workers pass their checks -* freeze master branch [0/] +* freeze master branch [0/1] - [ ] announced the freeze of the master branch on IRC (i.e. non-release PRs won't be merged until after release) * sync documentation [0/7] - - [ ] NEWS.rst: summarize user-visible changes, aim for one page of text + + - [ ] NEWS.rst: (run "tox -e news") - [ ] added final release name and date to top-most item in NEWS.rst - - [ ] updated relnotes.txt + - [ ] updated relnotes.txt (change next, last versions; summarize NEWS) - [ ] updated CREDITS - [ ] updated docs/known_issues.rst - [ ] docs/INSTALL.rst only points to current tahoe-lafs-X.Y.Z.tar.gz source code file diff --git a/docs/index.rst b/docs/index.rst index 566e78d4f..98b4f241b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,6 @@ Contents: frontends/CLI frontends/webapi frontends/FTP-and-SFTP - frontends/magic-folder frontends/download-status known_issues @@ -37,7 +36,6 @@ Contents: expenses cautions write_coordination - magic-folder-howto backupdb anonymity-configuration diff --git a/docs/magic-folder-howto.rst b/docs/magic-folder-howto.rst deleted file mode 100644 index b368972b8..000000000 --- a/docs/magic-folder-howto.rst +++ /dev/null @@ -1,176 +0,0 @@ -.. _magic-folder-howto: - -========================= -Magic Folder Set-up Howto -========================= - -#. `This document`_ -#. `Setting up a local test grid`_ -#. `Setting up Magic Folder`_ -#. `Testing`_ - - -This document -============= - -This is preliminary documentation of how to set up Magic Folder using a test -grid on a single Linux or Windows machine, with two clients and one server. -It is aimed at a fairly technical audience. - -For an introduction to Magic Folder and how to configure it -more generally, see :doc:`frontends/magic-folder`. - -It it possible to adapt these instructions to run the nodes on -different machines, to synchronize between three or more clients, -to mix Windows and Linux clients, and to use multiple servers -(if the Tahoe-LAFS encoding parameters are changed). - - -Setting up a local test grid -============================ - -Linux ------ - -Run these commands:: - - mkdir ../grid - bin/tahoe create-introducer ../grid/introducer - bin/tahoe start ../grid/introducer - export FURL=`cat ../grid/introducer/private/introducer.furl` - bin/tahoe create-node --introducer="$FURL" ../grid/server - bin/tahoe create-client --introducer="$FURL" ../grid/alice - bin/tahoe create-client --introducer="$FURL" ../grid/bob - - -Windows -------- - -Run:: - - mkdir ..\grid - bin\tahoe create-introducer ..\grid\introducer - bin\tahoe start ..\grid\introducer - -Leave the introducer running in that Command Prompt, -and in a separate Command Prompt (with the same current -directory), run:: - - set /p FURL=<..\grid\introducer\private\introducer.furl - bin\tahoe create-node --introducer=%FURL% ..\grid\server - bin\tahoe create-client --introducer=%FURL% ..\grid\alice - bin\tahoe create-client --introducer=%FURL% ..\grid\bob - - -Both Linux and Windows ----------------------- - -(Replace ``/`` with ``\`` for Windows paths.) - -Edit ``../grid/alice/tahoe.cfg``, and make the following -changes to the ``[node]`` and ``[client]`` sections:: - - [node] - nickname = alice - web.port = tcp:3457:interface=127.0.0.1 - - [client] - shares.needed = 1 - shares.happy = 1 - shares.total = 1 - -Edit ``../grid/bob/tahoe.cfg``, and make the following -change to the ``[node]`` section, and the same change as -above to the ``[client]`` section:: - - [node] - nickname = bob - web.port = tcp:3458:interface=127.0.0.1 - -Note that when running nodes on a single machine, -unique port numbers must be used for each node (and they -must not clash with ports used by other server software). -Here we have used the default of 3456 for the server, -3457 for alice, and 3458 for bob. - -Now start all of the nodes (the introducer should still be -running from above):: - - bin/tahoe start ../grid/server - bin/tahoe start ../grid/alice - bin/tahoe start ../grid/bob - -On Windows, a separate Command Prompt is needed to run each -node. - -Open a web browser on http://127.0.0.1:3457/ and verify that -alice is connected to the introducer and one storage server. -Then do the same for http://127.0.0.1:3568/ to verify that -bob is connected. Leave all of the nodes running for the -next stage. - - -Setting up Magic Folder -======================= - -Linux ------ - -Run:: - - mkdir -p ../local/alice ../local/bob - bin/tahoe -d ../grid/alice magic-folder create magic: alice ../local/alice - bin/tahoe -d ../grid/alice magic-folder invite magic: bob >invitecode - export INVITECODE=`cat invitecode` - bin/tahoe -d ../grid/bob magic-folder join "$INVITECODE" ../local/bob - - bin/tahoe restart ../grid/alice - bin/tahoe restart ../grid/bob - -Windows -------- - -Run:: - - mkdir ..\local\alice ..\local\bob - bin\tahoe -d ..\grid\alice magic-folder create magic: alice ..\local\alice - bin\tahoe -d ..\grid\alice magic-folder invite magic: bob >invitecode - set /p INVITECODE=`) invites and joins work this way -as well. +Introducer with new clients. This is a two-part process. Alice runs a grid and wishes to have her friend Bob use it as a client. She runs ``tahoe invite bob`` which diff --git a/docs/proposed/accounting-overview.txt b/docs/proposed/accounting-overview.txt index d2e5d26be..cc678bc62 100644 --- a/docs/proposed/accounting-overview.txt +++ b/docs/proposed/accounting-overview.txt @@ -546,16 +546,15 @@ The "restrictions dictionary" is a table which establishes an upper bound on how this authority (or any attenuations thereof) may be used. It is effectively a set of key-value pairs. -A "signing key" is an EC-DSA192 private key string, as supplied to the -pycryptopp SigningKey() constructor, and is 12 bytes long. A "verifying key" -is an EC-DSA192 public key string, as produced by pycryptopp, and is 24 bytes -long. A "key identifier" is a string which securely identifies a specific -signing/verifying keypair: for long RSA keys it would be a secure hash of the -public key, but since ECDSA192 keys are so short, we simply use the full -verifying key verbatim. A "key hint" is a variable-length prefix of the key -identifier, perhaps zero bytes long, used to help a recipient reduce the -number of verifying keys that it must search to find one that matches a -signed message. +A "signing key" is an EC-DSA192 private key string and is 12 bytes +long. A "verifying key" is an EC-DSA192 public key string, and is 24 +bytes long. A "key identifier" is a string which securely identifies a +specific signing/verifying keypair: for long RSA keys it would be a +secure hash of the public key, but since ECDSA192 keys are so short, +we simply use the full verifying key verbatim. A "key hint" is a +variable-length prefix of the key identifier, perhaps zero bytes long, +used to help a recipient reduce the number of verifying keys that it +must search to find one that matches a signed message. ==== Authority Chains ==== diff --git a/docs/proposed/index.rst b/docs/proposed/index.rst index a052baeff..d01d92d2d 100644 --- a/docs/proposed/index.rst +++ b/docs/proposed/index.rst @@ -14,8 +14,4 @@ index only lists the files that are in .rst format. :maxdepth: 2 leasedb - magic-folder/filesystem-integration - magic-folder/remote-to-local-sync - magic-folder/user-interface-design - magic-folder/multi-party-conflict-detection http-storage-node-protocol diff --git a/docs/proposed/magic-folder/filesystem-integration.rst b/docs/proposed/magic-folder/filesystem-integration.rst deleted file mode 100644 index 589db2322..000000000 --- a/docs/proposed/magic-folder/filesystem-integration.rst +++ /dev/null @@ -1,118 +0,0 @@ -Magic Folder local filesystem integration design -================================================ - -*Scope* - -This document describes how to integrate the local filesystem with Magic -Folder in an efficient and reliable manner. For now we ignore Remote to -Local synchronization; the design and implementation of this is scheduled -for a later time. We also ignore multiple writers for the same Magic -Folder, which may or may not be supported in future. The design here will -be updated to account for those features in later Objectives. Objective 3 -may require modifying the database schema or operation, and Objective 5 -may modify the User interface. - -Tickets on the Tahoe-LAFS trac with the `otf-magic-folder-objective2`_ -keyword are within the scope of the local filesystem integration for -Objective 2. - -.. _otf-magic-folder-objective2: https://tahoe-lafs.org/trac/tahoe-lafs/query?status=!closed&keywords=~otf-magic-folder-objective2 - -.. _filesystem_integration-local-scanning-and-database: - -*Local scanning and database* - -When a Magic-Folder-enabled node starts up, it scans all directories -under the local directory and adds every file to a first-in first-out -"scan queue". When processing the scan queue, redundant uploads are -avoided by using the same mechanism the Tahoe backup command uses: we -keep track of previous uploads by recording each file's metadata such as -size, ``ctime`` and ``mtime``. This information is stored in a database, -referred to from now on as the magic folder db. Using this recorded -state, we ensure that when Magic Folder is subsequently started, the -local directory tree can be scanned quickly by comparing current -filesystem metadata with the previously recorded metadata. Each file -referenced in the scan queue is uploaded only if its metadata differs at -the time it is processed. If a change event is detected for a file that -is already queued (and therefore will be processed later), the redundant -event is ignored. - -To implement the magic folder db, we will use an SQLite schema that -initially is the existing Tahoe-LAFS backup schema. This schema may -change in later objectives; this will cause no backward compatibility -problems, because this new feature will be developed on a branch that -makes no compatibility guarantees. However we will have a separate SQLite -database file and separate mutex lock just for Magic Folder. This avoids -usability problems related to mutual exclusion. (If a single file and -lock were used, a backup would block Magic Folder updates for a long -time, and a user would not be able to tell when backups are possible -because Magic Folder would acquire a lock at arbitrary times.) - - -*Eventual consistency property* - -During the process of reading a file in order to upload it, it is not -possible to prevent further local writes. Such writes will result in -temporary inconsistency (that is, the uploaded file will not reflect -what the contents of the local file were at any specific time). Eventual -consistency is reached when the queue of pending uploads is empty. That -is, a consistent snapshot will be achieved eventually when local writes -to the target folder cease for a sufficiently long period of time. - - -*Detecting filesystem changes* - -For the Linux implementation, we will use the `inotify`_ Linux kernel -subsystem to gather events on the local Magic Folder directory tree. This -implementation was already present in Tahoe-LAFS 1.9.0, but needs to be -changed to gather directory creation and move events, in addition to the -events indicating that a file has been written that are gathered by the -current code. - -.. _`inotify`: https://en.wikipedia.org/wiki/Inotify - -For the Windows implementation, we will use the ``ReadDirectoryChangesW`` -Win32 API. The prototype implementation simulates a Python interface to -the inotify API in terms of ``ReadDirectoryChangesW``, allowing most of -the code to be shared across platforms. - -The alternative of using `NTFS Change Journals`_ for Windows was -considered, but appears to be more complicated and does not provide any -additional functionality over the scanning approach described above. -The Change Journal mechanism is also only available for NTFS filesystems, -but FAT32 filesystems are still common in user installations of Windows. - -.. _`NTFS Change Journals`: https://msdn.microsoft.com/en-us/library/aa363803%28VS.85%29.aspx - -When we detect the creation of a new directory below the local Magic -Folder directory, we create it in the Tahoe-LAFS filesystem, and also -scan the new local directory for new files. This scan is necessary to -avoid missing events for creation of files in a new directory before it -can be watched, and to correctly handle cases where an existing directory -is moved to be under the local Magic Folder directory. - - -*User interface* - -The Magic Folder local filesystem integration will initially have a -provisional configuration file-based interface that may not be ideal from -a usability perspective. Creating our local filesystem integration in -this manner will allow us to use and test it independently of the rest of -the Magic Folder software components. We will focus greater attention on -user interface design as a later milestone in our development roadmap. - -The configuration file, ``tahoe.cfg``, must define a target local -directory to be synchronized. Provisionally, this configuration will -replace the current ``[drop_upload]`` section:: - - [magic_folder] - enabled = true - local.directory = "/home/human" - -When a filesystem directory is first configured for Magic Folder, the user -needs to create the remote Tahoe-LAFS directory using ``tahoe mkdir``, -and configure the Magic-Folder-enabled node with its URI (e.g. by putting -it in a file ``private/magic_folder_dircap``). If there are existing -files in the local directory, they will be uploaded as a result of the -initial scan described earlier. - diff --git a/docs/proposed/magic-folder/multi-party-conflict-detection.rst b/docs/proposed/magic-folder/multi-party-conflict-detection.rst deleted file mode 100644 index fb1ae8339..000000000 --- a/docs/proposed/magic-folder/multi-party-conflict-detection.rst +++ /dev/null @@ -1,373 +0,0 @@ -Multi-party Conflict Detection -============================== - -The current Magic-Folder remote conflict detection design does not properly detect remote conflicts -for groups of three or more parties. This design is specified in the "Fire Dragon" section of this document: -https://github.com/tahoe-lafs/tahoe-lafs/blob/2551.wip.2/docs/proposed/magic-folder/remote-to-local-sync.rst#fire-dragons-distinguishing-conflicts-from-overwrites - -This Tahoe-LAFS trac ticket comment outlines a scenario with -three parties in which a remote conflict is falsely detected: - -.. _`ticket comment`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2551#comment:22 - - -Summary and definitions -======================= - -Abstract file: a file being shared by a Magic Folder. - -Local file: a file in a client's local filesystem corresponding to an abstract file. - -Relative path: the path of an abstract or local file relative to the Magic Folder root. - -Version: a snapshot of an abstract file, with associated metadata, that is uploaded by a Magic Folder client. - -A version is associated with the file's relative path, its contents, and -mtime and ctime timestamps. Versions also have a unique identity. - -Follows relation: -* If and only if a change to a client's local file at relative path F that results in an upload of version V', -was made when the client already had version V of that file, then we say that V' directly follows V. -* The follows relation is the irreflexive transitive closure of the "directly follows" relation. - -The follows relation is transitive and acyclic, and therefore defines a DAG called the -Version DAG. Different abstract files correspond to disconnected sets of nodes in the Version DAG -(in other words there are no "follows" relations between different files). - -The DAG is only ever extended, not mutated. - -The desired behaviour for initially classifying overwrites and conflicts is as follows: - -* if a client Bob currently has version V of a file at relative path F, and it sees a new version V' - of that file in another client Alice's DMD, such that V' follows V, then the write of the new version - is initially an overwrite and should be to the same filename. -* if, in the same situation, V' does not follow V, then the write of the new version should be - classified as a conflict. - -The existing :doc:`remote-to-local-sync` document defines when an initial -overwrite should be reclassified as a conflict. - -The above definitions completely specify the desired solution of the false -conflict behaviour described in the `ticket comment`_. However, they do not give -a concrete algorithm to compute the follows relation, or a representation in the -Tahoe-LAFS file store of the metadata needed to compute it. - -We will consider two alternative designs, proposed by Leif Ryge and -Zooko Wilcox-O'Hearn, that aim to fill this gap. - - - -Leif's Proposal: Magic-Folder "single-file" snapshot design -=========================================================== - -Abstract --------- - -We propose a relatively simple modification to the initial Magic Folder design which -adds merkle DAGs of immutable historical snapshots for each file. The full history -does not necessarily need to be retained, and the choice of how much history to retain -can potentially be made on a per-file basis. - -Motivation: ------------ - -no SPOFs, no admins -``````````````````` - -Additionally, the initial design had two cases of excess authority: - -1. The magic folder administrator (inviter) has everyone's write-caps and is thus essentially "root" -2. Each client shares ambient authority and can delete anything or everything and - (assuming there is not a conflict) the data will be deleted from all clients. So, each client - is effectively "root" too. - -Thus, while it is useful for file synchronization, the initial design is a much less safe place -to store data than in a single mutable tahoe directory (because more client computers have the -possibility to delete it). - - -Glossary --------- - -- merkle DAG: like a merkle tree but with multiple roots, and with each node potentially having multiple parents -- magic folder: a logical directory that can be synchronized between many clients - (devices, users, ...) using a Tahoe-LAFS storage grid -- client: a Magic-Folder-enabled Tahoe-LAFS client instance that has access to a magic folder -- DMD: "distributed mutable directory", a physical Tahoe-LAFS mutable directory. - Each client has the write cap to their own DMD, and read caps to all other client's DMDs - (as in the original Magic Folder design). -- snapshot: a reference to a version of a file; represented as an immutable directory containing - an entry called "content" (pointing to the immutable file containing the file's contents), - and an entry called "parent0" (pointing to a parent snapshot), and optionally parent1 through - parentN pointing at other parents. The Magic Folder snapshot object is conceptually very similar - to a git commit object, except for that it is created automatically and it records the history of an - individual file rather than an entire repository. Also, commits do not need to have authors - (although an author field could be easily added later). -- deletion snapshot: immutable directory containing no content entry (only one or more parents) -- capability: a Tahoe-LAFS diminishable cryptographic capability -- cap: short for capability -- conflict: the situation when another client's current snapshot for a file is different than our current snapshot, and is not a descendant of ours. -- overwrite: the situation when another client's current snapshot for a file is a (not necessarily direct) descendant of our current snapshot. - - -Overview --------- - -This new design will track the history of each file using "snapshots" which are -created at each upload. Each snapshot will specify one or more parent snapshots, -forming a directed acyclic graph. A Magic-Folder user's DMD uses a flattened directory -hierarchy naming scheme, as in the original design. But, instead of pointing directly -at file contents, each file name will link to that user's latest snapshot for that file. - -Inside the dmd there will also be an immutable directory containing the client's subscriptions -(read-caps to other clients' dmds). - -Clients periodically poll each other's DMDs. When they see the current snapshot for a file is -different than their own current snapshot for that file, they immediately begin downloading its -contents and then walk backwards through the DAG from the new snapshot until they find their own -snapshot or a common ancestor. - -For the common ancestor search to be efficient, the client will need to keep a local store (in the magic folder db) of all of the snapshots -(but not their contents) between the oldest current snapshot of any of their subscriptions and their own current snapshot. -See "local cache purging policy" below for more details. - -If the new snapshot is a descendant of the client's existing snapshot, then this update -is an "overwrite" - like a git fast-forward. So, when the download of the new file completes it can overwrite -the existing local file with the new contents and update its dmd to point at the new snapshot. - -If the new snapshot is not a descendant of the client's current snapshot, then the update is a -conflict. The new file is downloaded and named $filename.conflict-$user1,$user2 (including a list -of other subscriptions who have that version as their current version). - -Changes to the local .conflict- file are not tracked. When that file disappears -(either by deletion, or being renamed) a new snapshot for the conflicting file is -created which has two parents - the client's snapshot prior to the conflict, and the -new conflicting snapshot. If multiple .conflict files are deleted or renamed in a short -period of time, a single conflict-resolving snapshot with more than two parents can be created. - -! I think this behavior will confuse users. - -Tahoe-LAFS snapshot objects ---------------------------- - -These Tahoe-LAFS snapshot objects only track the history of a single file, not a directory hierarchy. -Snapshot objects contain only two field types: -- ``Content``: an immutable capability of the file contents (omitted if deletion snapshot) -- ``Parent0..N``: immutable capabilities representing parent snapshots - -Therefore in this system an interesting side effect of this Tahoe snapshot object is that there is no -snapshot author. The only notion of an identity in the Magic-Folder system is the write capability of the user's DMD. - -The snapshot object is an immutable directory which looks like this: -content -> immutable cap to file content -parent0 -> immutable cap to a parent snapshot object -parent1..N -> more parent snapshots - - -Snapshot Author Identity ------------------------- - -Snapshot identity might become an important feature so that bad actors -can be recognized and other clients can stop "subscribing" to (polling for) updates from them. - -Perhaps snapshots could be signed by the user's Magic-Folder write key for this purpose? Probably a bad idea to reuse the write-cap key for this. Better to introduce ed25519 identity keys which can (optionally) sign snapshot contents and store the signature as another member of the immutable directory. - - -Conflict Resolution -------------------- - -detection of conflicts -`````````````````````` - -A Magic-Folder client updates a given file's current snapshot link to a snapshot which is a descendent -of the previous snapshot. For a given file, let's say "file1", Alice can detect that Bob's DMD has a "file1" -that links to a snapshot which conflicts. Two snapshots conflict if one is not an ancestor of the other. - - -a possible UI for resolving conflicts -````````````````````````````````````` - -If Alice links a conflicting snapshot object for a file named "file1", -Bob and Carole will see a file in their Magic-Folder called "file1.conflicted.Alice". -Alice conversely will see an additional file called "file1.conflicted.previous". -If Alice wishes to resolve the conflict with her new version of the file then -she simply deletes the file called "file1.conflicted.previous". If she wants to -choose the other version then she moves it into place: - - mv file1.conflicted.previous file1 - - -This scheme works for N number of conflicts. Bob for instance could choose -the same resolution for the conflict, like this: - - mv file1.Alice file1 - - -Deletion propagation and eventual Garbage Collection ----------------------------------------------------- - -When a user deletes a file, this is represented by a link from their DMD file -object to a deletion snapshot. Eventually all users will link this deletion -snapshot into their DMD. When all users have the link then they locally cache -the deletion snapshot and remove the link to that file in their DMD. -Deletions can of course be undeleted; this means creating a new snapshot -object that specifies itself a descent of the deletion snapshot. - -Clients periodically renew leases to all capabilities recursively linked -to in their DMD. Files which are unlinked by ALL the users of a -given Magic-Folder will eventually be garbage collected. - -Lease expirey duration must be tuned properly by storage servers such that -Garbage Collection does not occur too frequently. - - - -Performance Considerations --------------------------- - -local changes -````````````` - -Our old scheme requires two remote Tahoe-LAFS operations per local file modification: -1. upload new file contents (as an immutable file) -2. modify mutable directory (DMD) to link to the immutable file cap - -Our new scheme requires three remote operations: -1. upload new file contents (as in immutable file) -2. upload immutable directory representing Tahoe-LAFS snapshot object -3. modify mutable directory (DMD) to link to the immutable snapshot object - -remote changes -`````````````` - -Our old scheme requires one remote Tahoe-LAFS operation per remote file modification (not counting the polling of the dmd): -1. Download new file content - -Our new scheme requires a minimum of two remote operations (not counting the polling of the dmd) for conflicting downloads, or three remote operations for overwrite downloads: -1. Download new snapshot object -2. Download the content it points to -3. If the download is an overwrite, modify the DMD to indicate that the downloaded version is their current version. - -If the new snapshot is not a direct descendant of our current snapshot or the other party's previous snapshot we saw, we will also need to download more snapshots to determine if it is a conflict or an overwrite. However, those can be done in -parallel with the content download since we will need to download the content in either case. - -While the old scheme is obviously more efficient, we think that the properties provided by the new scheme make it worth the additional cost. - -Physical updates to the DMD overiouslly need to be serialized, so multiple logical updates should be combined when an update is already in progress. - -conflict detection and local caching -```````````````````````````````````` - -Local caching of snapshots is important for performance. -We refer to the client's local snapshot cache as the ``magic-folder db``. - -Conflict detection can be expensive because it may require the client -to download many snapshots from the other user's DMD in order to try -and find it's own current snapshot or a descendent. The cost of scanning -the remote DMDs should not be very high unless the client conducting the -scan has lots of history to download because of being offline for a long -time while many new snapshots were distributed. - - -local cache purging policy -`````````````````````````` - -The client's current snapshot for each file should be cached at all times. -When all clients' views of a file are synchronized (they all have the same -snapshot for that file), no ancestry for that file needs to be cached. -When clients' views of a file are *not* synchronized, the most recent -common ancestor of all clients' snapshots must be kept cached, as must -all intermediate snapshots. - - -Local Merge Property --------------------- - -Bob can in fact, set a pre-existing directory (with files) as his new Magic-Folder directory, resulting -in a merge of the Magic-Folder with Bob's local directory. Filename collisions will result in conflicts -because Bob's new snapshots are not descendent's of the existing Magic-Folder file snapshots. - - -Example: simultaneous update with four parties: - -1. A, B, C, D are in sync for file "foo" at snapshot X -2. A and B simultaneously change the file, creating snapshots XA and XB (both descendants of X). -3. C hears about XA first, and D hears about XB first. Both accept an overwrite. -4. All four parties hear about the other update they hadn't heard about yet. -5. Result: - - everyone's local file "foo" has the content pointed to by the snapshot in their DMD's "foo" entry - - A and C's DMDs each have the "foo" entry pointing at snapshot XA - - B and D's DMDs each have the "foo" entry pointing at snapshot XB - - A and C have a local file called foo.conflict-B,D with XB's content - - B and D have a local file called foo.conflict-A,C with XA's content - -Later: - - - Everyone ignores the conflict, and continue updating their local "foo". but slowly enough that there are no further conflicts, so that A and C remain in sync with eachother, and B and D remain in sync with eachother. - - - A and C's foo.conflict-B,D file continues to be updated with the latest version of the file B and D are working on, and vice-versa. - - - A and C edit the file at the same time again, causing a new conflict. - - - Local files are now: - - A: "foo", "foo.conflict-B,D", "foo.conflict-C" - - C: "foo", "foo.conflict-B,D", "foo.conflict-A" - - B and D: "foo", "foo.conflict-A", "foo.conflict-C" - - - Finally, D decides to look at "foo.conflict-A" and "foo.conflict-C", and they manually integrate (or decide to ignore) the differences into their own local file "foo". - - - D deletes their conflict files. - - - D's DMD now points to a snapshot that is a descendant of everyone else's current snapshot, resolving all conflicts. - - - The conflict files on A, B, and C disappear, and everyone's local file "foo" contains D's manually-merged content. - - -Daira: I think it is too complicated to include multiple nicknames in the .conflict files -(e.g. "foo.conflict-B,D"). It should be sufficient to have one file for each other client, -reflecting that client's latest version, regardless of who else it conflicts with. - - -Zooko's Design (as interpreted by Daira) -======================================== - -A version map is a mapping from client nickname to version number. - -Definition: a version map M' strictly-follows a mapping M iff for every entry c->v -in M, there is an entry c->v' in M' such that v' > v. - - -Each client maintains a 'local version map' and a 'conflict version map' for each file -in its magic folder db. -If it has never written the file, then the entry for its own nickname in the local version -map is zero. The conflict version map only contains entries for nicknames B where -"$FILENAME.conflict-$B" exists. - -When a client A uploads a file, it increments the version for its own nickname in its -local version map for the file, and includes that map as metadata with its upload. - -A download by client A from client B is an overwrite iff the downloaded version map -strictly-follows A's local version map for that file; in this case A replaces its local -version map with the downloaded version map. Otherwise it is a conflict, and the -download is put into "$FILENAME.conflict-$B"; in this case A's -local version map remains unchanged, and the entry B->v taken from the downloaded -version map is added to its conflict version map. - -If client A deletes or renames a conflict file "$FILENAME.conflict-$B", then A copies -the entry for B from its conflict version map to its local version map, deletes -the entry for B in its conflict version map, and performs another upload (with -incremented version number) of $FILENAME. - - -Example: - A, B, C = (10, 20, 30) everyone agrees. - A updates: (11, 20, 30) - B updates: (10, 21, 30) - -C will see either A or B first. Both would be an overwrite, if considered alone. - - - diff --git a/docs/proposed/magic-folder/remote-to-local-sync.rst b/docs/proposed/magic-folder/remote-to-local-sync.rst deleted file mode 100644 index 32c2fd5da..000000000 --- a/docs/proposed/magic-folder/remote-to-local-sync.rst +++ /dev/null @@ -1,951 +0,0 @@ -Magic Folder design for remote-to-local sync -============================================ - -Scope ------ - -In this Objective we will design remote-to-local synchronization: - -* How to efficiently determine which objects (files and directories) have - to be downloaded in order to bring the current local filesystem into sync - with the newly-discovered version of the remote filesystem. -* How to distinguish overwrites, in which the remote side was aware of - your most recent version and overwrote it with a new version, from - conflicts, in which the remote side was unaware of your most recent - version when it published its new version. The latter needs to be raised - to the user as an issue the user will have to resolve and the former must - not bother the user. -* How to overwrite the (stale) local versions of those objects with the - newly acquired objects, while preserving backed-up versions of those - overwritten objects in case the user didn't want this overwrite and wants - to recover the old version. - -Tickets on the Tahoe-LAFS trac with the `otf-magic-folder-objective4`_ -keyword are within the scope of the remote-to-local synchronization -design. - -.. _otf-magic-folder-objective4: https://tahoe-lafs.org/trac/tahoe-lafs/query?status=!closed&keywords=~otf-magic-folder-objective4 - - -Glossary -'''''''' - -Object: a file or directory - -DMD: distributed mutable directory - -Folder: an abstract directory that is synchronized between clients. -(A folder is not the same as the directory corresponding to it on -any particular client, nor is it the same as a DMD.) - -Collective: the set of clients subscribed to a given Magic Folder. - -Descendant: a direct or indirect child in a directory or folder tree - -Subfolder: a folder that is a descendant of a magic folder - -Subpath: the path from a magic folder to one of its descendants - -Write: a modification to a local filesystem object by a client - -Read: a read from a local filesystem object by a client - -Upload: an upload of a local object to the Tahoe-LAFS file store - -Download: a download from the Tahoe-LAFS file store to a local object - -Pending notification: a local filesystem change that has been detected -but not yet processed. - - -Representing the Magic Folder in Tahoe-LAFS -------------------------------------------- - -Unlike the local case where we use inotify or ReadDirectoryChangesW to -detect filesystem changes, we have no mechanism to register a monitor for -changes to a Tahoe-LAFS directory. Therefore, we must periodically poll -for changes. - -An important constraint on the solution is Tahoe-LAFS' ":doc:`write -coordination directive<../../write_coordination>`", which prohibits -concurrent writes by different storage clients to the same mutable object: - - Tahoe does not provide locking of mutable files and directories. If - there is more than one simultaneous attempt to change a mutable file - or directory, then an UncoordinatedWriteError may result. This might, - in rare cases, cause the file or directory contents to be accidentally - deleted. The user is expected to ensure that there is at most one - outstanding write or update request for a given file or directory at - a time. One convenient way to accomplish this is to make a different - file or directory for each person or process that wants to write. - -Since it is a goal to allow multiple users to write to a Magic Folder, -if the write coordination directive remains the same as above, then we -will not be able to implement the Magic Folder as a single Tahoe-LAFS -DMD. In general therefore, we will have multiple DMDs —spread across -clients— that together represent the Magic Folder. Each client in a -Magic Folder collective polls the other clients' DMDs in order to detect -remote changes. - -Six possible designs were considered for the representation of subfolders -of the Magic Folder: - -1. All subfolders written by a given Magic Folder client are collapsed -into a single client DMD, containing immutable files. The child name of -each file encodes the full subpath of that file relative to the Magic -Folder. - -2. The DMD tree under a client DMD is a direct copy of the folder tree -written by that client to the Magic Folder. Not all subfolders have -corresponding DMDs; only those to which that client has written files or -child subfolders. - -3. The directory tree under a client DMD is a ``tahoe backup`` structure -containing immutable snapshots of the folder tree written by that client -to the Magic Folder. As in design 2, only objects written by that client -are present. - -4. *Each* client DMD contains an eventually consistent mirror of all -files and folders written by *any* Magic Folder client. Thus each client -must also copy changes made by other Magic Folder clients to its own -client DMD. - -5. *Each* client DMD contains a ``tahoe backup`` structure containing -immutable snapshots of all files and folders written by *any* Magic -Folder client. Thus each client must also create another snapshot in its -own client DMD when changes are made by another client. (It can potentially -batch changes, subject to latency requirements.) - -6. The write coordination problem is solved by implementing `two-phase -commit`_. Then, the representation consists of a single DMD tree which is -written by all clients. - -.. _`two-phase commit`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1755 - -Here is a summary of advantages and disadvantages of each design: - -+----------------------------+ -| Key | -+=======+====================+ -| \+\+ | major advantage | -+-------+--------------------+ -| \+ | minor advantage | -+-------+--------------------+ -| ‒ | minor disadvantage | -+-------+--------------------+ -| ‒ ‒ | major disadvantage | -+-------+--------------------+ -| ‒ ‒ ‒ | showstopper | -+-------+--------------------+ - - -123456+: All designs have the property that a recursive add-lease operation -starting from a *collective directory* containing all of the client DMDs, -will find all of the files and directories used in the Magic Folder -representation. Therefore the representation is compatible with :doc:`garbage -collection <../../garbage-collection>`, even when a pre-Magic-Folder client -does the lease marking. - -123456+: All designs avoid "breaking" pre-Magic-Folder clients that read -a directory or file that is part of the representation. - -456++: Only these designs allow a readcap to one of the client -directories —or one of their subdirectories— to be directly shared -with other Tahoe-LAFS clients (not necessarily Magic Folder clients), -so that such a client sees all of the contents of the Magic Folder. -Note that this was not a requirement of the OTF proposal, although it -is useful. - -135+: A Magic Folder client has only one mutable Tahoe-LAFS object to -monitor per other client. This minimizes communication bandwidth for -polling, or alternatively the latency possible for a given polling -bandwidth. - -1236+: A client does not need to make changes to its own DMD that repeat -changes that another Magic Folder client had previously made. This reduces -write bandwidth and complexity. - -1‒: If the Magic Folder has many subfolders, their files will all be -collapsed into the same DMD, which could get quite large. In practice a -single DMD can easily handle the number of files expected to be written -by a client, so this is unlikely to be a significant issue. - -123‒ ‒: In these designs, the set of files in a Magic Folder is -represented as the union of the files in all client DMDs. However, -when a file is modified by more than one client, it will be linked -from multiple client DMDs. We therefore need a mechanism, such as a -version number or a monotonically increasing timestamp, to determine -which copy takes priority. - -35‒ ‒: When a Magic Folder client detects a remote change, it must -traverse an immutable directory structure to see what has changed. -Completely unchanged subtrees will have the same URI, allowing some of -this traversal to be shortcutted. - -24‒ ‒ ‒: When a Magic Folder client detects a remote change, it must -traverse a mutable directory structure to see what has changed. This is -more complex and less efficient than traversing an immutable structure, -because shortcutting is not possible (each DMD retains the same URI even -if a descendant object has changed), and because the structure may change -while it is being traversed. Also the traversal needs to be robust -against cycles, which can only occur in mutable structures. - -45‒ ‒: When a change occurs in one Magic Folder client, it will propagate -to all the other clients. Each client will therefore see multiple -representation changes for a single logical change to the Magic Folder -contents, and must suppress the duplicates. This is particularly -problematic for design 4 where it interacts with the preceding issue. - -4‒ ‒ ‒, 5‒ ‒: There is the potential for client DMDs to get "out of sync" -with each other, potentially for long periods if errors occur. Thus each -client must be able to "repair" its client directory (and its -subdirectory structure) concurrently with performing its own writes. This -is a significant complexity burden and may introduce failure modes that -could not otherwise happen. - -6‒ ‒ ‒: While two-phase commit is a well-established protocol, its -application to Tahoe-LAFS requires significant design work, and may still -leave some corner cases of the write coordination problem unsolved. - - -+------------------------------------------------+-----------------------------------------+ -| Design Property | Designs Proposed | -+================================================+======+======+======+======+======+======+ -| **advantages** | *1* | *2* | *3* | *4* | *5* | *6* | -+------------------------------------------------+------+------+------+------+------+------+ -| Compatible with garbage collection |\+ |\+ |\+ |\+ |\+ |\+ | -+------------------------------------------------+------+------+------+------+------+------+ -| Does not break old clients |\+ |\+ |\+ |\+ |\+ |\+ | -+------------------------------------------------+------+------+------+------+------+------+ -| Allows direct sharing | | | |\+\+ |\+\+ |\+\+ | -+------------------------------------------------+------+------+------+------+------+------+ -| Efficient use of bandwidth |\+ | |\+ | |\+ | | -+------------------------------------------------+------+------+------+------+------+------+ -| No repeated changes |\+ |\+ |\+ | | |\+ | -+------------------------------------------------+------+------+------+------+------+------+ -| **disadvantages** | *1* | *2* | *3* | *4* | *5* | *6* | -+------------------------------------------------+------+------+------+------+------+------+ -| Can result in large DMDs |‒ | | | | | | -+------------------------------------------------+------+------+------+------+------+------+ -| Need version number to determine priority |‒ ‒ |‒ ‒ |‒ ‒ | | | | -+------------------------------------------------+------+------+------+------+------+------+ -| Must traverse immutable directory structure | | |‒ ‒ | |‒ ‒ | | -+------------------------------------------------+------+------+------+------+------+------+ -| Must traverse mutable directory structure | |‒ ‒ ‒ | |‒ ‒ ‒ | | | -+------------------------------------------------+------+------+------+------+------+------+ -| Must suppress duplicate representation changes | | | |‒ ‒ |‒ ‒ | | -+------------------------------------------------+------+------+------+------+------+------+ -| "Out of sync" problem | | | |‒ ‒ ‒ |‒ ‒ | | -+------------------------------------------------+------+------+------+------+------+------+ -| Unsolved design problems | | | | | |‒ ‒ ‒ | -+------------------------------------------------+------+------+------+------+------+------+ - - -Evaluation of designs -''''''''''''''''''''' - -Designs 2 and 3 have no significant advantages over design 1, while -requiring higher polling bandwidth and greater complexity due to the need -to create subdirectories. These designs were therefore rejected. - -Design 4 was rejected due to the out-of-sync problem, which is severe -and possibly unsolvable for mutable structures. - -For design 5, the out-of-sync problem is still present but possibly -solvable. However, design 5 is substantially more complex, less efficient -in bandwidth/latency, and less scalable in number of clients and -subfolders than design 1. It only gains over design 1 on the ability to -share directory readcaps to the Magic Folder (or subfolders), which was -not a requirement. It would be possible to implement this feature in -future by switching to design 6. - -For the time being, however, design 6 was considered out-of-scope for -this project. - -Therefore, design 1 was chosen. That is: - - All subfolders written by a given Magic Folder client are collapsed - into a single client DMD, containing immutable files. The child name - of each file encodes the full subpath of that file relative to the - Magic Folder. - -Each directory entry in a DMD also stores a version number, so that the -latest version of a file is well-defined when it has been modified by -multiple clients. - -To enable representing empty directories, a client that creates a -directory should link a corresponding zero-length file in its DMD, -at a name that ends with the encoded directory separator character. - -We want to enable dynamic configuration of the membership of a Magic -Folder collective, without having to reconfigure or restart each client -when another client joins. To support this, we have a single collective -directory that links to all of the client DMDs, named by their client -nicknames. If the collective directory is mutable, then it is possible -to change its contents in order to add clients. Note that a client DMD -should not be unlinked from the collective directory unless all of its -files are first copied to some other client DMD. - -A client needs to be able to write to its own DMD, and read from other DMDs. -To be consistent with the `Principle of Least Authority`_, each client's -reference to its own DMD is a write capability, whereas its reference -to the collective directory is a read capability. The latter transitively -grants read access to all of the other client DMDs and the files linked -from them, as required. - -.. _`Principle of Least Authority`: http://www.eros-os.org/papers/secnotsep.pdf - -Design and implementation of the user interface for maintaining this -DMD structure and configuration will be addressed in Objectives 5 and 6. - -During operation, each client will poll for changes on other clients -at a predetermined frequency. On each poll, it will reread the collective -directory (to allow for added or removed clients), and then read each -client DMD linked from it. - -"Hidden" files, and files with names matching the patterns used for backup, -temporary, and conflicted files, will be ignored, i.e. not synchronized -in either direction. A file is hidden if it has a filename beginning with -"." (on any platform), or has the hidden or system attribute on Windows. - - -Conflict Detection and Resolution ---------------------------------- - -The combination of local filesystems and distributed objects is -an example of shared state concurrency, which is highly error-prone -and can result in race conditions that are complex to analyze. -Unfortunately we have no option but to use shared state in this -situation. - -We call the resulting design issues "dragons" (as in "Here be dragons"), -which as a convenient mnemonic we have named after the classical -Greek elements Earth, Fire, Air, and Water. - -Note: all filenames used in the following sections are examples, -and the filename patterns we use in the actual implementation may -differ. The actual patterns will probably include timestamps, and -for conflicted files, the nickname of the client that last changed -the file. - - -Earth Dragons: Collisions between local filesystem operations and downloads -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Write/download collisions -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Suppose that Alice's Magic Folder client is about to write a -version of ``foo`` that it has downloaded in response to a remote -change. - -The criteria for distinguishing overwrites from conflicts are -described later in the `Fire Dragons`_ section. Suppose that the -remote change has been initially classified as an overwrite. -(As we will see, it may be reclassified in some circumstances.) - -.. _`Fire Dragons`: #fire-dragons-distinguishing-conflicts-from-overwrites - -Note that writing a file that does not already have an entry in the -:ref:`magic folder db` is -initially classed as an overwrite. - -A *write/download collision* occurs when another program writes -to ``foo`` in the local filesystem, concurrently with the new -version being written by the Magic Folder client. We need to -ensure that this does not cause data loss, as far as possible. - -An important constraint on the design is that on Windows, it is -not possible to rename a file to the same name as an existing -file in that directory. Also, on Windows it may not be possible to -delete or rename a file that has been opened by another process -(depending on the sharing flags specified by that process). -Therefore we need to consider carefully how to handle failure -conditions. - -In our proposed design, Alice's Magic Folder client follows -this procedure for an overwrite in response to a remote change: - -1. Write a temporary file, say ``.foo.tmp``. -2. Use the procedure described in the `Fire Dragons_` section - to obtain an initial classification as an overwrite or a - conflict. (This takes as input the ``last_downloaded_uri`` - field from the directory entry of the changed ``foo``.) -3. Set the ``mtime`` of the replacement file to be at least *T* seconds - before the current local time. Stat the replacement file - to obtain its ``mtime`` and ``ctime`` as stored in the local - filesystem, and update the file's last-seen statinfo in - the magic folder db with this information. (Note that the - retrieved ``mtime`` may differ from the one that was set due - to rounding.) -4. Perform a ''file replacement'' operation (explained below) - with backup filename ``foo.backup``, replaced file ``foo``, - and replacement file ``.foo.tmp``. If any step of this - operation fails, reclassify as a conflict and stop. - -To reclassify as a conflict, attempt to rename ``.foo.tmp`` to -``foo.conflicted``, suppressing errors. - -The implementation of file replacement differs between Unix -and Windows. On Unix, it can be implemented as follows: - -* 4a. Stat the replaced path, and set the permissions of the - replacement file to be the same as the replaced file, - bitwise-or'd with octal 600 (``rw-------``). If the replaced - file does not exist, set the permissions according to the - user's umask. If there is a directory at the replaced path, - fail. -* 4b. Attempt to move the replaced file (``foo``) to the - backup filename (``foo.backup``). If an ``ENOENT`` error - occurs because the replaced file does not exist, ignore this - error and continue with steps 4c and 4d. -* 4c. Attempt to create a hard link at the replaced filename - (``foo``) pointing to the replacement file (``.foo.tmp``). -* 4d. Attempt to unlink the replacement file (``.foo.tmp``), - suppressing errors. - -Note that, if there is no conflict, the entry for ``foo`` -recorded in the :ref:`magic folder -db` will -reflect the ``mtime`` set in step 3. The move operation in step -4b will cause a ``MOVED_FROM`` event for ``foo``, and the link -operation in step 4c will cause an ``IN_CREATE`` event for -``foo``. However, these events will not trigger an upload, -because they are guaranteed to be processed only after the file -replacement has finished, at which point the last-seen statinfo -recorded in the database entry will exactly match the metadata -for the file's inode on disk. (The two hard links — ``foo`` -and, while it still exists, ``.foo.tmp`` — share the same inode -and therefore the same metadata.) - -On Windows, file replacement can be implemented by a call to -the `ReplaceFileW`_ API (with the -``REPLACEFILE_IGNORE_MERGE_ERRORS`` flag). If an error occurs -because the replaced file does not exist, then we ignore this -error and attempt to move the replacement file to the replaced -file. - -Similar to the Unix case, the `ReplaceFileW`_ operation will -cause one or more change notifications for ``foo``. The replaced -``foo`` has the same ``mtime`` as the replacement file, and so any -such notification(s) will not trigger an unwanted upload. - -.. _`ReplaceFileW`: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365512%28v=vs.85%29.aspx - -To determine whether this procedure adequately protects against data -loss, we need to consider what happens if another process attempts to -update ``foo``, for example by renaming ``foo.other`` to ``foo``. -This requires us to analyze all possible interleavings between the -operations performed by the Magic Folder client and the other process. -(Note that atomic operations on a directory are totally ordered.) -The set of possible interleavings differs between Windows and Unix. - -On Unix, for the case where the replaced file already exists, we have: - -* Interleaving A: the other process' rename precedes our rename in - step 4b, and we get an ``IN_MOVED_TO`` event for its rename by - step 2. Then we reclassify as a conflict; its changes end up at - ``foo`` and ours end up at ``foo.conflicted``. This avoids data - loss. - -* Interleaving B: its rename precedes ours in step 4b, and we do - not get an event for its rename by step 2. Its changes end up at - ``foo.backup``, and ours end up at ``foo`` after being linked there - in step 4c. This avoids data loss. - -* Interleaving C: its rename happens between our rename in step 4b, - and our link operation in step 4c of the file replacement. The - latter fails with an ``EEXIST`` error because ``foo`` already - exists. We reclassify as a conflict; the old version ends up at - ``foo.backup``, the other process' changes end up at ``foo``, and - ours at ``foo.conflicted``. This avoids data loss. - -* Interleaving D: its rename happens after our link in step 4c, and - causes an ``IN_MOVED_TO`` event for ``foo``. Its rename also changes - the ``mtime`` for ``foo`` so that it is different from the ``mtime`` - calculated in step 3, and therefore different from the metadata - recorded for ``foo`` in the magic folder db. (Assuming no system - clock changes, its rename will set an ``mtime`` timestamp - corresponding to a time after step 4c, which is after the timestamp - *T* seconds before step 4a, provided that *T* seconds is - sufficiently greater than the timestamp granularity.) Therefore, an - upload will be triggered for ``foo`` after its change, which is - correct and avoids data loss. - -If the replaced file did not already exist, an ``ENOENT`` error -occurs at step 4b, and we continue with steps 4c and 4d. The other -process' rename races with our link operation in step 4c. If the -other process wins the race then the effect is similar to -Interleaving C, and if we win the race this it is similar to -Interleaving D. Either case avoids data loss. - - -On Windows, the internal implementation of `ReplaceFileW`_ is similar -to what we have described above for Unix; it works like this: - -* 4a′. Copy metadata (which does not include ``mtime``) from the - replaced file (``foo``) to the replacement file (``.foo.tmp``). - -* 4b′. Attempt to move the replaced file (``foo``) onto the - backup filename (``foo.backup``), deleting the latter if it - already exists. - -* 4c′. Attempt to move the replacement file (``.foo.tmp``) to the - replaced filename (``foo``); fail if the destination already - exists. - -Notice that this is essentially the same as the algorithm we use -for Unix, but steps 4c and 4d on Unix are combined into a single -step 4c′. (If there is a failure at steps 4c′ after step 4b′ has -completed, the `ReplaceFileW`_ call will fail with return code -``ERROR_UNABLE_TO_MOVE_REPLACEMENT_2``. However, it is still -preferable to use this API over two `MoveFileExW`_ calls, because -it retains the attributes and ACLs of ``foo`` where possible. -Also note that if the `ReplaceFileW`_ call fails with -``ERROR_FILE_NOT_FOUND`` because the replaced file does not exist, -then the replacment operation ignores this error and continues with -the equivalent of step 4c′, as on Unix.) - -However, on Windows the other application will not be able to -directly rename ``foo.other`` onto ``foo`` (which would fail because -the destination already exists); it will have to rename or delete -``foo`` first. Without loss of generality, let's say ``foo`` is -deleted. This complicates the interleaving analysis, because we -have two operations done by the other process interleaving with -three done by the magic folder process (rather than one operation -interleaving with four as on Unix). - -So on Windows, for the case where the replaced file already exists, -we have: - -* Interleaving A′: the other process' deletion of ``foo`` and its - rename of ``foo.other`` to ``foo`` both precede our rename in - step 4b. We get an event corresponding to its rename by step 2. - Then we reclassify as a conflict; its changes end up at ``foo`` - and ours end up at ``foo.conflicted``. This avoids data loss. - -* Interleaving B′: the other process' deletion of ``foo`` and its - rename of ``foo.other`` to ``foo`` both precede our rename in - step 4b. We do not get an event for its rename by step 2. - Its changes end up at ``foo.backup``, and ours end up at ``foo`` - after being moved there in step 4c′. This avoids data loss. - -* Interleaving C′: the other process' deletion of ``foo`` precedes - our rename of ``foo`` to ``foo.backup`` done by `ReplaceFileW`_, - but its rename of ``foo.other`` to ``foo`` does not, so we get - an ``ERROR_FILE_NOT_FOUND`` error from `ReplaceFileW`_ indicating - that the replaced file does not exist. We ignore this error and - attempt to move ``foo.tmp`` to ``foo``, racing with the other - process which is attempting to move ``foo.other`` to ``foo``. - If we win the race, then our changes end up at ``foo``, and the - other process' move fails. If the other process wins the race, - then its changes end up at ``foo``, our move fails, and we - reclassify as a conflict, so that our changes end up at - ``foo.conflicted``. Either possibility avoids data loss. - -* Interleaving D′: the other process' deletion and/or rename happen - during the call to `ReplaceFileW`_, causing the latter to fail. - There are two subcases: - - * if the error is ``ERROR_UNABLE_TO_MOVE_REPLACEMENT_2``, then - ``foo`` is renamed to ``foo.backup`` and ``.foo.tmp`` remains - at its original name after the call. - * for all other errors, ``foo`` and ``.foo.tmp`` both remain at - their original names after the call. - - In both subcases, we reclassify as a conflict and rename ``.foo.tmp`` - to ``foo.conflicted``. This avoids data loss. - -* Interleaving E′: the other process' deletion of ``foo`` and attempt - to rename ``foo.other`` to ``foo`` both happen after all internal - operations of `ReplaceFileW`_ have completed. This causes deletion - and rename events for ``foo`` (which will in practice be merged due - to the pending delay, although we don't rely on that for - correctness). The rename also changes the ``mtime`` for ``foo`` so - that it is different from the ``mtime`` calculated in step 3, and - therefore different from the metadata recorded for ``foo`` in the - magic folder db. (Assuming no system clock changes, its rename will - set an ``mtime`` timestamp corresponding to a time after the - internal operations of `ReplaceFileW`_ have completed, which is - after the timestamp *T* seconds before `ReplaceFileW`_ is called, - provided that *T* seconds is sufficiently greater than the timestamp - granularity.) Therefore, an upload will be triggered for ``foo`` - after its change, which is correct and avoids data loss. - -.. _`MoveFileExW`: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365240%28v=vs.85%29.aspx - -If the replaced file did not already exist, we get an -``ERROR_FILE_NOT_FOUND`` error from `ReplaceFileW`_, and attempt to -move ``foo.tmp`` to ``foo``. This is similar to Interleaving C, and -either possibility for the resulting race avoids data loss. - -We also need to consider what happens if another process opens ``foo`` -and writes to it directly, rather than renaming another file onto it: - -* On Unix, open file handles refer to inodes, not paths. If the other - process opens ``foo`` before it has been renamed to ``foo.backup``, - and then closes the file, changes will have been written to the file - at the same inode, even if that inode is now linked at ``foo.backup``. - This avoids data loss. - -* On Windows, we have two subcases, depending on whether the sharing - flags specified by the other process when it opened its file handle - included ``FILE_SHARE_DELETE``. (This flag covers both deletion and - rename operations.) - - i. If the sharing flags *do not* allow deletion/renaming, the - `ReplaceFileW`_ operation will fail without renaming ``foo``. - In this case we will end up with ``foo`` changed by the other - process, and the downloaded file still in ``foo.tmp``. - This avoids data loss. - - ii. If the sharing flags *do* allow deletion/renaming, then - data loss or corruption may occur. This is unavoidable and - can be attributed to other process making a poor choice of - sharing flags (either explicitly if it used `CreateFile`_, or - via whichever higher-level API it used). - -.. _`CreateFile`: https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858%28v=vs.85%29.aspx - -Note that it is possible that another process tries to open the file -between steps 4b and 4c (or 4b′ and 4c′ on Windows). In this case the -open will fail because ``foo`` does not exist. Nevertheless, no data -will be lost, and in many cases the user will be able to retry the -operation. - -Above we only described the case where the download was initially -classified as an overwrite. If it was classed as a conflict, the -procedure is the same except that we choose a unique filename -for the conflicted file (say, ``foo.conflicted_unique``). We write -the new contents to ``.foo.tmp`` and then rename it to -``foo.conflicted_unique`` in such a way that the rename will fail -if the destination already exists. (On Windows this is a simple -rename; on Unix it can be implemented as a link operation followed -by an unlink, similar to steps 4c and 4d above.) If this fails -because another process wrote ``foo.conflicted_unique`` after we -chose the filename, then we retry with a different filename. - - -Read/download collisions -~~~~~~~~~~~~~~~~~~~~~~~~ - -A *read/download collision* occurs when another program reads -from ``foo`` in the local filesystem, concurrently with the new -version being written by the Magic Folder client. We want to -ensure that any successful attempt to read the file by the other -program obtains a consistent view of its contents. - -On Unix, the above procedure for writing downloads is sufficient -to achieve this. There are three cases: - -* A. The other process opens ``foo`` for reading before it is - renamed to ``foo.backup``. Then the file handle will continue to - refer to the old file across the rename, and the other process - will read the old contents. - -* B. The other process attempts to open ``foo`` after it has been - renamed to ``foo.backup``, and before it is linked in step c. - The open call fails, which is acceptable. - -* C. The other process opens ``foo`` after it has been linked to - the new file. Then it will read the new contents. - -On Windows, the analysis is very similar, but case A′ needs to -be split into two subcases, depending on the sharing mode the other -process uses when opening the file for reading: - -* A′. The other process opens ``foo`` before the Magic Folder - client's attempt to rename ``foo`` to ``foo.backup`` (as part - of the implementation of `ReplaceFileW`_). The subcases are: - - i. The other process uses sharing flags that deny deletion and - renames. The `ReplaceFileW`_ call fails, and the download is - reclassified as a conflict. The downloaded file ends up at - ``foo.conflicted``, which is correct. - - ii. The other process uses sharing flags that allow deletion - and renames. The `ReplaceFileW`_ call succeeds, and the - other process reads inconsistent data. This can be attributed - to a poor choice of sharing flags by the other process. - -* B′. The other process attempts to open ``foo`` at the point - during the `ReplaceFileW`_ call where it does not exist. - The open call fails, which is acceptable. - -* C′. The other process opens ``foo`` after it has been linked to - the new file. Then it will read the new contents. - - -For both write/download and read/download collisions, we have -considered only interleavings with a single other process, and -only the most common possibilities for the other process' -interaction with the file. If multiple other processes are -involved, or if a process performs operations other than those -considered, then we cannot say much about the outcome in general; -however, we believe that such cases will be much less common. - - - -Fire Dragons: Distinguishing conflicts from overwrites -'''''''''''''''''''''''''''''''''''''''''''''''''''''' - -When synchronizing a file that has changed remotely, the Magic Folder -client needs to distinguish between overwrites, in which the remote -side was aware of your most recent version (if any) and overwrote it -with a new version, and conflicts, in which the remote side was unaware -of your most recent version when it published its new version. Those two -cases have to be handled differently — the latter needs to be raised -to the user as an issue the user will have to resolve and the former -must not bother the user. - -For example, suppose that Alice's Magic Folder client sees a change -to ``foo`` in Bob's DMD. If the version it downloads from Bob's DMD -is "based on" the version currently in Alice's local filesystem at -the time Alice's client attempts to write the downloaded file ‒or if -there is no existing version in Alice's local filesystem at that time‒ -then it is an overwrite. Otherwise it is initially classified as a -conflict. - -This initial classification is used by the procedure for writing a -file described in the `Earth Dragons`_ section above. As explained -in that section, we may reclassify an overwrite as a conflict if an -error occurs during the write procedure. - -.. _`Earth Dragons`: #earth-dragons-collisions-between-local-filesystem-operations-and-downloads - -In order to implement this policy, we need to specify how the -"based on" relation between file versions is recorded and updated. - -We propose to record this information: - -* in the :ref:`magic folder - db`, for - local files; -* in the Tahoe-LAFS directory metadata, for files stored in the - Magic Folder. - -In the magic folder db we will add a *last-downloaded record*, -consisting of ``last_downloaded_uri`` and ``last_downloaded_timestamp`` -fields, for each path stored in the database. Whenever a Magic Folder -client downloads a file, it stores the downloaded version's URI and -the current local timestamp in this record. Since only immutable -files are used, the URI will be an immutable file URI, which is -deterministically and uniquely derived from the file contents and -the Tahoe-LAFS node's :doc:`convergence secret<../../convergence-secret>`. - -(Note that the last-downloaded record is updated regardless of -whether the download is an overwrite or a conflict. The rationale -for this to avoid "conflict loops" between clients, where every -new version after the first conflict would be considered as another -conflict.) - -Later, in response to a local filesystem change at a given path, the -Magic Folder client reads the last-downloaded record associated with -that path (if any) from the database and then uploads the current -file. When it links the uploaded file into its client DMD, it -includes the ``last_downloaded_uri`` field in the metadata of the -directory entry, overwriting any existing field of that name. If -there was no last-downloaded record associated with the path, this -field is omitted. - -Note that ``last_downloaded_uri`` field does *not* record the URI of -the uploaded file (which would be redundant); it records the URI of -the last download before the local change that caused the upload. -The field will be absent if the file has never been downloaded by -this client (i.e. if it was created on this client and no change -by any other client has been detected). - -A possible refinement also takes into account the -``last_downloaded_timestamp`` field from the magic folder db, and -compares it to the timestamp of the change that caused the upload -(which should be later, assuming no system clock changes). -If the duration between these timestamps is very short, then we -are uncertain about whether the process on Bob's system that wrote -the local file could have taken into account the last download. -We can use this information to be conservative about treating -changes as conflicts. So, if the duration is less than a configured -threshold, we omit the ``last_downloaded_uri`` field from the -metadata. This will have the effect of making other clients treat -this change as a conflict whenever they already have a copy of the -file. - -Conflict/overwrite decision algorithm -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Now we are ready to describe the algorithm for determining whether a -download for the file ``foo`` is an overwrite or a conflict (refining -step 2 of the procedure from the `Earth Dragons`_ section). - -Let ``last_downloaded_uri`` be the field of that name obtained from -the directory entry metadata for ``foo`` in Bob's DMD (this field -may be absent). Then the algorithm is: - -* 2a. Attempt to "stat" ``foo`` to get its *current statinfo* (size - in bytes, ``mtime``, and ``ctime``). If Alice has no local copy - of ``foo``, classify as an overwrite. - -* 2b. Read the following information for the path ``foo`` from the - local magic folder db: - - * the *last-seen statinfo*, if any (this is the size in - bytes, ``mtime``, and ``ctime`` stored in the ``local_files`` - table when the file was last uploaded); - * the ``last_uploaded_uri`` field of the ``local_files`` table - for this file, which is the URI under which the file was last - uploaded. - -* 2c. If any of the following are true, then classify as a conflict: - - * i. there are pending notifications of changes to ``foo``; - * ii. the last-seen statinfo is either absent (i.e. there is - no entry in the database for this path), or different from the - current statinfo; - * iii. either ``last_downloaded_uri`` or ``last_uploaded_uri`` - (or both) are absent, or they are different. - - Otherwise, classify as an overwrite. - - -Air Dragons: Collisions between local writes and uploads -'''''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Short of filesystem-specific features on Unix or the `shadow copy service`_ -on Windows (which is per-volume and therefore difficult to use in this -context), there is no way to *read* the whole contents of a file -atomically. Therefore, when we read a file in order to upload it, we -may read an inconsistent version if it was also being written locally. - -.. _`shadow copy service`: https://technet.microsoft.com/en-us/library/ee923636%28v=ws.10%29.aspx - -A well-behaved application can avoid this problem for its writes: - -* On Unix, if another process modifies a file by renaming a temporary - file onto it, then we will consistently read either the old contents - or the new contents. -* On Windows, if the other process uses sharing flags to deny reads - while it is writing a file, then we will consistently read either - the old contents or the new contents, unless a sharing error occurs. - In the case of a sharing error we should retry later, up to a - maximum number of retries. - -In the case of a not-so-well-behaved application writing to a file -at the same time we read from it, the magic folder will still be -eventually consistent, but inconsistent versions may be visible to -other users' clients. - -In Objective 2 we implemented a delay, called the *pending delay*, -after the notification of a filesystem change and before the file is -read in order to upload it (Tahoe-LAFS ticket `#1440`_). If another -change notification occurs within the pending delay time, the delay -is restarted. This helps to some extent because it means that if -files are written more quickly than the pending delay and less -frequently than the pending delay, we shouldn't encounter this -inconsistency. - -.. _`#1440`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1440 - -The likelihood of inconsistency could be further reduced, even for -writes by not-so-well-behaved applications, by delaying the actual -upload for a further period —called the *stability delay*— after the -file has finished being read. If a notification occurs between the -end of the pending delay and the end of the stability delay, then -the read would be aborted and the notification requeued. - -This would have the effect of ensuring that no write notifications -have been received for the file during a time window that brackets -the period when it was being read, with margin before and after -this period defined by the pending and stability delays. The delays -are intended to account for asynchronous notification of events, and -caching in the filesystem. - -Note however that we cannot guarantee that the delays will be long -enough to prevent inconsistency in any particular case. Also, the -stability delay would potentially affect performance significantly -because (unlike the pending delay) it is not overlapped when there -are multiple files on the upload queue. This performance impact -could be mitigated by uploading files in parallel where possible -(Tahoe-LAFS ticket `#1459`_). - -We have not yet decided whether to implement the stability delay, and -it is not planned to be implemented for the OTF objective 4 milestone. -Ticket `#2431`_ has been opened to track this idea. - -.. _`#1459`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1459 -.. _`#2431`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2431 - -Note that the situation of both a local process and the Magic Folder -client reading a file at the same time cannot cause any inconsistency. - - -Water Dragons: Handling deletion and renames -'''''''''''''''''''''''''''''''''''''''''''' - -Deletion of a file -~~~~~~~~~~~~~~~~~~ - -When a file is deleted from the filesystem of a Magic Folder client, -the most intuitive behavior is for it also to be deleted under that -name from other clients. To avoid data loss, the other clients should -actually rename their copies to a backup filename. - -It would not be sufficient for a Magic Folder client that deletes -a file to implement this simply by removing the directory entry from -its DMD. Indeed, the entry may not exist in the client's DMD if it -has never previously changed the file. - -Instead, the client links a zero-length file into its DMD and sets -``deleted: true`` in the directory entry metadata. Other clients -take this as a signal to rename their copies to the backup filename. - -Note that the entry for this zero-length file has a version number as -usual, and later versions may restore the file. - -When the downloader deletes a file (or renames it to a filename -ending in ``.backup``) in response to a remote change, a local -filesystem notification will occur, and we must make sure that this -is not treated as a local change. To do this we have the downloader -set the ``size`` field in the magic folder db to ``None`` (SQL NULL) -just before deleting the file, and suppress notifications for which -the local file does not exist, and the recorded ``size`` field is -``None``. - -When a Magic Folder client restarts, we can detect files that had -been downloaded but were deleted while it was not running, because -their paths will have last-downloaded records in the magic folder db -with a ``size`` other than ``None``, and without any corresponding -local file. - -Deletion of a directory -~~~~~~~~~~~~~~~~~~~~~~~ - -Local filesystems (unlike a Tahoe-LAFS filesystem) normally cannot -unlink a directory that has any remaining children. Therefore a -Magic Folder client cannot delete local copies of directories in -general, because they will typically contain backup files. This must -be done manually on each client if desired. - -Nevertheless, a Magic Folder client that deletes a directory should -set ``deleted: true`` on the metadata entry for the corresponding -zero-length file. This avoids the directory being recreated after -it has been manually deleted from a client. - -Renaming -~~~~~~~~ - -It is sufficient to handle renaming of a file by treating it as a -deletion and an addition under the new name. - -This also applies to directories, although users may find the -resulting behavior unintuitive: all of the files under the old name -will be renamed to backup filenames, and a new directory structure -created under the new name. We believe this is the best that can be -done without imposing unreasonable implementation complexity. - - -Summary -------- - -This completes the design of remote-to-local synchronization. -We realize that it may seem very complicated. Anecdotally, proprietary -filesystem synchronization designs we are aware of, such as Dropbox, -are said to incur similar or greater design complexity. diff --git a/docs/proposed/magic-folder/user-interface-design.rst b/docs/proposed/magic-folder/user-interface-design.rst deleted file mode 100644 index ea976bb00..000000000 --- a/docs/proposed/magic-folder/user-interface-design.rst +++ /dev/null @@ -1,205 +0,0 @@ -Magic Folder user interface design -================================== - -Scope ------ - -In this Objective we will design a user interface to allow users to conveniently -and securely indicate which folders on some devices should be "magically" linked -to which folders on other devices. - -This is a critical usability and security issue for which there is no known perfect -solution, but which we believe is amenable to a "good enough" trade-off solution. -This document explains the design and justifies its trade-offs in terms of security, -usability, and time-to-market. - -Tickets on the Tahoe-LAFS trac with the `otf-magic-folder-objective6`_ -keyword are within the scope of the user interface design. - -.. _otf-magic-folder-objective6: https://tahoe-lafs.org/trac/tahoe-lafs/query?status=!closed&keywords=~otf-magic-folder-objective6 - -Glossary -'''''''' - -Object: a file or directory - -DMD: distributed mutable directory - -Folder: an abstract directory that is synchronized between clients. -(A folder is not the same as the directory corresponding to it on -any particular client, nor is it the same as a DMD.) - -Collective: the set of clients subscribed to a given Magic Folder. - -Diminishing: the process of deriving, from an existing capability, -another capability that gives less authority (for example, deriving a -read cap from a read/write cap). - - -Design Constraints ------------------- - -The design of the Tahoe-side representation of a Magic Folder, and the -polling mechanism that the Magic Folder clients will use to detect remote -changes was discussed in :doc:`remote-to-local-sync`, -and we will not revisit that here. The assumption made by that design was -that each client would be configured with the following information: - -* a write cap to its own *client DMD*. -* a read cap to a *collective directory*. - -The collective directory contains links to each client DMD named by the -corresponding client's nickname. - -This design was chosen to allow straightforward addition of clients without -requiring each existing client to change its configuration. - -Note that each client in a Magic Folder collective has the authority to add, -modify or delete any object within the Magic Folder. It is also able to control -to some extent whether its writes will be treated by another client as overwrites -or as conflicts. However, there is still a reliability benefit to preventing a -client from accidentally modifying another client's DMD, or from accidentally -modifying the collective directory in a way that would lose data. This motivates -ensuring that each client only has access to the caps above, rather than, say, -every client having a write cap to the collective directory. - -Another important design constraint is that we cannot violate the :doc:`write -coordination directive<../../write_coordination>`; that is, we cannot write to -the same mutable directory from multiple clients, even during the setup phase -when adding a client. - -Within these constraints, for usability we want to minimize the number of steps -required to configure a Magic Folder collective. - - -Proposed Design ---------------- - -Three ``tahoe`` subcommands are added:: - - tahoe magic-folder create MAGIC: [MY_NICKNAME LOCAL_DIR] - - Create an empty Magic Folder. The MAGIC: local alias is set - to a write cap which can be used to refer to this Magic Folder - in future ``tahoe magic-folder invite`` commands. - - If MY_NICKNAME and LOCAL_DIR are given, the current client - immediately joins the newly created Magic Folder with that - nickname and local directory. - - - tahoe magic-folder invite MAGIC: THEIR_NICKNAME - - Print an "invitation" that can be used to invite another - client to join a Magic Folder, with the given nickname. - - The invitation must be sent to the user of the other client - over a secure channel (e.g. PGP email, OTR, or ssh). - - This command will normally be run by the same client that - created the Magic Folder. However, it may be run by a - different client if the ``MAGIC:`` alias is copied to - the ``private/aliases`` file of that other client, or if - ``MAGIC:`` is replaced by the write cap to which it points. - - - tahoe magic-folder join INVITATION LOCAL_DIR - - Accept an invitation created by ``tahoe magic-folder invite``. - The current client joins the specified Magic Folder, which will - appear in the local filesystem at the given directory. - - -There are no commands to remove a client or to revoke an -invitation, although those are possible features that could -be added in future. (When removing a client, it is necessary -to copy each file it added to some other client's DMD, if it -is the most recent version of that file.) - - -Implementation -'''''''''''''' - -For "``tahoe magic-folder create MAGIC: [MY_NICKNAME LOCAL_DIR]``" : - -1. Run "``tahoe create-alias MAGIC:``". -2. If ``MY_NICKNAME`` and ``LOCAL_DIR`` are given, do the equivalent of:: - - INVITATION=`tahoe invite-magic-folder MAGIC: MY_NICKNAME` - tahoe join-magic-folder INVITATION LOCAL_DIR - - -For "``tahoe magic-folder invite COLLECTIVE_WRITECAP NICKNAME``" : - -(``COLLECTIVE_WRITECAP`` can, as a special case, be an alias such as ``MAGIC:``.) - -1. Create an empty client DMD. Let its write URI be ``CLIENT_WRITECAP``. -2. Diminish ``CLIENT_WRITECAP`` to ``CLIENT_READCAP``, and - diminish ``COLLECTIVE_WRITECAP`` to ``COLLECTIVE_READCAP``. -3. Run "``tahoe ln CLIENT_READCAP COLLECTIVE_WRITECAP/NICKNAME``". -4. Print "``COLLECTIVE_READCAP+CLIENT_WRITECAP``" as the invitation, - accompanied by instructions on how to accept the invitation and - the need to send it over a secure channel. - - -For "``tahoe magic-folder join INVITATION LOCAL_DIR``" : - -1. Parse ``INVITATION`` as ``COLLECTIVE_READCAP+CLIENT_WRITECAP``. -2. Write ``CLIENT_WRITECAP`` to the file ``magic_folder_dircap`` - under the client's ``private`` directory. -3. Write ``COLLECTIVE_READCAP`` to the file ``collective_dircap`` - under the client's ``private`` directory. -4. Edit the client's ``tahoe.cfg`` to set - ``[magic_folder] enabled = True`` and - ``[magic_folder] local.directory = LOCAL_DIR``. - - -Discussion ----------- - -The proposed design has a minor violation of the -`Principle of Least Authority`_ in order to reduce the number -of steps needed. The invoker of "``tahoe magic-folder invite``" -creates the client DMD on behalf of the invited client, and -could retain its write cap (which is part of the invitation). - -.. _`Principle of Least Authority`: http://www.eros-os.org/papers/secnotsep.pdf - -A possible alternative design would be for the invited client -to create its own client DMD, and send it back to the inviter -to be linked into the collective directory. However this would -require another secure communication and another command -invocation per client. Given that, as mentioned earlier, each -client in a Magic Folder collective already has the authority -to add, modify or delete any object within the Magic Folder, -we considered the potential security/reliability improvement -here not to be worth the loss of usability. - -We also considered a design where each client had write access to -the collective directory. This would arguably be a more serious -violation of the Principle of Least Authority than the one above -(because all clients would have excess authority rather than just -the inviter). In any case, it was not clear how to make such a -design satisfy the :doc:`write coordination -directive<../../write_coordination>`, because the collective -directory would have needed to be written to by multiple clients. - -The reliance on a secure channel to send the invitation to its -intended recipient is not ideal, since it may involve additional -software such as clients for PGP, OTR, ssh etc. However, we believe -that this complexity is necessary rather than incidental, because -there must be some way to distinguish the intended recipient from -potential attackers who would try to become members of the Magic -Folder collective without authorization. By making use of existing -channels that have likely already been set up by security-conscious -users, we avoid reinventing the wheel or imposing substantial extra -implementation costs. - -The length of an invitation will be approximately the combined -length of a Tahoe-LAFS read cap and write cap. This is several -lines long, but still short enough to be cut-and-pasted successfully -if care is taken. Errors in copying the invitation can be detected -since Tahoe-LAFS cap URIs are self-authenticating. - -The implementation of the ``tahoe`` subcommands is straightforward -and raises no further difficult design issues. diff --git a/docs/windows.rst b/docs/windows.rst index aa1c3a086..568e502bc 100644 --- a/docs/windows.rst +++ b/docs/windows.rst @@ -77,9 +77,9 @@ If you're planning to hack on the source code, you might want to add Dependencies ------------ -Tahoe-LAFS depends upon several packages that use compiled C code, such as -zfec, pycryptopp, and others. This code must be built separately for each -platform (Windows, OS-X, and different flavors of Linux). +Tahoe-LAFS depends upon several packages that use compiled C code +(such as zfec). This code must be built separately for each platform +(Windows, OS-X, and different flavors of Linux). Pre-compiled "wheels" of all Tahoe's dependencies are hosted on the tahoe-lafs.org website in the ``deps/`` directory. The ``--find-links=`` diff --git a/integration/conftest.py b/integration/conftest.py index 98091c7ce..5395d7c5f 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -3,7 +3,7 @@ from __future__ import print_function import sys import shutil from time import sleep -from os import mkdir, listdir +from os import mkdir, listdir, environ from os.path import join, exists from tempfile import mkdtemp, mktemp from functools import partial @@ -15,6 +15,7 @@ from eliot import ( ) from twisted.python.procutils import which +from twisted.internet.defer import DeferredList from twisted.internet.error import ( ProcessExitedAlready, ProcessTerminated, @@ -30,7 +31,10 @@ from util import ( _ProcessExitedProtocol, _create_node, _run_node, - _cleanup_twistd_process, + _cleanup_tahoe_process, + _tahoe_runner_optional_coverage, + await_client_ready, + TahoeProcess, ) @@ -41,6 +45,10 @@ def pytest_addoption(parser): "--keep-tempdir", action="store_true", dest="keep", help="Keep the tmpdir with the client directories (introducer, etc)", ) + parser.addoption( + "--coverage", action="store_true", dest="coverage", + help="Collect coverage statistics", + ) @pytest.fixture(autouse=True, scope='session') def eliot_logging(): @@ -125,7 +133,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): pytest_twisted.blockon(twistd_protocol.magic_seen) def cleanup(): - _cleanup_twistd_process(twistd_process, twistd_protocol.exited) + _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) flog_file = mktemp('.flog_dump') flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) @@ -174,11 +182,11 @@ log_gatherer.furl = {log_furl} if not exists(intro_dir): mkdir(intro_dir) done_proto = _ProcessExitedProtocol() - reactor.spawnProcess( + _tahoe_runner_optional_coverage( done_proto, - sys.executable, + reactor, + request, ( - sys.executable, '-m', 'allmydata.scripts.runner', 'create-introducer', '--listen=tcp', '--hostname=localhost', @@ -195,19 +203,19 @@ log_gatherer.furl = {log_furl} # but on linux it means daemonize. "tahoe run" is consistent # between platforms. protocol = _MagicTextProtocol('introducer running') - process = reactor.spawnProcess( + transport = _tahoe_runner_optional_coverage( protocol, - sys.executable, + reactor, + request, ( - sys.executable, '-m', 'allmydata.scripts.runner', 'run', intro_dir, ), ) - request.addfinalizer(partial(_cleanup_twistd_process, process, protocol.exited)) + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) pytest_twisted.blockon(protocol.magic_seen) - return process + return TahoeProcess(transport, intro_dir) @pytest.fixture(scope='session') @@ -241,11 +249,11 @@ log_gatherer.furl = {log_furl} if not exists(intro_dir): mkdir(intro_dir) done_proto = _ProcessExitedProtocol() - reactor.spawnProcess( + _tahoe_runner_optional_coverage( done_proto, - sys.executable, + reactor, + request, ( - sys.executable, '-m', 'allmydata.scripts.runner', 'create-introducer', '--tor-control-port', 'tcp:localhost:8010', '--listen=tor', @@ -262,11 +270,11 @@ log_gatherer.furl = {log_furl} # but on linux it means daemonize. "tahoe run" is consistent # between platforms. protocol = _MagicTextProtocol('introducer running') - process = reactor.spawnProcess( + transport = _tahoe_runner_optional_coverage( protocol, - sys.executable, + reactor, + request, ( - sys.executable, '-m', 'allmydata.scripts.runner', 'run', intro_dir, ), @@ -274,14 +282,14 @@ log_gatherer.furl = {log_furl} def cleanup(): try: - process.signalProcess('TERM') + transport.signalProcess('TERM') pytest_twisted.blockon(protocol.exited) except ProcessExitedAlready: pass request.addfinalizer(cleanup) pytest_twisted.blockon(protocol.magic_seen) - return process + return transport @pytest.fixture(scope='session') @@ -301,31 +309,29 @@ def tor_introducer_furl(tor_introducer, temp_dir): include_result=False, ) def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request): - nodes = [] + nodes_d = [] # start all 5 nodes in parallel for x in range(5): name = 'node{}'.format(x) - # tub_port = 9900 + x - nodes.append( - pytest_twisted.blockon( - _create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, name, - web_port=None, storage=True, - ) + web_port= 9990 + x + nodes_d.append( + _create_node( + reactor, request, temp_dir, introducer_furl, flog_gatherer, name, + web_port="tcp:{}:interface=localhost".format(web_port), + storage=True, ) ) - #nodes = pytest_twisted.blockon(DeferredList(nodes)) + nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) + nodes = [] + for ok, process in nodes_status: + assert ok, "Storage node creation failed: {}".format(process) + nodes.append(process) return nodes @pytest.fixture(scope='session') @log_call(action_type=u"integration:alice", include_args=[], include_result=False) def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request): - try: - mkdir(join(temp_dir, 'magic-alice')) - except OSError: - pass - process = pytest_twisted.blockon( _create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", @@ -333,17 +339,13 @@ def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, requ storage=False, ) ) + await_client_ready(process) return process @pytest.fixture(scope='session') @log_call(action_type=u"integration:bob", include_args=[], include_result=False) def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request): - try: - mkdir(join(temp_dir, 'magic-bob')) - except OSError: - pass - process = pytest_twisted.blockon( _create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob", @@ -351,99 +353,10 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques storage=False, ) ) + await_client_ready(process) return process -@pytest.fixture(scope='session') -@log_call(action_type=u"integration:alice:invite", include_args=["temp_dir"]) -def alice_invite(reactor, alice, temp_dir, request): - node_dir = join(temp_dir, 'alice') - - with start_action(action_type=u"integration:alice:magic_folder:create"): - # FIXME XXX by the time we see "client running" in the logs, the - # storage servers aren't "really" ready to roll yet (uploads fairly - # consistently fail if we don't hack in this pause...) - import time ; time.sleep(5) - proto = _CollectOutputProtocol() - reactor.spawnProcess( - proto, - sys.executable, - [ - sys.executable, '-m', 'allmydata.scripts.runner', - 'magic-folder', 'create', - '--poll-interval', '2', - '--basedir', node_dir, 'magik:', 'alice', - join(temp_dir, 'magic-alice'), - ] - ) - pytest_twisted.blockon(proto.done) - - with start_action(action_type=u"integration:alice:magic_folder:invite") as a: - proto = _CollectOutputProtocol() - reactor.spawnProcess( - proto, - sys.executable, - [ - sys.executable, '-m', 'allmydata.scripts.runner', - 'magic-folder', 'invite', - '--basedir', node_dir, 'magik:', 'bob', - ] - ) - pytest_twisted.blockon(proto.done) - invite = proto.output.getvalue() - a.add_success_fields(invite=invite) - - with start_action(action_type=u"integration:alice:magic_folder:restart"): - # before magic-folder works, we have to stop and restart (this is - # crappy for the tests -- can we fix it in magic-folder?) - try: - alice.signalProcess('TERM') - pytest_twisted.blockon(alice.exited) - except ProcessExitedAlready: - pass - with start_action(action_type=u"integration:alice:magic_folder:magic-text"): - magic_text = 'Completed initial Magic Folder scan successfully' - pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text)) - return invite - - -@pytest.fixture(scope='session') -@log_call( - action_type=u"integration:magic_folder", - include_args=["alice_invite", "temp_dir"], -) -def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request): - print("pairing magic-folder") - bob_dir = join(temp_dir, 'bob') - proto = _CollectOutputProtocol() - reactor.spawnProcess( - proto, - sys.executable, - [ - sys.executable, '-m', 'allmydata.scripts.runner', - 'magic-folder', 'join', - '--poll-interval', '2', - '--basedir', bob_dir, - alice_invite, - join(temp_dir, 'magic-bob'), - ] - ) - pytest_twisted.blockon(proto.done) - - # before magic-folder works, we have to stop and restart (this is - # crappy for the tests -- can we fix it in magic-folder?) - try: - print("Sending TERM to Bob") - bob.signalProcess('TERM') - pytest_twisted.blockon(bob.exited) - except ProcessExitedAlready: - pass - - magic_text = 'Completed initial Magic Folder scan successfully' - pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text)) - return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob')) - - @pytest.fixture(scope='session') def chutney(reactor, temp_dir): chutney_dir = join(temp_dir, 'chutney') @@ -462,12 +375,13 @@ def chutney(reactor, temp_dir): proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, - '/usr/bin/git', + 'git', ( - '/usr/bin/git', 'clone', '--depth=1', + 'git', 'clone', '--depth=1', 'https://git.torproject.org/chutney.git', chutney_dir, - ) + ), + env=environ, ) pytest_twisted.blockon(proto.done) return chutney_dir @@ -483,6 +397,8 @@ def tor_network(reactor, temp_dir, chutney, request): # ./chutney configure networks/basic # ./chutney start networks/basic + env = environ.copy() + env.update({"PYTHONPATH": join(chutney_dir, "lib")}) proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -492,7 +408,7 @@ def tor_network(reactor, temp_dir, chutney, request): join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), - env={"PYTHONPATH": join(chutney_dir, "lib")}, + env=env, ) pytest_twisted.blockon(proto.done) @@ -505,7 +421,7 @@ def tor_network(reactor, temp_dir, chutney, request): join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), - env={"PYTHONPATH": join(chutney_dir, "lib")}, + env=env, ) pytest_twisted.blockon(proto.done) @@ -519,7 +435,7 @@ def tor_network(reactor, temp_dir, chutney, request): join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), - env={"PYTHONPATH": join(chutney_dir, "lib")}, + env=env, ) try: pytest_twisted.blockon(proto.done) @@ -538,7 +454,7 @@ def tor_network(reactor, temp_dir, chutney, request): join(chutney_dir, 'networks', 'basic'), ), path=join(chutney_dir), - env={"PYTHONPATH": join(chutney_dir, "lib")}, + env=env, ) pytest_twisted.blockon(proto.done) request.addfinalizer(cleanup) diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py index e4698a12a..4a2ef71a6 100644 --- a/integration/test_aaa_aardvark.py +++ b/integration/test_aaa_aardvark.py @@ -16,7 +16,3 @@ def test_create_introducer(introducer): def test_create_storage(storage_nodes): print("Created {} storage nodes".format(len(storage_nodes))) - - -def test_create_alice_bob_magicfolder(magic_folder): - print("Alice and Bob have paired magic-folders") diff --git a/integration/test_magic_folder.py b/integration/test_magic_folder.py deleted file mode 100644 index 2691639e6..000000000 --- a/integration/test_magic_folder.py +++ /dev/null @@ -1,461 +0,0 @@ -import sys -import time -import shutil -from os import mkdir, unlink, utime -from os.path import join, exists, getmtime - -import util - -import pytest_twisted - - -# tests converted from check_magicfolder_smoke.py -# see "conftest.py" for the fixtures (e.g. "magic_folder") - -def test_eliot_logs_are_written(alice, bob, temp_dir): - # The integration test configuration arranges for this logging - # configuration. Verify it actually does what we want. - # - # The alice and bob arguments looks unused but they actually tell pytest - # to set up all the magic-folder stuff. The assertions here are about - # side-effects of that setup. - assert exists(join(temp_dir, "alice", "logs", "eliot.json")) - assert exists(join(temp_dir, "bob", "logs", "eliot.json")) - - -def test_alice_writes_bob_receives(magic_folder): - alice_dir, bob_dir = magic_folder - - with open(join(alice_dir, "first_file"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents(join(bob_dir, "first_file"), "alice wrote this") - return - - -def test_alice_writes_bob_receives_multiple(magic_folder): - """ - When Alice does a series of updates, Bob should just receive them - with no .backup or .conflict files being produced. - """ - alice_dir, bob_dir = magic_folder - - unwanted_files = [ - join(bob_dir, "multiple.backup"), - join(bob_dir, "multiple.conflict") - ] - - # first update - with open(join(alice_dir, "multiple"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents( - join(bob_dir, "multiple"), "alice wrote this", - error_if=unwanted_files, - ) - - # second update - with open(join(alice_dir, "multiple"), "w") as f: - f.write("someone changed their mind") - - util.await_file_contents( - join(bob_dir, "multiple"), "someone changed their mind", - error_if=unwanted_files, - ) - - # third update - with open(join(alice_dir, "multiple"), "w") as f: - f.write("absolutely final version ship it") - - util.await_file_contents( - join(bob_dir, "multiple"), "absolutely final version ship it", - error_if=unwanted_files, - ) - - # forth update, but both "at once" so one should conflict - time.sleep(2) - with open(join(alice_dir, "multiple"), "w") as f: - f.write("okay one more attempt") - with open(join(bob_dir, "multiple"), "w") as f: - f.write("...but just let me add") - - bob_conflict = join(bob_dir, "multiple.conflict") - alice_conflict = join(alice_dir, "multiple.conflict") - - found = util.await_files_exist([ - bob_conflict, - alice_conflict, - ]) - - assert len(found) > 0, "Should have found a conflict" - print("conflict found (as expected)") - - -def test_alice_writes_bob_receives_old_timestamp(magic_folder): - alice_dir, bob_dir = magic_folder - fname = join(alice_dir, "ts_file") - ts = time.time() - (60 * 60 * 36) # 36 hours ago - - with open(fname, "w") as f: - f.write("alice wrote this") - utime(fname, (time.time(), ts)) - - fname = join(bob_dir, "ts_file") - util.await_file_contents(fname, "alice wrote this") - # make sure the timestamp is correct - assert int(getmtime(fname)) == int(ts) - return - - -def test_bob_writes_alice_receives(magic_folder): - alice_dir, bob_dir = magic_folder - - with open(join(bob_dir, "second_file"), "w") as f: - f.write("bob wrote this") - - util.await_file_contents(join(alice_dir, "second_file"), "bob wrote this") - return - - -def test_alice_deletes(magic_folder): - # alice writes a file, waits for bob to get it and then deletes it. - alice_dir, bob_dir = magic_folder - - with open(join(alice_dir, "delfile"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents(join(bob_dir, "delfile"), "alice wrote this") - - # bob has the file; now alices deletes it - unlink(join(alice_dir, "delfile")) - - # bob should remove his copy, but preserve a backup - util.await_file_vanishes(join(bob_dir, "delfile")) - util.await_file_contents(join(bob_dir, "delfile.backup"), "alice wrote this") - return - - -def test_alice_creates_bob_edits(magic_folder): - alice_dir, bob_dir = magic_folder - - # alice writes a file - with open(join(alice_dir, "editfile"), "w") as f: - f.write("alice wrote this") - - util.await_file_contents(join(bob_dir, "editfile"), "alice wrote this") - - # now bob edits it - with open(join(bob_dir, "editfile"), "w") as f: - f.write("bob says foo") - - util.await_file_contents(join(alice_dir, "editfile"), "bob says foo") - - -def test_bob_creates_sub_directory(magic_folder): - alice_dir, bob_dir = magic_folder - - # bob makes a sub-dir, with a file in it - mkdir(join(bob_dir, "subdir")) - with open(join(bob_dir, "subdir", "a_file"), "w") as f: - f.write("bob wuz here") - - # alice gets it - util.await_file_contents(join(alice_dir, "subdir", "a_file"), "bob wuz here") - - # now bob deletes it again - shutil.rmtree(join(bob_dir, "subdir")) - - # alice should delete it as well - util.await_file_vanishes(join(alice_dir, "subdir", "a_file")) - # i *think* it's by design that the subdir won't disappear, - # because a "a_file.backup" should appear... - util.await_file_contents(join(alice_dir, "subdir", "a_file.backup"), "bob wuz here") - - -def test_bob_creates_alice_deletes_bob_restores(magic_folder): - alice_dir, bob_dir = magic_folder - - # bob creates a file - with open(join(bob_dir, "boom"), "w") as f: - f.write("bob wrote this") - - util.await_file_contents( - join(alice_dir, "boom"), - "bob wrote this" - ) - - # alice deletes it (so bob should as well .. but keep a backup) - unlink(join(alice_dir, "boom")) - util.await_file_vanishes(join(bob_dir, "boom")) - assert exists(join(bob_dir, "boom.backup")) - - # bob restore it, with new contents - unlink(join(bob_dir, "boom.backup")) - with open(join(bob_dir, "boom"), "w") as f: - f.write("bob wrote this again, because reasons") - - # XXX double-check this behavior is correct! - - # alice sees bob's update, but marks it as a conflict (because - # .. she previously deleted it? does that really make sense) - - util.await_file_contents( - join(alice_dir, "boom"), - "bob wrote this again, because reasons", - ) - - -def test_bob_creates_alice_deletes_alice_restores(magic_folder): - alice_dir, bob_dir = magic_folder - - # bob creates a file - with open(join(bob_dir, "boom2"), "w") as f: - f.write("bob wrote this") - - util.await_file_contents( - join(alice_dir, "boom2"), - "bob wrote this" - ) - - # alice deletes it (so bob should as well) - unlink(join(alice_dir, "boom2")) - util.await_file_vanishes(join(bob_dir, "boom2")) - - # alice restore it, with new contents - with open(join(alice_dir, "boom2"), "w") as f: - f.write("alice re-wrote this again, because reasons") - - util.await_file_contents( - join(bob_dir, "boom2"), - "alice re-wrote this again, because reasons" - ) - - -def test_bob_conflicts_with_alice_fresh(magic_folder): - # both alice and bob make a file at "the same time". - alice_dir, bob_dir = magic_folder - - # either alice or bob will "win" by uploading to the DMD first. - with open(join(bob_dir, 'alpha'), 'w') as f0, open(join(alice_dir, 'alpha'), 'w') as f1: - f0.write("this is bob's alpha\n") - f1.write("this is alice's alpha\n") - - # there should be conflicts - _bob_conflicts_alice_await_conflicts('alpha', alice_dir, bob_dir) - - -def test_bob_conflicts_with_alice_preexisting(magic_folder): - # both alice and bob edit a file at "the same time" (similar to - # above, but the file already exists before the edits) - alice_dir, bob_dir = magic_folder - - # have bob create the file - with open(join(bob_dir, 'beta'), 'w') as f: - f.write("original beta (from bob)\n") - util.await_file_contents(join(alice_dir, 'beta'), "original beta (from bob)\n") - - # both alice and bob now have a "beta" file, at version 0 - - # either alice or bob will "win" by uploading to the DMD first - # (however, they should both detect a conflict) - with open(join(bob_dir, 'beta'), 'w') as f: - f.write("this is bob's beta\n") - with open(join(alice_dir, 'beta'), 'w') as f: - f.write("this is alice's beta\n") - - # both alice and bob should see a conflict - _bob_conflicts_alice_await_conflicts("beta", alice_dir, bob_dir) - - -def _bob_conflicts_alice_await_conflicts(name, alice_dir, bob_dir): - """ - shared code between _fresh and _preexisting conflict test - """ - found = util.await_files_exist( - [ - join(bob_dir, '{}.conflict'.format(name)), - join(alice_dir, '{}.conflict'.format(name)), - ], - ) - - assert len(found) >= 1, "should be at least one conflict" - assert open(join(bob_dir, name), 'r').read() == "this is bob's {}\n".format(name) - assert open(join(alice_dir, name), 'r').read() == "this is alice's {}\n".format(name) - - alice_conflict = join(alice_dir, '{}.conflict'.format(name)) - bob_conflict = join(bob_dir, '{}.conflict'.format(name)) - if exists(bob_conflict): - assert open(bob_conflict, 'r').read() == "this is alice's {}\n".format(name) - if exists(alice_conflict): - assert open(alice_conflict, 'r').read() == "this is bob's {}\n".format(name) - - -@pytest_twisted.inlineCallbacks -def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes): - """ - ticket 2880: if a magic-folder client uploads something, then - re-starts a spurious .backup file should not appear - """ - - edmond_dir = join(temp_dir, 'edmond') - edmond = yield util._create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, - "edmond", web_port="tcp:9985:interface=localhost", - storage=False, - ) - - - magic_folder = join(temp_dir, 'magic-edmond') - mkdir(magic_folder) - created = False - # create a magic-folder - # (how can we know that the grid is ready?) - for _ in range(10): # try 10 times - try: - proto = util._CollectOutputProtocol() - transport = reactor.spawnProcess( - proto, - sys.executable, - [ - sys.executable, '-m', 'allmydata.scripts.runner', - 'magic-folder', 'create', - '--poll-interval', '2', - '--basedir', edmond_dir, - 'magik:', - 'edmond_magic', - magic_folder, - ] - ) - yield proto.done - created = True - break - except Exception as e: - print("failed to create magic-folder: {}".format(e)) - time.sleep(1) - - assert created, "Didn't create a magic-folder" - - # to actually-start the magic-folder we have to re-start - edmond.signalProcess('TERM') - yield edmond._protocol.exited - time.sleep(1) - edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully') - - # add a thing to the magic-folder - with open(join(magic_folder, "its_a_file"), "w") as f: - f.write("edmond wrote this") - - # fixme, do status-update attempts in a loop below - time.sleep(5) - - # let it upload; poll the HTTP magic-folder status API until it is - # uploaded - from allmydata.scripts.magic_folder_cli import _get_json_for_fragment - - with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f: - token = f.read() - - uploaded = False - for _ in range(10): - options = { - "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(), - } - try: - magic_data = _get_json_for_fragment( - options, - 'magic_folder?t=json', - method='POST', - post_args=dict( - t='json', - name='default', - token=token, - ) - ) - for mf in magic_data: - if mf['status'] == u'success' and mf['path'] == u'its_a_file': - uploaded = True - break - except Exception as e: - time.sleep(1) - - assert uploaded, "expected to upload 'its_a_file'" - - # re-starting edmond right now would "normally" trigger the 2880 bug - - # kill edmond - edmond.signalProcess('TERM') - yield edmond._protocol.exited - time.sleep(1) - edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully') - - # XXX how can we say for sure if we've waited long enough? look at - # tail of logs for magic-folder ... somethingsomething? - print("waiting 20 seconds to see if a .backup appears") - for _ in range(20): - assert exists(join(magic_folder, "its_a_file")) - assert not exists(join(magic_folder, "its_a_file.backup")) - time.sleep(1) - - -@pytest_twisted.inlineCallbacks -def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder): - """ - Alice can add new files to a magic folder while Bob is offline. When Bob - comes back online his copy is updated to reflect the new files. - """ - alice_magic_dir, bob_magic_dir = magic_folder - alice_node_dir = join(temp_dir, "alice") - bob_node_dir = join(temp_dir, "bob") - - # Take Bob offline. - yield util.cli(reactor, bob_node_dir, "stop") - - # Create a couple files in Alice's local directory. - some_files = list( - (name * 3) + ".added-while-offline" - for name - in "xyz" - ) - for name in some_files: - with open(join(alice_magic_dir, name), "w") as f: - f.write(name + " some content") - - good = False - for i in range(15): - status = yield util.magic_folder_cli(reactor, alice_node_dir, "status") - good = status.count(".added-while-offline (36 B): good, version=0") == len(some_files) * 2 - if good: - # We saw each file as having a local good state and a remote good - # state. That means we're ready to involve Bob. - break - else: - time.sleep(1.0) - - assert good, ( - "Timed out waiting for good Alice state. Last status:\n{}".format(status) - ) - - # Start Bob up again - magic_text = 'Completed initial Magic Folder scan successfully' - yield util._run_node(reactor, bob_node_dir, request, magic_text) - - yield util.await_files_exist( - list( - join(bob_magic_dir, name) - for name - in some_files - ), - await_all=True, - ) - # Let it settle. It would be nicer to have a readable status output we - # could query. Parsing the current text format is more than I want to - # deal with right now. - time.sleep(1.0) - conflict_files = list(name + ".conflict" for name in some_files) - assert all( - list( - not exists(join(bob_magic_dir, name)) - for name - in conflict_files - ), - ) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index c9b654d9c..fd8f75e39 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -12,7 +12,7 @@ import pytest_twisted @pytest_twisted.inlineCallbacks def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request): - yield util._create_node( + edna = yield util._create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "edna", web_port="tcp:9983:interface=localhost", storage=False, @@ -20,13 +20,10 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto happy=7, total=10, ) - + util.await_client_ready(edna) node_dir = join(temp_dir, 'edna') - print("waiting 10 seconds unil we're maybe ready") - yield task.deferLater(reactor, 10, lambda: None) - # upload a file, which should fail because we have don't have 7 # storage servers (but happiness is set to 7) proto = util._CollectOutputProtocol() diff --git a/integration/test_tor.py b/integration/test_tor.py index 801efc3ca..a16c1b846 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -14,7 +14,7 @@ import pytest_twisted import util -# see "conftest.py" for the fixtures (e.g. "magic_folder") +# see "conftest.py" for the fixtures (e.g. "tor_network") @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): diff --git a/integration/test_web.py b/integration/test_web.py new file mode 100644 index 000000000..4ba0a6fd1 --- /dev/null +++ b/integration/test_web.py @@ -0,0 +1,521 @@ +""" +These tests were originally written to achieve some level of +coverage for the WebAPI functionality during Python3 porting (there +aren't many tests of the Web API period). + +Most of the tests have cursory asserts and encode 'what the WebAPI did +at the time of testing' -- not necessarily a cohesive idea of what the +WebAPI *should* do in every situation. It's not clear the latter +exists anywhere, however. +""" + +import sys +import time +import shutil +import json +import urllib2 +from os import mkdir, unlink, utime +from os.path import join, exists, getmtime + +import allmydata.uri + +import util + +import requests +import pytest_twisted +import html5lib +from bs4 import BeautifulSoup + + +def test_index(alice): + """ + we can download the index file + """ + util.web_get(alice, u"") + + +def test_index_json(alice): + """ + we can download the index file as json + """ + data = util.web_get(alice, u"", params={u"t": u"json"}) + # it should be valid json + json.loads(data) + + +def test_upload_download(alice): + """ + upload a file, then download it via readcap + """ + + FILE_CONTENTS = u"some contents" + + readcap = util.web_post( + alice, u"uri", + data={ + u"t": u"upload", + u"format": u"mdmf", + }, + files={ + u"file": FILE_CONTENTS, + }, + ) + readcap = readcap.strip() + + data = util.web_get( + alice, u"uri", + params={ + u"uri": readcap, + u"filename": u"boom", + } + ) + assert data == FILE_CONTENTS + + +def test_put(alice): + """ + use PUT to create a file + """ + + FILE_CONTENTS = b"added via PUT" * 20 + + resp = requests.put( + util.node_url(alice.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + cap = allmydata.uri.from_string(resp.text.strip().encode('ascii')) + cfg = alice.get_config() + assert isinstance(cap, allmydata.uri.CHKFileURI) + assert cap.size == len(FILE_CONTENTS) + assert cap.total_shares == int(cfg.get_config("client", "shares.total")) + assert cap.needed_shares == int(cfg.get_config("client", "shares.needed")) + + +def test_helper_status(storage_nodes): + """ + successfully GET the /helper_status page + """ + + url = util.node_url(storage_nodes[0].node_dir, "helper_status") + resp = requests.get(url) + assert resp.status_code >= 200 and resp.status_code < 300 + dom = BeautifulSoup(resp.content, "html5lib") + assert unicode(dom.h1.string) == u"Helper Status" + + +def test_deep_stats(alice): + """ + create a directory, do deep-stats on it and prove the /operations/ + URIs work + """ + resp = requests.post( + util.node_url(alice.node_dir, "uri"), + params={ + "format": "sdmf", + "t": "mkdir", + "redirect_to_result": "true", + }, + ) + assert resp.status_code >= 200 and resp.status_code < 300 + + # when creating a directory, we'll be re-directed to a URL + # containing our writecap.. + uri = urllib2.unquote(resp.url) + assert 'URI:DIR2:' in uri + dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') + dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap))) + + # POST a file into this directory + FILE_CONTENTS = u"a file in a directory" + + resp = requests.post( + dircap_uri, + data={ + u"t": u"upload", + u"when_done": u".", + }, + files={ + u"file": FILE_CONTENTS, + }, + ) + + # confirm the file is in the directory + resp = requests.get( + dircap_uri, + params={ + u"t": u"json", + }, + ) + d = json.loads(resp.content) + k, data = d + assert k == u"dirnode" + assert len(data['children']) == 1 + k, child = data['children'].values()[0] + assert k == u"filenode" + assert child['size'] == len(FILE_CONTENTS) + + # perform deep-stats on it... + resp = requests.post( + dircap_uri, + data={ + u"t": u"start-deep-stats", + u"ophandle": u"something_random", + }, + ) + assert resp.status_code >= 200 and resp.status_code < 300 + + # confirm we get information from the op .. after its done + tries = 10 + while tries > 0: + tries -= 1 + resp = requests.get( + util.node_url(alice.node_dir, u"operations/something_random"), + ) + d = json.loads(resp.content) + if d['size-literal-files'] == len(FILE_CONTENTS): + print("stats completed successfully") + break + else: + print("{} != {}; waiting".format(d['size-literal-files'], len(FILE_CONTENTS))) + time.sleep(.5) + + +def test_status(alice): + """ + confirm we get something sensible from /status and the various sub-types + """ + + # upload a file + # (because of the nature of the integration-tests, we can only + # assert things about "our" file because we don't know what other + # operations may have happened in the grid before our test runs). + + FILE_CONTENTS = u"all the Important Data of alice\n" * 1200 + + resp = requests.put( + util.node_url(alice.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + cap = resp.text.strip() + + print("Uploaded data, cap={}".format(cap)) + resp = requests.get( + util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))), + ) + + print("Downloaded {} bytes of data".format(len(resp.content))) + assert resp.content == FILE_CONTENTS + + resp = requests.get( + util.node_url(alice.node_dir, "status"), + ) + dom = html5lib.parse(resp.content) + + hrefs = [ + a.get('href') + for a in dom.iter(u'{http://www.w3.org/1999/xhtml}a') + ] + + found_upload = False + found_download = False + for href in hrefs: + if href.startswith(u"/") or not href: + continue + resp = requests.get( + util.node_url(alice.node_dir, u"status/{}".format(href)), + ) + if href.startswith(u'up'): + assert "File Upload Status" in resp.content + if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + found_upload = True + elif href.startswith(u'down'): + assert "File Download Status" in resp.content + if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + found_download = True + + # download the specialized event information + resp = requests.get( + util.node_url(alice.node_dir, u"status/{}/event_json".format(href)), + ) + js = json.loads(resp.content) + # there's usually just one "read" operation, but this can handle many .. + total_bytes = sum([st['bytes_returned'] for st in js['read']], 0) + assert total_bytes == len(FILE_CONTENTS) + + + assert found_upload, "Failed to find the file we uploaded in the status-page" + assert found_download, "Failed to find the file we downloaded in the status-page" + + +def test_directory_deep_check(alice): + """ + use deep-check and confirm the result pages work + """ + + # create a directory + resp = requests.post( + util.node_url(alice.node_dir, u"uri"), + params={ + u"t": u"mkdir", + u"redirect_to_result": u"true", + } + ) + + # get json information about our directory + dircap_url = resp.url + resp = requests.get( + dircap_url, + params={u"t": u"json"}, + ) + dir_meta = json.loads(resp.content) + + # upload a file of pangrams into the directory + FILE_CONTENTS = u"Sphinx of black quartz, judge my vow.\n" * (2048*10) + + resp = requests.post( + dircap_url, + params={ + u"t": u"upload", + u"upload-chk": u"upload-chk", + }, + files={ + u"file": FILE_CONTENTS, + } + ) + cap0 = resp.content + print("Uploaded data0, cap={}".format(cap0)) + + # a different pangram + FILE_CONTENTS = u"The five boxing wizards jump quickly.\n" * (2048*10) + + resp = requests.post( + dircap_url, + params={ + u"t": u"upload", + u"upload-chk": u"upload-chk", + }, + files={ + u"file": FILE_CONTENTS, + } + ) + cap1 = resp.content + print("Uploaded data1, cap={}".format(cap1)) + + resp = requests.get( + util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))), + params={u"t": u"info"}, + ) + + def check_repair_data(checkdata): + assert checkdata["healthy"] is True + assert checkdata["count-happiness"] == 4 + assert checkdata["count-good-share-hosts"] == 4 + assert checkdata["count-shares-good"] == 4 + assert checkdata["count-corrupt-shares"] == 0 + assert checkdata["list-corrupt-shares"] == [] + + # do a "check" (once for HTML, then with JSON for easier asserts) + resp = requests.post( + dircap_url, + params={ + u"t": u"check", + u"return_to": u".", + u"verify": u"true", + } + ) + resp = requests.post( + dircap_url, + params={ + u"t": u"check", + u"return_to": u".", + u"verify": u"true", + u"output": u"JSON", + } + ) + check_repair_data(json.loads(resp.content)["results"]) + + # "check and repair" + resp = requests.post( + dircap_url, + params={ + u"t": u"check", + u"return_to": u".", + u"verify": u"true", + u"repair": u"true", + } + ) + resp = requests.post( + dircap_url, + params={ + u"t": u"check", + u"return_to": u".", + u"verify": u"true", + u"repair": u"true", + u"output": u"JSON", + } + ) + check_repair_data(json.loads(resp.content)["post-repair-results"]["results"]) + + # start a "deep check and repair" + resp = requests.post( + dircap_url, + params={ + u"t": u"start-deep-check", + u"return_to": u".", + u"verify": u"on", + u"repair": u"on", + u"output": u"JSON", + u"ophandle": u"deadbeef", + } + ) + deepcheck_uri = resp.url + + data = json.loads(resp.content) + tries = 10 + while not data['finished'] and tries > 0: + tries -= 1 + time.sleep(0.5) + print("deep-check not finished, reloading") + resp = requests.get(deepcheck_uri, params={u"output": "JSON"}) + data = json.loads(resp.content) + print("deep-check finished") + assert data[u"stats"][u"count-immutable-files"] == 1 + assert data[u"stats"][u"count-literal-files"] == 0 + assert data[u"stats"][u"largest-immutable-file"] == 778240 + assert data[u"count-objects-checked"] == 2 + + # also get the HTML version + resp = requests.post( + dircap_url, + params={ + u"t": u"start-deep-check", + u"return_to": u".", + u"verify": u"on", + u"repair": u"on", + u"ophandle": u"definitely_random", + } + ) + deepcheck_uri = resp.url + + # if the operations isn't done, there's an

tag with the + # reload link; otherwise there's only an

tag..wait up to 5 + # seconds for this to respond properly. + for _ in range(5): + resp = requests.get(deepcheck_uri) + dom = BeautifulSoup(resp.content, "html5lib") + if dom.h1 and u'Results' in unicode(dom.h1.string): + break + if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string): + dom = None + time.sleep(1) + assert dom is not None, "Operation never completed" + + +def test_storage_info(storage_nodes): + """ + retrieve and confirm /storage URI for one storage node + """ + storage0 = storage_nodes[0] + + requests.get( + util.node_url(storage0.node_dir, u"storage"), + ) + + +def test_storage_info_json(storage_nodes): + """ + retrieve and confirm /storage?t=json URI for one storage node + """ + storage0 = storage_nodes[0] + + resp = requests.get( + util.node_url(storage0.node_dir, u"storage"), + params={u"t": u"json"}, + ) + data = json.loads(resp.content) + assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000 + + +def test_introducer_info(introducer): + """ + retrieve and confirm /introducer URI for the introducer + """ + resp = requests.get( + util.node_url(introducer.node_dir, u""), + ) + assert "Introducer" in resp.content + + resp = requests.get( + util.node_url(introducer.node_dir, u""), + params={u"t": u"json"}, + ) + data = json.loads(resp.content) + assert "announcement_summary" in data + assert "subscription_summary" in data + + +def test_mkdir_with_children(alice): + """ + create a directory using ?t=mkdir-with-children + """ + + # create a file to put in our directory + FILE_CONTENTS = u"some file contents\n" * 500 + resp = requests.put( + util.node_url(alice.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + filecap = resp.content.strip() + + # create a (sub) directory to put in our directory + resp = requests.post( + util.node_url(alice.node_dir, u"uri"), + params={ + u"t": u"mkdir", + } + ) + # (we need both the read-write and read-only URIs I guess) + dircap = resp.content + dircap_obj = allmydata.uri.from_string(dircap) + dircap_ro = dircap_obj.get_readonly().to_string() + + # create json information about our directory + meta = { + "a_file": [ + "filenode", { + "ro_uri": filecap, + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ], + "some_subdir": [ + "dirnode", { + "rw_uri": dircap, + "ro_uri": dircap_ro, + "metadata": { + "ctime": 1202778102.7589991, + "mtime": 1202778111.2160511, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ] + } + + # create a new directory with one file and one sub-dir (all-at-once) + resp = util.web_post( + alice, u"uri", + params={u"t": "mkdir-with-children"}, + data=json.dumps(meta), + ) + assert resp.startswith("URI:DIR2") + cap = allmydata.uri.from_string(resp) + assert isinstance(cap, allmydata.uri.DirectoryURI) diff --git a/integration/util.py b/integration/util.py index e0454533f..0c41c6bd3 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,5 +1,6 @@ import sys import time +import json from os import mkdir from os.path import exists, join from six.moves import StringIO @@ -10,11 +11,14 @@ from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone +import requests + from allmydata.util.configutil import ( get_config, set_config, write_config, ) +from allmydata import client import pytest_twisted @@ -127,19 +131,19 @@ class _MagicTextProtocol(ProcessProtocol): sys.stdout.write(data) -def _cleanup_twistd_process(twistd_process, exited): +def _cleanup_tahoe_process(tahoe_transport, exited): """ Terminate the given process with a kill signal (SIGKILL on POSIX, TerminateProcess on Windows). - :param twistd_process: The `IProcessTransport` representing the process. + :param tahoe_transport: The `IProcessTransport` representing the process. :param exited: A `Deferred` which fires when the process has exited. :return: After the process has exited. """ try: - print("signaling {} with KILL".format(twistd_process.pid)) - twistd_process.signalProcess('KILL') + print("signaling {} with TERM".format(tahoe_transport.pid)) + tahoe_transport.signalProcess('TERM') print("signaled, blocking on exit") pytest_twisted.blockon(exited) print("exited, goodbye") @@ -147,20 +151,68 @@ def _cleanup_twistd_process(twistd_process, exited): pass -def run_tahoe(reactor, *args, **kwargs): +def run_tahoe(reactor, request, *args): + """ + Helper to run tahoe with optional coverage + """ stdin = kwargs.get('stdin', None) protocol = _CollectOutputProtocol(stdin=stdin) - process = reactor.spawnProcess( - protocol, - sys.executable, - (sys.executable, '-m', 'allmydata.scripts.runner') + args - ) + process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) process.exited = protocol.done - return protocol.done +def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): + """ + Internal helper. Calls spawnProcess with `-m + allmydata.scripts.runner` and `other_args`, optionally inserting a + `--coverage` option if the `request` indicates we should. + """ + if request.config.getoption('coverage'): + args = [sys.executable, '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage'] + else: + args = [sys.executable, '-m', 'allmydata.scripts.runner'] + args += other_args + return reactor.spawnProcess( + proto, + sys.executable, + args, + ) + + +class TahoeProcess(object): + """ + A running Tahoe process, with associated information. + """ + + def __init__(self, process_transport, node_dir): + self._process_transport = process_transport # IProcessTransport instance + self._node_dir = node_dir # path + + @property + def transport(self): + return self._process_transport + + @property + def node_dir(self): + return self._node_dir + + def get_config(self): + return client.read_config( + self._node_dir, + u"portnum", + ) + + def __str__(self): + return "".format(self._node_dir) + + def _run_node(reactor, node_dir, request, magic_text): + """ + Run a tahoe process from its node_dir. + + :returns: a TahoeProcess for this node + """ if magic_text is None: magic_text = "client running" protocol = _MagicTextProtocol(magic_text) @@ -168,27 +220,29 @@ def _run_node(reactor, node_dir, request, magic_text): # on windows, "tahoe start" means: run forever in the foreground, # but on linux it means daemonize. "tahoe run" is consistent # between platforms. - process = reactor.spawnProcess( + + transport = _tahoe_runner_optional_coverage( protocol, - sys.executable, - ( - sys.executable, '-m', 'allmydata.scripts.runner', + reactor, + request, + [ '--eliot-destination', 'file:{}/logs/eliot.json'.format(node_dir), 'run', node_dir, - ), + ], ) - process.exited = protocol.exited + transport.exited = protocol.exited - request.addfinalizer(partial(_cleanup_twistd_process, process, protocol.exited)) + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) - # we return the 'process' ITransport instance - # XXX abusing the Deferred; should use .when_magic_seen() or something? + # XXX abusing the Deferred; should use .when_magic_seen() pattern def got_proto(proto): - process._protocol = proto - process._node_dir = node_dir - return process + transport._protocol = proto + return TahoeProcess( + transport, + node_dir, + ) protocol.magic_seen.addCallback(got_proto) return protocol.magic_seen @@ -213,7 +267,6 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam mkdir(node_dir) done_proto = _ProcessExitedProtocol() args = [ - sys.executable, '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, '--introducer', introducer_furl, @@ -223,16 +276,13 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam '--shares-needed', unicode(needed), '--shares-happy', unicode(happy), '--shares-total', unicode(total), + '--helper', ] if not storage: args.append('--no-storage') args.append(node_dir) - reactor.spawnProcess( - done_proto, - sys.executable, - args, - ) + _tahoe_runner_optional_coverage(done_proto, reactor, request, args) created_d = done_proto.done def created(_): @@ -365,17 +415,118 @@ def await_file_vanishes(path, timeout=10): raise FileShouldVanishException(path, timeout) -def cli(reactor, node_dir, *argv): +def cli(request, reactor, node_dir, *argv): + """ + Run a tahoe CLI subcommand for a given node, optionally running + under coverage if '--coverage' was supplied. + """ proto = _CollectOutputProtocol() - reactor.spawnProcess( - proto, - sys.executable, - [ - sys.executable, '-m', 'allmydata.scripts.runner', - '--node-directory', node_dir, - ] + list(argv), + _tahoe_runner_optional_coverage( + proto, reactor, request, + ['--node-directory', node_dir] + list(argv), ) return proto.done -def magic_folder_cli(reactor, node_dir, *argv): - return cli(reactor, node_dir, "magic-folder", *argv) + +def node_url(node_dir, uri_fragment): + """ + Create a fully qualified URL by reading config from `node_dir` and + adding the `uri_fragment` + """ + with open(join(node_dir, "node.url"), "r") as f: + base = f.read().strip() + url = base + uri_fragment + return url + + +def _check_status(response): + """ + Check the response code is a 2xx (raise an exception otherwise) + """ + if response.status_code < 200 or response.status_code >= 300: + raise ValueError( + "Expected a 2xx code, got {}".format(response.status_code) + ) + + +def web_get(tahoe, uri_fragment, **kwargs): + """ + Make a GET request to the webport of `tahoe` (a `TahoeProcess`, + usually from a fixture (e.g. `alice`). This will look like: + `http://localhost:/`. All `kwargs` are + passed on to `requests.get` + """ + url = node_url(tahoe.node_dir, uri_fragment) + resp = requests.get(url, **kwargs) + _check_status(resp) + return resp.content + + +def web_post(tahoe, uri_fragment, **kwargs): + """ + Make a POST request to the webport of `node` (a `TahoeProcess, + usually from a fixture e.g. `alice`). This will look like: + `http://localhost:/`. All `kwargs` are + passed on to `requests.post` + """ + url = node_url(tahoe.node_dir, uri_fragment) + resp = requests.post(url, **kwargs) + _check_status(resp) + return resp.content + + +def await_client_ready(tahoe, timeout=10, liveness=60*2): + """ + Uses the status API to wait for a client-type node (in `tahoe`, a + `TahoeProcess` instance usually from a fixture e.g. `alice`) to be + 'ready'. A client is deemed ready if: + + - it answers `http:///statistics/?t=json/` + - there is at least one storage-server connected + - every storage-server has a "last_received_data" and it is + within the last `liveness` seconds + + We will try for up to `timeout` seconds for the above conditions + to be true. Otherwise, an exception is raised + """ + start = time.time() + while (time.time() - start) < float(timeout): + try: + data = web_get(tahoe, u"", params={u"t": u"json"}) + js = json.loads(data) + except Exception as e: + print("waiting because '{}'".format(e)) + time.sleep(1) + continue + + if len(js['servers']) == 0: + print("waiting because no servers at all") + time.sleep(1) + continue + server_times = [ + server['last_received_data'] + for server in js['servers'] + ] + # if any times are null/None that server has never been + # contacted (so it's down still, probably) + if any(t is None for t in server_times): + print("waiting because at least one server not contacted") + time.sleep(1) + continue + + # check that all times are 'recent enough' + if any([time.time() - t > liveness for t in server_times]): + print("waiting because at least one server too old") + time.sleep(1) + continue + + # we have a status with at least one server, and all servers + # have been contacted recently + return True + # we only fall out of the loop when we've timed out + raise RuntimeError( + "Waited {} seconds for {} to be 'ready' but it never was".format( + timeout, + tahoe, + ) + ) diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index 6c76bcd69..3972d5c89 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -1,6 +1,6 @@ from __future__ import print_function -import sys, os, io +import sys, os, io, re from twisted.internet import reactor, protocol, task, defer from twisted.python.procutils import which from twisted.python import usage @@ -12,6 +12,7 @@ from twisted.python import usage class Options(usage.Options): optParameters = [ ["warnings", None, None, "file to write warnings into at end of test run"], + ["package", None, None, "Python package to which to restrict warning collection"] ] def parseArgs(self, command, *args): @@ -19,7 +20,7 @@ class Options(usage.Options): self["args"] = list(args) description = """Run as: -PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--warnings=STDERRFILE] COMMAND ARGS.. +PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] COMMAND ARGS.. """ class RunPP(protocol.ProcessProtocol): @@ -34,6 +35,34 @@ class RunPP(protocol.ProcessProtocol): rc = reason.value.exitCode self.d.callback((signal, rc)) + +def make_matcher(options): + """ + Make a function that matches a line with a relevant deprecation. + + A deprecation warning line looks something like this:: + + somepath/foo/bar/baz.py:43: DeprecationWarning: Foo is deprecated, try bar instead. + + Sadly there is no guarantee warnings begin at the beginning of a line + since they are written to output without coordination with whatever other + Python code is running in the process. + + :return: A one-argument callable that accepts a string and returns + ``True`` if it contains an interesting warning and ``False`` + otherwise. + """ + pattern = r".*\.py[oc]?:\d+:" # (Pending)?DeprecationWarning: .*" + if options["package"]: + pattern = r".*/{}/".format( + re.escape(options["package"]), + ) + pattern + expression = re.compile(pattern) + def match(line): + return expression.match(line) is not None + return match + + @defer.inlineCallbacks def run_command(main): config = Options() @@ -63,6 +92,8 @@ def run_command(main): reactor.spawnProcess(pp, exe, [exe] + config["args"], env=None) (signal, rc) = yield pp.d + match = make_matcher(config) + # maintain ordering, but ignore duplicates (for some reason, either the # 'warnings' module or twisted.python.deprecate isn't quashing them) already = set() @@ -75,12 +106,12 @@ def run_command(main): pp.stdout.seek(0) for line in pp.stdout.readlines(): - if "DeprecationWarning" in line: + if match(line): add(line) # includes newline pp.stderr.seek(0) for line in pp.stderr.readlines(): - if "DeprecationWarning" in line: + if match(line): add(line) if warnings: diff --git a/misc/build_helpers/show-tool-versions.py b/misc/build_helpers/show-tool-versions.py index 3b707aba0..c4fb79eff 100644 --- a/misc/build_helpers/show-tool-versions.py +++ b/misc/build_helpers/show-tool-versions.py @@ -146,8 +146,7 @@ print_py_pkg_ver('mock') print_py_pkg_ver('Nevow', 'nevow') print_py_pkg_ver('pyasn1') print_py_pkg_ver('pycparser') -print_py_pkg_ver('pycrypto', 'Crypto') -print_py_pkg_ver('pycryptopp') +print_py_pkg_ver('cryptography') print_py_pkg_ver('pyflakes') print_py_pkg_ver('pyOpenSSL', 'OpenSSL') print_py_pkg_ver('six') diff --git a/misc/build_helpers/test-osx-pkg.py b/misc/build_helpers/test-osx-pkg.py index c763193d7..aaf7bb47a 100644 --- a/misc/build_helpers/test-osx-pkg.py +++ b/misc/build_helpers/test-osx-pkg.py @@ -15,7 +15,6 @@ # allmydata-tahoe: 1.10.0.post185.dev0 [2249-deps-and-osx-packaging-1: 76ac53846042d9a4095995be92af66cdc09d5ad0-dirty] (/Applications/tahoe.app/src) # foolscap: 0.7.0 (/Applications/tahoe.app/support/lib/python2.7/site-packages/foolscap-0.7.0-py2.7.egg) -# pycryptopp: 0.6.0.1206569328141510525648634803928199668821045408958 (/Applications/tahoe.app/support/lib/python2.7/site-packages/pycryptopp-0.6.0.1206569328141510525648634803928199668821045408958-py2.7-macosx-10.9-intel.egg) # zfec: 1.4.24 (/Applications/tahoe.app/support/lib/python2.7/site-packages/zfec-1.4.24-py2.7-macosx-10.9-intel.egg) # Twisted: 13.0.0 (/Applications/tahoe.app/support/lib/python2.7/site-packages/Twisted-13.0.0-py2.7-macosx-10.9-intel.egg) # Nevow: 0.11.1 (/Applications/tahoe.app/support/lib/python2.7/site-packages/Nevow-0.11.1-py2.7.egg) @@ -23,7 +22,6 @@ # python: 2.7.5 (/usr/bin/python) # platform: Darwin-13.4.0-x86_64-i386-64bit (None) # pyOpenSSL: 0.13 (/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python) -# pycrypto: 2.6.1 (/Applications/tahoe.app/support/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-macosx-10.9-intel.egg) # pyasn1: 0.1.7 (/Applications/tahoe.app/support/lib/python2.7/site-packages/pyasn1-0.1.7-py2.7.egg) # mock: 1.0.1 (/Applications/tahoe.app/support/lib/python2.7/site-packages) # setuptools: 0.6c16dev6 (/Applications/tahoe.app/support/lib/python2.7/site-packages/setuptools-0.6c16dev6.egg) diff --git a/misc/coding_tools/check-miscaptures.py b/misc/coding_tools/check-miscaptures.py index c55f9fa9c..81e76f891 100644 --- a/misc/coding_tools/check-miscaptures.py +++ b/misc/coding_tools/check-miscaptures.py @@ -138,7 +138,7 @@ def report(out, path, results): print(path + (":%r %s captures %r assigned at line %d" % r), file=out) def check(sources, out): - class Counts: + class Counts(object): n = 0 processed_files = 0 suspect_files = 0 diff --git a/misc/operations_helpers/provisioning/reliability.py b/misc/operations_helpers/provisioning/reliability.py index a0d60769b..dc241b9d1 100644 --- a/misc/operations_helpers/provisioning/reliability.py +++ b/misc/operations_helpers/provisioning/reliability.py @@ -8,7 +8,7 @@ DAY=24*60*60 MONTH=31*DAY YEAR=365*DAY -class ReliabilityModel: +class ReliabilityModel(object): """Generate a model of system-wide reliability, given several input parameters. @@ -207,7 +207,7 @@ class ReliabilityModel: repair = matrix(new_repair_rows) return repair -class ReliabilityReport: +class ReliabilityReport(object): def __init__(self): self.samples = [] diff --git a/misc/operations_helpers/provisioning/test_provisioning.py b/misc/operations_helpers/provisioning/test_provisioning.py index d2b9dbd17..8835b79db 100644 --- a/misc/operations_helpers/provisioning/test_provisioning.py +++ b/misc/operations_helpers/provisioning/test_provisioning.py @@ -10,7 +10,7 @@ except ImportError: from nevow import inevow from zope.interface import implements -class MyRequest: +class MyRequest(object): implements(inevow.IRequest) pass diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh new file mode 100755 index 000000000..d5ad33bf7 --- /dev/null +++ b/misc/python3/depgraph.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -x +set -eo pipefail + +TAHOE="${PWD}" +git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git +cd tahoe-depgraph + +# Generate the maybe-changed data. +python "${TAHOE}"/misc/python3/tahoe-depgraph.py "${TAHOE}" + +if git diff-index --quiet HEAD; then + echo "Declining to commit without any changes." + exit 0 +fi + +git config user.name 'Build Automation' +git config user.email 'tahoe-dev@tahoe-lafs.org' + +git add tahoe-deps.json tahoe-ported.json +git commit -m "\ +Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1} + +tahoe-depgraph was $(git rev-parse HEAD) +" + +if [ "${CIRCLE_BRANCH}" != "master" ]; then + echo "Declining to update dependency graph for non-master build." + exit 0 +fi + +# Publish it on GitHub. +git push -q origin gh-pages diff --git a/misc/python3/tahoe-depgraph.py b/misc/python3/tahoe-depgraph.py new file mode 100644 index 000000000..1bfe89c73 --- /dev/null +++ b/misc/python3/tahoe-depgraph.py @@ -0,0 +1,123 @@ +# Copyright 2004, 2009 Toby Dickenson +# Copyright 2014-2015 Aaron Gallagher +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject +# to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import collections +import functools +import json +import os +import modulefinder +import sys +import tempfile + +from twisted.python import reflect + + +class mymf(modulefinder.ModuleFinder): + def __init__(self, *args, **kwargs): + self._depgraph = collections.defaultdict(set) + self._types = {} + self._last_caller = None + modulefinder.ModuleFinder.__init__(self, *args, **kwargs) + + def import_hook(self, name, caller=None, fromlist=None, level=None): + old_last_caller = self._last_caller + try: + self._last_caller = caller + return modulefinder.ModuleFinder.import_hook( + self, name, caller, fromlist) + finally: + self._last_caller = old_last_caller + + def import_module(self, partnam, fqname, parent): + if partnam.endswith('_py3'): + return None + r = modulefinder.ModuleFinder.import_module( + self, partnam, fqname, parent) + last_caller = self._last_caller + if r is not None and 'allmydata' in r.__name__: + if last_caller is None or last_caller.__name__ == '__main__': + self._depgraph[fqname] + else: + self._depgraph[last_caller.__name__].add(fqname) + return r + + def load_module(self, fqname, fp, pathname, (suffix, mode, type)): + r = modulefinder.ModuleFinder.load_module( + self, fqname, fp, pathname, (suffix, mode, type)) + if r is not None: + self._types[r.__name__] = type + return r + + def as_json(self): + return { + 'depgraph': { + name: dict.fromkeys(deps, 1) + for name, deps in self._depgraph.iteritems()}, + 'types': self._types, + } + + +json_dump = functools.partial( + json.dump, indent=4, separators=(',', ': '), sort_keys=True) + + +def main(target): + mf = mymf(sys.path[:], 0, []) + + moduleNames = [] + for path, dirnames, filenames in os.walk(os.path.join(target, 'src', 'allmydata')): + if 'test' in dirnames: + dirnames.remove('test') + for filename in filenames: + if not filename.endswith('.py'): + continue + if filename in ('setup.py',): + continue + if '-' in filename: + # a script like update-documentation.py + continue + if filename != '__init__.py': + filepath = os.path.join(path, filename) + else: + filepath = path + moduleNames.append(reflect.filenameToModuleName(filepath)) + + with tempfile.NamedTemporaryFile() as tmpfile: + for moduleName in moduleNames: + tmpfile.write('import %s\n' % moduleName) + tmpfile.flush() + mf.run_script(tmpfile.name) + + with open('tahoe-deps.json', 'wb') as outfile: + json_dump(mf.as_json(), outfile) + outfile.write('\n') + + ported_modules_path = os.path.join(target, "src", "allmydata", "ported-modules.txt") + with open(ported_modules_path) as ported_modules: + port_status = dict.fromkeys((line.strip() for line in ported_modules), "ported") + with open('tahoe-ported.json', 'wb') as outfile: + json_dump(port_status, outfile) + outfile.write('\n') + + +if __name__ == '__main__': + main(*sys.argv[1:]) diff --git a/misc/simulators/ringsim.py b/misc/simulators/ringsim.py index 83ed1302f..1eec7a466 100644 --- a/misc/simulators/ringsim.py +++ b/misc/simulators/ringsim.py @@ -54,7 +54,7 @@ print("average file size:", abbreviate_space(avg_filesize)) SERVER_CAPACITY = 10**12 -class Server: +class Server(object): def __init__(self, nodeid, capacity): self.nodeid = nodeid self.used = 0 @@ -75,7 +75,7 @@ class Server: else: return "<%s %s>" % (self.__class__.__name__, self.nodeid) -class Ring: +class Ring(object): SHOW_MINMAX = False def __init__(self, numservers, seed, permute): self.servers = [] diff --git a/misc/simulators/simulate_load.py b/misc/simulators/simulate_load.py index 5821ef7de..f522a6d93 100644 --- a/misc/simulators/simulate_load.py +++ b/misc/simulators/simulate_load.py @@ -8,7 +8,7 @@ import random SERVER_CAPACITY = 10**12 -class Server: +class Server(object): def __init__(self): self.si = random.randrange(0, 2**31) self.used = 0 diff --git a/misc/simulators/simulator.py b/misc/simulators/simulator.py index d0ef281db..ceeb05edf 100644 --- a/misc/simulators/simulator.py +++ b/misc/simulators/simulator.py @@ -17,7 +17,7 @@ def sha(s): def randomid(): return os.urandom(20) -class Node: +class Node(object): def __init__(self, nid, introducer, simulator): self.nid = nid self.introducer = introducer @@ -112,7 +112,7 @@ class Node: self.introducer.delete(fileid) return True -class Introducer: +class Introducer(object): def __init__(self, simulator): self.living_files = {} self.utilization = 0 # total size of all active files @@ -149,7 +149,7 @@ class Introducer: self.simulator.stamp_utilization(self.utilization) del self.living_files[fileid] -class Simulator: +class Simulator(object): NUM_NODES = 1000 EVENTS = ["ADDFILE", "DELFILE", "ADDNODE", "DELNODE"] RATE_ADDFILE = 1.0 / 10 diff --git a/misc/simulators/sizes.py b/misc/simulators/sizes.py index ca189e4e6..b55c664a5 100644 --- a/misc/simulators/sizes.py +++ b/misc/simulators/sizes.py @@ -37,7 +37,7 @@ GiB=1024*MiB TiB=1024*GiB PiB=1024*TiB -class Sizes: +class Sizes(object): def __init__(self, mode, file_size, arity=2): MAX_SEGSIZE = 128*KiB self.mode = mode diff --git a/misc/simulators/storage-overhead.py b/misc/simulators/storage-overhead.py index 547bc0adf..5a741834e 100644 --- a/misc/simulators/storage-overhead.py +++ b/misc/simulators/storage-overhead.py @@ -12,7 +12,7 @@ def roundup(size, blocksize=4096): return blocksize * mathutil.div_ceil(size, blocksize) -class BigFakeString: +class BigFakeString(object): def __init__(self, length): self.length = length self.fp = 0 diff --git a/newsfragments/1432.feature b/newsfragments/1432.feature deleted file mode 100644 index 2bb9a8b01..000000000 --- a/newsfragments/1432.feature +++ /dev/null @@ -1 +0,0 @@ -Magic-Folders are now supported on macOS. diff --git a/newsfragments/2870.bugfix b/newsfragments/2870.bugfix deleted file mode 100644 index d7f1b9135..000000000 --- a/newsfragments/2870.bugfix +++ /dev/null @@ -1 +0,0 @@ -refactor initialization code to be more async-friendly \ No newline at end of file diff --git a/newsfragments/2908.other b/newsfragments/2908.other deleted file mode 100644 index ccda51784..000000000 --- a/newsfragments/2908.other +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now uses towncrier to maintain the NEWS file. diff --git a/newsfragments/2920.other b/newsfragments/2920.other deleted file mode 100644 index b52772368..000000000 --- a/newsfragments/2920.other +++ /dev/null @@ -1 +0,0 @@ -The release process document has been updated. diff --git a/newsfragments/2933.other b/newsfragments/2933.other deleted file mode 100644 index c0f0ba8b8..000000000 --- a/newsfragments/2933.other +++ /dev/null @@ -1 +0,0 @@ -allmydata.test.test_system.SystemTest is now more reliable with respect to bound address collisions. diff --git a/newsfragments/2935.bugfix b/newsfragments/2935.bugfix deleted file mode 100644 index 23b985259..000000000 --- a/newsfragments/2935.bugfix +++ /dev/null @@ -1 +0,0 @@ -Configuration-checking code wasn't being called due to indenting \ No newline at end of file diff --git a/newsfragments/2936.bugfix b/newsfragments/2936.bugfix deleted file mode 100644 index 8bca69813..000000000 --- a/newsfragments/2936.bugfix +++ /dev/null @@ -1 +0,0 @@ -refactor configuration handling out of Node into _Config \ No newline at end of file diff --git a/newsfragments/2944.bugfix b/newsfragments/2944.bugfix deleted file mode 100644 index eb147950b..000000000 --- a/newsfragments/2944.bugfix +++ /dev/null @@ -1 +0,0 @@ -Updated the Tor release key, used by the integration tests. \ No newline at end of file diff --git a/newsfragments/2950.bugfix b/newsfragments/2950.bugfix deleted file mode 100644 index 8df25e4ff..000000000 --- a/newsfragments/2950.bugfix +++ /dev/null @@ -1 +0,0 @@ -`tahoe backup` no longer fails with an unhandled exception when it encounters a special file (device, fifo) in the backup source. diff --git a/newsfragments/2955.feature b/newsfragments/2955.feature deleted file mode 100644 index ad3aeba4d..000000000 --- a/newsfragments/2955.feature +++ /dev/null @@ -1 +0,0 @@ -Fedora 29 is now tested as part of the project's continuous integration system. \ No newline at end of file diff --git a/newsfragments/2955.removed b/newsfragments/2955.removed deleted file mode 100644 index 71c187d20..000000000 --- a/newsfragments/2955.removed +++ /dev/null @@ -1 +0,0 @@ -Fedora 27 is no longer tested as part of the project's continuous integration system. \ No newline at end of file diff --git a/newsfragments/2956.other b/newsfragments/2956.other deleted file mode 100644 index c69807477..000000000 --- a/newsfragments/2956.other +++ /dev/null @@ -1 +0,0 @@ -The Tox configuration has been fixed to work around a problem on Windows CI. diff --git a/newsfragments/2957.installation b/newsfragments/2957.installation deleted file mode 100644 index c3d2dff9a..000000000 --- a/newsfragments/2957.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now depends on Twisted 16.6 or newer. \ No newline at end of file diff --git a/newsfragments/2958.other b/newsfragments/2958.other deleted file mode 100644 index d4a7ca4cc..000000000 --- a/newsfragments/2958.other +++ /dev/null @@ -1 +0,0 @@ -The PyInstaller CI job now works around a pip/pyinstaller incompatibility. \ No newline at end of file diff --git a/newsfragments/2959.other b/newsfragments/2959.other deleted file mode 100644 index a8da6337a..000000000 --- a/newsfragments/2959.other +++ /dev/null @@ -1 +0,0 @@ -Some CI jobs for integration tests have been moved from TravisCI to CircleCI. diff --git a/newsfragments/2960.other b/newsfragments/2960.other deleted file mode 100644 index 2ef00c922..000000000 --- a/newsfragments/2960.other +++ /dev/null @@ -1 +0,0 @@ -Several warnings from a new release of pyflakes have been fixed. diff --git a/newsfragments/2961.other b/newsfragments/2961.other deleted file mode 100644 index e299bb020..000000000 --- a/newsfragments/2961.other +++ /dev/null @@ -1 +0,0 @@ -Some Slackware 14.2 continuous integration problems have been resolved. diff --git a/newsfragments/2962.other b/newsfragments/2962.other deleted file mode 100644 index de6cce22f..000000000 --- a/newsfragments/2962.other +++ /dev/null @@ -1 +0,0 @@ -Some macOS continuous integration failures have been fixed. diff --git a/newsfragments/2965.bugfix b/newsfragments/2965.bugfix deleted file mode 100644 index 58d4582e8..000000000 --- a/newsfragments/2965.bugfix +++ /dev/null @@ -1 +0,0 @@ -Magic-Folders now creates spurious conflict files in fewer cases. In particular, if files are added to the folder while a client is offline, that client will not create conflict files for all those new files when it starts up. diff --git a/newsfragments/2966.other b/newsfragments/2966.other deleted file mode 100644 index 92b5437f6..000000000 --- a/newsfragments/2966.other +++ /dev/null @@ -1 +0,0 @@ -The NoNetworkGrid implementation has been somewhat improved. diff --git a/newsfragments/2967.other b/newsfragments/2967.other deleted file mode 100644 index 7ee8118b3..000000000 --- a/newsfragments/2967.other +++ /dev/null @@ -1 +0,0 @@ -A bug in the test suite for the create-alias command has been fixed. diff --git a/newsfragments/2968.other b/newsfragments/2968.other deleted file mode 100644 index ca8f82d7f..000000000 --- a/newsfragments/2968.other +++ /dev/null @@ -1 +0,0 @@ -The integration test suite has been updated to use pytest-twisted instead of deprecated pytest APIs. \ No newline at end of file diff --git a/newsfragments/2969.other b/newsfragments/2969.other deleted file mode 100644 index 690b5bc78..000000000 --- a/newsfragments/2969.other +++ /dev/null @@ -1 +0,0 @@ -The magic-folder integration test suite now performs more aggressive cleanup of the processes it launches. diff --git a/newsfragments/2970.other b/newsfragments/2970.other deleted file mode 100644 index 881a1dd3b..000000000 --- a/newsfragments/2970.other +++ /dev/null @@ -1 +0,0 @@ -The integration tests now correctly document the `--keep-tempdir` option. \ No newline at end of file diff --git a/newsfragments/2971.other b/newsfragments/2971.other deleted file mode 100644 index 920ff983c..000000000 --- a/newsfragments/2971.other +++ /dev/null @@ -1 +0,0 @@ -A misuse of super() in the integration tests has been fixed. \ No newline at end of file diff --git a/newsfragments/2972.feature b/newsfragments/2972.feature deleted file mode 100644 index 94ed65ca3..000000000 --- a/newsfragments/2972.feature +++ /dev/null @@ -1 +0,0 @@ -The Magic-Folder frontend now emits structured, causal logs. This makes it easier for developers to make sense of its behavior and for users to submit useful debugging information alongside problem reports. diff --git a/newsfragments/2973.other b/newsfragments/2973.other deleted file mode 100644 index 4bd9b09a3..000000000 --- a/newsfragments/2973.other +++ /dev/null @@ -1 +0,0 @@ -Several utilities to facilitate the use of the Eliot causal logging library have been introduced. \ No newline at end of file diff --git a/newsfragments/2974.other b/newsfragments/2974.other deleted file mode 100644 index 80905b7c8..000000000 --- a/newsfragments/2974.other +++ /dev/null @@ -1 +0,0 @@ -The Windows CI configuration has been tweaked. \ No newline at end of file diff --git a/newsfragments/2975.feature b/newsfragments/2975.feature deleted file mode 100644 index 596a11d4e..000000000 --- a/newsfragments/2975.feature +++ /dev/null @@ -1 +0,0 @@ -The `tahoe` CLI now accepts arguments for configuring structured logging messages which Tahoe-LAFS is being converted to emit. This change does not introduce any new defaults for on-filesystem logging. diff --git a/newsfragments/2976.bugfix b/newsfragments/2976.bugfix deleted file mode 100644 index 69687630c..000000000 --- a/newsfragments/2976.bugfix +++ /dev/null @@ -1 +0,0 @@ -The confusing and misplaced sub-command group headings in `tahoe --help` output have been removed. diff --git a/newsfragments/2977.other b/newsfragments/2977.other deleted file mode 100644 index a30aba7a6..000000000 --- a/newsfragments/2977.other +++ /dev/null @@ -1 +0,0 @@ -The Magic-Folder frontend has had additional logging improvements. \ No newline at end of file diff --git a/newsfragments/2997.bugfix b/newsfragments/2997.bugfix deleted file mode 100644 index 5b8615575..000000000 --- a/newsfragments/2997.bugfix +++ /dev/null @@ -1 +0,0 @@ -The Magic-Folder frontend is now more responsive to subtree changes on Windows. diff --git a/newsfragments/3001.other b/newsfragments/3001.other deleted file mode 100644 index 8607600d1..000000000 --- a/newsfragments/3001.other +++ /dev/null @@ -1 +0,0 @@ -Added a simple sytax checker so that once a file has reached python3 compatibility, it will not regress. diff --git a/newsfragments/3002.other b/newsfragments/3002.other deleted file mode 100644 index a10cad243..000000000 --- a/newsfragments/3002.other +++ /dev/null @@ -1 +0,0 @@ -Converted all uses of the print statement to the print function in the ./misc/ directory. diff --git a/newsfragments/3003.other b/newsfragments/3003.other deleted file mode 100644 index e1875f5b5..000000000 --- a/newsfragments/3003.other +++ /dev/null @@ -1 +0,0 @@ -The contributor guidelines are now linked from the GitHub pull request creation page. diff --git a/newsfragments/3005.minor b/newsfragments/3005.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3006.feature b/newsfragments/3006.feature deleted file mode 100644 index d9ce19b54..000000000 --- a/newsfragments/3006.feature +++ /dev/null @@ -1 +0,0 @@ -The web API now publishes streaming Eliot logs via a token-protected WebSocket at /private/logs/v1. \ No newline at end of file diff --git a/newsfragments/3007.minor b/newsfragments/3007.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3008.other b/newsfragments/3008.other deleted file mode 100644 index c5947100e..000000000 --- a/newsfragments/3008.other +++ /dev/null @@ -1 +0,0 @@ -Updated the testing code to use the print function instead of the print statement. diff --git a/newsfragments/3009.other b/newsfragments/3009.other deleted file mode 100644 index 4115bdf90..000000000 --- a/newsfragments/3009.other +++ /dev/null @@ -1 +0,0 @@ -Replaced print statement with print fuction for all tahoe_* scripts. diff --git a/newsfragments/3010.other b/newsfragments/3010.other deleted file mode 100644 index fa1c406d1..000000000 --- a/newsfragments/3010.other +++ /dev/null @@ -1 +0,0 @@ -Replaced all remaining instances of the print statement with the print function. diff --git a/newsfragments/3011.other b/newsfragments/3011.other deleted file mode 100644 index b6a553df2..000000000 --- a/newsfragments/3011.other +++ /dev/null @@ -1 +0,0 @@ -Replace StringIO imports with six.moves. diff --git a/newsfragments/3012.minor b/newsfragments/3012.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3013.other b/newsfragments/3013.other deleted file mode 100644 index fd5a34e30..000000000 --- a/newsfragments/3013.other +++ /dev/null @@ -1 +0,0 @@ -Updated all Python files to use PEP-3110 exception syntax for Python3 compatibility. diff --git a/newsfragments/3014.other b/newsfragments/3014.other deleted file mode 100644 index 7be507e14..000000000 --- a/newsfragments/3014.other +++ /dev/null @@ -1 +0,0 @@ -Update raise syntax for Python3 compatibility. diff --git a/newsfragments/3015.other b/newsfragments/3015.other deleted file mode 100644 index 68841f923..000000000 --- a/newsfragments/3015.other +++ /dev/null @@ -1 +0,0 @@ -Updated instances of octal literals to use the format 0o123 for Python3 compatibility. diff --git a/newsfragments/3016.other b/newsfragments/3016.other deleted file mode 100644 index 0af735c73..000000000 --- a/newsfragments/3016.other +++ /dev/null @@ -1 +0,0 @@ -allmydata.test.no_network, allmydata.test.test_system, and allmydata.test.web.test_introducer are now more reliable with respect to bound address collisions. diff --git a/newsfragments/3017.minor b/newsfragments/3017.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3019.other b/newsfragments/3019.other deleted file mode 100644 index 76b07a5dd..000000000 --- a/newsfragments/3019.other +++ /dev/null @@ -1 +0,0 @@ -Removed tuple unpacking from function and lambda definitions for Python3 compatibility. diff --git a/newsfragments/3020.other b/newsfragments/3020.other deleted file mode 100644 index c66702d59..000000000 --- a/newsfragments/3020.other +++ /dev/null @@ -1 +0,0 @@ -Updated Python2 long numeric literals for Python3 compatibility. diff --git a/newsfragments/3021.minor b/newsfragments/3021.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3023.minor b/newsfragments/3023.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3024.other b/newsfragments/3024.other deleted file mode 100644 index a3ec47ebb..000000000 --- a/newsfragments/3024.other +++ /dev/null @@ -1 +0,0 @@ -CircleCI jobs are now faster as a result of pre-building configured Docker images for the CI jobs. diff --git a/newsfragments/3025.minor b/newsfragments/3025.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3026.minor b/newsfragments/3026.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3027.other b/newsfragments/3027.other deleted file mode 100644 index 8d548608c..000000000 --- a/newsfragments/3027.other +++ /dev/null @@ -1 +0,0 @@ -Removed used of backticks for "repr" for Python3 compatibility. diff --git a/newsfragments/3028.other b/newsfragments/3028.other deleted file mode 100644 index 4381305b2..000000000 --- a/newsfragments/3028.other +++ /dev/null @@ -1 +0,0 @@ -Updated string literal syntax for Python3 compatibility. diff --git a/newsfragments/3029.minor b/newsfragments/3029.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3030.other b/newsfragments/3030.other deleted file mode 100644 index 7c7b1f0b0..000000000 --- a/newsfragments/3030.other +++ /dev/null @@ -1 +0,0 @@ -Updated CI to enforce Python3 syntax for entire repo. diff --git a/newsfragments/3036.minor b/newsfragments/3036.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3038.minor b/newsfragments/3038.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3043.other b/newsfragments/3043.other deleted file mode 100644 index edc6bb77e..000000000 --- a/newsfragments/3043.other +++ /dev/null @@ -1 +0,0 @@ -Whitelisted "/bin/mv" as command for codechecks performed by tox. This fixes a current warning and prevents future errors (for tox 4). diff --git a/newsfragments/2980.minor b/newsfragments/3263.other similarity index 100% rename from newsfragments/2980.minor rename to newsfragments/3263.other diff --git a/newsfragments/2981.other b/newsfragments/3277.minor similarity index 100% rename from newsfragments/2981.other rename to newsfragments/3277.minor diff --git a/newsfragments/2982.other b/newsfragments/3278.minor similarity index 100% rename from newsfragments/2982.other rename to newsfragments/3278.minor diff --git a/newsfragments/3284.removed b/newsfragments/3284.removed new file mode 100644 index 000000000..7e31d352a --- /dev/null +++ b/newsfragments/3284.removed @@ -0,0 +1 @@ +The Magic Folder frontend has been split out into a stand-alone project. The functionality is no longer part of Tahoe-LAFS itself. Learn more at . diff --git a/newsfragments/2985.minor b/newsfragments/3292.minor similarity index 100% rename from newsfragments/2985.minor rename to newsfragments/3292.minor diff --git a/newsfragments/2986.minor b/newsfragments/3293.minor similarity index 100% rename from newsfragments/2986.minor rename to newsfragments/3293.minor diff --git a/newsfragments/2987.minor b/newsfragments/3294.minor similarity index 100% rename from newsfragments/2987.minor rename to newsfragments/3294.minor diff --git a/newsfragments/3296.installation b/newsfragments/3296.installation new file mode 100644 index 000000000..78cf83f60 --- /dev/null +++ b/newsfragments/3296.installation @@ -0,0 +1 @@ +Tahoe-LAFS now supports CentOS 8 and no longer supports CentOS 7. \ No newline at end of file diff --git a/newsfragments/2988.minor b/newsfragments/3297.minor similarity index 100% rename from newsfragments/2988.minor rename to newsfragments/3297.minor diff --git a/newsfragments/2989.minor b/newsfragments/3298.minor similarity index 100% rename from newsfragments/2989.minor rename to newsfragments/3298.minor diff --git a/newsfragments/2990.minor b/newsfragments/3299.minor similarity index 100% rename from newsfragments/2990.minor rename to newsfragments/3299.minor diff --git a/newsfragments/2991.minor b/newsfragments/3300.minor similarity index 100% rename from newsfragments/2991.minor rename to newsfragments/3300.minor diff --git a/newsfragments/2992.minor b/newsfragments/3302.minor similarity index 100% rename from newsfragments/2992.minor rename to newsfragments/3302.minor diff --git a/newsfragments/2995.minor b/newsfragments/3303.minor similarity index 100% rename from newsfragments/2995.minor rename to newsfragments/3303.minor diff --git a/newsfragments/3000.minor b/newsfragments/3305.minor similarity index 100% rename from newsfragments/3000.minor rename to newsfragments/3305.minor diff --git a/newsfragments/3004.minor b/newsfragments/3306.minor similarity index 100% rename from newsfragments/3004.minor rename to newsfragments/3306.minor diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 000000000..bd7460c2f --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,7 @@ +# This is the main entrypoint for the Tahoe-LAFS derivation. +{ pkgs ? import { } }: +# Add our Python packages to nixpkgs to simplify the expression for the +# Tahoe-LAFS derivation. +let pkgs' = pkgs.extend (import ./overlays.nix); +# Evaluate the expression for our Tahoe-LAFS derivation. +in pkgs'.python2.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/eliot.nix b/nix/eliot.nix new file mode 100644 index 000000000..c5975e990 --- /dev/null +++ b/nix/eliot.nix @@ -0,0 +1,31 @@ +{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons +, hypothesis, testtools, pytest }: +buildPythonPackage rec { + pname = "eliot"; + version = "1.7.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8"; + }; + + postPatch = '' + substituteInPlace setup.py \ + --replace "boltons >= 19.0.1" boltons + ''; + + # A seemingly random subset of the test suite fails intermittently. After + # Tahoe-LAFS is ported to Python 3 we can update to a newer Eliot and, if + # the test suite continues to fail, maybe it will be more likely that we can + # have upstream fix it for us. + doCheck = false; + + checkInputs = [ testtools pytest hypothesis ]; + propagatedBuildInputs = [ zope_interface pyrsistent boltons ]; + + meta = with lib; { + homepage = https://github.com/itamarst/eliot/; + description = "Logging library that tells you why it happened"; + license = licenses.asl20; + }; +} diff --git a/nix/nevow.nix b/nix/nevow.nix new file mode 100644 index 000000000..202a59722 --- /dev/null +++ b/nix/nevow.nix @@ -0,0 +1,45 @@ +{ stdenv, buildPythonPackage, fetchPypi, isPy3k, twisted }: + +buildPythonPackage rec { + pname = "Nevow"; + version = "0.14.5"; + name = "${pname}-${version}"; + disabled = isPy3k; + + src = fetchPypi { + inherit pname; + inherit version; + sha256 = "1wr3fai01h1bcp4qpia6indg4qmxvywwv3q1iibm669mln2vmdmg"; + }; + + propagatedBuildInputs = [ twisted ]; + + checkInputs = [ twisted ]; + + checkPhase = '' + trial formless nevow + ''; + + meta = with stdenv.lib; { + description = "Nevow, a web application construction kit for Python"; + longDescription = '' + Nevow - Pronounced as the French "nouveau", or "noo-voh", Nevow + is a web application construction kit written in Python. It is + designed to allow the programmer to express as much of the view + logic as desired in Python, and includes a pure Python XML + expression syntax named stan to facilitate this. However it + also provides rich support for designer-edited templates, using + a very small XML attribute language to provide bi-directional + template manipulation capability. + + Nevow also includes formless, a declarative syntax for + specifying the types of method parameters and exposing these + methods to the web. Forms can be rendered automatically, and + form posts will be validated and input coerced, rendering error + pages if appropriate. Once a form post has validated + successfully, the method will be called with the coerced values. + ''; + homepage = https://github.com/twisted/nevow; + license = licenses.mit; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix new file mode 100644 index 000000000..08d11306e --- /dev/null +++ b/nix/overlays.nix @@ -0,0 +1,12 @@ +self: super: { + python27 = super.python27.override { + packageOverrides = python-self: python-super: { + # eliot is not part of nixpkgs at all at this time. + eliot = python-self.callPackage ./eliot.nix { }; + # The packaged version of Nevow is very slightly out of date but also + # conflicts with the packaged version of Twisted. Supply our own + # slightly newer version. + nevow = python-super.callPackage ./nevow.nix { }; + }; + }; +} diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix new file mode 100644 index 000000000..cc0eea479 --- /dev/null +++ b/nix/tahoe-lafs.nix @@ -0,0 +1,66 @@ +{ fetchFromGitHub, lib +, nettools, python +, twisted, foolscap, nevow, zfec +, setuptools, setuptoolsTrial, pyasn1, zope_interface +, service-identity, pyyaml, magic-wormhole, treq, appdirs +, beautifulsoup4, eliot, autobahn, cryptography +, html5lib +}: +python.pkgs.buildPythonPackage rec { + version = "1.14.0.dev"; + name = "tahoe-lafs-${version}"; + src = lib.cleanSource ../.; + + postPatch = '' + # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around + # that. + for i in $(find src/allmydata/test -type f) + do + sed -i "$i" -e"s/localhost/127.0.0.1/g" + done + + # Some tests are flaky or fail to skip when dependencies are missing. + # This list is over-zealous because it's more work to disable individual + # tests with in a module. + + # test_system is a lot of integration-style tests that do a lot of real + # networking between many processes. They sometimes fail spuriously. + rm src/allmydata/test/test_system.py + + # Many of these tests don't properly skip when i2p or tor dependencies are + # not supplied (and we are not supplying them). + rm src/allmydata/test/test_i2p_provider.py + rm src/allmydata/test/test_connections.py + rm src/allmydata/test/cli/test_create.py + rm src/allmydata/test/test_client.py + rm src/allmydata/test/test_runner.py + + # Some eliot code changes behavior based on whether stdout is a tty or not + # and fails when it is not. + rm src/allmydata/test/test_eliotutil.py + ''; + + + propagatedNativeBuildInputs = [ + nettools + ]; + + propagatedBuildInputs = with python.pkgs; [ + twisted foolscap nevow zfec appdirs + setuptoolsTrial pyasn1 zope_interface + service-identity pyyaml magic-wormhole treq + eliot autobahn cryptography setuptools + ]; + + checkInputs = with python.pkgs; [ + hypothesis + testtools + fixtures + beautifulsoup4 + html5lib + ]; + + checkPhase = '' + ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata + ''; +} diff --git a/pyinstaller.spec b/pyinstaller.spec index bfd7d5acf..6d1bd21f2 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -31,7 +31,6 @@ hidden_imports = [ 'allmydata.introducer', 'allmydata.stats', 'cffi', - 'characteristic', 'Crypto', 'packaging.specifiers', 'six.moves.html_parser', diff --git a/relnotes.txt b/relnotes.txt index 0caadd8f2..a46996dfb 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.13.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.14.0 -The Tahoe-LAFS team is pleased to announce version 1.13.0 of +The Tahoe-LAFS team is pleased to announce version 1.14.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -18,22 +18,16 @@ unique security and fault-tolerance properties: The previous stable release of Tahoe-LAFS was v1.12.1, released on January 18, 2017. -v1.13.0 fixes several problems with the magic-folder frontend, adds ---json to more commands, adds the "tahoe status" and "tahoe -magic-folder status" commands and some internal refactoring. It is -also now possible to have multiple magic folders in a single Tahoe -client. +The v1.14.0 release: makes several Magic Folder improvements (MacOS +support, better logs, fewer conflict cases); adds an Eliot streaming +logs endpoint; adds an extension point for storage customization; +makes a bunch of bug-fixes and cleanups. NixOS is a supported +platform; Fedora 29 is no longer a supported platform. Several early +parts of Python3 porting have landed. -A long-standing feature to improve the share-placement algorithm -("servers of happiness") has been updated and merged. "tahoe backup" -now reports progress. - -New clients can now be added to a grid using "magic wormhole" [14], a -SPAKE2-based method of securely connecting two computers. This adds -the "tahoe invite" command for a current grid-participant to invite a -new client and a corresponding option "tahoe create-client --join" for -the other side. These use a server at ``wormhole.tahoe-lafs.org`` -currently operated by Least Authority. +DEPRECATED: ``tahoe start``, ``tahoe stop``, ``tahoe restart`` and +``tahoe daemonize`` are all deprecated in favour of using ``tahoe +run`` (along with a suitable process manager if desired). Please see ``NEWS.rst`` for a more complete list of changes. @@ -159,12 +153,12 @@ May 17, 2018 San Francisco, California, USA -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.13.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.14.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.13.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.13.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.13.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.14.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.14.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.14.0/INSTALL.html [7] https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS diff --git a/setup.cfg b/setup.cfg index 8e653f7d1..de40c9e23 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,4 @@ sdist = update_version sdist install = update_version install develop = update_version develop bdist_egg = update_version bdist_egg +bdist_wheel = update_version bdist_wheel diff --git a/setup.py b/setup.py index 2e7cabfb4..caa37f16e 100644 --- a/setup.py +++ b/setup.py @@ -30,18 +30,112 @@ def read_version_py(infname): VERSION_PY_FILENAME = 'src/allmydata/_version.py' version = read_version_py(VERSION_PY_FILENAME) -# Tahoe's dependencies are managed by the find_links= entry in setup.cfg and -# the _auto_deps.install_requires list, which is used in the call to setup() -# below. -adglobals = {} -auto_deps_fn = "src/allmydata/_auto_deps.py" -if sys.version_info[0] >= 3: - exec(compile(open(auto_deps_fn, 'rb').read(), auto_deps_fn, "exec"), - adglobals, adglobals) -else: - execfile(auto_deps_fn, adglobals) -install_requires = adglobals['install_requires'] -setup_requires = adglobals['setup_requires'] +install_requires = [ + # we don't need much out of setuptools but the version checking stuff + # needs pkg_resources and PEP 440 version specifiers. + "setuptools >= 28.8.0", + + "zfec >= 1.1.0", + + # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. + # zope.interface 3.6.3 and 3.6.4 are incompatible with Nevow (#1435). + "zope.interface >= 3.6.0, != 3.6.3, != 3.6.4", + + # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for + # transferring large mutable files of size N. + # * foolscap < 0.6 is incompatible with Twisted 10.2.0. + # * foolscap 0.6.1 quiets a DeprecationWarning. + # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. + # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, + # rather than 1024-bit RSA-with-MD5. This also allows us to work + # with a FIPS build of OpenSSL. + # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, + # and allocate_tcp_port + # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo + # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs + # * foolscap 0.13.2 drops i2p support completely + "foolscap == 0.13.1", + + # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that + # Twisted[conch] also depends on cryptography and Twisted[tls] + # transitively depends on cryptography. So it's anyone's guess what + # version of cryptography will *really* be installed. + "cryptography >= 2.6", + + # * We need Twisted 10.1.0 for the FTP frontend in order for + # Twisted's FTP server to support asynchronous close. + # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server + # rekeying bug + # * The FTP frontend depends on Twisted >= 11.1.0 for + # filepath.Permissions + # * Nevow 0.11.1 depends on Twisted >= 13.0.0. + # * The SFTP frontend and manhole depend on the conch extra. However, we + # can't explicitly declare that without an undesirable dependency on gmpy, + # as explained in ticket #2740. + # * Due to a setuptools bug, we need to declare a dependency on the tls + # extra even though we only depend on it via foolscap. + # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. + # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, + # which otherwise causes test_system to fail (DirtyReactorError, due to + # leftover timers) + # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed + # for coverage testing + # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch + # extra, letting us use that extra instead of trying to duplicate its + # dependencies here. Twisted[conch] >18.7 introduces a dependency on + # bcrypt. It is nice to avoid that if the user ends up with an older + # version of Twisted. That's hard to express except by using the extra. + # + # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the + # * initializer, needed by logic in our custom `Request` subclass. + # + # In a perfect world, Twisted[conch] would be a dependency of an "sftp" + # extra. However, pip fails to resolve the dependencies all + # dependencies when asked for Twisted[tls] *and* Twisted[conch]. + # Specifically, "Twisted[conch]" (as the later requirement) is ignored. + # If there were an Tahoe-LAFS sftp extra that dependended on + # Twisted[conch] and install_requires only included Twisted[tls] then + # `pip install tahoe-lafs[sftp]` would not install requirements + # specified by Twisted[conch]. Since this would be the *whole point* of + # an sftp extra in Tahoe-LAFS, there is no point in having one. + "Twisted[tls,conch] >= 18.4.0", + + # We need Nevow >= 0.11.1 which can be installed using pip. + "Nevow >= 0.11.1", + + "PyYAML >= 3.11", + + "six >= 1.10.0", + + # for 'tahoe invite' and 'tahoe join' + "magic-wormhole >= 0.10.2", + + # Eliot is contemplating dropping Python 2 support. Stick to a version we + # know works on Python 2.7. + "eliot ~= 1.7", + + # A great way to define types of values. + "attrs >= 18.2.0", + + # WebSocket library for twisted and asyncio + "autobahn >= 19.5.2", +] + +setup_requires = [ + 'setuptools >= 28.8.0', # for PEP-440 style versions +] + +tor_requires = [ + # This is exactly what `foolscap[tor]` means but pip resolves the pair of + # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose + # this if we don't declare it ourselves! + "txtorcon >= 0.17.0", +] + +i2p_requires = [ + # See the comment in tor_requires. + "txi2p >= 0.3.2", +] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': del sys.argv[1] @@ -254,21 +348,22 @@ setup(name="tahoe-lafs", # also set in __init__.py python_requires="<3.0", install_requires=install_requires, extras_require={ - ':sys_platform=="win32"': ["pypiwin32"], - ':sys_platform!="win32" and sys_platform!="linux2"': ["watchdog"], # For magic-folder on "darwin" (macOS) and the BSDs + # Duplicate the Twisted pywin32 dependency here. See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some + # discussion. + ':sys_platform=="win32"': ["pywin32 != 226"], "test": [ # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it # intentionally. "pyflakes == 2.1.0", - "coverage", + # coverage 5.0 breaks the integration tests in some opaque way. + # This probably needs to be addressed in a more permanent way + # eventually... + "coverage ~= 4.5", "mock", "tox", - "foolscap[tor] >= 0.12.5", - "txtorcon >= 0.17.0", # in case pip's resolver doesn't work - "foolscap[i2p] >= 0.12.6", - "txi2p >= 0.3.2", # in case pip's resolver doesn't work "pytest", "pytest-twisted", "hypothesis >= 3.6.1", @@ -276,22 +371,20 @@ setup(name="tahoe-lafs", # also set in __init__.py "towncrier", "testtools", "fixtures", - ], - "tor": [ - "foolscap[tor] >= 0.12.5", - "txtorcon >= 0.17.0", # in case pip's resolver doesn't work - ], - "i2p": [ - "foolscap[i2p] >= 0.12.6", - "txi2p >= 0.3.2", # in case pip's resolver doesn't work - ], + "beautifulsoup4", + "html5lib", + ] + tor_requires + i2p_requires, + "tor": tor_requires, + "i2p": i2p_requires, }, package_data={"allmydata.web": ["*.xhtml", "static/*.js", "static/*.png", "static/*.css", "static/img/*.png", "static/css/*.css", - ] + ], + "allmydata": ["ported-modules.txt"], }, + include_package_data=True, setup_requires=setup_requires, entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, **setup_args diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 195a92113..915b4c013 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -3,15 +3,14 @@ Decentralized storage grid. community web site: U{https://tahoe-lafs.org/} """ -import six - -class PackagingError(EnvironmentError): - """ - Raised when there is an error in packaging of Tahoe-LAFS or its - dependencies which makes it impossible to proceed safely. - """ - pass +__all__ = [ + "__version__", + "full_version", + "branch", + "__appname__", + "__full_version__", +] __version__ = "unknown" try: @@ -38,470 +37,3 @@ __appname__ = "tahoe-lafs" # in the "application" part of the Tahoe versioning scheme: # https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning __full_version__ = __appname__ + '/' + str(__version__) - -import os, platform, re, subprocess, sys, traceback -_distributor_id_cmdline_re = re.compile("(?:Distributor ID:)\s*(.*)", re.I) -_release_cmdline_re = re.compile("(?:Release:)\s*(.*)", re.I) - -_distributor_id_file_re = re.compile("(?:DISTRIB_ID\s*=)\s*(.*)", re.I) -_release_file_re = re.compile("(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I) - -_distname = None -_version = None - -def get_linux_distro(): - """ Tries to determine the name of the Linux OS distribution name. - - First, try to parse a file named "/etc/lsb-release". If it exists, and - contains the "DISTRIB_ID=" line and the "DISTRIB_RELEASE=" line, then return - the strings parsed from that file. - - If that doesn't work, then invoke platform.dist(). - - If that doesn't work, then try to execute "lsb_release", as standardized in - 2001: - - http://refspecs.freestandards.org/LSB_1.0.0/gLSB/lsbrelease.html - - The current version of the standard is here: - - http://refspecs.freestandards.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/lsbrelease.html - - that lsb_release emitted, as strings. - - Returns a tuple (distname,version). Distname is what LSB calls a - "distributor id", e.g. "Ubuntu". Version is what LSB calls a "release", - e.g. "8.04". - - A version of this has been submitted to python as a patch for the standard - library module "platform": - - http://bugs.python.org/issue3937 - """ - global _distname,_version - if _distname and _version: - return (_distname, _version) - - try: - etclsbrel = open("/etc/lsb-release", "rU") - for line in etclsbrel: - m = _distributor_id_file_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - m = _release_file_re.search(line) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - except EnvironmentError: - pass - - (_distname, _version) = platform.dist()[:2] - if _distname and _version: - return (_distname, _version) - - if os.path.isfile("/usr/bin/lsb_release") or os.path.isfile("/bin/lsb_release"): - try: - p = subprocess.Popen(["lsb_release", "--all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - rc = p.wait() - if rc == 0: - for line in p.stdout.readlines(): - m = _distributor_id_cmdline_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - - m = _release_cmdline_re.search(p.stdout.read()) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - except EnvironmentError: - pass - - if os.path.exists("/etc/arch-release"): - return ("Arch_Linux", "") - - return (_distname,_version) - -def get_platform(): - # Our version of platform.platform(), telling us both less and more than the - # Python Standard Library's version does. - # We omit details such as the Linux kernel version number, but we add a - # more detailed and correct rendition of the Linux distribution and - # distribution-version. - if "linux" in platform.system().lower(): - return platform.system()+"-"+"_".join(get_linux_distro())+"-"+platform.machine()+"-"+"_".join([x for x in platform.architecture() if x]) - else: - return platform.platform() - - -from allmydata.util import verlib -def normalized_version(verstr, what=None): - try: - suggested = verlib.suggest_normalized_version(verstr) or verstr - return verlib.NormalizedVersion(suggested) - except verlib.IrrationalVersionError: - raise - except StandardError: - cls, value, trace = sys.exc_info() - new_exc = PackagingError("could not parse %s due to %s: %s" - % (what or repr(verstr), cls.__name__, value)) - six.reraise(cls, new_exc, trace) - - -def get_openssl_version(): - try: - from OpenSSL import SSL - return extract_openssl_version(SSL) - except Exception: - return ("unknown", None, None) - -def extract_openssl_version(ssl_module): - openssl_version = ssl_module.SSLeay_version(ssl_module.SSLEAY_VERSION) - if openssl_version.startswith('OpenSSL '): - openssl_version = openssl_version[8 :] - - (version, _, comment) = openssl_version.partition(' ') - - try: - openssl_cflags = ssl_module.SSLeay_version(ssl_module.SSLEAY_CFLAGS) - if '-DOPENSSL_NO_HEARTBEATS' in openssl_cflags.split(' '): - comment += ", no heartbeats" - except Exception: - pass - - return (version, None, comment if comment else None) - -def get_package_versions_and_locations(): - import warnings - from _auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \ - runtime_warning_messages, warning_imports, ignorable - - def package_dir(srcfile): - return os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) - - # pkg_resources.require returns the distribution that pkg_resources attempted to put - # on sys.path, which can differ from the one that we actually import due to #1258, - # or any other bug that causes sys.path to be set up incorrectly. Therefore we - # must import the packages in order to check their versions and paths. - - # This is to suppress all UserWarnings and various DeprecationWarnings and RuntimeWarnings - # (listed in _auto_deps.py). - - warnings.filterwarnings("ignore", category=UserWarning, append=True) - - for msg in global_deprecation_messages + deprecation_messages: - warnings.filterwarnings("ignore", category=DeprecationWarning, message=msg, append=True) - for msg in runtime_warning_messages: - warnings.filterwarnings("ignore", category=RuntimeWarning, message=msg, append=True) - try: - for modulename in warning_imports: - try: - __import__(modulename) - except ImportError: - pass - finally: - # Leave suppressions for UserWarnings and global_deprecation_messages active. - for _ in runtime_warning_messages + deprecation_messages: - warnings.filters.pop() - - packages = [] - pkg_resources_vers_and_locs = dict() - - if not hasattr(sys, 'frozen'): - import pkg_resources - from _auto_deps import install_requires - - pkg_resources_vers_and_locs = dict([(p.project_name.lower(), (str(p.version), p.location)) - for p in pkg_resources.require(install_requires)]) - - def get_version(module): - if hasattr(module, '__version__'): - return str(getattr(module, '__version__')) - elif hasattr(module, 'version'): - ver = getattr(module, 'version') - if isinstance(ver, tuple): - return '.'.join(map(str, ver)) - else: - return str(ver) - else: - return 'unknown' - - for pkgname, modulename in [(__appname__, 'allmydata')] + package_imports: - if modulename: - try: - __import__(modulename) - module = sys.modules[modulename] - except ImportError: - etype, emsg, etrace = sys.exc_info() - trace_info = (etype, str(emsg), ([None] + traceback.extract_tb(etrace))[-1]) - packages.append( (pkgname, (None, None, trace_info)) ) - else: - comment = None - if pkgname == __appname__: - comment = "%s: %s" % (branch, full_version) - elif pkgname == 'setuptools' and hasattr(module, '_distribute'): - # distribute does not report its version in any module variables - comment = 'distribute' - ver = get_version(module) - loc = package_dir(module.__file__) - if ver == "unknown" and pkgname in pkg_resources_vers_and_locs: - (pr_ver, pr_loc) = pkg_resources_vers_and_locs[pkgname] - if loc == os.path.normcase(os.path.realpath(pr_loc)): - ver = pr_ver - packages.append( (pkgname, (ver, loc, comment)) ) - elif pkgname == 'python': - packages.append( (pkgname, (platform.python_version(), sys.executable, None)) ) - elif pkgname == 'platform': - packages.append( (pkgname, (get_platform(), None, None)) ) - elif pkgname == 'OpenSSL': - packages.append( (pkgname, get_openssl_version()) ) - - cross_check_errors = [] - - if len(pkg_resources_vers_and_locs) > 0: - imported_packages = set([p.lower() for (p, _) in packages]) - extra_packages = [] - - for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.iteritems(): - if pr_name not in imported_packages and pr_name not in ignorable: - extra_packages.append( (pr_name, (pr_ver, pr_loc, "according to pkg_resources")) ) - - cross_check_errors = cross_check(pkg_resources_vers_and_locs, packages) - packages += extra_packages - - return packages, cross_check_errors - - -def split_requirement(req): - """ - Split up a single requirement string into the different version constraint pieces. - - This is like req.split(",") except it doesn't split on , found inside []. - - :return: A list of the split up pieces. - """ - in_extras = False - pieces = [] - chunk = '' - for ch in req: - if in_extras: - if ch == ']': - in_extras = False - chunk += ch - else: - if ch == '[': - in_extras = True - chunk += ch - elif ch == ',': - pieces.append(chunk) - chunk = '' - else: - chunk += ch - pieces.append(chunk) - return pieces - - -def check_requirement(req, vers_and_locs): - # We support only conjunctions of <=, >=, and != - reqlist = split_requirement(req) - name = reqlist[0].split('<=')[0].split('>=')[0].split('!=')[0].strip(' ').split('[')[0] - if name not in vers_and_locs: - raise PackagingError("no version info for %s" % (name,)) - if req.strip(' ') == name: - return - (actual, location, comment) = vers_and_locs[name] - if actual is None: - # comment is (type, message, (filename, line number, function name, text)) for the original ImportError - raise ImportError("for requirement %r: %s" % (req, comment)) - if actual == 'unknown': - return - try: - actualver = normalized_version(actual, what="actual version %r of %s from %r" % - (actual, name, location)) - matched = match_requirement(req, reqlist, actualver) - except verlib.IrrationalVersionError: - # meh, it probably doesn't matter - return - - if not matched: - msg = ("We require %s, but could only find version %s.\n" % (req, actual)) - if location and location != 'unknown': - msg += "The version we found is from %r.\n" % (location,) - msg += ("To resolve this problem, uninstall that version, either using your\n" - "operating system's package manager or by moving aside the directory.") - raise PackagingError(msg) - - -def match_requirement(req, reqlist, actualver): - for r in reqlist: - s = r.split('<=') - if len(s) == 2: - required = s[1].strip(' ') - if not (actualver <= normalized_version(required, what="required maximum version %r in %r" % (required, req))): - return False # maximum requirement not met - else: - s = r.split('>=') - if len(s) == 2: - required = s[1].strip(' ') - if not (actualver >= normalized_version(required, what="required minimum version %r in %r" % (required, req))): - return False # minimum requirement not met - else: - s = r.split('!=') - if len(s) == 2: - required = s[1].strip(' ') - if not (actualver != normalized_version(required, what="excluded version %r in %r" % (required, req))): - return False # not-equal requirement not met - else: - raise PackagingError("no version info or could not understand requirement %r" % (req,)) - - return True - - -def cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list): - """This function returns a list of errors due to any failed cross-checks.""" - - from _auto_deps import not_import_versionable - - errors = [] - not_pkg_resourceable = ['python', 'platform', __appname__.lower(), 'openssl'] - - for name, (imp_ver, imp_loc, imp_comment) in imported_vers_and_locs_list: - name = name.lower() - if name not in not_pkg_resourceable: - if name not in pkg_resources_vers_and_locs: - if name == "setuptools" and "distribute" in pkg_resources_vers_and_locs: - pr_ver, pr_loc = pkg_resources_vers_and_locs["distribute"] - if not (os.path.normpath(os.path.realpath(pr_loc)) == os.path.normpath(os.path.realpath(imp_loc)) - and imp_comment == "distribute"): - errors.append("Warning: dependency 'setuptools' found to be version %r of 'distribute' from %r " - "by pkg_resources, but 'import setuptools' gave version %r [%s] from %r. " - "A version mismatch is expected, but a location mismatch is not." - % (pr_ver, pr_loc, imp_ver, imp_comment or 'probably *not* distribute', imp_loc)) - else: - errors.append("Warning: dependency %r (version %r imported from %r) was not found by pkg_resources." - % (name, imp_ver, imp_loc)) - continue - - pr_ver, pr_loc = pkg_resources_vers_and_locs[name] - if imp_ver is None and imp_loc is None: - errors.append("Warning: dependency %r could not be imported. pkg_resources thought it should be possible " - "to import version %r from %r.\nThe exception trace was %r." - % (name, pr_ver, pr_loc, imp_comment)) - continue - - # If the pkg_resources version is identical to the imported version, don't attempt - # to normalize them, since it is unnecessary and may fail (ticket #2499). - if imp_ver != 'unknown' and pr_ver == imp_ver: - continue - - try: - pr_normver = normalized_version(pr_ver) - except verlib.IrrationalVersionError: - continue - except Exception as e: - errors.append("Warning: version number %r found for dependency %r by pkg_resources could not be parsed. " - "The version found by import was %r from %r. " - "pkg_resources thought it should be found at %r. " - "The exception was %s: %s" - % (pr_ver, name, imp_ver, imp_loc, pr_loc, e.__class__.__name__, e)) - else: - if imp_ver == 'unknown': - if name not in not_import_versionable: - errors.append("Warning: unexpectedly could not find a version number for dependency %r imported from %r. " - "pkg_resources thought it should be version %r at %r." - % (name, imp_loc, pr_ver, pr_loc)) - else: - try: - imp_normver = normalized_version(imp_ver) - except verlib.IrrationalVersionError: - continue - except Exception as e: - errors.append("Warning: version number %r found for dependency %r (imported from %r) could not be parsed. " - "pkg_resources thought it should be version %r at %r. " - "The exception was %s: %s" - % (imp_ver, name, imp_loc, pr_ver, pr_loc, e.__class__.__name__, e)) - else: - if pr_ver == 'unknown' or (pr_normver != imp_normver): - if not os.path.normpath(os.path.realpath(pr_loc)) == os.path.normpath(os.path.realpath(imp_loc)): - errors.append("Warning: dependency %r found to have version number %r (normalized to %r, from %r) " - "by pkg_resources, but version %r (normalized to %r, from %r) by import." - % (name, pr_ver, str(pr_normver), pr_loc, imp_ver, str(imp_normver), imp_loc)) - - return errors - - -_vers_and_locs_list, _cross_check_errors = get_package_versions_and_locations() - - -def get_error_string(errors, debug=False): - from allmydata._auto_deps import install_requires - - msg = "\n%s\n" % ("\n".join(errors),) - if debug: - msg += ("\n" - "For debugging purposes, the PYTHONPATH was\n" - " %r\n" - "install_requires was\n" - " %r\n" - "sys.path after importing pkg_resources was\n" - " %s\n" - % (os.environ.get('PYTHONPATH'), install_requires, (os.pathsep+"\n ").join(sys.path)) ) - return msg - -def check_all_requirements(): - """This function returns a list of errors due to any failed checks.""" - - from allmydata._auto_deps import install_requires - - fatal_errors = [] - - # We require at least 2.6 on all platforms. - # (On Python 3, we'll have failed long before this point.) - if sys.version_info < (2, 6): - try: - version_string = ".".join(map(str, sys.version_info)) - except Exception: - version_string = repr(sys.version_info) - fatal_errors.append("Tahoe-LAFS currently requires Python v2.6 or greater (but less than v3), not %s" - % (version_string,)) - - vers_and_locs = dict(_vers_and_locs_list) - for requirement in install_requires: - try: - check_requirement(requirement, vers_and_locs) - except (ImportError, PackagingError) as e: - fatal_errors.append("%s: %s" % (e.__class__.__name__, e)) - - if fatal_errors: - raise PackagingError(get_error_string(fatal_errors + _cross_check_errors, debug=True)) - -check_all_requirements() - - -def get_package_versions(): - return dict([(k, v) for k, (v, l, c) in _vers_and_locs_list]) - -def get_package_locations(): - return dict([(k, l) for k, (v, l, c) in _vers_and_locs_list]) - -def get_package_versions_string(show_paths=False, debug=False): - res = [] - for p, (v, loc, comment) in _vers_and_locs_list: - info = str(p) + ": " + str(v) - if comment: - info = info + " [%s]" % str(comment) - if show_paths: - info = info + " (%s)" % str(loc) - res.append(info) - - output = "\n".join(res) + "\n" - - if _cross_check_errors: - output += get_error_string(_cross_check_errors, debug=debug) - - return output diff --git a/src/allmydata/_auto_deps.py b/src/allmydata/_auto_deps.py index 38b98e59c..cf98aae96 100644 --- a/src/allmydata/_auto_deps.py +++ b/src/allmydata/_auto_deps.py @@ -4,117 +4,11 @@ # It is ok to import modules from the Python Standard Library if they are # always available, or the import is protected by try...except ImportError. -# The semantics for requirement specs changed incompatibly in setuptools 8, -# which now follows PEP 440. The requirements used in this file must be valid -# under both the old and new semantics. That can be achieved by limiting -# requirement specs to one of the following forms: -# -# * >= X, <= Y where X < Y -# * >= X, != Y, != Z, ... where X < Y < Z... -# -# (In addition, check_requirement in allmydata/__init__.py only supports -# >=, <= and != operators.) - -install_requires = [ - # we don't need much out of setuptools, but the __init__.py stuff does - # need pkg_resources . We use >=11.3 here because that's what - # "cryptography" requires (which is a sub-dependency of TLS-using - # packages), so there's no point in requiring less. - "setuptools >= 28.8.0", - - "zfec >= 1.1.0", - - # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. - # zope.interface 3.6.3 and 3.6.4 are incompatible with Nevow (#1435). - "zope.interface >= 3.6.0, != 3.6.3, != 3.6.4", - - # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for - # transferring large mutable files of size N. - # * foolscap < 0.6 is incompatible with Twisted 10.2.0. - # * foolscap 0.6.1 quiets a DeprecationWarning. - # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. - # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, - # rather than 1024-bit RSA-with-MD5. This also allows us to work - # with a FIPS build of OpenSSL. - # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, - # and allocate_tcp_port - # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo - # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs - "foolscap >= 0.12.6", - - # pycryptopp-0.6.0 includes ed25519 - "pycryptopp >= 0.6.0", - - "service-identity", # this is needed to suppress complaints about being unable to verify certs - "characteristic >= 14.0.0", # latest service-identity depends on this version - "pyasn1 >= 0.1.8", # latest pyasn1-modules depends on this version - "pyasn1-modules >= 0.0.5", # service-identity depends on this - - # * On Linux we need at least Twisted 10.1.0 for inotify support - # used by the drop-upload frontend. - # * We also need Twisted 10.1.0 for the FTP frontend in order for - # Twisted's FTP server to support asynchronous close. - # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server - # rekeying bug - # * The FTP frontend depends on Twisted >= 11.1.0 for - # filepath.Permissions - # * Nevow 0.11.1 depends on Twisted >= 13.0.0. - # * The SFTP frontend and manhole depend on the conch extra. However, we - # can't explicitly declare that without an undesirable dependency on gmpy, - # as explained in ticket #2740. - # * Due to a setuptools bug, we need to declare a dependency on the tls - # extra even though we only depend on it via foolscap. - # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. - # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, - # which otherwise causes test_system to fail (DirtyReactorError, due to - # leftover timers) - # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed - # for coverage testing - - # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch - # extra, letting us use that extra instead of trying to duplicate its - # dependencies here. Twisted[conch] >18.7 introduces a dependency on - # bcrypt. It is nice to avoid that if the user ends up with an older - # version of Twisted. That's hard to express except by using the extra. - "Twisted[tls,conch] >= 16.6.0", - - # We need Nevow >= 0.11.1 which can be installed using pip. - "Nevow >= 0.11.1", - - # * pyOpenSSL is required in order for foolscap to provide secure connections. - # Since foolscap doesn't reliably declare this dependency in a machine-readable - # way, we need to declare a dependency on pyOpenSSL ourselves. Tahoe-LAFS does - # not *directly* depend on pyOpenSSL. - # * pyOpenSSL >= 0.13 is needed in order to avoid - # , and also to check the - # version of OpenSSL that pyOpenSSL is using. - # * pyOpenSSL >= 0.14 is needed in order to avoid - # . - "pyOpenSSL >= 0.14", - "PyYAML >= 3.11", - - "six >= 1.10.0", - - # for 'tahoe invite' and 'tahoe join' - "magic-wormhole >= 0.10.2", - - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. Because we don't have support for `==` - # constraints, pin 1.7.x this way. I feel pretty safe betting that we - # won't end up stuck on Eliot 1.7.100 with a critical fix only present in - # 1.7.101. And if we do, I know how to deal with that situation. - "eliot >= 1.7.0, <= 1.7.100", - - # A great way to define types of values. - "attrs >= 18.2.0", -] - # Includes some indirect dependencies, but does not include allmydata. # These are in the order they should be listed by --version, etc. package_imports = [ # package name module name ('foolscap', 'foolscap'), - ('pycryptopp', 'pycryptopp'), ('zfec', 'zfec'), ('Twisted', 'twisted'), ('Nevow', 'nevow'), @@ -125,7 +19,6 @@ package_imports = [ ('OpenSSL', None), ('pyasn1', 'pyasn1'), ('service-identity', 'service_identity'), - ('characteristic', 'characteristic'), ('pyasn1-modules', 'pyasn1_modules'), ('cryptography', 'cryptography'), ('cffi', 'cffi'), @@ -137,6 +30,7 @@ package_imports = [ ('setuptools', 'setuptools'), ('eliot', 'eliot'), ('attrs', 'attr'), + ('autobahn', 'autobahn'), ] # Dependencies for which we don't know how to get a version number at run-time. @@ -147,8 +41,6 @@ not_import_versionable = [ # Dependencies reported by pkg_resources that we can safely ignore. ignorable = [ 'argparse', - 'pyutil', - 'zbase32', 'distribute', 'twisted-web', 'twisted-core', @@ -156,11 +48,6 @@ ignorable = [ ] -setup_requires = [ - 'setuptools >= 28.8.0', # for PEP-440 style versions -] - - # These are suppressed globally: global_deprecation_messages = [ diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 23690266d..874ff95ca 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -17,7 +17,7 @@ class FileProhibited(Exception): self.reason = reason -class Blacklist: +class Blacklist(object): def __init__(self, blacklist_fn): self.blacklist_fn = blacklist_fn self.last_mtime = None diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index df6887a4e..068f77a25 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -169,7 +169,7 @@ class CheckAndRepairResults(object): return self.post_repair_results -class DeepResultsBase: +class DeepResultsBase(object): def __init__(self, root_storage_index): self.root_storage_index = root_storage_index diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 901004db4..ced282900 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -7,24 +7,36 @@ from allmydata import node from base64 import urlsafe_b64encode from functools import partial from errno import ENOENT, EPERM +from ConfigParser import NoSectionError +from foolscap.furl import ( + decode_furl, +) + +import attr from zope.interface import implementer + +from twisted.plugin import ( + getPlugins, +) from twisted.internet import reactor, defer from twisted.application import service from twisted.application.internet import TimerService from twisted.python.filepath import FilePath -from twisted.python.failure import Failure -from pycryptopp.publickey import rsa import allmydata +from allmydata.crypto import rsa, ed25519 +from allmydata.crypto.util import remove_prefix from allmydata.storage.server import StorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper from allmydata.control import ControlServer from allmydata.introducer.client import IntroducerClient -from allmydata.util import (hashutil, base32, pollmixin, log, keyutil, idlib, - yamlutil) +from allmydata.util import ( + hashutil, base32, pollmixin, log, idlib, + yamlutil, configutil, +) from allmydata.util.encodingutil import (get_filesystem_encoding, from_utf8_or_none) from allmydata.util.abbreviate import parse_abbreviated_size @@ -33,7 +45,14 @@ from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider from allmydata.stats import StatsProvider from allmydata.history import History -from allmydata.interfaces import IStatsProducer, SDMF_VERSION, MDMF_VERSION, DEFAULT_MAX_SEGMENT_SIZE +from allmydata.interfaces import ( + IStatsProducer, + SDMF_VERSION, + MDMF_VERSION, + DEFAULT_MAX_SEGMENT_SIZE, + IFoolscapStoragePlugin, + IAnnounceableStorageServer, +) from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist @@ -44,9 +63,20 @@ GiB=1024*MiB TiB=1024*GiB PiB=1024*TiB -def _valid_config_sections(): - cfg = node._common_config_sections() - cfg.update({ +def _is_valid_section(section_name): + """ + Check for valid dynamic configuration section names. + + Currently considers all possible storage server plugin sections valid. + """ + return ( + section_name.startswith(b"storageserver.plugins.") or + section_name.startswith(b"storageclient.plugins.") + ) + + +_client_config = configutil.ValidConfiguration( + static_valid_sections={ "client": ( "helper.furl", "introducer.furl", @@ -57,6 +87,7 @@ def _valid_config_sections(): "shares.needed", "shares.total", "stats_gatherer.furl", + "storage.plugins", ), "grid_managers": None, # means "any options valid" "grid_manager_certificates": None, @@ -72,6 +103,7 @@ def _valid_config_sections(): "storage": ( "debug_discard", "enabled", + "anonymous", "expire.cutoff_date", "expire.enabled", "expire.immutable", @@ -82,6 +114,7 @@ def _valid_config_sections(): "readonly", "reserved_space", "storage_dir", + "plugins", "grid_management", ), "sftpd": ( @@ -95,14 +128,16 @@ def _valid_config_sections(): "helper": ( "enabled", ), - "magic_folder": ( - "download.umask", - "enabled", - "local.directory", - "poll_interval", - ), - }) - return cfg + }, + is_valid_section=_is_valid_section, + # Anything in a valid section is a valid item, for now. + is_valid_item=lambda section, ignored: _is_valid_section(section), +) + + +def _valid_config(): + cfg = node._common_valid_config() + return cfg.update(_client_config) # this is put into README in new node-directories CLIENT_README = """ @@ -122,7 +157,7 @@ def _make_secret(): return base32.b2a(os.urandom(hashutil.CRYPTO_VAL_SIZE)) + "\n" -class SecretHolder: +class SecretHolder(object): def __init__(self, lease_secret, convergence_secret): self._lease_secret = lease_secret self._convergence_secret = convergence_secret @@ -136,7 +171,7 @@ class SecretHolder: def get_convergence_secret(self): return self._convergence_secret -class KeyGenerator: +class KeyGenerator(object): """I create RSA keys for mutable files. Each call to generate() returns a single keypair. The keysize is specified first by the keysize= argument to generate(), then with a default set by set_default_keysize(), then @@ -163,8 +198,7 @@ class KeyGenerator: keysize = keysize or self.default_keysize # RSA key generation for a 2048 bit key takes between 0.8 and 3.2 # secs - signer = rsa.generate(keysize) - verifier = signer.get_verifying_key() + signer, verifier = rsa.create_signing_keypair(keysize) return defer.succeed( (verifier, signer) ) class Terminator(service.Service): @@ -189,10 +223,16 @@ def read_config(basedir, portnumfile, generated_files=[]): return node.read_config( basedir, portnumfile, generated_files=generated_files, - _valid_config_sections=_valid_config_sections, + _valid_config=_valid_config(), ) +config_from_string = partial( + node.config_from_string, + _valid_config=_valid_config(), +) + + def create_client(basedir=u".", _client_factory=None): """ Creates a new client instance (a subclass of Node). @@ -214,10 +254,11 @@ def create_client(basedir=u".", _client_factory=None): _client_factory=_client_factory, ) except Exception: - return Failure() + return defer.fail() -def create_client_from_config(config, _client_factory=None): +@defer.inlineCallbacks +def create_client_from_config(config, _client_factory=None, _introducer_factory=None): """ Creates a new client instance (a subclass of Node). Most code should probably use `create_client` instead. @@ -229,46 +270,175 @@ def create_client_from_config(config, _client_factory=None): :param _client_factory: for testing; the class to instantiate instead of _Client + + :param _introducer_factory: for testing; the class to instantiate instead + of IntroducerClient """ - try: - if _client_factory is None: - _client_factory = _Client + if _client_factory is None: + _client_factory = _Client - i2p_provider = create_i2p_provider(reactor, config) - tor_provider = create_tor_provider(reactor, config) - handlers = node.create_connection_handlers(reactor, config, i2p_provider, tor_provider) - default_connection_handlers, foolscap_connection_handlers = handlers - tub_options = node.create_tub_options(config) + i2p_provider = create_i2p_provider(reactor, config) + tor_provider = create_tor_provider(reactor, config) + handlers = node.create_connection_handlers(reactor, config, i2p_provider, tor_provider) + default_connection_handlers, foolscap_connection_handlers = handlers + tub_options = node.create_tub_options(config) - main_tub = node.create_main_tub( - config, tub_options, default_connection_handlers, - foolscap_connection_handlers, i2p_provider, tor_provider, - ) - control_tub = node.create_control_tub() + main_tub = node.create_main_tub( + config, tub_options, default_connection_handlers, + foolscap_connection_handlers, i2p_provider, tor_provider, + ) + control_tub = node.create_control_tub() - introducer_clients = create_introducer_clients(config, main_tub) - storage_broker = create_storage_farm_broker( - config, default_connection_handlers, foolscap_connection_handlers, - tub_options, introducer_clients - ) + introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) + storage_broker = create_storage_farm_broker( + config, default_connection_handlers, foolscap_connection_handlers, + tub_options, introducer_clients + ) - client = _client_factory( + client = _client_factory( + config, + main_tub, + control_tub, + i2p_provider, + tor_provider, + introducer_clients, + storage_broker, + ) + + # Initialize storage separately after creating the client. This is + # necessary because we need to pass a reference to the client in to the + # storage plugins to allow them to initialize themselves (specifically, + # they may want the anonymous IStorageServer implementation so they don't + # have to duplicate all of its basic storage functionality). A better way + # to do this, eventually, may be to create that implementation first and + # then pass it in to both storage plugin creation and the client factory. + # This avoids making a partially initialized client object escape the + # client factory and removes the circular dependency between these + # objects. + storage_plugins = yield _StoragePlugins.from_config( + client.get_anonymous_storage_server, + config, + ) + client.init_storage(storage_plugins.announceable_storage_servers) + + i2p_provider.setServiceParent(client) + tor_provider.setServiceParent(client) + for ic in introducer_clients: + ic.setServiceParent(client) + storage_broker.setServiceParent(client) + defer.returnValue(client) + + +@attr.s +class _StoragePlugins(object): + """ + Functionality related to getting storage plugins set up and ready for use. + + :ivar list[IAnnounceableStorageServer] announceable_storage_servers: The + announceable storage servers that should be used according to node + configuration. + """ + announceable_storage_servers = attr.ib() + + @classmethod + @defer.inlineCallbacks + def from_config(cls, get_anonymous_storage_server, config): + """ + Load and configured storage plugins. + + :param get_anonymous_storage_server: A no-argument callable which + returns the node's anonymous ``IStorageServer`` implementation. + + :param _Config config: The node's configuration. + + :return: A ``_StoragePlugins`` initialized from the given + configuration. + """ + storage_plugin_names = cls._get_enabled_storage_plugin_names(config) + plugins = list(cls._collect_storage_plugins(storage_plugin_names)) + unknown_plugin_names = storage_plugin_names - {plugin.name for plugin in plugins} + if unknown_plugin_names: + raise configutil.UnknownConfigError( + "Storage plugins {} are enabled but not known on this system.".format( + unknown_plugin_names, + ), + ) + announceable_storage_servers = yield cls._create_plugin_storage_servers( + get_anonymous_storage_server, config, - main_tub, - control_tub, - i2p_provider, - tor_provider, - introducer_clients, - storage_broker, + plugins, ) - i2p_provider.setServiceParent(client) - tor_provider.setServiceParent(client) - for ic in introducer_clients: - ic.setServiceParent(client) - storage_broker.setServiceParent(client) - return defer.succeed(client) - except Exception: - return Failure() + defer.returnValue(cls( + announceable_storage_servers, + )) + + @classmethod + def _get_enabled_storage_plugin_names(cls, config): + """ + Get the names of storage plugins that are enabled in the configuration. + """ + return set( + config.get_config( + "storage", "plugins", b"" + ).decode("ascii").split(u",") + ) - {u""} + + @classmethod + def _collect_storage_plugins(cls, storage_plugin_names): + """ + Get the storage plugins with names matching those given. + """ + return list( + plugin + for plugin + in getPlugins(IFoolscapStoragePlugin) + if plugin.name in storage_plugin_names + ) + + @classmethod + def _create_plugin_storage_servers(cls, get_anonymous_storage_server, config, plugins): + """ + Cause each storage plugin to instantiate its storage server and return + them all. + + :return: A ``Deferred`` that fires with storage servers instantiated + by all of the given storage server plugins. + """ + return defer.gatherResults( + list( + plugin.get_storage_server( + cls._get_storage_plugin_configuration(config, plugin.name), + get_anonymous_storage_server, + ).addCallback( + partial( + _add_to_announcement, + {u"name": plugin.name}, + ), + ) + for plugin + # The order is fairly arbitrary and it is not meant to convey + # anything but providing *some* stable ordering makes the data + # a little easier to deal with (mainly in tests and when + # manually inspecting it). + in sorted(plugins, key=lambda p: p.name) + ), + ) + + @classmethod + def _get_storage_plugin_configuration(cls, config, storage_plugin_name): + """ + Load the configuration for a storage server plugin with the given name. + + :return dict[bytes, bytes]: The matching configuration. + """ + try: + config = config.items( + "storageserver.plugins." + storage_plugin_name, + ) + except NoSectionError: + config = [] + return dict(config) + def _sequencer(config): @@ -288,12 +458,18 @@ def _sequencer(config): return seqnum, nonce -def create_introducer_clients(config, main_tub): +def create_introducer_clients(config, main_tub, _introducer_factory=None): """ Read, validate and parse any 'introducers.yaml' configuration. + :param _introducer_factory: for testing; the class to instantiate instead + of IntroducerClient + :returns: a list of IntroducerClient instances """ + if _introducer_factory is None: + _introducer_factory = IntroducerClient + # we return this list introducer_clients = [] @@ -339,7 +515,7 @@ def create_introducer_clients(config, main_tub): for petname, introducer in introducers.items(): introducer_cache_filepath = FilePath(config.get_private_path("introducer_{}_cache.yaml".format(petname))) - ic = IntroducerClient( + ic = _introducer_factory( main_tub, introducer['furl'].encode("ascii"), config.nickname, @@ -370,8 +546,9 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con :param list introducer_clients: IntroducerClient instances if we're connecting to any """ - ps = config.get_config("client", "peers.preferred", "").split(",") - preferred_peers = tuple([p.strip() for p in ps if p != ""]) + storage_client_config = storage_client.StorageClientConfig.from_node_config( + config, + ) def tub_creator(handler_overrides=None, **kwargs): return node.create_tub( @@ -401,9 +578,10 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, + node_config=config, + storage_client_config=storage_client_config, preferred_peers=preferred_peers, grid_manager_keys=grid_manager_keys, -## node_pubkey=my_pubkey, ) for ic in introducer_clients: sb.use_introducer(ic) @@ -438,6 +616,89 @@ def _load_grid_manager_certificates(config): return grid_manager_certificates +def _register_reference(key, config, tub, referenceable): + """ + Register a referenceable in a tub with a stable fURL. + + Stability is achieved by storing the fURL in the configuration the first + time and then reading it back on for future calls. + + :param bytes key: An identifier for this reference which can be used to + identify its fURL in the configuration. + + :param _Config config: The configuration to use for fURL persistence. + + :param Tub tub: The tub in which to register the reference. + + :param Referenceable referenceable: The referenceable to register in the + Tub. + + :return bytes: The fURL at which the object is registered. + """ + persisted_furl = config.get_private_config( + key, + default=None, + ) + name = None + if persisted_furl is not None: + _, _, name = decode_furl(persisted_furl) + registered_furl = tub.registerReference( + referenceable, + name=name, + ) + if persisted_furl is None: + config.write_private_config(key, registered_furl) + return registered_furl + + +@implementer(IAnnounceableStorageServer) +@attr.s +class AnnounceableStorageServer(object): + announcement = attr.ib() + storage_server = attr.ib() + + + +def _add_to_announcement(information, announceable_storage_server): + """ + Create a new ``AnnounceableStorageServer`` based on + ``announceable_storage_server`` with ``information`` added to its + ``announcement``. + """ + updated_announcement = announceable_storage_server.announcement.copy() + updated_announcement.update(information) + return AnnounceableStorageServer( + updated_announcement, + announceable_storage_server.storage_server, + ) + + +def storage_enabled(config): + """ + Is storage enabled according to the given configuration object? + + :param _Config config: The configuration to inspect. + + :return bool: ``True`` if storage is enabled, ``False`` otherwise. + """ + return config.get_config(b"storage", b"enabled", True, boolean=True) + + +def anonymous_storage_enabled(config): + """ + Is anonymous access to storage enabled according to the given + configuration object? + + :param _Config config: The configuration to inspect. + + :return bool: ``True`` if storage is enabled, ``False`` otherwise. + """ + return ( + storage_enabled(config) and + config.get_config(b"storage", b"anonymous", True, boolean=True) + ) + + @implementer(IStatsProducer) class _Client(node.Node, pollmixin.PollMixin): @@ -467,7 +728,6 @@ class _Client(node.Node, pollmixin.PollMixin): """ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - self._magic_folders = dict() self.started_timestamp = time.time() self.logSource = "Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() @@ -478,7 +738,6 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_stats_provider() self.init_secrets() self.init_node_key() - self.init_storage() self.init_control() self._key_generator = KeyGenerator() key_gen_furl = config.get_config("client", "key_generator.furl", None) @@ -494,7 +753,6 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_helper() self.init_ftp_server() self.init_sftp_server() - self.init_magic_folder() # If the node sees an exit_trigger file, it will poll every second to see # whether the file still exists, and what its mtime is. If the file does not @@ -533,21 +791,30 @@ class _Client(node.Node, pollmixin.PollMixin): # we only create the key once. On all subsequent runs, we re-use the # existing key def _make_key(): - sk_vs,vk_vs = keyutil.make_keypair() - return sk_vs+"\n" - sk_vs = self.config.get_or_create_private_config("node.privkey", _make_key) - sk,vk_vs = keyutil.parse_privkey(sk_vs.strip()) - self.config.write_config_file("node.pubkey", vk_vs+"\n") - self._node_key = sk + private_key, _ = ed25519.create_signing_keypair() + return ed25519.string_from_signing_key(private_key) + "\n" + + private_key_str = self.config.get_or_create_private_config("node.privkey", _make_key) + private_key, public_key = ed25519.signing_keypair_from_string(private_key_str) + public_key_str = ed25519.string_from_verifying_key(public_key) + self.config.write_config_file("node.pubkey", public_key_str + "\n") + self._node_private_key = private_key + self._node_public_key = public_key def get_long_nodeid(self): # this matches what IServer.get_longname() says about us elsewhere - vk_bytes = self._node_key.get_verifying_key_bytes() - return "v0-"+base32.b2a(vk_bytes) + vk_string = ed25519.string_from_verifying_key(self._node_public_key) + return remove_prefix(vk_string, "pub-") def get_long_tubid(self): return idlib.nodeid_b2a(self.nodeid) + def get_web_service(self): + """ + :return: a reference to our web server + """ + return self.getServiceNamed("webish") + def _init_permutation_seed(self, ss): seed = self.config.get_config_from_file("permutation-seed") if not seed: @@ -564,18 +831,30 @@ class _Client(node.Node, pollmixin.PollMixin): else: # otherwise, we're free to use the more natural seed of our # pubkey-based serverid - vk_bytes = self._node_key.get_verifying_key_bytes() + vk_string = ed25519.string_from_verifying_key(self._node_public_key) + vk_bytes = remove_prefix(vk_string, ed25519.PUBLIC_KEY_PREFIX) seed = base32.b2a(vk_bytes) self.config.write_config_file("permutation-seed", seed+"\n") return seed.strip() - def init_storage(self): - # should we run a storage server (and publish it for others to use)? - if not self.config.get_config("storage", "enabled", True, boolean=True): - return - if not self._is_tub_listening(): - raise ValueError("config error: storage is enabled, but tub " - "is not listening ('tub.port=' is empty)") + def get_anonymous_storage_server(self): + """ + Get the anonymous ``IStorageServer`` implementation for this node. + + Note this will return an object even if storage is disabled on this + node (but the object will not be exposed, peers will not be able to + access it, and storage will remain disabled). + + The one and only instance for this node is always returned. It is + created first if necessary. + """ + try: + ss = self.getServiceNamed(StorageServer.name) + except KeyError: + pass + else: + return ss + readonly = self.config.get_config("storage", "readonly", False, boolean=True) config_storedir = self.get_config( @@ -630,6 +909,40 @@ class _Client(node.Node, pollmixin.PollMixin): expiration_sharetypes=expiration_sharetypes, ) ss.setServiceParent(self) + return ss + + def init_storage(self, announceable_storage_servers): + # should we run a storage server (and publish it for others to use)? + if not storage_enabled(self.config): + return + if not self._is_tub_listening(): + raise ValueError("config error: storage is enabled, but tub " + "is not listening ('tub.port=' is empty)") + + ss = self.get_anonymous_storage_server() + announcement = { + "permutation-seed-base32": self._init_permutation_seed(ss), + } + + if anonymous_storage_enabled(self.config): + furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) + furl = self.tub.registerReference(ss, furlFile=furl_file) + announcement["anonymous-storage-FURL"] = furl + + enabled_storage_servers = self._enable_storage_servers( + announceable_storage_servers, + ) + storage_options = list( + storage_server.announcement + for storage_server + in enabled_storage_servers + ) + plugins_announcement = {} + if storage_options: + # Only add the new key if there are any plugins enabled. + plugins_announcement[u"storage-options"] = storage_options + + announcement.update(plugins_announcement) grid_manager_certificates = [] @@ -650,7 +963,45 @@ class _Client(node.Node, pollmixin.PollMixin): "grid-manager-certificates": grid_manager_certificates, } for ic in self.introducer_clients: - ic.publish("storage", ann, self._node_key) + ic.publish("storage", announcement, self._node_private_key) + + def get_client_storage_plugin_web_resources(self): + """ + Get all of the client-side ``IResource`` implementations provided by + enabled storage plugins. + + :return dict[bytes, IResource provider]: The implementations. + """ + return self.storage_broker.get_client_storage_plugin_web_resources( + self.config, + ) + + def _enable_storage_servers(self, announceable_storage_servers): + """ + Register and announce the given storage servers. + """ + for announceable in announceable_storage_servers: + yield self._enable_storage_server(announceable) + + def _enable_storage_server(self, announceable_storage_server): + """ + Register a storage server. + """ + config_key = b"storage-plugin.{}.furl".format( + # Oops, why don't I have a better handle on this value? + announceable_storage_server.announcement[u"name"], + ) + furl = _register_reference( + config_key, + self.config, + self.tub, + announceable_storage_server.storage_server, + ) + announceable_storage_server = _add_to_announcement( + {u"storage-server-FURL": furl}, + announceable_storage_server, + ) + return announceable_storage_server def init_client(self): helper_furl = self.config.get_config("client", "helper.furl", None) @@ -682,9 +1033,6 @@ class _Client(node.Node, pollmixin.PollMixin): This returns a local authentication token, which is just some random data in "api_auth_token" which must be echoed to API calls. - - Currently only the URI '/magic' for magic-folder status; other - endpoints are invited to include this as well, as appropriate. """ return self.config.get_private_config('api_auth_token') @@ -802,40 +1150,6 @@ class _Client(node.Node, pollmixin.PollMixin): sftp_portstr, pubkey_file, privkey_file) s.setServiceParent(self) - def init_magic_folder(self): - #print "init_magic_folder" - if self.config.get_config("drop_upload", "enabled", False, boolean=True): - raise node.OldConfigOptionError( - "The [drop_upload] section must be renamed to [magic_folder].\n" - "See docs/frontends/magic-folder.rst for more information." - ) - - if self.config.get_config("magic_folder", "enabled", False, boolean=True): - from allmydata.frontends import magic_folder - - try: - magic_folders = magic_folder.load_magic_folders(self.config._basedir) - except Exception as e: - log.msg("Error loading magic-folder config: {}".format(e)) - raise - - # start processing the upload queue when we've connected to - # enough servers - threshold = min(self.encoding_params["k"], - self.encoding_params["happy"] + 1) - - for (name, mf_config) in magic_folders.items(): - self.log("Starting magic_folder '{}'".format(name)) - s = magic_folder.MagicFolder.from_config(self, name, mf_config) - self._magic_folders[name] = s - s.setServiceParent(self) - - connected_d = self.storage_broker.when_connected_enough(threshold) - def connected_enough(ign, mf): - mf.ready() # returns a Deferred we ignore - return None - connected_d.addCallback(connected_enough, s) - def _check_exit_trigger(self, exit_trigger_file): if os.path.exists(exit_trigger_file): mtime = os.stat(exit_trigger_file)[stat.ST_MTIME] diff --git a/src/allmydata/control.py b/src/allmydata/control.py index 33bbe22e6..07802efba 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -38,7 +38,7 @@ def log_memory_usage(where=""): where)) @implementer(IConsumer) -class FileWritingConsumer: +class FileWritingConsumer(object): def __init__(self, filename): self.done = False self.f = open(filename, "wb") @@ -123,9 +123,9 @@ class ControlServer(Referenceable, service.Service): return results server = everyone_left.pop(0) server_name = server.get_longname() - connection = server.get_rref() + storage_server = server.get_storage_server() start = time.time() - d = connection.callRemote("get_buckets", "\x00"*16) + d = storage_server.get_buckets("\x00" * 16) def _done(ignored): stop = time.time() elapsed = stop - start @@ -143,7 +143,7 @@ class ControlServer(Referenceable, service.Service): d.addCallback(_average) return d -class SpeedTest: +class SpeedTest(object): def __init__(self, parent, count, size, mutable): self.parent = parent self.count = count diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py new file mode 100644 index 000000000..ee92f223a --- /dev/null +++ b/src/allmydata/crypto/__init__.py @@ -0,0 +1,8 @@ +""" +Helper functions for cryptography-related operations inside Tahoe + +For the most part, these functions use and return objects that are +documented in the `cryptography` library -- however, code inside Tahoe +should only use these functions and not rely on features of any +objects that `cryptography` documents. +""" diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py new file mode 100644 index 000000000..4194c63df --- /dev/null +++ b/src/allmydata/crypto/aes.py @@ -0,0 +1,180 @@ +""" +Helper functions for cryptograhpy-related operations inside Tahoe +using AES + +These functions use and return objects that are documented in the +`cryptography` library -- however, code inside Tahoe should only use +functions from allmydata.crypto.aes and not rely on features of any +objects that `cryptography` documents. +""" + +import six + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import ( + Cipher, + algorithms, + modes, + CipherContext, +) +from zope.interface import ( + Interface, + directlyProvides, +) + + +DEFAULT_IV = b'\x00' * 16 + + +class IEncryptor(Interface): + """ + An object which can encrypt data. + + Create one using :func:`create_encryptor` and use it with + :func:`encrypt_data` + """ + + +class IDecryptor(Interface): + """ + An object which can decrypt data. + + Create one using :func:`create_decryptor` and use it with + :func:`decrypt_data` + """ + + +def create_encryptor(key, iv=None): + """ + Create and return a new object which can do AES encryptions with + the given key and initialization vector (IV). The default IV is 16 + zero-bytes. + + :param bytes key: the key bytes, should be 128 or 256 bits (16 or + 32 bytes) + + :param bytes iv: the Initialization Vector consisting of 16 bytes, + or None for the default (which is 16 zero bytes) + + :returns: an object suitable for use with :func:`encrypt_data` (an + :class:`IEncryptor`) + """ + cryptor = _create_cryptor(key, iv) + directlyProvides(cryptor, IEncryptor) + return cryptor + + +def encrypt_data(encryptor, plaintext): + """ + AES-encrypt `plaintext` with the given `encryptor`. + + :param encryptor: an instance of :class:`IEncryptor` previously + returned from `create_encryptor` + + :param bytes plaintext: the data to encrypt + + :returns: bytes of ciphertext + """ + + _validate_cryptor(encryptor, encrypt=True) + if not isinstance(plaintext, six.binary_type): + raise ValueError('Plaintext must be bytes') + + return encryptor.update(plaintext) + + +def create_decryptor(key, iv=None): + """ + Create and return a new object which can do AES decryptions with + the given key and initialization vector (IV). The default IV is 16 + zero-bytes. + + :param bytes key: the key bytes, should be 128 or 256 bits (16 or + 32 bytes) + + :param bytes iv: the Initialization Vector consisting of 16 bytes, + or None for the default (which is 16 zero bytes) + + :returns: an object suitable for use with :func:`decrypt_data` (an + :class:`IDecryptor` instance) + """ + cryptor = _create_cryptor(key, iv) + directlyProvides(cryptor, IDecryptor) + return cryptor + + +def decrypt_data(decryptor, plaintext): + """ + AES-decrypt `plaintext` with the given `decryptor`. + + :param decryptor: an instance of :class:`IDecryptor` previously + returned from `create_decryptor` + + :param bytes plaintext: the data to decrypt + + :returns: bytes of ciphertext + """ + + _validate_cryptor(decryptor, encrypt=False) + if not isinstance(plaintext, six.binary_type): + raise ValueError('Plaintext must be bytes') + + return decryptor.update(plaintext) + + +def _create_cryptor(key, iv): + """ + Internal helper. + + See :func:`create_encryptor` or :func:`create_decryptor`. + """ + key = _validate_key(key) + iv = _validate_iv(iv) + cipher = Cipher( + algorithms.AES(key), + modes.CTR(iv), + backend=default_backend() + ) + return cipher.encryptor() + + +def _validate_cryptor(cryptor, encrypt=True): + """ + raise ValueError if `cryptor` is not a valid object + """ + klass = IEncryptor if encrypt else IDecryptor + name = "encryptor" if encrypt else "decryptor" + if not isinstance(cryptor, CipherContext): + raise ValueError( + "'{}' must be a CipherContext".format(name) + ) + if not klass.providedBy(cryptor): + raise ValueError( + "'{}' must be created with create_{}()".format(name, name) + ) + + +def _validate_key(key): + """ + confirm `key` is suitable for AES encryption, or raise ValueError + """ + if not isinstance(key, six.binary_type): + raise TypeError('Key must be bytes') + if len(key) not in (16, 32): + raise ValueError('Key must be 16 or 32 bytes long') + return key + + +def _validate_iv(iv): + """ + Returns a suitable initialiation vector. If `iv` is `None`, a + default is returned. If `iv` is not a suitable initialization + vector an error is raised. `iv` is returned if it valid. + """ + if iv is None: + return DEFAULT_IV + if not isinstance(iv, six.binary_type): + raise TypeError('IV must be bytes') + if len(iv) != 16: + raise ValueError('IV must be 16 bytes long') + return iv diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py new file mode 100644 index 000000000..37e305d19 --- /dev/null +++ b/src/allmydata/crypto/ed25519.py @@ -0,0 +1,190 @@ +''' +Ed25519 keys and helpers. + +Key Formatting +-------------- + +- in base32, keys are 52 chars long (both signing and verifying keys) +- in base62, keys is 43 chars long +- in base64, keys is 43 chars long + +We can't use base64 because we want to reserve punctuation and preserve +cut-and-pasteability. The base62 encoding is shorter than the base32 form, +but the minor usability improvement is not worth the documentation and +specification confusion of using a non-standard encoding. So we stick with +base32. +''' + +import six + +from cryptography.exceptions import ( + InvalidSignature, +) +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + Ed25519PublicKey, +) +from cryptography.hazmat.primitives.serialization import ( + Encoding, + PrivateFormat, + NoEncryption, + PublicFormat, +) + +from allmydata.crypto.util import remove_prefix +from allmydata.crypto.error import BadSignature + +from allmydata.util.base32 import ( + a2b, + b2a, +) + +PRIVATE_KEY_PREFIX = b'priv-v0-' +PUBLIC_KEY_PREFIX = b'pub-v0-' + + +def create_signing_keypair(): + """ + Creates a new ed25519 keypair. + + :returns: 2-tuple of (private_key, public_key) + """ + private_key = Ed25519PrivateKey.generate() + return private_key, private_key.public_key() + + +def verifying_key_from_signing_key(private_key): + """ + :returns: the public key associated to the given `private_key` + """ + _validate_private_key(private_key) + return private_key.public_key() + + +def sign_data(private_key, data): + """ + Sign the given data using the given private key + + :param private_key: the private part returned from + `create_signing_keypair` or from + `signing_keypair_from_string` + + :param bytes data: the data to sign + + :returns: bytes representing the signature + """ + + _validate_private_key(private_key) + if not isinstance(data, six.binary_type): + raise ValueError('data must be bytes') + return private_key.sign(data) + + +def string_from_signing_key(private_key): + """ + Encode a private key to a string of bytes + + :param private_key: the private part returned from + `create_signing_keypair` or from + `signing_keypair_from_string` + + :returns: byte-string representing this key + """ + _validate_private_key(private_key) + raw_key_bytes = private_key.private_bytes( + Encoding.Raw, + PrivateFormat.Raw, + NoEncryption(), + ) + return PRIVATE_KEY_PREFIX + b2a(raw_key_bytes) + + +def signing_keypair_from_string(private_key_bytes): + """ + Load a signing keypair from a string of bytes (which includes the + PRIVATE_KEY_PREFIX) + + :returns: a 2-tuple of (private_key, public_key) + """ + + if not isinstance(private_key_bytes, six.binary_type): + raise ValueError('private_key_bytes must be bytes') + + private_key = Ed25519PrivateKey.from_private_bytes( + a2b(remove_prefix(private_key_bytes, PRIVATE_KEY_PREFIX)) + ) + return private_key, private_key.public_key() + + +def verify_signature(public_key, alleged_signature, data): + """ + :param public_key: a verifying key + + :param bytes alleged_signature: the bytes of the alleged signature + + :param bytes data: the data which was allegedly signed + + :raises: BadSignature if the signature is bad + :returns: None (or raises an exception). + """ + + if not isinstance(alleged_signature, six.binary_type): + raise ValueError('alleged_signature must be bytes') + + if not isinstance(data, six.binary_type): + raise ValueError('data must be bytes') + + _validate_public_key(public_key) + try: + public_key.verify(alleged_signature, data) + except InvalidSignature: + raise BadSignature() + + +def verifying_key_from_string(public_key_bytes): + """ + Load a verifying key from a string of bytes (which includes the + PUBLIC_KEY_PREFIX) + + :returns: a public_key + """ + if not isinstance(public_key_bytes, six.binary_type): + raise ValueError('public_key_bytes must be bytes') + + return Ed25519PublicKey.from_public_bytes( + a2b(remove_prefix(public_key_bytes, PUBLIC_KEY_PREFIX)) + ) + + +def string_from_verifying_key(public_key): + """ + Encode a public key to a string of bytes + + :param public_key: the public part of a keypair + + :returns: byte-string representing this key + """ + _validate_public_key(public_key) + raw_key_bytes = public_key.public_bytes( + Encoding.Raw, + PublicFormat.Raw, + ) + return PUBLIC_KEY_PREFIX + b2a(raw_key_bytes) + + +def _validate_public_key(public_key): + """ + Internal helper. Verify that `public_key` is an appropriate object + """ + if not isinstance(public_key, Ed25519PublicKey): + raise ValueError('public_key must be an Ed25519PublicKey') + return None + + +def _validate_private_key(private_key): + """ + Internal helper. Verify that `private_key` is an appropriate object + """ + if not isinstance(private_key, Ed25519PrivateKey): + raise ValueError('private_key must be an Ed25519PrivateKey') + return None diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py new file mode 100644 index 000000000..62c0b3e5b --- /dev/null +++ b/src/allmydata/crypto/error.py @@ -0,0 +1,15 @@ +""" +Exceptions raise by allmydata.crypto.* modules +""" + + +class BadSignature(Exception): + """ + An alleged signature did not match + """ + + +class BadPrefixError(Exception): + """ + A key did not start with the required prefix + """ diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py new file mode 100644 index 000000000..74f564ded --- /dev/null +++ b/src/allmydata/crypto/rsa.py @@ -0,0 +1,188 @@ +""" +Helper functions for cryptography-related operations inside Tahoe +using RSA public-key encryption and decryption. + +In cases where these functions happen to use and return objects that +are documented in the `cryptography` library, code outside this module +should only use functions from allmydata.crypto.rsa and not rely on +features of any objects that `cryptography` documents. + +That is, the public and private keys are opaque objects; DO NOT depend +on any of their methods. +""" + + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa, padding +from cryptography.hazmat.primitives.serialization import load_der_private_key, load_der_public_key, \ + Encoding, PrivateFormat, PublicFormat, NoEncryption + +from allmydata.crypto.error import BadSignature + + +# This is the value that was used by `pycryptopp`, and we must continue to use it for +# both backwards compatibility and interoperability. +# +# The docs for `cryptography` suggest to use the constant defined at +# `cryptography.hazmat.primitives.asymmetric.padding.PSS.MAX_LENGTH`, but this causes old +# signatures to fail to validate. +RSA_PSS_SALT_LENGTH = 32 + +RSA_PADDING = padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), + salt_length=RSA_PSS_SALT_LENGTH, +) + + + +def create_signing_keypair(key_size): + """ + Create a new RSA signing (private) keypair from scratch. Can be used with + `sign_data` function. + + :param int key_size: length of key in bits + + :returns: 2-tuple of (private_key, public_key) + """ + # Tahoe's original use of pycryptopp would use cryptopp's default + # public_exponent, which is 17 + # + # Thus, we are using 17 here as well. However, there are other + # choices; see this for more discussion: + # https://security.stackexchange.com/questions/2335/should-rsa-public-exponent-be-only-in-3-5-17-257-or-65537-due-to-security-c + # + # Another popular choice is 65537. See: + # https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#cryptography.hazmat.primitives.asymmetric.rsa.generate_private_key + # https://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html + priv_key = rsa.generate_private_key( + public_exponent=17, + key_size=key_size, + backend=default_backend() + ) + return priv_key, priv_key.public_key() + + +def create_signing_keypair_from_string(private_key_der): + """ + Create an RSA signing (private) key from previously serialized + private key bytes. + + :param bytes private_key_der: blob as returned from `der_string_from_signing_keypair` + + :returns: 2-tuple of (private_key, public_key) + """ + priv_key = load_der_private_key( + private_key_der, + password=None, + backend=default_backend(), + ) + return priv_key, priv_key.public_key() + + +def der_string_from_signing_key(private_key): + """ + Serializes a given RSA private key to a DER string + + :param private_key: a private key object as returned from + `create_signing_keypair` or `create_signing_keypair_from_string` + + :returns: bytes representing `private_key` + """ + _validate_private_key(private_key) + return private_key.private_bytes( + encoding=Encoding.DER, + format=PrivateFormat.PKCS8, + encryption_algorithm=NoEncryption(), + ) + + +def der_string_from_verifying_key(public_key): + """ + Serializes a given RSA public key to a DER string. + + :param public_key: a public key object as returned from + `create_signing_keypair` or `create_signing_keypair_from_string` + + :returns: bytes representing `public_key` + """ + _validate_public_key(public_key) + return public_key.public_bytes( + encoding=Encoding.DER, + format=PublicFormat.SubjectPublicKeyInfo, + ) + + +def create_verifying_key_from_string(public_key_der): + """ + Create an RSA verifying key from a previously serialized public key + + :param bytes public_key_der: a blob as returned by `der_string_from_verifying_key` + + :returns: a public key object suitable for use with other + functions in this module + """ + pub_key = load_der_public_key( + public_key_der, + backend=default_backend(), + ) + return pub_key + + +def sign_data(private_key, data): + """ + :param private_key: the private part of a keypair returned from + `create_signing_keypair_from_string` or `create_signing_keypair` + + :param bytes data: the bytes to sign + + :returns: bytes which are a signature of the bytes given as `data`. + """ + _validate_private_key(private_key) + return private_key.sign( + data, + RSA_PADDING, + hashes.SHA256(), + ) + +def verify_signature(public_key, alleged_signature, data): + """ + :param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key` + + :param bytes alleged_signature: the bytes of the alleged signature + + :param bytes data: the data which was allegedly signed + """ + _validate_public_key(public_key) + try: + public_key.verify( + alleged_signature, + data, + RSA_PADDING, + hashes.SHA256(), + ) + except InvalidSignature: + raise BadSignature() + + +def _validate_public_key(public_key): + """ + Internal helper. Checks that `public_key` is a valid cryptography + object + """ + if not isinstance(public_key, rsa.RSAPublicKey): + raise ValueError( + "public_key must be an RSAPublicKey" + ) + + +def _validate_private_key(private_key): + """ + Internal helper. Checks that `public_key` is a valid cryptography + object + """ + if not isinstance(private_key, rsa.RSAPrivateKey): + raise ValueError( + "private_key must be an RSAPrivateKey" + ) diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py new file mode 100644 index 000000000..6aa1f0973 --- /dev/null +++ b/src/allmydata/crypto/util.py @@ -0,0 +1,24 @@ +""" +Utilities used by allmydata.crypto modules +""" + +from allmydata.crypto.error import BadPrefixError + + +def remove_prefix(s_bytes, prefix): + """ + :param bytes s_bytes: a string of bytes whose prefix is removed + + :param bytes prefix: the bytes to remove from the beginning of `s_bytes` + + Removes `prefix` from `s_bytes` and returns the new bytes or + raises `BadPrefixError` if `s_bytes` did not start with the + `prefix` specified. + + :returns: `s_bytes` with `prefix` removed from the front. + """ + if s_bytes.startswith(prefix): + return s_bytes[len(prefix):] + raise BadPrefixError( + "did not see expected '{}' prefix".format(prefix) + ) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 7f90318d3..f1c95697b 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -6,6 +6,7 @@ from twisted.internet import defer from foolscap.api import fireEventually import json +from allmydata.crypto import aes from allmydata.deep_stats import DeepStats from allmydata.mutable.common import NotWriteableError from allmydata.mutable.filenode import MutableFileNode @@ -22,7 +23,6 @@ from allmydata.util.assertutil import precondition from allmydata.util.netstring import netstring, split_netstring from allmydata.util.consumer import download_to_data from allmydata.uri import wrap_dirnode_cap -from pycryptopp.cipher.aes import AES from allmydata.util.dictutil import AuxValueDict from eliot import ( @@ -111,7 +111,7 @@ def normalize(namex): # contents and end by repacking them. It might be better to apply them to # the unpacked contents. -class Deleter: +class Deleter(object): def __init__(self, node, namex, must_exist=True, must_be_directory=False, must_be_file=False): self.node = node self.name = normalize(namex) @@ -139,7 +139,7 @@ class Deleter: return new_contents -class MetadataSetter: +class MetadataSetter(object): def __init__(self, node, namex, metadata, create_readonly_node=None): self.node = node self.name = normalize(namex) @@ -164,7 +164,7 @@ class MetadataSetter: return new_contents -class Adder: +class Adder(object): def __init__(self, node, entries=None, overwrite=True, create_readonly_node=None): self.node = node if entries is None: @@ -214,8 +214,8 @@ def _encrypt_rw_uri(writekey, rw_uri): salt = hashutil.mutable_rwcap_salt_hash(rw_uri) key = hashutil.mutable_rwcap_key_hash(salt, writekey) - cryptor = AES(key) - crypttext = cryptor.process(rw_uri) + encryptor = aes.create_encryptor(key) + crypttext = aes.encrypt_data(encryptor, rw_uri) mac = hashutil.hmac(key, salt + crypttext) assert len(mac) == 32 return salt + crypttext + mac @@ -331,8 +331,8 @@ class DirectoryNode(object): salt = encwrcap[:16] crypttext = encwrcap[16:-32] key = hashutil.mutable_rwcap_key_hash(salt, self._node.get_writekey()) - cryptor = AES(key) - plaintext = cryptor.process(crypttext) + encryptor = aes.create_decryptor(key) + plaintext = aes.decrypt_data(encryptor, crypttext) return plaintext def _create_and_validate_node(self, rw_uri, ro_uri, name): @@ -861,7 +861,7 @@ class ManifestWalker(DeepStats): } -class DeepChecker: +class DeepChecker(object): def __init__(self, root, verify, repair, add_lease): root_si = root.get_storage_index() if root_si: diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 2c72a020e..49647bc60 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -16,7 +16,7 @@ class NeedRootcapLookupScheme(Exception): mechanism to translate name+passwd pairs into a rootcap, either a file of name/passwd/rootcap tuples, or a server to do the translation.""" -class FTPAvatarID: +class FTPAvatarID(object): def __init__(self, username, rootcap): self.username = username self.rootcap = rootcap diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py deleted file mode 100644 index bb2edce99..000000000 --- a/src/allmydata/frontends/magic_folder.py +++ /dev/null @@ -1,2111 +0,0 @@ -import six -import sys, os -import os.path -from errno import EEXIST -from collections import deque -from datetime import datetime -import time -import ConfigParser - -from twisted.python.log import msg as twmsg -from twisted.python.filepath import FilePath -from twisted.python.monkey import MonkeyPatcher -from twisted.internet import defer, reactor, task -from twisted.internet.error import AlreadyCancelled -from twisted.python.failure import Failure -from twisted.python import runtime -from twisted.application import service - -from zope.interface import Interface, Attribute, implementer - -from eliot import ( - Field, - Message, - start_action, - ActionType, - MessageType, - write_failure, - write_traceback, - log_call, -) -from eliot.twisted import ( - DeferredContext, -) - -from allmydata.util import ( - fileutil, - configutil, - yamlutil, - eliotutil, -) -from allmydata.interfaces import IDirectoryNode -from allmydata.util import log -from allmydata.util.fileutil import ( - precondition_abspath, - get_pathinfo, - ConflictError, - abspath_expanduser_unicode, -) -from allmydata.util.assertutil import precondition, _assert -from allmydata.util.deferredutil import HookMixin -from allmydata.util.progress import PercentProgress -from allmydata.util.encodingutil import listdir_filepath, to_filepath, \ - extend_filepath, unicode_from_filepath, unicode_segments_from, \ - quote_filepath, quote_local_unicode_path, FilenameEncodingError -from allmydata.util.time_format import format_time -from allmydata.immutable.upload import FileName, Data -from allmydata import magicfolderdb, magicpath - -if six.PY3: - long = int - - -# Mask off all non-owner permissions for magic-folders files by default. -_DEFAULT_DOWNLOAD_UMASK = 0o077 - -IN_EXCL_UNLINK = long(0x04000000) - - -class ConfigurationError(Exception): - """ - There was something wrong with some magic-folder configuration. - """ - - -def _get_inotify_module(): - try: - if sys.platform == "win32": - from allmydata.windows import inotify - elif runtime.platform.supportsINotify(): - from twisted.internet import inotify - elif not sys.platform.startswith("linux"): - from allmydata.watchdog import inotify - else: - raise NotImplementedError("filesystem notification needed for Magic Folder is not supported.\n" - "This currently requires Linux, Windows, or macOS.") - return inotify - except (ImportError, AttributeError) as e: - log.msg(e) - if sys.platform == "win32": - raise NotImplementedError("filesystem notification needed for Magic Folder is not supported.\n" - "Windows support requires at least Vista, and has only been tested on Windows 7.") - raise - - -def get_inotify_module(): - # Until Twisted #9579 is fixed, the Docker check just screws things up. - # Disable it. - monkey = MonkeyPatcher() - monkey.addPatch(runtime.platform, "isDocker", lambda: False) - return monkey.runWithPatches(_get_inotify_module) - - -def is_new_file(pathinfo, db_entry): - if db_entry is None: - return True - - if not pathinfo.exists and db_entry.size is None: - return False - - return ((pathinfo.size, pathinfo.ctime_ns, pathinfo.mtime_ns) != - (db_entry.size, db_entry.ctime_ns, db_entry.mtime_ns)) - - -def _upgrade_magic_folder_config(basedir): - """ - Helper that upgrades from single-magic-folder-only configs to - multiple magic-folder configuration style (in YAML) - """ - config_fname = os.path.join(basedir, "tahoe.cfg") - config = configutil.get_config(config_fname) - - collective_fname = os.path.join(basedir, "private", "collective_dircap") - upload_fname = os.path.join(basedir, "private", "magic_folder_dircap") - magic_folders = { - u"default": { - u"directory": config.get("magic_folder", "local.directory").decode("utf-8"), - u"collective_dircap": fileutil.read(collective_fname), - u"upload_dircap": fileutil.read(upload_fname), - u"poll_interval": int(config.get("magic_folder", "poll_interval")), - }, - } - fileutil.move_into_place( - source=os.path.join(basedir, "private", "magicfolderdb.sqlite"), - dest=os.path.join(basedir, "private", "magicfolder_default.sqlite"), - ) - save_magic_folders(basedir, magic_folders) - config.remove_option("magic_folder", "local.directory") - config.remove_option("magic_folder", "poll_interval") - configutil.write_config(os.path.join(basedir, 'tahoe.cfg'), config) - fileutil.remove_if_possible(collective_fname) - fileutil.remove_if_possible(upload_fname) - - -def maybe_upgrade_magic_folders(node_directory): - """ - If the given node directory is not already using the new-style - magic-folder config it will be upgraded to do so. (This should - only be done if the user is running a command that needs to modify - the config) - """ - yaml_fname = os.path.join(node_directory, u"private", u"magic_folders.yaml") - if os.path.exists(yaml_fname): - # we already have new-style magic folders - return - - config_fname = os.path.join(node_directory, "tahoe.cfg") - config = configutil.get_config(config_fname) - - # we have no YAML config; if we have config in tahoe.cfg then we - # can upgrade it to the YAML-based configuration - if config.has_option("magic_folder", "local.directory"): - _upgrade_magic_folder_config(node_directory) - - -def load_magic_folders(node_directory): - """ - Loads existing magic-folder configuration and returns it as a dict - mapping name -> dict of config. This will NOT upgrade from - old-style to new-style config (but WILL read old-style config and - return in the same way as if it was new-style). - - :param node_directory: path where node data is stored - :returns: dict mapping magic-folder-name to its config (also a dict) - """ - yaml_fname = os.path.join(node_directory, u"private", u"magic_folders.yaml") - folders = dict() - - config_fname = os.path.join(node_directory, "tahoe.cfg") - config = configutil.get_config(config_fname) - - if not os.path.exists(yaml_fname): - # there will still be a magic_folder section in a "new" - # config, but it won't have local.directory nor poll_interval - # in it. - if config.has_option("magic_folder", "local.directory"): - up_fname = os.path.join(node_directory, "private", "magic_folder_dircap") - coll_fname = os.path.join(node_directory, "private", "collective_dircap") - directory = config.get("magic_folder", "local.directory").decode('utf8') - try: - interval = int(config.get("magic_folder", "poll_interval")) - except ConfigParser.NoOptionError: - interval = 60 - - if config.has_option("magic_folder", "download.umask"): - umask = int(config.get("magic_folder", "download.umask"), 8) - else: - umask = _DEFAULT_DOWNLOAD_UMASK - - folders[u"default"] = { - u"directory": directory, - u"upload_dircap": fileutil.read(up_fname), - u"collective_dircap": fileutil.read(coll_fname), - u"poll_interval": interval, - u"umask": umask, - } - else: - # without any YAML file AND no local.directory option it's - # an error if magic-folder is "enabled" because we don't - # actually have enough config for any magic-folders at all - if config.has_section("magic_folder") \ - and config.getboolean("magic_folder", "enabled") \ - and not folders: - raise Exception( - "[magic_folder] is enabled but has no YAML file and no " - "'local.directory' option." - ) - - elif os.path.exists(yaml_fname): # yaml config-file exists - if config.has_option("magic_folder", "local.directory"): - raise Exception( - "magic-folder config has both old-style configuration" - " and new-style configuration; please remove the " - "'local.directory' key from tahoe.cfg or remove " - "'magic_folders.yaml' from {}".format(node_directory) - ) - with open(yaml_fname, "r") as f: - magic_folders = yamlutil.safe_load(f.read()) - if not isinstance(magic_folders, dict): - raise Exception( - "'{}' should contain a dict".format(yaml_fname) - ) - - folders = magic_folders['magic-folders'] - if not isinstance(folders, dict): - raise Exception( - "'magic-folders' in '{}' should be a dict".format(yaml_fname) - ) - - # check configuration - folders = dict( - (name, fix_magic_folder_config(yaml_fname, name, config)) - for (name, config) - in folders.items() - ) - return folders - - -def fix_magic_folder_config(yaml_fname, name, config): - """ - Check the given folder configuration for validity. - - If it refers to a local directory which does not exist, create that - directory with the configured permissions. - - :param unicode yaml_fname: The configuration file from which the - configuration was read. - - :param unicode name: The name of the magic-folder this particular - configuration blob is associated with. - - :param config: The configuration for a single magic-folder. This is - expected to be a ``dict`` with certain keys and values of certain - types but these properties will be checked. - - :raise ConfigurationError: If the given configuration object does not - conform to some magic-folder configuration requirement. - """ - if not isinstance(config, dict): - raise ConfigurationError( - "Each item in '{}' must itself be a dict".format(yaml_fname) - ) - - for k in ['collective_dircap', 'upload_dircap', 'directory', 'poll_interval']: - if k not in config: - raise ConfigurationError( - "Config for magic folder '{}' is missing '{}'".format( - name, k - ) - ) - - if not isinstance( - config.setdefault(u"umask", _DEFAULT_DOWNLOAD_UMASK), - int, - ): - raise Exception("magic-folder download umask must be an integer") - - # make sure directory for magic folder exists - dir_fp = to_filepath(config['directory']) - umask = config.setdefault('umask', 0o077) - - try: - os.mkdir(dir_fp.path, 0o777 & (~ umask)) - except OSError as e: - if EEXIST != e.errno: - # Report some unknown problem. - raise ConfigurationError( - "magic-folder {} configured path {} could not be created: " - "{}".format( - name, - dir_fp.path, - str(e), - ), - ) - elif not dir_fp.isdir(): - # Tell the user there's a collision. - raise ConfigurationError( - "magic-folder {} configured path {} exists and is not a " - "directory".format( - name, dir_fp.path, - ), - ) - - result_config = config.copy() - for k in ['collective_dircap', 'upload_dircap']: - if isinstance(config[k], unicode): - result_config[k] = config[k].encode('ascii') - return result_config - - - -def save_magic_folders(node_directory, folders): - fileutil.write_atomically( - os.path.join(node_directory, u"private", u"magic_folders.yaml"), - yamlutil.safe_dump({u"magic-folders": folders}), - ) - - config = configutil.get_config(os.path.join(node_directory, u"tahoe.cfg")) - configutil.set_config(config, "magic_folder", "enabled", "True") - configutil.write_config(os.path.join(node_directory, u"tahoe.cfg"), config) - - -class MagicFolder(service.MultiService): - - @classmethod - def from_config(cls, client_node, name, config): - """ - Create a ``MagicFolder`` from a client node and magic-folder - configuration. - - :param _Client client_node: The client node the magic-folder is - attached to. - - :param dict config: Magic-folder configuration like that in the list - returned by ``load_magic_folders``. - """ - db_filename = client_node.config.get_private_path("magicfolder_{}.sqlite".format(name)) - local_dir_config = config['directory'] - try: - poll_interval = int(config["poll_interval"]) - except ValueError: - raise ValueError("'poll_interval' option must be an int") - - return cls( - client=client_node, - upload_dircap=config["upload_dircap"], - collective_dircap=config["collective_dircap"], - # XXX surely a better way for this local_path_u business - local_path_u=abspath_expanduser_unicode( - local_dir_config, - base=client_node.config.get_config_path(), - ), - dbfile=abspath_expanduser_unicode(db_filename), - umask=config["umask"], - name=name, - downloader_delay=poll_interval, - ) - - def __init__(self, client, upload_dircap, collective_dircap, local_path_u, dbfile, umask, - name, uploader_delay=1.0, clock=None, downloader_delay=60): - precondition_abspath(local_path_u) - if not os.path.exists(local_path_u): - raise ValueError("'{}' does not exist".format(local_path_u)) - if not os.path.isdir(local_path_u): - raise ValueError("'{}' is not a directory".format(local_path_u)) - # this is used by 'service' things and must be unique in this Service hierarchy - self.name = 'magic-folder-{}'.format(name) - - service.MultiService.__init__(self) - - clock = clock or reactor - db = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1)) - if db is None: - raise Exception('ERROR: Unable to load magic folder db.') - - # for tests - self._client = client - self._db = db - - upload_dirnode = self._client.create_node_from_uri(upload_dircap) - collective_dirnode = self._client.create_node_from_uri(collective_dircap) - - self.uploader = Uploader(client, local_path_u, db, upload_dirnode, uploader_delay, clock) - self.downloader = Downloader(client, local_path_u, db, collective_dirnode, - upload_dirnode.get_readonly_uri(), clock, self.uploader.is_pending, umask, - self.set_public_status, poll_interval=downloader_delay) - self._public_status = (False, ['Magic folder has not yet started']) - - def get_public_status(self): - """ - For the web UI, basically. - """ - return self._public_status - - def set_public_status(self, status, *messages): - self._public_status = (status, messages) - - def startService(self): - service.MultiService.startService(self) - return self.uploader.start_monitoring() - - def stopService(self): - with MAGIC_FOLDER_STOP(nickname=self.name).context(): - d = DeferredContext(self._finish()) - d.addBoth( - lambda ign: service.MultiService.stopService(self) - ) - return d.addActionFinish() - - def ready(self): - """ready is used to signal us to start - processing the upload and download items... - """ - self.uploader.start_uploading() # synchronous, returns None - return self.downloader.start_downloading() - - def _finish(self): - d0 = self.downloader.stop() - d1 = self.uploader.stop() - return defer.DeferredList(list( - DeferredContext(d).addErrback(write_failure).result - for d in [d0, d1] - )) - - -_NICKNAME = Field.for_types( - u"nickname", - [unicode, bytes], - u"A Magic-Folder participant nickname.", -) - -_DIRECTION = Field.for_types( - u"direction", - [unicode], - u"A synchronization direction: uploader or downloader.", - eliotutil.validateSetMembership({u"uploader", u"downloader"}), -) - -PROCESSING_LOOP = ActionType( - u"magic-folder:processing-loop", - [_NICKNAME, _DIRECTION], - [], - u"A Magic-Folder is processing uploads or downloads.", -) - -ITERATION = ActionType( - u"magic-folder:iteration", - [_NICKNAME, _DIRECTION], - [], - u"A step towards synchronization in one direction.", -) - -_COUNT = Field.for_types( - u"count", - [int, long], - u"The number of items in the processing queue.", -) - -PROCESS_QUEUE = ActionType( - u"magic-folder:process-queue", - [_COUNT], - [], - u"A Magic-Folder is working through an item queue.", -) - -SCAN_REMOTE_COLLECTIVE = ActionType( - u"magic-folder:scan-remote-collective", - [], - [], - u"The remote collective is being scanned for peer DMDs.", -) - -_DMDS = Field( - u"dmds", - # The children of the collective directory are the participant DMDs. The - # keys in this dict give us the aliases of the participants. - lambda collective_directory_listing: collective_directory_listing.keys(), - u"The (D)istributed (M)utable (D)irectories belonging to each participant are being scanned for changes.", -) - -COLLECTIVE_SCAN = MessageType( - u"magic-folder:downloader:get-latest-file:collective-scan", - [_DMDS], - u"Participants in the collective are being scanned.", -) - - -SCAN_REMOTE_DMD = ActionType( - u"magic-folder:scan-remote-dmd", - [_NICKNAME], - [], - u"A peer DMD is being scanned for changes.", -) - -REMOTE_VERSION = Field.for_types( - u"remote_version", - [int, long], - u"The version of a path found in a peer DMD.", -) - -REMOTE_URI = Field.for_types( - u"remote_uri", - [bytes], - u"The filecap of a path found in a peer DMD.", -) - -REMOTE_DMD_ENTRY = MessageType( - u"magic-folder:remote-dmd-entry", - [eliotutil.RELPATH, magicfolderdb.PATHENTRY, REMOTE_VERSION, REMOTE_URI], - u"A single entry found by scanning a peer DMD.", -) - -ADD_TO_DOWNLOAD_QUEUE = MessageType( - u"magic-folder:add-to-download-queue", - [eliotutil.RELPATH], - u"An entry was found to be changed and is being queued for download.", -) - -MAGIC_FOLDER_STOP = ActionType( - u"magic-folder:stop", - [_NICKNAME], - [], - u"A Magic-Folder is being stopped.", -) - -MAYBE_UPLOAD = MessageType( - u"magic-folder:maybe-upload", - [eliotutil.RELPATH], - u"A decision is being made about whether to upload a file.", -) - -PENDING = Field( - u"pending", - lambda s: list(s), - u"The paths which are pending processing.", - eliotutil.validateInstanceOf(set), -) - -REMOVE_FROM_PENDING = ActionType( - u"magic-folder:remove-from-pending", - [eliotutil.RELPATH, PENDING], - [], - u"An item being processed is being removed from the pending set.", -) - -PATH = Field( - u"path", - lambda fp: fp.asTextMode().path, - u"A local filesystem path.", - eliotutil.validateInstanceOf(FilePath), -) - -NOTIFIED_OBJECT_DISAPPEARED = MessageType( - u"magic-folder:notified-object-disappeared", - [PATH], - u"A path which generated a notification was not found on the filesystem. This is normal.", -) - -PROPAGATE_DIRECTORY_DELETION = ActionType( - u"magic-folder:propagate-directory-deletion", - [], - [], - u"Children of a deleted directory are being queued for upload processing.", -) - -NO_DATABASE_ENTRY = MessageType( - u"magic-folder:no-database-entry", - [], - u"There is no local database entry for a particular relative path in the magic folder.", -) - -NOT_UPLOADING = MessageType( - u"magic-folder:not-uploading", - [], - u"An item being processed is not going to be uploaded.", -) - -SYMLINK = MessageType( - u"magic-folder:symlink", - [PATH], - u"An item being processed was a symlink and is being skipped", -) - -CREATED_DIRECTORY = Field.for_types( - u"created_directory", - [unicode], - u"The relative path of a newly created directory in a magic-folder.", -) - -PROCESS_DIRECTORY = ActionType( - u"magic-folder:process-directory", - [], - [CREATED_DIRECTORY], - u"An item being processed was a directory.", -) - -DIRECTORY_PATHENTRY = MessageType( - u"magic-folder:directory-dbentry", - [magicfolderdb.PATHENTRY], - u"Local database state relating to an item possibly being uploaded.", -) - -NOT_NEW_DIRECTORY = MessageType( - u"magic-folder:not-new-directory", - [], - u"A directory item being processed was found to not be new.", -) - -NOT_NEW_FILE = MessageType( - u"magic-folder:not-new-file", - [], - u"A file item being processed was found to not be new (or changed).", -) - -SPECIAL_FILE = MessageType( - u"magic-folder:special-file", - [], - u"An item being processed was found to be of a special type which is not supported.", -) - -_COUNTER_NAME = Field.for_types( - u"counter_name", - # Should really only be unicode - [unicode, bytes], - u"The name of a counter.", -) - -_DELTA = Field.for_types( - u"delta", - [int, long], - u"An amount of a specific change in a counter.", -) - -_VALUE = Field.for_types( - u"value", - [int, long], - u"The new value of a counter after a change.", -) - -COUNT_CHANGED = MessageType( - u"magic-folder:count", - [_COUNTER_NAME, _DELTA, _VALUE], - u"The value of a counter has changed.", -) - -START_MONITORING = ActionType( - u"magic-folder:start-monitoring", - [_NICKNAME, _DIRECTION], - [], - u"Uploader is beginning to monitor the filesystem for uploadable changes.", -) - -STOP_MONITORING = ActionType( - u"magic-folder:stop-monitoring", - [_NICKNAME, _DIRECTION], - [], - u"Uploader is terminating filesystem monitoring operation.", -) - -START_UPLOADING = ActionType( - u"magic-folder:start-uploading", - [_NICKNAME, _DIRECTION], - [], - u"Uploader is performing startup-time inspection of known files.", -) - -_IGNORED = Field.for_types( - u"ignored", - [bool], - u"A file proposed for queueing for processing is instead being ignored by policy.", -) - -_ALREADY_PENDING = Field.for_types( - u"already_pending", - [bool], - u"A file proposed for queueing for processing is already in the queue.", -) - -_SIZE = Field.for_types( - u"size", - [int, long, type(None)], - u"The size of a file accepted into the processing queue.", -) - -ADD_PENDING = ActionType( - u"magic-folder:add-pending", - [eliotutil.RELPATH], - [_IGNORED, _ALREADY_PENDING, _SIZE], - u"Uploader is adding a path to the processing queue.", -) - -FULL_SCAN = ActionType( - u"magic-folder:full-scan", - [_NICKNAME, _DIRECTION], - [], - u"A complete brute-force scan of the local directory is being performed.", -) - -SCAN = ActionType( - u"magic-folder:scan", - [eliotutil.RELPATH], - [], - u"A brute-force scan of a subset of the local directory is being performed.", -) - -NOTIFIED = ActionType( - u"magic-folder:notified", - [PATH, _NICKNAME, _DIRECTION], - [], - u"Magic-Folder received a notification of a local filesystem change for a certain path.", -) - -_NON_DIR_CREATED = Field.for_types( - u"non_dir_created", - [bool], - u"A creation event was for a non-directory and requires no further inspection.", -) - - -REACT_TO_INOTIFY = ActionType( - u"magic-folder:react-to-inotify", - [eliotutil.INOTIFY_EVENTS], - [_IGNORED, _NON_DIR_CREATED, _ALREADY_PENDING], - u"Magic-Folder is processing a notification from inotify(7) (or a clone) about a filesystem event.", -) - -_ABSPATH = Field.for_types( - u"abspath", - [unicode], - u"The absolute path of a file being written in a local directory.", -) - -_IS_CONFLICT = Field.for_types( - u"is_conflict", - [bool], - u"An indication of whether a file being written in a local directory is in a conflicted state.", -) - -_NOW = Field.for_types( - u"now", - [int, long, float], - u"The time at which a file is being written in a local directory.", -) - -_MTIME = Field.for_types( - u"mtime", - [int, long, float, type(None)], - u"A modification time to put into the metadata of a file being written in a local directory.", -) - -WRITE_DOWNLOADED_FILE = ActionType( - u"magic-folder:write-downloaded-file", - [_ABSPATH, _SIZE, _IS_CONFLICT, _NOW, _MTIME], - [], - u"A downloaded file is being written to the filesystem.", -) - -ALREADY_GONE = MessageType( - u"magic-folder:rename:already-gone", - [], - u"A deleted file could not be rewritten to a backup path because it no longer exists.", -) - -_REASON = Field( - u"reason", - lambda e: str(e), - u"An exception which may describe the form of the conflict.", - eliotutil.validateInstanceOf(Exception), -) - -OVERWRITE_BECOMES_CONFLICT = MessageType( - u"magic-folder:overwrite-becomes-conflict", - [_REASON], - u"An attempt to overwrite an existing file failed because that file is now conflicted.", -) - -_FILES = Field( - u"files", - lambda file_set: list(file_set), - u"All of the relative paths belonging to a Magic-Folder that are locally known.", -) - -ALL_FILES = MessageType( - u"magic-folder:all-files", - [_FILES], - u"A record of the rough state of the local database at the time of downloader start up.", -) - -_ITEMS = Field( - u"items", - lambda deque: list(dict(relpath=item.relpath_u, kind=item.kind) for item in deque), - u"Items in a processing queue.", -) - -ITEM_QUEUE = MessageType( - u"magic-folder:item-queue", - [_ITEMS], - u"A report of the items in the processing queue at this point.", -) - -_BATCH = Field( - u"batch", - # Just report the paths for now. Perhaps something from the values would - # also be useful, though? Consider it. - lambda batch: batch.keys(), - u"A batch of scanned items.", - eliotutil.validateInstanceOf(dict), -) - -SCAN_BATCH = MessageType( - u"magic-folder:scan-batch", - [_BATCH], - u"Items in a batch of files which were scanned from the DMD.", -) - -START_DOWNLOADING = ActionType( - u"magic-folder:start-downloading", - [_NICKNAME, _DIRECTION], - [], - u"A Magic-Folder downloader is initializing and beginning to manage downloads.", -) - -PERFORM_SCAN = ActionType( - u"magic-folder:perform-scan", - [], - [], - u"Remote storage is being scanned for changes which need to be synchronized.", -) - -_STATUS = Field.for_types( - u"status", - # Should just be unicode... - [unicode, bytes], - u"The status of an item in a processing queue.", -) - -QUEUED_ITEM_STATUS_CHANGE = MessageType( - u"magic-folder:item:status-change", - [eliotutil.RELPATH, _STATUS], - u"A queued item changed status.", -) - -_CONFLICT_REASON = Field.for_types( - u"conflict_reason", - [unicode, type(None)], - u"A human-readable explanation of why a file was in conflict.", - eliotutil.validateSetMembership({ - u"dbentry mismatch metadata", - u"dbentry newer version", - u"last_downloaded_uri mismatch", - u"file appeared", - None, - }), -) - -CHECKING_CONFLICTS = ActionType( - u"magic-folder:item:checking-conflicts", - [], - [_IS_CONFLICT, _CONFLICT_REASON], - u"A potential download item is being checked to determine if it is in a conflicted state.", -) - -REMOTE_DIRECTORY_CREATED = MessageType( - u"magic-folder:remote-directory-created", - [], - u"The downloader found a new directory in the DMD.", -) - -REMOTE_DIRECTORY_DELETED = MessageType( - u"magic-folder:remote-directory-deleted", - [], - u"The downloader found a directory has been deleted from the DMD.", -) - -class QueueMixin(HookMixin): - """ - A parent class for Uploader and Downloader that handles putting - IQueuedItem instances into a work queue and processing - them. Tracks some history of recent items processed (for the - "status" API). - - Subclasses implement _scan_delay, _perform_scan and _process - - :ivar unicode _name: Either "uploader" or "downloader". - - :ivar _deque: IQueuedItem instances to process - - :ivar _process_history: the last 20 items we processed - - :ivar _in_progress: current batch of items which are currently - being processed; chunks of work are removed from _deque and - worked on. As each finishes, it is added to _process_history - (with oldest items falling off the end). - """ - - def __init__(self, client, local_path_u, db, name, clock): - self._client = client - self._local_path_u = local_path_u - self._local_filepath = to_filepath(local_path_u) - self._db = db - self._name = name - self._clock = clock - self._log_fields = dict( - nickname=self._client.nickname, - direction=self._name, - ) - self._hooks = { - 'processed': None, - 'started': None, - 'iteration': None, - 'inotify': None, - 'item_processed': None, - } - self.started_d = self.set_hook('started') - - # we should have gotten nice errors already while loading the - # config, but just to be safe: - assert self._local_filepath.exists() - assert self._local_filepath.isdir() - - self._deque = deque() - # do we also want to bound on "maximum age"? - self._process_history = deque(maxlen=20) - self._in_progress = [] - - def get_status(self): - """ - Returns an iterable of instances that implement IQueuedItem - """ - for item in self._deque: - yield item - for item in self._in_progress: - yield item - for item in self._process_history: - yield item - - def _get_filepath(self, relpath_u): - return extend_filepath(self._local_filepath, relpath_u.split(u"/")) - - def stop(self): - """ - Don't process queued items anymore. - - :return Deferred: A ``Deferred`` that fires when processing has - completely stopped. - """ - d = self._processing - self._processing_loop.stop() - self._processing = None - self._processing_loop = None - return d - - def _begin_processing(self): - """ - Start a loop that looks for work to do and then does it. - """ - action = PROCESSING_LOOP(**self._log_fields) - - # Note that we don't put the processing iterations into the logging - # action because we expect this loop to run for the whole lifetime of - # the process. The tooling for dealing with incomplete action trees - # is still somewhat lacking. Putting the iteractions into the overall - # loop action would hamper reading those logs for now. - self._processing_loop = task.LoopingCall(self._processing_iteration) - self._processing_loop.clock = self._clock - self._processing = self._processing_loop.start(self._scan_delay(), now=True) - - with action.context(): - # We do make sure errors appear in the loop action though. - d = DeferredContext(self._processing) - d.addActionFinish() - - def _processing_iteration(self): - """ - One iteration runs self._process_deque which calls _perform_scan() and - then completely drains the _deque (processing each item). - """ - action = ITERATION(**self._log_fields) - with action.context(): - d = DeferredContext(defer.Deferred()) - - # During startup we scanned the collective for items to download. - # If we found work to do, we do not need to perform another scan - # here. More importantly, the logic for determining which items - # to download is *not correct* in the case where two scans are - # performed with no intermediate emptying of the work queue. - # Therefore, skip the scan any time there is queued work. The - # only time we expect there to be any, though, is on the first - # time through this loop. - if not self._deque: - # adds items to our deque - d.addCallback(lambda ignored: self._perform_scan()) - - # process anything in our queue - d.addCallback(lambda ignored: self._process_deque()) - - # Let the tests know we've made it this far. - d.addCallback(lambda ignored: self._call_hook(None, 'iteration')) - - # Get it out of the Eliot context - result = d.addActionFinish() - - # Kick it off - result.callback(None) - - # Give it back to LoopingCall so it can wait on us. - return result - - def _scan_delay(self): - raise NotImplementedError - - def _perform_scan(self): - return - - @eliotutil.inline_callbacks - def _process_deque(self): - # process everything currently in the queue. we're turning it - # into a list so that if any new items get added while we're - # processing, they'll not run until next time) - to_process = list(self._deque) - self._deque.clear() - self._count('objects_queued', -len(to_process)) - - # we want to include all these in the next status request, so - # we must put them 'somewhere' before the next yield (and it's - # not in _process_history because that gets trimmed and we - # don't want anything to disappear until after it is - # completed) - self._in_progress.extend(to_process) - - with PROCESS_QUEUE(count=len(to_process)): - for item in to_process: - self._process_history.appendleft(item) - self._in_progress.remove(item) - try: - proc = yield self._process(item) - if not proc: - self._process_history.remove(item) - self._call_hook(item, 'item_processed') - except: - write_traceback() - item.set_status('failed', self._clock.seconds()) - proc = Failure() - - self._call_hook(proc, 'processed') - - def _get_relpath(self, filepath): - segments = unicode_segments_from(filepath, self._local_filepath) - return u"/".join(segments) - - def _count(self, counter_name, delta=1): - ctr = 'magic_folder.%s.%s' % (self._name, counter_name) - self._client.stats_provider.count(ctr, delta) - COUNT_CHANGED.log( - counter_name=counter_name, - delta=delta, - value=self._client.stats_provider.counters[ctr], - ) - -# this isn't in interfaces.py because it's very specific to QueueMixin -class IQueuedItem(Interface): - relpath_u = Attribute("The path this item represents") - progress = Attribute("A PercentProgress instance") - - def set_status(self, status, current_time=None): - """ - """ - - def status_time(self, state): - """ - Get the time of particular state change, or None - """ - - def status_history(self): - """ - All status changes, sorted latest -> oldest - """ - - -@implementer(IQueuedItem) -class QueuedItem(object): - kind = None - - def __init__(self, relpath_u, progress, size): - self.relpath_u = relpath_u - self.progress = progress - self._status_history = dict() - self.size = size - - def set_status(self, status, current_time=None): - if current_time is None: - current_time = time.time() - self._status_history[status] = current_time - QUEUED_ITEM_STATUS_CHANGE.log( - relpath=self.relpath_u, - status=status, - ) - - def status_time(self, state): - """ - Returns None if there's no status-update for 'state', else returns - the timestamp when that state was reached. - """ - return self._status_history.get(state, None) - - def status_history(self): - """ - Returns a list of 2-tuples of (state, timestamp) sorted by timestamp - """ - hist = self._status_history.items() - hist.sort(lambda a, b: cmp(a[1], b[1])) - return hist - - def __eq__(self, other): - return ( - other.relpath_u == self.relpath_u, - other.status_history() == self.status_history(), - ) - - -class UploadItem(QueuedItem): - """ - Represents a single item the _deque of the Uploader - """ - kind = u"upload" - - -_ITEM = Field( - u"item", - lambda i: { - u"relpath": i.relpath_u, - u"size": i.size, - }, - u"An item to be uploaded or downloaded.", - eliotutil.validateInstanceOf(QueuedItem), -) - -PROCESS_ITEM = ActionType( - u"magic-folder:process-item", - [_ITEM], - [], - u"A path which was found wanting of an update is receiving an update.", -) - -DOWNLOAD_BEST_VERSION = ActionType( - u"magic-folder:download-best-version", - [], - [], - u"The content of a file in the Magic Folder is being downloaded.", -) - -class Uploader(QueueMixin): - - def __init__(self, client, local_path_u, db, upload_dirnode, pending_delay, clock): - QueueMixin.__init__(self, client, local_path_u, db, u'uploader', clock) - - self.is_ready = False - - if not IDirectoryNode.providedBy(upload_dirnode): - raise AssertionError("'upload_dircap' does not refer to a directory") - if upload_dirnode.is_unknown() or upload_dirnode.is_readonly(): - raise AssertionError("'upload_dircap' is not a writecap to a directory") - - self._upload_dirnode = upload_dirnode - self._inotify = get_inotify_module() - self._notifier = self._inotify.INotify() - - self._pending = set() # of unicode relpaths - self._pending_delay = pending_delay - self._periodic_full_scan_duration = 10 * 60 # perform a full scan every 10 minutes - self._periodic_callid = None - - if hasattr(self._notifier, 'set_pending_delay'): - self._notifier.set_pending_delay(pending_delay) - - # TODO: what about IN_MOVE_SELF and IN_UNMOUNT? - # - self.mask = ( self._inotify.IN_CREATE - | self._inotify.IN_CLOSE_WRITE - | self._inotify.IN_MOVED_TO - | self._inotify.IN_MOVED_FROM - | self._inotify.IN_DELETE - | self._inotify.IN_ONLYDIR - | IN_EXCL_UNLINK - ) - - def _add_watch(self, filepath): - self._notifier.watch( - filepath, - mask=self.mask, - callbacks=[self._notify], - recursive=True, - ) - - def start_monitoring(self): - action = START_MONITORING(**self._log_fields) - with action.context(): - d = DeferredContext(defer.succeed(None)) - - d.addCallback(lambda ign: self._add_watch(self._local_filepath)) - d.addCallback(lambda ign: self._notifier.startReading()) - d.addCallback(lambda ign: self._count('dirs_monitored')) - d.addBoth(self._call_hook, 'started') - return d.addActionFinish() - - def stop(self): - action = STOP_MONITORING(**self._log_fields) - with action.context(): - self._notifier.stopReading() - self._count('dirs_monitored', -1) - if self._periodic_callid: - try: - self._periodic_callid.cancel() - except AlreadyCancelled: - pass - - if hasattr(self._notifier, 'wait_until_stopped'): - d = DeferredContext(self._notifier.wait_until_stopped()) - else: - d = DeferredContext(defer.succeed(None)) - - d.addCallback(lambda ignored: QueueMixin.stop(self)) - return d.addActionFinish() - - def start_uploading(self): - action = START_UPLOADING(**self._log_fields) - with action: - self.is_ready = True - - all_relpaths = self._db.get_all_relpaths() - - for relpath_u in all_relpaths: - self._add_pending(relpath_u) - - self._full_scan() - self._begin_processing() - - def _scan_delay(self): - return self._pending_delay - - def _full_scan(self): - with FULL_SCAN(**self._log_fields): - self._periodic_callid = self._clock.callLater(self._periodic_full_scan_duration, self._full_scan) - self._scan(u"") - - def _add_pending(self, relpath_u): - with ADD_PENDING(relpath=relpath_u) as action: - if magicpath.should_ignore_file(relpath_u): - action.add_success_fields(ignored=True, already_pending=False, size=None) - return - if self.is_pending(relpath_u): - action.add_success_fields(ignored=False, already_pending=True, size=None) - return - - self._pending.add(relpath_u) - fp = self._get_filepath(relpath_u) - pathinfo = get_pathinfo(unicode_from_filepath(fp)) - progress = PercentProgress() - action.add_success_fields(ignored=False, already_pending=False, size=pathinfo.size) - item = UploadItem(relpath_u, progress, pathinfo.size) - item.set_status('queued', self._clock.seconds()) - self._deque.append(item) - self._count('objects_queued') - - def _scan(self, reldir_u): - # Scan a directory by (synchronously) adding the paths of all its children to self._pending. - # Note that this doesn't add them to the deque -- that will - with SCAN(relpath=reldir_u): - fp = self._get_filepath(reldir_u) - try: - children = listdir_filepath(fp) - except EnvironmentError: - raise Exception("WARNING: magic folder: permission denied on directory %s" - % quote_filepath(fp)) - except FilenameEncodingError: - raise Exception("WARNING: magic folder: could not list directory %s due to a filename encoding error" - % quote_filepath(fp)) - - for child in children: - _assert(isinstance(child, unicode), child=child) - self._add_pending("%s/%s" % (reldir_u, child) if reldir_u != u"" else child) - - def is_pending(self, relpath_u): - return relpath_u in self._pending - - def _notify(self, opaque, path, events_mask): - with NOTIFIED(path=path, **self._log_fields): - try: - return self._real_notify(opaque, path, events_mask) - except Exception: - write_traceback() - - def _real_notify(self, opaque, path, events_mask): - action = REACT_TO_INOTIFY( - # We could think about logging opaque here but ... it's opaque. - # All can do is id() or repr() it and neither of those actually - # produces very illuminating results. We drop opaque on the - # floor, anyway. - inotify_events=events_mask, - ) - success_fields = dict(non_dir_created=False, already_pending=False, ignored=False) - - with action: - relpath_u = self._get_relpath(path) - - # We filter out IN_CREATE events not associated with a directory. - # Acting on IN_CREATE for files could cause us to read and upload - # a possibly-incomplete file before the application has closed it. - # There should always be an IN_CLOSE_WRITE after an IN_CREATE, I think. - # It isn't possible to avoid watching for IN_CREATE at all, because - # it is the only event notified for a directory creation. - - if ((events_mask & self._inotify.IN_CREATE) != 0 and - (events_mask & self._inotify.IN_ISDIR) == 0): - success_fields[u"non_dir_created"] = True - elif relpath_u in self._pending: - success_fields[u"already_pending"] = True - elif magicpath.should_ignore_file(relpath_u): - success_fields[u"ignored"] = True - else: - self._add_pending(relpath_u) - - # Always fire the inotify hook. If an accident of timing causes a - # second inotify event for a particular path before the first has - # been processed, the expectation is still that any code that was - # waiting for the second inotify event should be notified. - self._call_hook(path, 'inotify') - action.add_success_fields(**success_fields) - - def _process(self, item): - """ - Possibly upload a single QueuedItem. If this returns False, the item is - removed from _process_history. - """ - # Uploader - with PROCESS_ITEM(item=item).context(): - relpath_u = item.relpath_u - precondition(isinstance(relpath_u, unicode), relpath_u) - precondition(not relpath_u.endswith(u'/'), relpath_u) - encoded_path_u = magicpath.path2magic(relpath_u) - - d = DeferredContext(defer.succeed(False)) - if relpath_u is None: - item.set_status('invalid_path', self._clock.seconds()) - return d.addActionFinish() - item.set_status('started', self._clock.seconds()) - - try: - # Take this item out of the pending set before we do any - # I/O-based processing related to it. If a further change - # takes place after we remove it from this set, we want it to - # end up in the set again. If we haven't gotten around to - # doing the I/O-based processing yet then the worst that will - # happen is we'll do a little redundant processing. - # - # If we did it the other way around, the sequence of events - # might be something like: we do some I/O, someone else does - # some I/O, a notification gets discarded because the path is - # still in the pending set, _then_ we remove it from the - # pending set. In such a circumstance, we've missed some I/O - # that we should have responded to. - with REMOVE_FROM_PENDING(relpath=relpath_u, pending=self._pending): - self._pending.remove(relpath_u) - except KeyError: - pass - - fp = self._get_filepath(relpath_u) - pathinfo = get_pathinfo(unicode_from_filepath(fp)) - - db_entry_is_dir = False - db_entry = self._db.get_db_entry(relpath_u) - if db_entry is None: - # Maybe it was a directory! - db_entry = self._db.get_db_entry(relpath_u + u"/") - if db_entry is None: - NO_DATABASE_ENTRY.log() - else: - db_entry_is_dir = True - - def _maybe_upload(ign, now=None): - MAYBE_UPLOAD.log(relpath=relpath_u) - if now is None: - now = time.time() - - if not pathinfo.exists: - # FIXME merge this with the 'isfile' case. - NOTIFIED_OBJECT_DISAPPEARED.log(path=fp) - self._count('objects_disappeared') - - if db_entry is None: - # If it exists neither on the filesystem nor in the - # database, it's neither a creation nor a deletion and - # there's nothing more to do. - return False - - if pathinfo.isdir or db_entry_is_dir: - with PROPAGATE_DIRECTORY_DELETION(): - for localpath in self._db.get_direct_children(relpath_u): - self._add_pending(localpath.relpath_u) - - last_downloaded_timestamp = now # is this correct? - - if is_new_file(pathinfo, db_entry): - new_version = db_entry.version + 1 - else: - NOT_UPLOADING.log() - self._count('objects_not_uploaded') - return False - - # look out! there's another place we set a "metadata" - # object like this (for new, not deleted files) - metadata = { - 'version': new_version, - 'deleted': True, - 'last_downloaded_timestamp': last_downloaded_timestamp, - 'user_mtime': pathinfo.ctime_ns / 1000000000.0, # why are we using ns in PathInfo?? - } - - # from the Fire Dragons part of the spec: - # Later, in response to a local filesystem change at a given path, the - # Magic Folder client reads the last-downloaded record associated with - # that path (if any) from the database and then uploads the current - # file. When it links the uploaded file into its client DMD, it - # includes the ``last_downloaded_uri`` field in the metadata of the - # directory entry, overwriting any existing field of that name. If - # there was no last-downloaded record associated with the path, this - # field is omitted. - # Note that ``last_downloaded_uri`` field does *not* record the URI of - # the uploaded file (which would be redundant); it records the URI of - # the last download before the local change that caused the upload. - # The field will be absent if the file has never been downloaded by - # this client (i.e. if it was created on this client and no change - # by any other client has been detected). - - # XXX currently not actually true: it will record the - # LAST THING we wrote to (or saw on) disk (not - # necessarily downloaded?) - - if db_entry.last_downloaded_uri is not None: - metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri - if db_entry.last_uploaded_uri is not None: - metadata['last_uploaded_uri'] = db_entry.last_uploaded_uri - - if db_entry_is_dir: - real_encoded_path_u = encoded_path_u + magicpath.path2magic(u"/") - real_relpath_u = relpath_u + u"/" - else: - real_encoded_path_u = encoded_path_u - real_relpath_u = relpath_u - - empty_uploadable = Data("", self._client.convergence) - d2 = DeferredContext(self._upload_dirnode.add_file( - real_encoded_path_u, - empty_uploadable, - metadata=metadata, - overwrite=True, - progress=item.progress, - )) - - def _add_db_entry(filenode): - filecap = filenode.get_uri() - # if we're uploading a file, we want to set - # last_downloaded_uri to the filecap so that we don't - # immediately re-download it when we start up next - last_downloaded_uri = metadata.get('last_downloaded_uri', filecap) - self._db.did_upload_version( - real_relpath_u, - new_version, - filecap, - last_downloaded_uri, - last_downloaded_timestamp, - pathinfo, - ) - self._count('files_uploaded') - d2.addCallback(_add_db_entry) - d2.addCallback(lambda ign: True) - return d2.result - elif pathinfo.islink: - SYMLINK.log(path=fp) - return False - elif pathinfo.isdir: - if not getattr(self._notifier, 'recursive_includes_new_subdirectories', False): - self._add_watch(fp) - - DIRECTORY_PATHENTRY.log(pathentry=db_entry) - if not is_new_file(pathinfo, db_entry): - NOT_NEW_DIRECTORY.log() - return False - - uploadable = Data("", self._client.convergence) - with PROCESS_DIRECTORY().context() as action: - upload_d = DeferredContext(self._upload_dirnode.add_file( - encoded_path_u + magicpath.path2magic(u"/"), - uploadable, - metadata={"version": 0}, - overwrite=True, - progress=item.progress, - )) - def _dir_succeeded(dirnode): - action.add_success_fields(created_directory=relpath_u) - self._count('directories_created') - self._db.did_upload_version( - relpath_u + u"/", - version=0, - last_uploaded_uri=dirnode.get_uri(), - last_downloaded_uri=None, - last_downloaded_timestamp=now, - pathinfo=pathinfo, - ) - upload_d.addCallback(_dir_succeeded) - upload_d.addCallback(lambda ign: self._scan(relpath_u)) - upload_d.addCallback(lambda ign: True) - return upload_d.addActionFinish() - elif pathinfo.isfile: - last_downloaded_timestamp = now - - if db_entry is None: - new_version = 0 - elif is_new_file(pathinfo, db_entry): - new_version = db_entry.version + 1 - else: - NOT_NEW_FILE.log() - self._count('objects_not_uploaded') - return False - - metadata = { - 'version': new_version, - 'last_downloaded_timestamp': last_downloaded_timestamp, - 'user_mtime': pathinfo.mtime_ns / 1000000000.0, # why are we using ns in PathInfo?? - } - if db_entry is not None: - if db_entry.last_downloaded_uri is not None: - metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri - if db_entry.last_uploaded_uri is not None: - metadata['last_uploaded_uri'] = db_entry.last_uploaded_uri - - uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) - d2 = DeferredContext(self._upload_dirnode.add_file( - encoded_path_u, uploadable, - metadata=metadata, - overwrite=True, - progress=item.progress, - )) - - def _add_db_entry(filenode): - filecap = filenode.get_uri() - # if we're uploading a file, we want to set - # last_downloaded_uri to the filecap so that we don't - # immediately re-download it when we start up next - last_downloaded_uri = filecap - self._db.did_upload_version( - relpath_u, - new_version, - filecap, - last_downloaded_uri, - last_downloaded_timestamp, - pathinfo - ) - self._count('files_uploaded') - return True - d2.addCallback(_add_db_entry) - return d2.result - else: - SPECIAL_FILE.log() - return False - - d.addCallback(_maybe_upload) - - def _succeeded(res): - if res: - self._count('objects_succeeded') - # TODO: maybe we want the status to be 'ignored' if res is False - item.set_status('success', self._clock.seconds()) - return res - def _failed(f): - self._count('objects_failed') - item.set_status('failure', self._clock.seconds()) - return f - d.addCallbacks(_succeeded, _failed) - return d.addActionFinish() - - def _get_metadata(self, encoded_path_u): - try: - d = self._upload_dirnode.get_metadata_for(encoded_path_u) - except KeyError: - return Failure() - return d - - def _get_filenode(self, encoded_path_u): - try: - d = self._upload_dirnode.get(encoded_path_u) - except KeyError: - return Failure() - return d - - -class WriteFileMixin(object): - FUDGE_SECONDS = 10.0 - - def _get_conflicted_filename(self, abspath_u): - return abspath_u + u".conflict" - - def _write_downloaded_file(self, local_path_u, abspath_u, file_contents, - is_conflict=False, now=None, mtime=None): - if now is None: - now = time.time() - action = WRITE_DOWNLOADED_FILE( - abspath=abspath_u, - size=len(file_contents), - is_conflict=is_conflict, - now=now, - mtime=mtime, - ) - with action: - return self._write_downloaded_file_logged( - local_path_u, - abspath_u, - file_contents, - is_conflict, - now, - mtime, - ) - - def _write_downloaded_file_logged(self, local_path_u, abspath_u, - file_contents, is_conflict, now, mtime): - # 1. Write a temporary file, say .foo.tmp. - # 2. is_conflict determines whether this is an overwrite or a conflict. - # 3. Set the mtime of the replacement file to be T seconds before the - # current local time, or mtime whichever is oldest - # 4. Perform a file replacement with backup filename foo.backup, - # replaced file foo, and replacement file .foo.tmp. If any step of - # this operation fails, reclassify as a conflict and stop. - # - # Returns the path of the destination file. - precondition_abspath(abspath_u) - replacement_path_u = abspath_u + u".tmp" # FIXME more unique - - initial_path_u = os.path.dirname(abspath_u) - fileutil.make_dirs_with_absolute_mode(local_path_u, initial_path_u, (~ self._umask) & 0o777) - fileutil.write(replacement_path_u, file_contents) - os.chmod(replacement_path_u, (~ self._umask) & 0o666) - - # FUDGE_SECONDS is used to determine if another process has - # written to the same file concurrently. This is described in - # the Earth Dragon section of our design document ("T" in the - # spec is FUDGE_SECONDS here): - # docs/proposed/magic-folder/remote-to-local-sync.rst - fudge_time = now - self.FUDGE_SECONDS - modified_time = min(fudge_time, mtime) if mtime else fudge_time - os.utime(replacement_path_u, (now, modified_time)) - if is_conflict: - return self._rename_conflicted_file(abspath_u, replacement_path_u) - else: - try: - fileutil.replace_file(abspath_u, replacement_path_u) - return abspath_u - except fileutil.ConflictError as e: - OVERWRITE_BECOMES_CONFLICT.log(reason=e) - return self._rename_conflicted_file(abspath_u, replacement_path_u) - - @log_call( - action_type=u"magic-folder:rename-conflicted", - include_args=["abspath_u", "replacement_path_u"], - ) - def _rename_conflicted_file(self, abspath_u, replacement_path_u): - conflict_path_u = self._get_conflicted_filename(abspath_u) - fileutil.rename_no_overwrite(replacement_path_u, conflict_path_u) - return conflict_path_u - - @log_call( - action_type=u"magic-folder:rename-deleted", - include_args=["abspath_u"], - ) - def _rename_deleted_file(self, abspath_u): - try: - fileutil.rename_no_overwrite(abspath_u, abspath_u + u'.backup') - except OSError: - ALREADY_GONE.log() - return abspath_u - - -def _is_empty_filecap(client, cap): - """ - Internal helper. - - :param cap: a capability URI - - :returns: True if "cap" represents an empty file - """ - node = client.create_node_from_uri( - None, - cap.encode('ascii'), - ) - return (not node.get_size()) - - -class DownloadItem(QueuedItem): - """ - Represents a single item in the _deque of the Downloader - """ - kind = u"download" - - def __init__(self, relpath_u, progress, filenode, metadata, size): - super(DownloadItem, self).__init__(relpath_u, progress, size) - self.file_node = filenode - self.metadata = metadata - - -class Downloader(QueueMixin, WriteFileMixin): - - def __init__(self, client, local_path_u, db, collective_dirnode, - upload_readonly_dircap, clock, is_upload_pending, umask, - status_reporter, poll_interval=60): - QueueMixin.__init__(self, client, local_path_u, db, u'downloader', clock) - - if not IDirectoryNode.providedBy(collective_dirnode): - raise AssertionError("'collective_dircap' does not refer to a directory") - if collective_dirnode.is_unknown() or not collective_dirnode.is_readonly(): - raise AssertionError("'collective_dircap' is not a readonly cap to a directory") - - self._collective_dirnode = collective_dirnode - self._upload_readonly_dircap = upload_readonly_dircap - self._is_upload_pending = is_upload_pending - self._umask = umask - self._status_reporter = status_reporter - self._poll_interval = poll_interval - - @eliotutil.inline_callbacks - def start_downloading(self): - action = START_DOWNLOADING(**self._log_fields) - with action: - ALL_FILES.log(files=self._db.get_all_relpaths()) - - while True: - try: - yield self._scan_remote_collective(scan_self=True) - # The integration tests watch for this log message (in the - # Twisted log) to decide when it is safe to proceed. - # Clearly, we need better programmatic interrogation of - # magic-folder state. - twmsg("Completed initial Magic Folder scan successfully ({})".format(self)) - self._begin_processing() - return - except Exception: - self._status_reporter( - False, "Initial scan has failed", - "Last tried at %s" % self.nice_current_time(), - ) - write_traceback() - yield task.deferLater(self._clock, self._scan_delay(), lambda: None) - - def nice_current_time(self): - return format_time(datetime.fromtimestamp(self._clock.seconds()).timetuple()) - - def _should_download(self, relpath_u, remote_version, remote_uri): - """ - _should_download returns a bool indicating whether or not a remote object should be downloaded. - We check the remote metadata version against our magic-folder db version number; - latest version wins. - """ - if magicpath.should_ignore_file(relpath_u): - return False - db_entry = self._db.get_db_entry(relpath_u) - if db_entry is None: - return True - if db_entry.version < remote_version: - return True - if db_entry.last_downloaded_uri is None and _is_empty_filecap(self._client, remote_uri): - pass - elif db_entry.last_downloaded_uri != remote_uri: - return True - return False - - def _get_local_latest(self, relpath_u): - """ - _get_local_latest takes a unicode path string checks to see if this file object - exists in our magic-folder db; if not then return None - else check for an entry in our magic-folder db and return it. - """ - if not self._get_filepath(relpath_u).exists(): - return None - return self._db.get_db_entry(relpath_u) - - def _get_collective_latest_file(self, filename): - """ - _get_collective_latest_file takes a file path pointing to a file managed by - magic-folder and returns a deferred that fires with the two tuple containing a - file node and metadata for the latest version of the file located in the - magic-folder collective directory. - """ - action = start_action( - action_type=u"magic-folder:downloader:get-latest-file", - name=filename, - ) - with action.context(): - collective_dirmap_d = DeferredContext(self._collective_dirnode.list()) - def scan_collective(result): - COLLECTIVE_SCAN.log(dmds=result) - list_of_deferreds = [] - for dir_name in result: - # XXX make sure it's a directory - d = DeferredContext(defer.succeed(None)) - d.addCallback(lambda x, dir_name=dir_name: result[dir_name][0].get_child_and_metadata(filename)) - list_of_deferreds.append(d) - deferList = defer.DeferredList(list_of_deferreds, consumeErrors=True) - return deferList - collective_dirmap_d.addCallback(scan_collective) - def highest_version(deferredList): - max_version = 0 - metadata = None - node = None - for success, result in deferredList: - if success: - Message.log( - message_type=u"magic-folder:downloader:get-latest-file:version", - version=result[1]['version'], - ) - if node is None or result[1]['version'] > max_version: - node, metadata = result - max_version = result[1]['version'] - else: - Message.log( - message_type="magic-folder:downloader:get-latest-file:failed", - ) - return node, metadata - collective_dirmap_d.addCallback(highest_version) - return collective_dirmap_d.addActionFinish() - - def _scan_remote_dmd(self, nickname, dirnode, scan_batch): - """ - Read the contents of a single DMD into the given batch. - - :param unicode nickname: The nickname for the participant owning the - DMD to scan. - - :param IDirectoryNode dirnode: The node representing the chosen - participant's DMD. - - :param dict scan_batch: A dictionary into which to collect the results - of the scan. This is mutated to add the results in-place. Keys - are the unicode relative paths of contents of the DMD. Values are - a list of two-tuples. The first element of each two-tuple is the - ``IFilesystemNode`` for the content. The second element is a - ``dict`` of metadata. - - :return Deferred: A ``Deferred`` which fires when the scan is - complete. - """ - with SCAN_REMOTE_DMD(nickname=nickname).context(): - d = DeferredContext(dirnode.list()) - def scan_listing(listing_map): - for encoded_relpath_u, (file_node, metadata) in listing_map.iteritems(): - relpath_u = magicpath.magic2path(encoded_relpath_u) - local_dbentry = self._get_local_latest(relpath_u) - - # XXX FIXME this is *awefully* similar to - # _should_download code in function etc -- can we - # share? - remote_version = metadata.get('version', None) - remote_uri = file_node.get_readonly_uri() - REMOTE_DMD_ENTRY.log( - relpath=relpath_u, - pathentry=local_dbentry, - remote_version=remote_version, - remote_uri=remote_uri, - ) - - if (local_dbentry is None or remote_version is None or - local_dbentry.version < remote_version or - (local_dbentry.version == remote_version and local_dbentry.last_downloaded_uri != remote_uri)): - ADD_TO_DOWNLOAD_QUEUE.log(relpath=relpath_u) - - # The scan_batch is shared across the scan of multiple - # DMDs. It is expected the DMDs will most often be mostly - # synchronized with each other. The common case, then, is - # that there is already an entry for relpath_u. So try to - # make that the fast path: assume there is a value already - # and extend it. If there's not, we'll do an extra lookup - # to initialize it. - try: - scan_batch[relpath_u] += [(file_node, metadata)] - except KeyError: - scan_batch[relpath_u] = [(file_node, metadata)] - self._status_reporter( - True, 'Magic folder is working', - 'Last scan: %s' % self.nice_current_time(), - ) - - d.addCallback(scan_listing) - return d.addActionFinish() - - @eliotutil.log_call_deferred(SCAN_REMOTE_COLLECTIVE.action_type) - def _scan_remote_collective(self, scan_self=False): - precondition(not self._deque, "Items in _deque invalidate should_download logic") - scan_batch = {} # path -> [(filenode, metadata)] - d = DeferredContext(self._collective_dirnode.list()) - def scan_collective(dirmap): - d2 = DeferredContext(defer.succeed(None)) - for dir_name in dirmap: - (dirnode, metadata) = dirmap[dir_name] - if scan_self or dirnode.get_readonly_uri() != self._upload_readonly_dircap: - d2.addCallback(lambda ign, dir_name=dir_name, dirnode=dirnode: - self._scan_remote_dmd(dir_name, dirnode, scan_batch)) - # XXX what should we do to make this failure more visible to users? - d2.addErrback(write_traceback) - return d2.result - d.addCallback(scan_collective) - - @log_call( - action_type=u"magic-folder:filter-batch-to-deque", - include_args=[], - include_result=False, - ) - def _filter_batch_to_deque(ign): - ITEM_QUEUE.log(items=self._deque) - SCAN_BATCH.log(batch=scan_batch) - for relpath_u, versions in scan_batch.iteritems(): - file_node, metadata = max(versions, key=lambda x: x[1]['version']) - - if self._should_download(relpath_u, metadata['version'], file_node.get_readonly_uri()): - to_dl = DownloadItem( - relpath_u, - PercentProgress(file_node.get_size()), - file_node, - metadata, - file_node.get_size(), - ) - to_dl.set_status('queued', self._clock.seconds()) - self._deque.append(to_dl) - self._count("objects_queued") - else: - self._call_hook(None, 'processed', async=True) # await this maybe-Deferred?? - - d.addCallback(_filter_batch_to_deque) - return d.result - - def _scan_delay(self): - return self._poll_interval - - @eliotutil.log_call_deferred(PERFORM_SCAN.action_type) - @eliotutil.inline_callbacks - def _perform_scan(self): - try: - yield self._scan_remote_collective() - self._status_reporter( - True, 'Magic folder is working', - 'Last scan: %s' % self.nice_current_time(), - ) - except Exception as e: - write_traceback() - self._status_reporter( - False, 'Remote scan has failed: %s' % str(e), - 'Last attempted at %s' % self.nice_current_time(), - ) - - def _process(self, item): - """ - Possibly upload a single QueuedItem. If this returns False, the item is - removed from _process_history. - """ - # Downloader - now = self._clock.seconds() - - item.set_status('started', now) - fp = self._get_filepath(item.relpath_u) - abspath_u = unicode_from_filepath(fp) - conflict_path_u = self._get_conflicted_filename(abspath_u) - last_uploaded_uri = item.metadata.get('last_uploaded_uri', None) - - with PROCESS_ITEM(item=item): - d = DeferredContext(defer.succeed(False)) - - def do_update_db(written_abspath_u): - filecap = item.file_node.get_uri() - if not item.file_node.get_size(): - filecap = None # ^ is an empty file - last_downloaded_uri = filecap - last_downloaded_timestamp = now - written_pathinfo = get_pathinfo(written_abspath_u) - - if not written_pathinfo.exists and not item.metadata.get('deleted', False): - raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u)) - - self._db.did_upload_version( - item.relpath_u, - item.metadata['version'], - last_uploaded_uri, - last_downloaded_uri, - last_downloaded_timestamp, - written_pathinfo, - ) - self._count('objects_downloaded') - item.set_status('success', self._clock.seconds()) - return True - - def failed(f): - item.set_status('failure', self._clock.seconds()) - self._count('objects_failed') - return f - - if os.path.isfile(conflict_path_u): - def fail(res): - raise ConflictError("download failed: already conflicted: %r" % (item.relpath_u,)) - d.addCallback(fail) - else: - - # Let ``last_downloaded_uri`` be the field of that name obtained from - # the directory entry metadata for ``foo`` in Bob's DMD (this field - # may be absent). Then the algorithm is: - - # * 2a. Attempt to "stat" ``foo`` to get its *current statinfo* (size - # in bytes, ``mtime``, and ``ctime``). If Alice has no local copy - # of ``foo``, classify as an overwrite. - - current_statinfo = get_pathinfo(abspath_u) - - is_conflict = False - db_entry = self._db.get_db_entry(item.relpath_u) - dmd_last_downloaded_uri = item.metadata.get('last_downloaded_uri', None) - - # * 2b. Read the following information for the path ``foo`` from the - # local magic folder db: - # * the *last-seen statinfo*, if any (this is the size in - # bytes, ``mtime``, and ``ctime`` stored in the ``local_files`` - # table when the file was last uploaded); - # * the ``last_uploaded_uri`` field of the ``local_files`` table - # for this file, which is the URI under which the file was last - # uploaded. - - with CHECKING_CONFLICTS() as action: - conflict_reason = None - if db_entry: - # * 2c. If any of the following are true, then classify as a conflict: - # * i. there are pending notifications of changes to ``foo``; - # * ii. the last-seen statinfo is either absent (i.e. there is - # no entry in the database for this path), or different from the - # current statinfo; - - if current_statinfo.exists: - if (db_entry.mtime_ns != current_statinfo.mtime_ns or \ - db_entry.ctime_ns != current_statinfo.ctime_ns or \ - db_entry.size != current_statinfo.size): - is_conflict = True - conflict_reason = u"dbentry mismatch metadata" - - if db_entry.last_downloaded_uri is None \ - or db_entry.last_uploaded_uri is None \ - or dmd_last_downloaded_uri is None: - # we've never downloaded anything before for this - # file, but the other side might have created a new - # file "at the same time" - if db_entry.version >= item.metadata['version']: - is_conflict = True - conflict_reason = u"dbentry newer version" - elif dmd_last_downloaded_uri != db_entry.last_downloaded_uri: - is_conflict = True - conflict_reason = u"last_downloaded_uri mismatch" - - else: # no local db_entry .. but has the file appeared locally meantime? - if current_statinfo.exists: - is_conflict = True - conflict_reason = u"file appeared" - - action.add_success_fields( - is_conflict=is_conflict, - conflict_reason=conflict_reason, - ) - - if is_conflict: - self._count('objects_conflicted') - - if item.relpath_u.endswith(u"/"): - if item.metadata.get('deleted', False): - REMOTE_DIRECTORY_DELETED.log() - else: - REMOTE_DIRECTORY_CREATED.log() - d.addCallback(lambda ign: fileutil.make_dirs(abspath_u)) - d.addCallback(lambda ign: abspath_u) - else: - if item.metadata.get('deleted', False): - d.addCallback(lambda ign: self._rename_deleted_file(abspath_u)) - else: - @eliotutil.log_call_deferred(DOWNLOAD_BEST_VERSION.action_type) - def download_best_version(ignored): - d = DeferredContext(item.file_node.download_best_version(progress=item.progress)) - d.addCallback(lambda contents: self._write_downloaded_file( - self._local_path_u, abspath_u, contents, - is_conflict=is_conflict, - mtime=item.metadata.get('user_mtime', item.metadata.get('tahoe', {}).get('linkmotime')), - )) - return d.result - - d.addCallback(download_best_version) - - d.addCallback(do_update_db) - d.addErrback(failed) - - def trap_conflicts(f): - f.trap(ConflictError) - return False - d.addErrback(trap_conflicts) - return d.addActionFinish() diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index 400b81efb..79157322f 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -951,7 +951,7 @@ class GeneralSFTPFile(PrefixingLogMixin): return d -class StoppableList: +class StoppableList(object): def __init__(self, items): self.items = items def __iter__(self): @@ -961,7 +961,7 @@ class StoppableList: pass -class Reason: +class Reason(object): def __init__(self, value): self.value = value diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 9ecd2f064..576d72a0e 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -68,7 +68,7 @@ def roundup_pow2(x): return ans -class CompleteBinaryTreeMixin: +class CompleteBinaryTreeMixin(object): """ Adds convenience methods to a complete binary tree. diff --git a/src/allmydata/history.py b/src/allmydata/history.py index f46eb6d4b..015b89107 100644 --- a/src/allmydata/history.py +++ b/src/allmydata/history.py @@ -1,7 +1,7 @@ import weakref -class History: +class History(object): """Keep track of recent operations, for a status display.""" name = "history" diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index 596bc8225..11f4341d4 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -496,16 +496,19 @@ class Checker(log.PrefixingLogMixin): that we want to track and report whether or not each server responded.)""" - rref = s.get_rref() + storage_server = s.get_storage_server() lease_seed = s.get_lease_seed() if self._add_lease: renew_secret = self._get_renewal_secret(lease_seed) cancel_secret = self._get_cancel_secret(lease_seed) - d2 = rref.callRemote("add_lease", storageindex, - renew_secret, cancel_secret) + d2 = storage_server.add_lease( + storageindex, + renew_secret, + cancel_secret, + ) d2.addErrback(self._add_lease_failed, s.get_name(), storageindex) - d = rref.callRemote("get_buckets", storageindex) + d = storage_server.get_buckets(storageindex) def _wrap_results(res): return (res, True) diff --git a/src/allmydata/immutable/downloader/fetcher.py b/src/allmydata/immutable/downloader/fetcher.py index ae76da876..f3cd41fe0 100644 --- a/src/allmydata/immutable/downloader/fetcher.py +++ b/src/allmydata/immutable/downloader/fetcher.py @@ -7,7 +7,7 @@ from allmydata.util.dictutil import DictOfSets from common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \ BadSegmentNumberError -class SegmentFetcher: +class SegmentFetcher(object): """I am responsible for acquiring blocks for a single segment. I will use the Share instances passed to my add_shares() method to locate, retrieve, and validate those blocks. I expect my parent node to call my diff --git a/src/allmydata/immutable/downloader/finder.py b/src/allmydata/immutable/downloader/finder.py index 8bcdca76f..3c6788537 100644 --- a/src/allmydata/immutable/downloader/finder.py +++ b/src/allmydata/immutable/downloader/finder.py @@ -20,11 +20,11 @@ def incidentally(res, f, *args, **kwargs): f(*args, **kwargs) return res -class RequestToken: +class RequestToken(object): def __init__(self, server): self.server = server -class ShareFinder: +class ShareFinder(object): OVERDUE_TIMEOUT = 10.0 def __init__(self, storage_broker, verifycap, node, download_status, @@ -139,7 +139,7 @@ class ShareFinder: # TODO: get the timer from a Server object, it knows best self.overdue_timers[req] = reactor.callLater(self.OVERDUE_TIMEOUT, self.overdue, req) - d = server.get_rref().callRemote("get_buckets", self._storage_index) + d = server.get_storage_server().get_buckets(self._storage_index) d.addBoth(incidentally, self._request_retired, req) d.addCallbacks(self._got_response, self._got_error, callbackArgs=(server, req, d_ev, time_sent, lp), @@ -221,5 +221,3 @@ class ShareFinder: self.log(format="got error from [%(name)s]", name=server.get_name(), failure=f, level=log.UNUSUAL, parent=lp, umid="zUKdCw") - - diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index a9f36810b..1c7ecca33 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -23,16 +23,18 @@ class IDownloadStatusHandlingConsumer(Interface): """Record the DownloadStatus 'read event', to be updated with the time it takes to decrypt each chunk of data.""" -class Cancel: +class Cancel(object): def __init__(self, f): self._f = f self.active = True + def cancel(self): if self.active: self.active = False self._f(self) -class DownloadNode: + +class DownloadNode(object): """Internal class which manages downloads and holds state. External callers use CiphertextFileNode instead.""" diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index e4ed429b5..5237a7a9b 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -24,7 +24,7 @@ class DataUnavailable(Exception): pass -class Share: +class Share(object): """I represent a single instance of a single share (e.g. I reference the shnum2 for share SI=abcde on server xy12t, not the one on server ab45q). I am associated with a CommonShare that remembers data that is held in @@ -825,7 +825,7 @@ class Share: o.notify(state=DEAD, f=f) -class CommonShare: +class CommonShare(object): # TODO: defer creation of the hashtree until somebody uses us. There will # be a lot of unused shares, and we shouldn't spend the memory on a large # hashtree unless necessary. diff --git a/src/allmydata/immutable/downloader/status.py b/src/allmydata/immutable/downloader/status.py index 1d6246cea..fd30bc5fe 100644 --- a/src/allmydata/immutable/downloader/status.py +++ b/src/allmydata/immutable/downloader/status.py @@ -3,25 +3,32 @@ import itertools from zope.interface import implementer from allmydata.interfaces import IDownloadStatus -class ReadEvent: +class ReadEvent(object): + def __init__(self, ev, ds): self._ev = ev self._ds = ds + def update(self, bytes, decrypttime, pausetime): self._ev["bytes_returned"] += bytes self._ev["decrypt_time"] += decrypttime self._ev["paused_time"] += pausetime + def finished(self, finishtime): self._ev["finish_time"] = finishtime self._ds.update_last_timestamp(finishtime) -class SegmentEvent: + +class SegmentEvent(object): + def __init__(self, ev, ds): self._ev = ev self._ds = ds + def activate(self, when): if self._ev["active_time"] is None: self._ev["active_time"] = when + def deliver(self, when, start, length, decodetime): assert self._ev["active_time"] is not None self._ev["finish_time"] = when @@ -30,34 +37,43 @@ class SegmentEvent: self._ev["segment_start"] = start self._ev["segment_length"] = length self._ds.update_last_timestamp(when) + def error(self, when): self._ev["finish_time"] = when self._ev["success"] = False self._ds.update_last_timestamp(when) -class DYHBEvent: + +class DYHBEvent(object): + def __init__(self, ev, ds): self._ev = ev self._ds = ds + def error(self, when): self._ev["finish_time"] = when self._ev["success"] = False self._ds.update_last_timestamp(when) + def finished(self, shnums, when): self._ev["finish_time"] = when self._ev["success"] = True self._ev["response_shnums"] = shnums self._ds.update_last_timestamp(when) -class BlockRequestEvent: + +class BlockRequestEvent(object): + def __init__(self, ev, ds): self._ev = ev self._ds = ds + def finished(self, received, when): self._ev["finish_time"] = when self._ev["success"] = True self._ev["response_length"] = received self._ds.update_last_timestamp(when) + def error(self, when): self._ev["finish_time"] = when self._ev["success"] = False diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index ff15843a8..670989c3a 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -7,12 +7,12 @@ from twisted.internet import defer from allmydata import uri from twisted.internet.interfaces import IConsumer +from allmydata.crypto import aes from allmydata.interfaces import IImmutableFileNode, IUploadResults from allmydata.util import consumer from allmydata.check_results import CheckResults, CheckAndRepairResults from allmydata.util.dictutil import DictOfSets from allmydata.util.happinessutil import servers_of_happiness -from pycryptopp.cipher.aes import AES # local imports from allmydata.immutable.checker import Checker @@ -21,7 +21,7 @@ from allmydata.immutable.downloader.node import DownloadNode, \ IDownloadStatusHandlingConsumer from allmydata.immutable.downloader.status import DownloadStatus -class CiphertextFileNode: +class CiphertextFileNode(object): def __init__(self, verifycap, storage_broker, secret_holder, terminator, history): assert isinstance(verifycap, uri.CHKFileVerifierURI) @@ -201,8 +201,9 @@ class DecryptingConsumer(object): offset_big = offset // 16 offset_small = offset % 16 iv = binascii.unhexlify("%032x" % offset_big) - self._decryptor = AES(readkey, iv=iv) - self._decryptor.process("\x00"*offset_small) + self._decryptor = aes.create_decryptor(readkey, iv) + # this is just to advance the counter + aes.decrypt_data(self._decryptor, b"\x00" * offset_small) def set_download_status_read_event(self, read_ev): self._read_ev = read_ev @@ -219,7 +220,7 @@ class DecryptingConsumer(object): self._consumer.unregisterProducer() def write(self, ciphertext): started = now() - plaintext = self._decryptor.process(ciphertext) + plaintext = aes.decrypt_data(self._decryptor, ciphertext) if self._read_ev: elapsed = now() - started self._read_ev.update(0, elapsed, 0) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index ac4bb795c..e04e94e8f 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -16,7 +16,7 @@ class NotEnoughWritersError(Exception): pass -class CHKCheckerAndUEBFetcher: +class CHKCheckerAndUEBFetcher(object): """I check to see if a file is already present in the grid. I also fetch the URI Extension Block, which is useful for an uploading client who wants to avoid the work of encryption and encoding. @@ -54,7 +54,7 @@ class CHKCheckerAndUEBFetcher: def _get_all_shareholders(self, storage_index): dl = [] for s in self._peer_getter(storage_index): - d = s.get_rref().callRemote("get_buckets", storage_index) + d = s.get_storage_server().get_buckets(storage_index) d.addCallbacks(self._got_response, self._got_error, callbackArgs=(s,)) dl.append(d) @@ -244,7 +244,7 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): self._helper.upload_finished(self._storage_index, 0) del self._reader -class AskUntilSuccessMixin: +class AskUntilSuccessMixin(object): # create me with a _reader array _last_failure = None diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 79c632c91..f22733f0d 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -5,6 +5,7 @@ from twisted.internet import defer from twisted.application import service from foolscap.api import Referenceable, Copyable, RemoteCopy, fireEventually +from allmydata.crypto import aes from allmydata.util.hashutil import file_renewal_secret_hash, \ file_cancel_secret_hash, bucket_renewal_secret_hash, \ bucket_cancel_secret_hash, plaintext_hasher, \ @@ -23,7 +24,6 @@ from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \ NoServersError, InsufficientVersionError, UploadUnhappinessError, \ DEFAULT_MAX_SEGMENT_SIZE, IProgress, IPeerSelector from allmydata.immutable import layout -from pycryptopp.cipher.aes import AES from six.moves import cStringIO as StringIO from happiness_upload import share_placement, calculate_happiness @@ -261,20 +261,21 @@ class ServerTracker(object): return self._server.get_name() def query(self, sharenums): - rref = self._server.get_rref() - d = rref.callRemote("allocate_buckets", - self.storage_index, - self.renew_secret, - self.cancel_secret, - sharenums, - self.allocated_size, - canary=Referenceable()) + storage_server = self._server.get_storage_server() + d = storage_server.allocate_buckets( + self.storage_index, + self.renew_secret, + self.cancel_secret, + sharenums, + self.allocated_size, + canary=Referenceable(), + ) d.addCallback(self._buckets_allocated) return d def ask_about_existing_shares(self): - rref = self._server.get_rref() - return rref.callRemote("get_buckets", self.storage_index) + storage_server = self._server.get_storage_server() + return storage_server.get_buckets(self.storage_index) def _buckets_allocated(self, alreadygot_and_buckets): #log.msg("%s._got_reply(%s)" % (self, (alreadygot, buckets))) @@ -415,7 +416,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): # field) from getting large shares (for files larger than about # 12GiB). See #439 for details. def _get_maxsize(server): - v0 = server.get_rref().version + v0 = server.get_version() v1 = v0["http://allmydata.org/tahoe/protocols/storage/v1"] return v1["maximum-immutable-share-size"] @@ -945,8 +946,7 @@ class EncryptAnUploadable(object): d = self.original.get_encryption_key() def _got(key): - e = AES(key) - self._encryptor = e + self._encryptor = aes.create_encryptor(key) storage_index = storage_index_hash(key) assert isinstance(storage_index, str) @@ -956,7 +956,7 @@ class EncryptAnUploadable(object): self._storage_index = storage_index if self._status: self._status.set_storage_index(storage_index) - return e + return self._encryptor d.addCallback(_got) return d @@ -1063,11 +1063,11 @@ class EncryptAnUploadable(object): self._plaintext_hasher.update(chunk) self._update_segment_hash(chunk) # TODO: we have to encrypt the data (even if hash_only==True) - # because pycryptopp's AES-CTR implementation doesn't offer a - # way to change the counter value. Once pycryptopp acquires + # because the AES-CTR implementation doesn't offer a + # way to change the counter value. Once it acquires # this ability, change this to simply update the counter - # before each call to (hash_only==False) _encryptor.process() - ciphertext = self._encryptor.process(chunk) + # before each call to (hash_only==False) encrypt_data + ciphertext = aes.encrypt_data(self._encryptor, chunk) if hash_only: self.log(" skipping encryption", level=log.NOISY) else: @@ -1355,7 +1355,7 @@ def read_this_many_bytes(uploadable, size, prepend_data=[]): d.addCallback(_got) return d -class LiteralUploader: +class LiteralUploader(object): def __init__(self, progress=None): self._status = s = UploadStatus() @@ -1477,7 +1477,7 @@ class RemoteEncryptedUploadable(Referenceable): return self._eu.close() -class AssistedUploader: +class AssistedUploader(object): def __init__(self, helper, storage_broker): self._helper = helper @@ -1632,7 +1632,7 @@ class AssistedUploader: def get_upload_status(self): return self._upload_status -class BaseUploadable: +class BaseUploadable(object): # this is overridden by max_segment_size default_max_segment_size = DEFAULT_MAX_SEGMENT_SIZE default_params_set = False diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index aced2b546..b2ae0196d 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,5 +1,8 @@ from zope.interface import Interface, Attribute +from twisted.plugin import ( + IPlugin, +) from foolscap.api import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \ ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable @@ -291,6 +294,81 @@ class RIStorageServer(RemoteInterface): """ +class IStorageServer(Interface): + """ + An object capable of storing shares for a storage client. + """ + def get_version(): + """ + :see: ``RIStorageServer.get_version`` + """ + + def allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ): + """ + :see: ``RIStorageServer.allocate_buckets`` + """ + + def add_lease( + storage_index, + renew_secret, + cancel_secret, + ): + """ + :see: ``RIStorageServer.add_lease`` + """ + + def renew_lease( + storage_index, + renew_secret, + ): + """ + :see: ``RIStorageServer.renew_lease`` + """ + + def get_buckets( + storage_index, + ): + """ + :see: ``RIStorageServer.get_buckets`` + """ + + def slot_readv( + storage_index, + shares, + readv, + ): + """ + :see: ``RIStorageServer.slot_readv`` + """ + + def slot_testv_and_readv_and_writev( + storage_index, + secrets, + tw_vectors, + r_vector, + ): + """ + :see: ``RIStorageServer.slot_testv_readv_and_writev`` + """ + + def advise_corrupt_share( + share_type, + storage_index, + shnum, + reason, + ): + """ + :see: ``RIStorageServer.advise_corrupt_share`` + """ + + class IStorageBucketWriter(Interface): """ Objects of this kind live on the client side. @@ -463,7 +541,9 @@ class IServer(IDisplayableServer): pass def get_rref(): - """Once a server is connected, I return a RemoteReference. + """Obsolete. Use ``get_storage_server`` instead. + + Once a server is connected, I return a RemoteReference. Before a server is connected for the first time, I return None. Note that the rref I return will start producing DeadReferenceErrors @@ -476,6 +556,15 @@ class IServer(IDisplayableServer): otherwise. """ + def get_storage_server(): + """ + Once a server is connected, I return an ``IStorageServer``. + Before a server is connected for the first time, I return None. + + Note that the ``IStorageServer`` I return will start producing + DeadReferenceErrors once the connection is lost. + """ + class IMutableSlotWriter(Interface): """ @@ -2942,3 +3031,131 @@ class IConnectionStatus(Interface): (pending, connected, refused, or other errors). """) + + +class IFoolscapStoragePlugin(IPlugin): + """ + An ``IStoragePlugin`` provides client- and server-side implementations of + a Foolscap-based protocol which can be used to store and retrieve data. + + Implementations are free to apply access control or authorization policies + to this storage service and doing so is a large part of the motivation for + providing this point of pluggability. + + There should be enough information and hook points to support at + least these use-cases: + + - anonymous, everything allowed (current default) + - "storage club" / "friend-net" (possibly identity based) + - cryptocurrencies (ideally, paying for each API call) + - anonymous tokens (payment for service, but without identities) + """ + name = Attribute( + """ + A name for referring to this plugin. This name is both user-facing + (for example, it is written in configuration files) and machine-facing + (for example, it may be used to construct URLs). It should be unique + across all plugins for this interface. Two plugins with the same name + cannot be used in one client. + + Because it is used to construct URLs, it is constrained to URL safe + characters (it must be a *segment* as defined by RFC 3986, section + 3.3). + + :type: ``unicode`` + """ + ) + + def get_storage_server(configuration, get_anonymous_storage_server): + """ + Get an ``IAnnounceableStorageServer`` provider that gives an announcement + for and an implementation of the server side of the storage protocol. + This will be exposed and offered to clients in the storage server's + announcement. + + :param dict configuration: Any configuration given in the section for + this plugin in the node's configuration file. As an example, the + configuration for the original anonymous-access filesystem-based + storage server might look like:: + + {u"storedir": u"/foo/bar/storage", + u"nodeid": u"abcdefg...", + u"reserved_space": 0, + u"discard_storage": False, + u"readonly_storage": False, + u"expiration_enabled": False, + u"expiration_mode": u"age", + u"expiration_override_lease_duration": None, + u"expiration_cutoff_date": None, + u"expiration_sharetypes": (u"mutable, u"immutable"), + } + + :param get_anonymous_storage_server: A no-argument callable which + returns a single instance of the original, anonymous-access + storage server. This may be helpful in providing actual storage + implementation behavior for a wrapper-style plugin. This is also + provided to keep the Python API offered by Tahoe-LAFS to plugin + developers narrow (do not try to find and instantiate the original + storage server yourself; if you want it, call this). + + :rtype: ``Deferred`` firing with ``IAnnounceableStorageServer`` + """ + + def get_storage_client(configuration, announcement, get_rref): + """ + Get an ``IStorageServer`` provider that implements the client side of the + storage protocol. + + :param allmydata.node._Config configuration: A representation of the + configuration for the node into which this plugin has been loaded. + + :param dict announcement: The announcement for the corresponding + server portion of this plugin received from a storage server which + is offering it. + + :param get_rref: A no-argument callable which returns a + ``foolscap.referenceable.RemoteReference`` which refers to the + server portion of this plugin on the currently active connection, + or ``None`` if no connection has been established yet. + + :rtype: ``IStorageServer`` + """ + + def get_client_resource(configuration): + """ + Get an ``IResource`` that can be published in the Tahoe-LAFS web interface + to expose information related to this plugin. + + :param allmydata.node._Config configuration: A representation of the + configuration for the node into which this plugin has been loaded. + + :rtype: ``IResource`` + """ + + +class IAnnounceableStorageServer(Interface): + announcement = Attribute( + """ + Data for an announcement for the associated storage server. + + :note: This does not include the storage server nickname nor Foolscap + fURL. These will be added to the announcement automatically. It + may be usual for this announcement to contain no information. + Once the client connects to this server it can use other methods + to query for additional information (eg, in the manner of + ``RIStorageServer.remote_get_version``). The announcement only + needs to contain information to help the client determine how to + connect. + + :type: ``dict`` of JSON-serializable types + """ + ) + + storage_server = Attribute( + """ + A Foolscap referenceable object implementing the server side of the + storage protocol. + + :type: ``IReferenceable`` provider + """ + ) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 4d41b5655..66cf9f8c9 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -1,4 +1,3 @@ - import time from zope.interface import implementer from twisted.application import service @@ -10,7 +9,7 @@ from allmydata.introducer.common import sign_to_foolscap, unsign_from_foolscap,\ get_tubid_string_from_ann from allmydata.util import log, yamlutil, connection_status from allmydata.util.rrefutil import add_version_to_remote_reference -from allmydata.util.keyutil import BadSignatureError +from allmydata.crypto.error import BadSignature from allmydata.util.assertutil import precondition class InvalidCacheError(Exception): @@ -240,7 +239,7 @@ class IntroducerClient(service.Service, Referenceable): ann, key_s = unsign_from_foolscap(ann_t) # key is "v0-base32abc123" precondition(isinstance(key_s, str), key_s) - except BadSignatureError: + except BadSignature: self.log("bad signature on inbound announcement: %s" % (ann_t,), parent=lp, level=log.WEIRD, umid="ZAU15Q") # process other announcements that arrived with the bad one diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index a1b47e173..abc0811f0 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -1,7 +1,9 @@ - import re import json -from allmydata.util import keyutil, base32, rrefutil +from allmydata.crypto.util import remove_prefix +from allmydata.crypto import ed25519 +from allmydata.util import base32, rrefutil + def get_tubid_string_from_ann(ann): return get_tubid_string(str(ann.get("anonymous-storage-FURL") @@ -13,35 +15,51 @@ def get_tubid_string(furl): return m.group(1).lower() -def sign_to_foolscap(ann, sk): +def sign_to_foolscap(announcement, signing_key): + """ + :param signing_key: a (private) signing key, as returned from + e.g. :func:`allmydata.crypto.ed25519.signing_keypair_from_string` + + :returns: 3-tuple of (msg, sig, vk) where msg is a UTF8 JSON + serialization of the `announcement` (bytes), sig is bytes (a + signature of msg) and vk is the verifying key bytes + """ # return (bytes, sig-str, pubkey-str). A future HTTP-based serialization # will use JSON({msg:b64(JSON(msg).utf8), sig:v0-b64(sig), # pubkey:v0-b64(pubkey)}) . - msg = json.dumps(ann).encode("utf-8") - sig = "v0-"+base32.b2a(sk.sign(msg)) - vk_bytes = sk.get_verifying_key_bytes() - ann_t = (msg, sig, "v0-"+base32.b2a(vk_bytes)) + msg = json.dumps(announcement).encode("utf-8") + sig = b"v0-" + base32.b2a( + ed25519.sign_data(signing_key, msg) + ) + verifying_key_string = ed25519.string_from_verifying_key( + ed25519.verifying_key_from_signing_key(signing_key) + ) + ann_t = (msg, sig, remove_prefix(verifying_key_string, b"pub-")) return ann_t + class UnknownKeyError(Exception): pass + def unsign_from_foolscap(ann_t): (msg, sig_vs, claimed_key_vs) = ann_t if not sig_vs or not claimed_key_vs: raise UnknownKeyError("only signed announcements recognized") - if not sig_vs.startswith("v0-"): + if not sig_vs.startswith(b"v0-"): raise UnknownKeyError("only v0- signatures recognized") - if not claimed_key_vs.startswith("v0-"): + if not claimed_key_vs.startswith(b"v0-"): raise UnknownKeyError("only v0- keys recognized") - claimed_key = keyutil.parse_pubkey("pub-"+claimed_key_vs) - sig_bytes = base32.a2b(keyutil.remove_prefix(sig_vs, "v0-")) - claimed_key.verify(sig_bytes, msg) + + claimed_key = ed25519.verifying_key_from_string(b"pub-" + claimed_key_vs) + sig_bytes = base32.a2b(remove_prefix(sig_vs, b"v0-")) + ed25519.verify_signature(claimed_key, sig_bytes, msg) key_vs = claimed_key_vs ann = json.loads(msg.decode("utf-8")) return (ann, key_vs) -class SubscriberDescriptor: + +class SubscriberDescriptor(object): """This describes a subscriber, for status display purposes. It contains the following attributes: @@ -65,7 +83,7 @@ class SubscriberDescriptor: self.remote_address = remote_address self.tubid = tubid -class AnnouncementDescriptor: +class AnnouncementDescriptor(object): """This describes an announcement, for status display purposes. It contains the following attributes, which will be empty ("" for strings) if the client did not provide them: diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 00fac36a7..0a933bd01 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -30,10 +30,7 @@ are set to disallow users other than its owner from reading the contents of the files. See the 'configuration.rst' documentation file for details. """ - -def _valid_config_sections(): - return node._common_config_sections() - +_valid_config = node._common_valid_config class FurlFileConflictError(Exception): pass @@ -52,7 +49,7 @@ def create_introducer(basedir=u"."): config = read_config( basedir, u"client.port", generated_files=["introducer.furl"], - _valid_config_sections=_valid_config_sections, + _valid_config=_valid_config(), ) i2p_provider = create_i2p_provider(reactor, config) @@ -229,7 +226,7 @@ class IntroducerService(service.MultiService, Referenceable): self._debug_counts["inbound_message"] += 1 self.log("introducer: announcement published: %s" % (ann_t,), umid="wKHgCw") - ann, key = unsign_from_foolscap(ann_t) # might raise BadSignatureError + ann, key = unsign_from_foolscap(ann_t) # might raise BadSignature service_name = str(ann["service-name"]) index = (service_name, key) diff --git a/src/allmydata/magicfolderdb.py b/src/allmydata/magicfolderdb.py deleted file mode 100644 index 9b09f6c95..000000000 --- a/src/allmydata/magicfolderdb.py +++ /dev/null @@ -1,204 +0,0 @@ -from __future__ import print_function - -import sys -from collections import namedtuple - -from allmydata.util.dbutil import get_db, DBError -from allmydata.util.eliotutil import ( - RELPATH, - VERSION, - LAST_UPLOADED_URI, - LAST_DOWNLOADED_URI, - LAST_DOWNLOADED_TIMESTAMP, - PATHINFO, - validateSetMembership, - validateInstanceOf, -) -from eliot import ( - Field, - ActionType, -) - -PathEntry = namedtuple('PathEntry', 'size mtime_ns ctime_ns version last_uploaded_uri ' - 'last_downloaded_uri last_downloaded_timestamp') - -PATHENTRY = Field( - u"pathentry", - lambda v: None if v is None else { - "size": v.size, - "mtime_ns": v.mtime_ns, - "ctime_ns": v.ctime_ns, - "version": v.version, - "last_uploaded_uri": v.last_uploaded_uri, - "last_downloaded_uri": v.last_downloaded_uri, - "last_downloaded_timestamp": v.last_downloaded_timestamp, - }, - u"The local database state of a file.", - validateInstanceOf((type(None), PathEntry)), -) - -_INSERT_OR_UPDATE = Field.for_types( - u"insert_or_update", - [unicode], - u"An indication of whether the record for this upload was new or an update to a previous entry.", - validateSetMembership({u"insert", u"update"}), -) - -UPDATE_ENTRY = ActionType( - u"magic-folder-db:update-entry", - [RELPATH, VERSION, LAST_UPLOADED_URI, LAST_DOWNLOADED_URI, LAST_DOWNLOADED_TIMESTAMP, PATHINFO], - [_INSERT_OR_UPDATE], - u"Record some metadata about a relative path in the magic-folder.", -) - - -# magic-folder db schema version 1 -SCHEMA_v1 = """ -CREATE TABLE version -( - version INTEGER -- contains one row, set to 1 -); - -CREATE TABLE local_files -( - path VARCHAR(1024) PRIMARY KEY, -- UTF-8 filename relative to local magic folder dir - size INTEGER, -- ST_SIZE, or NULL if the file has been deleted - mtime_ns INTEGER, -- ST_MTIME in nanoseconds - ctime_ns INTEGER, -- ST_CTIME in nanoseconds - version INTEGER, - last_uploaded_uri VARCHAR(256), -- URI:CHK:... - last_downloaded_uri VARCHAR(256), -- URI:CHK:... - last_downloaded_timestamp TIMESTAMP -); -""" - - -def get_magicfolderdb(dbfile, stderr=sys.stderr, - create_version=(SCHEMA_v1, 1), just_create=False): - # Open or create the given backupdb file. The parent directory must - # exist. - try: - (sqlite3, db) = get_db(dbfile, stderr, create_version, - just_create=just_create, dbname="magicfolderdb") - if create_version[1] in (1, 2): - return MagicFolderDB(sqlite3, db) - else: - print("invalid magicfolderdb schema version specified", file=stderr) - return None - except DBError as e: - print(e, file=stderr) - return None - -class LocalPath(object): - @classmethod - def fromrow(self, row): - p = LocalPath() - p.relpath_u = row[0] - p.entry = PathEntry(*row[1:]) - return p - - -class MagicFolderDB(object): - VERSION = 1 - - def __init__(self, sqlite_module, connection): - self.sqlite_module = sqlite_module - self.connection = connection - self.cursor = connection.cursor() - - def close(self): - self.connection.close() - - def get_db_entry(self, relpath_u): - """ - Retrieve the entry in the database for a given path, or return None - if there is no such entry. - """ - c = self.cursor - c.execute("SELECT size, mtime_ns, ctime_ns, version, last_uploaded_uri," - " last_downloaded_uri, last_downloaded_timestamp" - " FROM local_files" - " WHERE path=?", - (relpath_u,)) - row = self.cursor.fetchone() - if not row: - return None - else: - (size, mtime_ns, ctime_ns, version, last_uploaded_uri, - last_downloaded_uri, last_downloaded_timestamp) = row - return PathEntry(size=size, mtime_ns=mtime_ns, ctime_ns=ctime_ns, version=version, - last_uploaded_uri=last_uploaded_uri, - last_downloaded_uri=last_downloaded_uri, - last_downloaded_timestamp=last_downloaded_timestamp) - - def get_direct_children(self, relpath_u): - """ - Given the relative path to a directory, return ``LocalPath`` instances - representing all direct children of that directory. - """ - # It would be great to not be interpolating data into query - # statements. However, query parameters are not supported in the - # position where we need them. - sqlitesafe_relpath_u = relpath_u.replace(u"'", u"''") - statement = ( - """ - SELECT - path, size, mtime_ns, ctime_ns, version, last_uploaded_uri, - last_downloaded_uri, last_downloaded_timestamp - FROM - local_files - WHERE - -- The "_" used here ensures there is at least one character - -- after the /. This prevents matching the path itself. - path LIKE '{path}/_%' AND - - -- The "_" used here serves a similar purpose. This allows - -- matching directory children but avoids matching their - -- children. - path NOT LIKE '{path}/_%/_%' - """ - ).format(path=sqlitesafe_relpath_u) - - self.cursor.execute(statement) - rows = self.cursor.fetchall() - return list( - LocalPath.fromrow(row) - for row - in rows - ) - - def get_all_relpaths(self): - """ - Retrieve a set of all relpaths of files that have had an entry in magic folder db - (i.e. that have been downloaded at least once). - """ - self.cursor.execute("SELECT path FROM local_files") - rows = self.cursor.fetchall() - return set([r[0] for r in rows]) - - def did_upload_version(self, relpath_u, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, pathinfo): - action = UPDATE_ENTRY( - relpath=relpath_u, - version=version, - last_uploaded_uri=last_uploaded_uri, - last_downloaded_uri=last_downloaded_uri, - last_downloaded_timestamp=last_downloaded_timestamp, - pathinfo=pathinfo, - ) - with action: - try: - self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?,?,?)", - (relpath_u, pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns, - version, last_uploaded_uri, last_downloaded_uri, - last_downloaded_timestamp)) - action.add_success_fields(insert_or_update=u"insert") - except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): - self.cursor.execute("UPDATE local_files" - " SET size=?, mtime_ns=?, ctime_ns=?, version=?, last_uploaded_uri=?," - " last_downloaded_uri=?, last_downloaded_timestamp=?" - " WHERE path=?", - (pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns, version, - last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, - relpath_u)) - action.add_success_fields(insert_or_update=u"update") - self.connection.commit() diff --git a/src/allmydata/magicpath.py b/src/allmydata/magicpath.py deleted file mode 100644 index 1f46330d4..000000000 --- a/src/allmydata/magicpath.py +++ /dev/null @@ -1,32 +0,0 @@ -import re -import os.path - -from allmydata.util.assertutil import precondition, _assert - -def path2magic(path): - return re.sub(u'[/@]', lambda m: {u'/': u'@_', u'@': u'@@'}[m.group(0)], path) - -def magic2path(path): - return re.sub(u'@[_@]', lambda m: {u'@_': u'/', u'@@': u'@'}[m.group(0)], path) - - -IGNORE_SUFFIXES = [u'.backup', u'.tmp', u'.conflict'] -IGNORE_PREFIXES = [u'.'] - -def should_ignore_file(path_u): - precondition(isinstance(path_u, unicode), path_u=path_u) - - for suffix in IGNORE_SUFFIXES: - if path_u.endswith(suffix): - return True - - while path_u != u"": - oldpath_u = path_u - path_u, tail_u = os.path.split(path_u) - if tail_u.startswith(u"."): - return True - if path_u == oldpath_u: - return True # the path was absolute - _assert(len(path_u) < len(oldpath_u), path_u=path_u, oldpath_u=oldpath_u) - - return False diff --git a/src/allmydata/mutable/checker.py b/src/allmydata/mutable/checker.py index cf073b80b..6e083f8f5 100644 --- a/src/allmydata/mutable/checker.py +++ b/src/allmydata/mutable/checker.py @@ -8,7 +8,7 @@ from allmydata.mutable.common import MODE_CHECK, MODE_WRITE, CorruptShareError from allmydata.mutable.servermap import ServerMap, ServermapUpdater from allmydata.mutable.retrieve import Retrieve # for verifying -class MutableChecker: +class MutableChecker(object): SERVERMAP_MODE = MODE_CHECK def __init__(self, node, storage_broker, history, monitor): diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index a1604b180..48ce5d8b7 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1,9 +1,11 @@ - import random from zope.interface import implementer from twisted.internet import defer, reactor from foolscap.api import eventually + +from allmydata.crypto import aes +from allmydata.crypto import rsa from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \ NotEnoughSharesError, MDMF_VERSION, SDMF_VERSION, IMutableUploadable, \ IMutableFileVersion, IWriteable @@ -12,8 +14,6 @@ from allmydata.util.assertutil import precondition from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \ WriteableMDMFFileURI, ReadonlyMDMFFileURI from allmydata.monitor import Monitor -from pycryptopp.cipher.aes import AES - from allmydata.mutable.publish import Publish, MutableData,\ TransformingUploadable from allmydata.mutable.common import MODE_READ, MODE_WRITE, MODE_CHECK, UnrecoverableFileError, \ @@ -24,7 +24,7 @@ from allmydata.mutable.checker import MutableChecker, MutableCheckAndRepairer from allmydata.mutable.repairer import Repairer -class BackoffAgent: +class BackoffAgent(object): # these parameters are copied from foolscap.reconnector, which gets them # from twisted.internet.protocol.ReconnectingClientFactory initialDelay = 1.0 @@ -129,8 +129,8 @@ class MutableFileNode(object): """ (pubkey, privkey) = keypair self._pubkey, self._privkey = pubkey, privkey - pubkey_s = self._pubkey.serialize() - privkey_s = self._privkey.serialize() + pubkey_s = rsa.der_string_from_verifying_key(self._pubkey) + privkey_s = rsa.der_string_from_signing_key(self._privkey) self._writekey = hashutil.ssk_writekey_hash(privkey_s) self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s) self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) @@ -160,13 +160,13 @@ class MutableFileNode(object): return contents(self) def _encrypt_privkey(self, writekey, privkey): - enc = AES(writekey) - crypttext = enc.process(privkey) + encryptor = aes.create_encryptor(writekey) + crypttext = aes.encrypt_data(encryptor, privkey) return crypttext def _decrypt_privkey(self, enc_privkey): - enc = AES(self._writekey) - privkey = enc.process(enc_privkey) + decryptor = aes.create_decryptor(self._writekey) + privkey = aes.decrypt_data(decryptor, enc_privkey) return privkey def _populate_pubkey(self, pubkey): diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index dbfde5f68..20f2df3aa 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -230,7 +230,7 @@ class SDMFSlotWriteProxy(object): """ def __init__(self, shnum, - rref, # a remote reference to a storage server + storage_server, # an IStorageServer storage_index, secrets, # (write_enabler, renew_secret, cancel_secret) seqnum, # the sequence number of the mutable file @@ -239,7 +239,7 @@ class SDMFSlotWriteProxy(object): segment_size, data_length): # the length of the original file self.shnum = shnum - self._rref = rref + self._storage_server = storage_server self._storage_index = storage_index self._secrets = secrets self._seqnum = seqnum @@ -541,12 +541,13 @@ class SDMFSlotWriteProxy(object): tw_vectors = {} tw_vectors[self.shnum] = (self._testvs, datavs, None) - return self._rref.callRemote("slot_testv_and_readv_and_writev", - self._storage_index, - self._secrets, - tw_vectors, - # TODO is it useful to read something? - self._readvs) + return self._storage_server.slot_testv_and_readv_and_writev( + self._storage_index, + self._secrets, + tw_vectors, + # TODO is it useful to read something? + self._readvs, + ) MDMFHEADER = ">BQ32sBBQQ QQQQQQQQ" @@ -729,7 +730,7 @@ class MDMFSlotWriteProxy(object): # disruption. def __init__(self, shnum, - rref, # a remote reference to a storage server + storage_server, # a remote reference to a storage server storage_index, secrets, # (write_enabler, renew_secret, cancel_secret) seqnum, # the sequence number of the mutable file @@ -738,7 +739,7 @@ class MDMFSlotWriteProxy(object): segment_size, data_length): # the length of the original file self.shnum = shnum - self._rref = rref + self._storage_server = storage_server self._storage_index = storage_index self._seqnum = seqnum self._required_shares = required_shares @@ -1159,11 +1160,12 @@ class MDMFSlotWriteProxy(object): self._testvs = [(0, len(new_checkstring), "eq", new_checkstring)] on_success = _first_write tw_vectors[self.shnum] = (self._testvs, datavs, None) - d = self._rref.callRemote("slot_testv_and_readv_and_writev", - self._storage_index, - self._secrets, - tw_vectors, - self._readv) + d = self._storage_server.slot_testv_and_readv_and_writev( + self._storage_index, + self._secrets, + tw_vectors, + self._readv, + ) def _result(results): if isinstance(results, failure.Failure) or not results[0]: # Do nothing; the write was unsuccessful. @@ -1180,7 +1182,7 @@ def _handle_bad_struct(f): f.trap(struct.error) raise BadShareError(f.value.args[0]) -class MDMFSlotReadProxy: +class MDMFSlotReadProxy(object): """ I read from a mutable slot filled with data written in the MDMF data format (which is described above). @@ -1189,13 +1191,13 @@ class MDMFSlotReadProxy: it is valid) to eliminate some of the need to fetch it from servers. """ def __init__(self, - rref, + storage_server, storage_index, shnum, data="", data_is_everything=False): # Start the initialization process. - self._rref = rref + self._storage_server = storage_server self._storage_index = storage_index self.shnum = shnum @@ -1752,10 +1754,11 @@ class MDMFSlotReadProxy: results = {self.shnum: results} return defer.succeed(results) else: - return self._rref.callRemote("slot_readv", - self._storage_index, - [self.shnum], - readvs) + return self._storage_server.slot_readv( + self._storage_index, + [self.shnum], + readvs, + ) def is_sdmf(self): diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 8810bdf14..aa97c2242 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -4,15 +4,16 @@ from itertools import count from zope.interface import implementer from twisted.internet import defer from twisted.python import failure + +from allmydata.crypto import aes +from allmydata.crypto import rsa from allmydata.interfaces import IPublishStatus, SDMF_VERSION, MDMF_VERSION, \ IMutableUploadable from allmydata.util import base32, hashutil, mathutil, log from allmydata.util.dictutil import DictOfSets from allmydata import hashtree, codec from allmydata.storage.server import si_b2a -from pycryptopp.cipher.aes import AES from foolscap.api import eventually, fireEventually - from allmydata.mutable.common import MODE_WRITE, MODE_CHECK, MODE_REPAIR, \ UncoordinatedWriteError, NotEnoughServersError from allmydata.mutable.servermap import ServerMap @@ -100,7 +101,7 @@ class PublishStatus(object): class LoopLimitExceededError(Exception): pass -class Publish: +class Publish(object): """I represent a single act of publishing the mutable file to the grid. I will only publish my data if the servermap I am using still represents the current state of the world. @@ -269,7 +270,7 @@ class Publish: secrets = (write_enabler, renew_secret, cancel_secret) writer = writer_class(shnum, - server.get_rref(), + server.get_storage_server(), self._storage_index, secrets, self._new_seqnum, @@ -471,7 +472,7 @@ class Publish: secrets = (write_enabler, renew_secret, cancel_secret) writer = writer_class(shnum, - server.get_rref(), + server.get_storage_server(), self._storage_index, secrets, self._new_seqnum, @@ -711,8 +712,8 @@ class Publish: key = hashutil.ssk_readkey_data_hash(salt, self.readkey) self._status.set_status("Encrypting") - enc = AES(key) - crypttext = enc.process(data) + encryptor = aes.create_encryptor(key) + crypttext = aes.encrypt_data(encryptor, data) assert len(crypttext) == len(data) now = time.time() @@ -849,7 +850,7 @@ class Publish: started = time.time() self._status.set_status("Signing prefix") signable = self._get_some_writer().get_signable() - self.signature = self._privkey.sign(signable) + self.signature = rsa.sign_data(self._privkey, signable) for (shnum, writers) in self.writers.iteritems(): for writer in writers: @@ -864,7 +865,7 @@ class Publish: self._status.set_status("Pushing shares") self._started_pushing = started ds = [] - verification_key = self._pubkey.serialize() + verification_key = rsa.der_string_from_verifying_key(self._pubkey) for (shnum, writers) in self.writers.copy().iteritems(): for writer in writers: diff --git a/src/allmydata/mutable/repairer.py b/src/allmydata/mutable/repairer.py index 2f9377fc7..261ca9633 100644 --- a/src/allmydata/mutable/repairer.py +++ b/src/allmydata/mutable/repairer.py @@ -24,7 +24,7 @@ class RepairRequiresWritecapError(Exception): class MustForceRepairError(Exception): pass -class Repairer: +class Repairer(object): def __init__(self, node, check_results, storage_broker, history, monitor): self.node = node self.check_results = ICheckResults(check_results) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 3771a235c..b2d234a15 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -1,5 +1,5 @@ - import time + from itertools import count from zope.interface import implementer from twisted.internet import defer @@ -8,6 +8,8 @@ from twisted.internet.interfaces import IPushProducer, IConsumer from foolscap.api import eventually, fireEventually, DeadReferenceError, \ RemoteException +from allmydata.crypto import aes +from allmydata.crypto import rsa from allmydata.interfaces import IRetrieveStatus, NotEnoughSharesError, \ DownloadStopped, MDMF_VERSION, SDMF_VERSION from allmydata.util.assertutil import _assert, precondition @@ -15,8 +17,6 @@ from allmydata.util import hashutil, log, mathutil, deferredutil from allmydata.util.dictutil import DictOfSets from allmydata import hashtree, codec from allmydata.storage.server import si_b2a -from pycryptopp.cipher.aes import AES -from pycryptopp.publickey import rsa from allmydata.mutable.common import CorruptShareError, BadShareError, \ UncoordinatedWriteError @@ -89,7 +89,7 @@ class RetrieveStatus(object): serverid = server.get_serverid() self._problems[serverid] = f -class Marker: +class Marker(object): pass @implementer(IPushProducer) @@ -309,7 +309,7 @@ class Retrieve(object): if key in self.servermap.proxies: reader = self.servermap.proxies[key] else: - reader = MDMFSlotReadProxy(server.get_rref(), + reader = MDMFSlotReadProxy(server.get_storage_server(), self._storage_index, shnum, None) reader.server = server self.readers[shnum] = reader @@ -899,16 +899,20 @@ class Retrieve(object): self.log("decrypting segment %d" % self._current_segment) started = time.time() key = hashutil.ssk_readkey_data_hash(salt, self._node.get_readkey()) - decryptor = AES(key) - plaintext = decryptor.process(segment) + decryptor = aes.create_decryptor(key) + plaintext = aes.decrypt_data(decryptor, segment) self._status.accumulate_decrypt_time(time.time() - started) return plaintext def notify_server_corruption(self, server, shnum, reason): - rref = server.get_rref() - rref.callRemoteOnly("advise_corrupt_share", - "mutable", self._storage_index, shnum, reason) + storage_server = server.get_storage_server() + storage_server.advise_corrupt_share( + "mutable", + self._storage_index, + shnum, + reason, + ) def _try_to_validate_privkey(self, enc_privkey, reader, server): @@ -931,13 +935,11 @@ class Retrieve(object): # it's good self.log("got valid privkey from shnum %d on reader %s" % (reader.shnum, reader)) - privkey = rsa.create_signing_key_from_string(alleged_privkey_s) + privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) self._node._populate_encprivkey(enc_privkey) self._node._populate_privkey(privkey) self._need_privkey = False - - def _done(self): """ I am called by _download_current_segment when the download process @@ -968,7 +970,6 @@ class Retrieve(object): self._consumer.unregisterProducer() eventually(self._done_deferred.callback, ret) - def _raise_notenoughshareserror(self): """ I am called when there are not enough active servers left to complete diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index 4081df65f..8dba6d8b5 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -8,11 +8,12 @@ from twisted.internet import defer from twisted.python import failure from foolscap.api import DeadReferenceError, RemoteException, eventually, \ fireEventually +from allmydata.crypto.error import BadSignature +from allmydata.crypto import rsa from allmydata.util import base32, hashutil, log, deferredutil from allmydata.util.dictutil import DictOfSets from allmydata.storage.server import si_b2a from allmydata.interfaces import IServermapUpdaterStatus -from pycryptopp.publickey import rsa from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \ MODE_READ, MODE_REPAIR, CorruptShareError @@ -80,7 +81,7 @@ class UpdateStatus(object): def set_finished(self, when): self.finished = when -class ServerMap: +class ServerMap(object): """I record the placement of mutable shares. This object records which shares (of various versions) are located on @@ -378,7 +379,7 @@ class ServerMap: self.update_data.setdefault(shnum , []).append((verinfo, data)) -class ServermapUpdater: +class ServermapUpdater(object): def __init__(self, filenode, storage_broker, monitor, servermap, mode=MODE_READ, add_lease=False, update_range=None): """I update a servermap, locating a sufficient number of useful @@ -592,7 +593,7 @@ class ServermapUpdater: return d def _do_read(self, server, storage_index, shnums, readv): - ss = server.get_rref() + ss = server.get_storage_server() if self._add_lease: # send an add-lease message in parallel. The results are handled # separately. This is sent before the slot_readv() so that we can @@ -601,11 +602,14 @@ class ServermapUpdater: # add_lease is synchronous). renew_secret = self._node.get_renewal_secret(server) cancel_secret = self._node.get_cancel_secret(server) - d2 = ss.callRemote("add_lease", storage_index, - renew_secret, cancel_secret) + d2 = ss.add_lease( + storage_index, + renew_secret, + cancel_secret, + ) # we ignore success d2.addErrback(self._add_lease_failed, server, storage_index) - d = ss.callRemote("slot_readv", storage_index, shnums, readv) + d = ss.slot_readv(storage_index, shnums, readv) return d @@ -638,7 +642,7 @@ class ServermapUpdater: lp = self.log(format="got result from [%(name)s], %(numshares)d shares", name=server.get_name(), numshares=len(datavs)) - ss = server.get_rref() + ss = server.get_storage_server() now = time.time() elapsed = now - started def _done_processing(ignored=None): @@ -796,9 +800,13 @@ class ServermapUpdater: def notify_server_corruption(self, server, shnum, reason): - rref = server.get_rref() - rref.callRemoteOnly("advise_corrupt_share", - "mutable", self._storage_index, shnum, reason) + ss = server.get_storage_server() + ss.advise_corrupt_share( + "mutable", + self._storage_index, + shnum, + reason, + ) def _got_signature_one_share(self, results, shnum, server, lp): @@ -836,8 +844,9 @@ class ServermapUpdater: # This is a new version tuple, and we need to validate it # against the public key before keeping track of it. assert self._node.get_pubkey() - valid = self._node.get_pubkey().verify(prefix, signature[1]) - if not valid: + try: + rsa.verify_signature(self._node.get_pubkey(), signature[1], prefix) + except BadSignature: raise CorruptShareError(server, shnum, "signature is invalid") @@ -906,12 +915,10 @@ class ServermapUpdater: verinfo, update_data) - def _deserialize_pubkey(self, pubkey_s): verifier = rsa.create_verifying_key_from_string(pubkey_s) return verifier - def _try_to_validate_privkey(self, enc_privkey, server, shnum, lp): """ Given a writekey from a remote server, I validate it against the @@ -930,7 +937,7 @@ class ServermapUpdater: self.log("got valid privkey from shnum %d on serverid %s" % (shnum, server.get_name()), parent=lp) - privkey = rsa.create_signing_key_from_string(alleged_privkey_s) + privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) self._node._populate_encprivkey(enc_privkey) self._node._populate_privkey(privkey) self._need_privkey = False @@ -1220,5 +1227,3 @@ class ServermapUpdater: def _fatal_error(self, f): self.log("fatal error", failure=f, level=log.WEIRD, umid="1cNvlw") self._done_deferred.errback(f) - - diff --git a/src/allmydata/node.py b/src/allmydata/node.py index ae62e2fc3..404f8a125 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -17,7 +17,7 @@ from twisted.application import service from twisted.python.failure import Failure from foolscap.api import Tub, app_versions import foolscap.logging.log -from allmydata import get_package_versions, get_package_versions_string +from allmydata.version_checks import get_package_versions, get_package_versions_string from allmydata.util import log from allmydata.util import fileutil, iputil from allmydata.util.assertutil import _assert @@ -25,8 +25,8 @@ from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.encodingutil import get_filesystem_encoding, quote_output from allmydata.util import configutil -def _common_config_sections(): - return { +def _common_valid_config(): + return configutil.ValidConfiguration({ "connections": ( "tcp", ), @@ -63,7 +63,7 @@ def _common_config_sections(): "onion.external_port", "onion.private_key_file", ), - } + }) # Add our application versions to the data that Foolscap's LogPublisher # reports. @@ -152,7 +152,7 @@ def create_node_dir(basedir, readme_text): f.write(readme_text) -def read_config(basedir, portnumfile, generated_files=[], _valid_config_sections=None): +def read_config(basedir, portnumfile, generated_files=[], _valid_config=None): """ Read and validate configuration. @@ -163,15 +163,14 @@ def read_config(basedir, portnumfile, generated_files=[], _valid_config_sections :param list generated_files: a list of automatically-generated configuration files. - :param dict _valid_config_sections: (internal use, optional) a - dict-of-dicts structure defining valid configuration sections and - keys + :param ValidConfiguration _valid_config: (internal use, optional) a + structure defining valid configuration sections and keys :returns: :class:`allmydata.node._Config` instance """ basedir = abspath_expanduser_unicode(unicode(basedir)) - if _valid_config_sections is None: - _valid_config_sections = _common_config_sections + if _valid_config is None: + _valid_config = _common_valid_config() # complain if there's bad stuff in the config dir _error_about_old_config_files(basedir, generated_files) @@ -188,7 +187,7 @@ def read_config(basedir, portnumfile, generated_files=[], _valid_config_sections if e.errno != errno.ENOENT: raise - configutil.validate_config(config_fname, parser, _valid_config_sections()) + configutil.validate_config(config_fname, parser, _valid_config) # make sure we have a private configuration area fileutil.make_dirs(os.path.join(basedir, "private"), 0o700) @@ -196,13 +195,17 @@ def read_config(basedir, portnumfile, generated_files=[], _valid_config_sections return _Config(parser, portnumfile, basedir, config_fname) -def config_from_string(basedir, portnumfile, config_str): +def config_from_string(basedir, portnumfile, config_str, _valid_config=None): """ - load configuration from in-memory string + load and validate configuration from in-memory string """ + if _valid_config is None: + _valid_config = _common_valid_config() + # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) + configutil.validate_config(fname, parser, _valid_config) def write_new_config(cfg): """ @@ -321,6 +324,14 @@ class _Config(object): pass return answer + def items(self, section, default=_None): + try: + return self.config.items(section) + except ConfigParser.NoSectionError: + if default is _None: + raise + return default + def get_config(self, section, option, default=_None, boolean=False): try: if boolean: diff --git a/src/allmydata/ported-modules.txt b/src/allmydata/ported-modules.txt new file mode 100644 index 000000000..cd150cbfb --- /dev/null +++ b/src/allmydata/ported-modules.txt @@ -0,0 +1 @@ +allmydata.util.namespace diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 47ca4a0eb..0ac3c6479 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -24,11 +24,11 @@ Generate a public/private keypair, dumped to stdout as two lines of ASCII.. return t def print_keypair(options): - from allmydata.util.keyutil import make_keypair + from allmydata.crypto import ed25519 out = options.stdout - privkey_vs, pubkey_vs = make_keypair() - print("private:", privkey_vs, file=out) - print("public:", pubkey_vs, file=out) + private_key, public_key = ed25519.create_signing_keypair() + print("private:", ed25519.string_from_signing_key(private_key), file=out) + print("public:", ed25519.string_from_verifying_key(public_key), file=out) class DerivePubkeyOptions(BaseOptions): def parseArgs(self, privkey): @@ -48,11 +48,11 @@ generate-keypair, derive the public key and print it to stdout. def derive_pubkey(options): out = options.stdout - from allmydata.util import keyutil + from allmydata.crypto import ed25519 privkey_vs = options.privkey - sk, pubkey_vs = keyutil.parse_privkey(privkey_vs) - print("private:", privkey_vs, file=out) - print("public:", pubkey_vs, file=out) + private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) + print("private:", ed25519.string_from_signing_key(private_key), file=out) + print("public:", ed25519.string_from_verifying_key(public_key), file=out) return 0 diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index b86a06edc..234f8524a 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -78,7 +78,7 @@ def get_backupdb(dbfile, stderr=sys.stderr, return None -class FileResult: +class FileResult(object): def __init__(self, bdb, filecap, should_check, path, mtime, ctime, size): self.bdb = bdb @@ -106,7 +106,7 @@ class FileResult: self.bdb.did_check_file_healthy(self.filecap, results) -class DirectoryResult: +class DirectoryResult(object): def __init__(self, bdb, dirhash, dircap, should_check): self.bdb = bdb self.dircap = dircap @@ -128,7 +128,7 @@ class DirectoryResult: self.bdb.did_check_directory_healthy(self.dircap, results) -class BackupDB_v2: +class BackupDB_v2(object): VERSION = 2 NO_CHECK_BEFORE = 1*MONTH ALWAYS_CHECK_AFTER = 2*MONTH diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 16c723cd8..31ef26510 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -259,10 +259,6 @@ class UnlinkOptions(FileStoreOptions): synopsis = "[options] REMOTE_FILE" description = "Remove a named file from its parent directory." -class RmOptions(UnlinkOptions): - synopsis = "[options] REMOTE_FILE" - description = "Remove a named file from its parent directory." - class MvOptions(FileStoreOptions): def parseArgs(self, frompath, topath): self.from_file = argv_to_unicode(frompath) @@ -467,7 +463,6 @@ subCommands = [ ["put", None, PutOptions, "Upload a file into the grid."], ["cp", None, CpOptions, "Copy one or more files or directories."], ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."], - ["rm", None, RmOptions, "Unlink a file or directory on the grid (same as unlink)."], ["mv", None, MvOptions, "Move a file within the grid."], ["ln", None, LnOptions, "Make an additional link to an existing file or directory."], ["backup", None, BackupOptions, "Make target dir look like local dir."], diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 4a94e5fd0..d3dde9a72 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -151,7 +151,7 @@ def get_aliases(nodedir): pass return aliases -class DefaultAliasMarker: +class DefaultAliasMarker(object): pass pretend_platform_uses_lettercolon = False # for tests diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 084530f21..2634e0915 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -180,6 +180,7 @@ class CreateNodeOptions(CreateClientOptions): optFlags = [ ("no-storage", None, "Do not offer storage service to other nodes."), ("storage-dir", None, "Path where the storage will be placed."), + ("helper", None, "Enable helper"), ] + TOR_FLAGS + I2P_FLAGS synopsis = "[options] [NODEDIR]" @@ -334,7 +335,10 @@ def write_client_config(c, config): c.write("[helper]\n") c.write("# Shall this node run a helper service that clients can use?\n") - c.write("enabled = false\n") + if config.get("helper"): + c.write("enabled = true\n") + else: + c.write("enabled = false\n") c.write("\n") diff --git a/src/allmydata/scripts/magic_folder_cli.py b/src/allmydata/scripts/magic_folder_cli.py deleted file mode 100644 index 4165dec49..000000000 --- a/src/allmydata/scripts/magic_folder_cli.py +++ /dev/null @@ -1,610 +0,0 @@ -from __future__ import print_function - -import os -import urllib -from types import NoneType -from six.moves import cStringIO as StringIO -from datetime import datetime -import json - - -from twisted.python import usage - -from allmydata.util.assertutil import precondition - -from .common import BaseOptions, BasedirOptions, get_aliases -from .cli import MakeDirectoryOptions, LnOptions, CreateAliasOptions -import tahoe_mv -from allmydata.util.encodingutil import argv_to_abspath, argv_to_unicode, to_str, \ - quote_local_unicode_path -from allmydata.scripts.common_http import do_http, BadResponse -from allmydata.util import fileutil -from allmydata import uri -from allmydata.util.abbreviate import abbreviate_space, abbreviate_time -from allmydata.frontends.magic_folder import load_magic_folders -from allmydata.frontends.magic_folder import save_magic_folders -from allmydata.frontends.magic_folder import maybe_upgrade_magic_folders - - -INVITE_SEPARATOR = "+" - -class CreateOptions(BasedirOptions): - nickname = None # NOTE: *not* the "name of this magic-folder" - local_dir = None - synopsis = "MAGIC_ALIAS: [NICKNAME LOCAL_DIR]" - optParameters = [ - ("poll-interval", "p", "60", "How often to ask for updates"), - ("name", "n", "default", "The name of this magic-folder"), - ] - description = ( - "Create a new magic-folder. If you specify NICKNAME and " - "LOCAL_DIR, this client will also be invited and join " - "using the given nickname. A new alias (see 'tahoe list-aliases') " - "will be added with the master folder's writecap." - ) - - def parseArgs(self, alias, nickname=None, local_dir=None): - BasedirOptions.parseArgs(self) - alias = argv_to_unicode(alias) - if not alias.endswith(u':'): - raise usage.UsageError("An alias must end with a ':' character.") - self.alias = alias[:-1] - self.nickname = None if nickname is None else argv_to_unicode(nickname) - try: - if int(self['poll-interval']) <= 0: - raise ValueError("should be positive") - except ValueError: - raise usage.UsageError( - "--poll-interval must be a positive integer" - ) - - # Expand the path relative to the current directory of the CLI command, not the node. - self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False) - - if self.nickname and not self.local_dir: - raise usage.UsageError("If NICKNAME is specified then LOCAL_DIR must also be specified.") - node_url_file = os.path.join(self['node-directory'], u"node.url") - self['node-url'] = fileutil.read(node_url_file).strip() - -def _delegate_options(source_options, target_options): - target_options.aliases = get_aliases(source_options['node-directory']) - target_options["node-url"] = source_options["node-url"] - target_options["node-directory"] = source_options["node-directory"] - target_options["name"] = source_options["name"] - target_options.stdin = StringIO("") - target_options.stdout = StringIO() - target_options.stderr = StringIO() - return target_options - -def create(options): - precondition(isinstance(options.alias, unicode), alias=options.alias) - precondition(isinstance(options.nickname, (unicode, NoneType)), nickname=options.nickname) - precondition(isinstance(options.local_dir, (unicode, NoneType)), local_dir=options.local_dir) - - # make sure we don't already have a magic-folder with this name before we create the alias - maybe_upgrade_magic_folders(options["node-directory"]) - folders = load_magic_folders(options["node-directory"]) - if options['name'] in folders: - print("Already have a magic-folder named '{}'".format(options['name']), file=options.stderr) - return 1 - - # create an alias; this basically just remembers the cap for the - # master directory - from allmydata.scripts import tahoe_add_alias - create_alias_options = _delegate_options(options, CreateAliasOptions()) - create_alias_options.alias = options.alias - - rc = tahoe_add_alias.create_alias(create_alias_options) - if rc != 0: - print(create_alias_options.stderr.getvalue(), file=options.stderr) - return rc - print(create_alias_options.stdout.getvalue(), file=options.stdout) - - if options.nickname is not None: - print(u"Inviting myself as client '{}':".format(options.nickname), file=options.stdout) - invite_options = _delegate_options(options, InviteOptions()) - invite_options.alias = options.alias - invite_options.nickname = options.nickname - invite_options['name'] = options['name'] - rc = invite(invite_options) - if rc != 0: - print(u"magic-folder: failed to invite after create\n", file=options.stderr) - print(invite_options.stderr.getvalue(), file=options.stderr) - return rc - invite_code = invite_options.stdout.getvalue().strip() - print(u" created invite code", file=options.stdout) - join_options = _delegate_options(options, JoinOptions()) - join_options['poll-interval'] = options['poll-interval'] - join_options.nickname = options.nickname - join_options.local_dir = options.local_dir - join_options.invite_code = invite_code - rc = join(join_options) - if rc != 0: - print(u"magic-folder: failed to join after create\n", file=options.stderr) - print(join_options.stderr.getvalue(), file=options.stderr) - return rc - print(u" joined new magic-folder", file=options.stdout) - print( - u"Successfully created magic-folder '{}' with alias '{}:' " - u"and client '{}'\nYou must re-start your node before the " - u"magic-folder will be active." - .format(options['name'], options.alias, options.nickname), file=options.stdout) - return 0 - - -class ListOptions(BasedirOptions): - description = ( - "List all magic-folders this client has joined" - ) - optFlags = [ - ("json", "", "Produce JSON output") - ] - - -def list_(options): - folders = load_magic_folders(options["node-directory"]) - if options["json"]: - _list_json(options, folders) - return 0 - _list_human(options, folders) - return 0 - - -def _list_json(options, folders): - """ - List our magic-folders using JSON - """ - info = dict() - for name, details in folders.items(): - info[name] = { - u"directory": details["directory"], - } - print(json.dumps(info), file=options.stdout) - return 0 - - -def _list_human(options, folders): - """ - List our magic-folders for a human user - """ - if folders: - print("This client has the following magic-folders:", file=options.stdout) - biggest = max([len(nm) for nm in folders.keys()]) - fmt = " {:>%d}: {}" % (biggest, ) - for name, details in folders.items(): - print(fmt.format(name, details["directory"]), file=options.stdout) - else: - print("No magic-folders", file=options.stdout) - - -class InviteOptions(BasedirOptions): - nickname = None - synopsis = "MAGIC_ALIAS: NICKNAME" - stdin = StringIO("") - optParameters = [ - ("name", "n", "default", "The name of this magic-folder"), - ] - description = ( - "Invite a new participant to a given magic-folder. The resulting " - "invite-code that is printed is secret information and MUST be " - "transmitted securely to the invitee." - ) - - def parseArgs(self, alias, nickname=None): - BasedirOptions.parseArgs(self) - alias = argv_to_unicode(alias) - if not alias.endswith(u':'): - raise usage.UsageError("An alias must end with a ':' character.") - self.alias = alias[:-1] - self.nickname = argv_to_unicode(nickname) - node_url_file = os.path.join(self['node-directory'], u"node.url") - self['node-url'] = open(node_url_file, "r").read().strip() - aliases = get_aliases(self['node-directory']) - self.aliases = aliases - - -def invite(options): - precondition(isinstance(options.alias, unicode), alias=options.alias) - precondition(isinstance(options.nickname, unicode), nickname=options.nickname) - - from allmydata.scripts import tahoe_mkdir - mkdir_options = _delegate_options(options, MakeDirectoryOptions()) - mkdir_options.where = None - - rc = tahoe_mkdir.mkdir(mkdir_options) - if rc != 0: - print("magic-folder: failed to mkdir\n", file=options.stderr) - return rc - - # FIXME this assumes caps are ASCII. - dmd_write_cap = mkdir_options.stdout.getvalue().strip() - dmd_readonly_cap = uri.from_string(dmd_write_cap).get_readonly().to_string() - if dmd_readonly_cap is None: - print("magic-folder: failed to diminish dmd write cap\n", file=options.stderr) - return 1 - - magic_write_cap = get_aliases(options["node-directory"])[options.alias] - magic_readonly_cap = uri.from_string(magic_write_cap).get_readonly().to_string() - - # tahoe ln CLIENT_READCAP COLLECTIVE_WRITECAP/NICKNAME - ln_options = _delegate_options(options, LnOptions()) - ln_options.from_file = unicode(dmd_readonly_cap, 'utf-8') - ln_options.to_file = u"%s/%s" % (unicode(magic_write_cap, 'utf-8'), options.nickname) - rc = tahoe_mv.mv(ln_options, mode="link") - if rc != 0: - print("magic-folder: failed to create link\n", file=options.stderr) - print(ln_options.stderr.getvalue(), file=options.stderr) - return rc - - # FIXME: this assumes caps are ASCII. - print("%s%s%s" % (magic_readonly_cap, INVITE_SEPARATOR, dmd_write_cap), file=options.stdout) - return 0 - -class JoinOptions(BasedirOptions): - synopsis = "INVITE_CODE LOCAL_DIR" - dmd_write_cap = "" - magic_readonly_cap = "" - optParameters = [ - ("poll-interval", "p", "60", "How often to ask for updates"), - ("name", "n", "default", "Name of the magic-folder"), - ] - - def parseArgs(self, invite_code, local_dir): - BasedirOptions.parseArgs(self) - - try: - if int(self['poll-interval']) <= 0: - raise ValueError("should be positive") - except ValueError: - raise usage.UsageError( - "--poll-interval must be a positive integer" - ) - # Expand the path relative to the current directory of the CLI command, not the node. - self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False) - self.invite_code = to_str(argv_to_unicode(invite_code)) - -def join(options): - fields = options.invite_code.split(INVITE_SEPARATOR) - if len(fields) != 2: - raise usage.UsageError("Invalid invite code.") - magic_readonly_cap, dmd_write_cap = fields - - maybe_upgrade_magic_folders(options["node-directory"]) - existing_folders = load_magic_folders(options["node-directory"]) - - if options['name'] in existing_folders: - print("This client already has a magic-folder named '{}'".format(options['name']), file=options.stderr) - return 1 - - db_fname = os.path.join( - options["node-directory"], - u"private", - u"magicfolder_{}.sqlite".format(options['name']), - ) - if os.path.exists(db_fname): - print("Database '{}' already exists; not overwriting".format(db_fname), file=options.stderr) - return 1 - - folder = { - u"directory": options.local_dir.encode('utf-8'), - u"collective_dircap": magic_readonly_cap, - u"upload_dircap": dmd_write_cap, - u"poll_interval": options["poll-interval"], - } - existing_folders[options["name"]] = folder - - save_magic_folders(options["node-directory"], existing_folders) - return 0 - - -class LeaveOptions(BasedirOptions): - synopsis = "Remove a magic-folder and forget all state" - optParameters = [ - ("name", "n", "default", "Name of magic-folder to leave"), - ] - - -def leave(options): - from ConfigParser import SafeConfigParser - - existing_folders = load_magic_folders(options["node-directory"]) - - if not existing_folders: - print("No magic-folders at all", file=options.stderr) - return 1 - - if options["name"] not in existing_folders: - print("No such magic-folder '{}'".format(options["name"]), file=options.stderr) - return 1 - - privdir = os.path.join(options["node-directory"], u"private") - db_fname = os.path.join(privdir, u"magicfolder_{}.sqlite".format(options["name"])) - - # delete from YAML file and re-write it - del existing_folders[options["name"]] - save_magic_folders(options["node-directory"], existing_folders) - - # delete the database file - try: - fileutil.remove(db_fname) - except Exception as e: - print("Warning: unable to remove %s due to %s: %s" - % (quote_local_unicode_path(db_fname), e.__class__.__name__, str(e)), file=options.stderr) - - # if this was the last magic-folder, disable them entirely - if not existing_folders: - parser = SafeConfigParser() - parser.read(os.path.join(options["node-directory"], u"tahoe.cfg")) - parser.remove_section("magic_folder") - with open(os.path.join(options["node-directory"], u"tahoe.cfg"), "w") as f: - parser.write(f) - - return 0 - - -class StatusOptions(BasedirOptions): - synopsis = "" - stdin = StringIO("") - optParameters = [ - ("name", "n", "default", "Name for the magic-folder to show status"), - ] - - def parseArgs(self): - BasedirOptions.parseArgs(self) - node_url_file = os.path.join(self['node-directory'], u"node.url") - with open(node_url_file, "r") as f: - self['node-url'] = f.read().strip() - - -def _get_json_for_fragment(options, fragment, method='GET', post_args=None): - nodeurl = options['node-url'] - if nodeurl.endswith('/'): - nodeurl = nodeurl[:-1] - - url = u'%s/%s' % (nodeurl, fragment) - if method == 'POST': - if post_args is None: - raise ValueError("Must pass post_args= for POST method") - body = urllib.urlencode(post_args) - else: - body = '' - if post_args is not None: - raise ValueError("post_args= only valid for POST method") - resp = do_http(method, url, body=body) - if isinstance(resp, BadResponse): - # specifically NOT using format_http_error() here because the - # URL is pretty sensitive (we're doing /uri/). - raise RuntimeError( - "Failed to get json from '%s': %s" % (nodeurl, resp.error) - ) - - data = resp.read() - parsed = json.loads(data) - if parsed is None: - raise RuntimeError("No data from '%s'" % (nodeurl,)) - return parsed - - -def _get_json_for_cap(options, cap): - return _get_json_for_fragment( - options, - 'uri/%s?t=json' % urllib.quote(cap), - ) - -def _print_item_status(item, now, longest): - paddedname = (' ' * (longest - len(item['path']))) + item['path'] - if 'failure_at' in item: - ts = datetime.fromtimestamp(item['started_at']) - prog = 'Failed %s (%s)' % (abbreviate_time(now - ts), ts) - elif item['percent_done'] < 100.0: - if 'started_at' not in item: - prog = 'not yet started' - else: - so_far = now - datetime.fromtimestamp(item['started_at']) - if so_far.seconds > 0.0: - rate = item['percent_done'] / so_far.seconds - if rate != 0: - time_left = (100.0 - item['percent_done']) / rate - prog = '%2.1f%% done, around %s left' % ( - item['percent_done'], - abbreviate_time(time_left), - ) - else: - time_left = None - prog = '%2.1f%% done' % (item['percent_done'],) - else: - prog = 'just started' - else: - prog = '' - for verb in ['finished', 'started', 'queued']: - keyname = verb + '_at' - if keyname in item: - when = datetime.fromtimestamp(item[keyname]) - prog = '%s %s' % (verb, abbreviate_time(now - when)) - break - - print(" %s: %s" % (paddedname, prog)) - - -def status(options): - nodedir = options["node-directory"] - stdout, stderr = options.stdout, options.stderr - magic_folders = load_magic_folders(os.path.join(options["node-directory"])) - - with open(os.path.join(nodedir, u'private', u'api_auth_token'), 'rb') as f: - token = f.read() - - print("Magic-folder status for '{}':".format(options["name"]), file=stdout) - - if options["name"] not in magic_folders: - raise Exception( - "No such magic-folder '{}'".format(options["name"]) - ) - - dmd_cap = magic_folders[options["name"]]["upload_dircap"] - collective_readcap = magic_folders[options["name"]]["collective_dircap"] - - # do *all* our data-retrievals first in case there's an error - try: - dmd_data = _get_json_for_cap(options, dmd_cap) - remote_data = _get_json_for_cap(options, collective_readcap) - magic_data = _get_json_for_fragment( - options, - 'magic_folder?t=json', - method='POST', - post_args=dict( - t='json', - name=options["name"], - token=token, - ) - ) - except Exception as e: - print("failed to retrieve data: %s" % str(e), file=stderr) - return 2 - - for d in [dmd_data, remote_data, magic_data]: - if isinstance(d, dict) and 'error' in d: - print("Error from server: %s" % d['error'], file=stderr) - print("This means we can't retrieve the remote shared directory.", file=stderr) - return 3 - - captype, dmd = dmd_data - if captype != 'dirnode': - print("magic_folder_dircap isn't a directory capability", file=stderr) - return 2 - - now = datetime.now() - - print("Local files:", file=stdout) - for (name, child) in dmd['children'].items(): - captype, meta = child - status = 'good' - size = meta['size'] - created = datetime.fromtimestamp(meta['metadata']['tahoe']['linkcrtime']) - version = meta['metadata']['version'] - nice_size = abbreviate_space(size) - nice_created = abbreviate_time(now - created) - if captype != 'filenode': - print("%20s: error, should be a filecap" % name, file=stdout) - continue - print(" %s (%s): %s, version=%s, created %s" % (name, nice_size, status, version, nice_created), file=stdout) - - print(file=stdout) - print("Remote files:", file=stdout) - - captype, collective = remote_data - for (name, data) in collective['children'].items(): - if data[0] != 'dirnode': - print("Error: '%s': expected a dirnode, not '%s'" % (name, data[0]), file=stdout) - print(" %s's remote:" % name, file=stdout) - dmd = _get_json_for_cap(options, data[1]['ro_uri']) - if isinstance(dmd, dict) and 'error' in dmd: - print(" Error: could not retrieve directory", file=stdout) - continue - if dmd[0] != 'dirnode': - print("Error: should be a dirnode", file=stdout) - continue - for (n, d) in dmd[1]['children'].items(): - if d[0] != 'filenode': - print("Error: expected '%s' to be a filenode." % (n,), file=stdout) - - meta = d[1] - status = 'good' - size = meta['size'] - created = datetime.fromtimestamp(meta['metadata']['tahoe']['linkcrtime']) - version = meta['metadata']['version'] - nice_size = abbreviate_space(size) - nice_created = abbreviate_time(now - created) - print(" %s (%s): %s, version=%s, created %s" % (n, nice_size, status, version, nice_created), file=stdout) - - if len(magic_data): - uploads = [item for item in magic_data if item['kind'] == 'upload'] - downloads = [item for item in magic_data if item['kind'] == 'download'] - longest = max([len(item['path']) for item in magic_data]) - - # maybe gate this with --show-completed option or something? - uploads = [item for item in uploads if item['status'] != 'success'] - downloads = [item for item in downloads if item['status'] != 'success'] - - if len(uploads): - print() - print("Uploads:", file=stdout) - for item in uploads: - _print_item_status(item, now, longest) - - if len(downloads): - print() - print("Downloads:", file=stdout) - for item in downloads: - _print_item_status(item, now, longest) - - for item in magic_data: - if item['status'] == 'failure': - print("Failed:", item, file=stdout) - - return 0 - - -class MagicFolderCommand(BaseOptions): - subCommands = [ - ["create", None, CreateOptions, "Create a Magic Folder."], - ["invite", None, InviteOptions, "Invite someone to a Magic Folder."], - ["join", None, JoinOptions, "Join a Magic Folder."], - ["leave", None, LeaveOptions, "Leave a Magic Folder."], - ["status", None, StatusOptions, "Display status of uploads/downloads."], - ["list", None, ListOptions, "List Magic Folders configured in this client."], - ] - optFlags = [ - ["debug", "d", "Print full stack-traces"], - ] - description = ( - "A magic-folder has an owner who controls the writecap " - "containing a list of nicknames and readcaps. The owner can invite " - "new participants. Every participant has the writecap for their " - "own folder (the corresponding readcap is in the master folder). " - "All clients download files from all other participants using the " - "readcaps contained in the master magic-folder directory." - ) - - def postOptions(self): - if not hasattr(self, 'subOptions'): - raise usage.UsageError("must specify a subcommand") - def getSynopsis(self): - return "Usage: tahoe [global-options] magic-folder" - def getUsage(self, width=None): - t = BaseOptions.getUsage(self, width) - t += ( - "Please run e.g. 'tahoe magic-folder create --help' for more " - "details on each subcommand.\n" - ) - return t - -subDispatch = { - "create": create, - "invite": invite, - "join": join, - "leave": leave, - "status": status, - "list": list_, -} - -def do_magic_folder(options): - so = options.subOptions - so.stdout = options.stdout - so.stderr = options.stderr - f = subDispatch[options.subCommand] - try: - return f(so) - except Exception as e: - print("Error: %s" % (e,), file=options.stderr) - if options['debug']: - raise - -subCommands = [ - ["magic-folder", None, MagicFolderCommand, - "Magic Folder subcommands: use 'tahoe magic-folder' for a list."], -] - -dispatch = { - "magic-folder": do_magic_folder, -} diff --git a/src/allmydata/scripts/run_common.py b/src/allmydata/scripts/run_common.py new file mode 100644 index 000000000..fa19c2076 --- /dev/null +++ b/src/allmydata/scripts/run_common.py @@ -0,0 +1,263 @@ +from __future__ import print_function + +import os, sys +from allmydata.scripts.common import BasedirOptions +from twisted.scripts import twistd +from twisted.python import usage +from twisted.python.reflect import namedAny +from twisted.internet.defer import maybeDeferred, fail +from twisted.application.service import Service + +from allmydata.scripts.default_nodedir import _default_nodedir +from allmydata.util import fileutil +from allmydata.node import read_config +from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path +from allmydata.util.configutil import UnknownConfigError +from allmydata.util.deferredutil import HookMixin + + +def get_pidfile(basedir): + """ + Returns the path to the PID file. + :param basedir: the node's base directory + :returns: the path to the PID file + """ + return os.path.join(basedir, u"twistd.pid") + +def get_pid_from_pidfile(pidfile): + """ + Tries to read and return the PID stored in the node's PID file + (twistd.pid). + :param pidfile: try to read this PID file + :returns: A numeric PID on success, ``None`` if PID file absent or + inaccessible, ``-1`` if PID file invalid. + """ + try: + with open(pidfile, "r") as f: + pid = f.read() + except EnvironmentError: + return None + + try: + pid = int(pid) + except ValueError: + return -1 + + return pid + +def identify_node_type(basedir): + """ + :return unicode: None or one of: 'client', 'introducer', + 'key-generator' or 'stats-gatherer' + """ + tac = u'' + try: + for fn in listdir_unicode(basedir): + if fn.endswith(u".tac"): + tac = fn + break + except OSError: + return None + + for t in (u"client", u"introducer", u"key-generator", u"stats-gatherer"): + if t in tac: + return t + return None + + +class RunOptions(BasedirOptions): + optParameters = [ + ("basedir", "C", None, + "Specify which Tahoe base directory should be used." + " This has the same effect as the global --node-directory option." + " [default: %s]" % quote_local_unicode_path(_default_nodedir)), + ] + + def parseArgs(self, basedir=None, *twistd_args): + # This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon' + # looks like an option to the tahoe subcommand, not to twistd. So you + # can either use 'tahoe start' or 'tahoe start NODEDIR + # --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start + # --TWISTD-OPTIONS' also isn't allowed, unfortunately. + + BasedirOptions.parseArgs(self, basedir) + self.twistd_args = twistd_args + + def getSynopsis(self): + return ("Usage: %s [global-options] %s [options]" + " [NODEDIR [twistd-options]]" + % (self.command_name, self.subcommand_name)) + + def getUsage(self, width=None): + t = BasedirOptions.getUsage(self, width) + "\n" + twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0] + t += twistd_options.replace("Options:", "twistd-options:", 1) + t += """ + +Note that if any twistd-options are used, NODEDIR must be specified explicitly +(not by default or using -C/--basedir or -d/--node-directory), and followed by +the twistd-options. +""" + return t + + +class MyTwistdConfig(twistd.ServerOptions): + subCommands = [("DaemonizeTahoeNode", None, usage.Options, "node")] + + stderr = sys.stderr + + +class DaemonizeTheRealService(Service, HookMixin): + """ + this HookMixin should really be a helper; our hooks: + + - 'running': triggered when startup has completed; it triggers + with None of successful or a Failure otherwise. + """ + stderr = sys.stderr + + def __init__(self, nodetype, basedir, options): + super(DaemonizeTheRealService, self).__init__() + self.nodetype = nodetype + self.basedir = basedir + # setup for HookMixin + self._hooks = { + "running": None, + } + self.stderr = options.parent.stderr + + def startService(self): + + def key_generator_removed(): + return fail(ValueError("key-generator support removed, see #2783")) + + def start(): + node_to_instance = { + u"client": lambda: maybeDeferred(namedAny("allmydata.client.create_client"), self.basedir), + u"introducer": lambda: maybeDeferred(namedAny("allmydata.introducer.server.create_introducer"), self.basedir), + u"stats-gatherer": lambda: maybeDeferred(namedAny("allmydata.stats.StatsGathererService"), read_config(self.basedir, None), self.basedir, verbose=True), + u"key-generator": key_generator_removed, + } + + try: + service_factory = node_to_instance[self.nodetype] + except KeyError: + raise ValueError("unknown nodetype %s" % self.nodetype) + + def handle_config_error(fail): + if fail.check(UnknownConfigError): + self.stderr.write("\nConfiguration error:\n{}\n\n".format(fail.value)) + else: + self.stderr.write("\nUnknown error\n") + fail.printTraceback(self.stderr) + reactor.stop() + + d = service_factory() + + def created(srv): + srv.setServiceParent(self.parent) + d.addCallback(created) + d.addErrback(handle_config_error) + d.addBoth(self._call_hook, 'running') + return d + + from twisted.internet import reactor + reactor.callWhenRunning(start) + + +class DaemonizeTahoeNodePlugin(object): + tapname = "tahoenode" + def __init__(self, nodetype, basedir): + self.nodetype = nodetype + self.basedir = basedir + + def makeService(self, so): + return DaemonizeTheRealService(self.nodetype, self.basedir, so) + + +def run(config): + """ + Runs a Tahoe-LAFS node in the foreground. + + Sets up the IService instance corresponding to the type of node + that's starting and uses Twisted's twistd runner to disconnect our + process from the terminal. + """ + out = config.stdout + err = config.stderr + basedir = config['basedir'] + quoted_basedir = quote_local_unicode_path(basedir) + print("'tahoe {}' in {}".format(config.subcommand_name, quoted_basedir), file=out) + if not os.path.isdir(basedir): + print("%s does not look like a directory at all" % quoted_basedir, file=err) + return 1 + nodetype = identify_node_type(basedir) + if not nodetype: + print("%s is not a recognizable node directory" % quoted_basedir, file=err) + return 1 + # Now prepare to turn into a twistd process. This os.chdir is the point + # of no return. + os.chdir(basedir) + twistd_args = [] + if (nodetype in (u"client", u"introducer") + and "--nodaemon" not in config.twistd_args + and "--syslog" not in config.twistd_args + and "--logfile" not in config.twistd_args): + fileutil.make_dirs(os.path.join(basedir, u"logs")) + twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")]) + twistd_args.extend(config.twistd_args) + twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin + + twistd_config = MyTwistdConfig() + twistd_config.stdout = out + twistd_config.stderr = err + try: + twistd_config.parseOptions(twistd_args) + except usage.error as ue: + # these arguments were unsuitable for 'twistd' + print(config, file=err) + print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err) + return 1 + twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} + + # handle invalid PID file (twistd might not start otherwise) + pidfile = get_pidfile(basedir) + if get_pid_from_pidfile(pidfile) == -1: + print("found invalid PID file in %s - deleting it" % basedir, file=err) + os.remove(pidfile) + + # On Unix-like platforms: + # Unless --nodaemon was provided, the twistd.runApp() below spawns off a + # child process, and the parent calls os._exit(0), so there's no way for + # us to get control afterwards, even with 'except SystemExit'. If + # application setup fails (e.g. ImportError), runApp() will raise an + # exception. + # + # So if we wanted to do anything with the running child, we'd have two + # options: + # + # * fork first, and have our child wait for the runApp() child to get + # running. (note: just fork(). This is easier than fork+exec, since we + # don't have to get PATH and PYTHONPATH set up, since we're not + # starting a *different* process, just cloning a new instance of the + # current process) + # * or have the user run a separate command some time after this one + # exits. + # + # For Tahoe, we don't need to do anything with the child, so we can just + # let it exit. + # + # On Windows: + # twistd does not fork; it just runs in the current process whether or not + # --nodaemon is specified. (As on Unix, --nodaemon does have the side effect + # of causing us to log to stdout/stderr.) + + if "--nodaemon" in twistd_args or sys.platform == "win32": + verb = "running" + else: + verb = "starting" + + print("%s node in %s" % (verb, quoted_basedir), file=out) + twistd.runApp(twistd_config) + # we should only reach here if --nodaemon or equivalent was used + return 0 diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 421eca5d5..8d8c6fca6 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -6,9 +6,10 @@ from six.moves import StringIO from twisted.python import usage from twisted.internet import defer, task, threads +from allmydata.version_checks import get_package_versions_string from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ - stats_gatherer, admin, magic_folder_cli, tahoe_daemonize, tahoe_start, \ + stats_gatherer, admin, tahoe_daemonize, tahoe_start, \ tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite, tahoe_grid_manager from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding from allmydata.util.eliotutil import ( @@ -40,11 +41,11 @@ _control_node_dispatch = { } process_control_commands = [ - ["daemonize", None, tahoe_daemonize.DaemonizeOptions, "run a node in the background"], - ["start", None, tahoe_start.StartOptions, "start a node in the background and confirm it started"], ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], - ["stop", None, tahoe_stop.StopOptions, "stop a node"], - ["restart", None, tahoe_restart.RestartOptions, "restart a node"], + ["daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"], + ["start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"], + ["stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"], + ["restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"], ] @@ -60,7 +61,6 @@ class Options(usage.Options): + process_control_commands + debug.subCommands + cli.subCommands - + magic_folder_cli.subCommands + tahoe_invite.subCommands + tahoe_grid_manager.subCommands ) @@ -77,13 +77,11 @@ class Options(usage.Options): ] def opt_version(self): - import allmydata - print(allmydata.get_package_versions_string(debug=True), file=self.stdout) + print(get_package_versions_string(debug=True), file=self.stdout) self.no_command_needed = True def opt_version_and_path(self): - import allmydata - print(allmydata.get_package_versions_string(show_paths=True, debug=True), file=self.stdout) + print(get_package_versions_string(show_paths=True, debug=True), file=self.stdout) self.no_command_needed = True opt_eliot_destination = opt_eliot_destination @@ -156,10 +154,6 @@ def dispatch(config, # these are blocking, and must be run in a thread f0 = cli.dispatch[command] f = lambda so: threads.deferToThread(f0, so) - elif command in magic_folder_cli.dispatch: - # same - f0 = magic_folder_cli.dispatch[command] - f = lambda so: threads.deferToThread(f0, so) elif command in tahoe_grid_manager.dispatch: f = tahoe_grid_manager.dispatch[command] elif command in tahoe_invite.dispatch: @@ -203,7 +197,51 @@ def run(): # doesn't return: calls sys.exit(rc) task.react(_run_with_reactor) + +def _setup_coverage(reactor): + """ + Arrange for coverage to be collected if the 'coverage' package is + installed + """ + # can we put this _setup_coverage call after we hit + # argument-parsing? + if '--coverage' not in sys.argv: + return + sys.argv.remove('--coverage') + + try: + import coverage + except ImportError: + raise RuntimeError( + "The 'coveage' package must be installed to use --coverage" + ) + + # this doesn't change the shell's notion of the environment, but + # it makes the test in process_startup() succeed, which is the + # goal here. + os.environ["COVERAGE_PROCESS_START"] = '.coveragerc' + + # maybe-start the global coverage, unless it already got started + cov = coverage.process_startup() + if cov is None: + cov = coverage.process_startup.coverage + + def write_coverage_data(): + """ + Make sure that coverage has stopped; internally, it depends on + ataxit handlers running which doesn't always happen (Twisted's + shutdown hook also won't run if os._exit() is called, but it + runs more-often than atexit handlers). + """ + cov.stop() + cov.save() + reactor.addSystemEventTrigger('after', 'shutdown', write_coverage_data) + + def _run_with_reactor(reactor): + + _setup_coverage(reactor) + d = defer.maybeDeferred(parse_or_exit_with_explanation, sys.argv[1:]) d.addCallback(_maybe_enable_eliot_logging, reactor) d.addCallback(dispatch) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 8aa5adf12..ce25e9667 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -9,7 +9,7 @@ from allmydata.util.encodingutil import quote_output, is_printable_ascii import urllib import json -class SlowOperationRunner: +class SlowOperationRunner(object): def run(self, options): stderr = options.stderr diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 5694b8801..558c3d6d3 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -51,8 +51,8 @@ def mkdir(contents, options): return dircap def put_child(dirurl, childname, childcap): - assert dirurl[-1] == "/" - url = dirurl + urllib.quote(unicode_to_url(childname)) + "?t=uri" + assert dirurl[-1] != "/" + url = dirurl + "/" + urllib.quote(unicode_to_url(childname)) + "?t=uri" resp = do_http("PUT", url, childcap) if resp.status not in (200, 201): raise HTTPError("Error during put_child", resp) @@ -105,6 +105,9 @@ class BackerUpper(object): archives_url = to_url + "Archives/" + archives_url = archives_url.rstrip("/") + to_url = to_url.rstrip("/") + # first step: make sure the target directory exists, as well as the # Archives/ subdirectory. resp = do_http("GET", archives_url + "?t=json") diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 19967d505..89a716da9 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -8,7 +8,7 @@ from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util.encodingutil import quote_output, quote_path -class Checker: +class Checker(object): pass def _quote_serverid_index_share(serverid, storage_index, sharenum): @@ -111,10 +111,10 @@ def check(options): return errno return 0 -class FakeTransport: +class FakeTransport(object): disconnecting = False -class DeepCheckOutput(LineOnlyReceiver): +class DeepCheckOutput(LineOnlyReceiver, object): delimiter = "\n" def __init__(self, streamer, options): self.streamer = streamer @@ -173,7 +173,7 @@ class DeepCheckOutput(LineOnlyReceiver): print("done: %d objects checked, %d healthy, %d unhealthy" \ % (self.num_objects, self.files_healthy, self.files_unhealthy), file=stdout) -class DeepCheckAndRepairOutput(LineOnlyReceiver): +class DeepCheckAndRepairOutput(LineOnlyReceiver, object): delimiter = "\n" def __init__(self, streamer, options): self.streamer = streamer @@ -271,7 +271,7 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver): % (self.post_repair_files_healthy, self.post_repair_files_unhealthy), file=stdout) -class DeepCheckStreamer(LineOnlyReceiver): +class DeepCheckStreamer(LineOnlyReceiver, object): def deepcheck_location(self, options, where): stdout = options.stdout diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index e2c32fdfd..5d0849c56 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -70,7 +70,7 @@ def make_tahoe_subdirectory(nodeurl, parent_writecap, name): raise HTTPError("Error during mkdir", resp) -class LocalFileSource: +class LocalFileSource(object): def __init__(self, pathname, basename): precondition_abspath(pathname) self.pathname = pathname @@ -85,7 +85,7 @@ class LocalFileSource: def open(self, caps_only): return open(self.pathname, "rb") -class LocalFileTarget: +class LocalFileTarget(object): def __init__(self, pathname): precondition_abspath(pathname) self.pathname = pathname @@ -93,7 +93,7 @@ class LocalFileTarget: def put_file(self, inf): fileutil.put_file(self.pathname, inf) -class LocalMissingTarget: +class LocalMissingTarget(object): def __init__(self, pathname): precondition_abspath(pathname) self.pathname = pathname @@ -101,7 +101,7 @@ class LocalMissingTarget: def put_file(self, inf): fileutil.put_file(self.pathname, inf) -class LocalDirectorySource: +class LocalDirectorySource(object): def __init__(self, progressfunc, pathname, basename): precondition_abspath(pathname) @@ -133,7 +133,7 @@ class LocalDirectorySource: # TODO: output a warning pass -class LocalDirectoryTarget: +class LocalDirectoryTarget(object): def __init__(self, progressfunc, pathname): precondition_abspath(pathname) @@ -180,7 +180,7 @@ class LocalDirectoryTarget: pass -class TahoeFileSource: +class TahoeFileSource(object): def __init__(self, nodeurl, mutable, writecap, readcap, basename): self.nodeurl = nodeurl self.mutable = mutable @@ -205,7 +205,7 @@ class TahoeFileSource: def bestcap(self): return self.writecap or self.readcap -class TahoeFileTarget: +class TahoeFileTarget(object): def __init__(self, nodeurl, mutable, writecap, readcap, url): self.nodeurl = nodeurl self.mutable = mutable @@ -225,7 +225,7 @@ class TahoeFileTarget: # to always create mutable files, or to copy mutable files into new # mutable files. ticket #835 -class TahoeDirectorySource: +class TahoeDirectorySource(object): def __init__(self, nodeurl, cache, progressfunc, basename): self.nodeurl = nodeurl self.cache = cache @@ -298,7 +298,7 @@ class TahoeDirectorySource: "You probably need to use a later version of " "Tahoe-LAFS to copy this directory.") -class TahoeMissingTarget: +class TahoeMissingTarget(object): def __init__(self, url): self.url = url @@ -315,7 +315,7 @@ class TahoeMissingTarget: # I'm not sure this will always work return PUT(self.url + "?t=uri", filecap) -class TahoeDirectoryTarget: +class TahoeDirectoryTarget(object): def __init__(self, nodeurl, cache, progressfunc): self.nodeurl = nodeurl self.cache = cache @@ -459,7 +459,7 @@ FileTargets = (LocalFileTarget, TahoeFileTarget) DirectoryTargets = (LocalDirectoryTarget, TahoeDirectoryTarget) MissingTargets = (LocalMissingTarget, TahoeMissingTarget) -class Copier: +class Copier(object): def do_copy(self, options, progressfunc=None): if options['quiet']: diff --git a/src/allmydata/scripts/tahoe_daemonize.py b/src/allmydata/scripts/tahoe_daemonize.py index d7cfc89cf..ad2f92355 100644 --- a/src/allmydata/scripts/tahoe_daemonize.py +++ b/src/allmydata/scripts/tahoe_daemonize.py @@ -1,256 +1,16 @@ -from __future__ import print_function +from .run_common import ( + RunOptions as _RunOptions, + run, +) -import os, sys -from allmydata.scripts.common import BasedirOptions -from twisted.scripts import twistd -from twisted.python import usage -from twisted.python.reflect import namedAny -from twisted.internet.defer import maybeDeferred, fail -from twisted.application.service import Service - -from allmydata.scripts.default_nodedir import _default_nodedir -from allmydata.util import fileutil -from allmydata.node import read_config -from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path -from allmydata.util.configutil import UnknownConfigError -from allmydata.util.deferredutil import HookMixin - - -def get_pidfile(basedir): - """ - Returns the path to the PID file. - :param basedir: the node's base directory - :returns: the path to the PID file - """ - return os.path.join(basedir, u"twistd.pid") - -def get_pid_from_pidfile(pidfile): - """ - Tries to read and return the PID stored in the node's PID file - (twistd.pid). - :param pidfile: try to read this PID file - :returns: A numeric PID on success, ``None`` if PID file absent or - inaccessible, ``-1`` if PID file invalid. - """ - try: - with open(pidfile, "r") as f: - pid = f.read() - except EnvironmentError: - return None - - try: - pid = int(pid) - except ValueError: - return -1 - - return pid - -def identify_node_type(basedir): - """ - :return unicode: None or one of: 'client', 'introducer', - 'key-generator' or 'stats-gatherer' - """ - tac = u'' - try: - for fn in listdir_unicode(basedir): - if fn.endswith(u".tac"): - tac = fn - break - except OSError: - return None - - for t in (u"client", u"introducer", u"key-generator", u"stats-gatherer"): - if t in tac: - return t - return None - - -class DaemonizeOptions(BasedirOptions): - subcommand_name = "start" - optParameters = [ - ("basedir", "C", None, - "Specify which Tahoe base directory should be used." - " This has the same effect as the global --node-directory option." - " [default: %s]" % quote_local_unicode_path(_default_nodedir)), - ] - - def parseArgs(self, basedir=None, *twistd_args): - # This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon' - # looks like an option to the tahoe subcommand, not to twistd. So you - # can either use 'tahoe start' or 'tahoe start NODEDIR - # --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start - # --TWISTD-OPTIONS' also isn't allowed, unfortunately. - - BasedirOptions.parseArgs(self, basedir) - self.twistd_args = twistd_args - - def getSynopsis(self): - return ("Usage: %s [global-options] %s [options]" - " [NODEDIR [twistd-options]]" - % (self.command_name, self.subcommand_name)) - - def getUsage(self, width=None): - t = BasedirOptions.getUsage(self, width) + "\n" - twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0] - t += twistd_options.replace("Options:", "twistd-options:", 1) - t += """ - -Note that if any twistd-options are used, NODEDIR must be specified explicitly -(not by default or using -C/--basedir or -d/--node-directory), and followed by -the twistd-options. -""" - return t - - -class MyTwistdConfig(twistd.ServerOptions): - subCommands = [("DaemonizeTahoeNode", None, usage.Options, "node")] - - -class DaemonizeTheRealService(Service, HookMixin): - """ - this HookMixin should really be a helper; our hooks: - - - 'running': triggered when startup has completed; it triggers - with None of successful or a Failure otherwise. - """ - - def __init__(self, nodetype, basedir, options): - super(DaemonizeTheRealService, self).__init__() - self.nodetype = nodetype - self.basedir = basedir - # setup for HookMixin - self._hooks = { - "running": None, - } - - def startService(self): - - def key_generator_removed(): - return fail(ValueError("key-generator support removed, see #2783")) - - def start(): - node_to_instance = { - u"client": lambda: maybeDeferred(namedAny("allmydata.client.create_client"), self.basedir), - u"introducer": lambda: maybeDeferred(namedAny("allmydata.introducer.server.create_introducer"), self.basedir), - u"stats-gatherer": lambda: maybeDeferred(namedAny("allmydata.stats.StatsGathererService"), read_config(self.basedir, None), self.basedir, verbose=True), - u"key-generator": key_generator_removed, - } - - try: - service_factory = node_to_instance[self.nodetype] - except KeyError: - raise ValueError("unknown nodetype %s" % self.nodetype) - - def handle_config_error(fail): - fail.trap(UnknownConfigError) - sys.stderr.write("\nConfiguration error:\n{}\n\n".format(fail.value)) - reactor.stop() - return - - d = service_factory() - - def created(srv): - srv.setServiceParent(self.parent) - d.addCallback(created) - d.addErrback(handle_config_error) - d.addBoth(self._call_hook, 'running') - return d - - from twisted.internet import reactor - reactor.callWhenRunning(start) - - -class DaemonizeTahoeNodePlugin(object): - tapname = "tahoenode" - def __init__(self, nodetype, basedir): - self.nodetype = nodetype - self.basedir = basedir - - def makeService(self, so): - return DaemonizeTheRealService(self.nodetype, self.basedir, so) +__all__ = [ + "DaemonizeOptions", + "daemonize", +] +class DaemonizeOptions(_RunOptions): + subcommand_name = "daemonize" def daemonize(config): - """ - Runs the 'tahoe daemonize' command. - - Sets up the IService instance corresponding to the type of node - that's starting and uses Twisted's twistd runner to disconnect our - process from the terminal. - """ - out = config.stdout - err = config.stderr - basedir = config['basedir'] - quoted_basedir = quote_local_unicode_path(basedir) - print("daemonizing in {}".format(quoted_basedir), file=out) - if not os.path.isdir(basedir): - print("%s does not look like a directory at all" % quoted_basedir, file=err) - return 1 - nodetype = identify_node_type(basedir) - if not nodetype: - print("%s is not a recognizable node directory" % quoted_basedir, file=err) - return 1 - # Now prepare to turn into a twistd process. This os.chdir is the point - # of no return. - os.chdir(basedir) - twistd_args = [] - if (nodetype in (u"client", u"introducer") - and "--nodaemon" not in config.twistd_args - and "--syslog" not in config.twistd_args - and "--logfile" not in config.twistd_args): - fileutil.make_dirs(os.path.join(basedir, u"logs")) - twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")]) - twistd_args.extend(config.twistd_args) - twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin - - twistd_config = MyTwistdConfig() - try: - twistd_config.parseOptions(twistd_args) - except usage.error as ue: - # these arguments were unsuitable for 'twistd' - print(config, file=err) - print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err) - return 1 - twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} - - # handle invalid PID file (twistd might not start otherwise) - pidfile = get_pidfile(basedir) - if get_pid_from_pidfile(pidfile) == -1: - print("found invalid PID file in %s - deleting it" % basedir, file=err) - os.remove(pidfile) - - # On Unix-like platforms: - # Unless --nodaemon was provided, the twistd.runApp() below spawns off a - # child process, and the parent calls os._exit(0), so there's no way for - # us to get control afterwards, even with 'except SystemExit'. If - # application setup fails (e.g. ImportError), runApp() will raise an - # exception. - # - # So if we wanted to do anything with the running child, we'd have two - # options: - # - # * fork first, and have our child wait for the runApp() child to get - # running. (note: just fork(). This is easier than fork+exec, since we - # don't have to get PATH and PYTHONPATH set up, since we're not - # starting a *different* process, just cloning a new instance of the - # current process) - # * or have the user run a separate command some time after this one - # exits. - # - # For Tahoe, we don't need to do anything with the child, so we can just - # let it exit. - # - # On Windows: - # twistd does not fork; it just runs in the current process whether or not - # --nodaemon is specified. (As on Unix, --nodaemon does have the side effect - # of causing us to log to stdout/stderr.) - - if "--nodaemon" in twistd_args or sys.platform == "win32": - verb = "running" - else: - verb = "starting" - - print("%s node in %s" % (verb, quoted_basedir), file=out) - twistd.runApp(twistd_config) - # we should only reach here if --nodaemon or equivalent was used - return 0 + print("'tahoe daemonize' is deprecated; see 'tahoe run'") + return run(config) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index f1a5e7a99..b1daa7717 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -9,10 +9,10 @@ from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util.encodingutil import quote_output, quote_path -class FakeTransport: +class FakeTransport(object): disconnecting = False -class ManifestStreamer(LineOnlyReceiver): +class ManifestStreamer(LineOnlyReceiver, object): delimiter = "\n" def __init__(self): diff --git a/src/allmydata/scripts/tahoe_restart.py b/src/allmydata/scripts/tahoe_restart.py index 34d0b65c5..339db862f 100644 --- a/src/allmydata/scripts/tahoe_restart.py +++ b/src/allmydata/scripts/tahoe_restart.py @@ -9,6 +9,7 @@ class RestartOptions(StartOptions): def restart(config): + print("'tahoe restart' is deprecated; see 'tahoe run'") stderr = config.stderr rc = stop(config) if rc == COULD_NOT_STOP: diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index b5a0b9dfe..0a921cc71 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -1,10 +1,15 @@ -from .tahoe_daemonize import daemonize, DaemonizeOptions +from .run_common import ( + RunOptions as _RunOptions, + run, +) +__all__ = [ + "RunOptions", + "run", +] -class RunOptions(DaemonizeOptions): +class RunOptions(_RunOptions): subcommand_name = "run" - -def run(config): - config.twistd_args = config.twistd_args + ("--nodaemon",) - return daemonize(config) + def postOptions(self): + self.twistd_args += ("--nodaemon",) diff --git a/src/allmydata/scripts/tahoe_start.py b/src/allmydata/scripts/tahoe_start.py index 68a5df96f..bc076d1b7 100644 --- a/src/allmydata/scripts/tahoe_start.py +++ b/src/allmydata/scripts/tahoe_start.py @@ -11,7 +11,7 @@ from allmydata.scripts.common import BasedirOptions from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.encodingutil import quote_local_unicode_path -from .tahoe_daemonize import MyTwistdConfig, identify_node_type +from .run_common import MyTwistdConfig, identify_node_type class StartOptions(BasedirOptions): @@ -60,7 +60,7 @@ def start(config): (e.g. "introducer started"). If that doesn't happen within a few seconds, an error is printed along with all collected logs. """ - + print("'tahoe start' is deprecated; see 'tahoe run'") out = config.stdout err = config.stderr basedir = config['basedir'] diff --git a/src/allmydata/scripts/tahoe_stop.py b/src/allmydata/scripts/tahoe_stop.py index 2b7b98faf..28c0f8131 100644 --- a/src/allmydata/scripts/tahoe_stop.py +++ b/src/allmydata/scripts/tahoe_stop.py @@ -6,7 +6,7 @@ import signal from allmydata.scripts.common import BasedirOptions from allmydata.util.encodingutil import quote_local_unicode_path -from .tahoe_daemonize import get_pidfile, get_pid_from_pidfile +from .run_common import get_pidfile, get_pid_from_pidfile COULD_NOT_STOP = 2 @@ -21,6 +21,7 @@ class StopOptions(BasedirOptions): def stop(config): + print("'tahoe stop' is deprecated; see 'tahoe run'") out = config.stdout err = config.stderr basedir = config['basedir'] diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index fd846c63f..f66eec594 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -36,7 +36,7 @@ from allmydata.storage.common import UnknownImmutableContainerVersionError, \ # then the value stored in this field will be the actual share data length # modulo 2**32. -class ShareFile: +class ShareFile(object): LEASE_SIZE = struct.calcsize(">L32s32sL") sharetype = "immutable" diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index cd176aeef..5e3e01716 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -1,6 +1,6 @@ import struct, time -class LeaseInfo: +class LeaseInfo(object): def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, expiration_time=None, nodeid=None): self.owner_num = owner_num @@ -14,9 +14,11 @@ class LeaseInfo: def get_expiration_time(self): return self.expiration_time + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self.expiration_time - 31*24*60*60 + def get_age(self): return time.time() - self.get_grant_renew_time_time() @@ -27,6 +29,7 @@ class LeaseInfo: self.expiration_time) = struct.unpack(">L32s32sL", data) self.nodeid = None return self + def to_immutable_data(self): return struct.pack(">L32s32sL", self.owner_num, @@ -39,6 +42,7 @@ class LeaseInfo: int(self.expiration_time), self.renew_secret, self.cancel_secret, self.nodeid) + def from_mutable_data(self, data): (self.owner_num, self.expiration_time, diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index a8392eac9..37f773c0d 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -35,7 +35,7 @@ from allmydata.mutable.layout import MAX_MUTABLE_SHARE_SIZE assert struct.calcsize(">L") == 4, struct.calcsize(">L") assert struct.calcsize(">Q") == 8, struct.calcsize(">Q") -class MutableShareFile: +class MutableShareFile(object): sharetype = "mutable" DATA_LENGTH_OFFSET = struct.calcsize(">32s20s32s") @@ -443,7 +443,7 @@ def testv_compare(a, op, b): return a > b # never reached -class EmptyShare: +class EmptyShare(object): def check_testv(self, testv): test_good = True diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ac5567a30..eae6fd698 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -1,4 +1,5 @@ import os, re, weakref, struct, time +import six from foolscap.api import Referenceable from twisted.application import service @@ -391,8 +392,9 @@ class StorageServer(service.MultiService, Referenceable): bucket. Each lease is returned as a LeaseInfo instance. This method is not for client use. - """ + :note: Only for immutable shares. + """ # since all shares get the same lease data, we just grab the leases # from the first share try: @@ -402,20 +404,39 @@ class StorageServer(service.MultiService, Referenceable): except StopIteration: return iter([]) - def remote_slot_testv_and_readv_and_writev(self, storage_index, - secrets, - test_and_write_vectors, - read_vector): - start = time.time() - self.count("writev") - si_s = si_b2a(storage_index) - log.msg("storage: slot_writev %s" % si_s) - si_dir = storage_index_to_dir(storage_index) - (write_enabler, renew_secret, cancel_secret) = secrets - # shares exist if there is a file for them - bucketdir = os.path.join(self.sharedir, si_dir) + def get_slot_leases(self, storage_index): + """ + This method is not for client use. + + :note: Only for mutable shares. + + :return: An iterable of the leases attached to this slot. + """ + for _, share_filename in self._get_bucket_shares(storage_index): + share = MutableShareFile(share_filename) + return share.get_leases() + return [] + + def _collect_mutable_shares_for_storage_index(self, bucketdir, write_enabler, si_s): + """ + Gather up existing mutable shares for the given storage index. + + :param bytes bucketdir: The filesystem path containing shares for the + given storage index. + + :param bytes write_enabler: The write enabler secret for the shares. + + :param bytes si_s: The storage index in encoded (base32) form. + + :raise BadWriteEnablerError: If the write enabler is not correct for + any of the collected shares. + + :return dict[int, MutableShareFile]: The collected shares in a mapping + from integer share numbers to ``MutableShareFile`` instances. + """ shares = {} if os.path.isdir(bucketdir): + # shares exist if there is a file for them for sharenum_s in os.listdir(bucketdir): try: sharenum = int(sharenum_s) @@ -425,67 +446,197 @@ class StorageServer(service.MultiService, Referenceable): msf = MutableShareFile(filename, self) msf.check_write_enabler(write_enabler, si_s) shares[sharenum] = msf - # write_enabler is good for all existing shares. + return shares - # Now evaluate test vectors. - testv_is_good = True + def _evaluate_test_vectors(self, test_and_write_vectors, shares): + """ + Execute test vectors against share data. + + :param test_and_write_vectors: See + ``allmydata.interfaces.TestAndWriteVectorsForShares``. + + :param dict[int, MutableShareFile] shares: The shares against which to + execute the vectors. + + :return bool: ``True`` if and only if all of the test vectors succeed + against the given shares. + """ for sharenum in test_and_write_vectors: (testv, datav, new_length) = test_and_write_vectors[sharenum] if sharenum in shares: if not shares[sharenum].check_testv(testv): self.log("testv failed: [%d]: %r" % (sharenum, testv)) - testv_is_good = False - break + return False else: # compare the vectors against an empty share, in which all # reads return empty strings. if not EmptyShare().check_testv(testv): self.log("testv failed (empty): [%d] %r" % (sharenum, testv)) - testv_is_good = False - break + return False + return True - # now gather the read vectors, before we do any writes + def _evaluate_read_vectors(self, read_vector, shares): + """ + Execute read vectors against share data. + + :param read_vector: See ``allmydata.interfaces.ReadVector``. + + :param dict[int, MutableShareFile] shares: The shares against which to + execute the vector. + + :return dict[int, bytes]: The data read from the shares. + """ read_data = {} for sharenum, share in shares.items(): read_data[sharenum] = share.readv(read_vector) + return read_data + def _evaluate_write_vectors(self, bucketdir, secrets, test_and_write_vectors, shares): + """ + Execute write vectors against share data. + + :param bytes bucketdir: The parent directory holding the shares. This + is removed if the last share is removed from it. If shares are + created, they are created in it. + + :param secrets: A tuple of ``WriteEnablerSecret``, + ``LeaseRenewSecret``, and ``LeaseCancelSecret``. These secrets + are used to initialize new shares. + + :param test_and_write_vectors: See + ``allmydata.interfaces.TestAndWriteVectorsForShares``. + + :param dict[int, MutableShareFile]: The shares against which to + execute the vectors. + + :return dict[int, MutableShareFile]: The shares which still exist + after applying the vectors. + """ + remaining_shares = {} + + for sharenum in test_and_write_vectors: + (testv, datav, new_length) = test_and_write_vectors[sharenum] + if new_length == 0: + if sharenum in shares: + shares[sharenum].unlink() + else: + if sharenum not in shares: + # allocate a new share + allocated_size = 2000 # arbitrary, really + share = self._allocate_slot_share(bucketdir, secrets, + sharenum, + allocated_size, + owner_num=0) + shares[sharenum] = share + shares[sharenum].writev(datav, new_length) + remaining_shares[sharenum] = shares[sharenum] + + if new_length == 0: + # delete bucket directories that exist but are empty. They + # might not exist if a client showed up and asked us to + # truncate a share we weren't even holding. + if os.path.exists(bucketdir) and [] == os.listdir(bucketdir): + os.rmdir(bucketdir) + return remaining_shares + + def _make_lease_info(self, renew_secret, cancel_secret): + """ + :return LeaseInfo: Information for a new lease for a share. + """ ownerid = 1 # TODO expire_time = time.time() + 31*24*60*60 # one month lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) + return lease_info + + def _add_or_renew_leases(self, shares, lease_info): + """ + Put the given lease onto the given shares. + + :param dict[int, MutableShareFile] shares: The shares to put the lease + onto. + + :param LeaseInfo lease_info: The lease to put on the shares. + """ + for share in six.viewvalues(shares): + share.add_or_renew_lease(lease_info) + + def slot_testv_and_readv_and_writev( + self, + storage_index, + secrets, + test_and_write_vectors, + read_vector, + renew_leases, + ): + """ + Read data from shares and conditionally write some data to them. + + :param bool renew_leases: If and only if this is ``True`` and the test + vectors pass then shares in this slot will also have an updated + lease applied to them. + + See ``allmydata.interfaces.RIStorageServer`` for details about other + parameters and return value. + """ + start = time.time() + self.count("writev") + si_s = si_b2a(storage_index) + log.msg("storage: slot_writev %s" % si_s) + si_dir = storage_index_to_dir(storage_index) + (write_enabler, renew_secret, cancel_secret) = secrets + bucketdir = os.path.join(self.sharedir, si_dir) + + # If collection succeeds we know the write_enabler is good for all + # existing shares. + shares = self._collect_mutable_shares_for_storage_index( + bucketdir, + write_enabler, + si_s, + ) + + # Now evaluate test vectors. + testv_is_good = self._evaluate_test_vectors( + test_and_write_vectors, + shares, + ) + + # now gather the read vectors, before we do any writes + read_data = self._evaluate_read_vectors( + read_vector, + shares, + ) if testv_is_good: # now apply the write vectors - for sharenum in test_and_write_vectors: - (testv, datav, new_length) = test_and_write_vectors[sharenum] - if new_length == 0: - if sharenum in shares: - shares[sharenum].unlink() - else: - if sharenum not in shares: - # allocate a new share - allocated_size = 2000 # arbitrary, really - share = self._allocate_slot_share(bucketdir, secrets, - sharenum, - allocated_size, - owner_num=0) - shares[sharenum] = share - shares[sharenum].writev(datav, new_length) - # and update the lease - shares[sharenum].add_or_renew_lease(lease_info) - - if new_length == 0: - # delete empty bucket directories - if not os.listdir(bucketdir): - os.rmdir(bucketdir) - + remaining_shares = self._evaluate_write_vectors( + bucketdir, + secrets, + test_and_write_vectors, + shares, + ) + if renew_leases: + lease_info = self._make_lease_info(renew_secret, cancel_secret) + self._add_or_renew_leases(remaining_shares, lease_info) # all done self.add_latency("writev", time.time() - start) return (testv_is_good, read_data) + def remote_slot_testv_and_readv_and_writev(self, storage_index, + secrets, + test_and_write_vectors, + read_vector): + return self.slot_testv_and_readv_and_writev( + storage_index, + secrets, + test_and_write_vectors, + read_vector, + renew_leases=True, + ) + def _allocate_slot_share(self, bucketdir, secrets, sharenum, allocated_size, owner_num=0): (write_enabler, renew_secret, cancel_secret) = secrets diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e78470e88..c076624ab 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -29,19 +29,35 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . # 6: implement other sorts of IStorageClient classes: S3, etc -import re -import time -import hashlib -import json -from datetime import datetime - -from zope.interface import implementer +import re, time, hashlib +from ConfigParser import ( + NoSectionError, +) +import attr +from zope.interface import ( + Attribute, + Interface, + implementer, +) from twisted.internet import defer from twisted.application import service +from twisted.plugin import ( + getPlugins, +) +from eliot import ( + log_call, +) from foolscap.api import eventually -from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though - -from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer +from foolscap.reconnector import ( + ReconnectionInfo, +) +from allmydata.interfaces import ( + IStorageBroker, + IDisplayableServer, + IServer, + IStorageServer, + IFoolscapStoragePlugin, +) from allmydata.util import log, base32, connection_status from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList @@ -64,6 +80,58 @@ from allmydata.util.hashutil import permute_server_hash # look like? # don't pass signatures: only pass validated blessed-objects +@attr.s +class StorageClientConfig(object): + """ + Configuration for a node acting as a storage client. + + :ivar preferred_peers: An iterable of the server-ids (``bytes``) of the + storage servers where share placement is preferred, in order of + decreasing preference. See the *[client]peers.preferred* + documentation for details. + + :ivar dict[unicode, dict[bytes, bytes]] storage_plugins: A mapping from + names of ``IFoolscapStoragePlugin`` configured in *tahoe.cfg* to the + respective configuration. + """ + preferred_peers = attr.ib(default=()) + storage_plugins = attr.ib(default=attr.Factory(dict)) + + @classmethod + def from_node_config(cls, config): + """ + Create a ``StorageClientConfig`` from a complete Tahoe-LAFS node + configuration. + + :param _Config config: The loaded Tahoe-LAFS node configuration. + """ + ps = config.get_config("client", "peers.preferred", b"").split(b",") + preferred_peers = tuple([p.strip() for p in ps if p != b""]) + + enabled_storage_plugins = ( + name.strip() + for name + in config.get_config( + b"client", + b"storage.plugins", + b"", + ).decode("utf-8").split(u",") + if name.strip() + ) + + storage_plugins = {} + for plugin_name in enabled_storage_plugins: + try: + plugin_config = config.items(b"storageclient.plugins." + plugin_name) + except NoSectionError: + plugin_config = [] + storage_plugins[plugin_name] = dict(plugin_config) + + return cls( + preferred_peers, + storage_plugins, + ) + @implementer(IStorageBroker) class StorageFarmBroker(service.MultiService): @@ -72,15 +140,35 @@ class StorageFarmBroker(service.MultiService): remember enough information to establish a connection to it on demand. I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. + + :ivar StorageClientConfig storage_client_config: Values from the node + configuration file relating to storage behavior. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=None, grid_manager_keys=None): + + @property + def preferred_peers(self): + return self.storage_client_config.preferred_peers + + def __init__( + self, + permute_peers, + tub_maker, + node_config, + storage_client_config=None, + grid_manager_keys=None, + ): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker - self.preferred_peers = preferred_peers if preferred_peers else tuple() self._grid_manager_keys = grid_manager_keys if grid_manager_keys else list() + self.node_config = node_config + + if storage_client_config is None: + storage_client_config = StorageClientConfig() + self.storage_client_config = storage_client_config + # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its # own Reconnector, and will give us a RemoteReference when we ask @@ -91,25 +179,73 @@ class StorageFarmBroker(service.MultiService): self._threshold_listeners = [] # tuples of (threshold, Deferred) self._connected_high_water_mark = 0 + @log_call(action_type=u"storage-client:broker:set-static-servers") def set_static_servers(self, servers): - for (server_id, server) in servers.items(): - assert isinstance(server_id, unicode) # from YAML - server_id = server_id.encode("ascii") - self._static_server_ids.add(server_id) - handler_overrides = server.get("connections", {}) - s = NativeStorageServer( - server_id, - server["ann"], - self._tub_maker, - handler_overrides, - self._grid_manager_keys, - [], # XXX FIXME? need grid_manager_certs too? - ) - print("SET STATIC {}".format(s)) - s.on_status_changed(lambda _: self._got_connection()) - s.setServiceParent(self) - self.servers[server_id] = s - s.start_connecting(self._trigger_connections) + # Sorting the items gives us a deterministic processing order. This + # doesn't really matter but it makes the logging behavior more + # predictable and easier to test (and at least one test does depend on + # this sorted order). + for (server_id, server) in sorted(servers.items()): + try: + storage_server = self._make_storage_server(server_id, server) + except Exception: + # TODO: The _make_storage_server failure is logged but maybe + # we should write a traceback here. Notably, tests don't + # automatically fail just because we hit this case. Well + # written tests will still fail if a surprising exception + # arrives here but they might be harder to debug without this + # information. + pass + else: + self._static_server_ids.add(server_id) + self.servers[server_id] = storage_server + storage_server.setServiceParent(self) + storage_server.start_connecting(self._trigger_connections) + + def get_client_storage_plugin_web_resources(self, node_config): + """ + Get all of the client-side ``IResource`` implementations provided by + enabled storage plugins. + + :param allmydata.node._Config node_config: The complete node + configuration for the node from which these web resources will be + served. + + :return dict[unicode, IResource]: Resources for all of the plugins. + """ + plugins = { + plugin.name: plugin + for plugin + in getPlugins(IFoolscapStoragePlugin) + } + return { + name: plugins[name].get_client_resource(node_config) + for (name, config) + in self.storage_client_config.storage_plugins.items() + } + + @log_call( + action_type=u"storage-client:broker:make-storage-server", + include_args=["server_id"], + include_result=False, + ) + def _make_storage_server(self, server_id, server): + assert isinstance(server_id, unicode) # from YAML + server_id = server_id.encode("ascii") + handler_overrides = server.get("connections", {}) + print("ANN", server["ann"]) + s = NativeStorageServer( + server_id, + server["ann"], + self._tub_maker, + handler_overrides, + self.node_config, + self.storage_client_config, + self._grid_manager_keys, + server["ann"].get("grid-manager-certificates", []), + ) + s.on_status_changed(lambda _: self._got_connection()) + return s def when_connected_enough(self, threshold): """ @@ -124,8 +260,11 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], []) - s.rref = rref + s = self._make_storage_server( + serverid.decode("ascii"), + {"ann": ann.copy()}, + ) + s._rref = rref s._is_connected = True self.servers[serverid] = s @@ -138,7 +277,7 @@ class StorageFarmBroker(service.MultiService): ic.subscribe_to("storage", self._got_announcement) def _got_connection(self): - # this is called by NativeStorageClient when it is connected + # this is called by NativeStorageServer when it is connected self._check_connected_high_water_mark() def _check_connected_high_water_mark(self): @@ -165,11 +304,10 @@ class StorageFarmBroker(service.MultiService): facility="tahoe.storage_broker", umid="AlxzqA", level=log.UNUSUAL) return - - grid_manager_certs = ann.get("grid-manager-certificates", []) - print("certs for {}: {}".format(key_s, grid_manager_certs)) - s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, grid_manager_certs) - s.on_status_changed(lambda _: self._got_connection()) + s = self._make_storage_server( + server_id.decode("utf-8"), + {u"ann": ann}, + ) server_id = s.get_serverid() old = self.servers.get(server_id) if old: @@ -263,7 +401,7 @@ class StorageFarmBroker(service.MultiService): # tubids. This clause maps the old tubids to our existing servers. for s in self.servers.values(): if isinstance(s, NativeStorageServer): - if serverid == s._tubid: + if serverid == s.get_tubid(): return s return StubServer(serverid) @@ -348,6 +486,212 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): return True +class IFoolscapStorageServer(Interface): + """ + An internal interface that mediates between ``NativeStorageServer`` and + Foolscap-based ``IStorageServer`` implementations. + """ + nickname = Attribute(""" + A name for this server for presentation to users. + """) + permutation_seed = Attribute(""" + A stable value associated with this server which a client can use as an + input to the server selection permutation ordering. + """) + tubid = Attribute(""" + The identifier for the Tub in which the server is run. + """) + storage_server = Attribute(""" + An IStorageServer provide which implements a concrete Foolscap-based + protocol for communicating with the server. + """) + name = Attribute(""" + Another name for this server for presentation to users. + """) + longname = Attribute(""" + *Another* name for this server for presentation to users. + """) + lease_seed = Attribute(""" + A stable value associated with this server which a client can use as an + input to a lease secret generation function. + """) + + def connect_to(tub, got_connection): + """ + Attempt to establish and maintain a connection to the server. + + :param Tub tub: A Foolscap Tub from which the connection is to + originate. + + :param got_connection: A one-argument callable which is called with a + Foolscap ``RemoteReference`` when a connection is established. + This may be called multiple times if the connection is lost and + then re-established. + + :return foolscap.reconnector.Reconnector: An object which manages the + connection and reconnection attempts. + """ + + +@implementer(IFoolscapStorageServer) +@attr.s(frozen=True) +class _FoolscapStorage(object): + """ + Abstraction for connecting to a storage server exposed via Foolscap. + """ + nickname = attr.ib() + permutation_seed = attr.ib() + tubid = attr.ib() + + storage_server = attr.ib(validator=attr.validators.provides(IStorageServer)) + + _furl = attr.ib() + _short_description = attr.ib() + _long_description = attr.ib() + + + @property + def name(self): + return self._short_description + + @property + def longname(self): + return self._long_description + + @property + def lease_seed(self): + return self.tubid + + @classmethod + def from_announcement(cls, server_id, furl, ann, storage_server): + """ + Create an instance from a fURL and an announcement like:: + + {"permutation-seed-base32": "...", + "nickname": "...", + "grid-manager-certificates": [..], + } + + *nickname* and *grid-manager-certificates* are optional. + """ + m = re.match(r'pb://(\w+)@', furl) + assert m, furl + tubid_s = m.group(1).lower() + tubid = base32.a2b(tubid_s) + if "permutation-seed-base32" in ann: + ps = base32.a2b(str(ann["permutation-seed-base32"])) + elif re.search(r'^v0-[0-9a-zA-Z]{52}$', server_id): + ps = base32.a2b(server_id[3:]) + else: + log.msg("unable to parse serverid '%(server_id)s as pubkey, " + "hashing it to get permutation-seed, " + "may not converge with other clients", + server_id=server_id, + facility="tahoe.storage_broker", + level=log.UNUSUAL, umid="qu86tw") + ps = hashlib.sha256(server_id).digest() + permutation_seed = ps + + assert server_id + long_description = server_id + if server_id.startswith("v0-"): + # remove v0- prefix from abbreviated name + short_description = server_id[3:3+8] + else: + short_description = server_id[:8] + nickname = ann.get("nickname", "") + + ## XXX FIXME post-merge, need to parse out + ## "grid-manager-certificates" and .. do something with them + ## (like store them on this object) + + return cls( + nickname=nickname, + permutation_seed=permutation_seed, + tubid=tubid, + storage_server=storage_server, + furl=furl, + short_description=short_description, + long_description=long_description, + ) + + def connect_to(self, tub, got_connection): + return tub.connectTo(self._furl, got_connection) + + +@implementer(IFoolscapStorageServer) +class _NullStorage(object): + """ + Abstraction for *not* communicating with a storage server of a type with + which we can't communicate. + """ + nickname = "" + permutation_seed = hashlib.sha256("").digest() + tubid = hashlib.sha256("").digest() + storage_server = None + + lease_seed = hashlib.sha256("").digest() + + name = "" + longname = "" + + def connect_to(self, tub, got_connection): + return NonReconnector() + + +class NonReconnector(object): + """ + A ``foolscap.reconnector.Reconnector``-alike that doesn't do anything. + """ + def stopConnecting(self): + pass + + def reset(self): + pass + + def getReconnectionInfo(self): + return ReconnectionInfo() + +_null_storage = _NullStorage() + + +class AnnouncementNotMatched(Exception): + """ + A storage server announcement wasn't matched by any of the locally enabled + plugins. + """ + + +def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): + """ + Construct an ``IStorageServer`` from the most locally-preferred plugin + that is offered in the given announcement. + + :param allmydata.node._Config node_config: The node configuration to + pass to the plugin. + """ + plugins = { + plugin.name: plugin + for plugin + in getPlugins(IFoolscapStoragePlugin) + } + storage_options = announcement.get(u"storage-options", []) + for plugin_name, plugin_config in config.storage_plugins.items(): + try: + plugin = plugins[plugin_name] + except KeyError: + raise ValueError("{} not installed".format(plugin_name)) + for option in storage_options: + if plugin_name == option[u"name"]: + furl = option[u"storage-server-FURL"] + return furl, plugin.get_storage_client( + node_config, + option, + get_rref, + ) + raise AnnouncementNotMatched() + + @implementer(IServer) class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. @@ -376,8 +720,8 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): - # print("CREATE {}: {}".format(server_id, grid_manager_certs)) + def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=None, + grid_manager_keys=None, grid_manager_certs=None): service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id @@ -385,6 +729,9 @@ class NativeStorageServer(service.MultiService): self._tub_maker = tub_maker self._handler_overrides = handler_overrides + if config is None: + config = StorageClientConfig() + # XXX we should validate as much as we can about the # certificates right now -- the only thing we HAVE to be lazy # about is the expiry, which should be checked before any @@ -392,45 +739,18 @@ class NativeStorageServer(service.MultiService): # any public-keys which the user has configured (if none, it # means use any storage servers) - self._grid_manager_keys = grid_manager_keys - # print("keys: {}".format(self._grid_manager_keys)) + self._grid_manager_keys = grid_manager_keys or [] + # any storage-certificates that this storage-server included # in its announcement - self._grid_manager_certificates = grid_manager_certs - # print("certs: {}".format(self._grid_manager_certificates)) + self._grid_manager_certificates = grid_manager_certs or [] - assert "anonymous-storage-FURL" in ann, ann - furl = str(ann["anonymous-storage-FURL"]) - m = re.match(r'pb://(\w+)@', furl) - assert m, furl - tubid_s = m.group(1).lower() - self._tubid = base32.a2b(tubid_s) - if "permutation-seed-base32" in ann: - ps = base32.a2b(str(ann["permutation-seed-base32"])) - elif re.search(r'^v0-[0-9a-zA-Z]{52}$', server_id): - ps = base32.a2b(server_id[3:]) - else: - log.msg("unable to parse serverid '%(server_id)s as pubkey, " - "hashing it to get permutation-seed, " - "may not converge with other clients", - server_id=server_id, - facility="tahoe.storage_broker", - level=log.UNUSUAL, umid="qu86tw") - ps = hashlib.sha256(server_id).digest() - self._permutation_seed = ps - - assert server_id - self._long_description = server_id - if server_id.startswith("v0-"): - # remove v0- prefix from abbreviated name - self._short_description = server_id[3:3+8] - else: - self._short_description = server_id[:8] + self._storage = self._make_storage_system(node_config, config, ann) self.last_connect_time = None self.last_loss_time = None self.remote_host = None - self.rref = None + self._rref = None self._is_connected = False self._reconnector = None self._trigger_cb = None @@ -466,6 +786,79 @@ class NativeStorageServer(service.MultiService): # print("didn't validate {} keys".format(len(self._grid_manager_keys))) return False + def _make_storage_system(self, node_config, config, ann): + """ + :param allmydata.node._Config node_config: The node configuration to pass + to any configured storage plugins. + + :param StorageClientConfig config: Configuration specifying desired + storage client behavior. + + :param dict ann: The storage announcement from the storage server we + are meant to communicate with. + + :return IFoolscapStorageServer: An object enabling communication via + Foolscap with the server which generated the announcement. + """ + # Try to match the announcement against a plugin. + try: + furl, storage_server = _storage_from_foolscap_plugin( + node_config, + config, + ann, + # Pass in an accessor for our _rref attribute. The value of + # the attribute may change over time as connections are lost + # and re-established. The _StorageServer should always be + # able to get the most up-to-date value. + self.get_rref, + ) + except AnnouncementNotMatched: + # Nope. + pass + else: + return _FoolscapStorage.from_announcement( + self._server_id, + furl.encode("utf-8"), + ann, + storage_server, + ) + + # Try to match the announcement against the anonymous access scheme. + try: + furl = ann[u"anonymous-storage-FURL"] + except KeyError: + # Nope + pass + else: + # See comment above for the _storage_from_foolscap_plugin case + # about passing in get_rref. + storage_server = _StorageServer(get_rref=self.get_rref) + return _FoolscapStorage.from_announcement( + self._server_id, + furl.encode("utf-8"), + ann, + storage_server, + ) + + # Nothing matched so we can't talk to this server. + return _null_storage + + def get_permutation_seed(self): + return self._storage.permutation_seed + def get_name(self): # keep methodname short + # TODO: decide who adds [] in the short description. It should + # probably be the output side, not here. + return self._storage.name + def get_longname(self): + return self._storage.longname + def get_tubid(self): + return self._storage.tubid + def get_lease_seed(self): + return self._storage.lease_seed + def get_foolscap_write_enabler_seed(self): + return self._storage.tubid + def get_nickname(self): + return self._storage.nickname def on_status_changed(self, status_changed): """ @@ -487,25 +880,10 @@ class NativeStorageServer(service.MultiService): return "" % self.get_name() def get_serverid(self): return self._server_id - def get_permutation_seed(self): - return self._permutation_seed def get_version(self): - if self.rref: - return self.rref.version + if self._rref: + return self._rref.version return None - def get_name(self): # keep methodname short - # TODO: decide who adds [] in the short description. It should - # probably be the output side, not here. - return self._short_description - def get_longname(self): - return self._long_description - def get_lease_seed(self): - return self._tubid - def get_foolscap_write_enabler_seed(self): - return self._tubid - - def get_nickname(self): - return self.announcement.get("nickname", "") def get_announcement(self): return self.announcement def get_remote_host(self): @@ -513,8 +891,8 @@ class NativeStorageServer(service.MultiService): def get_connection_status(self): last_received = None - if self.rref: - last_received = self.rref.getDataLastReceivedAt() + if self._rref: + last_received = self._rref.getDataLastReceivedAt() return connection_status.from_foolscap_reconnector(self._reconnector, last_received) @@ -531,13 +909,11 @@ class NativeStorageServer(service.MultiService): available_space = protocol_v1_version.get('maximum-immutable-share-size', None) return available_space - def start_connecting(self, trigger_cb): self._tub = self._tub_maker(self._handler_overrides) self._tub.setServiceParent(self) - furl = str(self.announcement["anonymous-storage-FURL"]) self._trigger_cb = trigger_cb - self._reconnector = self._tub.connectTo(furl, self._got_connection) + self._reconnector = self._storage.connect_to(self._tub, self._got_connection) def _got_connection(self, rref): lp = log.msg(format="got connection to %(name)s, getting versions", @@ -560,18 +936,26 @@ class NativeStorageServer(service.MultiService): self.last_connect_time = time.time() self.remote_host = rref.getLocationHints() - self.rref = rref + self._rref = rref self._is_connected = True rref.notifyOnDisconnect(self._lost) def get_rref(self): - return self.rref + return self._rref + + def get_storage_server(self): + """ + See ``IServer.get_storage_server``. + """ + if self._rref is None: + return None + return self._storage.storage_server def _lost(self): log.msg(format="lost connection to %(name)s", name=self.get_name(), facility="tahoe.storage_broker", umid="zbRllw") self.last_loss_time = time.time() - # self.rref is now stale: all callRemote()s will get a + # self._rref is now stale: all callRemote()s will get a # DeadReferenceError. We leave the stale reference in place so that # uploader/downloader code (which received this IServer through # get_connected_servers() or get_servers_for_psi()) can continue to @@ -589,3 +973,117 @@ class NativeStorageServer(service.MultiService): class UnknownServerTypeError(Exception): pass + + +@implementer(IStorageServer) +@attr.s +class _StorageServer(object): + """ + ``_StorageServer`` is a direct pass-through to an ``RIStorageServer`` via + a ``RemoteReference``. + """ + _get_rref = attr.ib() + + @property + def _rref(self): + return self._get_rref() + + def get_version(self): + return self._rref.callRemote( + "get_version", + ) + + def allocate_buckets( + self, + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ): + return self._rref.callRemote( + "allocate_buckets", + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ) + + def add_lease( + self, + storage_index, + renew_secret, + cancel_secret, + ): + return self._rref.callRemote( + "add_lease", + storage_index, + renew_secret, + cancel_secret, + ) + + def renew_lease( + self, + storage_index, + renew_secret, + ): + return self._rref.callRemote( + "renew_lease", + storage_index, + renew_secret, + ) + + def get_buckets( + self, + storage_index, + ): + return self._rref.callRemote( + "get_buckets", + storage_index, + ) + + def slot_readv( + self, + storage_index, + shares, + readv, + ): + return self._rref.callRemote( + "slot_readv", + storage_index, + shares, + readv, + ) + + def slot_testv_and_readv_and_writev( + self, + storage_index, + secrets, + tw_vectors, + r_vector, + ): + return self._rref.callRemote( + "slot_testv_and_readv_and_writev", + storage_index, + secrets, + tw_vectors, + r_vector, + ) + + def advise_corrupt_share( + self, + share_type, + storage_index, + shnum, + reason, + ): + return self._rref.callRemoteOnly( + "advise_corrupt_share", + share_type, + storage_index, + shnum, + reason, + ) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 4ee3e8060..abbde919f 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -1,6 +1,27 @@ +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Some setup that should apply across the entire test suite. + +Rather than defining interesting APIs for other code to use, this just causes +some side-effects which make things better when the test suite runs. +""" + +from traceback import extract_stack, format_list +from foolscap.pb import Listener +from twisted.python.log import err +from twisted.application import service + from foolscap.logging.incident import IncidentQualifier -class NonQualifier(IncidentQualifier): +class NonQualifier(IncidentQualifier, object): def check_event(self, ev): return False @@ -52,6 +73,39 @@ def _configure_hypothesis(): settings.load_profile(profile_name) _configure_hypothesis() +def logging_for_pb_listener(): + """ + Make Foolscap listen error reports include Listener creation stack + information. + """ + original__init__ = Listener.__init__ + def _listener__init__(self, *a, **kw): + original__init__(self, *a, **kw) + # Capture the stack here, where Listener is instantiated. This is + # likely to explain what code is responsible for this Listener, useful + # information to have when the Listener eventually fails to listen. + self._creation_stack = extract_stack() + + # Override the Foolscap implementation with one that has an errback + def _listener_startService(self): + service.Service.startService(self) + d = self._ep.listen(self) + def _listening(lp): + self._lp = lp + d.addCallbacks( + _listening, + # Make sure that this listen failure is reported promptly and with + # the creation stack. + err, + errbackArgs=( + "Listener created at {}".format( + "".join(format_list(self._creation_stack)), + ), + ), + ) + Listener.__init__ = _listener__init__ + Listener.startService = _listener_startService +logging_for_pb_listener() import sys if sys.platform == "win32": diff --git a/src/allmydata/test/_twisted_9607.py b/src/allmydata/test/_twisted_9607.py index 8f5d3845c..c4e37ef38 100644 --- a/src/allmydata/test/_twisted_9607.py +++ b/src/allmydata/test/_twisted_9607.py @@ -15,7 +15,7 @@ from io import BytesIO from twisted.internet import protocol, defer -class _EverythingGetter(protocol.ProcessProtocol): +class _EverythingGetter(protocol.ProcessProtocol, object): def __init__(self, deferred, stdinBytes=None): self.deferred = deferred diff --git a/src/allmydata/test/bench_dirnode.py b/src/allmydata/test/bench_dirnode.py deleted file mode 100644 index 2b3a11a60..000000000 --- a/src/allmydata/test/bench_dirnode.py +++ /dev/null @@ -1,139 +0,0 @@ -from __future__ import print_function - -import hotshot.stats, os, random, sys - -from pyutil import benchutil, randutil # http://tahoe-lafs.org/trac/pyutil - -from zope.interface import implementer -from allmydata import dirnode, uri -from allmydata.interfaces import IFileNode -from allmydata.mutable.filenode import MutableFileNode -from allmydata.immutable.filenode import ImmutableFileNode - -@implementer(IFileNode) -class ContainerNode(object): - # dirnodes sit on top of a "container" filenode, from which it extracts a - # writekey - def __init__(self): - self._writekey = randutil.insecurerandstr(16) - self._fingerprint = randutil.insecurerandstr(32) - self._cap = uri.WriteableSSKFileURI(self._writekey, self._fingerprint) - def get_writekey(self): - return self._writekey - def get_cap(self): - return self._cap - def get_uri(self): - return self._cap.to_string() - def is_readonly(self): - return False - def is_mutable(self): - return True - -class FakeNode: - def raise_error(self): - return None - -class FakeNodeMaker: - def create_from_cap(self, writecap, readcap=None, deep_immutable=False, name=''): - return FakeNode() - -def random_unicode(n=140, b=3, codec='utf-8'): - l = [] - while len(l) < n: - try: - u = os.urandom(b).decode(codec)[0] - except UnicodeDecodeError: - pass - else: - l.append(u) - return u''.join(l) - -encoding_parameters = {"k": 3, "n": 10} -def random_metadata(): - d = {} - d['tahoe'] = {} - d['tahoe']['linkcrtime'] = random.random() - d['tahoe']['linkmotime'] = random.random() - return d - -PROF_FILE_NAME="bench_dirnode.prof" - -class B(object): - def __init__(self): - self.children = [] # tuples of (k, v) (suitable for passing to dict()) - self.packstr = None - self.nodemaker = FakeNodeMaker() - self.testdirnode = dirnode.DirectoryNode(ContainerNode(), self.nodemaker, uploader=None) - - def random_fsnode(self): - coin = random.randrange(0, 3) - if coin == 0: - cap = uri.CHKFileURI(randutil.insecurerandstr(16), - randutil.insecurerandstr(32), - random.randrange(1, 5), - random.randrange(6, 15), - random.randrange(99, 1000000000000)) - return ImmutableFileNode(cap, None, None, None, None, None) - elif coin == 1: - cap = uri.WriteableSSKFileURI(randutil.insecurerandstr(16), - randutil.insecurerandstr(32)) - n = MutableFileNode(None, None, encoding_parameters, None) - return n.init_from_cap(cap) - else: - assert coin == 2 - cap = uri.WriteableSSKFileURI(randutil.insecurerandstr(16), - randutil.insecurerandstr(32)) - n = MutableFileNode(None, None, encoding_parameters, None) - n.init_from_cap(cap) - return dirnode.DirectoryNode(n, self.nodemaker, uploader=None) - - def random_child(self): - return self.random_fsnode(), random_metadata() - - def init_for_pack(self, N): - for i in xrange(len(self.children), N): - name = random_unicode(random.randrange(0, 10)) - self.children.append( (name, self.random_child()) ) - - def init_for_unpack(self, N): - self.init_for_pack(N) - self.packstr = self.pack(N) - - def pack(self, N): - return self.testdirnode._pack_contents(dict(self.children[:N])) - - def unpack(self, N): - return self.testdirnode._unpack_contents(self.packstr) - - def unpack_and_repack(self, N): - return self.testdirnode._pack_contents(self.testdirnode._unpack_contents(self.packstr)) - - def run_benchmarks(self, profile=False): - for (initfunc, func) in [(self.init_for_unpack, self.unpack), - (self.init_for_pack, self.pack), - (self.init_for_unpack, self.unpack_and_repack)]: - print("benchmarking %s" % (func,)) - for N in 16, 512, 2048, 16384: - print("%5d" % N, end=' ') - benchutil.rep_bench(func, N, initfunc=initfunc, MAXREPS=20, UNITS_PER_SECOND=1000) - benchutil.print_bench_footer(UNITS_PER_SECOND=1000) - print("(milliseconds)") - - def prof_benchmarks(self): - # This requires pyutil >= v1.3.34. - self.run_benchmarks(profile=True) - - def print_stats(self): - s = hotshot.stats.load(PROF_FILE_NAME) - s.strip_dirs().sort_stats("time").print_stats(32) - -if __name__ == "__main__": - if '--profile' in sys.argv: - if os.path.exists(PROF_FILE_NAME): - print("WARNING: profiling results file '%s' already exists -- the profiling results from this run will be added into the profiling results stored in that file and then the sum of them will be printed out after this run." % (PROF_FILE_NAME,)) - b = B() - b.prof_benchmarks() - b.print_stats() - else: - b = B() - b.run_benchmarks() diff --git a/src/allmydata/test/check_grid.py b/src/allmydata/test/check_grid.py index b47067e1e..d3993ee5e 100644 --- a/src/allmydata/test/check_grid.py +++ b/src/allmydata/test/check_grid.py @@ -72,7 +72,7 @@ class GridTesterOptions(usage.Options): class CommandFailed(Exception): pass -class GridTester: +class GridTester(object): def __init__(self, config): self.config = config self.tahoe = config.tahoe @@ -140,7 +140,7 @@ class GridTester: if f not in oldfiles: raise CommandFailed("um, '%s' was supposed to already be in %s" % (f, dirname)) - self.cli("rm", absfilename) + self.cli("unlink", absfilename) newfiles = self.listdir(dirname) if f in newfiles: raise CommandFailed("failed to remove '%s' from %s" % (f, dirname)) diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 33e593bf8..abb903240 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -270,4 +270,3 @@ while True: f.write("directories-written: %d\n" % directories_written) f.close() os.rename(stats_out+".tmp", stats_out) - diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index a6e35be09..e12ed3d54 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -14,7 +14,7 @@ from allmydata.util.encodingutil import get_filesystem_encoding from foolscap.api import Tub, fireEventually, flushEventualQueue from twisted.python import log, procutils -class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter): +class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter, object): full_speed_ahead = False _bytes_so_far = 0 stalled = None @@ -41,7 +41,7 @@ class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter): self.stalled = None return tw_client.HTTPPageGetter.handleResponseEnd(self) -class StallableDiscardingHTTPClientFactory(tw_client.HTTPClientFactory): +class StallableDiscardingHTTPClientFactory(tw_client.HTTPClientFactory, object): protocol = StallableHTTPGetterDiscarder def discardPage(url, stall=False, *args, **kwargs): @@ -477,7 +477,7 @@ this file are ignored. return d -class ClientWatcher(protocol.ProcessProtocol): +class ClientWatcher(protocol.ProcessProtocol, object): ended = False def outReceived(self, data): print("OUT:", data) @@ -504,4 +504,3 @@ if __name__ == '__main__': # removed each time we run. sf = SystemFramework("_test_memory", mode) sf.run() - diff --git a/src/allmydata/test/check_speed.py b/src/allmydata/test/check_speed.py index c23938337..2fce53387 100644 --- a/src/allmydata/test/check_speed.py +++ b/src/allmydata/test/check_speed.py @@ -8,7 +8,7 @@ from foolscap.api import Tub, fireEventually MB = 1000000 -class SpeedTest: +class SpeedTest(object): DO_IMMUTABLE = True DO_MUTABLE_CREATE = True DO_MUTABLE = True diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 3f695f414..1e77325bb 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,9 +1,8 @@ - import os.path from six.moves import cStringIO as StringIO import urllib, sys import re -from mock import patch +from mock import patch, Mock from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher @@ -11,14 +10,14 @@ from twisted.internet import task from twisted.python.filepath import FilePath import allmydata -from allmydata.util import fileutil, hashutil, base32, keyutil +from allmydata.crypto import ed25519 +from allmydata.util import fileutil, hashutil, base32 from allmydata.util.namespace import Namespace from allmydata import uri from allmydata.immutable import upload from allmydata.dirnode import normalize from allmydata.scripts.common_http import socket_error import allmydata.scripts.common_http -from pycryptopp.publickey import ed25519 # Test that the scripts can be imported. from allmydata.scripts import create_node, debug, tahoe_start, tahoe_restart, \ @@ -35,10 +34,10 @@ from allmydata.scripts.common import DEFAULT_ALIAS, get_aliases, get_alias, \ DefaultAliasMarker from allmydata.scripts import cli, debug, runner -from ..common_util import (ReallyEqualMixin, skip_if_cannot_represent_filename, - run_cli) -from ..no_network import GridTestMixin -from .common import CLITestMixin, parse_options +from allmydata.test.common_util import (ReallyEqualMixin, skip_if_cannot_represent_filename, + run_cli) +from allmydata.test.no_network import GridTestMixin +from allmydata.test.cli.common import CLITestMixin, parse_options from twisted.python import usage from allmydata.util.encodingutil import listdir_unicode, get_io_encoding @@ -526,7 +525,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnlessEqual(exitcode, 1) def fake_react(f): - d = f("reactor") + reactor = Mock() + d = f(reactor) # normally this Deferred would be errbacked with SystemExit, but # since we mocked out sys.exit, it will be fired with None. So # it's safe to drop it on the floor. @@ -570,10 +570,6 @@ class Help(unittest.TestCase): help = str(cli.UnlinkOptions()) self.failUnlessIn("[options] REMOTE_FILE", help) - def test_rm(self): - help = str(cli.RmOptions()) - self.failUnlessIn("[options] REMOTE_FILE", help) - def test_mv(self): help = str(cli.MvOptions()) self.failUnlessIn("[options] FROM TO", help) @@ -734,16 +730,20 @@ class Admin(unittest.TestCase): self.failUnlessEqual(pubkey_bits[0], vk_header, lines[1]) self.failUnless(privkey_bits[1].startswith("priv-v0-"), lines[0]) self.failUnless(pubkey_bits[1].startswith("pub-v0-"), lines[1]) - sk_bytes = base32.a2b(keyutil.remove_prefix(privkey_bits[1], "priv-v0-")) - sk = ed25519.SigningKey(sk_bytes) - vk_bytes = base32.a2b(keyutil.remove_prefix(pubkey_bits[1], "pub-v0-")) - self.failUnlessEqual(sk.get_verifying_key_bytes(), vk_bytes) + sk, pk = ed25519.signing_keypair_from_string(privkey_bits[1]) + vk_bytes = pubkey_bits[1] + self.assertEqual( + ed25519.string_from_verifying_key(pk), + vk_bytes, + ) d.addCallback(_done) return d def test_derive_pubkey(self): - priv1,pub1 = keyutil.make_keypair() - d = run_cli("admin", "derive-pubkey", priv1) + priv_key, pub_key = ed25519.create_signing_keypair() + priv_key_str = ed25519.string_from_signing_key(priv_key) + pub_key_str = ed25519.string_from_verifying_key(pub_key) + d = run_cli("admin", "derive-pubkey", priv_key_str) def _done(args): (rc, stdout, stderr) = args lines = stdout.split("\n") @@ -753,8 +753,8 @@ class Admin(unittest.TestCase): vk_header = "public: pub-v0-" self.failUnless(privkey_line.startswith(sk_header), privkey_line) self.failUnless(pubkey_line.startswith(vk_header), pubkey_line) - pub2 = pubkey_line[len(vk_header):] - self.failUnlessEqual("pub-v0-"+pub2, pub1) + pub_key_str2 = pubkey_line[len(vk_header):] + self.assertEqual("pub-v0-" + pub_key_str2, pub_key_str) d.addCallback(_done) return d @@ -911,8 +911,14 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(err, "") self.failUnlessIn(st, out) return out + def _mkdir(ign, mutable_type, uri_prefix, dirname): - d2 = self.do_cli("mkdir", "--format="+mutable_type, dirname) + """ + :param str mutable_type: 'sdmf' or 'mdmf' (or uppercase versions) + :param str uri_prefix: kind of URI + :param str dirname: the directory alias + """ + d2 = self.do_cli("mkdir", "--format={}".format(mutable_type), dirname) d2.addCallback(_check, uri_prefix) def _stash_filecap(cap): u = uri.from_string(cap) @@ -1071,11 +1077,6 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): return d -class Rm(Unlink): - """Test that 'tahoe rm' behaves in the same way as 'tahoe unlink'.""" - command = "rm" - - class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): def test_empty_directory(self): self.basedir = "cli/Stats/empty_directory" @@ -1342,8 +1343,8 @@ class Stop(unittest.TestCase): class Start(unittest.TestCase): - @patch('allmydata.scripts.tahoe_daemonize.os.chdir') - @patch('allmydata.scripts.tahoe_daemonize.twistd') + @patch('allmydata.scripts.run_common.os.chdir') + @patch('allmydata.scripts.run_common.twistd') def test_non_numeric_pid(self, mock_twistd, chdir): """ If the pidfile exists but does not contain a numeric value, a complaint to diff --git a/src/allmydata/test/cli/test_daemonize.py b/src/allmydata/test/cli/test_daemonize.py index 78df0aa1a..b1365329a 100644 --- a/src/allmydata/test/cli/test_daemonize.py +++ b/src/allmydata/test/cli/test_daemonize.py @@ -1,16 +1,28 @@ import os +from io import ( + BytesIO, +) from os.path import dirname, join from mock import patch, Mock from six.moves import StringIO from sys import getfilesystemencoding from twisted.trial import unittest from allmydata.scripts import runner -from allmydata.scripts.tahoe_daemonize import identify_node_type -from allmydata.scripts.tahoe_daemonize import DaemonizeTahoeNodePlugin -from allmydata.scripts.tahoe_daemonize import DaemonizeOptions +from allmydata.scripts.run_common import ( + identify_node_type, + DaemonizeTahoeNodePlugin, + MyTwistdConfig, +) +from allmydata.scripts.tahoe_daemonize import ( + DaemonizeOptions, +) class Util(unittest.TestCase): + def setUp(self): + self.twistd_options = MyTwistdConfig() + self.twistd_options.parseOptions(["DaemonizeTahoeNode"]) + self.options = self.twistd_options.subOptions def test_node_type_nothing(self): tmpdir = self.mktemp() @@ -39,7 +51,7 @@ class Util(unittest.TestCase): fn() r.stop = lambda: None r.callWhenRunning = call - service = plug.makeService(None) + service = plug.makeService(self.options) service.parent = Mock() service.startService() @@ -47,6 +59,7 @@ class Util(unittest.TestCase): def test_daemonize_no_keygen(self): tmpdir = self.mktemp() + stderr = BytesIO() plug = DaemonizeTahoeNodePlugin('key-generator', tmpdir) with patch('twisted.internet.reactor') as r: @@ -54,8 +67,8 @@ class Util(unittest.TestCase): d = fn() d.addErrback(lambda _: None) # ignore the error we'll trigger r.callWhenRunning = call - r.stop = 'foo' - service = plug.makeService(None) + service = plug.makeService(self.options) + service.stderr = stderr service.parent = Mock() # we'll raise ValueError because there's no key-generator # .. BUT we do this in an async function called via @@ -65,7 +78,7 @@ class Util(unittest.TestCase): def done(f): self.assertIn( "key-generator support removed", - str(f), + stderr.getvalue(), ) return None d.addBoth(done) @@ -80,7 +93,7 @@ class Util(unittest.TestCase): fn() r.stop = lambda: None r.callWhenRunning = call - service = plug.makeService(None) + service = plug.makeService(self.options) service.parent = Mock() with self.assertRaises(ValueError) as ctx: service.startService() @@ -109,7 +122,7 @@ class RunDaemonizeTests(unittest.TestCase): d = super(RunDaemonizeTests, self).setUp() self._reactor = patch('twisted.internet.reactor') self._reactor.stop = lambda: None - self._twistd = patch('allmydata.scripts.tahoe_daemonize.twistd') + self._twistd = patch('allmydata.scripts.run_common.twistd') self.node_dir = self.mktemp() os.mkdir(self.node_dir) for cm in [self._reactor, self._twistd]: diff --git a/src/allmydata/test/cli/test_magic_folder.py b/src/allmydata/test/cli/test_magic_folder.py deleted file mode 100644 index b8d26b893..000000000 --- a/src/allmydata/test/cli/test_magic_folder.py +++ /dev/null @@ -1,814 +0,0 @@ -import json -import shutil -import os.path -import mock -import re -import time -from datetime import datetime - -from eliot import ( - log_call, - start_action, -) -from eliot.twisted import ( - DeferredContext, -) - -from twisted.trial import unittest -from twisted.internet import defer -from twisted.internet import reactor -from twisted.python import usage - -from allmydata.util.assertutil import precondition -from allmydata.util import fileutil -from allmydata.scripts.common import get_aliases -from ..no_network import GridTestMixin -from ..common_util import parse_cli -from .common import CLITestMixin -from allmydata.test.common_util import NonASCIIPathMixin -from allmydata.scripts import magic_folder_cli -from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import unicode_to_argv -from allmydata.frontends.magic_folder import MagicFolder -from allmydata import uri -from ...util.eliotutil import ( - log_call_deferred, -) - -class MagicFolderCLITestMixin(CLITestMixin, GridTestMixin, NonASCIIPathMixin): - def setUp(self): - GridTestMixin.setUp(self) - self.alice_nickname = self.unicode_or_fallback(u"Alice\u00F8", u"Alice", io_as_well=True) - self.bob_nickname = self.unicode_or_fallback(u"Bob\u00F8", u"Bob", io_as_well=True) - - def do_create_magic_folder(self, client_num): - with start_action(action_type=u"create-magic-folder", client_num=client_num).context(): - d = DeferredContext( - self.do_cli( - "magic-folder", "--debug", "create", "magic:", - client_num=client_num, - ) - ) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0, stdout + stderr) - self.assertIn("Alias 'magic' created", stdout) -# self.failUnlessIn("joined new magic-folder", stdout) -# self.failUnlessIn("Successfully created magic-folder", stdout) - self.failUnlessEqual(stderr, "") - aliases = get_aliases(self.get_clientdir(i=client_num)) - self.assertIn("magic", aliases) - self.failUnless(aliases["magic"].startswith("URI:DIR2:")) - d.addCallback(_done) - return d.addActionFinish() - - def do_invite(self, client_num, nickname): - nickname_arg = unicode_to_argv(nickname) - action = start_action( - action_type=u"invite-to-magic-folder", - client_num=client_num, - nickname=nickname, - ) - with action.context(): - d = DeferredContext( - self.do_cli( - "magic-folder", - "invite", - "magic:", - nickname_arg, - client_num=client_num, - ) - ) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - return (rc, stdout, stderr) - d.addCallback(_done) - return d.addActionFinish() - - def do_list(self, client_num, json=False): - args = ("magic-folder", "list",) - if json: - args = args + ("--json",) - d = self.do_cli(*args, client_num=client_num) - def _done(args): - (rc, stdout, stderr) = args - return (rc, stdout, stderr) - d.addCallback(_done) - return d - - def do_status(self, client_num, name=None): - args = ("magic-folder", "status",) - if name is not None: - args = args + ("--name", name) - d = self.do_cli(*args, client_num=client_num) - def _done(args): - (rc, stdout, stderr) = args - return (rc, stdout, stderr) - d.addCallback(_done) - return d - - def do_join(self, client_num, local_dir, invite_code): - action = start_action( - action_type=u"join-magic-folder", - client_num=client_num, - local_dir=local_dir, - invite_code=invite_code, - ) - with action.context(): - precondition(isinstance(local_dir, unicode), local_dir=local_dir) - precondition(isinstance(invite_code, str), invite_code=invite_code) - local_dir_arg = unicode_to_argv(local_dir) - d = DeferredContext( - self.do_cli( - "magic-folder", - "join", - invite_code, - local_dir_arg, - client_num=client_num, - ) - ) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(stdout, "") - self.failUnlessEqual(stderr, "") - return (rc, stdout, stderr) - d.addCallback(_done) - return d.addActionFinish() - - def do_leave(self, client_num): - d = self.do_cli("magic-folder", "leave", client_num=client_num) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - return (rc, stdout, stderr) - d.addCallback(_done) - return d - - def check_joined_config(self, client_num, upload_dircap): - """Tests that our collective directory has the readonly cap of - our upload directory. - """ - action = start_action(action_type=u"check-joined-config") - with action.context(): - collective_readonly_cap = self.get_caps_from_files(client_num)[0] - d = DeferredContext( - self.do_cli( - "ls", "--json", - collective_readonly_cap, - client_num=client_num, - ) - ) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - return (rc, stdout, stderr) - d.addCallback(_done) - def test_joined_magic_folder(args): - (rc, stdout, stderr) = args - readonly_cap = unicode(uri.from_string(upload_dircap).get_readonly().to_string(), 'utf-8') - s = re.search(readonly_cap, stdout) - self.failUnless(s is not None) - return None - d.addCallback(test_joined_magic_folder) - return d.addActionFinish() - - def get_caps_from_files(self, client_num): - from allmydata.frontends.magic_folder import load_magic_folders - folders = load_magic_folders(self.get_clientdir(i=client_num)) - mf = folders["default"] - return mf['collective_dircap'], mf['upload_dircap'] - - @log_call - def check_config(self, client_num, local_dir): - client_config = fileutil.read(os.path.join(self.get_clientdir(i=client_num), "tahoe.cfg")) - mf_yaml = fileutil.read(os.path.join(self.get_clientdir(i=client_num), "private", "magic_folders.yaml")) - local_dir_utf8 = local_dir.encode('utf-8') - magic_folder_config = "[magic_folder]\nenabled = True" - self.assertIn(magic_folder_config, client_config) - self.assertIn(local_dir_utf8, mf_yaml) - - def create_invite_join_magic_folder(self, nickname, local_dir): - nickname_arg = unicode_to_argv(nickname) - local_dir_arg = unicode_to_argv(local_dir) - # the --debug means we get real exceptions on failures - d = self.do_cli("magic-folder", "--debug", "create", "magic:", nickname_arg, local_dir_arg) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0, stdout + stderr) - - client = self.get_client() - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - self.collective_dirnode = client.create_node_from_uri(self.collective_dircap) - self.upload_dirnode = client.create_node_from_uri(self.upload_dircap) - d.addCallback(_done) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, local_dir)) - return d - - # XXX should probably just be "tearDown"... - @log_call_deferred(action_type=u"test:cli:magic-folder:cleanup") - def cleanup(self, res): - d = DeferredContext(defer.succeed(None)) - def _clean(ign): - return self.magicfolder.disownServiceParent() - - d.addCallback(_clean) - d.addCallback(lambda ign: res) - return d.result - - def init_magicfolder(self, client_num, upload_dircap, collective_dircap, local_magic_dir, clock): - dbfile = abspath_expanduser_unicode(u"magicfolder_default.sqlite", base=self.get_clientdir(i=client_num)) - magicfolder = MagicFolder( - client=self.get_client(client_num), - upload_dircap=upload_dircap, - collective_dircap=collective_dircap, - local_path_u=local_magic_dir, - dbfile=dbfile, - umask=0o077, - name='default', - clock=clock, - uploader_delay=0.2, - downloader_delay=0, - ) - - magicfolder.setServiceParent(self.get_client(client_num)) - magicfolder.ready() - return magicfolder - - def setup_alice_and_bob(self, alice_clock=reactor, bob_clock=reactor): - self.set_up_grid(num_clients=2, oneshare=True) - - self.alice_magicfolder = None - self.bob_magicfolder = None - - alice_magic_dir = abspath_expanduser_unicode(u"Alice-magic", base=self.basedir) - self.mkdir_nonascii(alice_magic_dir) - bob_magic_dir = abspath_expanduser_unicode(u"Bob-magic", base=self.basedir) - self.mkdir_nonascii(bob_magic_dir) - - # Alice creates a Magic Folder, invites herself and joins. - d = self.do_create_magic_folder(0) - d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname)) - def get_invite_code(result): - self.invite_code = result[1].strip() - d.addCallback(get_invite_code) - d.addCallback(lambda ign: self.do_join(0, alice_magic_dir, self.invite_code)) - def get_alice_caps(ign): - self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_alice_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, alice_magic_dir)) - def get_Alice_magicfolder(result): - self.alice_magicfolder = self.init_magicfolder(0, self.alice_upload_dircap, - self.alice_collective_dircap, - alice_magic_dir, alice_clock) - return result - d.addCallback(get_Alice_magicfolder) - - # Alice invites Bob. Bob joins. - d.addCallback(lambda ign: self.do_invite(0, self.bob_nickname)) - def get_invite_code(result): - self.invite_code = result[1].strip() - d.addCallback(get_invite_code) - d.addCallback(lambda ign: self.do_join(1, bob_magic_dir, self.invite_code)) - def get_bob_caps(ign): - self.bob_collective_dircap, self.bob_upload_dircap = self.get_caps_from_files(1) - d.addCallback(get_bob_caps) - d.addCallback(lambda ign: self.check_joined_config(1, self.bob_upload_dircap)) - d.addCallback(lambda ign: self.check_config(1, bob_magic_dir)) - def get_Bob_magicfolder(result): - self.bob_magicfolder = self.init_magicfolder(1, self.bob_upload_dircap, - self.bob_collective_dircap, - bob_magic_dir, bob_clock) - return result - d.addCallback(get_Bob_magicfolder) - return d - - -class ListMagicFolder(MagicFolderCLITestMixin, unittest.TestCase): - - @defer.inlineCallbacks - def setUp(self): - yield super(ListMagicFolder, self).setUp() - self.basedir="mf_list" - self.set_up_grid(oneshare=True) - self.local_dir = os.path.join(self.basedir, "magic") - os.mkdir(self.local_dir) - self.abs_local_dir_u = abspath_expanduser_unicode(unicode(self.local_dir), long_path=False) - - yield self.do_create_magic_folder(0) - (rc, stdout, stderr) = yield self.do_invite(0, self.alice_nickname) - invite_code = stdout.strip() - yield self.do_join(0, unicode(self.local_dir), invite_code) - - @defer.inlineCallbacks - def tearDown(self): - yield super(ListMagicFolder, self).tearDown() - shutil.rmtree(self.basedir) - - @defer.inlineCallbacks - def test_list(self): - rc, stdout, stderr = yield self.do_list(0) - self.failUnlessEqual(rc, 0) - self.assertIn("default:", stdout) - - @defer.inlineCallbacks - def test_list_none(self): - yield self.do_leave(0) - rc, stdout, stderr = yield self.do_list(0) - self.failUnlessEqual(rc, 0) - self.assertIn("No magic-folders", stdout) - - @defer.inlineCallbacks - def test_list_json(self): - rc, stdout, stderr = yield self.do_list(0, json=True) - self.failUnlessEqual(rc, 0) - res = json.loads(stdout) - self.assertEqual( - dict(default=dict(directory=self.abs_local_dir_u)), - res, - ) - - -class StatusMagicFolder(MagicFolderCLITestMixin, unittest.TestCase): - - @defer.inlineCallbacks - def setUp(self): - yield super(StatusMagicFolder, self).setUp() - self.basedir="mf_list" - self.set_up_grid(oneshare=True) - self.local_dir = os.path.join(self.basedir, "magic") - os.mkdir(self.local_dir) - self.abs_local_dir_u = abspath_expanduser_unicode(unicode(self.local_dir), long_path=False) - - yield self.do_create_magic_folder(0) - (rc, stdout, stderr) = yield self.do_invite(0, self.alice_nickname) - invite_code = stdout.strip() - yield self.do_join(0, unicode(self.local_dir), invite_code) - - @defer.inlineCallbacks - def tearDown(self): - yield super(StatusMagicFolder, self).tearDown() - shutil.rmtree(self.basedir) - - @defer.inlineCallbacks - def test_status(self): - now = datetime.now() - then = now.replace(year=now.year - 5) - five_year_interval = (now - then).total_seconds() - - def json_for_cap(options, cap): - if cap.startswith('URI:DIR2:'): - return ( - 'dirnode', - { - "children": { - "foo": ('filenode', { - "size": 1234, - "metadata": { - "tahoe": { - "linkcrtime": (time.time() - five_year_interval), - }, - "version": 1, - }, - "ro_uri": "read-only URI", - }) - } - } - ) - else: - return ('dirnode', {"children": {}}) - jc = mock.patch( - "allmydata.scripts.magic_folder_cli._get_json_for_cap", - side_effect=json_for_cap, - ) - - def json_for_frag(options, fragment, method='GET', post_args=None): - return {} - jf = mock.patch( - "allmydata.scripts.magic_folder_cli._get_json_for_fragment", - side_effect=json_for_frag, - ) - - with jc, jf: - rc, stdout, stderr = yield self.do_status(0) - self.failUnlessEqual(rc, 0) - self.assertIn("default", stdout) - - self.assertIn( - "foo (1.23 kB): good, version=1, created 5 years ago", - stdout, - ) - - @defer.inlineCallbacks - def test_status_child_not_dirnode(self): - def json_for_cap(options, cap): - if cap.startswith('URI:DIR2'): - return ( - 'dirnode', - { - "children": { - "foo": ('filenode', { - "size": 1234, - "metadata": { - "tahoe": { - "linkcrtime": 0.0, - }, - "version": 1, - }, - "ro_uri": "read-only URI", - }) - } - } - ) - elif cap == "read-only URI": - return { - "error": "bad stuff", - } - else: - return ('dirnode', {"children": {}}) - jc = mock.patch( - "allmydata.scripts.magic_folder_cli._get_json_for_cap", - side_effect=json_for_cap, - ) - - def json_for_frag(options, fragment, method='GET', post_args=None): - return {} - jf = mock.patch( - "allmydata.scripts.magic_folder_cli._get_json_for_fragment", - side_effect=json_for_frag, - ) - - with jc, jf: - rc, stdout, stderr = yield self.do_status(0) - self.failUnlessEqual(rc, 0) - - self.assertIn( - "expected a dirnode", - stdout + stderr, - ) - - @defer.inlineCallbacks - def test_status_error_not_dircap(self): - def json_for_cap(options, cap): - if cap.startswith('URI:DIR2:'): - return ( - 'filenode', - {} - ) - else: - return ('dirnode', {"children": {}}) - jc = mock.patch( - "allmydata.scripts.magic_folder_cli._get_json_for_cap", - side_effect=json_for_cap, - ) - - def json_for_frag(options, fragment, method='GET', post_args=None): - return {} - jf = mock.patch( - "allmydata.scripts.magic_folder_cli._get_json_for_fragment", - side_effect=json_for_frag, - ) - - with jc, jf: - rc, stdout, stderr = yield self.do_status(0) - self.failUnlessEqual(rc, 2) - self.assertIn( - "magic_folder_dircap isn't a directory capability", - stdout + stderr, - ) - - @defer.inlineCallbacks - def test_status_nothing(self): - rc, stdout, stderr = yield self.do_status(0, name="blam") - self.assertIn("No such magic-folder 'blam'", stderr) - - -class CreateMagicFolder(MagicFolderCLITestMixin, unittest.TestCase): - def test_create_and_then_invite_join(self): - self.basedir = "cli/MagicFolder/create-and-then-invite-join" - self.set_up_grid(oneshare=True) - local_dir = os.path.join(self.basedir, "magic") - os.mkdir(local_dir) - abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False) - - d = self.do_create_magic_folder(0) - d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname)) - def get_invite_code_and_join(args): - (rc, stdout, stderr) = args - invite_code = stdout.strip() - return self.do_join(0, unicode(local_dir), invite_code) - d.addCallback(get_invite_code_and_join) - def get_caps(ign): - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u)) - return d - - def test_create_error(self): - self.basedir = "cli/MagicFolder/create-error" - self.set_up_grid(oneshare=True) - - d = self.do_cli("magic-folder", "create", "m a g i c:", client_num=0) - def _done(args): - (rc, stdout, stderr) = args - self.failIfEqual(rc, 0) - self.failUnlessIn("Alias names cannot contain spaces.", stderr) - d.addCallback(_done) - return d - - @defer.inlineCallbacks - def test_create_duplicate_name(self): - self.basedir = "cli/MagicFolder/create-dup" - self.set_up_grid(oneshare=True) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "create", "magic:", "--name", "foo", - client_num=0, - ) - self.assertEqual(rc, 0) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "create", "magic:", "--name", "foo", - client_num=0, - ) - self.assertEqual(rc, 1) - self.assertIn( - "Already have a magic-folder named 'default'", - stderr - ) - - @defer.inlineCallbacks - def test_leave_wrong_folder(self): - self.basedir = "cli/MagicFolder/leave_wrong_folders" - yield self.set_up_grid(oneshare=True) - magic_dir = os.path.join(self.basedir, 'magic') - os.mkdir(magic_dir) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "create", "--name", "foo", "magic:", "my_name", magic_dir, - client_num=0, - ) - self.assertEqual(rc, 0) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "leave", "--name", "bar", - client_num=0, - ) - self.assertNotEqual(rc, 0) - self.assertIn( - "No such magic-folder 'bar'", - stdout + stderr, - ) - - @defer.inlineCallbacks - def test_leave_no_folder(self): - self.basedir = "cli/MagicFolder/leave_no_folders" - yield self.set_up_grid(oneshare=True) - magic_dir = os.path.join(self.basedir, 'magic') - os.mkdir(magic_dir) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "create", "--name", "foo", "magic:", "my_name", magic_dir, - client_num=0, - ) - self.assertEqual(rc, 0) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "leave", "--name", "foo", - client_num=0, - ) - self.assertEqual(rc, 0) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "leave", "--name", "foo", - client_num=0, - ) - self.assertEqual(rc, 1) - self.assertIn( - "No magic-folders at all", - stderr, - ) - - @defer.inlineCallbacks - def test_leave_no_folders_at_all(self): - self.basedir = "cli/MagicFolder/leave_no_folders_at_all" - yield self.set_up_grid(oneshare=True) - - rc, stdout, stderr = yield self.do_cli( - "magic-folder", "leave", - client_num=0, - ) - self.assertEqual(rc, 1) - self.assertIn( - "No magic-folders at all", - stderr, - ) - - def test_create_invite_join(self): - self.basedir = "cli/MagicFolder/create-invite-join" - self.set_up_grid(oneshare=True) - local_dir = os.path.join(self.basedir, "magic") - abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False) - - d = self.do_cli("magic-folder", "create", "magic:", "Alice", local_dir) - def _done(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - d.addCallback(_done) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u)) - return d - - def test_help_synopsis(self): - self.basedir = "cli/MagicFolder/help_synopsis" - os.makedirs(self.basedir) - - o = magic_folder_cli.CreateOptions() - o.parent = magic_folder_cli.MagicFolderCommand() - o.parent.getSynopsis() - - def test_create_invite_join_failure(self): - self.basedir = "cli/MagicFolder/create-invite-join-failure" - os.makedirs(self.basedir) - - o = magic_folder_cli.CreateOptions() - o.parent = magic_folder_cli.MagicFolderCommand() - o.parent['node-directory'] = self.basedir - try: - o.parseArgs("magic:", "Alice", "-foo") - except usage.UsageError as e: - self.failUnlessIn("cannot start with '-'", str(e)) - else: - self.fail("expected UsageError") - - def test_join_failure(self): - self.basedir = "cli/MagicFolder/create-join-failure" - os.makedirs(self.basedir) - - o = magic_folder_cli.JoinOptions() - o.parent = magic_folder_cli.MagicFolderCommand() - o.parent['node-directory'] = self.basedir - try: - o.parseArgs("URI:invite+URI:code", "-foo") - except usage.UsageError as e: - self.failUnlessIn("cannot start with '-'", str(e)) - else: - self.fail("expected UsageError") - - def test_join_twice_failure(self): - self.basedir = "cli/MagicFolder/create-join-twice-failure" - os.makedirs(self.basedir) - self.set_up_grid(oneshare=True) - local_dir = os.path.join(self.basedir, "magic") - abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False) - - d = self.do_create_magic_folder(0) - d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname)) - def get_invite_code_and_join(args): - (rc, stdout, stderr) = args - self.invite_code = stdout.strip() - return self.do_join(0, unicode(local_dir), self.invite_code) - d.addCallback(get_invite_code_and_join) - def get_caps(ign): - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u)) - def join_again(ignore): - return self.do_cli("magic-folder", "join", self.invite_code, local_dir, client_num=0) - d.addCallback(join_again) - def get_results(result): - (rc, out, err) = result - self.failUnlessEqual(out, "") - self.failUnlessIn("This client already has a magic-folder", err) - self.failIfEqual(rc, 0) - d.addCallback(get_results) - return d - - def test_join_leave_join(self): - self.basedir = "cli/MagicFolder/create-join-leave-join" - os.makedirs(self.basedir) - self.set_up_grid(oneshare=True) - local_dir = os.path.join(self.basedir, "magic") - abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False) - - self.invite_code = None - d = self.do_create_magic_folder(0) - d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname)) - def get_invite_code_and_join(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - self.invite_code = stdout.strip() - return self.do_join(0, unicode(local_dir), self.invite_code) - d.addCallback(get_invite_code_and_join) - def get_caps(ign): - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u)) - d.addCallback(lambda ign: self.do_leave(0)) - - d.addCallback(lambda ign: self.do_join(0, unicode(local_dir), self.invite_code)) - def get_caps(ign): - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u)) - - return d - - def test_join_failures(self): - self.basedir = "cli/MagicFolder/create-join-failures" - os.makedirs(self.basedir) - self.set_up_grid(oneshare=True) - local_dir = os.path.join(self.basedir, "magic") - os.mkdir(local_dir) - abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False) - - self.invite_code = None - d = self.do_create_magic_folder(0) - d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname)) - def get_invite_code_and_join(args): - (rc, stdout, stderr) = args - self.failUnlessEqual(rc, 0) - self.invite_code = stdout.strip() - return self.do_join(0, unicode(local_dir), self.invite_code) - d.addCallback(get_invite_code_and_join) - def get_caps(ign): - self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u)) - - def check_success(result): - (rc, out, err) = result - self.failUnlessEqual(rc, 0, out + err) - def check_failure(result): - (rc, out, err) = result - self.failIfEqual(rc, 0) - - def leave(ign): - return self.do_cli("magic-folder", "leave", client_num=0) - d.addCallback(leave) - d.addCallback(check_success) - - magic_folder_db_file = os.path.join(self.get_clientdir(i=0), u"private", u"magicfolder_default.sqlite") - - def check_join_if_file(my_file): - fileutil.write(my_file, "my file data") - d2 = self.do_cli("magic-folder", "join", self.invite_code, local_dir, client_num=0) - d2.addCallback(check_failure) - return d2 - - for my_file in [magic_folder_db_file]: - d.addCallback(lambda ign, my_file: check_join_if_file(my_file), my_file) - d.addCallback(leave) - # we didn't successfully join, so leaving should be an error - d.addCallback(check_failure) - - return d - -class CreateErrors(unittest.TestCase): - def test_poll_interval(self): - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "create", "--poll-interval=frog", - "alias:") - self.assertEqual(str(e), "--poll-interval must be a positive integer") - - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "create", "--poll-interval=-4", - "alias:") - self.assertEqual(str(e), "--poll-interval must be a positive integer") - - def test_alias(self): - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "create", "no-colon") - self.assertEqual(str(e), "An alias must end with a ':' character.") - - def test_nickname(self): - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "create", "alias:", "nickname") - self.assertEqual(str(e), "If NICKNAME is specified then LOCAL_DIR must also be specified.") - -class InviteErrors(unittest.TestCase): - def test_alias(self): - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "invite", "no-colon") - self.assertEqual(str(e), "An alias must end with a ':' character.") - -class JoinErrors(unittest.TestCase): - def test_poll_interval(self): - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "join", "--poll-interval=frog", - "code", "localdir") - self.assertEqual(str(e), "--poll-interval must be a positive integer") - - e = self.assertRaises(usage.UsageError, parse_cli, - "magic-folder", "join", "--poll-interval=-2", - "code", "localdir") - self.assertEqual(str(e), "--poll-interval must be a positive integer") diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index fa9b61cf4..9d1a64974 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -110,7 +110,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): original_do_http = tahoe_mv.do_http def mock_do_http(method, url, body=""): if method == "DELETE": - class FakeResponse: + class FakeResponse(object): def read(self): return "response" resp = FakeResponse() diff --git a/src/allmydata/test/cli/test_start.py b/src/allmydata/test/cli/test_start.py index 790b3ce24..b38ca7fd8 100644 --- a/src/allmydata/test/cli/test_start.py +++ b/src/allmydata/test/cli/test_start.py @@ -1,5 +1,4 @@ import os -import sys import shutil import subprocess from os.path import join @@ -255,10 +254,9 @@ class RunTests(unittest.TestCase): ]) i, o, e = StringIO(), StringIO(), StringIO() - with patch.object(sys, 'stdout', o), patch.object(sys, 'stderr', e): - runner.dispatch(config, i, o, e) + runner.dispatch(config, i, o, e) - output = o.getvalue() + output = e.getvalue() # should print out the collected logs and an error-code self.assertIn( "invalid section", diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index f6ee6015e..8453fbca2 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -49,7 +49,7 @@ from ..util.eliotutil import ( inline_callbacks, ) -class Expect(Protocol): +class Expect(Protocol, object): def __init__(self): self._expectations = [] @@ -79,7 +79,7 @@ class Expect(Protocol): d.errback(reason) -class _ProcessProtocolAdapter(ProcessProtocol): +class _ProcessProtocolAdapter(ProcessProtocol, object): def __init__(self, fds): self._fds = fds @@ -218,7 +218,7 @@ class CLINodeAPI(object): return stopping -class _WaitForEnd(ProcessProtocol): +class _WaitForEnd(ProcessProtocol, object): def __init__(self, ended): self._ended = ended diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index e5f16a007..e7a94ff40 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -4,6 +4,7 @@ __all__ = [ "SyncTestCase", "AsyncTestCase", "AsyncBrokenTestCase", + "TrialTestCase", "flush_logged_errors", "skip", @@ -11,6 +12,7 @@ __all__ = [ ] import os, random, struct +import six import tempfile from tempfile import mktemp from functools import partial @@ -26,6 +28,8 @@ from errno import ( EADDRINUSE, ) +import attr + import treq from zope.interface import implementer @@ -55,6 +59,7 @@ from twisted.internet.interfaces import ( IReactorSocket, ) from twisted.internet.endpoints import AdoptedStreamServerEndpoint +from twisted.trial.unittest import TestCase as _TrialTestCase from allmydata import uri from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\ @@ -72,7 +77,14 @@ from allmydata.util.assertutil import precondition from allmydata.util.consumer import download_to_data import allmydata.test.common_util as testutil from allmydata.immutable.upload import Uploader +from allmydata.client import ( + config_from_string, + create_client_from_config, +) +from ..crypto import ( + ed25519, +) from .eliotutil import ( EliotLoggedRunTest, ) @@ -80,6 +92,194 @@ from .eliotutil import ( TEST_RSA_KEY_SIZE = 522 +EMPTY_CLIENT_CONFIG = config_from_string( + b"/dev/null", + b"tub.port", + b"" +) + + +@attr.s +class MemoryIntroducerClient(object): + """ + A model-only (no behavior) stand-in for ``IntroducerClient``. + """ + tub = attr.ib() + introducer_furl = attr.ib() + nickname = attr.ib() + my_version = attr.ib() + oldest_supported = attr.ib() + app_versions = attr.ib() + sequencer = attr.ib() + cache_filepath = attr.ib() + + subscribed_to = attr.ib(default=attr.Factory(list)) + published_announcements = attr.ib(default=attr.Factory(list)) + + + def setServiceParent(self, parent): + pass + + + def subscribe_to(self, service_name, cb, *args, **kwargs): + self.subscribed_to.append(Subscription(service_name, cb, args, kwargs)) + + + def publish(self, service_name, ann, signing_key): + self.published_announcements.append(Announcement( + service_name, + ann, + ed25519.string_from_signing_key(signing_key), + )) + + +@attr.s +class Subscription(object): + """ + A model of an introducer subscription. + """ + service_name = attr.ib() + cb = attr.ib() + args = attr.ib() + kwargs = attr.ib() + + +@attr.s +class Announcement(object): + """ + A model of an introducer announcement. + """ + service_name = attr.ib() + ann = attr.ib() + signing_key_bytes = attr.ib(type=bytes) + + @property + def signing_key(self): + return ed25519.signing_keypair_from_string(self.signing_key_bytes)[0] + + +def get_published_announcements(client): + """ + Get a flattened list of all announcements sent using all introducer + clients. + """ + return list( + announcement + for introducer_client + in client.introducer_clients + for announcement + in introducer_client.published_announcements + ) + + +class UseTestPlugins(object): + """ + A fixture which enables loading Twisted plugins from the Tahoe-LAFS test + suite. + """ + def setUp(self): + """ + Add the testing package ``plugins`` directory to the ``twisted.plugins`` + aggregate package. + """ + import twisted.plugins + testplugins = FilePath(__file__).sibling("plugins") + twisted.plugins.__path__.insert(0, testplugins.path) + + def cleanUp(self): + """ + Remove the testing package ``plugins`` directory from the + ``twisted.plugins`` aggregate package. + """ + import twisted.plugins + testplugins = FilePath(__file__).sibling("plugins") + twisted.plugins.__path__.remove(testplugins.path) + + def getDetails(self): + return {} + + +@attr.s +class UseNode(object): + """ + A fixture which creates a client node. + + :ivar dict[bytes, bytes] plugin_config: Configuration items to put in the + node's configuration. + + :ivar bytes storage_plugin: The name of a storage plugin to enable. + + :ivar FilePath basedir: The base directory of the node. + + :ivar bytes introducer_furl: The introducer furl with which to + configure the client. + + :ivar dict[bytes, bytes] node_config: Configuration items for the *node* + section of the configuration. + + :ivar _Config config: The complete resulting configuration. + """ + plugin_config = attr.ib() + storage_plugin = attr.ib() + basedir = attr.ib() + introducer_furl = attr.ib() + node_config = attr.ib(default=attr.Factory(dict)) + + config = attr.ib(default=None) + + def setUp(self): + def format_config_items(config): + return b"\n".join( + b" = ".join((key, value)) + for (key, value) + in config.items() + ) + + if self.plugin_config is None: + plugin_config_section = b"" + else: + plugin_config_section = b""" +[storageclient.plugins.{storage_plugin}] +{config} +""".format( + storage_plugin=self.storage_plugin, + config=format_config_items(self.plugin_config), +) + + self.config = config_from_string( + self.basedir.asTextMode().path, + u"tub.port", +b""" +[node] +{node_config} + +[client] +introducer.furl = {furl} +storage.plugins = {storage_plugin} +{plugin_config_section} +""".format( + furl=self.introducer_furl, + storage_plugin=self.storage_plugin, + node_config=format_config_items(self.node_config), + plugin_config_section=plugin_config_section, +) + ) + + def create_node(self): + return create_client_from_config( + self.config, + _introducer_factory=MemoryIntroducerClient, + ) + + def cleanUp(self): + pass + + + def getDetails(self): + return {} + + + @implementer(IPlugin, IStreamServerEndpointStringParser) class AdoptedServerPort(object): """ @@ -135,23 +335,17 @@ class SameProcessStreamEndpointAssigner(object): """ def setUp(self): self._cleanups = [] + # Make sure the `adopt-socket` endpoint is recognized. We do this + # instead of providing a dropin because we don't want to make this + # endpoint available to random other applications. + f = UseTestPlugins() + f.setUp() + self._cleanups.append(f.cleanUp) def tearDown(self): for c in self._cleanups: c() - def _patch_plugins(self): - """ - Add the testing package ``plugins`` directory to the ``twisted.plugins`` - aggregate package. Arrange for it to be removed again when the - fixture is torn down. - """ - import twisted.plugins - testplugins = FilePath(__file__).sibling("plugins") - twisted.plugins.__path__.insert(0, testplugins.path) - self._cleanups.append(lambda: twisted.plugins.__path__.remove(testplugins.path)) - - def assign(self, reactor): """ Make a new streaming server endpoint and return its string description. @@ -183,10 +377,6 @@ class SameProcessStreamEndpointAssigner(object): host, port = s.getsockname() location_hint = "tcp:%s:%d" % (host, port) port_endpoint = "adopt-socket:fd=%d" % (s.fileno(),) - # Make sure `adopt-socket` is recognized. We do this instead of - # providing a dropin because we don't want to make this endpoint - # available to random other applications. - self._patch_plugins() else: # On other platforms, we blindly guess and hope we get lucky. portnum = iputil.allocate_tcp_port() @@ -201,7 +391,7 @@ class DummyProducer(object): pass @implementer(IImmutableFileNode) -class FakeCHKFileNode: +class FakeCHKFileNode(object): """I provide IImmutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -339,7 +529,7 @@ def create_chk_filenode(contents, all_contents): @implementer(IMutableFileNode, ICheckable) -class FakeMutableFileNode: +class FakeMutableFileNode(object): """I provide IMutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -597,7 +787,7 @@ class LoggingServiceParent(service.MultiService): TEST_DATA="\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1) -class ShouldFailMixin: +class ShouldFailMixin(object): def shouldFail(self, expected_failure, which, substring, callable, *args, **kwargs): """Assert that a function call raises some exception. This is a @@ -638,7 +828,7 @@ class ShouldFailMixin: d.addBoth(done) return d -class WebErrorMixin: +class WebErrorMixin(object): def explain_web_error(self, f): # an error on the server side causes the client-side getPage() to # return a failure(t.web.error.Error), and its str() doesn't show the @@ -1055,3 +1245,29 @@ class AsyncBrokenTestCase(_TestCaseMixin, TestCase): run_tests_with = EliotLoggedRunTest.make_factory( AsynchronousDeferredRunTestForBrokenTwisted.make_factory(timeout=60.0), ) + + +class TrialTestCase(_TrialTestCase): + """ + A twisted.trial.unittest.TestCaes with Tahoe required fixes + applied. Currently these are: + + - ensure that .fail() passes a bytes msg on Python2 + """ + + def fail(self, msg): + """ + Ensure our msg is a native string on Python2. If it was Unicode, + we encode it as utf8 and hope for the best. On Python3 we take + no action. + + This is necessary because Twisted passes the 'msg' argument + along to the constructor of an exception; on Python2, + Exception will accept a `unicode` instance but will fail if + you try to turn that Exception instance into a string. + """ + + if six.PY2: + if isinstance(msg, six.text_type): + return super(self, TrialTestCase).fail(msg.encode("utf8")) + return super(self, TrialTestCase).fail(msg) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 91afa6814..f9e0f2b01 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -83,13 +83,13 @@ def flip_one_bit(s, offset=0, size=None): return result -class ReallyEqualMixin: +class ReallyEqualMixin(object): def failUnlessReallyEqual(self, a, b, msg=None): self.assertEqual(a, b, msg) self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg)) -class NonASCIIPathMixin: +class NonASCIIPathMixin(object): def mkdir_nonascii(self, dirpath): # Kludge to work around the fact that buildbot can't remove a directory tree that has # any non-ASCII directory names on Windows. (#1472) @@ -143,13 +143,13 @@ class SignalMixin(object): signal.signal(signal.SIGCHLD, self.sigchldHandler) return super(SignalMixin, self).tearDown() -class StallMixin: +class StallMixin(object): def stall(self, res=None, delay=1): d = defer.Deferred() reactor.callLater(delay, d.callback, res) return d -class ShouldFailMixin: +class ShouldFailMixin(object): def shouldFail(self, expected_failure, which, substring, callable, *args, **kwargs): @@ -173,26 +173,10 @@ class ShouldFailMixin: class TestMixin(SignalMixin): - def setUp(self, repeatable=False): - """ - @param repeatable: install the repeatable_randomness hacks to attempt - to without access to real randomness and real time.time from the - code under test - """ - self.repeatable = repeatable - if self.repeatable: - import repeatable_random - repeatable_random.force_repeatability() - if hasattr(time, 'realtime'): - self.teststarttime = time.realtime() - else: - self.teststarttime = time.time() + def setUp(self): return super(TestMixin, self).setUp() def tearDown(self): - if self.repeatable: - import repeatable_random - repeatable_random.restore_non_repeatability() self.clean_pending(required_to_quiesce=True) return super(TestMixin, self).tearDown() diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index fc8fe6e3b..791a8d9ab 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -6,7 +6,7 @@ from twisted.web.error import Error from nevow.testutil import FakeRequest from nevow import inevow, context -class WebRenderingMixin: +class WebRenderingMixin(object): # d=page.renderString() or s=page.renderSynchronously() will exercise # docFactory, render_*/data_* . It won't exercise want_json(), or my # renderHTTP() override which tests want_json(). To exercise args=, we diff --git a/src/allmydata/test/data/pycryptopp-rsa-2048-priv.txt b/src/allmydata/test/data/pycryptopp-rsa-2048-priv.txt new file mode 100644 index 000000000..f01eed504 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-2048-priv.txt @@ -0,0 +1 @@ +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC0JwgBbVsI+XlOopqjvBatKkQbJPXuap7Psbe5i4EoMfiYI2PC2UB7GuYeTdE79TvDtmfjFD/RVWA3Y/RTQYQz/lKyCFS4w3wa/TPkZwF1r3OjIMSsCYe2J3W9NV3cK+PVw2A8D2y5DvUIAdO+Mi6aH26p2UV8FTnPqHWvJubrcLQt6979/BQnqKCFJ+SPx4se5XsMZ3vrbs6MCqM2qS9RnNEhexlNrJd1wXezILKsmQdf/QiZiY7LXjEdD6BNG8OYQ2iSbCa8aGEoSPQfdnZZxcTFE02QwKcScZKhU9fRv0Ttqr3i8xiliw9gn4UzptEZO6MVO2BrptS30SjJDXC7AgERAoIBADpI3PFnJPtfxV00m3E1UqFvjoFAqetAnMq5fzR/9RSIo0BHr1Wgo+uXwuuvw7GEC85gqSPR2GlfYuS+dLGGIz3/dRt7KngDAoEzzQYhU0u4w4eZqQp7jcn9tSagUxKGq5f7cfVQSNJ1x77TaibyHiLN7xjVWj67krQf6dbI0j0cYvnxu+4EZbzNdvFw93ddoOZB/dFjLu0kVKVl/mWyCX9GNr2nCSHe9wYipOz5b9WkdD0J2Oy0v8Wkn4y3yOOvo/EgrNYfo4IVslsDo9Yw3Yk32Eml0ZsdwSqu+wM4c+jRbTJ+sBGqci4etPpMhcsH0Vt9+97Lnuan2Jza9xjrL2ECgYEA8wj+/bfjTCXsu22f8V7Z40vJUyM7j4WvUoA9khAQ7qAlnFdqdzq5a7ArA9vRjeN6ya16j36IXCkpT+FGe6YWCsZCKd1ZVy7sZ1Uh7X2hRqf0vxJsSJvG/OmofFUfuwCgLFLKI4SDhHaB+pWAdkAIL4MkJQADg/qVlAdrWoPsfhECgYEAvcNHhSCW010SRudwmTRX5QtndHk/LM6VAgyR0ElarvmG6K5pbpL8MD5CpJ3AhUwKp96SlMsBEG3a9BR5zv6Jvhc/KHxT7W/EjLnV9PSD90+BgHHsTonjg6TayJ9XE6RpO3MqeifVG/2S5WhhFFGGd5KSFnvZwr9ni+LYRuDVpgsCgYEAgKpo4KylgqqqgVgnf8jNtJGIs4sfiDe3K61NxcxFMwl9UsTeAuLaomxTAgr2eEtBAVvXeSTex2EV3v7K9irAYA6bf5NNamQizUswFFGRneByg0X9F2GHdtYN53hcF7UJgOCJIdy+GPNx/SH4txLXKDZebfDyzWaLbHxmAr5QBoECgYBC+aDFkwgOXRWCb81jP6aNExV0Zwc8/Z4AuSRnoWtM0In3xRYnBrNcUjWjgvinhD//A0LLGnjYnz44BzoM0k67j7vwK+Fi3CdAug9HZVvAsqYtVWJ2EoyI0MWwODzZwY6Nc/Df0dK+lbtgBrjZ/qft937awkzbUp0EMfH65fENbQKBgQCSVWXy+WLQXeHtx/+nNv9HyjQnowalp3SwWRf0YoK/xa526xg+ixViVZvT6e2KTcJGdHFQ+cbCsc1Vx6E13n3Mu9y0N3a4WRQkZHPgnsNouPLaKn0SmVY7RX/I/Rz2r0hRE+gDM6+1/99zPuwP3FW5eLoTBX021Y35kBFHbZ4r+w== diff --git a/src/allmydata/test/data/pycryptopp-rsa-2048-pub.txt b/src/allmydata/test/data/pycryptopp-rsa-2048-pub.txt new file mode 100644 index 000000000..74b773931 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-2048-pub.txt @@ -0,0 +1 @@ +MIIBIDANBgkqhkiG9w0BAQEFAAOCAQ0AMIIBCAKCAQEAtCcIAW1bCPl5TqKao7wWrSpEGyT17mqez7G3uYuBKDH4mCNjwtlAexrmHk3RO/U7w7Zn4xQ/0VVgN2P0U0GEM/5SsghUuMN8Gv0z5GcBda9zoyDErAmHtid1vTVd3Cvj1cNgPA9suQ71CAHTvjIumh9uqdlFfBU5z6h1rybm63C0Leve/fwUJ6ighSfkj8eLHuV7DGd7627OjAqjNqkvUZzRIXsZTayXdcF3syCyrJkHX/0ImYmOy14xHQ+gTRvDmENokmwmvGhhKEj0H3Z2WcXExRNNkMCnEnGSoVPX0b9E7aq94vMYpYsPYJ+FM6bRGTujFTtga6bUt9EoyQ1wuwIBEQ== diff --git a/src/allmydata/test/data/pycryptopp-rsa-2048-sig.txt b/src/allmydata/test/data/pycryptopp-rsa-2048-sig.txt new file mode 100644 index 000000000..ae6b2ed40 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-2048-sig.txt @@ -0,0 +1 @@ +ItsyW1XTOIvet6WsS68AJ/ernMG62aoeJKzyBBZ9fdeB2mVzURCBmgX5P0hTPgxHa1sEI6oIbREv4lIQnWHcPgjvz5qBkDtbOp1YHkkFAFOh533dH4s2MiRECIzHh19sBsqTGe0w/pRTHhwV+nStFqZ0IMsdxv0Qsgk5IClIY/WgBSnHQZpVbxyfL7qwvm1JK2GRuygRRsrSsxLiSnA5RWlOsDkDikVu5nhZI31K+PWa9v1i6U7ZkV4uD9triJkHW2XBIRkCyqT6wgM4KBN6V4H9nqlxZhJSQoSn1U5Rh3pL+XG6yevaZq7+pwOnRUcFkEwiJ2wT/NIK0Bjng8Szmw== diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 2ba221284..f5972c929 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -52,7 +52,7 @@ def eliot_logged_test(f): """ # A convenient, mutable container into which nested functions can write # state to be shared among them. - class storage: + class storage(object): pass @wraps(f) diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py new file mode 100644 index 000000000..eaf7d13a5 --- /dev/null +++ b/src/allmydata/test/matchers.py @@ -0,0 +1,120 @@ +""" +Testtools-style matchers useful to the Tahoe-LAFS test suite. +""" + +import attr + +from testtools.matchers import ( + Mismatch, + AfterPreprocessing, + MatchesStructure, + MatchesDict, + MatchesListwise, + Always, + Equals, +) + +from foolscap.furl import ( + decode_furl, +) + +from allmydata.util import ( + base32, +) +from allmydata.node import ( + read_config, +) +from allmydata.crypto import ( + ed25519, + error, +) + +@attr.s +class MatchesNodePublicKey(object): + """ + Match an object representing the node's private key. + + To verify, the private key is loaded from the node's private config + directory at the time the match is checked. + """ + basedir = attr.ib() + + def match(self, other): + """ + Match a private key which is the same as the private key in the node at + ``self.basedir``. + + :param other: A signing key (aka "private key") from + ``allmydata.crypto.ed25519``. This is the key to check against + the node's key. + + :return Mismatch: If the keys don't match. + """ + config = read_config(self.basedir, u"tub.port") + privkey_bytes = config.get_private_config("node.privkey") + private_key = ed25519.signing_keypair_from_string(privkey_bytes)[0] + signature = ed25519.sign_data(private_key, b"") + other_public_key = ed25519.verifying_key_from_signing_key(other) + try: + ed25519.verify_signature(other_public_key, signature, b"") + except error.BadSignature: + return Mismatch("The signature did not verify.") + + +def matches_storage_announcement(basedir, anonymous=True, options=None): + """ + Match a storage announcement. + + :param bytes basedir: The path to the node base directory which is + expected to emit the announcement. This is used to determine the key + which is meant to sign the announcement. + + :param bool anonymous: If True, matches a storage announcement containing + an anonymous access fURL. Otherwise, fails to match such an + announcement. + + :param list[matcher]|NoneType options: If a list, matches a storage + announcement containing a list of storage plugin options matching the + elements of the list. If None, fails to match an announcement with + storage plugin options. + + :return: A matcher with the requested behavior. + """ + announcement = { + u"permutation-seed-base32": matches_base32(), + } + if anonymous: + announcement[u"anonymous-storage-FURL"] = matches_furl() + if options: + announcement[u"storage-options"] = MatchesListwise(options) + return MatchesStructure( + # Has each of these keys with associated values that match + service_name=Equals(u"storage"), + ann=MatchesDict(announcement), + signing_key=MatchesNodePublicKey(basedir), + ) + + +def matches_furl(): + """ + Match any Foolscap fURL byte string. + """ + return AfterPreprocessing(decode_furl, Always()) + + +def matches_base32(): + """ + Match any base32 encoded byte string. + """ + return AfterPreprocessing(base32.a2b, Always()) + + + +class MatchesSameElements(object): + """ + Match if the two-tuple value given contains two elements that are equal to + each other. + """ + def match(self, value): + left, right = value + return Equals(left).match(right) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 62b720442..fdc19d5cb 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -11,7 +11,11 @@ from allmydata.mutable.publish import MutableData from ..test_download import PausingConsumer, PausingAndStoppingConsumer, \ StoppingConsumer, ImmediatelyStoppingConsumer from .. import common_util as testutil -from .util import FakeStorage, make_nodemaker +from .util import ( + FakeStorage, + make_nodemaker_with_peers, + make_peer, +) class Filenode(unittest.TestCase, testutil.ShouldFailMixin): # this used to be in Publish, but we removed the limit. Some of @@ -19,8 +23,15 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): # larger than the limit. OLD_MAX_SEGMENT_SIZE = 3500000 def setUp(self): - self._storage = s = FakeStorage() - self.nodemaker = make_nodemaker(s) + self._storage = FakeStorage() + self._peers = list( + make_peer(self._storage, n) + for n + # 10 is the default for N. We're trying to make enough servers + # here so that each only gets one share. + in range(10) + ) + self.nodemaker = make_nodemaker_with_peers(self._peers) def test_create(self): d = self.nodemaker.create_mutable_file() @@ -352,16 +363,19 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_mdmf_write_count(self): - # Publishing an MDMF file should only cause one write for each - # share that is to be published. Otherwise, we introduce - # undesirable semantics that are a regression from SDMF + """ + Publishing an MDMF file causes exactly one write for each share that is to + be published. Otherwise, we introduce undesirable semantics that are a + regression from SDMF. + """ upload = MutableData("MDMF" * 100000) # about 400 KiB d = self.nodemaker.create_mutable_file(upload, version=MDMF_VERSION) def _check_server_write_counts(ignored): - sb = self.nodemaker.storage_broker - for server in sb.servers.itervalues(): - self.failUnlessEqual(server.get_rref().queries, 1) + for peer in self._peers: + # There were enough servers for each to only get a single + # share. + self.assertEqual(peer.storage_server.queries, 1) d.addCallback(_check_server_write_counts) return d diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index f2669d9e0..08990c180 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -5,6 +5,7 @@ from twisted.trial import unittest from twisted.internet import defer from foolscap.logging import log from allmydata import uri +from allmydata.crypto import rsa from allmydata.interfaces import NotEnoughSharesError, SDMF_VERSION, MDMF_VERSION from allmydata.util import fileutil from allmydata.util.hashutil import ssk_writekey_hash, ssk_pubkey_fingerprint_hash @@ -19,14 +20,14 @@ from ..no_network import GridTestMixin from .. import common_util as testutil from ..common_util import DevNullDictionary -class SameKeyGenerator: +class SameKeyGenerator(object): def __init__(self, pubkey, privkey): self.pubkey = pubkey self.privkey = privkey def generate(self, keysize=None): return defer.succeed( (self.pubkey, self.privkey) ) -class FirstServerGetsKilled: +class FirstServerGetsKilled(object): done = False def notify(self, retval, wrapper, methname): if not self.done: @@ -34,7 +35,7 @@ class FirstServerGetsKilled: self.done = True return retval -class FirstServerGetsDeleted: +class FirstServerGetsDeleted(object): def __init__(self): self.done = False self.silenced = None @@ -211,8 +212,8 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): def _got_key(keypair): (pubkey, privkey) = keypair nm.key_generator = SameKeyGenerator(pubkey, privkey) - pubkey_s = pubkey.serialize() - privkey_s = privkey.serialize() + pubkey_s = rsa.der_string_from_verifying_key(pubkey) + privkey_s = rsa.der_string_from_signing_key(privkey) u = uri.WriteableSSKFileURI(ssk_writekey_hash(privkey_s), ssk_pubkey_fingerprint_hash(pubkey_s)) self._storage_index = u.get_storage_index() diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index c09f111c4..a664c1e08 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -1,4 +1,5 @@ from six.moves import cStringIO as StringIO +import attr from twisted.internet import defer, reactor from foolscap.api import eventually, fireEventually from allmydata import client @@ -9,7 +10,10 @@ from allmydata.util.hashutil import tagged_hash from allmydata.storage_client import StorageFarmBroker from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.mutable.publish import MutableData -from ..common import TEST_RSA_KEY_SIZE +from ..common import ( + TEST_RSA_KEY_SIZE, + EMPTY_CLIENT_CONFIG, +) def eventuaaaaaly(res=None): d = fireEventually(res) @@ -21,7 +25,7 @@ def eventuaaaaaly(res=None): # network connections, both to speed up the tests and to reduce the amount of # non-mutable.py code being exercised. -class FakeStorage: +class FakeStorage(object): # this class replaces the collection of storage servers, allowing the # tests to examine and manipulate the published shares. It also lets us # control the order in which read queries are answered, to exercise more @@ -78,11 +82,13 @@ class FakeStorage: shares[shnum] = f.getvalue() -class FakeStorageServer: +class FakeStorageServer(object): + def __init__(self, peerid, storage): self.peerid = peerid self.storage = storage self.queries = 0 + def callRemote(self, methname, *args, **kwargs): self.queries += 1 def _call(): @@ -197,21 +203,100 @@ def corrupt(res, s, offset, shnums_to_corrupt=None, offset_offset=0): dl.addCallback(lambda ignored: res) return dl +@attr.s +class Peer(object): + peerid = attr.ib() + storage_server = attr.ib() + announcement = attr.ib() + +def make_peer(s, i): + """ + Create a "peer" suitable for use with ``make_storagebroker_with_peers`` or + ``make_nodemaker_with_peers``. + + :param IServer s: The server with which to associate the peers. + + :param int i: A unique identifier for this peer within the whole group of + peers to be used. For example, a sequence number. This is used to + generate a unique peer id. + + :rtype: ``Peer`` + """ + peerid = base32.b2a(tagged_hash("peerid", "%d" % i)[:20]) + fss = FakeStorageServer(peerid, s) + ann = { + "anonymous-storage-FURL": "pb://%s@nowhere/fake" % (peerid,), + "permutation-seed-base32": peerid, + } + return Peer(peerid=peerid, storage_server=fss, announcement=ann) + + def make_storagebroker(s=None, num_peers=10): + """ + Make a ``StorageFarmBroker`` connected to some number of fake storage + servers. + + :param IServer s: The server with which to associate the fake storage + servers. + + :param int num_peers: The number of fake storage servers to associate with + the broker. + """ if not s: s = FakeStorage() - peerids = [tagged_hash("peerid", "%d" % i)[:20] - for i in range(num_peers)] - storage_broker = StorageFarmBroker(True, None) - for peerid in peerids: - fss = FakeStorageServer(peerid, s) - ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(peerid), - "permutation-seed-base32": base32.b2a(peerid) } - storage_broker.test_add_rref(peerid, fss, ann) + peers = [] + for peer_num in range(num_peers): + peers.append(make_peer(s, peer_num)) + return make_storagebroker_with_peers(peers) + + +def make_storagebroker_with_peers(peers): + """ + Make a ``StorageFarmBroker`` connected to the given storage servers. + + :param list peers: The storage servers to associate with the storage + broker. + """ + storage_broker = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) + for peer in peers: + storage_broker.test_add_rref( + peer.peerid, + peer.storage_server, + peer.announcement, + ) return storage_broker + def make_nodemaker(s=None, num_peers=10, keysize=TEST_RSA_KEY_SIZE): + """ + Make a ``NodeMaker`` connected to some number of fake storage servers. + + :param IServer s: The server with which to associate the fake storage + servers. + + :param int num_peers: The number of fake storage servers to associate with + the node maker. + """ storage_broker = make_storagebroker(s, num_peers) + return make_nodemaker_with_storage_broker(storage_broker, keysize) + + +def make_nodemaker_with_peers(peers, keysize=TEST_RSA_KEY_SIZE): + """ + Make a ``NodeMaker`` connected to the given storage servers. + + :param list peers: The storage servers to associate with the node maker. + """ + storage_broker = make_storagebroker_with_peers(peers) + return make_nodemaker_with_storage_broker(storage_broker, keysize) + + +def make_nodemaker_with_storage_broker(storage_broker, keysize): + """ + Make a ``NodeMaker`` using the given storage broker. + + :param StorageFarmBroker peers: The storage broker to use. + """ sh = client.SecretHolder("lease secret", "convergence secret") keygen = client.KeyGenerator() if keysize: @@ -221,7 +306,8 @@ def make_nodemaker(s=None, num_peers=10, keysize=TEST_RSA_KEY_SIZE): {"k": 3, "n": 10}, SDMF_VERSION, keygen) return nodemaker -class PublishMixin: + +class PublishMixin(object): def publish_one(self): # publish a file and create shares, which can then be manipulated # later. @@ -331,7 +417,7 @@ class PublishMixin: index = versionmap[shnum] shares[peerid][shnum] = oldshares[index][peerid][shnum] -class CheckerMixin: +class CheckerMixin(object): def check_good(self, r, where): self.failUnless(r.is_healthy(), where) return r @@ -349,4 +435,3 @@ class CheckerMixin: return self.fail("%s: didn't see expected exception %s in problems %s" % (where, expected_exception, r.get_share_problems())) - diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index fb1da0635..81f09379f 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -32,6 +32,9 @@ from allmydata.util import fileutil, idlib, hashutil from allmydata.util.hashutil import permute_server_hash from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IStorageBroker, IServer +from allmydata.storage_client import ( + _StorageServer, +) from .common import ( TEST_RSA_KEY_SIZE, SameProcessStreamEndpointAssigner, @@ -41,10 +44,10 @@ from .common import ( class IntentionalError(Exception): pass -class Marker: +class Marker(object): pass -class LocalWrapper: +class LocalWrapper(object): def __init__(self, original): self.original = original self.broken = False @@ -170,6 +173,10 @@ class NoNetworkServer(object): return "nickname" def get_rref(self): return self.rref + def get_storage_server(self): + if self.rref is None: + return None + return _StorageServer(lambda: self.rref) def get_version(self): return self.rref.version @@ -254,7 +261,7 @@ class _NoNetworkClient(_Client): pass #._servers will be set by the NoNetworkGrid which creates us -class SimpleStats: +class SimpleStats(object): def __init__(self): self.counters = {} self.stats_producers = [] @@ -277,6 +284,13 @@ class NoNetworkGrid(service.MultiService): def __init__(self, basedir, num_clients, num_servers, client_config_hooks, port_assigner): service.MultiService.__init__(self) + + # We really need to get rid of this pattern here (and + # everywhere) in Tahoe where "async work" is started in + # __init__ For now, we at least keep the errors so they can + # cause tests to fail less-improperly (see _check_clients) + self._setup_errors = [] + self.port_assigner = port_assigner self.basedir = basedir fileutil.make_dirs(basedir) @@ -297,6 +311,20 @@ class NoNetworkGrid(service.MultiService): d = self.make_client(i) d.addCallback(lambda c: self.clients.append(c)) + def _bad(f): + self._setup_errors.append(f) + d.addErrback(_bad) + + def _check_clients(self): + """ + The anti-pattern of doing async work in __init__ means we need to + check if that work completed successfully. This method either + returns nothing or raises an exception in case __init__ failed + to complete properly + """ + if self._setup_errors: + raise self._setup_errors[0].value + @defer.inlineCallbacks def make_client(self, i, write_config=True): clientid = hashutil.tagged_hash("clientid", str(i))[:20] @@ -361,6 +389,7 @@ class NoNetworkGrid(service.MultiService): return self.proxies_by_id.keys() def rebuild_serverlist(self): + self._check_clients() self.all_servers = frozenset(self.proxies_by_id.values()) for c in self.clients: c._servers = self.all_servers @@ -437,12 +466,14 @@ class GridTestMixin(object): self._record_webports_and_baseurls() def _record_webports_and_baseurls(self): + self.g._check_clients() self.client_webports = [c.getServiceNamed("webish").getPortnum() for c in self.g.clients] self.client_baseurls = [c.getServiceNamed("webish").getURL() for c in self.g.clients] def get_client_config(self, i=0): + self.g._check_clients() return self.g.clients[i].config def get_clientdir(self, i=0): @@ -451,9 +482,11 @@ class GridTestMixin(object): return self.get_client_config(i).get_config_path() def get_client(self, i=0): + self.g._check_clients() return self.g.clients[i] def restart_client(self, i=0): + self.g._check_clients() client = self.g.clients[i] d = defer.succeed(None) d.addCallback(lambda ign: self.g.removeService(client)) diff --git a/src/allmydata/test/plugins/tahoe_lafs_dropin.py b/src/allmydata/test/plugins/tahoe_lafs_dropin.py index 9faf5f07f..24651e388 100644 --- a/src/allmydata/test/plugins/tahoe_lafs_dropin.py +++ b/src/allmydata/test/plugins/tahoe_lafs_dropin.py @@ -2,4 +2,11 @@ from allmydata.test.common import ( AdoptedServerPort, ) +from allmydata.test.storage_plugin import ( + DummyStorage, +) + adoptedEndpointParser = AdoptedServerPort() + +dummyStoragev1 = DummyStorage(u"tahoe-lafs-dummy-v1") +dummyStoragev2 = DummyStorage(u"tahoe-lafs-dummy-v2") diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py new file mode 100644 index 000000000..43186122c --- /dev/null +++ b/src/allmydata/test/storage_plugin.py @@ -0,0 +1,120 @@ +""" +A storage server plugin the test suite can use to validate the +functionality. +""" + +from json import ( + dumps, +) + +import attr + +from zope.interface import ( + implementer, +) + +from twisted.internet.defer import ( + succeed, +) +from twisted.web.resource import ( + Resource, +) +from twisted.web.static import ( + Data, +) +from foolscap.api import ( + RemoteInterface, +) + +from allmydata.interfaces import ( + IFoolscapStoragePlugin, + IStorageServer, +) +from allmydata.client import ( + AnnounceableStorageServer, +) + + +class RIDummy(RemoteInterface): + __remote_name__ = "RIDummy.tahoe.allmydata.com" + + def just_some_method(): + """ + Just some method so there is something callable on this object. We won't + pretend to actually offer any storage capabilities. + """ + + + +@implementer(IFoolscapStoragePlugin) +@attr.s +class DummyStorage(object): + name = attr.ib() + + @property + def _client_section_name(self): + return u"storageclient.plugins.{}".format(self.name) + + def get_storage_server(self, configuration, get_anonymous_storage_server): + if u"invalid" in configuration: + raise Exception("The plugin is unhappy.") + + announcement = {u"value": configuration.get(u"some", u"default-value")} + storage_server = DummyStorageServer(get_anonymous_storage_server) + return succeed( + AnnounceableStorageServer( + announcement, + storage_server, + ), + ) + + def get_storage_client(self, configuration, announcement, get_rref): + return DummyStorageClient( + get_rref, + dict(configuration.items(self._client_section_name, [])), + announcement, + ) + + def get_client_resource(self, configuration): + """ + :return: A static data resource that produces the given configuration when + rendered, as an aid to testing. + """ + items = configuration.items(self._client_section_name, []) + resource = Data( + dumps(dict(items)), + b"text/json", + ) + # Give it some dynamic stuff too. + resource.putChild(b"counter", GetCounter()) + return resource + + +class GetCounter(Resource, object): + """ + ``GetCounter`` is a resource that returns a count of the number of times + it has rendered a response to a GET request. + + :ivar int value: The number of ``GET`` requests rendered so far. + """ + value = 0 + def render_GET(self, request): + self.value += 1 + return dumps({"value": self.value}) + + +@implementer(RIDummy) +@attr.s(frozen=True) +class DummyStorageServer(object): + get_anonymous_storage_server = attr.ib() + + def remote_just_some_method(self): + pass + + +@implementer(IStorageServer) +@attr.s +class DummyStorageClient(object): + get_rref = attr.ib() + configuration = attr.ib() + announcement = attr.ib() diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 7f89e9e08..5eed6f21f 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -15,14 +15,18 @@ from allmydata.immutable.upload import Data from allmydata.test.common_web import WebRenderingMixin from allmydata.mutable.publish import MutableData -class FakeClient: +from .common import ( + EMPTY_CLIENT_CONFIG, +) + +class FakeClient(object): def get_storage_broker(self): return self.storage_broker class WebResultsRendering(unittest.TestCase, WebRenderingMixin): def create_fake_client(self): - sb = StorageFarmBroker(True, None) + sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) # s.get_name() (the "short description") will be "v0-00000000". # s.get_longname() will include the -long suffix. servers = [("v0-00000000-long", "\x00"*20, "peer-0"), @@ -41,7 +45,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "my-version": "ver", "oldest-supported": "oldest", } - s = NativeStorageServer(server_id, ann, None, None, {}, None) + s = NativeStorageServer(server_id, ann, None, None, None) sb.test_add_server(server_id, s) c = FakeClient() c.storage_broker = sb diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 1d0fa085c..41a44d5a6 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,26 +1,79 @@ import os, sys import mock +from functools import ( + partial, +) + import twisted +from yaml import ( + safe_dump, +) +from fixtures import ( + Fixture, + TempDir, +) +from eliot.testing import ( + capture_logging, + assertHasAction, +) from twisted.trial import unittest from twisted.application import service from twisted.internet import defer +from twisted.python.filepath import ( + FilePath, +) +from testtools.matchers import ( + Equals, + AfterPreprocessing, + MatchesListwise, + MatchesDict, + Always, + Is, + raises, +) +from testtools.twistedsupport import ( + succeeded, + failed, +) import allmydata -import allmydata.frontends.magic_folder import allmydata.util.log -from allmydata.node import OldConfigError, OldConfigOptionError, UnescapedHashError, _Config, read_config, create_node_dir -from allmydata.node import config_from_string +from allmydata.node import OldConfigError, UnescapedHashError, _Config, create_node_dir from allmydata.frontends.auth import NeedRootcapLookupScheme +from allmydata.version_checks import ( + get_package_versions_string, +) from allmydata import client -from allmydata.storage_client import StorageFarmBroker -from allmydata.util import base32, fileutil, encodingutil +from allmydata.storage_client import ( + StorageClientConfig, + StorageFarmBroker, +) +from allmydata.util import ( + base32, + fileutil, + encodingutil, + configutil, +) from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode from foolscap.api import flushEventualQueue import allmydata.test.common_util as testutil +from .common import ( + EMPTY_CLIENT_CONFIG, + SyncTestCase, + UseTestPlugins, + MemoryIntroducerClient, + get_published_announcements, +) +from .matchers import ( + MatchesSameElements, + matches_storage_announcement, + matches_furl, +) +SOME_FURL = b"pb://abcde@nowhere/fake" BASECONFIG = ("[client]\n" "introducer.furl = \n" @@ -100,10 +153,9 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test try: e = self.assertRaises( EnvironmentError, - read_config, + client.read_config, basedir, "client.port", - _valid_config_sections=client._valid_config_sections, ) self.assertIn("Permission denied", str(e)) finally: @@ -128,10 +180,9 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test e = self.failUnlessRaises( OldConfigError, - read_config, + client.read_config, basedir, "client.port", - _valid_config_sections=client._valid_config_sections, ) abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding()) self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0]) @@ -195,6 +246,69 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test c = yield client.create_client(basedir) self.failUnless(c.get_long_nodeid().startswith("v0-")) + def test_storage_anonymous_enabled_by_default(self): + """ + Anonymous storage access is enabled if storage is enabled and *anonymous* + is not set. + """ + config = client.config_from_string( + b"test_storage_default_anonymous_enabled", + b"tub.port", + BASECONFIG + ( + b"[storage]\n" + b"enabled = true\n" + ) + ) + self.assertTrue(client.anonymous_storage_enabled(config)) + + def test_storage_anonymous_enabled_explicitly(self): + """ + Anonymous storage access is enabled if storage is enabled and *anonymous* + is set to true. + """ + config = client.config_from_string( + self.id(), + b"tub.port", + BASECONFIG + ( + b"[storage]\n" + b"enabled = true\n" + b"anonymous = true\n" + ) + ) + self.assertTrue(client.anonymous_storage_enabled(config)) + + def test_storage_anonymous_disabled_explicitly(self): + """ + Anonymous storage access is disabled if storage is enabled and *anonymous* + is set to false. + """ + config = client.config_from_string( + self.id(), + b"tub.port", + BASECONFIG + ( + b"[storage]\n" + b"enabled = true\n" + b"anonymous = false\n" + ) + ) + self.assertFalse(client.anonymous_storage_enabled(config)) + + def test_storage_anonymous_disabled_by_storage(self): + """ + Anonymous storage access is disabled if storage is disabled and *anonymous* + is set to true. + """ + config = client.config_from_string( + self.id(), + b"tub.port", + BASECONFIG + ( + b"[storage]\n" + b"enabled = false\n" + b"anonymous = true\n" + ) + ) + self.assertFalse(client.anonymous_storage_enabled(config)) + @defer.inlineCallbacks def test_reserved_1(self): """ @@ -466,9 +580,9 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] def test_permute(self): - sb = StorageFarmBroker(True, None) + sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) for k in ["%d" % i for i in range(5)]: - ann = {"anonymous-storage-FURL": "pb://abcde@nowhere/fake", + ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } sb.test_add_rref(k, "rref", ann) @@ -478,9 +592,14 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test self.failUnlessReallyEqual(self._permute(sb, "one"), []) def test_permute_with_preferred(self): - sb = StorageFarmBroker(True, None, preferred_peers=['1','4']) + sb = StorageFarmBroker( + True, + None, + EMPTY_CLIENT_CONFIG, + StorageClientConfig(preferred_peers=['1','4']), + ) for k in ["%d" % i for i in range(5)]: - ann = {"anonymous-storage-FURL": "pb://abcde@nowhere/fake", + ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } sb.test_add_rref(k, "rref", ann) @@ -508,7 +627,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test self.failIfEqual(str(allmydata.__version__), "unknown") self.failUnless("." in str(allmydata.__full_version__), "non-numeric version in '%s'" % allmydata.__version__) - all_versions = allmydata.get_package_versions_string() + all_versions = get_package_versions_string() self.failUnless(allmydata.__appname__ in all_versions) # also test stats stats = c.get_stats() @@ -538,104 +657,6 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test yield _check("helper.furl = None", None) yield _check("helper.furl = pb://blah\n", "pb://blah") - @defer.inlineCallbacks - def test_create_magic_folder_service(self): - """ - providing magic-folder options actually creates a MagicFolder service - """ - boom = False - class Boom(Exception): - pass - - class MockMagicFolder(allmydata.frontends.magic_folder.MagicFolder): - name = 'magic-folder' - - def __init__(self, client, upload_dircap, collective_dircap, local_path_u, dbfile, umask, name, - inotify=None, uploader_delay=1.0, clock=None, downloader_delay=3): - if boom: - raise Boom() - - service.MultiService.__init__(self) - self.client = client - self._umask = umask - self.upload_dircap = upload_dircap - self.collective_dircap = collective_dircap - self.local_dir = local_path_u - self.dbfile = dbfile - self.inotify = inotify - - def startService(self): - self.running = True - - def stopService(self): - self.running = False - - def ready(self): - pass - - self.patch(allmydata.frontends.magic_folder, 'MagicFolder', MockMagicFolder) - - upload_dircap = "URI:DIR2:blah" - local_dir_u = self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir") - local_dir_utf8 = local_dir_u.encode('utf-8') - config = (BASECONFIG + - "[storage]\n" + - "enabled = false\n" + - "[magic_folder]\n" + - "enabled = true\n") - - basedir1 = "test_client.Basic.test_create_magic_folder_service1" - os.mkdir(basedir1) - os.mkdir(local_dir_u) - - # which config-entry should be missing? - fileutil.write(os.path.join(basedir1, "tahoe.cfg"), - config + "local.directory = " + local_dir_utf8 + "\n") - with self.assertRaises(IOError): - yield client.create_client(basedir1) - - # local.directory entry missing .. but that won't be an error - # now, it'll just assume there are not magic folders - # .. hrm...should we make that an error (if enabled=true but - # there's not yaml AND no local.directory?) - fileutil.write(os.path.join(basedir1, "tahoe.cfg"), config) - fileutil.write(os.path.join(basedir1, "private", "magic_folder_dircap"), "URI:DIR2:blah") - fileutil.write(os.path.join(basedir1, "private", "collective_dircap"), "URI:DIR2:meow") - - fileutil.write(os.path.join(basedir1, "tahoe.cfg"), - config.replace("[magic_folder]\n", "[drop_upload]\n")) - - with self.assertRaises(OldConfigOptionError): - yield client.create_client(basedir1) - - fileutil.write(os.path.join(basedir1, "tahoe.cfg"), - config + "local.directory = " + local_dir_utf8 + "\n") - c1 = yield client.create_client(basedir1) - magicfolder = c1.getServiceNamed('magic-folder') - self.failUnless(isinstance(magicfolder, MockMagicFolder), magicfolder) - self.failUnlessReallyEqual(magicfolder.client, c1) - self.failUnlessReallyEqual(magicfolder.upload_dircap, upload_dircap) - self.failUnlessReallyEqual(os.path.basename(magicfolder.local_dir), local_dir_u) - self.failUnless(magicfolder.inotify is None, magicfolder.inotify) - # It doesn't start until the client starts. - self.assertFalse(magicfolder.running) - - # See above. - boom = True - - basedir2 = "test_client.Basic.test_create_magic_folder_service2" - os.mkdir(basedir2) - os.mkdir(os.path.join(basedir2, "private")) - fileutil.write(os.path.join(basedir2, "tahoe.cfg"), - BASECONFIG + - "[magic_folder]\n" + - "enabled = true\n" + - "local.directory = " + local_dir_utf8 + "\n") - fileutil.write(os.path.join(basedir2, "private", "magic_folder_dircap"), "URI:DIR2:blah") - fileutil.write(os.path.join(basedir2, "private", "collective_dircap"), "URI:DIR2:meow") - with self.assertRaises(Boom): - yield client.create_client(basedir2) - def flush_but_dont_ignore(res): d = flushEventualQueue() @@ -645,6 +666,129 @@ def flush_but_dont_ignore(res): return d +class AnonymousStorage(SyncTestCase): + """ + Tests for behaviors of the client object with respect to the anonymous + storage service. + """ + @defer.inlineCallbacks + def test_anonymous_storage_enabled(self): + """ + If anonymous storage access is enabled then the client announces it. + """ + basedir = self.id() + os.makedirs(basedir + b"/private") + config = client.config_from_string( + basedir, + b"tub.port", + BASECONFIG_I % (SOME_FURL,) + ( + b"[storage]\n" + b"enabled = true\n" + b"anonymous = true\n" + ) + ) + node = yield client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ) + self.assertThat( + get_published_announcements(node), + MatchesListwise([ + matches_storage_announcement( + basedir, + anonymous=True, + ), + ]), + ) + + @defer.inlineCallbacks + def test_anonymous_storage_disabled(self): + """ + If anonymous storage access is disabled then the client does not announce + it nor does it write a fURL for it to beneath the node directory. + """ + basedir = self.id() + os.makedirs(basedir + b"/private") + config = client.config_from_string( + basedir, + b"tub.port", + BASECONFIG_I % (SOME_FURL,) + ( + b"[storage]\n" + b"enabled = true\n" + b"anonymous = false\n" + ) + ) + node = yield client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ) + self.expectThat( + get_published_announcements(node), + MatchesListwise([ + matches_storage_announcement( + basedir, + anonymous=False, + ), + ]), + ) + self.expectThat( + config.get_private_config(b"storage.furl", default=None), + Is(None), + ) + + @defer.inlineCallbacks + def test_anonymous_storage_enabled_then_disabled(self): + """ + If a node is run with anonymous storage enabled and then later anonymous + storage is disabled in the configuration for that node, it is not + possible to reach the anonymous storage server via the originally + published fURL. + """ + basedir = self.id() + os.makedirs(basedir + b"/private") + enabled_config = client.config_from_string( + basedir, + b"tub.port", + BASECONFIG_I % (SOME_FURL,) + ( + b"[storage]\n" + b"enabled = true\n" + b"anonymous = true\n" + ) + ) + node = yield client.create_client_from_config( + enabled_config, + _introducer_factory=MemoryIntroducerClient, + ) + anonymous_storage_furl = enabled_config.get_private_config(b"storage.furl") + def check_furl(): + return node.tub.getReferenceForURL(anonymous_storage_furl) + # Perform a sanity check that our test code makes sense: is this a + # legit way to verify whether a fURL will refer to an object? + self.assertThat( + check_furl(), + # If it doesn't raise a KeyError we're in business. + Always(), + ) + + disabled_config = client.config_from_string( + basedir, + b"tub.port", + BASECONFIG_I % (SOME_FURL,) + ( + b"[storage]\n" + b"enabled = true\n" + b"anonymous = false\n" + ) + ) + node = yield client.create_client_from_config( + disabled_config, + _introducer_factory=MemoryIntroducerClient, + ) + self.assertThat( + check_furl, + raises(KeyError), + ) + + class IntroducerClients(unittest.TestCase): def test_invalid_introducer_furl(self): @@ -656,7 +800,7 @@ class IntroducerClients(unittest.TestCase): "[client]\n" "introducer.furl = None\n" ) - config = config_from_string("basedir", "client.port", cfg) + config = client.config_from_string("basedir", "client.port", cfg) with self.assertRaises(ValueError) as ctx: client.create_introducer_clients(config, main_tub=None) @@ -666,6 +810,143 @@ class IntroducerClients(unittest.TestCase): ) +def get_known_server_details(a_client): + """ + Get some details about known storage servers from a client. + + :param _Client a_client: The client to inspect. + + :return: A ``list`` of two-tuples. Each element of the list corresponds + to a "known server". The first element of each tuple is a server id. + The second is the server's announcement. + """ + return list( + (s.get_serverid(), s.get_announcement()) + for s + in a_client.storage_broker.get_known_servers() + ) + + +class StaticServers(Fixture): + """ + Create a ``servers.yaml`` file. + """ + def __init__(self, basedir, server_details): + super(StaticServers, self).__init__() + self._basedir = basedir + self._server_details = server_details + + def _setUp(self): + private = self._basedir.child(u"private") + private.makedirs() + servers = private.child(u"servers.yaml") + servers.setContent(safe_dump({ + u"storage": { + serverid: { + u"ann": announcement, + } + for (serverid, announcement) + in self._server_details + }, + })) + + +class StorageClients(SyncTestCase): + """ + Tests for storage-related behavior of ``_Client``. + """ + def setUp(self): + super(StorageClients, self).setUp() + # Some other tests create Nodes and Node mutates tempfile.tempdir and + # that screws us up because we're *not* making a Node. "Fix" it. See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3052 for the real fix, + # though. + import tempfile + tempfile.tempdir = None + + tempdir = TempDir() + self.useFixture(tempdir) + self.basedir = FilePath(tempdir.path) + + @capture_logging( + lambda case, logger: assertHasAction( + case, + logger, + actionType=u"storage-client:broker:set-static-servers", + succeeded=True, + ), + ) + def test_static_servers(self, logger): + """ + Storage servers defined in ``private/servers.yaml`` are loaded into the + storage broker. + """ + serverid = u"v0-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + announcement = { + u"nickname": u"some-storage-server", + u"anonymous-storage-FURL": u"pb://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@tcp:storage.example:100/swissnum", + } + self.useFixture( + StaticServers( + self.basedir, + [(serverid, announcement)], + ), + ) + self.assertThat( + client.create_client(self.basedir.asTextMode().path), + succeeded( + AfterPreprocessing( + get_known_server_details, + Equals([(serverid, announcement)]), + ), + ), + ) + + @capture_logging( + lambda case, logger: assertHasAction( + case, + logger, + actionType=u"storage-client:broker:make-storage-server", + succeeded=False, + ), + ) + def test_invalid_static_server(self, logger): + """ + An invalid announcement for a static server does not prevent other static + servers from being loaded. + """ + # Some good details + serverid = u"v1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + announcement = { + u"nickname": u"some-storage-server", + u"anonymous-storage-FURL": u"pb://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@tcp:storage.example:100/swissnum", + } + self.useFixture( + StaticServers( + self.basedir, + [(serverid, announcement), + # Along with a "bad" server announcement. Order in this list + # doesn't matter, yaml serializer and Python dicts are going + # to shuffle everything around kind of randomly. + (u"v0-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + {u"nickname": u"another-storage-server", + u"anonymous-storage-FURL": None, + }), + ], + ), + ) + self.assertThat( + client.create_client(self.basedir.asTextMode().path), + succeeded( + AfterPreprocessing( + get_known_server_details, + # It should have the good server details. + Equals([(serverid, announcement)]), + ), + ), + ) + + class Run(unittest.TestCase, testutil.StallMixin): def setUp(self): @@ -821,3 +1102,326 @@ class NodeMaker(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(n.get_uri(), unknown_rw) self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw) self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro) + + + +def matches_dummy_announcement(name, value): + """ + Matches the portion of an announcement for the ``DummyStorage`` storage + server plugin. + + :param unicode name: The name of the dummy plugin. + + :param unicode value: The arbitrary value in the dummy plugin + announcement. + + :return: a testtools-style matcher + """ + return MatchesDict({ + # Everyone gets a name and a fURL added to their announcement. + u"name": Equals(name), + u"storage-server-FURL": matches_furl(), + # The plugin can contribute things, too. + u"value": Equals(value), + }) + + + +class StorageAnnouncementTests(SyncTestCase): + """ + Tests for the storage announcement published by the client. + """ + def setUp(self): + super(StorageAnnouncementTests, self).setUp() + self.basedir = self.useFixture(TempDir()).path + create_node_dir(self.basedir, u"") + + + def get_config(self, storage_enabled, more_storage=b"", more_sections=b""): + return b""" +[node] +tub.location = tcp:192.0.2.0:1234 + +[storage] +enabled = {storage_enabled} +{more_storage} + +[client] +introducer.furl = pb://abcde@nowhere/fake + +{more_sections} +""".format( + storage_enabled=storage_enabled, + more_storage=more_storage, + more_sections=more_sections, +) + + + def test_no_announcement(self): + """ + No storage announcement is published if storage is not enabled. + """ + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config(storage_enabled=False), + ) + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + succeeded(AfterPreprocessing( + get_published_announcements, + Equals([]), + )), + ) + + + def test_anonymous_storage_announcement(self): + """ + A storage announcement with the anonymous storage fURL is published when + storage is enabled. + """ + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config(storage_enabled=True), + ) + client_deferred = client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ) + self.assertThat( + client_deferred, + # The Deferred succeeds + succeeded(AfterPreprocessing( + # The announcements published by the client should ... + get_published_announcements, + # Match the following list (of one element) ... + MatchesListwise([ + # The only element in the list ... + matches_storage_announcement(self.basedir), + ]), + )), + ) + + + def test_single_storage_plugin_announcement(self): + """ + The announcement from a single enabled storage plugin is published when + storage is enabled. + """ + self.useFixture(UseTestPlugins()) + + value = u"thing" + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config( + storage_enabled=True, + more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_sections=( + b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + b"some = {}\n".format(value) + ), + ), + ) + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + succeeded(AfterPreprocessing( + get_published_announcements, + MatchesListwise([ + matches_storage_announcement( + self.basedir, + options=[ + matches_dummy_announcement( + u"tahoe-lafs-dummy-v1", + value, + ), + ], + ), + ]), + )), + ) + + + def test_multiple_storage_plugin_announcements(self): + """ + The announcements from several enabled storage plugins are published when + storage is enabled. + """ + self.useFixture(UseTestPlugins()) + + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config( + storage_enabled=True, + more_storage=b"plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2", + more_sections=( + b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + b"some = thing-1\n" + b"[storageserver.plugins.tahoe-lafs-dummy-v2]\n" + b"some = thing-2\n" + ), + ), + ) + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + succeeded(AfterPreprocessing( + get_published_announcements, + MatchesListwise([ + matches_storage_announcement( + self.basedir, + options=[ + matches_dummy_announcement( + u"tahoe-lafs-dummy-v1", + u"thing-1", + ), + matches_dummy_announcement( + u"tahoe-lafs-dummy-v2", + u"thing-2", + ), + ], + ), + ]), + )), + ) + + + def test_stable_storage_server_furl(self): + """ + The value for the ``storage-server-FURL`` item in the announcement for a + particular storage server plugin is stable across different node + instantiations. + """ + self.useFixture(UseTestPlugins()) + + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config( + storage_enabled=True, + more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_sections=( + b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + b"some = thing\n" + ), + ), + ) + node_a = client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ) + node_b = client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ) + + self.assertThat( + defer.gatherResults([node_a, node_b]), + succeeded(AfterPreprocessing( + partial(map, get_published_announcements), + MatchesSameElements(), + )), + ) + + + def test_storage_plugin_without_configuration(self): + """ + A storage plugin with no configuration is loaded and announced. + """ + self.useFixture(UseTestPlugins()) + + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config( + storage_enabled=True, + more_storage=b"plugins=tahoe-lafs-dummy-v1", + ), + ) + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + succeeded(AfterPreprocessing( + get_published_announcements, + MatchesListwise([ + matches_storage_announcement( + self.basedir, + options=[ + matches_dummy_announcement( + u"tahoe-lafs-dummy-v1", + u"default-value", + ), + ], + ), + ]), + )), + ) + + + def test_broken_storage_plugin(self): + """ + A storage plugin that raises an exception from ``get_storage_server`` + causes ``client.create_client_from_config`` to return ``Deferred`` + that fails. + """ + self.useFixture(UseTestPlugins()) + + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config( + storage_enabled=True, + more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_sections=( + b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + # This will make it explode on instantiation. + b"invalid = configuration\n" + ) + ), + ) + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + failed(Always()), + ) + + def test_storage_plugin_not_found(self): + """ + ``client.create_client_from_config`` raises ``UnknownConfigError`` when + called with a configuration which enables a storage plugin that is not + available on the system. + """ + config = client.config_from_string( + self.basedir, + u"tub.port", + self.get_config( + storage_enabled=True, + more_storage=b"plugins=tahoe-lafs-dummy-vX", + ), + ) + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + failed( + AfterPreprocessing( + lambda f: f.type, + Equals(configutil.UnknownConfigError), + ), + ), + ) diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index 5786be8e0..45eb6ac25 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -9,6 +9,16 @@ from .. import client class ConfigUtilTests(GridTestMixin, unittest.TestCase): + def setUp(self): + super(ConfigUtilTests, self).setUp() + self.static_valid_config = configutil.ValidConfiguration( + dict(node=['valid']), + ) + self.dynamic_valid_config = configutil.ValidConfiguration( + dict(), + lambda section_name: section_name == "node", + lambda section_name, item_name: (section_name, item_name) == ("node", "valid"), + ) def test_config_utils(self): self.basedir = "cli/ConfigUtilTests/test-config-utils" @@ -44,7 +54,32 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): config = configutil.get_config(fname) # should succeed, no exceptions - configutil.validate_config(fname, config, dict(node=['valid'])) + configutil.validate_config( + fname, + config, + self.static_valid_config, + ) + + def test_config_dynamic_validation_success(self): + """ + A configuration with sections and items that are not matched by the static + validation but are matched by the dynamic validation is considered + valid. + """ + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + + with open(fname, 'w') as f: + f.write('[node]\nvalid = foo\n') + + config = configutil.get_config(fname) + # should succeed, no exceptions + configutil.validate_config( + fname, + config, + self.dynamic_valid_config, + ) def test_config_validation_invalid_item(self): d = self.mktemp() @@ -58,11 +93,16 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): e = self.assertRaises( configutil.UnknownConfigError, configutil.validate_config, - fname, config, dict(node=['valid']), + fname, config, + self.static_valid_config, ) self.assertIn("section [node] contains unknown option 'invalid'", str(e)) def test_config_validation_invalid_section(self): + """ + A configuration with a section that is matched by neither the static nor + dynamic validators is rejected. + """ d = self.mktemp() os.mkdir(d) fname = os.path.join(d, 'tahoe.cfg') @@ -74,10 +114,53 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): e = self.assertRaises( configutil.UnknownConfigError, configutil.validate_config, - fname, config, dict(node=['valid']), + fname, config, + self.static_valid_config, ) self.assertIn("contains unknown section [invalid]", str(e)) + def test_config_dynamic_validation_invalid_section(self): + """ + A configuration with a section that is matched by neither the static nor + dynamic validators is rejected. + """ + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + + with open(fname, 'w') as f: + f.write('[node]\nvalid = foo\n[invalid]\n') + + config = configutil.get_config(fname) + e = self.assertRaises( + configutil.UnknownConfigError, + configutil.validate_config, + fname, config, + self.dynamic_valid_config, + ) + self.assertIn("contains unknown section [invalid]", str(e)) + + def test_config_dynamic_validation_invalid_item(self): + """ + A configuration with a section, item pair that is matched by neither the + static nor dynamic validators is rejected. + """ + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + + with open(fname, 'w') as f: + f.write('[node]\nvalid = foo\ninvalid = foo\n') + + config = configutil.get_config(fname) + e = self.assertRaises( + configutil.UnknownConfigError, + configutil.validate_config, + fname, config, + self.dynamic_valid_config, + ) + self.assertIn("section [node] contains unknown option 'invalid'", str(e)) + def test_create_client_config(self): d = self.mktemp() os.mkdir(d) @@ -97,5 +180,8 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): config = configutil.get_config(fname) # should succeed, no exceptions - configutil.validate_config(fname, config, - client._valid_config_sections()) + configutil.validate_config( + fname, + config, + client._valid_config(), + ) diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 5ba6a0db5..ccec13336 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -12,9 +12,7 @@ from ..util.i2p_provider import create as create_i2p_provider from ..util.tor_provider import create as create_tor_provider -BASECONFIG = ("[client]\n" - "introducer.furl = \n" - ) +BASECONFIG = "" class TCP(unittest.TestCase): @@ -568,4 +566,3 @@ class Status(unittest.TestCase): {"h1 via hand1": "st1", "h2": "st2"}) self.assertEqual(cs.last_connection_time, None) self.assertEqual(cs.last_received_time, 5) - diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py new file mode 100644 index 000000000..53ba344a4 --- /dev/null +++ b/src/allmydata/test/test_crypto.py @@ -0,0 +1,496 @@ +import six +import unittest + +from base64 import b64decode +from binascii import a2b_hex, b2a_hex + +from twisted.python.filepath import FilePath + +from allmydata.crypto import ( + aes, + ed25519, + rsa, +) +from allmydata.crypto.util import remove_prefix +from allmydata.crypto.error import BadPrefixError + + + +RESOURCE_DIR = FilePath(__file__).parent().child('data') + + +class TestRegression(unittest.TestCase): + ''' + These tests are regression tests to ensure that the upgrade from `pycryptopp` to `cryptography` + doesn't break anything. They check that data encrypted with old keys can be decrypted with new + keys. + ''' + + AES_KEY = b'My\x9c\xc0f\xd3\x03\x9a1\x8f\xbd\x17W_\x1f2' + IV = b'\x96\x1c\xa0\xbcUj\x89\xc1\x85J\x1f\xeb=\x17\x04\xca' + + with RESOURCE_DIR.child('pycryptopp-rsa-2048-priv.txt').open('r') as f: + # Created using `pycryptopp`: + # + # from base64 import b64encode + # from pycryptopp.publickey import rsa + # priv = rsa.generate(2048) + # priv_str = b64encode(priv.serialize()) + # pub_str = b64encode(priv.get_verifying_key().serialize()) + RSA_2048_PRIV_KEY = six.b(b64decode(f.read().strip())) + + with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f: + # Signature created using `RSA_2048_PRIV_KEY` via: + # + # sig = priv.sign(b'test') + RSA_2048_SIG = six.b(b64decode(f.read().strip())) + + with RESOURCE_DIR.child('pycryptopp-rsa-2048-pub.txt').open('r') as f: + # The public key corresponding to `RSA_2048_PRIV_KEY`. + RSA_2048_PUB_KEY = six.b(b64decode(f.read().strip())) + + def test_old_start_up_test(self): + """ + This was the old startup test run at import time in `pycryptopp.cipher.aes`. + """ + enc0 = b"dc95c078a2408989ad48a21492842087530f8afbc74536b9a963b4f1c4cb738b" + cryptor = aes.create_decryptor(key=b"\x00" * 32) + ct = aes.decrypt_data(cryptor, b"\x00" * 32) + self.assertEqual(enc0, b2a_hex(ct)) + + cryptor = aes.create_decryptor(key=b"\x00" * 32) + ct1 = aes.decrypt_data(cryptor, b"\x00" * 15) + ct2 = aes.decrypt_data(cryptor, b"\x00" * 17) + self.assertEqual(enc0, b2a_hex(ct1+ct2)) + + enc0 = b"66e94bd4ef8a2c3b884cfa59ca342b2e" + cryptor = aes.create_decryptor(key=b"\x00" * 16) + ct = aes.decrypt_data(cryptor, b"\x00" * 16) + self.assertEqual(enc0, b2a_hex(ct)) + + cryptor = aes.create_decryptor(key=b"\x00" * 16) + ct1 = aes.decrypt_data(cryptor, b"\x00" * 8) + ct2 = aes.decrypt_data(cryptor, b"\x00" * 8) + self.assertEqual(enc0, b2a_hex(ct1+ct2)) + + def _test_from_Niels_AES(keysize, result): + def fake_ecb_using_ctr(k, p): + encryptor = aes.create_encryptor(key=k, iv=p) + return aes.encrypt_data(encryptor, b'\x00' * 16) + + E = fake_ecb_using_ctr + b = 16 + k = keysize + S = b'\x00' * (k + b) + + for i in range(1000): + K = S[-k:] + P = S[-k-b:-k] + S += E(K, E(K, P)) + + self.assertEqual(S[-b:], a2b_hex(result)) + + _test_from_Niels_AES(16, b'bd883f01035e58f42f9d812f2dacbcd8') + _test_from_Niels_AES(32, b'c84b0f3a2c76dd9871900b07f09bdd3e') + + def test_aes_no_iv_process_short_input(self): + ''' + The old code used the following patterns with AES ciphers. + + import os + from pycryptopp.cipher.aes import AES + key = = os.urandom(16) + ciphertext = AES(key).process(plaintext) + + This test verifies that using the new AES wrapper generates the same output. + ''' + plaintext = b'test' + expected_ciphertext = b'\x7fEK\\' + + k = aes.create_decryptor(self.AES_KEY) + ciphertext = aes.decrypt_data(k, plaintext) + + self.assertEqual(ciphertext, expected_ciphertext) + + def test_aes_no_iv_process_long_input(self): + ''' + The old code used the following patterns with AES ciphers. + + import os + from pycryptopp.cipher.aes import AES + key = = os.urandom(16) + ciphertext = AES(key).process(plaintext) + + This test verifies that using the new AES wrapper generates the same output. + ''' + plaintext = b'hi' * 32 + expected_ciphertext = ( + b'cIPAY%o:\xce\xfex\x8e@^.\x90\xb1\x80a\xff\xd8^\xac\x8d\xa7/\x1d\xe6\x92\xa1\x04\x92' + b'\x1f\xa1|\xd2$E\xb5\xe7\x9d\xae\xd1\x1f)\xe4\xc7\x83\xb8\xd5|dHhU\xc8\x9a\xb1\x10\xed' + b'\xd1\xe7|\xd1') + + k = aes.create_decryptor(self.AES_KEY) + ciphertext = aes.decrypt_data(k, plaintext) + + self.assertEqual(ciphertext, expected_ciphertext) + + def test_aes_with_iv_process_short_input(self): + ''' + The old code used the following patterns with AES ciphers. + + import os + from pycryptopp.cipher.aes import AES + key = = os.urandom(16) + ciphertext = AES(key).process(plaintext) + + This test verifies that using the new AES wrapper generates the same output. + ''' + plaintext = b'test' + expected_ciphertext = b'\x82\x0e\rt' + + k = aes.create_decryptor(self.AES_KEY, iv=self.IV) + ciphertext = aes.decrypt_data(k, plaintext) + + self.assertEqual(ciphertext, expected_ciphertext) + + def test_aes_with_iv_process_long_input(self): + ''' + The old code used the following patterns with AES ciphers. + + import os + from pycryptopp.cipher.aes import AES + key = = os.urandom(16) + ciphertext = AES(key).process(plaintext) + + This test verifies that using the new AES wrapper generates the same output. + ''' + plaintext = b'hi' * 32 + expected_ciphertext = ( + b'\x9e\x02\x16i}WL\xbf\x83\xac\xb4K\xf7\xa0\xdf\xa3\xba!3\x15\xd3(L\xb7\xb3\x91\xbcb' + b'\x97a\xdc\x100?\xf5L\x9f\xd9\xeeO\x98\xda\xf5g\x93\xa7q\xe1\xb1~\xf8\x1b\xe8[\\s' + b'\x144$\x86\xeaC^f') + + k = aes.create_decryptor(self.AES_KEY, iv=self.IV) + ciphertext = aes.decrypt_data(k, plaintext) + + self.assertEqual(ciphertext, expected_ciphertext) + + def test_decode_ed15519_keypair(self): + ''' + Created using the old code: + + from allmydata.util.keyutil import make_keypair, parse_privkey, parse_pubkey + test_data = b'test' + priv_str, pub_str = make_keypair() + priv, _ = parse_privkey(priv_str) + pub = parse_pubkey(pub_str) + sig = priv.sign(test_data) + pub.verify(sig, test_data) + + This simply checks that keys and signatures generated using the old code are still valid + using the new code. + ''' + priv_str = b'priv-v0-lqcj746bqa4npkb6zpyc6esd74x3bl6mbcjgqend7cvtgmcpawhq' + pub_str = b'pub-v0-yzpqin3of3ep363lwzxwpvgai3ps43dao46k2jds5kw5ohhpcwhq' + test_data = b'test' + sig = (b'\xde\x0e\xd6\xe2\xf5\x03]8\xfe\xa71\xad\xb4g\x03\x11\x81\x8b\x08\xffz\xf4K\xa0' + b'\x86 ier!\xe8\xe5#*\x9d\x8c\x0bI\x02\xd90\x0e7\xbeW\xbf\xa3\xfe\xc1\x1c\xf5+\xe9)' + b'\xa3\xde\xc9\xc6s\xc9\x90\xf7x\x08') + + private_key, derived_public_key = ed25519.signing_keypair_from_string(priv_str) + public_key = ed25519.verifying_key_from_string(pub_str) + + self.assertEqual( + ed25519.string_from_verifying_key(public_key), + ed25519.string_from_verifying_key(derived_public_key), + ) + + new_sig = ed25519.sign_data(private_key, test_data) + self.assertEqual(new_sig, sig) + + ed25519.verify_signature(public_key, new_sig, test_data) + ed25519.verify_signature(derived_public_key, new_sig, test_data) + ed25519.verify_signature(public_key, sig, test_data) + ed25519.verify_signature(derived_public_key, sig, test_data) + + def test_decode_rsa_keypair(self): + ''' + This simply checks that keys and signatures generated using the old code are still valid + using the new code. + ''' + priv_key, pub_key = rsa.create_signing_keypair_from_string(self.RSA_2048_PRIV_KEY) + rsa.verify_signature(pub_key, self.RSA_2048_SIG, b'test') + + def test_encrypt_data_not_bytes(self): + ''' + only bytes can be encrypted + ''' + key = b'\x00' * 16 + encryptor = aes.create_encryptor(key) + with self.assertRaises(ValueError) as ctx: + aes.encrypt_data(encryptor, u"not bytes") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_key_incorrect_size(self): + ''' + keys that aren't 16 or 32 bytes are rejected + ''' + key = b'\x00' * 12 + with self.assertRaises(ValueError) as ctx: + aes.create_encryptor(key) + self.assertIn( + "16 or 32 bytes long", + str(ctx.exception) + ) + + def test_iv_not_bytes(self): + ''' + iv must be bytes + ''' + key = b'\x00' * 16 + with self.assertRaises(TypeError) as ctx: + aes.create_encryptor(key, iv=u"1234567890abcdef") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_incorrect_iv_size(self): + ''' + iv must be 16 bytes + ''' + key = b'\x00' * 16 + with self.assertRaises(ValueError) as ctx: + aes.create_encryptor(key, iv=b'\x00' * 3) + self.assertIn( + "16 bytes long", + str(ctx.exception) + ) + + +class TestEd25519(unittest.TestCase): + """ + Test allmydata.crypto.ed25519 + """ + + def test_key_serialization(self): + """ + a serialized+deserialized keypair is the same as the original + """ + private_key, public_key = ed25519.create_signing_keypair() + private_key_str = ed25519.string_from_signing_key(private_key) + + self.assertIsInstance(private_key_str, six.string_types) + + private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str) + + # the deserialized signing keys are the same as the original + self.assertEqual( + ed25519.string_from_signing_key(private_key), + ed25519.string_from_signing_key(private_key2), + ) + self.assertEqual( + ed25519.string_from_verifying_key(public_key), + ed25519.string_from_verifying_key(public_key2), + ) + + # ditto, but for the verifying keys + public_key_str = ed25519.string_from_verifying_key(public_key) + self.assertIsInstance(public_key_str, six.string_types) + + public_key2 = ed25519.verifying_key_from_string(public_key_str) + self.assertEqual( + ed25519.string_from_verifying_key(public_key), + ed25519.string_from_verifying_key(public_key2), + ) + + def test_deserialize_private_not_bytes(self): + ''' + serialized key must be bytes + ''' + with self.assertRaises(ValueError) as ctx: + ed25519.signing_keypair_from_string(u"not bytes") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_deserialize_public_not_bytes(self): + ''' + serialized key must be bytes + ''' + with self.assertRaises(ValueError) as ctx: + ed25519.verifying_key_from_string(u"not bytes") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_signed_data_not_bytes(self): + ''' + data to sign must be bytes + ''' + priv, pub = ed25519.create_signing_keypair() + with self.assertRaises(ValueError) as ctx: + ed25519.sign_data(priv, u"not bytes") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_signature_not_bytes(self): + ''' + signature must be bytes + ''' + priv, pub = ed25519.create_signing_keypair() + with self.assertRaises(ValueError) as ctx: + ed25519.verify_signature(pub, u"not bytes", b"data") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_signature_data_not_bytes(self): + ''' + signed data must be bytes + ''' + priv, pub = ed25519.create_signing_keypair() + with self.assertRaises(ValueError) as ctx: + ed25519.verify_signature(pub, b"signature", u"not bytes") + self.assertIn( + "must be bytes", + str(ctx.exception) + ) + + def test_sign_invalid_pubkey(self): + ''' + pubkey must be correct kind of object + ''' + priv, pub = ed25519.create_signing_keypair() + with self.assertRaises(ValueError) as ctx: + ed25519.sign_data(object(), b"data") + self.assertIn( + "must be an Ed25519PrivateKey", + str(ctx.exception) + ) + + def test_verify_invalid_pubkey(self): + ''' + pubkey must be correct kind of object + ''' + priv, pub = ed25519.create_signing_keypair() + with self.assertRaises(ValueError) as ctx: + ed25519.verify_signature(object(), b"signature", b"data") + self.assertIn( + "must be an Ed25519PublicKey", + str(ctx.exception) + ) + + +class TestRsa(unittest.TestCase): + """ + Tests related to allmydata.crypto.rsa module + """ + + def test_keys(self): + """ + test that two instances of 'the same' key sign and verify data + in the same way + """ + priv_key, pub_key = rsa.create_signing_keypair(2048) + priv_key_str = rsa.der_string_from_signing_key(priv_key) + + self.assertIsInstance(priv_key_str, six.string_types) + + priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str) + + # instead of asking "are these two keys equal", we can instead + # test their function: can the second key verify a signature + # produced by the first (and FAIL a signature with different + # data) + + data_to_sign = b"test data" + sig0 = rsa.sign_data(priv_key, data_to_sign) + rsa.verify_signature(pub_key2, sig0, data_to_sign) + + # ..and the other way + sig1 = rsa.sign_data(priv_key2, data_to_sign) + rsa.verify_signature(pub_key, sig1, data_to_sign) + + # ..and a failed way + with self.assertRaises(rsa.BadSignature): + rsa.verify_signature(pub_key, sig1, data_to_sign + b"more") + + def test_sign_invalid_pubkey(self): + ''' + signing data using an invalid key-object fails + ''' + priv, pub = rsa.create_signing_keypair(1024) + with self.assertRaises(ValueError) as ctx: + rsa.sign_data(object(), b"data") + self.assertIn( + "must be an RSAPrivateKey", + str(ctx.exception) + ) + + def test_verify_invalid_pubkey(self): + ''' + verifying a signature using an invalid key-object fails + ''' + priv, pub = rsa.create_signing_keypair(1024) + with self.assertRaises(ValueError) as ctx: + rsa.verify_signature(object(), b"signature", b"data") + self.assertIn( + "must be an RSAPublicKey", + str(ctx.exception) + ) + + +class TestUtil(unittest.TestCase): + """ + tests related to allmydata.crypto utils + """ + + def test_remove_prefix_good(self): + """ + remove a simple prefix properly + """ + self.assertEquals( + remove_prefix(b"foobar", b"foo"), + b"bar" + ) + + def test_remove_prefix_bad(self): + """ + attempt to remove a prefix that doesn't exist fails with exception + """ + with self.assertRaises(BadPrefixError): + remove_prefix(b"foobar", b"bar") + + def test_remove_prefix_zero(self): + """ + removing a zero-length prefix does nothing + """ + self.assertEquals( + remove_prefix(b"foobar", b""), + b"foobar", + ) + + def test_remove_prefix_entire_string(self): + """ + removing a prefix which is the whole string is empty + """ + self.assertEquals( + remove_prefix(b"foobar", b"foobar"), + b"", + ) + + def test_remove_prefix_partial(self): + """ + removing a prefix with only partial match fails with exception + """ + with self.assertRaises(BadPrefixError): + remove_prefix(b"foobar", b"fooz"), diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 488bd6052..48ffff45a 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -9,6 +9,7 @@ from twisted.internet.interfaces import IConsumer from allmydata import uri, dirnode from allmydata.client import _Client from allmydata.immutable import upload +from allmydata.immutable.literal import LiteralFileNode from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \ MustBeDeepImmutableError, MustBeReadonlyError, \ @@ -27,6 +28,9 @@ from allmydata.nodemaker import NodeMaker from base64 import b32decode import allmydata.test.common_util as testutil +from hypothesis import given +from hypothesis.strategies import text + if six.PY3: long = int @@ -1380,7 +1384,7 @@ class Dirnode(GridTestMixin, unittest.TestCase, self.set_up_grid(oneshare=True) return self._do_initial_children_test(mdmf=True) -class MinimalFakeMutableFile: +class MinimalFakeMutableFile(object): def get_writekey(self): return "writekey" @@ -1460,6 +1464,33 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids[unicode(name)] = (nm.create_from_cap(caps[name]), {}) return kids + @given(text(min_size=1, max_size=20)) + def test_pack_unpack_unicode_hypothesis(self, name): + """ + pack -> unpack results in the same objects (with a unicode name) + """ + nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None) + fn = MinimalFakeMutableFile() + + # FIXME TODO: we shouldn't have to do this out here, but + # Hypothesis found that a name with "\x2000" does not make the + # round-trip properly .. so for now we'll only give the packer + # normalized names. + # See also: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2606 + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1076 + name = unicodedata.normalize('NFC', name) + + kids = { + name: (LiteralFileNode(uri.from_string(one_uri)), {}), + } + packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False) + write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + filenode = nm.create_from_cap(write_uri) + dn = dirnode.DirectoryNode(filenode, nm, None) + unkids = dn._unpack_contents(packed) + self.assertEqual(kids, unkids) + def test_deep_immutable(self): nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None) fn = MinimalFakeMutableFile() diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index fa9cc9745..5c78a9805 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -601,8 +601,8 @@ class DownloadTest(_Base, unittest.TestCase): # that they're old and can't handle reads that overrun the length of # the share. This exercises a different code path. for s in self.c0.storage_broker.get_connected_servers(): - rref = s.get_rref() - v1 = rref.version["http://allmydata.org/tahoe/protocols/storage/v1"] + v = s.get_version() + v1 = v["http://allmydata.org/tahoe/protocols/storage/v1"] v1["tolerates-immutable-read-overrun"] = False n = self.c0.create_node_from_uri(immutable_uri) @@ -1178,8 +1178,8 @@ class DownloadV2(_Base, unittest.TestCase): # that they're old and can't handle reads that overrun the length of # the share. This exercises a different code path. for s in self.c0.storage_broker.get_connected_servers(): - rref = s.get_rref() - v1 = rref.version["http://allmydata.org/tahoe/protocols/storage/v1"] + v = s.get_version() + v1 = v["http://allmydata.org/tahoe/protocols/storage/v1"] v1["tolerates-immutable-read-overrun"] = False # upload a file @@ -1198,8 +1198,8 @@ class DownloadV2(_Base, unittest.TestCase): self.c0 = self.g.clients[0] for s in self.c0.storage_broker.get_connected_servers(): - rref = s.get_rref() - v1 = rref.version["http://allmydata.org/tahoe/protocols/storage/v1"] + v = s.get_version() + v1 = v["http://allmydata.org/tahoe/protocols/storage/v1"] v1["tolerates-immutable-read-overrun"] = False # upload a file @@ -1287,11 +1287,12 @@ def make_servers(clientids): servers[clientid] = make_server(clientid) return servers -class MyShare: +class MyShare(object): def __init__(self, shnum, server, rtt): self._shnum = shnum self._server = server self._dyhb_rtt = rtt + def __repr__(self): return "sh%d-on-%s" % (self._shnum, self._server.get_name()) @@ -1302,21 +1303,26 @@ class MySegmentFetcher(SegmentFetcher): def _start_share(self, share, shnum): self._test_start_shares.append(share) -class FakeNode: +class FakeNode(object): def __init__(self): self.want_more = 0 self.failed = None self.processed = None self._si_prefix = "si_prefix" + def want_more_shares(self): self.want_more += 1 + def fetch_failed(self, fetcher, f): self.failed = f + def process_blocks(self, segnum, blocks): self.processed = (segnum, blocks) + def get_num_segments(self): return 1, True + class Selection(unittest.TestCase): def test_no_shares(self): node = FakeNode() diff --git a/src/allmydata/test/test_filenode.py b/src/allmydata/test/test_filenode.py index abc74981f..559beac9d 100644 --- a/src/allmydata/test/test_filenode.py +++ b/src/allmydata/test/test_filenode.py @@ -8,10 +8,10 @@ from allmydata.mutable.filenode import MutableFileNode from allmydata.util import hashutil from allmydata.util.consumer import download_to_data -class NotANode: +class NotANode(object): pass -class FakeClient: +class FakeClient(object): # just enough to let the node acquire a downloader (which it won't use), # and to get default encoding parameters def getServiceNamed(self, name): diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 471c8bb87..3774704c6 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -5,12 +5,16 @@ from twisted.application import service from foolscap.api import Tub, fireEventually, flushEventualQueue +from allmydata.crypto import aes from allmydata.storage.server import si_b2a from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import offloaded, upload from allmydata import uri, client from allmydata.util import hashutil, fileutil, mathutil -from pycryptopp.cipher.aes import AES + +from .common import ( + EMPTY_CLIENT_CONFIG, +) MiB = 1024*1024 @@ -118,7 +122,11 @@ class AssistedUpload(unittest.TestCase): self.tub = t = Tub() t.setOption("expose-remote-exception-types", False) self.s = FakeClient() - self.s.storage_broker = StorageFarmBroker(True, lambda h: self.tub) + self.s.storage_broker = StorageFarmBroker( + True, + lambda h: self.tub, + EMPTY_CLIENT_CONFIG, + ) self.s.secret_holder = client.SecretHolder("lease secret", "converge") self.s.startService() @@ -189,12 +197,12 @@ class AssistedUpload(unittest.TestCase): key = hashutil.convergence_hash(k, n, segsize, DATA, "test convergence string") assert len(key) == 16 - encryptor = AES(key) + encryptor = aes.create_encryptor(key) SI = hashutil.storage_index_hash(key) SI_s = si_b2a(SI) encfile = os.path.join(self.basedir, "CHK_encoding", SI_s) f = open(encfile, "wb") - f.write(encryptor.process(DATA)) + f.write(aes.encrypt_data(encryptor, DATA)) f.close() u = upload.Uploader(self.helper_furl) diff --git a/src/allmydata/test/test_immutable.py b/src/allmydata/test/test_immutable.py index be4c3dc20..0c7a15199 100644 --- a/src/allmydata/test/test_immutable.py +++ b/src/allmydata/test/test_immutable.py @@ -16,6 +16,9 @@ from allmydata.interfaces import NotEnoughSharesError from allmydata.immutable.upload import Data from allmydata.immutable.downloader import finder +from .no_network import ( + NoNetworkServer, +) class MockShareHashTree(object): def needed_hashes(self): @@ -106,19 +109,6 @@ class TestShareFinder(unittest.TestCase): eventually(_give_buckets_and_hunger_again) return d - class MockIServer(object): - def __init__(self, serverid, rref): - self.serverid = serverid - self.rref = rref - def get_serverid(self): - return self.serverid - def get_rref(self): - return self.rref - def get_name(self): - return "name-%s" % self.serverid - def get_version(self): - return self.rref.version - class MockStorageBroker(object): def __init__(self, servers): self.servers = servers @@ -136,9 +126,9 @@ class TestShareFinder(unittest.TestCase): mockserver1 = MockServer({1: MockBuckets(), 2: MockBuckets()}) mockserver2 = MockServer({}) mockserver3 = MockServer({3: MockBuckets()}) - servers = [ MockIServer("ms1", mockserver1), - MockIServer("ms2", mockserver2), - MockIServer("ms3", mockserver3), ] + servers = [ NoNetworkServer("ms1", mockserver1), + NoNetworkServer("ms2", mockserver2), + NoNetworkServer("ms3", mockserver3), ] mockstoragebroker = MockStorageBroker(servers) mockdownloadstatus = MockDownloadStatus() mocknode = MockNode(check_reneging=True, check_fetch_failed=True) diff --git a/src/allmydata/test/test_import.py b/src/allmydata/test/test_import.py index b34012d61..e69de29bb 100644 --- a/src/allmydata/test/test_import.py +++ b/src/allmydata/test/test_import.py @@ -1,29 +0,0 @@ - -from twisted.trial import unittest -from twisted.python.monkey import MonkeyPatcher - -import allmydata -import __builtin__ - - -class T(unittest.TestCase): - def test_report_import_error(self): - marker = "wheeeyo" - real_import_func = __import__ - def raiseIE_from_this_particular_func(name, *args): - if name == "foolscap": - raise ImportError(marker + " foolscap cant be imported") - else: - return real_import_func(name, *args) - - # Let's run as little code as possible with __import__ patched. - patcher = MonkeyPatcher((__builtin__, '__import__', raiseIE_from_this_particular_func)) - vers_and_locs, errors = patcher.runWithPatches(allmydata.get_package_versions_and_locations) - - foolscap_stuffs = [stuff for (pkg, stuff) in vers_and_locs if pkg == 'foolscap'] - self.failUnlessEqual(len(foolscap_stuffs), 1) - comment = str(foolscap_stuffs[0][2]) - self.failUnlessIn(marker, comment) - self.failUnlessIn('raiseIE_from_this_particular_func', comment) - - self.failUnless([e for e in errors if "dependency \'foolscap\' could not be imported" in e]) diff --git a/src/allmydata/test/test_inotify.py b/src/allmydata/test/test_inotify.py deleted file mode 100644 index 9f618a34a..000000000 --- a/src/allmydata/test/test_inotify.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Tests for the inotify-alike implementation L{allmydata.watchdog}. -""" - -# Note: See https://twistedmatrix.com/trac/ticket/8915 for a proposal -# to avoid all of this duplicated code from Twisted. - -from twisted.internet import defer, reactor -from twisted.python import filepath, runtime - -from allmydata.frontends.magic_folder import get_inotify_module -from .common import ( - AsyncTestCase, - skipIf, -) -inotify = get_inotify_module() - - -@skipIf(runtime.platformType == "win32", "inotify does not yet work on windows") -class INotifyTests(AsyncTestCase): - """ - Define all the tests for the basic functionality exposed by - L{inotify.INotify}. - """ - def setUp(self): - self.dirname = filepath.FilePath(self.mktemp()) - self.dirname.createDirectory() - self.inotify = inotify.INotify() - self.inotify.startReading() - self.addCleanup(self.inotify.stopReading) - return super(INotifyTests, self).setUp() - - - def _notificationTest(self, mask, operation, expectedPath=None): - """ - Test notification from some filesystem operation. - - @param mask: The event mask to use when setting up the watch. - - @param operation: A function which will be called with the - name of a file in the watched directory and which should - trigger the event. - - @param expectedPath: Optionally, the name of the path which is - expected to come back in the notification event; this will - also be passed to C{operation} (primarily useful when the - operation is being done to the directory itself, not a - file in it). - - @return: A L{Deferred} which fires successfully when the - expected event has been received or fails otherwise. - """ - if expectedPath is None: - expectedPath = self.dirname.child("foo.bar") - notified = defer.Deferred() - def cbNotified(result): - (watch, filename, events) = result - self.assertEqual(filename.asBytesMode(), expectedPath.asBytesMode()) - self.assertTrue(events & mask) - self.inotify.ignore(self.dirname) - notified.addCallback(cbNotified) - - def notify_event(*args): - notified.callback(args) - self.inotify.watch( - self.dirname, mask=mask, - callbacks=[notify_event]) - operation(expectedPath) - return notified - - - def test_modify(self): - """ - Writing to a file in a monitored directory sends an - C{inotify.IN_MODIFY} event to the callback. - """ - def operation(path): - with path.open("w") as fObj: - fObj.write(b'foo') - - return self._notificationTest(inotify.IN_MODIFY, operation) - - - def test_attrib(self): - """ - Changing the metadata of a file in a monitored directory - sends an C{inotify.IN_ATTRIB} event to the callback. - """ - def operation(path): - # Create the file. - path.touch() - # Modify the file's attributes. - path.touch() - - return self._notificationTest(inotify.IN_ATTRIB, operation) - - - def test_closeWrite(self): - """ - Closing a file which was open for writing in a monitored - directory sends an C{inotify.IN_CLOSE_WRITE} event to the - callback. - """ - def operation(path): - path.open("w").close() - - return self._notificationTest(inotify.IN_CLOSE_WRITE, operation) - - - def test_delete(self): - """ - Deleting a file in a monitored directory sends an - C{inotify.IN_DELETE} event to the callback. - """ - expectedPath = self.dirname.child("foo.bar") - expectedPath.touch() - notified = defer.Deferred() - def cbNotified(result): - (watch, filename, events) = result - self.assertEqual(filename.asBytesMode(), expectedPath.asBytesMode()) - self.assertTrue(events & inotify.IN_DELETE) - notified.addCallback(cbNotified) - self.inotify.watch( - self.dirname, mask=inotify.IN_DELETE, - callbacks=[lambda *args: notified.callback(args)]) - expectedPath.remove() - return notified - - - def test_humanReadableMask(self): - """ - L{inotify.humanReadableMask} translates all the possible event masks to a - human readable string. - """ - for mask, value in inotify._FLAG_TO_HUMAN: - self.assertEqual(inotify.humanReadableMask(mask)[0], value) - - checkMask = ( - inotify.IN_CLOSE_WRITE | inotify.IN_ACCESS | inotify.IN_OPEN) - self.assertEqual( - set(inotify.humanReadableMask(checkMask)), - set(['close_write', 'access', 'open'])) - - - def test_noAutoAddSubdirectory(self): - """ - L{inotify.INotify.watch} with autoAdd==False will stop inotify - from watching subdirectories created under the watched one. - """ - def _callback(wp, fp, mask): - # We are notified before we actually process new - # directories, so we need to defer this check. - def _(): - try: - self.assertFalse(self.inotify._isWatched(subdir)) - d.callback(None) - except Exception: - d.errback() - reactor.callLater(0, _) - - checkMask = inotify.IN_ISDIR | inotify.IN_CREATE - self.inotify.watch( - self.dirname, mask=checkMask, autoAdd=False, - callbacks=[_callback]) - subdir = self.dirname.child('test') - d = defer.Deferred() - subdir.createDirectory() - return d diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 9e5fc9bf5..d99e18c4a 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -11,9 +11,13 @@ from testtools.matchers import ( from twisted.internet import defer, address from twisted.python import log from twisted.python.filepath import FilePath +from twisted.web.template import flattenString from foolscap.api import Tub, Referenceable, fireEventually, flushEventualQueue from twisted.application import service +from allmydata.crypto import ed25519 +from allmydata.crypto.util import remove_prefix +from allmydata.crypto.error import BadSignature from allmydata.interfaces import InsufficientVersionError from allmydata.introducer.client import IntroducerClient from allmydata.introducer.server import IntroducerService, FurlFileConflictError @@ -31,12 +35,12 @@ from allmydata.client import ( create_client, create_introducer_clients, ) -from allmydata.util import pollmixin, keyutil, idlib, fileutil, yamlutil +from allmydata.util import pollmixin, idlib, fileutil, yamlutil from allmydata.util.iputil import ( listenOnUnused, ) import allmydata.test.common_util as testutil -from .common import ( +from allmydata.test.common import ( SyncTestCase, AsyncTestCase, AsyncBrokenTestCase, @@ -200,21 +204,21 @@ class Client(AsyncTestCase): furl1a = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp" furl2 = "pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo" - privkey_s, pubkey_vs = keyutil.make_keypair() - privkey, _ignored = keyutil.parse_privkey(privkey_s) - pubkey_s = keyutil.remove_prefix(pubkey_vs, "pub-") + private_key, public_key = ed25519.create_signing_keypair() + public_key_str = ed25519.string_from_verifying_key(public_key) + pubkey_s = remove_prefix(public_key_str, "pub-") # ann1: ic1, furl1 # ann1a: ic1, furl1a (same SturdyRef, different connection hints) # ann1b: ic2, furl1 # ann2: ic2, furl2 - self.ann1 = make_ann_t(ic1, furl1, privkey, seqnum=10) - self.ann1old = make_ann_t(ic1, furl1, privkey, seqnum=9) - self.ann1noseqnum = make_ann_t(ic1, furl1, privkey, seqnum=None) - self.ann1b = make_ann_t(ic2, furl1, privkey, seqnum=11) - self.ann1a = make_ann_t(ic1, furl1a, privkey, seqnum=12) - self.ann2 = make_ann_t(ic2, furl2, privkey, seqnum=13) + self.ann1 = make_ann_t(ic1, furl1, private_key, seqnum=10) + self.ann1old = make_ann_t(ic1, furl1, private_key, seqnum=9) + self.ann1noseqnum = make_ann_t(ic1, furl1, private_key, seqnum=None) + self.ann1b = make_ann_t(ic2, furl1, private_key, seqnum=11) + self.ann1a = make_ann_t(ic1, furl1a, private_key, seqnum=12) + self.ann2 = make_ann_t(ic2, furl2, private_key, seqnum=13) ic1.remote_announce_v2([self.ann1]) # queues eventual-send d = fireEventually() @@ -298,14 +302,13 @@ class Server(AsyncTestCase): FilePath(self.mktemp())) furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp" - privkey_s, _ = keyutil.make_keypair() - privkey, _ = keyutil.parse_privkey(privkey_s) + private_key, _ = ed25519.create_signing_keypair() - ann1 = make_ann_t(ic1, furl1, privkey, seqnum=10) - ann1_old = make_ann_t(ic1, furl1, privkey, seqnum=9) - ann1_new = make_ann_t(ic1, furl1, privkey, seqnum=11) - ann1_noseqnum = make_ann_t(ic1, furl1, privkey, seqnum=None) - ann1_badseqnum = make_ann_t(ic1, furl1, privkey, seqnum="not an int") + ann1 = make_ann_t(ic1, furl1, private_key, seqnum=10) + ann1_old = make_ann_t(ic1, furl1, private_key, seqnum=9) + ann1_new = make_ann_t(ic1, furl1, private_key, seqnum=11) + ann1_noseqnum = make_ann_t(ic1, furl1, private_key, seqnum=None) + ann1_badseqnum = make_ann_t(ic1, furl1, private_key, seqnum="not an int") i.remote_publish_v2(ann1, None) all = i.get_announcements() @@ -396,22 +399,24 @@ class Queue(SystemTestMixin, AsyncTestCase): u"nickname", "version", "oldest", {}, fakeseq, FilePath(self.mktemp())) furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short") - sk_s, vk_s = keyutil.make_keypair() - sk, _ignored = keyutil.parse_privkey(sk_s) + private_key, _ = ed25519.create_signing_keypair() d = introducer.disownServiceParent() + def _offline(ign): # now that the introducer server is offline, create a client and # publish some messages c.setServiceParent(self.parent) # this starts the reconnector - c.publish("storage", make_ann(furl1), sk) + c.publish("storage", make_ann(furl1), private_key) introducer.setServiceParent(self.parent) # restart the server # now wait for the messages to be delivered def _got_announcement(): return bool(introducer.get_announcements()) return self.poll(_got_announcement) + d.addCallback(_offline) + def _done(ign): v = introducer.get_announcements()[0] furl = v.announcement["anonymous-storage-FURL"] @@ -427,6 +432,7 @@ class Queue(SystemTestMixin, AsyncTestCase): return False return True return self.poll(_idle) + d.addCallback(_wait_until_idle) return d @@ -482,16 +488,15 @@ class SystemTest(SystemTestMixin, AsyncTestCase): expected_announcements[i] += 1 # all expect a 'storage' announcement node_furl = tub.registerReference(Referenceable()) - privkey_s, pubkey_s = keyutil.make_keypair() - privkey, _ignored = keyutil.parse_privkey(privkey_s) - privkeys[i] = privkey - pubkeys[i] = pubkey_s + private_key, public_key = ed25519.create_signing_keypair() + public_key_str = ed25519.string_from_verifying_key(public_key) + privkeys[i] = private_key + pubkeys[i] = public_key_str if i < NUM_STORAGE: # sign all announcements - c.publish("storage", make_ann(node_furl), privkey) - assert pubkey_s.startswith("pub-") - printable_serverids[i] = pubkey_s[len("pub-"):] + c.publish("storage", make_ann(node_furl), private_key) + printable_serverids[i] = remove_prefix(public_key_str, b"pub-") publishing_clients.append(c) else: # the last one does not publish anything @@ -500,13 +505,12 @@ class SystemTest(SystemTestMixin, AsyncTestCase): if i == 2: # also publish something that nobody cares about boring_furl = tub.registerReference(Referenceable()) - c.publish("boring", make_ann(boring_furl), privkey) + c.publish("boring", make_ann(boring_furl), private_key) c.setServiceParent(self.parent) clients.append(c) tubs[c] = tub - def _wait_for_connected(ign): def _connected(): for c in clients: @@ -589,7 +593,12 @@ class SystemTest(SystemTestMixin, AsyncTestCase): # now check the web status, make sure it renders without error ir = introweb.IntroducerRoot(self.parent) self.parent.nodeid = "NODEID" - text = ir.renderSynchronously().decode("utf-8") + log.msg("_check1 done") + return flattenString(None, ir._create_element()) + d.addCallback(_check1) + + def _check2(flattened_bytes): + text = flattened_bytes.decode("utf-8") self.assertIn(NICKNAME % "0", text) # a v2 client self.assertIn(NICKNAME % "1", text) # another v2 client for i in range(NUM_STORAGE): @@ -598,8 +607,8 @@ class SystemTest(SystemTestMixin, AsyncTestCase): # make sure there isn't a double-base32ed string too self.assertNotIn(idlib.nodeid_b2a(printable_serverids[i]), text, (i,printable_serverids[i],text)) - log.msg("_check1 done") - d.addCallback(_check1) + log.msg("_check2 done") + d.addCallback(_check2) # force an introducer reconnect, by shutting down the Tub it's using # and starting a new Tub (with the old introducer). Everybody should @@ -716,7 +725,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase): os.makedirs(self.basedir) return self.do_system_test() -class FakeRemoteReference: +class FakeRemoteReference(object): def notifyOnDisconnect(self, *args, **kwargs): pass def getRemoteTubID(self): return "62ubehyunnyhzs7r6vdonnm2hpi52w6y" def getLocationHints(self): return ["tcp:here.example.com:1234", @@ -746,6 +755,7 @@ class ClientInfo(AsyncTestCase): self.failUnlessEqual(s0.nickname, NICKNAME % u"v2") self.failUnlessEqual(s0.version, "my_version") + class Announcements(AsyncTestCase): def test_client_v2_signed(self): introducer = IntroducerService() @@ -755,16 +765,17 @@ class Announcements(AsyncTestCase): "my_version", "oldest", app_versions, fakeseq, FilePath(self.mktemp())) furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" - sk_s, vk_s = keyutil.make_keypair() - sk, _ignored = keyutil.parse_privkey(sk_s) - pks = keyutil.remove_prefix(vk_s, "pub-") - ann_t0 = make_ann_t(client_v2, furl1, sk, 10) + + private_key, public_key = ed25519.create_signing_keypair() + public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), "pub-") + + ann_t0 = make_ann_t(client_v2, furl1, private_key, 10) canary0 = Referenceable() introducer.remote_publish_v2(ann_t0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.assertThat(a[0].canary, Is(canary0)) - self.failUnlessEqual(a[0].index, ("storage", pks)) + self.failUnlessEqual(a[0].index, ("storage", public_key_str)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") @@ -786,20 +797,18 @@ class Announcements(AsyncTestCase): # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting # confused by this, disable storage. - f = open(os.path.join(basedir, "tahoe.cfg"), "w") - f.write("[client]\n") - f.write("introducer.furl = nope\n") - f.write("[storage]\n") - f.write("enabled = false\n") - f.close() + with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: + f.write("[client]\n") + f.write("introducer.furl = nope\n") + f.write("[storage]\n") + f.write("enabled = false\n") c = yield create_client(basedir) ic = c.introducer_clients[0] - sk_s, vk_s = keyutil.make_keypair() - sk, _ignored = keyutil.parse_privkey(sk_s) - pub1 = keyutil.remove_prefix(vk_s, "pub-") + private_key, public_key = ed25519.create_signing_keypair() + public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), "pub-") furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short") - ann_t = make_ann_t(ic, furl1, sk, 1) + ann_t = make_ann_t(ic, furl1, private_key, 1) ic.got_announcements([ann_t]) yield flushEventualQueue() @@ -807,7 +816,7 @@ class Announcements(AsyncTestCase): # check the cache for the announcement announcements = self._load_cache(cache_filepath) self.failUnlessEqual(len(announcements), 1) - self.failUnlessEqual(announcements[0]['key_s'], pub1) + self.failUnlessEqual(announcements[0]['key_s'], public_key_str) ann = announcements[0]["ann"] self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["seqnum"], 1) @@ -815,29 +824,28 @@ class Announcements(AsyncTestCase): # a new announcement that replaces the first should replace the # cached entry, not duplicate it furl2 = furl1 + "er" - ann_t2 = make_ann_t(ic, furl2, sk, 2) + ann_t2 = make_ann_t(ic, furl2, private_key, 2) ic.got_announcements([ann_t2]) yield flushEventualQueue() announcements = self._load_cache(cache_filepath) self.failUnlessEqual(len(announcements), 1) - self.failUnlessEqual(announcements[0]['key_s'], pub1) + self.failUnlessEqual(announcements[0]['key_s'], public_key_str) ann = announcements[0]["ann"] self.failUnlessEqual(ann["anonymous-storage-FURL"], furl2) self.failUnlessEqual(ann["seqnum"], 2) # but a third announcement with a different key should add to the # cache - sk_s2, vk_s2 = keyutil.make_keypair() - sk2, _ignored = keyutil.parse_privkey(sk_s2) - pub2 = keyutil.remove_prefix(vk_s2, "pub-") + private_key2, public_key2 = ed25519.create_signing_keypair() + public_key_str2 = remove_prefix(ed25519.string_from_verifying_key(public_key2), "pub-") furl3 = "pb://onug64tu@127.0.0.1:456/short" - ann_t3 = make_ann_t(ic, furl3, sk2, 1) + ann_t3 = make_ann_t(ic, furl3, private_key2, 1) ic.got_announcements([ann_t3]) yield flushEventualQueue() announcements = self._load_cache(cache_filepath) self.failUnlessEqual(len(announcements), 2) - self.failUnlessEqual(set([pub1, pub2]), + self.failUnlessEqual(set([public_key_str, public_key_str2]), set([a["key_s"] for a in announcements])) self.failUnlessEqual(set([furl2, furl3]), set([a["ann"]["anonymous-storage-FURL"] @@ -855,17 +863,17 @@ class Announcements(AsyncTestCase): ic2._load_announcements() # normally happens when connection fails yield flushEventualQueue() - self.failUnless(pub1 in announcements) - self.failUnlessEqual(announcements[pub1]["anonymous-storage-FURL"], + self.failUnless(public_key_str in announcements) + self.failUnlessEqual(announcements[public_key_str]["anonymous-storage-FURL"], furl2) - self.failUnlessEqual(announcements[pub2]["anonymous-storage-FURL"], + self.failUnlessEqual(announcements[public_key_str2]["anonymous-storage-FURL"], furl3) c2 = yield create_client(basedir) c2.introducer_clients[0]._load_announcements() yield flushEventualQueue() self.assertEqual(c2.storage_broker.get_all_serverids(), - frozenset([pub1, pub2])) + frozenset([public_key_str, public_key_str2])) class ClientSeqnums(AsyncBrokenTestCase): @@ -894,7 +902,7 @@ class ClientSeqnums(AsyncBrokenTestCase): f.close() return int(seqnum) - ic.publish("sA", {"key": "value1"}, c._node_key) + ic.publish("sA", {"key": "value1"}, c._node_private_key) self.failUnlessEqual(read_seqnum(), 1) self.failUnless("sA" in outbound) self.failUnlessEqual(outbound["sA"]["seqnum"], 1) @@ -906,7 +914,7 @@ class ClientSeqnums(AsyncBrokenTestCase): # publishing a second service causes both services to be # re-published, with the next higher sequence number - ic.publish("sB", {"key": "value2"}, c._node_key) + ic.publish("sB", {"key": "value2"}, c._node_private_key) self.failUnlessEqual(read_seqnum(), 2) self.failUnless("sB" in outbound) self.failUnlessEqual(outbound["sB"]["seqnum"], 2) @@ -978,11 +986,12 @@ class DecodeFurl(SyncTestCase): self.failUnlessEqual(nodeid, "\x9fM\xf2\x19\xcckU0\xbf\x03\r\x10\x99\xfb&\x9b-\xc7A\x1d") class Signatures(SyncTestCase): + def test_sign(self): ann = {"key1": "value1"} - sk_s,vk_s = keyutil.make_keypair() - sk,ignored = keyutil.parse_privkey(sk_s) - ann_t = sign_to_foolscap(ann, sk) + private_key, public_key = ed25519.create_signing_keypair() + public_key_str = ed25519.string_from_verifying_key(public_key) + ann_t = sign_to_foolscap(ann, private_key) (msg, sig, key) = ann_t self.failUnlessEqual(type(msg), type("".encode("utf-8"))) # bytes self.failUnlessEqual(json.loads(msg.decode("utf-8")), ann) @@ -990,7 +999,7 @@ class Signatures(SyncTestCase): self.failUnless(key.startswith("v0-")) (ann2,key2) = unsign_from_foolscap(ann_t) self.failUnlessEqual(ann2, ann) - self.failUnlessEqual("pub-"+key2, vk_s) + self.failUnlessEqual("pub-" + key2, public_key_str) # not signed self.failUnlessRaises(UnknownKeyError, @@ -1000,14 +1009,34 @@ class Signatures(SyncTestCase): # bad signature bad_ann = {"key1": "value2"} bad_msg = json.dumps(bad_ann).encode("utf-8") - self.failUnlessRaises(keyutil.BadSignatureError, - unsign_from_foolscap, (bad_msg,sig,key)) + self.failUnlessRaises(BadSignature, + unsign_from_foolscap, (bad_msg, sig, key)) # unrecognized signatures self.failUnlessRaises(UnknownKeyError, - unsign_from_foolscap, (bad_msg,"v999-sig",key)) + unsign_from_foolscap, (bad_msg, "v999-sig", key)) self.failUnlessRaises(UnknownKeyError, - unsign_from_foolscap, (bad_msg,sig,"v999-key")) + unsign_from_foolscap, (bad_msg, sig, "v999-key")) + + def test_unsigned_announcement(self): + ed25519.verifying_key_from_string(b"pub-v0-wodst6ly4f7i7akt2nxizsmmy2rlmer6apltl56zctn67wfyu5tq") + mock_tub = Mock() + ic = IntroducerClient( + mock_tub, + u"pb://", + u"fake_nick", + "0.0.0", + "1.2.3", + {}, + (0, u"i am a nonce"), + "invalid", + ) + self.assertEqual(0, ic._debug_counts["inbound_announcement"]) + ic.got_announcements([ + ("message", "v0-aaaaaaa", "v0-wodst6ly4f7i7akt2nxizsmmy2rlmer6apltl56zctn67wfyu5tq") + ]) + # we should have rejected this announcement due to a bad signature + self.assertEqual(0, ic._debug_counts["inbound_announcement"]) # add tests of StorageFarmBroker: if it receives duplicate announcements, it diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 71a8752ac..ebb77c3c8 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -86,7 +86,7 @@ WINDOWS_TEST_ADDRESSES = set(["127.0.0.1", "10.0.2.15", "192.168.0.10"]) CYGWIN_TEST_ADDRESSES = set(["127.0.0.1", "192.168.0.10"]) -class FakeProcess: +class FakeProcess(object): def __init__(self, output, err): self.output = output self.err = err diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py deleted file mode 100644 index c39027eea..000000000 --- a/src/allmydata/test/test_magic_folder.py +++ /dev/null @@ -1,2425 +0,0 @@ -from __future__ import print_function - -import os, sys, time -import stat, shutil, json -import mock -from os.path import join, exists, isdir -from errno import ENOENT - -from twisted.internet import defer, task, reactor -from twisted.python.runtime import platform -from twisted.python.filepath import FilePath - -from testtools.matchers import ( - Not, - Is, - ContainsDict, - Equals, -) - -from eliot import ( - Message, - start_action, - log_call, -) -from eliot.twisted import DeferredContext - -from allmydata.interfaces import ( - IDirectoryNode, - NoSharesError, -) -from allmydata.util.assertutil import precondition - -from allmydata.util import fake_inotify, fileutil, configutil, yamlutil -from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath -from allmydata.util.consumer import download_to_data -from allmydata.test.no_network import GridTestMixin -from allmydata.test.common_util import ReallyEqualMixin -from .common import ( - ShouldFailMixin, - SyncTestCase, - AsyncTestCase, - skipIf, -) -from .cli.test_magic_folder import MagicFolderCLITestMixin - -from allmydata.frontends import magic_folder -from allmydata.frontends.magic_folder import ( - MagicFolder, WriteFileMixin, - ConfigurationError, -) -from allmydata import magicfolderdb, magicpath -from allmydata.util.fileutil import get_pathinfo -from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.immutable.upload import Data -from allmydata.mutable.common import ( - UnrecoverableFileError, -) - -from ..util.eliotutil import ( - inline_callbacks, - log_call_deferred, -) - -_debug = False - -try: - magic_folder.get_inotify_module() -except NotImplementedError: - support_missing = True - support_message = ( - "Magic Folder support can only be tested for-real on an OS that " - "supports inotify or equivalent." - ) -else: - support_missing = False - support_message = None - -if platform.isMacOSX(): - def modified_mtime_barrier(path): - """ - macOS filesystem (HFS+) has one second resolution on filesystem - modification time metadata. Make sure that code running after this - function which modifies the file will produce a changed mtime on that - file. - """ - try: - mtime = path.getModificationTime() - except OSError as e: - if e.errno == ENOENT: - # If the file does not exist yet, there is no current mtime - # value that might match a future mtime value. We have - # nothing to do. - return - # Propagate any other errors as we don't know what's going on. - raise - if int(time.time()) == int(mtime): - # The current time matches the file's modification time, to the - # resolution of the filesystem metadata. Therefore, change the - # current time. - time.sleep(1) -else: - def modified_mtime_barrier(path): - """ - non-macOS platforms have sufficiently high-resolution file modification - time metadata that nothing in particular is required to ensure a - modified mtime as a result of a future write. - """ - - -class NewConfigUtilTests(SyncTestCase): - - def setUp(self): - # some tests look at the umask of created directories or files - # so we set an explicit one - old_umask = os.umask(0o022) - self.addCleanup(lambda: os.umask(old_umask)) - self.basedir = abspath_expanduser_unicode(unicode(self.mktemp())) - os.mkdir(self.basedir) - self.local_dir = abspath_expanduser_unicode(unicode(self.mktemp())) - os.mkdir(self.local_dir) - privdir = join(self.basedir, "private") - os.mkdir(privdir) - - self.poll_interval = 60 - self.collective_dircap = u"a" * 32 - self.magic_folder_dircap = u"b" * 32 - - self.folders = { - u"default": { - u"directory": self.local_dir, - u"upload_dircap": self.magic_folder_dircap, - u"collective_dircap": self.collective_dircap, - u"poll_interval": self.poll_interval, - } - } - - # we need a bit of tahoe.cfg - self.write_tahoe_config( - self.basedir, - u"[magic_folder]\n" - u"enabled = True\n", - ) - # ..and the yaml - self.write_magic_folder_config(self.basedir, self.folders) - return super(NewConfigUtilTests, self).setUp() - - def write_tahoe_config(self, basedir, tahoe_config): - with open(join(basedir, u"tahoe.cfg"), "w") as f: - f.write(tahoe_config) - - def write_magic_folder_config(self, basedir, folder_configuration): - yaml_fname = join(basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write(yamlutil.safe_dump({u"magic-folders": folder_configuration})) - - def test_load(self): - folders = magic_folder.load_magic_folders(self.basedir) - self.assertEqual(['default'], list(folders.keys())) - self.assertEqual(folders['default'][u'umask'], 0o077) - - def test_load_makes_directory(self): - """ - If the *directory* does not exist then it is created by - ``load_magic_folders``. - """ - os.rmdir(self.local_dir) - # Just pick some arbitrary bits. - # rwxr-xr-- - perm = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH - self.folders[u"default"][u"umask"] = (0o777 & ~perm) - self.write_magic_folder_config(self.basedir, self.folders) - - magic_folder.load_magic_folders(self.basedir) - - # It is created. - self.assertTrue( - isdir(self.local_dir), - "magic-folder local directory {} was not created".format( - self.local_dir, - ), - ) - # It has permissions determined by the configured umask. - if sys.platform != "win32": - self.assertEqual( - perm, - stat.S_IMODE(os.stat(self.local_dir).st_mode), - ) - else: - # Do directories even have permissions on Windows? - print("Not asserting directory-creation mode on windows") - - def test_directory_collision(self): - """ - If a non-directory already exists at the magic folder's configured local - directory path, ``load_magic_folders`` raises an exception. - """ - os.rmdir(self.local_dir) - open(self.local_dir, "w").close() - - with self.assertRaises(ConfigurationError) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "exists and is not a directory", - str(ctx.exception), - ) - - def test_directory_creation_error(self): - """ - If a directory at the magic folder's configured local directory path - cannot be created for some other reason, ``load_magic_folders`` raises - an exception. - """ - os.rmdir(self.local_dir) - open(self.local_dir, "w").close() - self.folders[u"default"][u"directory"] = self.local_dir + "/foo" - self.write_magic_folder_config(self.basedir, self.folders) - - with self.assertRaises(ConfigurationError) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "could not be created", - str(ctx.exception), - ) - - def test_both_styles_of_config(self): - os.unlink(join(self.basedir, u"private", u"magic_folders.yaml")) - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "[magic_folder] is enabled but has no YAML file and no 'local.directory' option", - str(ctx.exception) - ) - - def test_wrong_obj(self): - yaml_fname = join(self.basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write('----\n') - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "should contain a dict", - str(ctx.exception) - ) - - def test_no_magic_folders(self): - yaml_fname = join(self.basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write('') - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "should contain a dict", - str(ctx.exception) - ) - - def test_magic_folders_not_dict(self): - yaml_fname = join(self.basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write('magic-folders: "foo"\n') - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "should be a dict", - str(ctx.exception) - ) - self.assertIn( - "'magic-folders'", - str(ctx.exception) - ) - - def test_wrong_umask_obj(self): - """ - If a umask is given for a magic-folder that is not an integer, an - exception is raised. - """ - self.folders[u"default"][u"umask"] = "0077" - yaml_fname = join(self.basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write(yamlutil.safe_dump({u"magic-folders": self.folders})) - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "umask must be an integer", - str(ctx.exception) - ) - - def test_wrong_sub_obj(self): - yaml_fname = join(self.basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write("magic-folders:\n default: foo\n") - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "must itself be a dict", - str(ctx.exception) - ) - - def test_missing_interval(self): - del self.folders[u"default"]["poll_interval"] - yaml_fname = join(self.basedir, u"private", u"magic_folders.yaml") - with open(yaml_fname, "w") as f: - f.write(yamlutil.safe_dump({u"magic-folders": self.folders})) - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "missing 'poll_interval'", - str(ctx.exception) - ) - - -class LegacyConfigUtilTests(SyncTestCase): - - def setUp(self): - # create a valid 'old style' magic-folder configuration - self.basedir = abspath_expanduser_unicode(unicode(self.mktemp())) - os.mkdir(self.basedir) - self.local_dir = abspath_expanduser_unicode(unicode(self.mktemp())) - os.mkdir(self.local_dir) - privdir = join(self.basedir, "private") - os.mkdir(privdir) - - # state tests might need to know - self.poll_interval = 60 - self.collective_dircap = u"a" * 32 - self.magic_folder_dircap = u"b" * 32 - - # write fake config structure - with open(join(self.basedir, u"tahoe.cfg"), "w") as f: - f.write( - u"[magic_folder]\n" - u"enabled = True\n" - u"local.directory = {}\n" - u"poll_interval = {}\n".format( - self.local_dir, - self.poll_interval, - ) - ) - with open(join(privdir, "collective_dircap"), "w") as f: - f.write("{}\n".format(self.collective_dircap)) - with open(join(privdir, "magic_folder_dircap"), "w") as f: - f.write("{}\n".format(self.magic_folder_dircap)) - with open(join(privdir, "magicfolderdb.sqlite"), "w") as f: - pass - return super(LegacyConfigUtilTests, self).setUp() - - def test_load_legacy_no_dir(self): - expected = self.local_dir + 'foo' - with open(join(self.basedir, u"tahoe.cfg"), "w") as f: - f.write( - u"[magic_folder]\n" - u"enabled = True\n" - u"local.directory = {}\n" - u"poll_interval = {}\n".format( - expected, - self.poll_interval, - ) - ) - - magic_folder.load_magic_folders(self.basedir) - - self.assertTrue( - isdir(expected), - "magic-folder local directory {} was not created".format( - expected, - ), - ) - - def test_load_legacy_not_a_dir(self): - with open(join(self.basedir, u"tahoe.cfg"), "w") as f: - f.write( - u"[magic_folder]\n" - u"enabled = True\n" - u"local.directory = {}\n" - u"poll_interval = {}\n".format( - self.local_dir + "foo", - self.poll_interval, - ) - ) - with open(self.local_dir + "foo", "w") as f: - f.write("not a directory") - - with self.assertRaises(ConfigurationError) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "is not a directory", - str(ctx.exception) - ) - - def test_load_legacy_and_new(self): - with open(join(self.basedir, u"private", u"magic_folders.yaml"), "w") as f: - f.write("---") - - with self.assertRaises(Exception) as ctx: - magic_folder.load_magic_folders(self.basedir) - self.assertIn( - "both old-style configuration and new-style", - str(ctx.exception) - ) - - def test_upgrade(self): - # test data is created in setUp; upgrade config - magic_folder._upgrade_magic_folder_config(self.basedir) - - # ensure old stuff is gone - self.assertFalse( - exists(join(self.basedir, "private", "collective_dircap")) - ) - self.assertFalse( - exists(join(self.basedir, "private", "magic_folder_dircap")) - ) - self.assertFalse( - exists(join(self.basedir, "private", "magicfolderdb.sqlite")) - ) - - # ensure we've got the new stuff - self.assertTrue( - exists(join(self.basedir, "private", "magicfolder_default.sqlite")) - ) - # what about config? - config = configutil.get_config(join(self.basedir, u"tahoe.cfg")) - self.assertFalse(config.has_option("magic_folder", "local.directory")) - - def test_load_legacy(self): - folders = magic_folder.load_magic_folders(self.basedir) - - self.assertEqual(['default'], list(folders.keys())) - self.assertTrue( - exists(join(self.basedir, "private", "collective_dircap")) - ) - self.assertTrue( - exists(join(self.basedir, "private", "magic_folder_dircap")) - ) - self.assertTrue( - exists(join(self.basedir, "private", "magicfolderdb.sqlite")) - ) - - def test_load_legacy_upgrade(self): - magic_folder.maybe_upgrade_magic_folders(self.basedir) - folders = magic_folder.load_magic_folders(self.basedir) - - self.assertEqual(['default'], list(folders.keys())) - # 'legacy' files should be gone - self.assertFalse( - exists(join(self.basedir, "private", "collective_dircap")) - ) - self.assertFalse( - exists(join(self.basedir, "private", "magic_folder_dircap")) - ) - self.assertFalse( - exists(join(self.basedir, "private", "magicfolderdb.sqlite")) - ) - - - -class MagicFolderDbTests(SyncTestCase): - - def setUp(self): - self.temp = abspath_expanduser_unicode(unicode(self.mktemp())) - os.mkdir(self.temp) - self.addCleanup(lambda: shutil.rmtree(self.temp)) - dbfile = abspath_expanduser_unicode(u"testdb.sqlite", base=self.temp) - self.db = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1)) - self.addCleanup(lambda: self.db.close()) - self.failUnless(self.db, "unable to create magicfolderdb from %r" % (dbfile,)) - self.failUnlessEqual(self.db.VERSION, 1) - return super(MagicFolderDbTests, self).setUp() - - def test_create(self): - self.db.did_upload_version( - relpath_u=u'fake_path', - version=0, - last_uploaded_uri=None, - last_downloaded_uri='URI:foo', - last_downloaded_timestamp=1234.5, - pathinfo=get_pathinfo(self.temp), # a directory, but should be fine for test - ) - - entry = self.db.get_db_entry(u'fake_path') - self.assertTrue(entry is not None) - self.assertEqual(entry.last_downloaded_uri, 'URI:foo') - - def test_update(self): - self.db.did_upload_version( - relpath_u=u'fake_path', - version=0, - last_uploaded_uri=None, - last_downloaded_uri='URI:foo', - last_downloaded_timestamp=1234.5, - pathinfo=get_pathinfo(self.temp), # a directory, but should be fine for test - ) - self.db.did_upload_version( - relpath_u=u'fake_path', - version=1, - last_uploaded_uri=None, - last_downloaded_uri='URI:bar', - last_downloaded_timestamp=1234.5, - pathinfo=get_pathinfo(self.temp), # a directory, but should be fine for test - ) - - entry = self.db.get_db_entry(u'fake_path') - self.assertTrue(entry is not None) - self.assertEqual(entry.last_downloaded_uri, 'URI:bar') - self.assertEqual(entry.version, 1) - - def test_same_content_different_path(self): - content_uri = 'URI:CHK:27d2yruqwk6zb2w7hkbbfxxbue:ipmszjysmn4vdeaxz7rtxtv3gwv6vrqcg2ktrdmn4oxqqucltxxq:2:4:1052835840' - self.db.did_upload_version( - relpath_u=u'path0', - version=0, - last_uploaded_uri=None, - last_downloaded_uri=content_uri, - last_downloaded_timestamp=1234.5, - pathinfo=get_pathinfo(self.temp), # a directory, but should be fine for test - ) - self.db.did_upload_version( - relpath_u=u'path1', - version=0, - last_uploaded_uri=None, - last_downloaded_uri=content_uri, - last_downloaded_timestamp=1234.5, - pathinfo=get_pathinfo(self.temp), # a directory, but should be fine for test - ) - - entry = self.db.get_db_entry(u'path0') - self.assertTrue(entry is not None) - self.assertEqual(entry.last_downloaded_uri, content_uri) - - entry = self.db.get_db_entry(u'path1') - self.assertTrue(entry is not None) - self.assertEqual(entry.last_downloaded_uri, content_uri) - - def test_get_direct_children(self): - """ - ``get_direct_children`` returns a list of ``PathEntry`` representing each - local file in the database which is a direct child of the given path. - """ - def add_file(relpath_u): - self.db.did_upload_version( - relpath_u=relpath_u, - version=0, - last_uploaded_uri=None, - last_downloaded_uri=None, - last_downloaded_timestamp=1234, - pathinfo=get_pathinfo(self.temp), - ) - paths = [ - u"some_random_file", - u"the_target_directory_is_elsewhere", - u"the_target_directory_is_not_this/", - u"the_target_directory_is_not_this/and_not_in_here", - u"the_target_directory/", - u"the_target_directory/foo", - u"the_target_directory/bar", - u"the_target_directory/baz", - u"the_target_directory/quux/", - u"the_target_directory/quux/exclude_grandchildren", - u"the_target_directory/quux/and_great_grandchildren/", - u"the_target_directory/quux/and_great_grandchildren/foo", - u"the_target_directory_is_over/stuff", - u"please_ignore_this_for_sure", - ] - for relpath_u in paths: - add_file(relpath_u) - - expected_paths = [ - u"the_target_directory/foo", - u"the_target_directory/bar", - u"the_target_directory/baz", - u"the_target_directory/quux/", - ] - - actual_paths = list( - localpath.relpath_u - for localpath - in self.db.get_direct_children(u"the_target_directory") - ) - self.assertEqual(expected_paths, actual_paths) - - -def iterate_downloader(magic): - return magic.downloader._processing_iteration() - - -def iterate_uploader(magic): - return magic.uploader._processing_iteration() - -@inline_callbacks -def iterate(magic): - yield iterate_uploader(magic) - yield iterate_downloader(magic) - - -@inline_callbacks -def notify_when_pending(uploader, filename): - with start_action(action_type=u"notify-when-pending", filename=filename): - relpath = uploader._get_relpath(FilePath(filename)) - while not uploader.is_pending(relpath): - Message.log(message_type=u"not-pending") - yield uploader.set_hook('inotify') - - -class FileOperationsHelper(object): - """ - This abstracts all file operations we might do in magic-folder unit-tests. - - This is so we can correctly wait for inotify events to 'actually' - propagate. For the mock tests this is easy, since we're sending - them sychronously. For the Real tests we have to wait for the - actual inotify thing. - """ - _timeout = 30.0 - - def __init__(self, uploader, inject_events=False): - self._uploader = uploader - self._inotify = fake_inotify # fixme? - self._fake_inotify = inject_events - - @log_call_deferred(action_type=u"fileops:move") - def move(self, from_path_u, to_path_u): - from_fname = from_path_u - to_fname = to_path_u - d = self._uploader.set_hook('inotify') - os.rename(from_fname, to_fname) - - self._maybe_notify(to_fname, self._inotify.IN_MOVED_TO) - # hmm? we weren't faking IN_MOVED_FROM previously .. but seems like we should have been? - # self._uploader._notifier.event(to_filepath(from_fname), self._inotify.IN_MOVED_FROM) - return d.addTimeout(self._timeout, reactor) - - @log_call_deferred(action_type=u"fileops:write") - def write(self, path_u, contents): - fname = path_u - if not os.path.exists(fname): - self._maybe_notify(fname, self._inotify.IN_CREATE) - - d = notify_when_pending(self._uploader, path_u) - - modified_mtime_barrier(FilePath(fname)) - with open(fname, "wb") as f: - f.write(contents) - - self._maybe_notify(fname, self._inotify.IN_CLOSE_WRITE) - return d.addTimeout(self._timeout, reactor) - - @log_call_deferred(action_type=u"fileops:mkdir") - def mkdir(self, path_u): - fname = path_u - d = self._uploader.set_hook('inotify') - os.mkdir(fname) - self._maybe_notify(fname, self._inotify.IN_CREATE | self._inotify.IN_ISDIR) - return d.addTimeout(self._timeout, reactor) - - @log_call_deferred(action_type=u"fileops:delete") - def delete(self, path_u): - fname = path_u - d = self._uploader.set_hook('inotify') - if os.path.isdir(fname): - remove = os.rmdir - else: - remove = os.unlink - remove(fname) - - self._maybe_notify(fname, self._inotify.IN_DELETE) - return d.addTimeout(self._timeout, reactor) - - def _maybe_notify(self, fname, mask): - if self._fake_inotify: - self._uploader._notifier.event(to_filepath(fname), mask) - - -class CheckerMixin(object): - """ - Factored out of one of the many test classes. - - *Ideally* these should just be bare helper methods, but many of - them already depended upon self.* state. One major problem is that - they're using self.magicfolder *but* some of the alice/bob tests - use this, too, and they just do "self.magicfolder = - self.bob_magicfolder" or whatever before calling them, which is - *horrible*. - """ - def _check_mkdir(self, name_u): - return self._check_file(name_u + u"/", "", directory=True) - - @defer.inlineCallbacks - def _check_file(self, name_u, data, temporary=False, directory=False): - precondition(not (temporary and directory), temporary=temporary, directory=directory) - - # print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory) - previously_uploaded = self._get_count('uploader.objects_succeeded') - previously_disappeared = self._get_count('uploader.objects_disappeared') - - path_u = abspath_expanduser_unicode(name_u, base=self.local_dir) - - if directory: - yield self.fileops.mkdir(path_u) - else: - # We don't use FilePath.setContent() here because it creates a temporary file that - # is renamed into place, which causes events that the test is not expecting. - yield self.fileops.write(path_u, data) - yield iterate(self.magicfolder) - if temporary: - yield iterate(self.magicfolder) - yield self.fileops.delete(path_u) - - yield iterate(self.magicfolder) - encoded_name_u = magicpath.path2magic(name_u) - - yield self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0) - if temporary: - yield self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'), - previously_disappeared + 1) - else: - yield self.magicfolder.uploader._upload_dirnode.list() - x = yield self.magicfolder.uploader._upload_dirnode.get(encoded_name_u) - actual_data = yield download_to_data(x) - self.failUnlessReallyEqual(actual_data, data) - self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), - previously_uploaded + 1) - - self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0) - - @defer.inlineCallbacks - def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version): - encoded_name_u = magicpath.path2magic(relpath_u) - result = yield magicfolder.downloader._get_collective_latest_file(encoded_name_u) - self.assertIsNot( - result, - None, - "collective_latest_file({}) is None".format(encoded_name_u), - ) - node, metadata = result - self.assertIsNot( - metadata, - None, - "collective_latest_file({}) metadata is None".format(encoded_name_u), - ) - self.failUnlessEqual(metadata['version'], expected_version) - - def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version): - db_entry = magicfolder._db.get_db_entry(relpath_u) - if db_entry is not None: - #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version) - self.failUnlessEqual(db_entry.version, expected_version) - - def _check_file_gone(self, magicfolder, relpath_u): - path = os.path.join(magicfolder.uploader._local_path_u, relpath_u) - self.assertTrue(not os.path.exists(path)) - - def _check_uploader_count(self, name, expected, magic=None): - if magic is None: - magic = self.alice_magicfolder - self.failUnlessReallyEqual( - self._get_count( - 'uploader.'+name, - client=magic._client, - ), - expected, - "Pending: {}\n" - "Deque: {}\n".format(magic.uploader._pending, magic.uploader._deque), - ) - - def _check_downloader_count(self, name, expected, magic=None): - self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client), - expected) - - def _get_count(self, name, client=None): - counters = (client or self.get_client()).stats_provider.get_stats()["counters"] - return counters.get('magic_folder.%s' % (name,), 0) - - - -class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, CheckerMixin): - inject_inotify = False - - def setUp(self): - MagicFolderCLITestMixin.setUp(self) - temp = self.mktemp() - self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding())) - # set_up_grid depends on self.basedir existing - with start_action(action_type=u"set_up_grid"): - self.set_up_grid(num_clients=2, oneshare=True) - - self.alice_clock = task.Clock() - self.bob_clock = task.Clock() - - # this is all just .setup_alice_and_bob(), essentially - self.alice_magicfolder = None - self.bob_magicfolder = None - - self.alice_magic_dir = abspath_expanduser_unicode(u"Alice-magic", base=self.basedir) - self.mkdir_nonascii(self.alice_magic_dir) - self.bob_magic_dir = abspath_expanduser_unicode(u"Bob-magic", base=self.basedir) - self.mkdir_nonascii(self.bob_magic_dir) - - # Alice creates a Magic Folder, invites herself and joins. - d = DeferredContext(self.do_create_magic_folder(0)) - d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname)) - def get_invite_code(result): - self.invite_code = result[1].strip() - d.addCallback(get_invite_code) - d.addCallback(lambda ign: self.do_join(0, self.alice_magic_dir, self.invite_code)) - def get_alice_caps(ign): - self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0) - d.addCallback(get_alice_caps) - d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap)) - d.addCallback(lambda ign: self.check_config(0, self.alice_magic_dir)) - def get_Alice_magicfolder(result): - self.alice_magicfolder = self.init_magicfolder(0, self.alice_upload_dircap, - self.alice_collective_dircap, - self.alice_magic_dir, self.alice_clock) - self.alice_fileops = FileOperationsHelper(self.alice_magicfolder.uploader, self.inject_inotify) - d0 = self.alice_magicfolder.uploader.set_hook('iteration') - d1 = self.alice_magicfolder.downloader.set_hook('iteration') - self.alice_clock.advance(self.alice_magicfolder.uploader._pending_delay + 1) - d0.addCallback(lambda ign: d1) - d0.addCallback(lambda ign: result) - return d0 - d.addCallback(get_Alice_magicfolder) - - # Alice invites Bob. Bob joins. - d.addCallback(lambda ign: self.do_invite(0, self.bob_nickname)) - def get_invite_code(result): - self.invite_code = result[1].strip() - d.addCallback(get_invite_code) - d.addCallback(lambda ign: self.do_join(1, self.bob_magic_dir, self.invite_code)) - def get_bob_caps(ign): - self.bob_collective_dircap, self.bob_upload_dircap = self.get_caps_from_files(1) - d.addCallback(get_bob_caps) - d.addCallback(lambda ign: self.check_joined_config(1, self.bob_upload_dircap)) - d.addCallback(lambda ign: self.check_config(1, self.bob_magic_dir)) - def get_Bob_magicfolder(result): - self.bob_magicfolder = self.init_magicfolder(1, self.bob_upload_dircap, - self.bob_collective_dircap, - self.bob_magic_dir, self.bob_clock) - self.bob_fileops = FileOperationsHelper(self.bob_magicfolder.uploader, self.inject_inotify) - d0 = self.bob_magicfolder.uploader.set_hook('iteration') - d1 = self.bob_magicfolder.downloader.set_hook('iteration') - self.bob_clock.advance(self.alice_magicfolder.uploader._pending_delay + 1) - d0.addCallback(lambda ign: d1) - d0.addCallback(lambda ign: result) - return d0 - d.addCallback(get_Bob_magicfolder) - return d.result - - @defer.inlineCallbacks - def tearDown(self): - yield GridTestMixin.tearDown(self) - - for mf in [self.alice_magicfolder, self.bob_magicfolder]: - mf.uploader._clock.advance(mf.uploader._pending_delay + 1) - mf.downloader._clock.advance(mf.downloader._poll_interval + 1) - - @inline_callbacks - def test_alice_delete_bob_restore(self): - alice_fname = os.path.join(self.alice_magic_dir, 'blam') - bob_fname = os.path.join(self.bob_magic_dir, 'blam') - - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - - with start_action(action_type=u"alice:create"): - yield self.alice_fileops.write(alice_fname, 'contents0\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"alice:upload"): - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc - - with start_action(action_type=u"alice:check-upload"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) - - with start_action(action_type=u"bob:download"): - yield iterate_downloader(self.bob_magicfolder) - - with start_action(action_type=u"alice:recheck-upload"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) - - with start_action(action_type=u"bob:check-download"): - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0) - yield self.failUnlessReallyEqual( - self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client), - 0 - ) - yield self.failUnlessReallyEqual( - self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client), - 1 - ) - - yield iterate(self.bob_magicfolder) # for windows - - - bob_proc = self.bob_magicfolder.uploader.set_hook('processed') - alice_proc = self.alice_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"bob:delete"): - yield self.bob_fileops.delete(bob_fname) - yield iterate(self.bob_magicfolder) # for windows - - with start_action(action_type=u"bob:upload"): - yield iterate_uploader(self.bob_magicfolder) - yield bob_proc - - with start_action(action_type=u"alice:download"): - yield iterate_downloader(self.alice_magicfolder) - yield alice_proc - - # check versions - with start_action(action_type=u"bob:check-upload"): - node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam') - self.assertTrue(metadata['deleted']) - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1) - - with start_action(action_type=u"alice:check-download"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1) - - with start_action(action_type=u"alice:mysterious-iterate"): - # not *entirely* sure why we need to iterate Alice for the - # real test here. But, we do. - yield iterate(self.alice_magicfolder) - - # now alice restores it (alice should upload, bob download) - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"alice:rewrite"): - yield self.alice_fileops.write(alice_fname, 'new contents\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"alice:reupload"): - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc - - with start_action(action_type=u"bob:redownload"): - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc - - # check versions - with start_action(action_type=u"bob:recheck-download"): - node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam') - self.assertTrue('deleted' not in metadata or not metadata['deleted']) - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2) - - with start_action(action_type=u"alice:final-check-upload"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2) - - @inline_callbacks - def test_alice_sees_bobs_delete_with_error(self): - # alice creates a file, bob deletes it -- and we also arrange - # for Alice's file to have "gone missing" as well. - alice_fname = os.path.join(self.alice_magic_dir, 'blam') - bob_fname = os.path.join(self.bob_magic_dir, 'blam') - - # alice creates a file, bob downloads it - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"alice:create"): - yield self.alice_fileops.write(alice_fname, 'contents0\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"alice:upload"): - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc # alice uploads - - with start_action(action_type=u"bob:download"): - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc # bob downloads - - with start_action(action_type=u"mysterious:iterate"): - yield iterate(self.alice_magicfolder) # for windows - yield iterate(self.bob_magicfolder) # for windows - - # check the state (XXX I had to switch the versions to 0; is that really right? why?) - with start_action(action_type=u"alice:check"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) - - with start_action(action_type=u"bob:check"): - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0) - self.failUnlessReallyEqual( - self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client), - 0 - ) - self.failUnlessReallyEqual( - self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client), - 1 - ) - - bob_proc = self.bob_magicfolder.uploader.set_hook('processed') - alice_proc = self.alice_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"bob:delete"): - yield self.bob_fileops.delete(bob_fname) - - with start_action(action_type=u"alice:delete"): - # just after notifying bob, we also delete alice's, - # covering the 'except' flow in _rename_deleted_file() - yield self.alice_fileops.delete(alice_fname) - - with start_action(action_type=u"bob:upload-delete"): - yield iterate_uploader(self.bob_magicfolder) - yield bob_proc - - with start_action(action_type=u"alice:download-delete"): - yield iterate_downloader(self.alice_magicfolder) - yield alice_proc - - # check versions - with start_action(action_type=u"bob:check"): - node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam') - self.assertTrue(metadata['deleted']) - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1) - - with start_action(action_type=u"alice:check"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1) - - @inline_callbacks - def test_alice_create_bob_update(self): - alice_fname = os.path.join(self.alice_magic_dir, 'blam') - bob_fname = os.path.join(self.bob_magic_dir, 'blam') - - # alice creates a file, bob downloads it - yield self.alice_fileops.write(alice_fname, 'contents0\n') - - yield iterate(self.alice_magicfolder) - yield iterate(self.alice_magicfolder) - yield iterate(self.bob_magicfolder) - - # check the state (XXX ditto, had to switch to veresion 0; right?) - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) - self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0) - self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0) - self.failUnlessReallyEqual( - self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client), - 0 - ) - self.failUnlessReallyEqual( - self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client), - 1 - ) - - yield iterate(self.bob_magicfolder) - # now bob updates it (bob should upload, alice download) - yield self.bob_fileops.write(bob_fname, 'bob wuz here\n') - - yield iterate(self.bob_magicfolder) - yield iterate(self.bob_magicfolder) - yield iterate(self.alice_magicfolder) - - # check the state - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1) - self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1) - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1) - self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1) - - @inline_callbacks - def test_download_retry(self): - alice_fname = os.path.join(self.alice_magic_dir, 'blam') - # bob_fname = os.path.join(self.bob_magic_dir, 'blam') - - # Alice creates a file - yield self.alice_fileops.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)])) - yield iterate(self.alice_magicfolder) - # check alice created the file - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) - self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) - - # now, we ONLY want to do the scan, not a full iteration of - # the process loop. So we do just the scan part "by hand" in - # Bob's downloader - with start_action(action_type=u"test:perform-scan"): - yield self.bob_magicfolder.downloader._perform_scan() - # while we're delving into internals, I guess we might as well - # confirm that we did queue up an item to download - self.assertEqual(1, len(self.bob_magicfolder.downloader._deque)) - - # break all the servers so the download fails. count=1 because we - # only want the download attempted by _process_deque to fail. After - # that, we want it to work again. - for server_id in self.g.get_all_serverids(): - self.g.break_server(server_id, count=1) - - # now let bob try to do the download. Reach in and call - # _process_deque directly because we are already half-way through a - # logical iteration thanks to the _perform_scan call above. - with start_action(action_type=u"test:process-deque"): - yield self.bob_magicfolder.downloader._process_deque() - - self.eliot_logger.flushTracebacks(UnrecoverableFileError) - logged = self.eliot_logger.flushTracebacks(NoSharesError) - self.assertEqual( - 1, - len(logged), - "Got other than expected single NoSharesError: {}".format(logged), - ) - - # ...however Bob shouldn't have downloaded anything - self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0) - # bob should *not* have downloaded anything, as we failed all the servers - self.failUnlessReallyEqual( - self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client), - 0 - ) - self.failUnlessReallyEqual( - self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client), - 1 - ) - - with start_action(action_type=u"test:iterate"): - # now we let Bob try again - yield iterate(self.bob_magicfolder) - - # ...and he should have succeeded - self.failUnlessReallyEqual( - self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client), - 1 - ) - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0) - - @inline_callbacks - def test_conflict_local_change_fresh(self): - alice_fname = os.path.join(self.alice_magic_dir, 'localchange0') - bob_fname = os.path.join(self.bob_magic_dir, 'localchange0') - - # alice creates a file, bob downloads it - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - yield self.alice_fileops.write(alice_fname, 'contents0\n') - yield iterate(self.alice_magicfolder) # for windows - - # before bob downloads, we make a local file for bob by the - # same name - with open(bob_fname, 'w') as f: - f.write("not the right stuff") - - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc # alice uploads - - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc # bob downloads - - # ...so now bob should produce a conflict - self.assertTrue(os.path.exists(bob_fname + '.conflict')) - - @inline_callbacks - def test_conflict_local_change_existing(self): - alice_fname = os.path.join(self.alice_magic_dir, 'localchange1') - bob_fname = os.path.join(self.bob_magic_dir, 'localchange1') - - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"alice:create"): - yield self.alice_fileops.write(alice_fname, 'contents0\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"alice:upload"): - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc # alice uploads - self.assertEqual( - 1, - self._get_count( - 'uploader.files_uploaded', - client=self.alice_magicfolder._client, - ), - ) - - with start_action(action_type=u"bob:download"): - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc # bob downloads - self.assertEqual( - 1, - self._get_count( - 'downloader.objects_downloaded', - client=self.bob_magicfolder._client, - ), - ) - - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"alice:rewrite"): - yield self.alice_fileops.write(alice_fname, 'contents1\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"bob:rewrite"): - # before bob downloads, make a local change - with open(bob_fname, "w") as f: - f.write("bob's local change") - - with start_action(action_type=u"alice:reupload"): - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc # alice uploads - self.assertEqual( - 2, - self._get_count( - 'uploader.files_uploaded', - client=self.alice_magicfolder._client, - ), - ) - - with start_action(action_type=u"bob:redownload-and-conflict"): - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc # bob downloads - - self.assertEqual( - 2, - self._get_count( - 'downloader.objects_downloaded', - client=self.bob_magicfolder._client, - ), - ) - self.assertEqual( - 1, - self._get_count( - 'downloader.objects_conflicted', - client=self.bob_magicfolder._client, - ), - ) - - # ...so now bob should produce a conflict - self.assertTrue(os.path.exists(bob_fname + '.conflict')) - - @inline_callbacks - def test_alice_delete_and_restore(self): - alice_fname = os.path.join(self.alice_magic_dir, 'blam') - bob_fname = os.path.join(self.bob_magic_dir, 'blam') - - # alice creates a file, bob downloads it - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"alice:create"): - yield self.alice_fileops.write(alice_fname, 'contents0\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"alice:upload"): - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc # alice uploads - - with start_action(action_type=u"bob:download"): - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc # bob downloads - - with start_action(action_type=u"alice:check"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) - - with start_action(action_type=u"bob:check"): - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0) - yield self.failUnlessReallyEqual( - self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client), - 0 - ) - yield self.failUnlessReallyEqual( - self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client), - 1 - ) - self.failUnless(os.path.exists(bob_fname)) - self.failUnless(not os.path.exists(bob_fname + '.backup')) - self.failUnless(not os.path.exists(bob_fname + '.conflict')) - - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - - with start_action(action_type=u"alice:delete"): - yield self.alice_fileops.delete(alice_fname) - yield iterate_uploader(self.alice_magicfolder) - yield alice_proc - - with start_action(action_type=u"bob:redownload"): - yield iterate_downloader(self.bob_magicfolder) - yield bob_proc - - with start_action(action_type=u"bob:recheck"): - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1) - self.assertFalse(os.path.exists(bob_fname)) - self.assertTrue(os.path.exists(bob_fname + '.backup')) - self.assertFalse(os.path.exists(bob_fname + '.conflict')) - - with start_action(action_type=u"alice:recheck"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1) - - with start_action(action_type=u"alice:restore"): - os.unlink(bob_fname + '.backup') - alice_proc = self.alice_magicfolder.uploader.set_hook('processed') - bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - yield self.alice_fileops.write(alice_fname, 'alice wuz here\n') - yield iterate(self.alice_magicfolder) # for windows - - with start_action(action_type=u"alice:reupload"): - yield iterate_uploader(self.alice_magicfolder) - yield iterate_downloader(self.alice_magicfolder) # why? - yield alice_proc - - with start_action(action_type=u"bob:final-redownload"): - yield iterate_downloader(self.bob_magicfolder) - yield iterate_uploader(self.bob_magicfolder) - yield bob_proc - - with start_action(action_type=u"bob:final-check"): - yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2) - yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2) - self.failUnless(os.path.exists(bob_fname)) - - with start_action(action_type=u"alice:final-check"): - yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2) - yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2) - - # XXX this should be shortened -- as in, any cases not covered by - # the other tests in here should get their own minimal test-case. - @skipIf(sys.platform == "win32", "Still inotify problems on Windows (FIXME)") - def test_alice_bob(self): - d = DeferredContext(defer.succeed(None)) - - # XXX FIXME just quickly porting this test via aliases -- the - # "real" solution is to break out any relevant test-cases as - # their own (smaller!) tests. - alice_clock = self.alice_magicfolder.uploader._clock - bob_clock = self.bob_magicfolder.uploader._clock - - def _wait_for_Alice(ign, downloaded_d): - if _debug: print("Now waiting for Alice to download\n") - alice_clock.advance(4) - return downloaded_d - - def _wait_for_Bob(ign, downloaded_d): - if _debug: print("Now waiting for Bob to download\n") - bob_clock.advance(4) - return downloaded_d - - def _wait_for(ign, something_to_do, alice=True): - if alice: - downloaded_d = self.bob_magicfolder.downloader.set_hook('processed') - uploaded_d = self.alice_magicfolder.uploader.set_hook('processed') - else: - downloaded_d = self.alice_magicfolder.downloader.set_hook('processed') - uploaded_d = self.bob_magicfolder.uploader.set_hook('processed') - - d = something_to_do() - - def advance(ign): - if alice: - if _debug: print("Waiting for Alice to upload 3\n") - alice_clock.advance(4) - uploaded_d.addCallback(_wait_for_Bob, downloaded_d) - else: - if _debug: print("Waiting for Bob to upload\n") - bob_clock.advance(4) - uploaded_d.addCallback(_wait_for_Alice, downloaded_d) - return uploaded_d - d.addCallback(advance) - return d - - @inline_callbacks - def Alice_to_write_a_file(): - if _debug: print("Alice writes a file\n\n\n\n\n") - self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u) - yield self.alice_fileops.write(self.file_path, "meow, meow meow. meow? meow meow! meow.") - yield iterate(self.alice_magicfolder) - d.addCallback(_wait_for, Alice_to_write_a_file) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0) - self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0) - self._check_uploader_count('objects_failed', 0) - self._check_uploader_count('objects_succeeded', 1) - self._check_uploader_count('files_uploaded', 1) - self._check_uploader_count('objects_queued', 0) - self._check_uploader_count('directories_created', 0) - self._check_uploader_count('objects_conflicted', 0) - self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder) - - self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0) - self._check_downloader_count('objects_failed', 0) - self._check_downloader_count('objects_downloaded', 1) - self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder) - self._check_downloader_count('objects_downloaded', 1, magic=self.bob_magicfolder) - d.addCallback(check_state) - - @inline_callbacks - def Alice_to_delete_file(): - if _debug: print("Alice deletes the file!\n\n\n\n") - yield self.alice_fileops.delete(self.file_path) - yield iterate(self.alice_magicfolder) - yield iterate(self.bob_magicfolder) - d.addCallback(_wait_for, Alice_to_delete_file) - - @inline_callbacks - def notify_bob_moved(ign): - # WARNING: this is just directly notifying for the mock - # tests, because in the Real* tests the .backup file will - # me moved into place (from the original) - p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u) - if self.bob_fileops._fake_inotify: - self.bob_magicfolder.uploader._notifier.event(to_filepath(p + u'.backup'), fake_inotify.IN_MOVED_TO) - yield iterate(self.bob_magicfolder) - d.addCallback(notify_bob_moved) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1) - self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1) - self._check_uploader_count('objects_failed', 0) - self._check_uploader_count('objects_succeeded', 2) - self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder) - - self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1) - self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1) - self._check_file_gone(self.bob_magicfolder, u"file1") - self._check_downloader_count('objects_failed', 0) - self._check_downloader_count('objects_downloaded', 2) - self._check_downloader_count('objects_downloaded', 2, magic=self.bob_magicfolder) - d.addCallback(check_state) - - @inline_callbacks - def Alice_to_rewrite_file(): - if _debug: print("Alice rewrites file\n") - self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u) - yield self.alice_fileops.write( - self.file_path, - "Alice suddenly sees the white rabbit running into the forest.", - ) - yield iterate(self.alice_magicfolder) - d.addCallback(_wait_for, Alice_to_rewrite_file) - d.addCallback(lambda ign: iterate(self.bob_magicfolder)) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2) - self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2) - self._check_uploader_count('objects_failed', 0) - self._check_uploader_count('objects_succeeded', 3) - self._check_uploader_count('files_uploaded', 3) - self._check_uploader_count('objects_queued', 0) - self._check_uploader_count('directories_created', 0) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 0) - - self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2) - self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2) - self._check_downloader_count('objects_failed', 0) - self._check_downloader_count('objects_downloaded', 3) - self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder) - d.addCallback(check_state) - - path_u = u"/tmp/magic_folder_test" - encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test") - - def Alice_tries_to_p0wn_Bob(ign): - if _debug: print("Alice tries to p0wn Bob\n") - iter_d = iterate(self.bob_magicfolder) - processed_d = self.bob_magicfolder.downloader.set_hook('processed') - - # upload a file that would provoke the security bug from #2506 - uploadable = Data("", self.alice_magicfolder._client.convergence) - alice_dmd = self.alice_magicfolder.uploader._upload_dirnode - - d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True) - d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u))) - d2.addCallback(lambda ign: iter_d) - d2.addCallback(_wait_for_Bob, processed_d) - return d2 - d.addCallback(Alice_tries_to_p0wn_Bob) - - @log_call(action_type=u"check_state", include_args=[], include_result=False) - def check_state(ignored): - self.failIf(os.path.exists(path_u)) - self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None) - self._check_downloader_count('objects_downloaded', 3) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 0) - d.addCallback(check_state) - - @inline_callbacks - def Bob_to_rewrite_file(): - if _debug: print("Bob rewrites file\n") - self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u) - if _debug: print("---- bob's file is %r" % (self.file_path,)) - yield self.bob_fileops.write(self.file_path, "No white rabbit to be found.") - yield iterate(self.bob_magicfolder) - d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False)) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3) - self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3) - self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder) - self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder) - self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder) - self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder) - self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.bob_magicfolder) - - self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3) - self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - d.addCallback(check_state) - - def Alice_conflicts_with_Bobs_last_downloaded_uri(): - if _debug: print("Alice conflicts with Bob\n") - downloaded_d = self.bob_magicfolder.downloader.set_hook('processed') - uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence) - alice_dmd = self.alice_magicfolder.uploader._upload_dirnode - d2 = alice_dmd.add_file(u"file1", uploadable, - metadata={"version": 5, - "last_downloaded_uri" : "URI:LIT:" }, - overwrite=True) - if _debug: print("Waiting for Alice to upload\n") - d2.addCallback(lambda ign: bob_clock.advance(6)) - d2.addCallback(lambda ign: downloaded_d) - d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u))) - return d2 - d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri()) - - @log_call(action_type=u"check_state", include_args=[], include_result=False) - def check_state(ignored): - self._check_downloader_count('objects_downloaded', 4, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 1, magic=self.bob_magicfolder) - self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder) - self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder) - d.addCallback(check_state) - - # prepare to perform another conflict test - @log_call_deferred(action_type=u"alice:to-write:file2") - @inline_callbacks - def Alice_to_write_file2(): - if _debug: print("Alice writes a file2\n") - self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u) - d = self.alice_fileops.write(self.file_path, "something") - self.bob_clock.advance(4) - yield d - d.addCallback(_wait_for, Alice_to_write_file2) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0) - self._check_version_in_local_db(self.alice_magicfolder, u"file2", 0) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder) - d.addCallback(check_state) - - def advance(ign): - alice_clock.advance(4) - bob_clock.advance(4) - # we need to pause here, or make "is_new_file()" more - # robust, because this is now fast enough that the mtime - # of the allegedly-new file matches, so Bob decides not to - # upload (and the test hangs). Not sure why it worked - # before; must have been *just* slow enough? - # XXX FIXME for the new real-test had to jack this to 0.5; - # related to the 0.1 notify pause?? - return task.deferLater(reactor, 0.5, lambda: None) - d.addCallback(advance) - d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file2", 0)) - - @inline_callbacks - def Bob_to_rewrite_file2(): - if _debug: print("Bob rewrites file2\n") - self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u) - if _debug: print("---- bob's file is %r" % (self.file_path,)) - yield iterate(self.bob_magicfolder) - yield self.bob_fileops.write(self.file_path, "roger roger. what vector?") - if _debug: print("---- bob rewrote file2") - yield iterate(self.bob_magicfolder) - if _debug: print("---- iterated bob's magicfolder") - d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False)) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1) - self._check_downloader_count('objects_downloaded', 5, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 1, magic=self.bob_magicfolder) - self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder) - self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder) - self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder) - self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder) - self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder) - d.addCallback(check_state) - - # XXX here we advance the clock and then test again to make sure no values are monotonically increasing - # with each queue turn ;-p - alice_clock.advance(6) - bob_clock.advance(6) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1) - self._check_downloader_count('objects_downloaded', 5) - self._check_downloader_count('objects_conflicted', 1) - self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder) - self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder) - self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder) - self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder) - self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder) - d.addCallback(check_state) - - def Alice_conflicts_with_Bobs_last_uploaded_uri(): - if _debug: print("Alice conflicts with Bob\n") - encoded_path_u = magicpath.path2magic(u"file2") - downloaded_d = self.bob_magicfolder.downloader.set_hook('processed') - uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence) - alice_dmd = self.alice_magicfolder.uploader._upload_dirnode - d2 = alice_dmd.add_file(u"file2", uploadable, - metadata={"version": 5, - "last_uploaded_uri" : "URI:LIT:" }, - overwrite=True) - if _debug: print("Waiting for Alice to upload\n") - d2.addCallback(lambda ign: bob_clock.advance(6)) - d2.addCallback(lambda ign: downloaded_d) - d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u))) - return d2 - d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri()) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5) - self._check_downloader_count('objects_downloaded', 6) - self._check_downloader_count('objects_conflicted', 1) - self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder) - self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder) - self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder) - self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder) - d.addCallback(check_state) - - def foo(ign): - alice_clock.advance(6) - bob_clock.advance(6) - alice_clock.advance(6) - bob_clock.advance(6) - d.addCallback(foo) - - @log_call(action_type=u"check_state", include_args=[], include_result=False) - def check_state(ignored): - self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 1) - self._check_downloader_count('objects_downloaded', 6) - d.addCallback(check_state) - - # prepare to perform another conflict test - @inline_callbacks - def Alice_to_write_file3(): - if _debug: print("Alice writes a file\n") - self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u) - yield self.alice_fileops.write(self.file_path, "something") - yield iterate(self.alice_magicfolder) - # Make sure Bob gets the file before we do anything else. - yield iterate(self.bob_magicfolder) - d.addCallback(_wait_for, Alice_to_write_file3) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_downloaded', 7) - self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder) - self._check_downloader_count('objects_conflicted', 1) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - d.addCallback(check_state) - - @inline_callbacks - def Bob_to_rewrite_file3(): - if _debug: print("Bob rewrites file3\n") - self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u) - if _debug: print("---- bob's file is %r" % (self.file_path,)) - yield iterate(self.bob_magicfolder) - yield self.bob_fileops.write(self.file_path, "roger roger") - yield iterate(self.bob_magicfolder) - d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False)) - - @log_call_deferred(action_type=u"check_state") - @inline_callbacks - def check_state(ignored): - yield self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1) - self._check_downloader_count('objects_downloaded', 7) - self._check_downloader_count('objects_conflicted', 1) - self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder) - self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder) - self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder) - self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder) - self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder) - self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder) - self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder) - d.addCallback(check_state) - - return d.addActionFinish() - - -class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, CheckerMixin): - """ - These tests will be run both with a mock notifier, and (on platforms that support it) - with the real INotify. - """ - - def setUp(self): - self.assertIs(None, super(SingleMagicFolderTestMixin, self).setUp()) - temp = self.mktemp() - self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding())) - self.magicfolder = None - self.set_up_grid(oneshare=True) - self.local_dir = os.path.join(self.basedir, u"local_dir") - self.mkdir_nonascii(self.local_dir) - - # Magic-folder implementation somehow manages to leave a DelayedCall - # in the reactor from the eventual queue by the end of the test. It - # may have something to do with the upload process but it's not - # entirely clear. It's difficult to track things through the eventual - # queue. It is almost certainly the case that some other Deferred - # involved in magic-folder that is already being waited on elsewhere - # *should* encompass this DelayedCall but I wasn't able to figure out - # where that association needs to be made. So, as a work-around, - # explicitly flush the eventual queue at the end of the test, too. - from foolscap.eventual import flushEventualQueue - self.addCleanup(flushEventualQueue) - - # Sometimes a collective scan fails with UnrecoverableFileError. It's - # not clear to me why. :/ This fixes the issue, though, and all other - # asserted-about behavior is provided whether this case is hit or not. - self.addCleanup( - lambda: self.eliot_logger.flushTracebacks(UnrecoverableFileError) - ) - - d = DeferredContext(self.create_invite_join_magic_folder(self.alice_nickname, self.local_dir)) - d.addCallback(self._restart_client) - # note: _restart_client ultimately sets self.magicfolder to not-None - return d.result - - def tearDown(self): - d = DeferredContext(super(SingleMagicFolderTestMixin, self).tearDown()) - d.addCallback(self.cleanup) - return d.result - - def _createdb(self): - dbfile = abspath_expanduser_unicode(u"magicfolder_default.sqlite", base=self.basedir) - mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1)) - self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,)) - self.failUnlessEqual(mdb.VERSION, 1) - return mdb - - @log_call_deferred(action_type=u"restart-client") - def _restart_client(self, ign): - #print "_restart_client" - d = DeferredContext(self.restart_client()) - d.addCallback(self._wait_until_started) - return d.result - - @log_call_deferred(action_type=u"wait-until-started") - def _wait_until_started(self, ign): - #print "_wait_until_started" - self.magicfolder = self.get_client().getServiceNamed('magic-folder-default') - self.fileops = FileOperationsHelper(self.magicfolder.uploader, self.inject_inotify) - self.up_clock = task.Clock() - self.down_clock = task.Clock() - self.magicfolder.uploader._clock = self.up_clock - self.magicfolder.downloader._clock = self.down_clock - - # XXX should probably be passing the reactor to instances when - # they're created, but that's a ton of re-factoring, so we - # side-step that issue by hacking it in here. However, we - # *have* to "hack it in" before we call ready() so that the - # first iteration of the loop doesn't call the "real" - # reactor's callLater. :( - return self.magicfolder.ready() - - def test_db_basic(self): - fileutil.make_dirs(self.basedir) - self._createdb() - - @inline_callbacks - def test_scan_once_on_startup(self): - # What is this test? Maybe it is just a stub and needs finishing. - self.magicfolder.uploader._clock.advance(99) - - yield self._check_uploader_count('files_uploaded', 0, magic=self.magicfolder) - yield self._check_uploader_count('objects_queued', 0, magic=self.magicfolder) - yield self._check_downloader_count('objects_conflicted', 0, magic=self.magicfolder) - yield self._check_uploader_count('objects_succeeded', 0, magic=self.magicfolder) - yield self._check_downloader_count('objects_failed', 0, magic=self.magicfolder) - yield self._check_downloader_count('objects_downloaded', 0, magic=self.magicfolder) - - def test_db_persistence(self): - """Test that a file upload creates an entry in the database.""" - - fileutil.make_dirs(self.basedir) - db = self._createdb() - - relpath1 = u"myFile1" - pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False, - exists=True, size=1, mtime_ns=123, ctime_ns=456) - db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo) - - c = db.cursor - c.execute("SELECT size, mtime_ns, ctime_ns" - " FROM local_files" - " WHERE path=?", - (relpath1,)) - row = c.fetchone() - self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns)) - - # Second test uses magic_folder.is_new_file instead of SQL query directly - # to confirm the previous upload entry in the db. - relpath2 = u"myFile2" - path2 = os.path.join(self.basedir, relpath2) - fileutil.write(path2, "meow\n") - pathinfo = fileutil.get_pathinfo(path2) - db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo) - db_entry = db.get_db_entry(relpath2) - self.assertFalse(magic_folder.is_new_file(pathinfo, db_entry)) - - different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False, - exists=True, size=0, mtime_ns=pathinfo.mtime_ns, - ctime_ns=pathinfo.ctime_ns) - self.assertTrue(magic_folder.is_new_file(different_pathinfo, db_entry)) - - def _test_magicfolder_start_service(self): - # what is this even testing? - d = defer.succeed(None) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0)) - - d.addCallback(lambda ign: self.create_invite_join_magic_folder(self.alice_nickname, self.local_dir)) - d.addCallback(self._restart_client) - - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1)) - d.addBoth(self.cleanup) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0)) - return d - - @skipIf(sys.platform == "linux2", "fails on certain linux flavors: see ticket #2834") - def test_move_tree(self): - """ - create an empty directory tree and 'mv' it into the magic folder, - noting the new directory and uploading it. - - also creates a directory tree with one file in it and 'mv's it - into the magic folder, so we upload the file and record the - directory. (XXX split to separate test) - """ - empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree") - empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir) - new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir) - - small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree") - small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir) - new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir) - - d = DeferredContext(defer.succeed(None)) - - @inline_callbacks - def _check_move_empty_tree(res): - self.mkdir_nonascii(empty_tree_dir) - yield self.fileops.move(empty_tree_dir, new_empty_tree_dir) - yield iterate(self.magicfolder) - - d.addCallback(_check_move_empty_tree) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1)) - - @inline_callbacks - def _check_move_small_tree(res): - self.mkdir_nonascii(small_tree_dir) - what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir) - fileutil.write(what_path, "say when") - yield self.fileops.move(small_tree_dir, new_small_tree_dir) - upstatus = list(self.magicfolder.uploader.get_status()) - downstatus = list(self.magicfolder.downloader.get_status()) - - self.assertEqual(2, len(upstatus)) - self.assertEqual(0, len(downstatus)) - yield iterate(self.magicfolder) - - # when we add the dir, we queue a scan of it; so we want - # the upload to "go" as well requiring 1 more iteration - yield iterate(self.magicfolder) - - d.addCallback(_check_move_small_tree) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2)) - - @inline_callbacks - def _check_moved_tree_is_watched(res): - another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir) - yield self.fileops.write(another_path, "file") - yield iterate(self.magicfolder) - yield iterate(self.magicfolder) # windows; why? - - d.addCallback(_check_moved_tree_is_watched) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2)) - - return d.result - - def test_persistence(self): - """ - Perform an upload of a given file and then stop the client. - Start a new client and magic-folder service... and verify that the file is NOT uploaded - a second time. This test is meant to test the database persistence along with - the startup and shutdown code paths of the magic-folder service. - """ - self.collective_dircap = "" # XXX hmmm? - - d = DeferredContext(defer.succeed(None)) - - @inline_callbacks - def create_test_file(filename): - test_file = abspath_expanduser_unicode(filename, base=self.local_dir) - yield self.fileops.write(test_file, "meow %s" % filename) - yield iterate(self.magicfolder) - yield iterate(self.magicfolder) # windows; why? - - d.addCallback(lambda ign: create_test_file(u"what1")) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0)) - d.addCallback(self.cleanup) - - d.addCallback(self._restart_client) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0)) - d.addCallback(lambda ign: create_test_file(u"what2")) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0)) - return d.result - - # all this "self.*" state via 9000 mix-ins is really really - # hard to read, keep track of, etc. Very hard to understand - # what each test uses for setup, etc. :( - - @inline_callbacks - def test_delete(self): - # setup: create a file 'foo' - path = os.path.join(self.local_dir, u'foo') - yield self.fileops.write(path, 'foo\n') - yield iterate_uploader(self.magicfolder) - yield iterate_uploader(self.magicfolder) # req'd for windows; not sure why? - self.assertTrue(os.path.exists(path)) - node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo') - self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,)) - - # the test: delete the file (and do fake notifies) - yield self.fileops.delete(path) - - yield iterate_uploader(self.magicfolder) - self.assertFalse(os.path.exists(path)) - - yield iterate_downloader(self.magicfolder) - # ensure we still have a DB entry, and that the version is 1 - node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo') - self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,)) - self.failUnlessEqual(metadata['version'], 1) - - @inline_callbacks - def test_batched_process(self): - """ - status APIs correctly function when there are 2 items queued at - once for processing - """ - # setup: get at least two items into the deque - path0 = os.path.join(self.local_dir, u'foo') - yield self.fileops.write(path0, 'foo\n') - path1 = os.path.join(self.local_dir, u'bar') - yield self.fileops.write(path1, 'bar\n') - - # get the status before we've processed anything - upstatus0 = list(self.magicfolder.uploader.get_status()) - upstatus1 = [] - - def one_item(item): - # grab status after we've processed a single item - us = list(self.magicfolder.uploader.get_status()) - upstatus1.extend(us) - one_d = self.magicfolder.uploader.set_hook('item_processed') - # can't 'yield' here because the hook isn't called until - # inside iterate() - one_d.addCallbacks(one_item, self.fail) - - yield iterate_uploader(self.magicfolder) - yield iterate_uploader(self.magicfolder) # req'd for windows; not sure why? - - # no matter which part of the queue the items are in, we - # should see the same status from the outside - self.assertEqual(upstatus0, upstatus1) - - @inline_callbacks - def test_real_notify_failure(self): - """ - Simulate an exception from the _real_notify helper in - magic-folder's uploader, confirming error-handling works. - """ - - orig_notify = self.magicfolder.uploader._real_notify - - class BadStuff(Exception): - pass - - def bad_stuff(*args, **kw): - # call original method .. - orig_notify(*args, **kw) - # ..but then cause a special problem - raise BadStuff("the bad stuff") - - patch_notify = mock.patch.object( - self.magicfolder.uploader, - '_real_notify', - mock.Mock(side_effect=bad_stuff), - ) - with patch_notify: - path0 = os.path.join(self.local_dir, u'foo') - yield self.fileops.write(path0, 'foo\n') - # this actually triggers two notifies - - # do a reactor turn; this is necessary because our "bad_stuff" - # method calls the hook (so the above 'yield' resumes) right - # *before* it raises the exception; thus, we ensure all the - # pending callbacks including the exception are processed - # before we flush the errors. - yield task.deferLater(reactor, 0, lambda: None) - - errors = self.eliot_logger.flushTracebacks(BadStuff) - # it seems on Windows the "RealTest" variant only produces 1 - # notification for some reason.. - self.assertTrue(len(errors) >= 1) - - @inline_callbacks - def test_delete_and_restore(self): - # setup: create a file - path = os.path.join(self.local_dir, u'foo') - yield self.fileops.write(path, 'foo\n') - yield iterate_uploader(self.magicfolder) - yield iterate_uploader(self.magicfolder) # req'd for windows; why? - self.assertTrue(os.path.exists(path)) - - # ...and delete the file - yield self.fileops.delete(path) - yield iterate_uploader(self.magicfolder) - self.assertFalse(os.path.exists(path)) - - # ensure we still have a DB entry, and that the version is 1 - node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo') - self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,)) - self.failUnlessEqual(metadata['version'], 1) - - # restore the file, with different contents - path = os.path.join(self.local_dir, u'foo') - yield self.fileops.write(path, 'bar\n') - yield iterate_uploader(self.magicfolder) - - # ensure we still have a DB entry, and that the version is 2 - node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo') - self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,)) - self.failUnlessEqual(metadata['version'], 2) - - def test_write_short_file(self): - # Write something short enough for a LIT file. - return self._check_file(u"short", "test") - - def test_magic_folder(self): - d = DeferredContext(defer.succeed(None)) - # Write something short enough for a LIT file. - d.addCallback(lambda ign: self._check_file(u"short", "test")) - - # Write to the same file again with different data. - d.addCallback(lambda ign: self._check_file(u"short", "different")) - - # Test that temporary files are not uploaded. - d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True)) - - # Test creation of a subdirectory. - d.addCallback(lambda ign: self._check_mkdir(u"directory")) - - # Write something longer, and also try to test a Unicode name if the fs can represent it. - name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long") - d.addCallback(lambda ign: self._check_file(name_u, "test"*100)) - - # TODO: test that causes an upload failure. - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0)) - - return d.result - - @inline_callbacks - def _create_directory_with_file(self, relpath_u, content): - path_f = os.path.join(self.local_dir, relpath_u) - path_d = os.path.dirname(path_f) - # Create a new directory in the monitored directory. - yield self.fileops.mkdir(path_d) - # Give the system a chance to notice and process it. - yield iterate(self.magicfolder) - # Create a new file in that new directory. - yield self.fileops.write(path_f, content) - # Another opportunity to process. - yield iterate(self.magicfolder) - - @inline_callbacks - def test_create_file_in_sub_directory(self): - reldir_u = u'subdir' - # The OS and the DMD may have conflicting conventions for directory - # the separator. Construct a value for each. - dmd_relpath_u = u'/'.join((reldir_u, u'some-file')) - platform_relpath_u = join(reldir_u, u'some-file') - content = u'some great content' - yield self._create_directory_with_file( - platform_relpath_u, - content, - ) - # The new directory and file should have been noticed and uploaded. - downloader = self.magicfolder.downloader - encoded_dir_u = magicpath.path2magic(reldir_u + u"/") - encoded_path_u = magicpath.path2magic(dmd_relpath_u) - - with start_action(action_type=u"retrieve-metadata"): - dir_node, dir_meta = yield downloader._get_collective_latest_file( - encoded_dir_u, - ) - path_node, path_meta = yield downloader._get_collective_latest_file( - encoded_path_u, - ) - - self.expectThat(dir_node, Not(Is(None)), "dir node") - self.expectThat(dir_meta, ContainsDict({'version': Equals(0)}), "dir meta") - self.expectThat(path_node, Not(Is(None)), "path node") - self.expectThat(path_meta, ContainsDict({'version': Equals(0)}), "path meta") - - @inline_callbacks - def test_delete_file_in_sub_directory(self): - dmd_relpath_u = u'/'.join((u'subdir', u'some-file')) - platform_relpath_u = join(u'subdir', u'some-file') - content = u'some great content' - yield self._create_directory_with_file( - platform_relpath_u, - content, - ) - # Delete the file in the sub-directory. - yield self.fileops.delete(os.path.join(self.local_dir, platform_relpath_u)) - # Let the deletion be processed. - yield iterate(self.magicfolder) - # Verify the deletion was uploaded. - encoded_path_u = magicpath.path2magic(dmd_relpath_u) - downloader = self.magicfolder.downloader - node, metadata = yield downloader._get_collective_latest_file(encoded_path_u) - self.assertThat(node, Not(Is(None))) - self.assertThat(metadata['version'], Equals(1)) - self.assertThat(metadata['deleted'], Equals(True)) - - def test_delete_sub_directory_containing_file(self): - reldir_u = u'subdir' - relpath_u = os.path.join(reldir_u, u'some-file') - content = u'some great content' - yield self._create_directory_with_file( - relpath_u, - content, - ) - # Delete the sub-directory and the file in it. Don't wait in between - # because the case where all events are delivered before any - # processing happens is interesting. And don't use the fileops API to - # delete the contained file so that we don't necessarily generate a - # notification for that path at all. We require that the - # implementation behave correctly when receiving only the notification - # for the containing directory. - os.unlink(os.path.join(self.local_dir, relpath_u)) - yield self.fileops.delete(os.path.join(self.local_dir, reldir_u)) - - # Now allow processing. - yield iterate(self.magicfolder) - # Give it some extra time because of recursive directory processing. - yield iterate(self.magicfolder) - - # Deletion of both entities should have been uploaded. - downloader = self.magicfolder.downloader - encoded_dir_u = magicpath.path2magic(reldir_u + u"/") - encoded_path_u = magicpath.path2magic(relpath_u) - - dir_node, dir_meta = yield downloader._get_collective_latest_file(encoded_dir_u) - path_node, path_meta = yield downloader._get_collective_latest_file(encoded_path_u) - - self.expectThat(dir_node, Not(Is(None)), "dir node") - self.expectThat(dir_meta, ContainsDict({ - "version": Equals(1), - "deleted": Equals(True), - }), "dir meta") - - self.expectThat(path_node, Not(Is(None)), "path node") - self.expectThat(path_meta, ContainsDict({ - "version": Equals(1), - "deleted": Equals(True), - }), "path meta") - - -@skipIf(support_missing, support_message) -class MockTestAliceBob(MagicFolderAliceBobTestMixin, AsyncTestCase): - inject_inotify = True - - def setUp(self): - self.inotify = fake_inotify - self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify) - return super(MockTestAliceBob, self).setUp() - - -@skipIf(support_missing, support_message) -class MockTest(SingleMagicFolderTestMixin, AsyncTestCase): - """This can run on any platform, and even if twisted.internet.inotify can't be imported.""" - inject_inotify = True - - def setUp(self): - self.inotify = fake_inotify - self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify) - return super(MockTest, self).setUp() - - def test_errors(self): - self.set_up_grid(oneshare=True) - - errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir) - os.mkdir(errors_dir) - not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir) - fileutil.write(not_a_dir, "") - magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir) - doesnotexist = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir) - - client = self.g.clients[0] - d = DeferredContext(client.create_dirnode()) - def _check_errors(n): - self.failUnless(IDirectoryNode.providedBy(n)) - upload_dircap = n.get_uri() - readonly_dircap = n.get_readonly_uri() - - self.shouldFail(ValueError, 'does not exist', 'does not exist', - MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0o077, 'default') - self.shouldFail(ValueError, 'is not a directory', 'is not a directory', - MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0o077, 'default') - self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory', - MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0o077, 'default') - self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory', - MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0o077, 'default') - self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory', - MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0o077, 'default') - self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory', - MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0o077, 'default') - - def _not_implemented(): - raise NotImplementedError("blah") - self.patch(magic_folder, 'get_inotify_module', _not_implemented) - self.shouldFail(NotImplementedError, 'unsupported', 'blah', - MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0o077, 'default') - d.addCallback(_check_errors) - return d.result - - def test_write_downloaded_file(self): - workdir = fileutil.abspath_expanduser_unicode(u"cli/MagicFolder/write-downloaded-file") - local_file = fileutil.abspath_expanduser_unicode(u"foobar", base=workdir) - - class TestWriteFileMixin(WriteFileMixin): - def _log(self, msg): - pass - - writefile = TestWriteFileMixin() - writefile._umask = 0o077 - - # create a file with name "foobar" with content "foo" - # write downloaded file content "bar" into "foobar" with is_conflict = False - fileutil.make_dirs(workdir) - fileutil.write(local_file, "foo") - - # if is_conflict is False, then the .conflict file shouldn't exist. - now = time.time() - writefile._write_downloaded_file(workdir, local_file, "bar", False, now=now) - conflicted_path = local_file + u".conflict" - self.failIf(os.path.exists(conflicted_path)) - - # no backup - backup_path = local_file + u".backup" - self.failIf(os.path.exists(backup_path)) - - # .tmp file shouldn't exist - self.failIf(os.path.exists(local_file + u".tmp")) - - # The original file should have the new content - self.failUnlessEqual(fileutil.read(local_file), "bar") - - # .. and approximately the correct timestamp. - pathinfo = fileutil.get_pathinfo(local_file) - error_ns = pathinfo.mtime_ns - fileutil.seconds_to_ns(now - WriteFileMixin.FUDGE_SECONDS) - permitted_error_ns = fileutil.seconds_to_ns(WriteFileMixin.FUDGE_SECONDS)/4 - self.failUnless(abs(error_ns) < permitted_error_ns, (error_ns, permitted_error_ns)) - - # now a test for conflicted case - writefile._write_downloaded_file(workdir, local_file, "bar", True, None) - self.failUnless(os.path.exists(conflicted_path)) - - # .tmp file shouldn't exist - self.failIf(os.path.exists(local_file + u".tmp")) - - def test_periodic_full_scan(self): - """ - Create a file in a subdir without doing a notify on it and - fast-forward time to prove we do a full scan periodically. - """ - sub_dir = abspath_expanduser_unicode(u"subdir", base=self.local_dir) - self.mkdir_nonascii(sub_dir) - - d = DeferredContext(defer.succeed(None)) - - def _create_file_without_event(res): - processed_d = self.magicfolder.uploader.set_hook('processed') - what_path = abspath_expanduser_unicode(u"what", base=sub_dir) - fileutil.write(what_path, "say when") - self.magicfolder.uploader._clock.advance(self.magicfolder.uploader._periodic_full_scan_duration + 1) - # this will have now done the full scan, so we have to do - # an iteration to process anything from it - iterate_d = iterate_uploader(self.magicfolder) - return processed_d.addCallback(lambda ignored: iterate_d) - d.addCallback(_create_file_without_event) - def _advance_clock(res): - processed_d = self.magicfolder.uploader.set_hook('processed') - self.magicfolder.uploader._clock.advance(4) - return processed_d - d.addCallback(_advance_clock) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1)) - return d.result - - def test_statistics(self): - d = DeferredContext(defer.succeed(None)) - # Write something short enough for a LIT file. - d.addCallback(lambda ign: self._check_file(u"short", "test")) - - # test magic-folder statistics - d.addCallback(lambda res: self.GET("statistics")) - def _got_stats(res): - self.assertIn("Operational Statistics", res) - self.assertIn("Magic Folder", res) - self.assertIn("
  • Local Directories Monitored: 1 directories
    • ", res) - self.assertIn("
  • Files Uploaded: 1 files
    • ", res) - self.assertIn("
  • Files Queued for Upload: 0 files
    • ", res) - self.assertIn("
  • Failed Uploads: 0 files
    • ", res) - self.assertIn("
  • Files Downloaded: 0 files
    • ", res) - self.assertIn("
  • Files Queued for Download: 0 files
    • ", res) - self.assertIn("
  • Failed Downloads: 0 files
    • ", res) - d.addCallback(_got_stats) - d.addCallback(lambda res: self.GET("statistics?t=json")) - def _got_stats_json(res): - data = json.loads(res) - self.assertEqual(data["counters"]["magic_folder.uploader.dirs_monitored"], 1) - self.assertEqual(data["counters"]["magic_folder.uploader.objects_succeeded"], 1) - self.assertEqual(data["counters"]["magic_folder.uploader.files_uploaded"], 1) - self.assertEqual(data["counters"]["magic_folder.uploader.objects_queued"], 0) - d.addCallback(_got_stats_json) - return d.result - - -@skipIf(support_missing, support_message) -class RealTest(SingleMagicFolderTestMixin, AsyncTestCase): - """This is skipped unless both Twisted and the platform support inotify.""" - inject_inotify = False - - def setUp(self): - d = super(RealTest, self).setUp() - self.inotify = magic_folder.get_inotify_module() - return d - - -@skipIf(support_missing, support_message) -class RealTestAliceBob(MagicFolderAliceBobTestMixin, AsyncTestCase): - """This is skipped unless both Twisted and the platform support inotify.""" - inject_inotify = False - - def setUp(self): - d = super(RealTestAliceBob, self).setUp() - self.inotify = magic_folder.get_inotify_module() - return d diff --git a/src/allmydata/test/test_magicpath.py b/src/allmydata/test/test_magicpath.py deleted file mode 100644 index 1227a2c4d..000000000 --- a/src/allmydata/test/test_magicpath.py +++ /dev/null @@ -1,28 +0,0 @@ - -from twisted.trial import unittest - -from allmydata import magicpath - - -class MagicPath(unittest.TestCase): - tests = { - u"Documents/work/critical-project/qed.txt": u"Documents@_work@_critical-project@_qed.txt", - u"Documents/emails/bunnyfufu@hoppingforest.net": u"Documents@_emails@_bunnyfufu@@hoppingforest.net", - u"foo/@/bar": u"foo@_@@@_bar", - } - - def test_path2magic(self): - for test, expected in self.tests.items(): - self.failUnlessEqual(magicpath.path2magic(test), expected) - - def test_magic2path(self): - for expected, test in self.tests.items(): - self.failUnlessEqual(magicpath.magic2path(test), expected) - - def test_should_ignore(self): - self.failUnlessEqual(magicpath.should_ignore_file(u".bashrc"), True) - self.failUnlessEqual(magicpath.should_ignore_file(u"bashrc."), False) - self.failUnlessEqual(magicpath.should_ignore_file(u"forest/tree/branch/.bashrc"), True) - self.failUnlessEqual(magicpath.should_ignore_file(u"forest/tree/.branch/bashrc"), True) - self.failUnlessEqual(magicpath.should_ignore_file(u"forest/.tree/branch/bashrc"), True) - self.failUnlessEqual(magicpath.should_ignore_file(u"forest/tree/branch/bashrc"), False) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index a7ed6d528..b77acf4b0 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -4,6 +4,7 @@ import stat import sys import time import mock +from textwrap import dedent from unittest import skipIf @@ -175,6 +176,29 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): with self.assertRaises(Exception): config.get_config_from_file("it_does_not_exist", required=True) + def test_config_items(self): + """ + All items in a config section can be retrieved. + """ + basedir = u"test_node/test_config_items" + create_node_dir(basedir, "testing") + + with open(os.path.join(basedir, 'tahoe.cfg'), 'wt') as f: + f.write(dedent( + """ + [node] + nickname = foo + timeout.disconnect = 12 + """ + )) + config = read_config(basedir, "portnum") + self.assertEqual( + config.items("node"), + [(b"nickname", b"foo"), + (b"timeout.disconnect", b"12"), + ], + ) + @skipIf( "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), "We don't know how to set permissions on Windows.", @@ -526,7 +550,7 @@ ENABLE_HELPER = """ enabled = true """ -class FakeTub: +class FakeTub(object): def __init__(self): self.tubID = base64.b32encode("foo") self.listening_ports = [] diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py new file mode 100644 index 000000000..84484f1cf --- /dev/null +++ b/src/allmydata/test/test_python2_regressions.py @@ -0,0 +1,67 @@ +""" +Tests to check for Python2 regressions +""" + +from inspect import isclass + +from twisted.python.modules import getModule + +from testtools import ( + TestCase, +) +from testtools.matchers import ( + Equals, +) + +BLACKLIST = { + "allmydata.test.check_load", + "allmydata.windows.registry", +} + + +def is_new_style(cls): + """ + :return bool: ``True`` if and only if the given class is "new style". + """ + # All new-style classes are instances of type. By definition. + return isinstance(cls, type) + +def defined_here(cls, where): + """ + :return bool: ``True`` if and only if the given class was defined in a + module with the given name. + + :note: Classes can lie about where they are defined. Try not to do that. + """ + return cls.__module__ == where + +class PythonTwoRegressions(TestCase): + """ + Regression tests for Python 2 behaviors related to Python 3 porting. + """ + def test_new_style_classes(self): + """ + All classes in Tahoe-LAFS are new-style. + """ + newstyle = set() + classic = set() + for mod in getModule("allmydata").walkModules(): + if mod.name in BLACKLIST: + continue + + # iterAttributes will only work on loaded modules. So, load it. + mod.load() + + for attr in mod.iterAttributes(): + value = attr.load() + if isclass(value) and defined_here(value, mod.name): + if is_new_style(value): + newstyle.add(value) + else: + classic.add(value) + + self.assertThat( + classic, + Equals(set()), + "Expected to find no classic classes.", + ) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py new file mode 100644 index 000000000..8a25af889 --- /dev/null +++ b/src/allmydata/test/test_python3.py @@ -0,0 +1,107 @@ +""" +Tests related to the Python 3 porting effort itself. +""" + +from pkg_resources import ( + resource_stream, +) + +from twisted.python.modules import ( + getModule, +) +from twisted.trial.unittest import ( + SynchronousTestCase, +) + + +class Python3PortingEffortTests(SynchronousTestCase): + def test_finished_porting(self): + """ + Tahoe-LAFS has been ported to Python 3. + """ + tahoe_lafs_module_names = set(all_module_names("allmydata")) + ported_names = set(ported_module_names()) + self.assertEqual( + tahoe_lafs_module_names - ported_names, + set(), + "Some unported modules remain: {}".format( + unported_report( + tahoe_lafs_module_names, + ported_names, + ), + ), + ) + test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed" + + def test_ported_modules_exist(self): + """ + All modules listed as ported exist and belong to Tahoe-LAFS. + """ + tahoe_lafs_module_names = set(all_module_names("allmydata")) + ported_names = set(ported_module_names()) + unknown = ported_names - tahoe_lafs_module_names + self.assertEqual( + unknown, + set(), + "Some supposedly-ported modules weren't found: {}.".format(sorted(unknown)), + ) + + def test_ported_modules_distinct(self): + """ + The ported modules list doesn't contain duplicates. + """ + ported_names_list = ported_module_names() + ported_names_list.sort() + ported_names_set = set(ported_names_list) + ported_names_unique_list = list(ported_names_set) + ported_names_unique_list.sort() + self.assertEqual( + ported_names_list, + ported_names_unique_list, + ) + + +def all_module_names(toplevel): + """ + :param unicode toplevel: The name of a top-level Python package. + + :return iterator[unicode]: An iterator of ``unicode`` giving the names of + all modules within the given top-level Python package. + """ + allmydata = getModule(toplevel) + for module in allmydata.walkModules(): + yield module.name.decode("utf-8") + + +def ported_module_names(): + """ + :return list[unicode]: A ``set`` of ``unicode`` giving the names of + Tahoe-LAFS modules which have been ported to Python 3. + """ + return resource_stream( + "allmydata", + u"ported-modules.txt", + ).read().splitlines() + + +def unported_report(tahoe_lafs_module_names, ported_names): + return """ +Ported files: {} / {} +Ported lines: {} / {} +""".format( + len(ported_names), + len(tahoe_lafs_module_names), + sum(map(count_lines, ported_names)), + sum(map(count_lines, tahoe_lafs_module_names)), +) + +def count_lines(module_name): + module = getModule(module_name) + try: + source = module.filePath.getContent() + except Exception as e: + print(module_name, e) + return 0 + lines = source.splitlines() + nonblank = filter(None, lines) + return len(nonblank) diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index 9475741b4..ec521fe96 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -19,7 +19,7 @@ MAX_DELTA_READS = 10 * READ_LEEWAY # N = 10 timeout=240 # François's ARM box timed out after 120 seconds of Verifier.test_corrupt_crypttext_hashtree -class RepairTestMixin: +class RepairTestMixin(object): def failUnlessIsInstance(self, x, xtype): self.failUnless(isinstance(x, xtype), x) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 1456fdf4e..d7fa08a0c 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -23,6 +23,7 @@ from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv, unicode_to_output, \ get_filesystem_encoding from allmydata.test import common_util +from allmydata.version_checks import normalized_version import allmydata from allmydata import __appname__ from .common_util import parse_cli, run_cli @@ -55,7 +56,9 @@ def get_root_from_file(src): srcfile = allmydata.__file__ rootdir = get_root_from_file(srcfile) -class RunBinTahoeMixin: + +class RunBinTahoeMixin(object): + @inlineCallbacks def find_import_location(self): res = yield self.run_bintahoe(["--version-and-path"]) @@ -110,8 +113,6 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin): def test_path(self): d = self.run_bintahoe(["--version-and-path"]) def _cb(res): - from allmydata import normalized_version - out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) @@ -169,21 +170,6 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin): d.addCallback(_cb) return d - def test_version_no_noise(self): - d = self.run_bintahoe(["--version"]) - def _cb(res): - out, err, rc_or_sig = res - self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.failUnless(out.startswith(allmydata.__appname__+':'), str(res)) - self.failIfIn("DeprecationWarning", out, str(res)) - errlines = err.split("\n") - self.failIf([True for line in errlines if (line != "" and "UserWarning: Unbuilt egg for setuptools" not in line - and "from pkg_resources import load_entry_point" not in line)], str(res)) - if err != "": - raise unittest.SkipTest("This test is known not to pass on Ubuntu Lucid; see #1235.") - d.addCallback(_cb) - return d - @inlineCallbacks def test_help_eliot_destinations(self): out, err, rc_or_sig = yield self.run_bintahoe(["--help-eliot-destinations"]) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index 0c200d12c..b6f1fbc8a 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -12,18 +12,15 @@ from allmydata.util import deferredutil conch_interfaces = None sftp = None sftpd = None -have_pycrypto = False -try: - from Crypto import Util - Util # hush pyflakes - have_pycrypto = True -except ImportError: - pass -if have_pycrypto: +try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd +except ImportError as e: + conch_unavailable_reason = e +else: + conch_unavailable_reason = None from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError @@ -38,8 +35,10 @@ from allmydata.test.common_util import ReallyEqualMixin class Handler(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, unittest.TestCase): """This is a no-network unit test of the SFTPUserHandler and the abstractions it uses.""" - if not have_pycrypto: - skip = "SFTP support requires pycrypto, which is not installed" + if conch_unavailable_reason: + skip = "SFTP support requires Twisted Conch which is not available: {}".format( + conch_unavailable_reason, + ) def shouldFailWithSFTPError(self, expected_code, which, callable, *args, **kwargs): assert isinstance(expected_code, int), repr(expected_code) @@ -1393,7 +1392,7 @@ class Handler(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, unittest.TestCas return d def test_execCommand_and_openShell(self): - class MockProtocol: + class MockProtocol(object): def __init__(self): self.output = "" self.error = "" diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 10e31a698..9f3aee9b8 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1,4 +1,3 @@ - import time, os.path, platform, stat, re, json, struct, shutil from twisted.trial import unittest @@ -32,10 +31,14 @@ from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import StorageStatus, remove_prefix +from allmydata.storage_client import ( + _StorageServer, +) -class Marker: +class Marker(object): pass -class FakeCanary: + +class FakeCanary(object): def __init__(self, ignore_disconnectors=False): self.ignore = ignore_disconnectors self.disconnectors = {} @@ -50,7 +53,7 @@ class FakeCanary: return del self.disconnectors[marker] -class FakeStatsProvider: +class FakeStatsProvider(object): def count(self, name, delta=1): pass def register_producer(self, producer): @@ -159,9 +162,10 @@ class Bucket(unittest.TestCase): result_of_read = br.remote_read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) -class RemoteBucket: +class RemoteBucket(object): - def __init__(self): + def __init__(self, target): + self.target = target self.read_count = 0 self.write_count = 0 @@ -187,8 +191,7 @@ class BucketProxy(unittest.TestCase): fileutil.make_dirs(os.path.join(basedir, "tmp")) bw = BucketWriter(self, incoming, final, size, self.make_lease(), FakeCanary()) - rb = RemoteBucket() - rb.target = bw + rb = RemoteBucket(bw) return bw, rb, final def make_lease(self): @@ -260,8 +263,7 @@ class BucketProxy(unittest.TestCase): # now read everything back def _start_reading(res): br = BucketReader(self, sharefname) - rb = RemoteBucket() - rb.target = br + rb = RemoteBucket(br) server = NoNetworkServer("abc", None) rbp = rbp_class(rb, server, storage_index="") self.failUnlessIn("to peer", repr(rbp)) @@ -1367,14 +1369,89 @@ class MutableServer(unittest.TestCase): self.failUnless(os.path.exists(prefixdir), prefixdir) self.failIf(os.path.exists(bucketdir), bucketdir) + def test_writev_without_renew_lease(self): + """ + The helper method ``slot_testv_and_readv_and_writev`` does not renew + leases if ``False`` is passed for the ``renew_leases`` parameter. + """ + ss = self.create("test_writev_without_renew_lease") + + storage_index = "si2" + secrets = ( + self.write_enabler(storage_index), + self.renew_secret(storage_index), + self.cancel_secret(storage_index), + ) + + sharenum = 3 + datav = [(0, b"Hello, world")] + + ss.slot_testv_and_readv_and_writev( + storage_index=storage_index, + secrets=secrets, + test_and_write_vectors={ + sharenum: ([], datav, None), + }, + read_vector=[], + renew_leases=False, + ) + leases = list(ss.get_slot_leases(storage_index)) + self.assertEqual([], leases) + + def test_get_slot_leases_empty_slot(self): + """ + When ``get_slot_leases`` is called for a slot for which the server has no + shares, it returns an empty iterable. + """ + ss = self.create(b"test_get_slot_leases_empty_slot") + self.assertEqual( + list(ss.get_slot_leases(b"si1")), + [], + ) + + def test_remove_non_present(self): + """ + A write vector which would remove a share completely is applied as a no-op + by a server which does not have the share. + """ + ss = self.create("test_remove_non_present") + + storage_index = "si1" + secrets = ( + self.write_enabler(storage_index), + self.renew_secret(storage_index), + self.cancel_secret(storage_index), + ) + + sharenum = 3 + testv = [] + datav = [] + new_length = 0 + read_vector = [] + + # We don't even need to create any shares to exercise this + # functionality. Just go straight to sending a truncate-to-zero + # write. + testv_is_good, read_data = ss.remote_slot_testv_and_readv_and_writev( + storage_index=storage_index, + secrets=secrets, + test_and_write_vectors={ + sharenum: (testv, datav, new_length), + }, + read_vector=read_vector, + ) + + self.assertTrue(testv_is_good) + self.assertEqual({}, read_data) + class MDMFProxies(unittest.TestCase, ShouldFailMixin): def setUp(self): self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() self.ss = self.create("MDMFProxies storage test server") - self.rref = RemoteBucket() - self.rref.target = self.ss + self.rref = RemoteBucket(self.ss) + self.storage_server = _StorageServer(lambda: self.rref) self.secrets = (self.write_enabler("we_secret"), self.renew_secret("renew_secret"), self.cancel_secret("cancel_secret")) @@ -1604,7 +1681,6 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): empty=False): # Some tests need SDMF shares to verify that we can still # read them. This method writes one, which resembles but is not - assert self.rref write = self.ss.remote_slot_testv_and_readv_and_writev share = self.build_test_sdmf_share(empty) testvs = [(0, 1, "eq", "")] @@ -1617,7 +1693,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_read(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) # Check that every method equals what we expect it to. d = defer.succeed(None) def _check_block_and_salt(block_and_salt): @@ -1689,7 +1765,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_read_with_different_tail_segment_size(self): self.write_test_share_to_server("si1", tail_segment=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = mr.get_block_and_salt(5) def _check_tail_segment(results): block, salt = results @@ -1701,7 +1777,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_get_block_with_invalid_segnum(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "test invalid segnum", @@ -1712,7 +1788,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_get_encoding_parameters_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = mr.get_encoding_parameters() def _check_encoding_parameters(args): (k, n, segment_size, datalen) = args @@ -1726,7 +1802,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_get_seqnum_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = mr.get_seqnum() d.addCallback(lambda seqnum: self.failUnlessEqual(seqnum, 0)) @@ -1735,7 +1811,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_get_root_hash_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = mr.get_root_hash() d.addCallback(lambda root_hash: self.failUnlessEqual(root_hash, self.root_hash)) @@ -1744,7 +1820,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_get_checkstring_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = mr.get_checkstring() d.addCallback(lambda checkstring: self.failUnlessEqual(checkstring, self.checkstring)) @@ -2059,7 +2135,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # size of 6, we know that it has 6 byte segments, which will # be split into blocks of 2 bytes because our FEC k # parameter is 3. - mw = MDMFSlotWriteProxy(share, self.rref, si, self.secrets, 0, 3, 10, + mw = MDMFSlotWriteProxy(share, self.storage_server, si, self.secrets, 0, 3, 10, 6, datalength) return mw @@ -2262,7 +2338,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mw.finish_publishing()) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) def _check_block_and_salt(block_and_salt): (block, salt) = block_and_salt self.failUnlessEqual(block, self.block) @@ -2330,7 +2406,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # since it will encounter them on the grid. Callers use the # is_sdmf method to test this. self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = mr.is_sdmf() d.addCallback(lambda issdmf: self.failUnless(issdmf)) @@ -2341,7 +2417,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # The slot read proxy should, naturally, know how to tell us # about data in the SDMF format self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: mr.is_sdmf()) @@ -2412,7 +2488,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # read more segments than that. The reader should know this and # complain if we try to do that. self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: mr.is_sdmf()) @@ -2434,7 +2510,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mdmf_data = self.build_test_mdmf_share() self.write_test_share_to_server("si1") def _make_mr(ignored, length): - mr = MDMFSlotReadProxy(self.rref, "si1", 0, mdmf_data[:length]) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, mdmf_data[:length]) return mr d = defer.succeed(None) @@ -2495,7 +2571,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): sdmf_data = self.build_test_sdmf_share() self.write_sdmf_share_to_server("si1") def _make_mr(ignored, length): - mr = MDMFSlotReadProxy(self.rref, "si1", 0, sdmf_data[:length]) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, sdmf_data[:length]) return mr d = defer.succeed(None) @@ -2561,7 +2637,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # unrelated to the actual handling of the content of the file. # The reader should behave intelligently in these cases. self.write_test_share_to_server("si1", empty=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) # We should be able to get the encoding parameters, and they # should be correct. d = defer.succeed(None) @@ -2587,7 +2663,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_read_with_empty_sdmf_file(self): self.write_sdmf_share_to_server("si1", empty=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) # We should be able to get the encoding parameters, and they # should be correct d = defer.succeed(None) @@ -2613,7 +2689,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_verinfo_with_sdmf_file(self): self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) # We should be able to get the version information. d = defer.succeed(None) d.addCallback(lambda ignored: @@ -2654,7 +2730,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def test_verinfo_with_mdmf_file(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MDMFSlotReadProxy(self.storage_server, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: mr.get_verinfo()) @@ -2700,7 +2776,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # set the way we want them for the tests below. data = self.build_test_sdmf_share() sdmfr = SDMFSlotWriteProxy(0, - self.rref, + self.storage_server, "si1", self.secrets, 0, 3, 10, 36, 36) @@ -2743,7 +2819,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # write, we need to set the checkstring correctly. When we # don't, no write should occur. sdmfw = SDMFSlotWriteProxy(0, - self.rref, + self.storage_server, "si1", self.secrets, 1, 3, 10, 36, 36) @@ -3052,8 +3128,9 @@ class InstrumentedLeaseCheckingCrawler(LeaseCheckingCrawler): if not self.stop_after_first_bucket: self.cpu_slice = 500 -class BrokenStatResults: +class BrokenStatResults(object): pass + class No_ST_BLOCKS_LeaseCheckingCrawler(LeaseCheckingCrawler): def stat(self, fn): s = os.stat(fn) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 6a4ae5d06..00343ea20 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -1,13 +1,80 @@ import hashlib from mock import Mock -from allmydata.util import base32, yamlutil +from json import ( + dumps, + loads, +) +from fixtures import ( + TempDir, +) +from testtools.content import ( + text_content, +) +from testtools.matchers import ( + MatchesAll, + IsInstance, + MatchesStructure, + Equals, + Is, + AfterPreprocessing, +) + +from zope.interface import ( + implementer, +) +from zope.interface.verify import ( + verifyObject, +) + +from hyperlink import ( + URL, +) + +from twisted.application.service import ( + Service, +) from twisted.trial import unittest from twisted.internet.defer import succeed, inlineCallbacks +from twisted.python.filepath import ( + FilePath, +) -from allmydata.storage_client import NativeStorageServer -from allmydata.storage_client import StorageFarmBroker +from foolscap.api import ( + Tub, +) +from .common import ( + EMPTY_CLIENT_CONFIG, + SyncTestCase, + AsyncTestCase, + UseTestPlugins, + UseNode, + SameProcessStreamEndpointAssigner, +) +from .common_web import ( + do_http, +) +from .storage_plugin import ( + DummyStorageClient, +) +from allmydata.webish import ( + WebishServer, +) +from allmydata.util import base32, yamlutil +from allmydata.storage_client import ( + IFoolscapStorageServer, + NativeStorageServer, + StorageFarmBroker, + _FoolscapStorage, + _NullStorage, +) +from allmydata.interfaces import ( + IConnectionStatus, + IStorageServer, +) + +SOME_FURL = b"pb://abcde@nowhere/fake" class NativeStorageServerWithVersion(NativeStorageServer): def __init__(self, version): @@ -40,22 +107,411 @@ class TestNativeStorageServer(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - nss = NativeStorageServer("server_id", ann, None, {}, {}, None) + nss = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) self.assertEqual(nss.get_nickname(), "") + +class GetConnectionStatus(unittest.TestCase): + """ + Tests for ``NativeStorageServer.get_connection_status``. + """ + def test_unrecognized_announcement(self): + """ + When ``NativeStorageServer`` is constructed with a storage announcement it + doesn't recognize, its ``get_connection_status`` nevertheless returns + an object which provides ``IConnectionStatus``. + """ + # Pretty hard to recognize anything from an empty announcement. + ann = {} + nss = NativeStorageServer("server_id", ann, Tub, {}, EMPTY_CLIENT_CONFIG) + nss.start_connecting(lambda: None) + connection_status = nss.get_connection_status() + self.assertTrue(IConnectionStatus.providedBy(connection_status)) + + +class UnrecognizedAnnouncement(unittest.TestCase): + """ + Tests for handling of announcements that aren't recognized and don't use + *anonymous-storage-FURL*. + + Recognition failure is created by making up something completely novel for + these tests. In real use, recognition failure would most likely come from + an announcement generated by a storage server plugin which is not loaded + in the client. + """ + ann = { + u"name": u"tahoe-lafs-testing-v1", + u"any-parameter": 12345, + } + server_id = b"abc" + + def _tub_maker(self, overrides): + return Service() + + def native_storage_server(self): + """ + Make a ``NativeStorageServer`` out of an unrecognizable announcement. + """ + return NativeStorageServer( + self.server_id, + self.ann, + self._tub_maker, + {}, + EMPTY_CLIENT_CONFIG, + ) + + def test_no_exceptions(self): + """ + ``NativeStorageServer`` can be instantiated with an unrecognized + announcement. + """ + self.native_storage_server() + + def test_start_connecting(self): + """ + ``NativeStorageServer.start_connecting`` does not raise an exception. + """ + server = self.native_storage_server() + server.start_connecting(None) + + def test_stop_connecting(self): + """ + ``NativeStorageServer.stop_connecting`` does not raise an exception. + """ + server = self.native_storage_server() + server.start_connecting(None) + server.stop_connecting() + + def test_try_to_connect(self): + """ + ``NativeStorageServer.try_to_connect`` does not raise an exception. + """ + server = self.native_storage_server() + server.start_connecting(None) + server.try_to_connect() + + def test_various_data_methods(self): + """ + The data accessors of ``NativeStorageServer`` that depend on the + announcement do not raise an exception. + """ + server = self.native_storage_server() + server.get_permutation_seed() + server.get_name() + server.get_longname() + server.get_tubid() + server.get_lease_seed() + server.get_foolscap_write_enabler_seed() + server.get_nickname() + + + +class PluginMatchedAnnouncement(SyncTestCase): + """ + Tests for handling by ``NativeStorageServer`` of storage server + announcements that are handled by an ``IFoolscapStoragePlugin``. + """ + @inlineCallbacks + def make_node(self, introducer_furl, storage_plugin, plugin_config): + """ + Create a client node with the given configuration. + + :param bytes introducer_furl: The introducer furl with which to + configure the client. + + :param bytes storage_plugin: The name of a storage plugin to enable. + + :param dict[bytes, bytes] plugin_config: Configuration to supply to + the enabled plugin. May also be ``None`` for no configuration + section (distinct from ``{}`` which creates an empty configuration + section). + """ + tempdir = TempDir() + self.useFixture(tempdir) + self.basedir = FilePath(tempdir.path) + self.basedir.child(u"private").makedirs() + self.useFixture(UseTestPlugins()) + + self.node_fixture = self.useFixture(UseNode( + plugin_config, + storage_plugin, + self.basedir, + introducer_furl, + )) + self.config = self.node_fixture.config + self.node = yield self.node_fixture.create_node() + [self.introducer_client] = self.node.introducer_clients + + + def publish(self, server_id, announcement, introducer_client): + for subscription in introducer_client.subscribed_to: + if subscription.service_name == u"storage": + subscription.cb( + server_id, + announcement, + *subscription.args, + **subscription.kwargs + ) + + def get_storage(self, server_id, node): + storage_broker = node.get_storage_broker() + native_storage_server = storage_broker.servers[server_id] + return native_storage_server._storage + + def set_rref(self, server_id, node, rref): + storage_broker = node.get_storage_broker() + native_storage_server = storage_broker.servers[server_id] + native_storage_server._rref = rref + + @inlineCallbacks + def test_ignored_non_enabled_plugin(self): + """ + An announcement that could be matched by a plugin that is not enabled is + not matched. + """ + yield self.make_node( + introducer_furl=SOME_FURL, + storage_plugin=b"tahoe-lafs-dummy-v1", + plugin_config=None, + ) + server_id = b"v0-abcdef" + ann = { + u"service-name": u"storage", + u"storage-options": [{ + # notice how the announcement is for a different storage plugin + # than the one that is enabled. + u"name": u"tahoe-lafs-dummy-v2", + u"storage-server-FURL": SOME_FURL.decode("ascii"), + }], + } + self.publish(server_id, ann, self.introducer_client) + storage = self.get_storage(server_id, self.node) + self.assertIsInstance(storage, _NullStorage) + + @inlineCallbacks + def test_enabled_plugin(self): + """ + An announcement that could be matched by a plugin that is enabled with + configuration is matched and the plugin's storage client is used. + """ + plugin_config = { + b"abc": b"xyz", + } + plugin_name = b"tahoe-lafs-dummy-v1" + yield self.make_node( + introducer_furl=SOME_FURL, + storage_plugin=plugin_name, + plugin_config=plugin_config, + ) + server_id = b"v0-abcdef" + ann = { + u"service-name": u"storage", + u"storage-options": [{ + # and this announcement is for a plugin with a matching name + u"name": plugin_name, + u"storage-server-FURL": SOME_FURL.decode("ascii"), + }], + } + self.publish(server_id, ann, self.introducer_client) + storage = self.get_storage(server_id, self.node) + self.assertTrue( + verifyObject( + IFoolscapStorageServer, + storage, + ), + ) + expected_rref = object() + # Can't easily establish a real Foolscap connection so fake the result + # of doing so... + self.set_rref(server_id, self.node, expected_rref) + self.expectThat( + storage.storage_server, + MatchesAll( + IsInstance(DummyStorageClient), + MatchesStructure( + get_rref=AfterPreprocessing( + lambda get_rref: get_rref(), + Is(expected_rref), + ), + configuration=Equals(plugin_config), + announcement=Equals({ + u'name': plugin_name, + u'storage-server-FURL': u'pb://abcde@nowhere/fake', + }), + ), + ), + ) + + @inlineCallbacks + def test_enabled_no_configuration_plugin(self): + """ + An announcement that could be matched by a plugin that is enabled with no + configuration is matched and the plugin's storage client is used. + """ + plugin_name = b"tahoe-lafs-dummy-v1" + yield self.make_node( + introducer_furl=SOME_FURL, + storage_plugin=plugin_name, + plugin_config=None, + ) + server_id = b"v0-abcdef" + ann = { + u"service-name": u"storage", + u"storage-options": [{ + # and this announcement is for a plugin with a matching name + u"name": plugin_name, + u"storage-server-FURL": SOME_FURL.decode("ascii"), + }], + } + self.publish(server_id, ann, self.introducer_client) + storage = self.get_storage(server_id, self.node) + self.addDetail("storage", text_content(str(storage))) + self.expectThat( + storage.storage_server, + MatchesAll( + IsInstance(DummyStorageClient), + MatchesStructure( + configuration=Equals({}), + ), + ), + ) + + +class FoolscapStorageServers(unittest.TestCase): + """ + Tests for implementations of ``IFoolscapStorageServer``. + """ + def test_null_provider(self): + """ + Instances of ``_NullStorage`` provide ``IFoolscapStorageServer``. + """ + self.assertTrue( + verifyObject( + IFoolscapStorageServer, + _NullStorage(), + ), + ) + + def test_foolscap_provider(self): + """ + Instances of ``_FoolscapStorage`` provide ``IFoolscapStorageServer``. + """ + @implementer(IStorageServer) + class NotStorageServer(object): + pass + self.assertTrue( + verifyObject( + IFoolscapStorageServer, + _FoolscapStorage.from_announcement( + u"server-id", + SOME_FURL, + {u"permutation-seed-base32": base32.b2a(b"permutationseed")}, + NotStorageServer(), + ), + ), + ) + + +class StoragePluginWebPresence(AsyncTestCase): + """ + Tests for the web resources ``IFoolscapStorageServer`` plugins may expose. + """ + @inlineCallbacks + def setUp(self): + super(StoragePluginWebPresence, self).setUp() + + self.useFixture(UseTestPlugins()) + + self.port_assigner = SameProcessStreamEndpointAssigner() + self.port_assigner.setUp() + self.addCleanup(self.port_assigner.tearDown) + self.storage_plugin = b"tahoe-lafs-dummy-v1" + + from twisted.internet import reactor + _, port_endpoint = self.port_assigner.assign(reactor) + + tempdir = TempDir() + self.useFixture(tempdir) + self.basedir = FilePath(tempdir.path) + self.basedir.child(u"private").makedirs() + self.node_fixture = self.useFixture(UseNode( + plugin_config={ + b"web": b"1", + }, + node_config={ + b"tub.location": b"127.0.0.1:1", + b"web.port": port_endpoint, + }, + storage_plugin=self.storage_plugin, + basedir=self.basedir, + introducer_furl=SOME_FURL, + )) + self.node = yield self.node_fixture.create_node() + self.webish = self.node.getServiceNamed(WebishServer.name) + self.node.startService() + self.addCleanup(self.node.stopService) + self.port = self.webish.getPortnum() + + @inlineCallbacks + def test_plugin_resource_path(self): + """ + The plugin's resource is published at */storage-plugins/*. + """ + url = u"http://127.0.0.1:{port}/storage-plugins/{plugin_name}".format( + port=self.port, + plugin_name=self.storage_plugin, + ).encode("utf-8") + result = yield do_http(b"get", url) + self.assertThat(result, Equals(dumps({b"web": b"1"}))) + + @inlineCallbacks + def test_plugin_resource_persistent_across_requests(self): + """ + The plugin's resource is loaded and then saved and re-used for future + requests. + """ + url = URL( + scheme=u"http", + host=u"127.0.0.1", + port=self.port, + path=( + u"storage-plugins", + self.storage_plugin.decode("utf-8"), + u"counter", + ), + ).to_text().encode("utf-8") + values = { + loads((yield do_http(b"get", url)))[u"value"], + loads((yield do_http(b"get", url)))[u"value"], + } + self.assertThat( + values, + # If the counter manages to go up then the state stuck around. + Equals({1, 2}), + ) + + +def make_broker(tub_maker=lambda h: Mock()): + """ + Create a ``StorageFarmBroker`` with the given tub maker and an empty + client configuration. + """ + return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG) + + class TestStorageFarmBroker(unittest.TestCase): def test_static_servers(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = make_broker() key_s = 'v0-1234-1' - servers_yaml = """\ + servers_yaml = b"""\ storage: v0-1234-1: ann: - anonymous-storage-FURL: pb://ge@nowhere/fake + anonymous-storage-FURL: {furl} permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa -""" +""".format(furl=SOME_FURL) servers = yamlutil.safe_load(servers_yaml) permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa") broker.set_static_servers(servers["storage"]) @@ -80,22 +536,22 @@ storage: self.assertEqual(s2.get_permutation_seed(), permseed) def test_static_permutation_seed_pubkey(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = make_broker() server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" ann = { - "anonymous-storage-FURL": "pb://abcde@nowhere/fake", + "anonymous-storage-FURL": SOME_FURL, } broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}}) s = broker.servers[server_id] self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) def test_static_permutation_seed_explicit(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = make_broker() server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7" ann = { - "anonymous-storage-FURL": "pb://abcde@nowhere/fake", + "anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": k, } broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}}) @@ -103,10 +559,10 @@ storage: self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) def test_static_permutation_seed_hashed(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = make_broker() server_id = "unparseable" ann = { - "anonymous-storage-FURL": "pb://abcde@nowhere/fake", + "anonymous-storage-FURL": SOME_FURL, } broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}}) s = broker.servers[server_id] @@ -119,7 +575,7 @@ storage: new_tubs = [] def make_tub(*args, **kwargs): return new_tubs.pop() - broker = StorageFarmBroker(True, make_tub) + broker = make_broker(make_tub) done = broker.when_connected_enough(5) broker.use_introducer(introducer) # subscribes to "storage" to learn of new storage nodes diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index ef87517e9..3f8a363d2 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -3,6 +3,8 @@ from __future__ import print_function import os, re, sys, time, json from functools import partial +from bs4 import BeautifulSoup + from twisted.internet import reactor from twisted.trial import unittest from twisted.internet import defer @@ -38,6 +40,9 @@ from .common import ( SameProcessStreamEndpointAssigner, ) from .common_web import do_http, Error +from .web.common import ( + assert_soup_has_tag_with_attributes +) # TODO: move this to common or common_util from allmydata.test.test_runner import RunBinTahoeMixin @@ -1771,8 +1776,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # get the welcome page from the node that uses the helper too d.addCallback(lambda res: do_http("get", self.helper_webish_url)) def _got_welcome_helper(page): - html = page.replace('\n', ' ') - self.failUnless(re.search('', html), page) + soup = BeautifulSoup(page, 'html5lib') + assert_soup_has_tag_with_attributes( + self, soup, u"img", + { u"alt": u"Connected", u"src": u"img/connected-yes.png" } + ) self.failUnlessIn("Not running helper", page) d.addCallback(_got_welcome_helper) @@ -2227,8 +2235,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): self.failUnlessEqual(data, "data to be uploaded: file1\n") d.addCallback(_check_outfile1) - d.addCallback(run, "rm", "tahoe-file0") - d.addCallback(run, "rm", "tahoe:file2") + d.addCallback(run, "unlink", "tahoe-file0") + d.addCallback(run, "unlink", "tahoe:file2") d.addCallback(run, "ls") d.addCallback(_check_ls, [], ["tahoe-file0", "file2"]) @@ -2428,7 +2436,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _run_in_subprocess(ignored, verb, *args, **kwargs): stdin = kwargs.get("stdin") - env = kwargs.get("env") + env = kwargs.get("env", os.environ) + # Python warnings from the child process don't matter. + env["PYTHONWARNINGS"] = "ignore" newargs = ["--node-directory", self.getdir("client0"), verb] + list(args) return self.run_bintahoe(newargs, stdin=stdin, env=env) @@ -2511,9 +2521,9 @@ class Connections(SystemTestMixin, unittest.TestCase): self.failUnlessEqual(len(nonclients), 1) self.s1 = nonclients[0] # s1 is the server, not c0 - self.s1_rref = self.s1.get_rref() - self.failIfEqual(self.s1_rref, None) - self.failUnless(self.s1.is_connected()) + self.s1_storage_server = self.s1.get_storage_server() + self.assertIsNot(self.s1_storage_server, None) + self.assertTrue(self.s1.is_connected()) d.addCallback(_start) # now shut down the server @@ -2524,9 +2534,9 @@ class Connections(SystemTestMixin, unittest.TestCase): d.addCallback(lambda ign: self.poll(_poll)) def _down(ign): - self.failIf(self.s1.is_connected()) - rref = self.s1.get_rref() - self.failUnless(rref) - self.failUnlessIdentical(rref, self.s1_rref) + self.assertFalse(self.s1.is_connected()) + storage_server = self.s1.get_storage_server() + self.assertIsNot(storage_server, None) + self.assertEqual(storage_server, self.s1_storage_server) d.addCallback(_down) return d diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index c413c9b5a..165ca17d7 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -22,6 +22,10 @@ from allmydata.storage_client import StorageFarmBroker from allmydata.storage.server import storage_index_to_dir from allmydata.client import _Client +from .common import ( + EMPTY_CLIENT_CONFIG, +) + MiB = 1024*1024 def extract_uri(results): @@ -86,7 +90,7 @@ class Uploadable(unittest.TestCase): class ServerError(Exception): pass -class SetDEPMixin: +class SetDEPMixin(object): def set_encoding_parameters(self, k, happy, n, max_segsize=1*MiB): p = {"k": k, "happy": happy, @@ -95,7 +99,7 @@ class SetDEPMixin: } self.node.encoding_params = p -class FakeStorageServer: +class FakeStorageServer(object): def __init__(self, mode, reactor=None): self.mode = mode self.allocated = [] @@ -162,7 +166,7 @@ class FakeStorageServer: -class FakeBucketWriter: +class FakeBucketWriter(object): # a diagnostic version of storageserver.BucketWriter def __init__(self, size): self.data = StringIO() @@ -217,7 +221,11 @@ class FakeClient(object): ("%20d" % fakeid, FakeStorageServer(mode[fakeid], reactor=reactor)) for fakeid in range(self.num_servers) ] - self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) for (serverid, rref) in servers: ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(serverid), "permutation-seed-base32": base32.b2a(serverid) } @@ -856,7 +864,7 @@ def is_happy_enough(servertoshnums, h, k): return False return True -class FakeServerTracker: +class FakeServerTracker(object): def __init__(self, serverid, buckets): self._serverid = serverid self.buckets = buckets diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 76e3f8155..c0ad0e62d 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -3,16 +3,18 @@ from __future__ import print_function def foo(): pass # keep the line number constant +import binascii import six +import hashlib import os, time, sys import yaml + from six.moves import StringIO from datetime import timedelta from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log -from pycryptopp.hash.sha256 import SHA256 as _hash from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil from allmydata.util import assertutil, fileutil, deferredutil, abbreviate @@ -20,12 +22,22 @@ from allmydata.util import limiter, time_format, pollmixin, cachedir from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans +from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin, TimezoneMixin if six.PY3: long = int +def sha256(data): + """ + :param bytes data: data to hash + + :returns: a hex-encoded SHA256 hash of the data + """ + return binascii.hexlify(hashlib.sha256(data).digest()) + + class Base32(unittest.TestCase): def test_b2a_matches_Pythons(self): import base64 @@ -777,6 +789,11 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessFalse(symlinkinfo.isfile) self.failUnlessFalse(symlinkinfo.isdir) + def test_encrypted_tempfile(self): + f = EncryptedTemporaryFile() + f.write("foobar") + f.close() + class PollMixinTests(unittest.TestCase): def setUp(self): @@ -1333,7 +1350,7 @@ class CacheDir(unittest.TestCase): del b2 ctr = [0] -class EqButNotIs: +class EqButNotIs(object): def __init__(self, x): self.x = x self.hash = ctr[0] @@ -1615,7 +1632,7 @@ class Log(unittest.TestCase): self.flushLoggedErrors(SampleError) -class SimpleSpans: +class SimpleSpans(object): # this is a simple+inefficient form of util.spans.Spans . We compare the # behavior of this reference model against the real (efficient) form. @@ -1825,7 +1842,7 @@ class ByteSpans(unittest.TestCase): def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): - what = _hash(subseed+str(i)).hexdigest() + what = sha256(subseed+str(i)) start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) ns1.add(start, length); ns2.add(start, length) @@ -1833,7 +1850,7 @@ class ByteSpans(unittest.TestCase): #print for i in range(1000): - what = _hash(seed+str(i)).hexdigest() + what = sha256(seed+str(i)) op = what[0] subop = what[1] start = int(what[2:4], 16) @@ -1879,7 +1896,7 @@ class ByteSpans(unittest.TestCase): self.failUnlessEqual(bool(s1), bool(s2)) self.failUnlessEqual(list(s1), list(s2)) for j in range(10): - what = _hash(what[12:14]+str(j)).hexdigest() + what = sha256(what[12:14]+str(j)) start = int(what[2:4], 16) length = max(1, int(what[5:6], 16)) span = (start, length) @@ -1943,7 +1960,7 @@ def replace(s, start, data): assert len(s) >= start+len(data) return s[:start] + data + s[start+len(data):] -class SimpleDataSpans: +class SimpleDataSpans(object): def __init__(self, other=None): self.missing = "" # "1" where missing, "0" where found self.data = "" @@ -2148,14 +2165,14 @@ class StringSpans(unittest.TestCase): created = 0 pieces = [] while created < length: - piece = _hash(seed + str(created)).hexdigest() + piece = sha256(seed + str(created)) pieces.append(piece) created += len(piece) return "".join(pieces)[:length] def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): - what = _hash(subseed+str(i)).hexdigest() + what = sha256(subseed+str(i)) start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) ns1.add(start, _randstr(length, what[7:9])); @@ -2164,7 +2181,7 @@ class StringSpans(unittest.TestCase): #print for i in range(1000): - what = _hash(seed+str(i)).hexdigest() + what = sha256(seed+str(i)) op = what[0] subop = what[1] start = int(what[2:4], 16) @@ -2192,7 +2209,7 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(s1.len(), s2.len()) self.failUnlessEqual(list(s1._dump()), list(s2._dump())) for j in range(100): - what = _hash(what[12:14]+str(j)).hexdigest() + what = sha256(what[12:14]+str(j)) start = int(what[2:4], 16) length = max(1, int(what[5:6], 16)) d1 = s1.get(start, length); d2 = s2.get(start, length) diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index c33f8e9ef..fd0cb0e20 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -1,12 +1,16 @@ import sys import pkg_resources -from pkg_resources import Requirement - +from operator import ( + setitem, +) from twisted.trial import unittest -from allmydata import check_requirement, cross_check, get_package_versions_and_locations, \ - extract_openssl_version, PackagingError +from allmydata.version_checks import ( + _cross_check as cross_check, + _extract_openssl_version as extract_openssl_version, + _get_package_versions_and_locations as get_package_versions_and_locations, +) from allmydata.util.verlib import NormalizedVersion as V, \ IrrationalVersionError, \ suggest_normalized_version as suggest @@ -28,43 +32,6 @@ class MockSSL(object): class CheckRequirement(unittest.TestCase): - def test_check_requirement(self): - self._check_success("setuptools >= 0.6c6", {"setuptools": ("0.6", "", None)}) - self._check_success("setuptools >= 0.6c6", {"setuptools": ("0.6", "", "distribute")}) - self._check_success("pycrypto >= 2.1.0, != 2.2, != 2.4", {"pycrypto": ("2.1.0", "", None)}) - self._check_success("pycrypto >= 2.1.0, != 2.2, != 2.4", {"pycrypto": ("2.3.0", "", None)}) - self._check_success("pycrypto >= 2.1.0, != 2.2, != 2.4", {"pycrypto": ("2.4.1", "", None)}) - self._check_success("Twisted >= 11.0.0, <= 12.2.0", {"Twisted": ("11.0.0", "", None)}) - self._check_success("Twisted >= 11.0.0, <= 12.2.0", {"Twisted": ("12.2.0", "", None)}) - - self._check_success("zope.interface", {"zope.interface": ("unknown", "", None)}) - self._check_success("mock", {"mock": ("0.6.0", "", None)}) - self._check_success("foo >= 1.0", {"foo": ("1.0", "", None), "bar": ("2.0", "", None)}) - - self._check_success("foolscap[secure_connections] >= 0.6.0", {"foolscap": ("0.7.0", "", None)}) - - self._check_failure("foolscap[secure_connections] >= 0.6.0", {"foolscap": ("0.5.1", "", None)}) - self._check_failure("pycrypto >= 2.1.0, != 2.2, != 2.4", {"pycrypto": ("2.2.0", "", None)}) - self._check_failure("pycrypto >= 2.1.0, != 2.2, != 2.4", {"pycrypto": ("2.0.0", "", None)}) - self._check_failure("Twisted >= 11.0.0, <= 12.2.0", {"Twisted": ("10.2.0", "", None)}) - self._check_failure("Twisted >= 11.0.0, <= 12.2.0", {"Twisted": ("13.0.0", "", None)}) - self._check_failure("foo >= 1.0", {}) - - self.failUnlessRaises(ImportError, check_requirement, - "foo >= 1.0", {"foo": (None, None, "foomodule")}) - - def _check_success(self, req, vers_and_locs): - check_requirement(req, vers_and_locs) - - for pkg, ver in vers_and_locs.items(): - self.failUnless(ver[0] in Requirement.parse(req), str((ver, req))) - - def _check_failure(self, req, vers_and_locs): - self.failUnlessRaises(PackagingError, check_requirement, req, vers_and_locs) - - for pkg, ver in vers_and_locs.items(): - self.failIf(ver[0] in Requirement.parse(req), str((ver, req))) - def test_packages_from_pkg_resources(self): if hasattr(sys, 'frozen'): raise unittest.SkipTest("This test doesn't apply to frozen builds.") @@ -270,3 +237,26 @@ class VersionTestCase(unittest.TestCase): # zetuptoolz self.failUnlessEqual(suggest('0.6c16dev3'), '0.6c16.dev3') + + +class T(unittest.TestCase): + def test_report_import_error(self): + """ + get_package_versions_and_locations reports a dependency if a dependency + cannot be imported. + """ + # Make sure we don't leave the system in a bad state. + self.addCleanup( + lambda foolscap=sys.modules["foolscap"]: setitem( + sys.modules, + "foolscap", + foolscap, + ), + ) + # Make it look like Foolscap isn't installed. + sys.modules["foolscap"] = None + vers_and_locs, errors = get_package_versions_and_locations() + + foolscap_stuffs = [stuff for (pkg, stuff) in vers_and_locs if pkg == 'foolscap'] + self.failUnlessEqual(len(foolscap_stuffs), 1) + self.failUnless([e for e in errors if "dependency \'foolscap\' could not be imported" in e]) diff --git a/src/allmydata/test/test_websocket_logs.py b/src/allmydata/test/test_websocket_logs.py new file mode 100644 index 000000000..e666a4902 --- /dev/null +++ b/src/allmydata/test/test_websocket_logs.py @@ -0,0 +1,54 @@ +import json + +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks + +from eliot import log_call + +from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper + +from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol + + +class TestStreamingLogs(unittest.TestCase): + """ + Test websocket streaming of logs + """ + + def setUp(self): + self.reactor = MemoryReactorClockResolver() + self.pumper = create_pumper() + self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) + return self.pumper.start() + + def tearDown(self): + return self.pumper.stop() + + @inlineCallbacks + def test_one_log(self): + """ + write a single Eliot log and see it streamed via websocket + """ + + proto = yield self.agent.open( + transport_config=u"ws://localhost:1234/ws", + options={}, + ) + + messages = [] + def got_message(msg, is_binary=False): + messages.append(json.loads(msg)) + proto.on("message", got_message) + + @log_call(action_type=u"test:cli:some-exciting-action") + def do_a_thing(): + pass + + do_a_thing() + + proto.transport.loseConnection() + yield proto.is_closed + + self.assertEqual(len(messages), 2) + self.assertEqual("started", messages[0]["action_status"]) + self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 3ea67bf83..871cdeb26 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -1,6 +1,87 @@ +import re + unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8') unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8') unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8') FAVICON_MARKUP = '' + + +def assert_soup_has_favicon(testcase, soup): + """ + Using a ``TestCase`` object ``testcase``, assert that the passed in + ``BeautifulSoup`` object ``soup`` contains the tahoe favicon link. + """ + links = soup.find_all(u'link', rel=u'shortcut icon') + testcase.assert_( + any(t[u'href'] == u'/icon.png' for t in links), soup) + + +def assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs): + """ + Using a ``TestCase`` object ``testcase``, assert that the passed + in ``BeatufulSoup`` object ``soup`` contains a tag ``tag_name`` + (unicode) which has all the attributes in ``attrs`` (dict). + """ + tags = soup.find_all(tag_name) + for tag in tags: + if all(v in tag.attrs.get(k, []) for k, v in attrs.items()): + return # we found every attr in this tag; done + testcase.fail( + u"No <{}> tags contain attributes: {}".format(tag_name, attrs) + ) + + +def assert_soup_has_tag_with_attributes_and_content(testcase, soup, tag_name, content, attrs): + """ + Using a ``TestCase`` object ``testcase``, assert that the passed + in ``BeatufulSoup`` object ``soup`` contains a tag ``tag_name`` + (unicode) which has all the attributes in ``attrs`` (dict) and + contains the string ``content`` (unicode). + """ + assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs) + assert_soup_has_tag_with_content(testcase, soup, tag_name, content) + + +def _normalized_contents(tag): + """ + :returns: all the text contents of the tag with whitespace + normalized: all newlines removed and at most one space between + words. + """ + return u" ".join(tag.text.split()) + + +def assert_soup_has_tag_with_content(testcase, soup, tag_name, content): + """ + Using a ``TestCase`` object ``testcase``, assert that the passed + in ``BeatufulSoup`` object ``soup`` contains a tag ``tag_name`` + (unicode) which contains the string ``content`` (unicode). + """ + tags = soup.find_all(tag_name) + for tag in tags: + if content in tag.contents: + return + + # make these "fuzzy" options? + for c in tag.contents: + if content in c: + return + + if content in _normalized_contents(tag): + return + testcase.fail( + u"No <{}> tag contains the text '{}'".format(tag_name, content) + ) + + +def assert_soup_has_text(testcase, soup, text): + """ + Using a ``TestCase`` object ``testcase``, assert that the passed in + ``BeautifulSoup`` object ``soup`` contains the passed in ``text`` anywhere + as a text node. + """ + testcase.assert_( + soup.find_all(string=re.compile(re.escape(text))), + soup) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 33a6197c6..2b953b82c 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -3,7 +3,10 @@ from __future__ import print_function import os.path, re, urllib import json from six.moves import StringIO -from nevow import rend + +from bs4 import BeautifulSoup + +from twisted.web import resource from twisted.trial import unittest from allmydata import uri, dirnode from allmydata.util import base32 @@ -24,8 +27,9 @@ DIR_HTML_TAG = '' class CompletelyUnhandledError(Exception): pass -class ErrorBoom(rend.Page): - def beforeRender(self, ctx): + +class ErrorBoom(object, resource.Resource): + def render(self, req): raise CompletelyUnhandledError("whoops") class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): @@ -325,8 +329,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _stash_root_and_create_file(n): self.rootnode = n - self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/" - self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/" + self.rooturl = "uri/" + urllib.quote(n.get_uri()) + self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) if not immutable: return self.rootnode.set_node(name, future_node) d.addCallback(_stash_root_and_create_file) @@ -386,7 +390,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.GET(expected_info_url)) d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False) - d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name)))) + d.addCallback(lambda ign: self.GET("%s/%s?t=info" % (self.rooturl, str(name)))) d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True) def _check_json(res, expect_rw_uri): @@ -410,7 +414,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # TODO: check metadata contents self.failUnlessIn("metadata", data[1]) - d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name)))) + d.addCallback(lambda ign: self.GET("%s/%s?t=json" % (self.rooturl, str(name)))) d.addCallback(_check_json, expect_rw_uri=not immutable) # and make sure that a read-only version of the directory can be @@ -425,7 +429,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.GET(self.rourl+"?t=json")) d.addCallback(_check_directory_json, expect_rw_uri=False) - d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name)))) + d.addCallback(lambda ign: self.GET("%s/%s?t=json" % (self.rourl, str(name)))) d.addCallback(_check_json, expect_rw_uri=False) # TODO: check that getting t=info from the Info link in the ro directory @@ -492,7 +496,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("CHK", cap.to_string()) self.cap = cap self.rootnode = dn - self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/" + self.rooturl = "uri/" + urllib.quote(dn.get_uri()) return download_to_data(dn._node) d.addCallback(_created) @@ -534,19 +538,28 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # Make sure the lonely child can be listed in HTML... d.addCallback(lambda ign: self.GET(self.rooturl)) def _check_html(res): + soup = BeautifulSoup(res, 'html5lib') self.failIfIn("URI:SSK", res) - get_lonely = "".join([r'FILE', - r'\s+', - r'lonely' % (urllib.quote(lonely_uri),), - r'', - r'\s+%d' % len("one"), - ]) - self.failUnless(re.search(get_lonely, res), res) + found = False + for td in soup.find_all(u"td"): + if td.text != u"FILE": + continue + a = td.findNextSibling()(u"a")[0] + self.assertIn(urllib.quote(lonely_uri), a[u"href"]) + self.assertEqual(u"lonely", a.text) + self.assertEqual(a[u"rel"], [u"noreferrer"]) + self.assertEqual(u"{}".format(len("one")), td.findNextSibling().findNextSibling().text) + found = True + break + self.assertTrue(found) - # find the More Info link for name, should be relative - mo = re.search(r'More Info', res) - info_url = mo.group(1) - self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url) + infos = list( + a[u"href"] + for a in soup.find_all(u"a") + if a.text == u"More Info" + ) + self.assertEqual(1, len(infos)) + self.assertTrue(infos[0].endswith(urllib.quote(lonely_uri) + "?t=info")) d.addCallback(_check_html) # ... and in JSON. @@ -573,7 +586,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/" + self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) return n.add_file(u"good", upload.Data(DATA, convergence="")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): @@ -747,7 +760,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/" + self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) return n.add_file(u"good", upload.Data(DATA, convergence="")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): @@ -960,7 +973,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _stash_root_and_create_file(n): self.rootnode = n self.uris["root"] = n.get_uri() - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/" + self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) return n.add_file(u"one", upload.Data(DATA, convergence="")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): @@ -1027,8 +1040,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi DATA = "data" * 100 d = c0.create_dirnode() def _stash_root(n): - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/" - self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary" + self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + self.fileurls["imaginary"] = self.fileurls["root"] + "/imaginary" return n d.addCallback(_stash_root) d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence=""))) @@ -1044,14 +1057,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: c0.create_dirnode()) def _mangle_dirnode_1share(n): u = n.get_uri() - url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/" + url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) self.fileurls["dir-1share-json"] = url + "?t=json" self.delete_shares_numbered(u, range(1,10)) d.addCallback(_mangle_dirnode_1share) d.addCallback(lambda ign: c0.create_dirnode()) def _mangle_dirnode_0share(n): u = n.get_uri() - url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/" + url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) self.fileurls["dir-0share-json"] = url + "?t=json" self.delete_shares_numbered(u, range(0,10)) d.addCallback(_mangle_dirnode_0share) @@ -1330,8 +1343,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.dir_si_b32 = base32.b2a(dn.get_storage_index()) self.dir_url_base = "uri/"+dn.get_write_uri() self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json" - self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json" - self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json" + self.dir_url_json2 = "uri/"+dn.get_write_uri()+"?t=json" + self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"?t=json" self.child_url = "uri/"+dn.get_readonly_uri()+"/child" d.addCallback(_get_dircap) d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True)) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 7b14e46e7..55db61a13 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,3 +1,4 @@ +from bs4 import BeautifulSoup from os.path import join from twisted.trial import unittest from twisted.internet import reactor @@ -6,13 +7,15 @@ from twisted.internet import defer from allmydata.introducer import create_introducer from allmydata import node from .common import ( - FAVICON_MARKUP, + assert_soup_has_favicon, + assert_soup_has_text, ) from ..common import ( SameProcessStreamEndpointAssigner, ) from ..common_web import do_http + class IntroducerWeb(unittest.TestCase): def setUp(self): self.node = None @@ -47,7 +50,8 @@ class IntroducerWeb(unittest.TestCase): url = "http://localhost:%d/" % self.ws.getPortnum() res = yield do_http("get", url) - self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res) - self.failUnlessIn(FAVICON_MARKUP, res) - self.failUnlessIn('Page rendered at', res) - self.failUnlessIn('Tahoe-LAFS code imported from:', res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_text(self, soup, u'Welcome to the Tahoe-LAFS Introducer') + assert_soup_has_favicon(self, soup) + assert_soup_has_text(self, soup, u'Page rendered at') + assert_soup_has_text(self, soup, u'Tahoe-LAFS code imported from:') diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 81b531113..5a888587c 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,38 +1,110 @@ +from mock import Mock + +import time + from twisted.trial import unittest +from twisted.web.template import Tag +from twisted.web.test.requesthelper import DummyRequest +from twisted.application import service -from ...storage_client import NativeStorageServer -from ...web.root import Root +from ...storage_client import ( + NativeStorageServer, + StorageFarmBroker, +) +from ...web.root import RootElement from ...util.connection_status import ConnectionStatus +from allmydata.web.root import URIHandler +from allmydata.web.common import WebError +from allmydata.client import _Client -class FakeRoot(Root): - def __init__(self): - pass - def now_fn(self): - return 0 +from hypothesis import given +from hypothesis.strategies import text + + +from ..common import ( + EMPTY_CLIENT_CONFIG, +) + +class RenderSlashUri(unittest.TestCase): + """ + Ensure that URIs starting with /uri?uri= only accept valid + capabilities + """ + + def setUp(self): + self.request = DummyRequest(b"/uri") + self.request.fields = {} + + def prepathURL(): + return b"http://127.0.0.1.99999/" + b"/".join(self.request.prepath) + + self.request.prePathURL = prepathURL + self.client = Mock() + self.res = URIHandler(self.client) + + def test_valid(self): + """ + A valid capbility does not result in error + """ + self.request.args[b"uri"] = [( + b"URI:CHK:nt2xxmrccp7sursd6yh2thhcky:" + b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882" + )] + self.res.render_GET(self.request) + + def test_invalid(self): + """ + A (trivially) invalid capbility is an error + """ + self.request.args[b"uri"] = [b"not a capability"] + with self.assertRaises(WebError): + self.res.render_GET(self.request) + + @given( + text() + ) + def test_hypothesis_error_caps(self, cap): + """ + Let hypothesis try a bunch of invalid capabilities + """ + self.request.args[b"uri"] = [cap.encode('utf8')] + with self.assertRaises(WebError): + self.res.render_GET(self.request) -class FakeContext: - def __init__(self): - self.slots = {} - self.tag = self - def fillSlots(self, slotname, contents): - self.slots[slotname] = contents class RenderServiceRow(unittest.TestCase): def test_missing(self): - # minimally-defined static servers just need anonymous-storage-FURL - # and permutation-seed-base32. The WUI used to have problems - # rendering servers that lacked nickname and version. This tests that - # we can render such minimal servers. + """ + minimally-defined static servers just need anonymous-storage-FURL + and permutation-seed-base32. The WUI used to have problems + rendering servers that lacked nickname and version. This tests that + we can render such minimal servers. + """ ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - s = NativeStorageServer("server_id", ann, None, {}, {}, None) - cs = ConnectionStatus(False, "summary", {}, 0, 0) - s.get_connection_status = lambda: cs + srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) - r = FakeRoot() - ctx = FakeContext() - res = r.render_service_row(ctx, s) - self.assertIdentical(res, ctx) - self.assertEqual(ctx.slots["version"], "") - self.assertEqual(ctx.slots["nickname"], "") + class FakeClient(_Client): + def __init__(self): + service.MultiService.__init__(self) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) + self.storage_broker.test_add_server("test-srv", srv) + + root = RootElement(FakeClient(), time.time) + req = DummyRequest(b"") + tag = Tag(b"") + + # Pick all items from services table. + items = root.services_table(req, tag).item(req, tag) + + # Coerce `items` to list and pick the first item from it. + item = list(items)[0] + + self.assertEqual(item.slotData.get("version"), "") + self.assertEqual(item.slotData.get("nickname"), "") diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 120c59c31..5be0b2f7b 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -3,10 +3,10 @@ from __future__ import print_function import os.path, re, urllib, time, cgi import json import treq -import mock + +from bs4 import BeautifulSoup from twisted.application import service -from twisted.trial import unittest from twisted.internet import defer from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred from twisted.internet.task import Clock @@ -33,16 +33,30 @@ from allmydata.immutable import upload from allmydata.immutable.downloader.status import DownloadStatus from allmydata.dirnode import DirectoryNode from allmydata.nodemaker import NodeMaker -from allmydata.frontends.magic_folder import QueuedItem from allmydata.web import status from allmydata.web.common import WebError, MultiFormatPage from allmydata.util import fileutil, base32, hashutil from allmydata.util.consumer import download_to_data from allmydata.util.encodingutil import to_str from ...util.connection_status import ConnectionStatus -from ..common import FakeCHKFileNode, FakeMutableFileNode, \ - create_chk_filenode, WebErrorMixin, \ - make_mutable_file_uri, create_mutable_filenode +from ..common import ( + EMPTY_CLIENT_CONFIG, + FakeCHKFileNode, + FakeMutableFileNode, + create_chk_filenode, + WebErrorMixin, + make_mutable_file_uri, + create_mutable_filenode, + TrialTestCase, +) +from .common import ( + assert_soup_has_favicon, + assert_soup_has_text, + assert_soup_has_tag_with_attributes, + assert_soup_has_tag_with_content, + assert_soup_has_tag_with_attributes_and_content, +) + from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION from allmydata.mutable import servermap, publish, retrieve from .. import common_util as testutil @@ -52,12 +66,11 @@ from ..common_web import ( ) from allmydata.client import _Client, SecretHolder from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP -from ..status import FakeStatus # create a fake uploader/downloader, and a couple of fake dirnodes, then # create a webserver that works against them -class FakeStatsProvider: +class FakeStatsProvider(object): def get_stats(self): stats = {'stats': {}, 'counters': {}} return stats @@ -113,29 +126,6 @@ class FakeUploader(service.Service): return (self.helper_furl, self.helper_connected) -def create_test_queued_item(relpath_u, history=[]): - progress = mock.Mock() - progress.progress = 100.0 - item = QueuedItem(relpath_u, progress, 1234) - for the_status, timestamp in history: - item.set_status(the_status, current_time=timestamp) - return item - - -class FakeMagicFolder(object): - def __init__(self): - self.uploader = FakeStatus() - self.downloader = FakeStatus() - - def get_public_status(self): - return ( - True, - [ - 'a magic-folder status message' - ], - ) - - def build_one_ds(): ds = DownloadStatus("storage_index", 1234) now = time.time() @@ -179,7 +169,7 @@ def build_one_ds(): return ds -class FakeHistory: +class FakeHistory(object): _all_upload_status = [upload.UploadStatus()] _all_download_status = [build_one_ds()] _all_mapupdate_statuses = [servermap.UpdateStatus()] @@ -270,7 +260,6 @@ class FakeClient(_Client): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset service.MultiService.__init__(self) - self._magic_folders = dict() self.all_contents = {} self.nodeid = "fake_nodeid" self.nickname = u"fake_nickname \u263A" @@ -280,7 +269,11 @@ class FakeClient(_Client): self._secret_holder = SecretHolder("lease secret", "convergence secret") self.helper = None self.convergence = "some random string" - self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) # fake knowledge of another server self.storage_broker.test_add_server("other_nodeid", FakeDisplayableServer( @@ -663,7 +656,7 @@ class WebMixin(testutil.TimezoneMixin): -class MultiFormatPageTests(unittest.TestCase): +class MultiFormatPageTests(TrialTestCase): """ Tests for ``MultiFormatPage``. """ @@ -772,7 +765,7 @@ class MultiFormatPageTests(unittest.TestCase): -class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase): +class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, TrialTestCase): maxDiff = None def test_create(self): @@ -825,64 +818,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_check) return d - def test_welcome(self): - d = self.GET("/") - def _check(res): - # TODO: replace this with a parser - self.failUnlessIn('Tahoe-LAFS - Welcome', res) - self.failUnlessIn(FAVICON_MARKUP, res) - self.failUnlessIn('Recent and Active Operations', res) - self.failUnlessIn('Operational Statistics', res) - self.failUnless(re.search('',res), res) - self.failUnlessIn('Page rendered at', res) - self.failUnlessIn('Tahoe-LAFS code imported from:', res) - res_u = res.decode('utf-8') - self.failUnlessIn(u'fake_nickname \u263A', res_u) - self.failUnlessIn(u'
    other_nickname \u263B
    ', res_u) - self.failUnlessIn(u'Connected to 1\n of 2 known storage servers', res_u) - def timestamp(t): - return (u'"%s"' % (t,)) if self.have_working_tzset() else u'"[^"]*"' - - # TODO: use a real parser to make sure these two nodes are siblings - self.failUnless(re.search( - u'
    ' - u'\s+' - u'
    other_nickname \u263B
    ', - res_u), repr(res_u)) - self.failUnless(re.search( - u'\s+1d\u00A00h\u00A00m\u00A050s\s+' - % timestamp(u'1970-01-01 13:00:10'), res_u), repr(res_u)) - - # same for these two nodes - self.failUnless(re.search( - u'
    ' - u'\s+' - u'
    disconnected_nickname \u263B
    ', - res_u), repr(res_u)) - self.failUnless(re.search( - u'\s+N/A\s+', - res_u), repr(res_u)) - - self.failUnless(re.search( - u'' - u'1d\u00A00h\u00A00m\u00A030s' - % timestamp(u'1970-01-01 13:00:30'), res_u), repr(res_u)) - self.failUnless(re.search( - u'' - u'1d\u00A00h\u00A00m\u00A025s' - % timestamp(u'1970-01-01 13:00:35'), res_u), repr(res_u)) - - self.failUnlessIn(u'\u00A9 Tahoe-LAFS Software Foundation', res_u) - self.failUnlessIn('

    Available

    ', res) - self.failUnlessIn('123.5kB', res) - - self.s.basedir = 'web/test_welcome' - fileutil.make_dirs("web/test_welcome") - fileutil.make_dirs("web/test_welcome/private") - return self.GET("/") - d.addCallback(_check) - return d - def test_introducer_status(self): class MockIntroducerClient(object): def __init__(self, connected): @@ -899,10 +834,16 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return self.GET("/") d.addCallback(_set_introducer_not_connected_unguessable) def _check_introducer_not_connected_unguessable(res): - html = res.replace('\n', ' ') - self.failIfIn('pb://someIntroducer/secret', html) - self.failUnless(re.search('[ ]*
    No introducers connected
    ', html), res) - + soup = BeautifulSoup(res, 'html5lib') + self.failIfIn('pb://someIntroducer/secret', res) + assert_soup_has_tag_with_attributes( + self, soup, u"img", + {u"alt": u"Disconnected", u"src": u"img/connected-no.png"} + ) + assert_soup_has_tag_with_content( + self, soup, u"div", + u"No introducers connected" + ) d.addCallback(_check_introducer_not_connected_unguessable) # introducer connected, unguessable furl @@ -912,10 +853,21 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return self.GET("/") d.addCallback(_set_introducer_connected_unguessable) def _check_introducer_connected_unguessable(res): - html = res.replace('\n', ' ') - self.failUnlessIn('
    summary
    ', html) - self.failIfIn('pb://someIntroducer/secret', html) - self.failUnless(re.search('[ ]*
    1 introducer connected
    ', html), res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"div", + u"summary", + { u"class": u"connection-status", u"title": u"(no other hints)" } + ) + self.failIfIn('pb://someIntroducer/secret', res) + assert_soup_has_tag_with_attributes( + self, soup, u"img", + { u"alt": u"Connected", u"src": u"img/connected-yes.png" } + ) + assert_soup_has_tag_with_content( + self, soup, u"div", + u"1 introducer connected" + ) d.addCallback(_check_introducer_connected_unguessable) # introducer connected, guessable furl @@ -925,9 +877,20 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return self.GET("/") d.addCallback(_set_introducer_connected_guessable) def _check_introducer_connected_guessable(res): - html = res.replace('\n', ' ') - self.failUnlessIn('
    summary
    ', html) - self.failUnless(re.search('[ ]*
    1 introducer connected
    ', html), res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"div", + u"summary", + { u"class": u"connection-status", u"title": u"(no other hints)" } + ) + assert_soup_has_tag_with_attributes( + self, soup, u"img", + { u"src": u"img/connected-yes.png", u"alt": u"Connected" } + ) + assert_soup_has_tag_with_content( + self, soup, u"div", + u"1 introducer connected" + ) d.addCallback(_check_introducer_connected_guessable) return d @@ -940,8 +903,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return self.GET("/") d.addCallback(_set_no_helper) def _check_no_helper(res): - html = res.replace('\n', ' ') - self.failUnless(re.search('', html), res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_tag_with_attributes( + self, soup, u"img", + { u"src": u"img/connected-not-configured.png", u"alt": u"Not Configured" } + ) d.addCallback(_check_no_helper) # enable helper, not connected @@ -951,10 +917,17 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return self.GET("/") d.addCallback(_set_helper_not_connected) def _check_helper_not_connected(res): - html = res.replace('\n', ' ') - self.failUnlessIn('
    pb://someHelper/[censored]
    ', html) - self.failIfIn('pb://someHelper/secret', html) - self.failUnless(re.search('', html), res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"div", + u"pb://someHelper/[censored]", + { u"class": u"furl" } + ) + self.failIfIn('pb://someHelper/secret', res) + assert_soup_has_tag_with_attributes( + self, soup, u"img", + { u"src": u"img/connected-no.png", u"alt": u"Disconnected" } + ) d.addCallback(_check_helper_not_connected) # enable helper, connected @@ -964,10 +937,17 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return self.GET("/") d.addCallback(_set_helper_connected) def _check_helper_connected(res): - html = res.replace('\n', ' ') - self.failUnlessIn('
    pb://someHelper/[censored]
    ', html) - self.failIfIn('pb://someHelper/secret', html) - self.failUnless(re.search('', html), res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"div", + u"pb://someHelper/[censored]", + { u"class": u"furl" } + ) + self.failIfIn('pb://someHelper/secret', res) + assert_soup_has_tag_with_attributes( + self, soup, u"img", + { u"src": u"img/connected-yes.png", "alt": u"Connected" } + ) d.addCallback(_check_helper_connected) return d @@ -981,79 +961,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_check) return d - @defer.inlineCallbacks - def test_magicfolder_status_bad_token(self): - with self.assertRaises(Error): - yield self.POST( - '/magic_folder?t=json', - t='json', - name='default', - token='not the token you are looking for', - ) - - @defer.inlineCallbacks - def test_magicfolder_status_wrong_folder(self): - with self.assertRaises(Exception) as ctx: - yield self.POST( - '/magic_folder?t=json', - t='json', - name='a non-existent magic-folder', - token=self.s.get_auth_token(), - ) - self.assertIn( - "Not Found", - str(ctx.exception) - ) - - @defer.inlineCallbacks - def test_magicfolder_status_success(self): - self.s._magic_folders['default'] = mf = FakeMagicFolder() - mf.uploader.status = [ - create_test_queued_item(u"rel/uppath", [('done', 12345)]) - ] - mf.downloader.status = [ - create_test_queued_item(u"rel/downpath", [('done', 23456)]) - ] - data = yield self.POST( - '/magic_folder?t=json', - t='json', - name='default', - token=self.s.get_auth_token(), - ) - data = json.loads(data) - self.assertEqual( - data, - [ - { - "status": "done", - "path": "rel/uppath", - "kind": "upload", - "percent_done": 100.0, - "done_at": 12345, - "size": 1234, - }, - { - "status": "done", - "path": "rel/downpath", - "kind": "download", - "percent_done": 100.0, - "done_at": 23456, - "size": 1234, - }, - ] - ) - - @defer.inlineCallbacks - def test_magicfolder_root_success(self): - self.s._magic_folders['default'] = mf = FakeMagicFolder() - mf.uploader.status = [ - create_test_queued_item(u"rel/path", [('done', 12345)]) - ] - data = yield self.GET( - '/', - ) - del data - def test_status(self): h = self.s.get_history() dl_num = h.list_all_download_statuses()[0].get_counter() @@ -1375,7 +1282,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_GET_FILE_URI_badchild(self): base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri) - errmsg = "Files have no children, certainly not named 'boguschild'" + errmsg = "Files have no children named 'boguschild'" d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild", "400 Bad Request", errmsg, self.GET, base) @@ -1385,7 +1292,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri) errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory" d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild", - "400 Bad Request", errmsg, + "409 Conflict", errmsg, self.PUT, base, "") return d @@ -1840,134 +1747,171 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addBoth(self.should404, "test_GET_FILEURL_uri_missing") return d - def _check_upload_and_mkdir_forms(self, html): - # We should have a form to create a file, with radio buttons that allow - # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file. - self.failUnless(re.search('', html), html) - self.failUnless(re.search('', html), html) - self.failUnless(re.search(' tags") + assert_soup_has_favicon(self, soup) + @inlineCallbacks def test_GET_DIRECTORY_html(self): - d = self.GET(self.public_url + "/foo", followRedirect=True) - def _check(html): - self.failUnlessIn('
  • Return to Welcome page
    • ', html) - self._check_upload_and_mkdir_forms(html) - self.failUnlessIn("quux", html) - d.addCallback(_check) - return d + data = yield self.GET(self.public_url + "/foo", followRedirect=True) + soup = BeautifulSoup(data, 'html5lib') + self._check_upload_and_mkdir_forms(soup) + toolbars = soup.find_all(u"li", {u"class": u"toolbar-item"}) + self.assertTrue(any(li.text == u"Return to Welcome page" for li in toolbars)) + self.failUnlessIn("quux", data) + @inlineCallbacks def test_GET_DIRECTORY_html_filenode_encoding(self): - d = self.GET(self.public_url + "/foo", followRedirect=True) - def _check(html): - # Check if encoded entries are there - self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '" rel="noreferrer">' - + self._htmlname_escaped + '', html) - self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html) - self.failIfIn(self._htmlname_escaped_double, html) - # Make sure that Nevow escaping actually works by checking for unsafe characters - # and that '&' is escaped. - for entity in '<>': - self.failUnlessIn(entity, self._htmlname_raw) - self.failIfIn(entity, self._htmlname_escaped) - self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw)) - self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped)) - d.addCallback(_check) - return d + data = yield self.GET(self.public_url + "/foo", followRedirect=True) + soup = BeautifulSoup(data, 'html5lib') + # Check if encoded entries are there + target_ref = u'@@named=/{}'.format(self._htmlname_urlencoded) + # at least one tag has our weirdly-named file properly + # encoded (or else BeautifulSoup would produce an error) + self.assertTrue( + any( + a.text == self._htmlname_unicode and a[u"href"].endswith(target_ref) + for a in soup.find_all(u"a", {u"rel": u"noreferrer"}) + ) + ) + # XXX leaving this as-is, but consider using beautfulsoup here too? + # Make sure that Nevow escaping actually works by checking for unsafe characters + # and that '&' is escaped. + for entity in '<>': + self.failUnlessIn(entity, self._htmlname_raw) + self.failIfIn(entity, self._htmlname_escaped) + self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw)) + self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped)) + + @inlineCallbacks def test_GET_root_html(self): - d = self.GET("/") - d.addCallback(self._check_upload_and_mkdir_forms) - return d + data = yield self.GET("/") + soup = BeautifulSoup(data, 'html5lib') + self._check_upload_and_mkdir_forms(soup) + @inlineCallbacks def test_GET_DIRURL(self): - # the addSlash means we get a redirect here + data = yield self.GET(self.public_url + "/foo", followRedirect=True) + soup = BeautifulSoup(data, 'html5lib') + # from /uri/$URI/foo/ , we need ../../../ to get back to the root - ROOT = "../../.." - d = self.GET(self.public_url + "/foo", followRedirect=True) - def _check(res): - self.failUnlessIn('Return to Welcome page' % ROOT, res) + root = u"../../.." + self.assertTrue( + any( + a.text == u"Return to Welcome page" + for a in soup.find_all(u"a", {u"href": root}) + ) + ) - # the FILE reference points to a URI, but it should end in bar.txt - bar_url = ("%s/file/%s/@@named=/bar.txt" % - (ROOT, urllib.quote(self._bar_txt_uri))) - get_bar = "".join([r'FILE', - r'\s+', - r'bar.txt' % bar_url, - r'', - r'\s+%d' % len(self.BAR_CONTENTS), - ]) - self.failUnless(re.search(get_bar, res), res) - for label in ['unlink', 'rename/relink']: - for line in res.split("\n"): - # find the line that contains the relevant button for bar.txt - if ("form action" in line and - ('value="%s"' % (label,)) in line and - 'value="bar.txt"' in line): - # the form target should use a relative URL - foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri)) - self.failUnlessIn('action="%s"' % foo_url, line) - # and the when_done= should too - #done_url = urllib.quote(???) - #self.failUnlessIn('name="when_done" value="%s"' % done_url, line) + # the FILE reference points to a URI, but it should end in bar.txt + bar_url = "{}/file/{}/@@named=/bar.txt".format(root, urllib.quote(self._bar_txt_uri)) + self.assertTrue( + any( + a.text == u"bar.txt" + for a in soup.find_all(u"a", {u"href": bar_url}) + ) + ) + self.assertTrue( + any( + td.text == u"{}".format(len(self.BAR_CONTENTS)) + for td in soup.find_all(u"td", {u"align": u"right"}) + ) + ) + foo_url = urllib.quote("{}/uri/{}/".format(root, self._foo_uri)) + forms = soup.find_all(u"form", {u"action": foo_url}) + found = [] + for form in forms: + if form.find_all(u"input", {u"name": u"name", u"value": u"bar.txt"}): + kind = form.find_all(u"input", {u"type": u"submit"})[0][u"value"] + found.append(kind) + if kind == u"unlink": + self.assertTrue(form[u"method"] == u"post") + self.assertEqual( + set(found), + {u"unlink", u"rename/relink"} + ) - # 'unlink' needs to use POST because it directly has a side effect - if label == 'unlink': - self.failUnlessIn('method="post"', line) - break - else: - self.fail("unable to find '%s bar.txt' line" % (label,)) - - # the DIR reference just points to a URI - sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri))) - get_sub = ((r'DIR') - +r'\s+sub' % sub_url) - self.failUnless(re.search(get_sub, res), res) - d.addCallback(_check) + sub_url = "{}/uri/{}/".format(root, urllib.quote(self._sub_uri)) + self.assertTrue( + any( + td.findNextSibling()(u"a")[0][u"href"] == sub_url + for td in soup.find_all(u"td") + if td.text == u"DIR" + ) + ) + @inlineCallbacks + def test_GET_DIRURL_readonly(self): # look at a readonly directory - d.addCallback(lambda res: - self.GET(self.public_url + "/reedownlee", followRedirect=True)) - def _check2(res): - self.failUnlessIn("(read-only)", res) - self.failIfIn("Upload a file", res) - d.addCallback(_check2) + data = yield self.GET(self.public_url + "/reedownlee", followRedirect=True) + self.failUnlessIn("(read-only)", data) + self.failIfIn("Upload a file", data) - # and at a directory that contains a readonly directory - d.addCallback(lambda res: - self.GET(self.public_url, followRedirect=True)) - def _check3(res): - self.failUnless(re.search('DIR-RO' - r'\s+reedownlee', res), res) - d.addCallback(_check3) + @inlineCallbacks + def test_GET_DIRURL_readonly_dir(self): + # look at a directory that contains a readonly directory + data = yield self.GET(self.public_url, followRedirect=True) + soup = BeautifulSoup(data, 'html5lib') + ro_links = list( + td.findNextSibling()(u"a")[0] + for td in soup.find_all(u"td") + if td.text == u"DIR-RO" + ) + self.assertEqual(1, len(ro_links)) + self.assertEqual(u"reedownlee", ro_links[0].text) + self.assertTrue(u"URI%3ADIR2-RO%3A" in ro_links[0][u"href"]) - # and an empty directory - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/")) - def _check4(res): - self.failUnlessIn("directory is empty", res) - MKDIR_BUTTON_RE=re.compile('.*Create a new directory in this directory.*', re.I) - self.failUnless(MKDIR_BUTTON_RE.search(res), res) - d.addCallback(_check4) + @inlineCallbacks + def test_GET_DIRURL_empty(self): + # look at an empty directory + data = yield self.GET(self.public_url + "/foo/empty") + soup = BeautifulSoup(data, 'html5lib') + self.failUnlessIn("directory is empty", data) + mkdir_inputs = soup.find_all(u"input", {u"type": u"hidden", u"name": u"t", u"value": u"mkdir"}) + self.assertEqual(1, len(mkdir_inputs)) + self.assertEqual( + u"Create a new directory in this directory", + mkdir_inputs[0].parent(u"legend")[0].text + ) - # and at a literal directory + @inlineCallbacks + def test_GET_DIRURL_literal(self): + # look at a literal directory tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT - d.addCallback(lambda res: - self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True)) - def _check5(res): - self.failUnlessIn('(immutable)', res) - self.failUnless(re.search('FILE' - r'\s+short', res), res) - d.addCallback(_check5) - return d + data = yield self.GET("/uri/" + tiny_litdir_uri, followRedirect=True) + soup = BeautifulSoup(data, 'html5lib') + self.failUnlessIn('(immutable)', data) + file_links = list( + td.findNextSibling()(u"a")[0] + for td in soup.find_all(u"td") + if td.text == u"FILE" + ) + self.assertEqual(1, len(file_links)) + self.assertEqual(u"short", file_links[0].text) + self.assertTrue(file_links[0][u"href"].endswith(u"/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short")) @inlineCallbacks def test_GET_DIRURL_badtype(self): @@ -2020,7 +1964,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_DIRURL_manifest(self): d = defer.succeed(None) def getman(ignored, output): - url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=125" + url = self.webish_url + self.public_url + "/foo?t=start-manifest&ophandle=125" d = do_http("post", url, allow_redirects=True, browser_like_redirects=True) d.addCallback(self.wait_for_operation, "125") @@ -2028,11 +1972,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d d.addCallback(getman, None) def _got_html(manifest): - self.failUnlessIn("Manifest of SI=", manifest) - self.failUnlessIn("sub", manifest) - self.failUnlessIn(self._sub_uri, manifest) - self.failUnlessIn("sub/baz.txt", manifest) - self.failUnlessIn(FAVICON_MARKUP, manifest) + soup = BeautifulSoup(manifest, 'html5lib') + assert_soup_has_text(self, soup, "Manifest of SI=") + assert_soup_has_text(self, soup, "sub") + assert_soup_has_text(self, soup, self._sub_uri) + assert_soup_has_text(self, soup, "sub/baz.txt") + assert_soup_has_favicon(self, soup) d.addCallback(_got_html) # both t=status and unadorned GET should be identical @@ -2071,7 +2016,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_DIRURL_deepsize(self): - url = self.webish_url + self.public_url + "/foo/?t=start-deep-size&ophandle=126" + url = self.webish_url + self.public_url + "/foo?t=start-deep-size&ophandle=126" d = do_http("post", url, allow_redirects=True, browser_like_redirects=True) d.addCallback(self.wait_for_operation, "126") @@ -2100,7 +2045,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_DIRURL_deepstats(self): - url = self.webish_url + self.public_url + "/foo/?t=start-deep-stats&ophandle=127" + url = self.webish_url + self.public_url + "/foo?t=start-deep-stats&ophandle=127" d = do_http("post", url, allow_redirects=True, browser_like_redirects=True) d.addCallback(self.wait_for_operation, "127") @@ -2129,7 +2074,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_DIRURL_stream_manifest(self): - d = self.POST(self.public_url + "/foo/?t=stream-manifest") + d = self.POST(self.public_url + "/foo?t=stream-manifest") def _check(res): self.failUnless(res.endswith("\n")) units = [json.loads(t) for t in res[:-1].split("\n")] @@ -2791,7 +2736,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi # slightly differently d.addCallback(lambda res: - self.GET(self.public_url + "/foo/", + self.GET(self.public_url + "/foo", followRedirect=True)) def _check_page(res): # TODO: assert more about the contents @@ -2809,7 +2754,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi # look at the JSON form of the enclosing directory d.addCallback(lambda res: - self.GET(self.public_url + "/foo/?t=json", + self.GET(self.public_url + "/foo?t=json", followRedirect=True)) def _check_page_json(res): parsed = json.loads(res) @@ -2947,7 +2892,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi body, headers = self.build_form(t="upload", when_done="/THERE", file=("new.txt", self.NEWFILE_CONTENTS)) yield self.shouldRedirectTo(self.webish_url + self.public_url + "/foo", - self.webish_url + "/THERE", + "/THERE", method="post", data=body, headers=headers, code=http.FOUND) fn = self._foo_node @@ -3023,7 +2968,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_POST_DIRURL_check(self): - foo_url = self.public_url + "/foo/" + foo_url = self.public_url + "/foo" res = yield self.POST(foo_url, t="check") self.failUnlessIn("Healthy :", res) @@ -3045,7 +2990,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_POST_DIRURL_check_and_repair(self): - foo_url = self.public_url + "/foo/" + foo_url = self.public_url + "/foo" res = yield self.POST(foo_url, t="check", repair="true") self.failUnlessIn("Healthy :", res) @@ -3601,7 +3546,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi body, headers = self.build_form(t="mkdir", name="newdir", when_done="/THERE") yield self.shouldRedirectTo(self.webish_url + self.public_url + "/foo", - self.webish_url + "/THERE", + "/THERE", method="post", data=body, headers=headers, code=http.FOUND) res = yield self._foo_node.get(u"newdir") @@ -3611,7 +3556,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_mkdir_whendone_queryarg(self): body, headers = self.build_form(t="mkdir", name="newdir") url = self.webish_url + self.public_url + "/foo?when_done=/THERE" - yield self.shouldRedirectTo(url, self.webish_url + "/THERE", + yield self.shouldRedirectTo(url, "/THERE", method="post", data=body, headers=headers, code=http.FOUND) res = yield self._foo_node.get(u"newdir") @@ -4160,15 +4105,22 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.GET, "/uri") return d + @inlineCallbacks def test_GET_rename_form(self): - d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt", - followRedirect=True) - def _check(res): - self.failUnless(re.search('', res), res) - self.failUnless(re.search(r'', res), res) - self.failUnlessIn(FAVICON_MARKUP, res) - d.addCallback(_check) - return d + data = yield self.GET( + self.public_url + "/foo?t=rename-form&name=bar.txt", + followRedirect=True + ) + soup = BeautifulSoup(data, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_attributes( + self, soup, u"input", + {u"name": u"when_done", u"value": u".", u"type": u"hidden"}, + ) + assert_soup_has_tag_with_attributes( + self, soup, u"input", + {u"readonly": u"true", u"name": u"from_name", u"value": u"bar.txt", u"type": u"text"}, + ) def log(self, res, msg): #print "MSG: %s RES: %s" % (msg, res) @@ -4515,13 +4467,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_ophandle_cancel(self): - url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=128" + url = self.webish_url + self.public_url + "/foo?t=start-manifest&ophandle=128" yield do_http("post", url, allow_redirects=True, browser_like_redirects=True) res = yield self.GET("/operations/128?t=status&output=JSON") data = json.loads(res) self.failUnless("finished" in data, res) - monitor = self.ws.root.child_operations.handles["128"][0] + monitor = self.ws.getServiceNamed("operations").handles["128"][0] res = yield self.POST("/operations/128?t=cancel&output=JSON") data = json.loads(res) @@ -4534,7 +4486,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_ophandle_retainfor(self): - url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60" + url = self.webish_url + self.public_url + "/foo?t=start-manifest&ophandle=129&retain-for=60" yield do_http("post", url, allow_redirects=True, browser_like_redirects=True) res = yield self.GET("/operations/129?t=status&output=JSON&retain-for=0") @@ -4548,7 +4500,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_ophandle_release_after_complete(self): - url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=130" + url = self.webish_url + self.public_url + "/foo?t=start-manifest&ophandle=130" yield do_http("post", url, allow_redirects=True, browser_like_redirects=True) yield self.wait_for_operation(None, "130") @@ -4562,7 +4514,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi # uncollected ophandles should expire after 4 days def _make_uncollected_ophandle(ophandle): url = (self.webish_url + self.public_url + - "/foo/?t=start-manifest&ophandle=%d" % ophandle) + "/foo?t=start-manifest&ophandle=%d" % ophandle) # When we start the operation, the webapi server will want to # redirect us to the page for the ophandle, so we get # confirmation that the operation has started. If the manifest @@ -4600,7 +4552,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi # collected ophandles should expire after 1 day def _make_collected_ophandle(ophandle): url = (self.webish_url + self.public_url + - "/foo/?t=start-manifest&ophandle=%d" % ophandle) + "/foo?t=start-manifest&ophandle=%d" % ophandle) # By following the initial redirect, we collect the ophandle # we've just created. return do_http("post", url, diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 8d462d537..051b45f79 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -707,7 +707,7 @@ class ImmutableDirectoryURIVerifier(DirectoryURIVerifier): INNER_URI_CLASS=CHKFileVerifierURI -class UnknownURI: +class UnknownURI(object): def __init__(self, uri, error=None): self._uri = uri self._error = error diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index fb8cb6352..2017aaed4 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -1,4 +1,5 @@ # from the Python Standard Library +import six import string from allmydata.util.assertutil import precondition @@ -179,13 +180,13 @@ def init_s5(): s5 = init_s5() def could_be_base32_encoded(s, s8=s8, tr=string.translate, identitytranstable=identitytranstable, chars=chars): - precondition(isinstance(s, str), s) + precondition(isinstance(s, six.binary_type), s) if s == '': return True return s8[len(s)%8][ord(s[-1])] and not tr(s, identitytranstable, chars) def could_be_base32_encoded_l(s, lengthinbits, s5=s5, tr=string.translate, identitytranstable=identitytranstable, chars=chars): - precondition(isinstance(s, str), s) + precondition(isinstance(s, six.binary_type), s) if s == '': return True assert lengthinbits%5 < len(s5), lengthinbits @@ -201,7 +202,7 @@ def a2b(cs): @param cs the base-32 encoded data (a string) """ precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs) - precondition(isinstance(cs, str), cs) + precondition(isinstance(cs, six.binary_type), cs) return a2b_l(cs, num_octets_that_encode_to_this_many_quintets(len(cs))*8) @@ -226,7 +227,7 @@ def a2b_l(cs, lengthinbits): @return the data encoded in cs """ precondition(could_be_base32_encoded_l(cs, lengthinbits), "cs is required to be possibly base32 encoded data.", cs=cs, lengthinbits=lengthinbits) - precondition(isinstance(cs, str), cs) + precondition(isinstance(cs, six.binary_type), cs) if cs == '': return '' diff --git a/src/allmydata/util/cachedir.py b/src/allmydata/util/cachedir.py index c4902c3ed..118a8a7eb 100644 --- a/src/allmydata/util/cachedir.py +++ b/src/allmydata/util/cachedir.py @@ -35,7 +35,7 @@ class CacheDirectoryManager(service.MultiService): if now - mtime > self.old: os.remove(absfn) -class CacheFile: +class CacheFile(object): def __init__(self, absfn): self.filename = absfn diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 0e22767cb..d58bc4217 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -1,6 +1,7 @@ from ConfigParser import SafeConfigParser +import attr class UnknownConfigError(Exception): """ @@ -36,28 +37,82 @@ def write_config(tahoe_cfg, config): finally: f.close() -def validate_config(fname, cfg, valid_sections): +def validate_config(fname, cfg, valid_config): """ - raises UnknownConfigError if there are any unknown sections or config - values. + :param ValidConfiguration valid_config: The definition of a valid + configuration. + + :raises UnknownConfigError: if there are any unknown sections or config + values. """ for section in cfg.sections(): - try: - valid_in_section = valid_sections[section] - except KeyError: + if not valid_config.is_valid_section(section): raise UnknownConfigError( "'{fname}' contains unknown section [{section}]".format( fname=fname, section=section, ) ) - if valid_in_section is not None: - for option in cfg.options(section): - if option not in valid_in_section: - raise UnknownConfigError( - "'{fname}' section [{section}] contains unknown option '{option}'".format( - fname=fname, - section=section, - option=option, - ) + for option in cfg.options(section): + if not valid_config.is_valid_item(section, option): + raise UnknownConfigError( + "'{fname}' section [{section}] contains unknown option '{option}'".format( + fname=fname, + section=section, + option=option, ) + ) + + +@attr.s +class ValidConfiguration(object): + """ + :ivar dict[bytes, tuple[bytes]] _static_valid_sections: A mapping from + valid section names to valid items in those sections. + + :ivar _is_valid_section: A callable which accepts a section name as bytes + and returns True if that section name is valid, False otherwise. + + :ivar _is_valid_item: A callable which accepts a section name as bytes and + an item name as bytes and returns True if that section, item pair is + valid, False otherwise. + """ + _static_valid_sections = attr.ib() + _is_valid_section = attr.ib(default=lambda section_name: False) + _is_valid_item = attr.ib(default=lambda section_name, item_name: False) + + def is_valid_section(self, section_name): + """ + :return: True if the given section name is valid, False otherwise. + """ + return ( + section_name in self._static_valid_sections or + self._is_valid_section(section_name) + ) + + def is_valid_item(self, section_name, item_name): + """ + :return: True if the given section name, ite name pair is valid, False + otherwise. + """ + return ( + item_name in self._static_valid_sections.get(section_name, ()) or + self._is_valid_item(section_name, item_name) + ) + + + def update(self, valid_config): + static_valid_sections = self._static_valid_sections.copy() + static_valid_sections.update(valid_config._static_valid_sections) + return ValidConfiguration( + static_valid_sections, + _either(self._is_valid_section, valid_config._is_valid_section), + _either(self._is_valid_item, valid_config._is_valid_item), + ) + + +def _either(f, g): + """ + :return: A function which returns True if either f or g returns True. + """ + return lambda *a, **kw: f(*a, **kw) or g(*a, **kw) diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index 3f5dd5278..44c12f220 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -12,6 +12,20 @@ class ConnectionStatus(object): self.last_connection_time = last_connection_time self.last_received_time = last_received_time + @classmethod + def unstarted(cls): + """ + Create a ``ConnectionStatus`` representing a connection for which no + attempts have yet been made. + """ + return cls( + connected=False, + summary=u"unstarted", + non_connected_statuses=[], + last_connection_time=None, + last_received_time=None, + ) + def _hint_statuses(which, handlers, statuses): non_connected_statuses = {} for hint in which: @@ -23,10 +37,12 @@ def _hint_statuses(which, handlers, statuses): def from_foolscap_reconnector(rc, last_received): ri = rc.getReconnectionInfo() + # See foolscap/reconnector.py, ReconnectionInfo, for details about + # possible states. state = ri.state - # the Reconnector shouldn't even be exposed until it is started, so we - # should never see "unstarted" - assert state in ("connected", "connecting", "waiting"), state + if state == "unstarted": + return ConnectionStatus.unstarted() + ci = ri.connectionInfo connected = False last_connected = None diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index d48faa0c7..671d8bfd9 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -102,7 +102,7 @@ def eventual_chain(source, target): source.addCallbacks(eventually_callback(target), eventually_errback(target)) -class HookMixin: +class HookMixin(object): """ I am a helper mixin that maintains a collection of named hooks, primarily for use in tests. Each hook is set to an unfired Deferred using 'set_hook', diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 7b36a27d1..f6f40945d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -16,15 +16,6 @@ __all__ = [ "opt_help_eliot_destinations", "validateInstanceOf", "validateSetMembership", - "MAYBE_NOTIFY", - "CALLBACK", - "INOTIFY_EVENTS", - "RELPATH", - "VERSION", - "LAST_UPLOADED_URI", - "LAST_DOWNLOADED_URI", - "LAST_DOWNLOADED_TIMESTAMP", - "PATHINFO", ] from sys import ( @@ -51,8 +42,6 @@ from attr.validators import ( from eliot import ( ILogger, Message, - Field, - ActionType, FileDestination, add_destinations, remove_destination, @@ -86,14 +75,6 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service - -from .fileutil import ( - PathInfo, -) -from .fake_inotify import ( - humanReadableMask, -) - def validateInstanceOf(t): """ Return an Eliot validator that requires values to be instances of ``t``. @@ -112,72 +93,6 @@ def validateSetMembership(s): raise ValidationError("{} not in {}".format(v, s)) return validator -RELPATH = Field.for_types( - u"relpath", - [unicode], - u"The relative path of a file in a magic-folder.", -) - -VERSION = Field.for_types( - u"version", - [int, long], - u"The version of the file.", -) - -LAST_UPLOADED_URI = Field.for_types( - u"last_uploaded_uri", - [unicode, bytes, None], - u"The filecap to which this version of this file was uploaded.", -) - -LAST_DOWNLOADED_URI = Field.for_types( - u"last_downloaded_uri", - [unicode, bytes, None], - u"The filecap from which the previous version of this file was downloaded.", -) - -LAST_DOWNLOADED_TIMESTAMP = Field.for_types( - u"last_downloaded_timestamp", - [float, int, long], - u"(XXX probably not really, don't trust this) The timestamp of the last download of this file.", -) - -PATHINFO = Field( - u"pathinfo", - lambda v: None if v is None else { - "isdir": v.isdir, - "isfile": v.isfile, - "islink": v.islink, - "exists": v.exists, - "size": v.size, - "mtime_ns": v.mtime_ns, - "ctime_ns": v.ctime_ns, - }, - u"The metadata for this version of this file.", - validateInstanceOf((type(None), PathInfo)), -) - -INOTIFY_EVENTS = Field( - u"inotify_events", - humanReadableMask, - u"Details about a filesystem event generating a notification event.", - validateInstanceOf((int, long)), -) - -MAYBE_NOTIFY = ActionType( - u"filesystem:notification:maybe-notify", - [], - [], - u"A filesystem event is being considered for dispatch to an application handler.", -) - -CALLBACK = ActionType( - u"filesystem:notification:callback", - [INOTIFY_EVENTS], - [], - u"A filesystem event is being dispatched to an application callback." -) - def eliot_logging_service(reactor, destinations): """ Parse the given Eliot destination descriptions and return an ``IService`` diff --git a/src/allmydata/util/fake_inotify.py b/src/allmydata/util/fake_inotify.py deleted file mode 100644 index c6d0b16e2..000000000 --- a/src/allmydata/util/fake_inotify.py +++ /dev/null @@ -1,109 +0,0 @@ - -# Most of this is copied from Twisted 11.0. The reason for this hack is that -# twisted.internet.inotify can't be imported when the platform does not support inotify. - -import six - -if six.PY3: - long = int - -# from /usr/src/linux/include/linux/inotify.h - -IN_ACCESS = long(0x00000001) # File was accessed -IN_MODIFY = long(0x00000002) # File was modified -IN_ATTRIB = long(0x00000004) # Metadata changed -IN_CLOSE_WRITE = long(0x00000008) # Writeable file was closed -IN_CLOSE_NOWRITE = long(0x00000010) # Unwriteable file closed -IN_OPEN = long(0x00000020) # File was opened -IN_MOVED_FROM = long(0x00000040) # File was moved from X -IN_MOVED_TO = long(0x00000080) # File was moved to Y -IN_CREATE = long(0x00000100) # Subfile was created -IN_DELETE = long(0x00000200) # Subfile was delete -IN_DELETE_SELF = long(0x00000400) # Self was deleted -IN_MOVE_SELF = long(0x00000800) # Self was moved -IN_UNMOUNT = long(0x00002000) # Backing fs was unmounted -IN_Q_OVERFLOW = long(0x00004000) # Event queued overflowed -IN_IGNORED = long(0x00008000) # File was ignored - -IN_ONLYDIR = 0x01000000 # only watch the path if it is a directory -IN_DONT_FOLLOW = 0x02000000 # don't follow a sym link -IN_MASK_ADD = 0x20000000 # add to the mask of an already existing watch -IN_ISDIR = 0x40000000 # event occurred against dir -IN_ONESHOT = 0x80000000 # only send event once - -IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE # closes -IN_MOVED = IN_MOVED_FROM | IN_MOVED_TO # moves -IN_CHANGED = IN_MODIFY | IN_ATTRIB # changes - -IN_WATCH_MASK = (IN_MODIFY | IN_ATTRIB | - IN_CREATE | IN_DELETE | - IN_DELETE_SELF | IN_MOVE_SELF | - IN_UNMOUNT | IN_MOVED_FROM | IN_MOVED_TO) - - -_FLAG_TO_HUMAN = [ - (IN_ACCESS, 'access'), - (IN_MODIFY, 'modify'), - (IN_ATTRIB, 'attrib'), - (IN_CLOSE_WRITE, 'close_write'), - (IN_CLOSE_NOWRITE, 'close_nowrite'), - (IN_OPEN, 'open'), - (IN_MOVED_FROM, 'moved_from'), - (IN_MOVED_TO, 'moved_to'), - (IN_CREATE, 'create'), - (IN_DELETE, 'delete'), - (IN_DELETE_SELF, 'delete_self'), - (IN_MOVE_SELF, 'move_self'), - (IN_UNMOUNT, 'unmount'), - (IN_Q_OVERFLOW, 'queue_overflow'), - (IN_IGNORED, 'ignored'), - (IN_ONLYDIR, 'only_dir'), - (IN_DONT_FOLLOW, 'dont_follow'), - (IN_MASK_ADD, 'mask_add'), - (IN_ISDIR, 'is_dir'), - (IN_ONESHOT, 'one_shot') -] - - - -def humanReadableMask(mask): - """ - Auxiliary function that converts an hexadecimal mask into a series - of human readable flags. - """ - s = [] - for k, v in _FLAG_TO_HUMAN: - if k & mask: - s.append(v) - return s - - -from eliot import start_action - -# This class is not copied from Twisted; it acts as a mock. -class INotify(object): - def startReading(self): - pass - - def stopReading(self): - pass - - def loseConnection(self): - pass - - def watch(self, filepath, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None, recursive=False): - self.callbacks = callbacks - - def event(self, filepath, mask): - with start_action(action_type=u"fake-inotify:event", path=filepath.asTextMode().path, mask=mask): - for cb in self.callbacks: - cb(None, filepath, mask) - - -__all__ = ["INotify", "humanReadableMask", "IN_WATCH_MASK", "IN_ACCESS", - "IN_MODIFY", "IN_ATTRIB", "IN_CLOSE_NOWRITE", "IN_CLOSE_WRITE", - "IN_OPEN", "IN_MOVED_FROM", "IN_MOVED_TO", "IN_CREATE", - "IN_DELETE", "IN_DELETE_SELF", "IN_MOVE_SELF", "IN_UNMOUNT", - "IN_Q_OVERFLOW", "IN_IGNORED", "IN_ONLYDIR", "IN_DONT_FOLLOW", - "IN_MASK_ADD", "IN_ISDIR", "IN_ONESHOT", "IN_CLOSE", - "IN_MOVED", "IN_CHANGED"] diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 0bb38e776..162852c38 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -16,8 +16,7 @@ if sys.platform == "win32": from twisted.python import log -from pycryptopp.cipher.aes import AES - +from allmydata.crypto import aes from allmydata.util.assertutil import _assert @@ -75,7 +74,7 @@ def remove(f, tries=4, basedelay=0.1): basedelay *= 2 return os.remove(f) # The last try. -class ReopenableNamedTemporaryFile: +class ReopenableNamedTemporaryFile(object): """ This uses tempfile.mkstemp() to generate a secure temp file. It then closes the file, leaving a zero-length file as a placeholder. You can get the @@ -99,7 +98,7 @@ class ReopenableNamedTemporaryFile: def shutdown(self): remove(self.name) -class EncryptedTemporaryFile: +class EncryptedTemporaryFile(object): # not implemented: next, readline, readlines, xreadlines, writelines def __init__(self): @@ -110,9 +109,10 @@ class EncryptedTemporaryFile: offset_big = offset // 16 offset_small = offset % 16 iv = binascii.unhexlify("%032x" % offset_big) - cipher = AES(self.key, iv=iv) - cipher.process("\x00"*offset_small) - return cipher.process(data) + cipher = aes.create_encryptor(self.key, iv) + # this is just to advance the counter + aes.encrypt_data(cipher, b"\x00" * offset_small) + return aes.encrypt_data(cipher, data) def close(self): self.file.close() diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index 3c57cdf6b..fd8ab4190 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -1,4 +1,3 @@ -from pycryptopp.hash.sha256 import SHA256 import os import hashlib from allmydata.util.netstring import netstring @@ -12,40 +11,44 @@ from allmydata.util.netstring import netstring # randomly-generated secrets such as the lease secret, and symmetric encryption # keys. In the near future we will add DSA private keys, and salts of various # kinds. -CRYPTO_VAL_SIZE=32 +CRYPTO_VAL_SIZE = 32 -class _SHA256d_Hasher: + +class _SHA256d_Hasher(object): # use SHA-256d, as defined by Ferguson and Schneier: hash the output # again to prevent length-extension attacks def __init__(self, truncate_to=None): - self.h = SHA256() + self.h = hashlib.sha256() self.truncate_to = truncate_to self._digest = None + def update(self, data): - assert isinstance(data, str) # no unicode + assert isinstance(data, bytes) # no unicode self.h.update(data) + def digest(self): if self._digest is None: h1 = self.h.digest() del self.h - h2 = SHA256(h1).digest() + h2 = hashlib.sha256(h1).digest() if self.truncate_to: h2 = h2[:self.truncate_to] self._digest = h2 return self._digest - def tagged_hasher(tag, truncate_to=None): hasher = _SHA256d_Hasher(truncate_to) hasher.update(netstring(tag)) return hasher + def tagged_hash(tag, val, truncate_to=None): hasher = tagged_hasher(tag, truncate_to) hasher.update(val) return hasher.digest() + def tagged_pair_hash(tag, val1, val2, truncate_to=None): s = _SHA256d_Hasher(truncate_to) s.update(netstring(tag)) @@ -53,7 +56,8 @@ def tagged_pair_hash(tag, val1, val2, truncate_to=None): s.update(netstring(val2)) return s.digest() -## specific hash tags that we use +# specific hash tags that we use + # immutable STORAGE_INDEX_TAG = "allmydata_immutable_key_to_storage_index_v1" @@ -85,6 +89,7 @@ MUTABLE_STORAGEINDEX_TAG = "allmydata_mutable_readkey_to_storage_index_v1" DIRNODE_CHILD_WRITECAP_TAG = "allmydata_mutable_writekey_and_salt_to_dirnode_child_capkey_v1" DIRNODE_CHILD_SALT_TAG = "allmydata_dirnode_child_rwcap_to_salt_v1" + def storage_index_hash(key): # storage index is truncated to 128 bits (16 bytes). We're only hashing a # 16-byte value to get it, so there's no point in using a larger value. We @@ -93,115 +98,165 @@ def storage_index_hash(key): # files. Mutable files use ssk_storage_index_hash(). return tagged_hash(STORAGE_INDEX_TAG, key, 16) + def block_hash(data): return tagged_hash(BLOCK_TAG, data) + + def block_hasher(): return tagged_hasher(BLOCK_TAG) + def uri_extension_hash(data): return tagged_hash(UEB_TAG, data) + + def uri_extension_hasher(): return tagged_hasher(UEB_TAG) + def plaintext_hash(data): return tagged_hash(PLAINTEXT_TAG, data) + + def plaintext_hasher(): return tagged_hasher(PLAINTEXT_TAG) + def crypttext_hash(data): return tagged_hash(CIPHERTEXT_TAG, data) + + def crypttext_hasher(): return tagged_hasher(CIPHERTEXT_TAG) + def crypttext_segment_hash(data): return tagged_hash(CIPHERTEXT_SEGMENT_TAG, data) + + def crypttext_segment_hasher(): return tagged_hasher(CIPHERTEXT_SEGMENT_TAG) + def plaintext_segment_hash(data): return tagged_hash(PLAINTEXT_SEGMENT_TAG, data) + + def plaintext_segment_hasher(): return tagged_hasher(PLAINTEXT_SEGMENT_TAG) + KEYLEN = 16 IVLEN = 16 + def convergence_hash(k, n, segsize, data, convergence): h = convergence_hasher(k, n, segsize, convergence) h.update(data) return h.digest() + + def convergence_hasher(k, n, segsize, convergence): assert isinstance(convergence, str) param_tag = netstring("%d,%d,%d" % (k, n, segsize)) tag = CONVERGENT_ENCRYPTION_TAG + netstring(convergence) + param_tag return tagged_hasher(tag, KEYLEN) + def random_key(): return os.urandom(KEYLEN) + def my_renewal_secret_hash(my_secret): return tagged_hash(my_secret, CLIENT_RENEWAL_TAG) + + def my_cancel_secret_hash(my_secret): return tagged_hash(my_secret, CLIENT_CANCEL_TAG) + def file_renewal_secret_hash(client_renewal_secret, storage_index): return tagged_pair_hash(FILE_RENEWAL_TAG, client_renewal_secret, storage_index) + def file_cancel_secret_hash(client_cancel_secret, storage_index): return tagged_pair_hash(FILE_CANCEL_TAG, client_cancel_secret, storage_index) + def bucket_renewal_secret_hash(file_renewal_secret, peerid): - assert len(peerid) == 20, "%s: %r" % (len(peerid), peerid) # binary! + assert len(peerid) == 20, "%s: %r" % (len(peerid), peerid) # binary! return tagged_pair_hash(BUCKET_RENEWAL_TAG, file_renewal_secret, peerid) + def bucket_cancel_secret_hash(file_cancel_secret, peerid): - assert len(peerid) == 20, "%s: %r" % (len(peerid), peerid) # binary! + assert len(peerid) == 20, "%s: %r" % (len(peerid), peerid) # binary! return tagged_pair_hash(BUCKET_CANCEL_TAG, file_cancel_secret, peerid) def _xor(a, b): return "".join([chr(ord(c) ^ ord(b)) for c in a]) + def hmac(tag, data): ikey = _xor(tag, "\x36") okey = _xor(tag, "\x5c") - h1 = SHA256(ikey + data).digest() - h2 = SHA256(okey + h1).digest() + h1 = hashlib.sha256(ikey + data).digest() + h2 = hashlib.sha256(okey + h1).digest() return h2 + def mutable_rwcap_key_hash(iv, writekey): return tagged_pair_hash(DIRNODE_CHILD_WRITECAP_TAG, iv, writekey, KEYLEN) + + def mutable_rwcap_salt_hash(writekey): return tagged_hash(DIRNODE_CHILD_SALT_TAG, writekey, IVLEN) + def ssk_writekey_hash(privkey): return tagged_hash(MUTABLE_WRITEKEY_TAG, privkey, KEYLEN) + + def ssk_write_enabler_master_hash(writekey): return tagged_hash(MUTABLE_WRITE_ENABLER_MASTER_TAG, writekey) + + def ssk_write_enabler_hash(writekey, peerid): - assert len(peerid) == 20, "%s: %r" % (len(peerid), peerid) # binary! + assert len(peerid) == 20, "%s: %r" % (len(peerid), peerid) # binary! wem = ssk_write_enabler_master_hash(writekey) return tagged_pair_hash(MUTABLE_WRITE_ENABLER_TAG, wem, peerid) + def ssk_pubkey_fingerprint_hash(pubkey): return tagged_hash(MUTABLE_PUBKEY_TAG, pubkey) + def ssk_readkey_hash(writekey): return tagged_hash(MUTABLE_READKEY_TAG, writekey, KEYLEN) + + def ssk_readkey_data_hash(IV, readkey): return tagged_pair_hash(MUTABLE_DATAKEY_TAG, IV, readkey, KEYLEN) + + def ssk_storage_index_hash(readkey): return tagged_hash(MUTABLE_STORAGEINDEX_TAG, readkey, KEYLEN) + def timing_safe_compare(a, b): n = os.urandom(32) return bool(tagged_hash(n, a) == tagged_hash(n, b)) + BACKUPDB_DIRHASH_TAG = "allmydata_backupdb_dirhash_v1" + + def backupdb_dirhash(contents): return tagged_hash(BACKUPDB_DIRHASH_TAG, contents) + def permute_server_hash(peer_selection_index, server_permutation_seed): return hashlib.sha1(peer_selection_index + server_permutation_seed).digest() diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 1a5576189..4d97effe9 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -1,7 +1,7 @@ import exceptions, os from repr import Repr -class BetterRepr(Repr): +class BetterRepr(Repr, object): def __init__(self): Repr.__init__(self) diff --git a/src/allmydata/util/keyutil.py b/src/allmydata/util/keyutil.py deleted file mode 100644 index ee28bd746..000000000 --- a/src/allmydata/util/keyutil.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -from pycryptopp.publickey import ed25519 -from allmydata.util.base32 import a2b, b2a - -BadSignatureError = ed25519.BadSignatureError - -class BadPrefixError(Exception): - pass - -def remove_prefix(s_bytes, prefix): - if not s_bytes.startswith(prefix): - raise BadPrefixError("did not see expected '%s' prefix" % (prefix,)) - return s_bytes[len(prefix):] - -# in base32, keys are 52 chars long (both signing and verifying keys) -# in base62, keys is 43 chars long -# in base64, keys is 43 chars long -# -# We can't use base64 because we want to reserve punctuation and preserve -# cut-and-pasteability. The base62 encoding is shorter than the base32 form, -# but the minor usability improvement is not worth the documentation and -# specification confusion of using a non-standard encoding. So we stick with -# base32. - -def make_keypair(): - sk_bytes = os.urandom(32) - sk = ed25519.SigningKey(sk_bytes) - vk_bytes = sk.get_verifying_key_bytes() - return ("priv-v0-"+b2a(sk_bytes), "pub-v0-"+b2a(vk_bytes)) - -def parse_privkey(privkey_vs): - sk_bytes = a2b(remove_prefix(privkey_vs, "priv-v0-")) - sk = ed25519.SigningKey(sk_bytes) - vk_bytes = sk.get_verifying_key_bytes() - return (sk, "pub-v0-"+b2a(vk_bytes)) - -def parse_pubkey(pubkey_vs): - vk_bytes = a2b(remove_prefix(pubkey_vs, "pub-v0-")) - return ed25519.VerifyingKey(vk_bytes) diff --git a/src/allmydata/util/limiter.py b/src/allmydata/util/limiter.py index 7ba27742c..0391ede11 100644 --- a/src/allmydata/util/limiter.py +++ b/src/allmydata/util/limiter.py @@ -2,7 +2,7 @@ from twisted.internet import defer from foolscap.api import eventually -class ConcurrencyLimiter: +class ConcurrencyLimiter(object): """I implement a basic concurrency limiter. Add work to it in the form of (callable, args, kwargs) tuples. No more than LIMIT callables will be outstanding at any one time. diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index 3dc1d2768..30eb92329 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -10,7 +10,7 @@ something happens. The way this is typically implemented is that the observed has an ObserverList whose when_fired method is called in the observed's 'when_something'.""" -class OneShotObserverList: +class OneShotObserverList(object): """A one-shot event distributor.""" def __init__(self): self._fired = False @@ -18,6 +18,12 @@ class OneShotObserverList: self._watchers = [] self.__repr__ = self._unfired_repr + def __repr__(self): + """string representation of the OneshotObserverList""" + if self._fired: + return self._fired_repr() + return self._unfired_repr() + def _unfired_repr(self): return "" % (self._watchers, ) @@ -77,7 +83,7 @@ class LazyOneShotObserverList(OneShotObserverList): if self._watchers: # if not, don't call result_producer self._fire(self._get_result()) -class ObserverList: +class ObserverList(object): """A simple class to distribute events to a number of subscribers.""" def __init__(self): @@ -93,7 +99,7 @@ class ObserverList: for o in self._watchers: eventually(o, *args, **kwargs) -class EventStreamObserver: +class EventStreamObserver(object): """A simple class to distribute multiple events to a single subscriber. It accepts arbitrary kwargs, but no posargs.""" def __init__(self): diff --git a/src/allmydata/util/pipeline.py b/src/allmydata/util/pipeline.py index b072dc502..285a06b98 100644 --- a/src/allmydata/util/pipeline.py +++ b/src/allmydata/util/pipeline.py @@ -19,7 +19,7 @@ class SingleFileError(Exception): """You are not permitted to add a job to a full pipeline.""" -class ExpandableDeferredList(defer.Deferred): +class ExpandableDeferredList(defer.Deferred, object): # like DeferredList(fireOnOneErrback=True) with a built-in # gatherResults(), but you can add new Deferreds until you close it. This # gives you a chance to add don't-complain-about-unhandled-error errbacks @@ -66,7 +66,7 @@ class ExpandableDeferredList(defer.Deferred): return f -class Pipeline: +class Pipeline(object): """I manage a size-limited pipeline of Deferred operations, usually callRemote() messages.""" diff --git a/src/allmydata/util/pkgresutil.py b/src/allmydata/util/pkgresutil.py deleted file mode 100644 index bb1d985c6..000000000 --- a/src/allmydata/util/pkgresutil.py +++ /dev/null @@ -1,46 +0,0 @@ - -def install(): - """ - This installs a hook into setuptools' pkg_resources infrastructure, so that resource - files can be found in files relative to the runnin executable, in addition to the - usual egg and source lookup mechanisms. This overrides the ZipProvider, since that - is the lookup mechanism triggered within pkg_resources when running code out of a - py2exe or py2app build's library.zip. - """ - import os, sys - import pkg_resources, zipimport - - platform_libdirs = { - 'darwin': '../Resources/pkg_resources', - } - exedir = os.path.dirname(sys.executable) - libdir = platform_libdirs.get(sys.platform, 'pkg_resources') - - class Provider(pkg_resources.ZipProvider): - - def __init__(self, module): - self._module_name = module.__name__ - pkg_resources.ZipProvider.__init__(self, module) - - def get_resource_filename(self, manager, resource_name): - #print 'get_resource_filename(%s, %s)' % (manager, resource_name) - path = [exedir, libdir] + self._module_name.split('.') + [resource_name] - localfile = os.path.join(*path) - #print ' checking(%s)' % (localfile,) - if os.path.exists(localfile): - #print 'found locally' - return localfile - else: - try: - ret = pkg_resources.ZipProvider.get_resource_filename(self, manager, resource_name) - #print 'returning %s' % (ret,) - return ret - except NotImplementedError: - #print 'get_resource_filename(%s,%s): not found' % (self._module_name, resource_name) - #import traceback - #traceback.print_exc() - return '' - - pkg_resources.register_loader_type(zipimport.zipimporter, Provider) - - diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 971c543a4..14f5cd390 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -9,7 +9,7 @@ class TimeoutError(Exception): class PollComplete(Exception): pass -class PollMixin: +class PollMixin(object): _poll_should_ignore_these_errors = [] def poll(self, check_f, pollinterval=0.01, timeout=1000): diff --git a/src/allmydata/util/repeatable_random.py b/src/allmydata/util/repeatable_random.py deleted file mode 100644 index a64445efc..000000000 --- a/src/allmydata/util/repeatable_random.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -If you execute force_repeatability() then the following things are changed in the runtime: - -1. random.random() and its sibling functions, and random.Random.seed() in the random module are seeded with a known seed so that they will return the same sequence on each run. -2. os.urandom() is replaced by a fake urandom that returns a pseudorandom sequence. -3. time.time() is replaced by a fake time that returns an incrementing number. (Original time.time is available as time.realtime.) - -Which seed will be used? - -If the environment variable REPEATABLE_RANDOMNESS_SEED is set, then it will use that. Else, it will use the current real time. In either case it logs the seed that it used. - -Caveats: - -1. If some code has acquired a random.Random object before force_repeatability() is executed, then that Random object will produce non-reproducible results. For example, the tempfile module in the Python Standard Library does this. -2. Likewise if some code called time.time() before force_repeatability() was called, then it will have gotten a real time stamp. For example, trial does this. (Then it later subtracts that real timestamp from a faketime timestamp to calculate elapsed time, resulting in a large negative elapsed time.) -3. The output from the fake urandom has weird distribution for performance reasons-- every byte after the first 20 bytes resulting from a single call to os.urandom() is zero. In practice this hasn't caused any problems. -""" - -import os, random, time -if not hasattr(time, "realtime"): - time.realtime = time.time -if not hasattr(os, "realurandom"): - os.realurandom = os.urandom -if not hasattr(random, "realseed"): - random.realseed = random.seed - -tdelta = 0 -seeded = False -def force_repeatability(): - now = 1043659734.0 - def faketime(): - global tdelta - tdelta += 1 - return now + tdelta - time.faketime = faketime - time.time = faketime - - from allmydata.util.idlib import i2b - def fakeurandom(n): - if n > 20: - z = i2b(random.getrandbits(20*8)) - elif n == 0: - return '' - else: - z = i2b(random.getrandbits(n*8)) - x = z + "0" * (n-len(z)) - assert len(x) == n - return x - os.fakeurandom = fakeurandom - os.urandom = fakeurandom - - global seeded - if not seeded: - SEED = os.environ.get('REPEATABLE_RANDOMNESS_SEED', None) - - if SEED is None: - # Generate a seed which is integral and fairly short (to ease cut-and-paste, writing it down, etc.). - t = time.realtime() - subsec = t % 1 - t += (subsec * 1000000) - t %= 1000000 - SEED = long(t) - import sys - sys.stdout.write("REPEATABLE_RANDOMNESS_SEED: %s\n" % SEED) ; sys.stdout.flush() - sys.stdout.write("In order to reproduce this run of the code, set the environment variable \"REPEATABLE_RANDOMNESS_SEED\" to %s before executing.\n" % SEED) ; sys.stdout.flush() - random.seed(SEED) - - def seed_which_refuses(a): - sys.stdout.write("I refuse to reseed to %s. Go away!\n" % (a,)) ; sys.stdout.flush() - return - random.realseed = random.seed - random.seed = seed_which_refuses - seeded = True - - import setutil - setutil.RandomSet.DETERMINISTIC = True - -def restore_real_clock(): - time.time = time.realtime - -def restore_real_urandom(): - os.urandom = os.realurandom - -def restore_real_seed(): - random.seed = random.realseed - -def restore_non_repeatability(): - restore_real_seed() - restore_real_urandom() - restore_real_clock() diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 94aef6216..1576f059a 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -1,7 +1,7 @@ from __future__ import print_function -class Spans: +class Spans(object): """I represent a compressed list of booleans, one per index (an integer). Typically, each index represents an offset into a large string, pointing to a specific byte of a share. In this context, True means that byte has @@ -222,7 +222,7 @@ def adjacent(start0, length0, start1, length1): return True return False -class DataSpans: +class DataSpans(object): """I represent portions of a large string. Equivalently, I can be said to maintain a large array of characters (with gaps of empty elements). I can be used to manage access to a remote share, where some pieces have been diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py new file mode 100644 index 000000000..9f084ff4b --- /dev/null +++ b/src/allmydata/version_checks.py @@ -0,0 +1,399 @@ +""" +Produce reports about the versions of Python software in use by Tahoe-LAFS +for debugging and auditing purposes. +""" + +__all__ = [ + "PackagingError", + "get_package_versions", + "get_package_versions_string", + "normalized_version", +] + +import os, platform, re, subprocess, sys, traceback, pkg_resources + +import six + +from . import ( + __appname__, + full_version, + branch, +) +from .util import ( + verlib, +) + +if getattr(sys, 'frozen', None): + # "Frozen" python interpreters (i.e., standalone executables + # generated by PyInstaller and other, similar utilities) run + # independently of a traditional setuptools-based packaging + # environment, and so pkg_resources.get_distribution() cannot be + # used in such cases to gather a list of requirements at runtime + # (and because a frozen application is one that has already been + # "installed", an empty list suffices here). + _INSTALL_REQUIRES = [] +else: + _INSTALL_REQUIRES = list( + str(req) + for req + in pkg_resources.get_distribution(__appname__).requires() + ) + +class PackagingError(EnvironmentError): + """ + Raised when there is an error in packaging of Tahoe-LAFS or its + dependencies which makes it impossible to proceed safely. + """ + +def get_package_versions(): + return dict([(k, v) for k, (v, l, c) in _vers_and_locs_list]) + +def get_package_versions_string(show_paths=False, debug=False): + res = [] + for p, (v, loc, comment) in _vers_and_locs_list: + info = str(p) + ": " + str(v) + if comment: + info = info + " [%s]" % str(comment) + if show_paths: + info = info + " (%s)" % str(loc) + res.append(info) + + output = "\n".join(res) + "\n" + + if _cross_check_errors: + output += _get_error_string(_cross_check_errors, debug=debug) + + return output + +_distributor_id_cmdline_re = re.compile("(?:Distributor ID:)\s*(.*)", re.I) +_release_cmdline_re = re.compile("(?:Release:)\s*(.*)", re.I) + +_distributor_id_file_re = re.compile("(?:DISTRIB_ID\s*=)\s*(.*)", re.I) +_release_file_re = re.compile("(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I) + +_distname = None +_version = None + +def normalized_version(verstr, what=None): + try: + suggested = verlib.suggest_normalized_version(verstr) or verstr + return verlib.NormalizedVersion(suggested) + except verlib.IrrationalVersionError: + raise + except StandardError: + cls, value, trace = sys.exc_info() + new_exc = PackagingError("could not parse %s due to %s: %s" + % (what or repr(verstr), cls.__name__, value)) + six.reraise(cls, new_exc, trace) + +def _get_error_string(errors, debug=False): + + msg = "\n%s\n" % ("\n".join(errors),) + if debug: + msg += ( + "\n" + "For debugging purposes, the PYTHONPATH was\n" + " %r\n" + "install_requires was\n" + " %r\n" + "sys.path after importing pkg_resources was\n" + " %s\n" + % ( + os.environ.get('PYTHONPATH'), + _INSTALL_REQUIRES, + (os.pathsep+"\n ").join(sys.path), + ) + ) + return msg + +def _cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list): + """This function returns a list of errors due to any failed cross-checks.""" + + from _auto_deps import not_import_versionable + + errors = [] + not_pkg_resourceable = ['python', 'platform', __appname__.lower(), 'openssl'] + + for name, (imp_ver, imp_loc, imp_comment) in imported_vers_and_locs_list: + name = name.lower() + if name not in not_pkg_resourceable: + if name not in pkg_resources_vers_and_locs: + if name == "setuptools" and "distribute" in pkg_resources_vers_and_locs: + pr_ver, pr_loc = pkg_resources_vers_and_locs["distribute"] + if not (os.path.normpath(os.path.realpath(pr_loc)) == os.path.normpath(os.path.realpath(imp_loc)) + and imp_comment == "distribute"): + errors.append("Warning: dependency 'setuptools' found to be version %r of 'distribute' from %r " + "by pkg_resources, but 'import setuptools' gave version %r [%s] from %r. " + "A version mismatch is expected, but a location mismatch is not." + % (pr_ver, pr_loc, imp_ver, imp_comment or 'probably *not* distribute', imp_loc)) + else: + errors.append("Warning: dependency %r (version %r imported from %r) was not found by pkg_resources." + % (name, imp_ver, imp_loc)) + continue + + pr_ver, pr_loc = pkg_resources_vers_and_locs[name] + if imp_ver is None and imp_loc is None: + errors.append("Warning: dependency %r could not be imported. pkg_resources thought it should be possible " + "to import version %r from %r.\nThe exception trace was %r." + % (name, pr_ver, pr_loc, imp_comment)) + continue + + # If the pkg_resources version is identical to the imported version, don't attempt + # to normalize them, since it is unnecessary and may fail (ticket #2499). + if imp_ver != 'unknown' and pr_ver == imp_ver: + continue + + try: + pr_normver = normalized_version(pr_ver) + except verlib.IrrationalVersionError: + continue + except Exception as e: + errors.append("Warning: version number %r found for dependency %r by pkg_resources could not be parsed. " + "The version found by import was %r from %r. " + "pkg_resources thought it should be found at %r. " + "The exception was %s: %s" + % (pr_ver, name, imp_ver, imp_loc, pr_loc, e.__class__.__name__, e)) + else: + if imp_ver == 'unknown': + if name not in not_import_versionable: + errors.append("Warning: unexpectedly could not find a version number for dependency %r imported from %r. " + "pkg_resources thought it should be version %r at %r." + % (name, imp_loc, pr_ver, pr_loc)) + else: + try: + imp_normver = normalized_version(imp_ver) + except verlib.IrrationalVersionError: + continue + except Exception as e: + errors.append("Warning: version number %r found for dependency %r (imported from %r) could not be parsed. " + "pkg_resources thought it should be version %r at %r. " + "The exception was %s: %s" + % (imp_ver, name, imp_loc, pr_ver, pr_loc, e.__class__.__name__, e)) + else: + if pr_ver == 'unknown' or (pr_normver != imp_normver): + if not os.path.normpath(os.path.realpath(pr_loc)) == os.path.normpath(os.path.realpath(imp_loc)): + errors.append("Warning: dependency %r found to have version number %r (normalized to %r, from %r) " + "by pkg_resources, but version %r (normalized to %r, from %r) by import." + % (name, pr_ver, str(pr_normver), pr_loc, imp_ver, str(imp_normver), imp_loc)) + + return errors + +def _get_openssl_version(): + try: + from OpenSSL import SSL + return _extract_openssl_version(SSL) + except Exception: + return ("unknown", None, None) + +def _extract_openssl_version(ssl_module): + openssl_version = ssl_module.SSLeay_version(ssl_module.SSLEAY_VERSION) + if openssl_version.startswith('OpenSSL '): + openssl_version = openssl_version[8 :] + + (version, _, comment) = openssl_version.partition(' ') + + try: + openssl_cflags = ssl_module.SSLeay_version(ssl_module.SSLEAY_CFLAGS) + if '-DOPENSSL_NO_HEARTBEATS' in openssl_cflags.split(' '): + comment += ", no heartbeats" + except Exception: + pass + + return (version, None, comment if comment else None) + +def _get_linux_distro(): + """ Tries to determine the name of the Linux OS distribution name. + + First, try to parse a file named "/etc/lsb-release". If it exists, and + contains the "DISTRIB_ID=" line and the "DISTRIB_RELEASE=" line, then return + the strings parsed from that file. + + If that doesn't work, then invoke platform.dist(). + + If that doesn't work, then try to execute "lsb_release", as standardized in + 2001: + + http://refspecs.freestandards.org/LSB_1.0.0/gLSB/lsbrelease.html + + The current version of the standard is here: + + http://refspecs.freestandards.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/lsbrelease.html + + that lsb_release emitted, as strings. + + Returns a tuple (distname,version). Distname is what LSB calls a + "distributor id", e.g. "Ubuntu". Version is what LSB calls a "release", + e.g. "8.04". + + A version of this has been submitted to python as a patch for the standard + library module "platform": + + http://bugs.python.org/issue3937 + """ + global _distname,_version + if _distname and _version: + return (_distname, _version) + + try: + etclsbrel = open("/etc/lsb-release", "rU") + for line in etclsbrel: + m = _distributor_id_file_re.search(line) + if m: + _distname = m.group(1).strip() + if _distname and _version: + return (_distname, _version) + m = _release_file_re.search(line) + if m: + _version = m.group(1).strip() + if _distname and _version: + return (_distname, _version) + except EnvironmentError: + pass + + (_distname, _version) = platform.dist()[:2] + if _distname and _version: + return (_distname, _version) + + if os.path.isfile("/usr/bin/lsb_release") or os.path.isfile("/bin/lsb_release"): + try: + p = subprocess.Popen(["lsb_release", "--all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + rc = p.wait() + if rc == 0: + for line in p.stdout.readlines(): + m = _distributor_id_cmdline_re.search(line) + if m: + _distname = m.group(1).strip() + if _distname and _version: + return (_distname, _version) + + m = _release_cmdline_re.search(p.stdout.read()) + if m: + _version = m.group(1).strip() + if _distname and _version: + return (_distname, _version) + except EnvironmentError: + pass + + if os.path.exists("/etc/arch-release"): + return ("Arch_Linux", "") + + return (_distname,_version) + +def _get_platform(): + # Our version of platform.platform(), telling us both less and more than the + # Python Standard Library's version does. + # We omit details such as the Linux kernel version number, but we add a + # more detailed and correct rendition of the Linux distribution and + # distribution-version. + if "linux" in platform.system().lower(): + return ( + platform.system() + "-" + + "_".join(_get_linux_distro()) + "-" + + platform.machine() + "-" + + "_".join([x for x in platform.architecture() if x]) + ) + else: + return platform.platform() + +def _get_package_versions_and_locations(): + import warnings + from _auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \ + runtime_warning_messages, warning_imports, ignorable + + def package_dir(srcfile): + return os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) + + # pkg_resources.require returns the distribution that pkg_resources attempted to put + # on sys.path, which can differ from the one that we actually import due to #1258, + # or any other bug that causes sys.path to be set up incorrectly. Therefore we + # must import the packages in order to check their versions and paths. + + # This is to suppress all UserWarnings and various DeprecationWarnings and RuntimeWarnings + # (listed in _auto_deps.py). + + warnings.filterwarnings("ignore", category=UserWarning, append=True) + + for msg in global_deprecation_messages + deprecation_messages: + warnings.filterwarnings("ignore", category=DeprecationWarning, message=msg, append=True) + for msg in runtime_warning_messages: + warnings.filterwarnings("ignore", category=RuntimeWarning, message=msg, append=True) + try: + for modulename in warning_imports: + try: + __import__(modulename) + except ImportError: + pass + finally: + # Leave suppressions for UserWarnings and global_deprecation_messages active. + for _ in runtime_warning_messages + deprecation_messages: + warnings.filters.pop() + + packages = [] + pkg_resources_vers_and_locs = dict() + + if not hasattr(sys, 'frozen'): + pkg_resources_vers_and_locs = { + p.project_name.lower(): (str(p.version), p.location) + for p + in pkg_resources.require(_INSTALL_REQUIRES) + } + + def get_version(module): + if hasattr(module, '__version__'): + return str(getattr(module, '__version__')) + elif hasattr(module, 'version'): + ver = getattr(module, 'version') + if isinstance(ver, tuple): + return '.'.join(map(str, ver)) + else: + return str(ver) + else: + return 'unknown' + + for pkgname, modulename in [(__appname__, 'allmydata')] + package_imports: + if modulename: + try: + __import__(modulename) + module = sys.modules[modulename] + except ImportError: + etype, emsg, etrace = sys.exc_info() + trace_info = (etype, str(emsg), ([None] + traceback.extract_tb(etrace))[-1]) + packages.append( (pkgname, (None, None, trace_info)) ) + else: + comment = None + if pkgname == __appname__: + comment = "%s: %s" % (branch, full_version) + elif pkgname == 'setuptools' and hasattr(module, '_distribute'): + # distribute does not report its version in any module variables + comment = 'distribute' + ver = get_version(module) + loc = package_dir(module.__file__) + if ver == "unknown" and pkgname in pkg_resources_vers_and_locs: + (pr_ver, pr_loc) = pkg_resources_vers_and_locs[pkgname] + if loc == os.path.normcase(os.path.realpath(pr_loc)): + ver = pr_ver + packages.append( (pkgname, (ver, loc, comment)) ) + elif pkgname == 'python': + packages.append( (pkgname, (platform.python_version(), sys.executable, None)) ) + elif pkgname == 'platform': + packages.append( (pkgname, (_get_platform(), None, None)) ) + elif pkgname == 'OpenSSL': + packages.append( (pkgname, _get_openssl_version()) ) + + cross_check_errors = [] + + if len(pkg_resources_vers_and_locs) > 0: + imported_packages = set([p.lower() for (p, _) in packages]) + extra_packages = [] + + for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.iteritems(): + if pr_name not in imported_packages and pr_name not in ignorable: + extra_packages.append( (pr_name, (pr_ver, pr_loc, "according to pkg_resources")) ) + + cross_check_errors = _cross_check(pkg_resources_vers_and_locs, packages) + packages += extra_packages + + return packages, cross_check_errors + + +_vers_and_locs_list, _cross_check_errors = _get_package_versions_and_locations() diff --git a/src/allmydata/watchdog/__init__.py b/src/allmydata/watchdog/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/allmydata/watchdog/_watchdog_541.py b/src/allmydata/watchdog/_watchdog_541.py deleted file mode 100644 index 87686ce6d..000000000 --- a/src/allmydata/watchdog/_watchdog_541.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Hotfix for https://github.com/gorakhargosh/watchdog/issues/541 -""" - -from watchdog.observers.fsevents import FSEventsEmitter - -# The class object has already been bundled up in the default arguments to -# FSEventsObserver.__init__. So mutate the class object (instead of replacing -# it with a safer version). -original_on_thread_stop = FSEventsEmitter.on_thread_stop -def safe_on_thread_stop(self): - if self.is_alive(): - return original_on_thread_stop(self) - -def patch(): - FSEventsEmitter.on_thread_stop = safe_on_thread_stop diff --git a/src/allmydata/watchdog/inotify.py b/src/allmydata/watchdog/inotify.py deleted file mode 100644 index 68a121a60..000000000 --- a/src/allmydata/watchdog/inotify.py +++ /dev/null @@ -1,212 +0,0 @@ - -""" -An implementation of an inotify-like interface on top of the ``watchdog`` library. -""" - -from __future__ import ( - unicode_literals, - print_function, - absolute_import, - division, -) - -__all__ = [ - "humanReadableMask", "INotify", - "IN_WATCH_MASK", "IN_ACCESS", "IN_MODIFY", "IN_ATTRIB", "IN_CLOSE_NOWRITE", - "IN_CLOSE_WRITE", "IN_OPEN", "IN_MOVED_FROM", "IN_MOVED_TO", "IN_CREATE", - "IN_DELETE", "IN_DELETE_SELF", "IN_MOVE_SELF", "IN_UNMOUNT", "IN_ONESHOT", - "IN_Q_OVERFLOW", "IN_IGNORED", "IN_ONLYDIR", "IN_DONT_FOLLOW", "IN_MOVED", - "IN_MASK_ADD", "IN_ISDIR", "IN_CLOSE", "IN_CHANGED", "_FLAG_TO_HUMAN", -] - -from watchdog.observers import Observer -from watchdog.events import ( - FileSystemEvent, - FileSystemEventHandler, DirCreatedEvent, FileCreatedEvent, - DirDeletedEvent, FileDeletedEvent, FileModifiedEvent -) - -from twisted.internet import reactor -from twisted.python.filepath import FilePath -from allmydata.util.fileutil import abspath_expanduser_unicode - -from eliot import ( - ActionType, - Message, - Field, - preserve_context, - start_action, -) - -from allmydata.util.pollmixin import PollMixin -from allmydata.util.assertutil import _assert, precondition -from allmydata.util import encodingutil -from allmydata.util.fake_inotify import humanReadableMask, \ - IN_WATCH_MASK, IN_ACCESS, IN_MODIFY, IN_ATTRIB, IN_CLOSE_NOWRITE, IN_CLOSE_WRITE, \ - IN_OPEN, IN_MOVED_FROM, IN_MOVED_TO, IN_CREATE, IN_DELETE, IN_DELETE_SELF, \ - IN_MOVE_SELF, IN_UNMOUNT, IN_Q_OVERFLOW, IN_IGNORED, IN_ONLYDIR, IN_DONT_FOLLOW, \ - IN_MASK_ADD, IN_ISDIR, IN_ONESHOT, IN_CLOSE, IN_MOVED, IN_CHANGED, \ - _FLAG_TO_HUMAN - -from ..util.eliotutil import ( - MAYBE_NOTIFY, - CALLBACK, - validateInstanceOf, -) - -from . import _watchdog_541 - -_watchdog_541.patch() - -NOT_STARTED = "NOT_STARTED" -STARTED = "STARTED" -STOPPING = "STOPPING" -STOPPED = "STOPPED" - -_PATH = Field.for_types( - u"path", - [bytes, unicode], - u"The path an inotify event concerns.", -) - -_EVENT = Field( - u"event", - lambda e: e.__class__.__name__, - u"The watchdog event that has taken place.", - validateInstanceOf(FileSystemEvent), -) - -ANY_INOTIFY_EVENT = ActionType( - u"watchdog:inotify:any-event", - [_PATH, _EVENT], - [], - u"An inotify event is being dispatched.", -) - -class INotifyEventHandler(FileSystemEventHandler): - def __init__(self, path, mask, callbacks, pending_delay): - FileSystemEventHandler.__init__(self) - self._path = path - self._mask = mask - self._callbacks = callbacks - self._pending_delay = pending_delay - self._pending = set() - - def _maybe_notify(self, path, event): - with MAYBE_NOTIFY(): - event_mask = IN_CHANGED - if isinstance(event, FileModifiedEvent): - event_mask = event_mask | IN_CLOSE_WRITE - event_mask = event_mask | IN_MODIFY - if isinstance(event, (DirCreatedEvent, FileCreatedEvent)): - # For our purposes, IN_CREATE is irrelevant. - event_mask = event_mask | IN_CLOSE_WRITE - if isinstance(event, (DirDeletedEvent, FileDeletedEvent)): - event_mask = event_mask | IN_DELETE - if event.is_directory: - event_mask = event_mask | IN_ISDIR - if not (self._mask & event_mask): - return - for cb in self._callbacks: - try: - with CALLBACK(inotify_events=event_mask): - cb(None, FilePath(path), event_mask) - except: - # Eliot already logged the exception for us. - # There's nothing else we can do about it here. - pass - - def process(self, event): - event_filepath_u = event.src_path.decode(encodingutil.get_filesystem_encoding()) - event_filepath_u = abspath_expanduser_unicode(event_filepath_u, base=self._path) - - if event_filepath_u == self._path: - # ignore events for parent directory - return - - self._maybe_notify(event_filepath_u, event) - - def on_any_event(self, event): - with ANY_INOTIFY_EVENT(path=event.src_path, event=event): - reactor.callFromThread( - preserve_context(self.process), - event, - ) - - -class INotify(PollMixin): - """ - I am a prototype INotify, made to work on Mac OS X (Darwin) - using the Watchdog python library. This is actually a simplified subset - of the twisted Linux INotify class because we do not utilize the watch mask - and only implement the following methods: - - watch - - startReading - - stopReading - - wait_until_stopped - - set_pending_delay - """ - def __init__(self): - self._pending_delay = 1.0 - self.recursive_includes_new_subdirectories = False - self._callbacks = {} - self._watches = {} - self._state = NOT_STARTED - self._observer = Observer(timeout=self._pending_delay) - - def set_pending_delay(self, delay): - Message.log(message_type=u"watchdog:inotify:set-pending-delay", delay=delay) - assert self._state != STARTED - self._pending_delay = delay - - def startReading(self): - with start_action(action_type=u"watchdog:inotify:start-reading"): - assert self._state != STARTED - try: - # XXX twisted.internet.inotify doesn't require watches to - # be set before startReading is called. - # _assert(len(self._callbacks) != 0, "no watch set") - self._observer.start() - self._state = STARTED - except: - self._state = STOPPED - raise - - def stopReading(self): - with start_action(action_type=u"watchdog:inotify:stop-reading"): - if self._state != STOPPED: - self._state = STOPPING - self._observer.unschedule_all() - self._observer.stop() - self._observer.join() - self._state = STOPPED - - def wait_until_stopped(self): - return self.poll(lambda: self._state == STOPPED) - - def _isWatched(self, path_u): - return path_u in self._callbacks.keys() - - def ignore(self, path): - path_u = path.path - self._observer.unschedule(self._watches[path_u]) - del self._callbacks[path_u] - del self._watches[path_u] - - def watch(self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None, recursive=False): - precondition(isinstance(autoAdd, bool), autoAdd=autoAdd) - precondition(isinstance(recursive, bool), recursive=recursive) - assert autoAdd == False - - path_u = path.path - if not isinstance(path_u, unicode): - path_u = path_u.decode('utf-8') - _assert(isinstance(path_u, unicode), path_u=path_u) - - if path_u not in self._callbacks.keys(): - self._callbacks[path_u] = callbacks or [] - self._watches[path_u] = self._observer.schedule( - INotifyEventHandler(path_u, mask, self._callbacks[path_u], self._pending_delay), - path=path_u, - recursive=False, - ) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d5178d476..7a9badad4 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -63,7 +63,7 @@ def json_check_and_repair_results(r): data["post-repair-results"] = json_check_results(post) return data -class ResultsBase: +class ResultsBase(object): # self.client must point to the Client, so we can get nicknames and # determine the permuted peer order @@ -205,7 +205,7 @@ class LiteralCheckResultsRenderer(rend.Page, ResultsBase): return T.div[T.a(href=return_to)["Return to file."]] return "" -class CheckerBase: +class CheckerBase(object): def renderHTTP(self, ctx): if self.want_json(ctx): diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 7909d6ad3..e7b9f8ed1 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -2,10 +2,9 @@ import time import json -from twisted.web import http, server, resource +from twisted.web import http, server, resource, template from twisted.python import log from twisted.python.failure import Failure -from zope.interface import Interface from nevow import loaders, appserver from nevow.rend import Page from nevow.inevow import IRequest @@ -39,9 +38,6 @@ def get_filenode_metadata(filenode): metadata['size'] = size return metadata -class IOpHandleTable(Interface): - pass - def getxmlfile(name): return loaders.xmlfile(resource_filename('allmydata.web', '%s' % name)) @@ -104,8 +100,7 @@ def parse_offset_arg(offset): def get_root(ctx_or_req): req = IRequest(ctx_or_req) - # the addSlash=True gives us one extra (empty) segment - depth = len(req.prepath) + len(req.postpath) - 1 + depth = len(req.prepath) + len(req.postpath) link = "/".join([".."] * depth) return link @@ -207,8 +202,7 @@ def plural(sequence_or_length): return "" return "s" -def text_plain(text, ctx): - req = IRequest(ctx) +def text_plain(text, req): req.setHeader("content-type", "text/plain") req.setHeader("content-length", b"%d" % len(text)) return text @@ -322,7 +316,7 @@ def humanize_failure(f): return (f.getTraceback(), http.REQUEST_ENTITY_TOO_LARGE) return (str(f), None) -class MyExceptionHandler(appserver.DefaultExceptionHandler): +class MyExceptionHandler(appserver.DefaultExceptionHandler, object): def simple(self, ctx, text, code=http.BAD_REQUEST): req = IRequest(ctx) req.setResponseCode(code) @@ -371,7 +365,10 @@ class NeedOperationHandleError(WebError): pass -class RenderMixin: +# XXX should be phased out by the nevow -> twisted.web port (that is, +# this whole class should have no users and can be deleted once the +# port away from nevow is complete) +class RenderMixin(object): def renderHTTP(self, ctx): request = IRequest(ctx) @@ -460,8 +457,108 @@ class MultiFormatPage(Page): return lambda ctx: renderer(IRequest(ctx)) +class MultiFormatResource(resource.Resource, object): + """ + ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in + a number of different formats. -class TokenOnlyWebApi(resource.Resource): + Rendered format is controlled by a query argument (given by + ``self.formatArgument``). Different resources may support different + formats but ``json`` is a pretty common one. ``html`` is the default + format if nothing else is given as the ``formatDefault``. + """ + formatArgument = "t" + formatDefault = None + + def render(self, req): + """ + Dispatch to a renderer for a particular format, as selected by a query + argument. + + A renderer for the format given by the query argument matching + ``formatArgument`` will be selected and invoked. render_HTML will be + used as a default if no format is selected (either by query arguments + or by ``formatDefault``). + + :return: The result of the selected renderer. + """ + t = get_arg(req, self.formatArgument, self.formatDefault) + renderer = self._get_renderer(t) + return renderer(req) + + def _get_renderer(self, fmt): + """ + Get the renderer for the indicated format. + + :param str fmt: The format. If a method with a prefix of ``render_`` + and a suffix of this format (upper-cased) is found, it will be + used. + + :return: A callable which takes a twisted.web Request and renders a + response. + """ + renderer = None + + if fmt is not None: + try: + renderer = getattr(self, "render_{}".format(fmt.upper())) + except AttributeError: + raise WebError( + "Unknown {} value: {!r}".format(self.formatArgument, fmt), + ) + + if renderer is None: + renderer = self.render_HTML + + return renderer + + +class SlotsSequenceElement(template.Element): + """ + ``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for + twisted.web.template. + + Tags passed in to be templated will have two renderers available: ``item`` + and ``tag``. + """ + + def __init__(self, tag, seq): + self.loader = template.TagLoader(tag) + self.seq = seq + + @template.renderer + def header(self, request, tag): + return tag + + @template.renderer + def item(self, request, tag): + """ + A template renderer for each sequence item. + + ``tag`` will be cloned for each item in the sequence provided, and its + slots filled from the sequence item. Each item must be dict-like enough + for ``tag.fillSlots(**item)``. Each cloned tag will be siblings with no + separator beween them. + """ + for item in self.seq: + yield tag.clone(deep=False).fillSlots(**item) + + @template.renderer + def empty(self, request, tag): + """ + A template renderer for empty sequences. + + This renderer will either return ``tag`` unmodified if the provided + sequence has no items, or return the empty string if there are any + items. + """ + if len(self.seq) > 0: + return u'' + else: + return tag + + +class TokenOnlyWebApi(resource.Resource, object): """ I provide a rend.Page implementation that only accepts POST calls, and only if they have a 'token=' arg with the correct diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index a08e89f32..57c222788 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -1,33 +1,62 @@ import json import urllib +from datetime import timedelta from zope.interface import implementer from twisted.internet import defer from twisted.internet.interfaces import IPushProducer from twisted.python.failure import Failure from twisted.web import http -from nevow import url, rend, inevow, tags as T -from nevow.inevow import IRequest - -from foolscap.api import fireEventually +from twisted.web.resource import ErrorPage +from twisted.web.resource import Resource +from twisted.web.template import ( + Element, + XMLFile, + renderElement, + renderer, + tags, +) +from hyperlink import URL +from twisted.python.filepath import FilePath from allmydata.util import base32 -from allmydata.util.encodingutil import to_str -from allmydata.uri import from_string_dirnode +from allmydata.util.encodingutil import ( + to_str, + quote_output, +) +from allmydata.uri import ( + from_string_dirnode, + from_string, + CHKFileURI, + WriteableSSKFileURI, + ReadonlySSKFileURI, +) from allmydata.interfaces import IDirectoryNode, IFileNode, IFilesystemNode, \ IImmutableFileNode, IMutableFileNode, ExistingChildError, \ NoSuchChildError, EmptyPathnameComponentError, SDMF_VERSION, MDMF_VERSION from allmydata.blacklist import ProhibitedNode from allmydata.monitor import Monitor, OperationCancelledError from allmydata import dirnode -from allmydata.web.common import text_plain, WebError, \ - IOpHandleTable, NeedOperationHandleError, \ - boolean_of_arg, get_arg, get_root, parse_replace_arg, \ - should_create_intermediate_directories, \ - getxmlfile, RenderMixin, humanize_failure, convert_children_json, \ - get_format, get_mutable_type, get_filenode_metadata, render_time, \ - MultiFormatPage +from allmydata.web.common import ( + text_plain, + WebError, + NeedOperationHandleError, + boolean_of_arg, + get_arg, + get_root, + parse_replace_arg, + should_create_intermediate_directories, + humanize_failure, + convert_children_json, + get_format, + get_mutable_type, + get_filenode_metadata, + render_time, + MultiFormatPage, + MultiFormatResource, + SlotsSequenceElement, +) from allmydata.web.filenode import ReplaceMeMixin, \ FileNodeHandler, PlaceHolderNodeHandler from allmydata.web.check_results import CheckResultsRenderer, \ @@ -38,11 +67,13 @@ from allmydata.web.operations import ReloadMixin from allmydata.web.check_results import json_check_results, \ json_check_and_repair_results + class BlockingFileError(Exception): # TODO: catch and transform """We cannot auto-create a parent directory, because there is a file in the way""" + def make_handler_for(node, client, parentnode=None, name=None): if parentnode: assert IDirectoryNode.providedBy(parentnode) @@ -52,31 +83,47 @@ def make_handler_for(node, client, parentnode=None, name=None): return DirectoryNodeHandler(client, node, parentnode, name) return UnknownNodeHandler(client, node, parentnode, name) -class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): - addSlash = True + +class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): def __init__(self, client, node, parentnode=None, name=None): - rend.Page.__init__(self) + super(DirectoryNodeHandler, self).__init__() self.client = client assert node self.node = node self.parentnode = parentnode self.name = name + self._operations = client.get_web_service().get_operations() - def childFactory(self, ctx, name): - name = name.decode("utf-8") + def getChild(self, name, req): + """ + Dynamically create a child for the given request and name + """ + # trying to replicate what I have observed as Nevow behavior + # for these nodes, which is that a URI like + # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash + # or no further children) renders "this" page + name = name.decode('utf8') if not name: - raise EmptyPathnameComponentError() + raise EmptyPathnameComponentError( + u"The webapi does not allow empty pathname components", + ) + d = self.node.get(name) - d.addBoth(self.got_child, ctx, name) - # got_child returns a handler resource: FileNodeHandler or - # DirectoryNodeHandler + d.addBoth(self._got_child, req, name) return d - def got_child(self, node_or_failure, ctx, name): - req = IRequest(ctx) - method = req.method - nonterminal = len(req.postpath) > 1 + def _got_child(self, node_or_failure, req, name): + """ + Callback when self.node.get has returned, meaning we have received + whatever child was requested -- that is `node.get` has + returned something (maybe an error). This method then performs + the rest of the work of the Twisted API getChild(): returning + a suitable child resource to Twisted Web. + """ + terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name + nonterminal = not terminal #len(req.postpath) > 0 + t = get_arg(req, "t", "").strip() if isinstance(node_or_failure, Failure): f = node_or_failure @@ -91,29 +138,43 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): return d else: # terminal node - if (method,t) in [ ("POST","mkdir"), ("PUT","mkdir"), - ("POST", "mkdir-with-children"), - ("POST", "mkdir-immutable") ]: + terminal_requests = ( + ("POST", "mkdir"), + ("PUT", "mkdir"), + ("POST", "mkdir-with-children"), + ("POST", "mkdir-immutable") + ) + if (req.method, t) in terminal_requests: # final directory kids = {} if t in ("mkdir-with-children", "mkdir-immutable"): req.content.seek(0) kids_json = req.content.read() - kids = convert_children_json(self.client.nodemaker, - kids_json) + kids = convert_children_json( + self.client.nodemaker, + kids_json, + ) file_format = get_format(req, None) mutable = True mt = get_mutable_type(file_format) if t == "mkdir-immutable": mutable = False - d = self.node.create_subdirectory(name, kids, - mutable=mutable, - mutable_version=mt) - d.addCallback(make_handler_for, - self.client, self.node, name) + d = self.node.create_subdirectory( + name, kids, + mutable=mutable, + mutable_version=mt, + ) + d.addCallback( + make_handler_for, + self.client, self.node, name, + ) return d - if (method,t) in ( ("PUT",""), ("PUT","uri"), ): + leaf_requests = ( + ("PUT",""), + ("PUT","uri"), + ) + if (req.method, t) in leaf_requests: # we were trying to find the leaf filenode (to put a new # file in its place), and it didn't exist. That's ok, # since that's the leaf node that we're about to create. @@ -128,19 +189,20 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): if not IDirectoryNode.providedBy(node): # we would have put a new directory here, but there was a # file in the way. - raise WebError("Unable to create directory '%s': " - "a file was in the way" % name, - http.CONFLICT) + return ErrorPage( + http.CONFLICT, + "Unable to create directory %s: a file was in the way" % quote_output(name), + "no details", + ) return make_handler_for(node, self.client, self.node, name) - def render_DELETE(self, ctx): + def render_DELETE(self, req): assert self.parentnode and self.name d = self.parentnode.delete(self.name) d.addCallback(lambda res: self.node.get_uri()) return d - def render_GET(self, ctx): - req = IRequest(ctx) + def render_GET(self, req): # This is where all of the directory-related ?t=* code goes. t = get_arg(req, "t", "").strip() @@ -153,33 +215,39 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): return "" if not t: - # render the directory as HTML, using the docFactory and Nevow's - # whole templating thing. - return DirectoryAsHTML(self.node, - self.client.mutable_file_default) + # render the directory as HTML + return renderElement( + req, + DirectoryAsHTML( + self.node, + self.client.mutable_file_default, + ) + ) if t == "json": - return DirectoryJSONMetadata(ctx, self.node) + return _directory_json_metadata(req, self.node) if t == "info": return MoreInfo(self.node) if t == "uri": - return DirectoryURI(ctx, self.node) + return _directory_uri(req, self.node) if t == "readonly-uri": - return DirectoryReadonlyURI(ctx, self.node) + return _directory_readonly_uri(req, self.node) if t == 'rename-form': - return RenameForm(self.node) + return renderElement( + req, + RenameForm(self.node), + ) raise WebError("GET directory: bad t=%s" % t) - def render_PUT(self, ctx): - req = IRequest(ctx) + def render_PUT(self, req): t = get_arg(req, "t", "").strip() replace = parse_replace_arg(get_arg(req, "replace", "true")) if t == "mkdir": # our job was done by the traversal/create-intermediate-directory # process that got us here. - return text_plain(self.node.get_uri(), ctx) # TODO: urlencode + return text_plain(self.node.get_uri(), req) # TODO: urlencode if t == "uri": if not replace: # they're trying to set_uri and that name is already occupied @@ -191,8 +259,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): raise WebError("PUT to a directory") - def render_POST(self, ctx): - req = IRequest(ctx) + def render_POST(self, req): t = get_arg(req, "t", "").strip() if t == "mkdir": @@ -202,7 +269,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): elif t == "mkdir-immutable": d = self._POST_mkdir_immutable(req) elif t == "upload": - d = self._POST_upload(ctx) # this one needs the context + d = self._POST_upload(req) # this one needs the context elif t == "uri": d = self._POST_uri(req) elif t == "delete" or t == "unlink": @@ -214,17 +281,17 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): elif t == "check": d = self._POST_check(req) elif t == "start-deep-check": - d = self._POST_start_deep_check(ctx) + d = self._POST_start_deep_check(req) elif t == "stream-deep-check": - d = self._POST_stream_deep_check(ctx) + d = self._POST_stream_deep_check(req) elif t == "start-manifest": - d = self._POST_start_manifest(ctx) + d = self._POST_start_manifest(req) elif t == "start-deep-size": - d = self._POST_start_deep_size(ctx) + d = self._POST_start_deep_size(req) elif t == "start-deep-stats": - d = self._POST_start_deep_stats(ctx) + d = self._POST_start_deep_stats(req) elif t == "stream-manifest": - d = self._POST_stream_manifest(ctx) + d = self._POST_stream_manifest(req) elif t == "set_children" or t == "set-children": d = self._POST_set_children(req) else: @@ -232,7 +299,10 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): when_done = get_arg(req, "when_done", None) if when_done: - d.addCallback(lambda res: url.URL.fromString(when_done)) + def done(res): + req.redirect(when_done) + return res + d.addCallback(done) return d def _POST_mkdir(self, req): @@ -284,8 +354,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): d.addCallback(lambda child: child.get_uri()) # TODO: urlencode return d - def _POST_upload(self, ctx): - req = IRequest(ctx) + def _POST_upload(self, req): charset = get_arg(req, "_charset", "utf-8") contents = req.fields["file"] assert contents.filename is None or isinstance(contents.filename, str) @@ -305,7 +374,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): # since POST /uri/path/file?t=upload is equivalent to # POST /uri/path/dir?t=upload&name=foo, just do the same thing that # childFactory would do. Things are cleaner if we only do a subset of - # them, though, so we don't do: d = self.childFactory(ctx, name) + # them, though, so we don't do: d = self.childFactory(req, name) d = self.node.get(name) def _maybe_got_node(node_or_failure): @@ -322,7 +391,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): # delegate to it. We could return the resource back out of # DirectoryNodeHandler.renderHTTP, and nevow would recurse into it, # but the addCallback() that handles when_done= would break. - d.addCallback(lambda child: child.renderHTTP(ctx)) + d.addCallback(lambda child: child.render(req)) return d def _POST_uri(self, req): @@ -442,37 +511,36 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): return d def _start_operation(self, monitor, renderer, ctx): - table = IOpHandleTable(ctx) - table.add_monitor(ctx, monitor, renderer) - return table.redirect_to(ctx) + self._operations.add_monitor(ctx, monitor, renderer) + return self._operations.redirect_to(ctx) - def _POST_start_deep_check(self, ctx): + def _POST_start_deep_check(self, req): # check this directory and everything reachable from it - if not get_arg(ctx, "ophandle"): + if not get_arg(req, "ophandle"): raise NeedOperationHandleError("slow operation requires ophandle=") - verify = boolean_of_arg(get_arg(ctx, "verify", "false")) - repair = boolean_of_arg(get_arg(ctx, "repair", "false")) - add_lease = boolean_of_arg(get_arg(ctx, "add-lease", "false")) + verify = boolean_of_arg(get_arg(req, "verify", "false")) + repair = boolean_of_arg(get_arg(req, "repair", "false")) + add_lease = boolean_of_arg(get_arg(req, "add-lease", "false")) if repair: monitor = self.node.start_deep_check_and_repair(verify, add_lease) renderer = DeepCheckAndRepairResultsRenderer(self.client, monitor) else: monitor = self.node.start_deep_check(verify, add_lease) renderer = DeepCheckResultsRenderer(self.client, monitor) - return self._start_operation(monitor, renderer, ctx) + return self._start_operation(monitor, renderer, req) - def _POST_stream_deep_check(self, ctx): - verify = boolean_of_arg(get_arg(ctx, "verify", "false")) - repair = boolean_of_arg(get_arg(ctx, "repair", "false")) - add_lease = boolean_of_arg(get_arg(ctx, "add-lease", "false")) - walker = DeepCheckStreamer(ctx, self.node, verify, repair, add_lease) + def _POST_stream_deep_check(self, req): + verify = boolean_of_arg(get_arg(req, "verify", "false")) + repair = boolean_of_arg(get_arg(req, "repair", "false")) + add_lease = boolean_of_arg(get_arg(req, "add-lease", "false")) + walker = DeepCheckStreamer(req, self.node, verify, repair, add_lease) monitor = self.node.deep_traverse(walker) walker.setMonitor(monitor) # register to hear stopProducing. The walker ignores pauseProducing. - IRequest(ctx).registerProducer(walker, True) + req.registerProducer(walker, True) d = monitor.when_done() def _done(res): - IRequest(ctx).unregisterProducer() + req.unregisterProducer() return res d.addBoth(_done) def _cancelled(f): @@ -488,36 +556,36 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): d.addErrback(_error) return d - def _POST_start_manifest(self, ctx): - if not get_arg(ctx, "ophandle"): + def _POST_start_manifest(self, req): + if not get_arg(req, "ophandle"): raise NeedOperationHandleError("slow operation requires ophandle=") monitor = self.node.build_manifest() renderer = ManifestResults(self.client, monitor) - return self._start_operation(monitor, renderer, ctx) + return self._start_operation(monitor, renderer, req) - def _POST_start_deep_size(self, ctx): - if not get_arg(ctx, "ophandle"): + def _POST_start_deep_size(self, req): + if not get_arg(req, "ophandle"): raise NeedOperationHandleError("slow operation requires ophandle=") monitor = self.node.start_deep_stats() renderer = DeepSizeResults(self.client, monitor) - return self._start_operation(monitor, renderer, ctx) + return self._start_operation(monitor, renderer, req) - def _POST_start_deep_stats(self, ctx): - if not get_arg(ctx, "ophandle"): + def _POST_start_deep_stats(self, req): + if not get_arg(req, "ophandle"): raise NeedOperationHandleError("slow operation requires ophandle=") monitor = self.node.start_deep_stats() renderer = DeepStatsResults(self.client, monitor) - return self._start_operation(monitor, renderer, ctx) + return self._start_operation(monitor, renderer, req) - def _POST_stream_manifest(self, ctx): - walker = ManifestStreamer(ctx, self.node) + def _POST_stream_manifest(self, req): + walker = ManifestStreamer(req, self.node) monitor = self.node.deep_traverse(walker) walker.setMonitor(monitor) # register to hear stopProducing. The walker ignores pauseProducing. - IRequest(ctx).registerProducer(walker, True) + req.registerProducer(walker, True) d = monitor.when_done() def _done(res): - IRequest(ctx).unregisterProducer() + req.unregisterProducer() return res d.addBoth(_done) def _cancelled(f): @@ -564,51 +632,59 @@ def abbreviated_dirnode(dirnode): SPACE = u"\u00A0"*2 -class DirectoryAsHTML(rend.Page): +class DirectoryAsHTML(Element): # The remainder of this class is to render the directory into # human+browser -oriented HTML. - docFactory = getxmlfile("directory.xhtml") - addSlash = True + loader = XMLFile(FilePath(__file__).sibling("directory.xhtml")) def __init__(self, node, default_mutable_format): - rend.Page.__init__(self) + super(DirectoryAsHTML, self).__init__() self.node = node - - assert default_mutable_format in (MDMF_VERSION, SDMF_VERSION) + if default_mutable_format not in (MDMF_VERSION, SDMF_VERSION): + raise ValueError( + "Uknown mutable format '{}'".format(default_mutable_format) + ) self.default_mutable_format = default_mutable_format - def beforeRender(self, ctx): + @defer.inlineCallbacks + def render(self, request): + """ + Override Element.render .. we have async work to do before we flatten our template + """ + # this could be improved; doing a more-straightforward port + # here, which used to be in "beforeRender" .. so lots of the + # renderers don't yield/wait properly, they expect the self.* + # state that is set up here to "just be there" :/ + yield self._get_children(request) + template = Element.render(self, request) + defer.returnValue(template) + + @defer.inlineCallbacks + def _get_children(self, req): # attempt to get the dirnode's children, stashing them (or the # failure that results) for later use - d = self.node.list() - def _good(children): - # Deferreds don't optimize out tail recursion, and the way - # Nevow's flattener handles Deferreds doesn't take this into - # account. As a result, large lists of Deferreds that fire in the - # same turn (i.e. the output of defer.succeed) will cause a stack - # overflow. To work around this, we insert a turn break after - # every 100 items, using foolscap's fireEventually(). This gives - # the stack a chance to be popped. It would also work to put - # every item in its own turn, but that'd be a lot more - # inefficient. This addresses ticket #237, for which I was never - # able to create a failing unit test. - output = [] - for i,item in enumerate(sorted(children.items())): - if i % 100 == 0: - output.append(fireEventually(item)) - else: - output.append(item) - self.dirnode_children = output - return ctx - def _bad(f): - text, code = humanize_failure(f) - self.dirnode_children = None + try: + children = yield self.node.list() + except Exception as e: + text, code = humanize_failure(Failure(e)) + children = None self.dirnode_children_error = text - return ctx - d.addCallbacks(_good, _bad) - return d - def render_title(self, ctx, data): + self.dirnode_children = children + defer.returnValue(self.dirnode_children) + + @renderer + def children(self, req, tag): + return SlotsSequenceElement( + tag, + [ + self._child_slots(req, fname, data[0], data[1]) + for fname, data in self.dirnode_children.items() + ] + ) + + @renderer + def title(self, req, tag): si_s = abbreviated_dirnode(self.node) header = ["Tahoe-LAFS - Directory SI=%s" % si_s] if self.node.is_unknown(): @@ -619,51 +695,57 @@ class DirectoryAsHTML(rend.Page): header.append(" (read-only)") else: header.append(" (modifiable)") - return ctx.tag[header] + return tag(header) - def render_header(self, ctx, data): + @renderer + def header(self, req, tag): si_s = abbreviated_dirnode(self.node) - header = ["Tahoe-LAFS Directory SI=", T.span(class_="data-chars")[si_s]] + header = ["Tahoe-LAFS Directory SI=", tags.span(si_s, class_="data-chars")] if self.node.is_unknown(): header.append(" (unknown)") elif not self.node.is_mutable(): header.append(" (immutable)") elif self.node.is_readonly(): header.append(" (read-only)") - return ctx.tag[header] + return tag(header) - def render_welcome(self, ctx, data): - link = get_root(ctx) - return ctx.tag[T.a(href=link)["Return to Welcome page"]] + @renderer + def welcome(self, req, tag): + link = get_root(req) + return tag(tags.a("Return to Welcome page", href=link)) - def render_show_readonly(self, ctx, data): + @renderer + def show_readonly(self, req, tag): if self.node.is_unknown() or self.node.is_readonly(): return "" rocap = self.node.get_readonly_uri() - root = get_root(ctx) + root = get_root(req) uri_link = "%s/uri/%s/" % (root, urllib.quote(rocap)) - return ctx.tag[T.a(href=uri_link)["Read-Only Version"]] + return tag(tags.a("Read-Only Version", href=uri_link)) - def render_try_children(self, ctx, data): + @renderer + def try_children(self, req, tag): # if the dirnode can be retrived, render a table of children. # Otherwise, render an apologetic error message. if self.dirnode_children is not None: - return ctx.tag + return tag else: - return T.div[T.p["Error reading directory:"], - T.p[self.dirnode_children_error]] + return tags.div( + tags.p("Error reading directory:"), + tags.p(self.dirnode_children_error), + ) - def data_children(self, ctx, data): - return self.dirnode_children - - def render_row(self, ctx, data): - name, (target, metadata) = data + def _child_slots(self, req, name, target, metadata): + """ + :returns: a dict of key/values to give to each item in the table + of sub-items that directory.xhtml defines (this method is + called by the 'children' renderer) + """ name = name.encode("utf-8") - assert not isinstance(name, unicode) nameurl = urllib.quote(name, safe="") # encode any slashes too - root = get_root(ctx) - here = "%s/uri/%s/" % (root, urllib.quote(self.node.get_uri())) + root = get_root(req) + here = "{}/uri/{}/".format(root, urllib.quote(self.node.get_uri())) if self.node.is_unknown() or self.node.is_readonly(): unlink = "-" rename = "-" @@ -671,22 +753,29 @@ class DirectoryAsHTML(rend.Page): # this creates a button which will cause our _POST_unlink method # to be invoked, which unlinks the file and then redirects the # browser back to this directory - unlink = T.form(action=here, method="post")[ - T.input(type='hidden', name='t', value='unlink'), - T.input(type='hidden', name='name', value=name), - T.input(type='hidden', name='when_done', value="."), - T.input(type='submit', _class='btn', value='unlink', name="unlink"), - ] + unlink = tags.form( + [ + tags.input(type='hidden', name='t', value='unlink'), + tags.input(type='hidden', name='name', value=name), + tags.input(type='hidden', name='when_done', value="."), + tags.input(type='submit', class_='btn', value='unlink', name="unlink"), + ], + action=here, method="post" + ) + rename = tags.form( + [ + tags.input(type='hidden', name='t', value='rename-form'), + tags.input(type='hidden', name='name', value=name), + tags.input(type='hidden', name='when_done', value="."), + tags.input(type='submit', class_='btn', value='rename/relink', name="rename"), + ], + action=here, method="get", + ) - rename = T.form(action=here, method="get")[ - T.input(type='hidden', name='t', value='rename-form'), - T.input(type='hidden', name='name', value=name), - T.input(type='hidden', name='when_done', value="."), - T.input(type='submit', _class='btn', value='rename/relink', name="rename"), - ] - - ctx.fillSlots("unlink", unlink) - ctx.fillSlots("rename", rename) + slots = { + "unlink": unlink, + "rename": rename, + } times = [] linkcrtime = metadata.get('tahoe', {}).get("linkcrtime") @@ -700,16 +789,16 @@ class DirectoryAsHTML(rend.Page): linkmotime = metadata.get('tahoe', {}).get("linkmotime") if linkmotime is not None: if times: - times.append(T.br()) + times.append(tags.br()) times.append("lmo: " + render_time(linkmotime)) else: # For backwards-compatibility with links last modified by Tahoe < 1.4.0: if "mtime" in metadata: mtime = render_time(metadata["mtime"]) if times: - times.append(T.br()) + times.append(tags.br()) times.append("m: " + mtime) - ctx.fillSlots("times", times) + slots["times"] = times assert IFilesystemNode.providedBy(target), target target_uri = target.get_uri() or "" @@ -720,145 +809,187 @@ class DirectoryAsHTML(rend.Page): # secret directory URI from the URL, send the browser to a URI-based # page that doesn't know about the directory at all dlurl = "%s/file/%s/@@named=/%s" % (root, quoted_uri, nameurl) - - ctx.fillSlots("filename", T.a(href=dlurl, rel="noreferrer")[name]) - ctx.fillSlots("type", "SSK") - - ctx.fillSlots("size", "?") - - info_link = "%s/uri/%s?t=info" % (root, quoted_uri) + slots["filename"] = tags.a(name, href=dlurl, rel="noreferrer") + slots["type"] = "SSK" + slots["size"] = "?" + info_link = "{}/uri/{}?t=info".format(root, quoted_uri) elif IImmutableFileNode.providedBy(target): dlurl = "%s/file/%s/@@named=/%s" % (root, quoted_uri, nameurl) - - ctx.fillSlots("filename", T.a(href=dlurl, rel="noreferrer")[name]) - ctx.fillSlots("type", "FILE") - - ctx.fillSlots("size", target.get_size()) - - info_link = "%s/uri/%s?t=info" % (root, quoted_uri) + slots["filename"] = tags.a(name, href=dlurl, rel="noreferrer") + slots["type"] = "FILE" + slots["size"] = str(target.get_size()) + info_link = "{}/uri/{}?t=info".format(root, quoted_uri) elif IDirectoryNode.providedBy(target): # directory uri_link = "%s/uri/%s/" % (root, urllib.quote(target_uri)) - ctx.fillSlots("filename", T.a(href=uri_link)[name]) + slots["filename"] = tags.a(name, href=uri_link) if not target.is_mutable(): dirtype = "DIR-IMM" elif target.is_readonly(): dirtype = "DIR-RO" else: dirtype = "DIR" - ctx.fillSlots("type", dirtype) - ctx.fillSlots("size", "-") + slots["type"] = dirtype + slots["size"] = "-" info_link = "%s/uri/%s/?t=info" % (root, quoted_uri) elif isinstance(target, ProhibitedNode): - ctx.fillSlots("filename", T.strike[name]) if IDirectoryNode.providedBy(target.wrapped_node): blacklisted_type = "DIR-BLACKLISTED" else: blacklisted_type = "BLACKLISTED" - ctx.fillSlots("type", blacklisted_type) - ctx.fillSlots("size", "-") + slots["type"] = blacklisted_type + slots["size"] = "-" + slots["info"] = ["Access Prohibited:", tags.br, target.reason] + slots["filename"] = tags.strike(name) info_link = None - ctx.fillSlots("info", ["Access Prohibited:", T.br, target.reason]) else: # unknown - ctx.fillSlots("filename", name) if target.get_write_uri() is not None: unknowntype = "?" elif not self.node.is_mutable() or target.is_alleged_immutable(): unknowntype = "?-IMM" else: unknowntype = "?-RO" - ctx.fillSlots("type", unknowntype) - ctx.fillSlots("size", "-") + slots["filename"] = name + slots["type"] = unknowntype + slots["size"] = "-" # use a directory-relative info link, so we can extract both the # writecap and the readcap info_link = "%s?t=info" % urllib.quote(name) if info_link: - ctx.fillSlots("info", T.a(href=info_link)["More Info"]) + slots["info"] = tags.a("More Info", href=info_link) - return ctx.tag + return slots # XXX: similar to render_upload_form and render_mkdir_form in root.py. - def render_forms(self, ctx, data): + # XXX: also, is generating so much HTML in code a great idea? -> templates? + @renderer + def forms(self, req, data): forms = [] + # making this consistent with the other forms, and also + # because action="." doesn't get us back to the dir page (but + # instead /uri itself) + root = get_root(req) + here = "{}/uri/{}/".format(root, urllib.quote(self.node.get_uri())) + if self.node.is_readonly(): - return T.div["No upload forms: directory is read-only"] + return tags.div("No upload forms: directory is read-only") if self.dirnode_children is None: - return T.div["No upload forms: directory is unreadable"] + return tags.div("No upload forms: directory is unreadable") - mkdir_sdmf = T.input(type='radio', name='format', - value='sdmf', id='mkdir-sdmf', - checked='checked') - mkdir_mdmf = T.input(type='radio', name='format', - value='mdmf', id='mkdir-mdmf') + mkdir_sdmf = tags.input( + type='radio', + name='format', + value='sdmf', + id='mkdir-sdmf', + checked='checked', + ) + mkdir_mdmf = tags.input( + type='radio', + name='format', + value='mdmf', + id='mkdir-mdmf', + ) - mkdir_form = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ - T.input(type="hidden", name="t", value="mkdir"), - T.input(type="hidden", name="when_done", value="."), - T.legend(class_="freeform-form-label")["Create a new directory in this directory"], - "New directory name:"+SPACE, T.br, - T.input(type="text", name="name"), SPACE, - T.div(class_="form-inline")[ - mkdir_sdmf, T.label(for_='mutable-directory-sdmf')[SPACE, "SDMF"], SPACE*2, - mkdir_mdmf, T.label(for_='mutable-directory-mdmf')[SPACE, "MDMF (experimental)"] + mkdir_form = tags.form( + [ + tags.fieldset([ + tags.input(type="hidden", name="t", value="mkdir"), + tags.input(type="hidden", name="when_done", value="."), + tags.legend("Create a new directory in this directory", class_="freeform-form-label"), + "New directory name:"+SPACE, tags.br, + tags.input(type="text", name="name"), SPACE, + tags.div( + [ + mkdir_sdmf, tags.label(SPACE, "SDMF", for_='mutable-directory-sdmf'), SPACE*2, + mkdir_mdmf, tags.label(SPACE, "MDMF (experimental)", for_='mutable-directory-mdmf'), + ], + class_="form-inline", + ), + tags.input( + type="submit", + class_="btn", + value="Create", + ), + ]) ], - T.input(type="submit", class_="btn", value="Create") - ]] - forms.append(T.div(class_="freeform-form")[mkdir_form]) + action=here, + method="post", + enctype="multipart/form-data", + ) + forms.append(tags.div(mkdir_form, class_="freeform-form")) - upload_chk = T.input(type='radio', name='format', - value='chk', id='upload-chk', - checked='checked') - upload_sdmf = T.input(type='radio', name='format', - value='sdmf', id='upload-sdmf') - upload_mdmf = T.input(type='radio', name='format', - value='mdmf', id='upload-mdmf') + upload_chk = tags.input( + type='radio', + name='format', + value='chk', + id='upload-chk', + checked='checked', + ) + upload_sdmf = tags.input( + type='radio', + name='format', + value='sdmf', + id='upload-sdmf', + ) + upload_mdmf = tags.input( + type='radio', + name='format', + value='mdmf', + id='upload-mdmf', + ) - upload_form = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ - T.input(type="hidden", name="t", value="upload"), - T.input(type="hidden", name="when_done", value="."), - T.legend(class_="freeform-form-label")["Upload a file to this directory"], - "Choose a file to upload:"+SPACE, - T.input(type="file", name="file", class_="freeform-input-file"), SPACE, - T.div(class_="form-inline")[ - upload_chk, T.label(for_="upload-chk") [SPACE, "Immutable"], SPACE*2, - upload_sdmf, T.label(for_="upload-sdmf")[SPACE, "SDMF"], SPACE*2, - upload_mdmf, T.label(for_="upload-mdmf")[SPACE, "MDMF (experimental)"] - ], - T.input(type="submit", class_="btn", value="Upload"), SPACE*2, - ]] - forms.append(T.div(class_="freeform-form")[upload_form]) + upload_form = tags.form( + tags.fieldset([ + tags.input(type="hidden", name="t", value="upload"), + tags.input(type="hidden", name="when_done", value=req.uri), + tags.legend("Upload a file to this directory", class_="freeform-form-label"), + "Choose a file to upload:"+SPACE, + tags.input(type="file", name="file", class_="freeform-input-file"), SPACE, + tags.div([ + upload_chk, tags.label(SPACE, "Immutable", for_="upload-chk"), SPACE*2, + upload_sdmf, tags.label(SPACE, "SDMF", for_="upload-sdmf"), SPACE*2, + upload_mdmf, tags.label(SPACE, "MDMF (experimental)", for_="upload-mdmf"), + ], class_="form-inline"), + tags.input(type="submit", class_="btn", value="Upload"), SPACE*2, + ]), + action=req.uri, + method="post", + enctype="multipart/form-data", + ) + forms.append(tags.div(upload_form, class_="freeform-form")) - attach_form = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ T.div(class_="form-inline")[ - T.input(type="hidden", name="t", value="uri"), - T.input(type="hidden", name="when_done", value="."), - T.legend(class_="freeform-form-label")["Add a link to a file or directory which is already in Tahoe-LAFS."], - "New child name:"+SPACE, - T.input(type="text", name="name"), SPACE*2, T.br, - "URI of new child:"+SPACE, - T.input(type="text", name="uri"), SPACE, - T.input(type="submit", class_="btn", value="Attach"), - ]]] - forms.append(T.div(class_="freeform-form")[attach_form]) + attach_form = tags.form( + tags.fieldset( + tags.div([ + tags.input(type="hidden", name="t", value="uri"), + tags.input(type="hidden", name="when_done", value="."), + tags.legend("Add a link to a file or directory which is already in Tahoe-LAFS.", class_="freeform-form-label"), + "New child name:"+SPACE, + tags.input(type="text", name="name"), SPACE*2, tags.br, + "URI of new child:"+SPACE, + tags.input(type="text", name="uri"), SPACE, + tags.input(type="submit", class_="btn", value="Attach"), + ], class_="form-inline"), + ), + action=here, + method="post", + enctype="multipart/form-data", + ) + forms.append(tags.div(attach_form, class_="freeform-form")) return forms - def render_results(self, ctx, data): - req = IRequest(ctx) + @renderer + def results(self, req, tag): return get_arg(req, "results", "") -def DirectoryJSONMetadata(ctx, dirnode): +def _directory_json_metadata(req, dirnode): d = dirnode.list() def _got(children): kids = {} @@ -898,11 +1029,10 @@ def DirectoryJSONMetadata(ctx, dirnode): data = ("dirnode", contents) return json.dumps(data, indent=1) + "\n" d.addCallback(_got) - d.addCallback(text_plain, ctx) + d.addCallback(text_plain, req) def error(f): message, code = humanize_failure(f) - req = IRequest(ctx) req.setResponseCode(code) return json.dumps({ "error": message, @@ -911,40 +1041,168 @@ def DirectoryJSONMetadata(ctx, dirnode): return d -def DirectoryURI(ctx, dirnode): - return text_plain(dirnode.get_uri(), ctx) +def _directory_uri(req, dirnode): + return text_plain(dirnode.get_uri(), req) -def DirectoryReadonlyURI(ctx, dirnode): - return text_plain(dirnode.get_readonly_uri(), ctx) +def _directory_readonly_uri(req, dirnode): + return text_plain(dirnode.get_readonly_uri(), req) -class RenameForm(rend.Page): - addSlash = True - docFactory = getxmlfile("rename-form.xhtml") +class RenameForm(Element, object): - def render_title(self, ctx, data): - return ctx.tag["Directory SI=%s" % abbreviated_dirnode(self.original)] + loader = XMLFile(FilePath(__file__).sibling("rename-form.xhtml")) - def render_header(self, ctx, data): - header = ["Rename " - "in directory SI=%s" % abbreviated_dirnode(self.original), - ] + def __init__(self, original): + self.original = original + super(RenameForm, self).__init__() + + @renderer + def title(self, req, tag): + return tag("Directory SI={}".format(abbreviated_dirnode(self.original))) + + @renderer + def header(self, req, tag): + header = [ + "Rename " + "in directory SI=%s" % abbreviated_dirnode(self.original), + ] if self.original.is_readonly(): header.append(" (readonly!)") header.append(":") - return ctx.tag[header] + return tag(header) - def render_when_done(self, ctx, data): - return T.input(type="hidden", name="when_done", value=".") + @renderer + def when_done(self, req, tag): + return tags.input(type="hidden", name="when_done", value=".") - def render_get_name(self, ctx, data): - req = IRequest(ctx) + @renderer + def get_name(self, req, tag): name = get_arg(req, "name", "") - ctx.tag.attributes['value'] = name - return ctx.tag + tag.attributes['value'] = name + return tag -class ManifestResults(MultiFormatPage, ReloadMixin): - docFactory = getxmlfile("manifest.xhtml") + +class ReloadableMonitorElement(Element): + """ + Like 'ReloadMixin', but for twisted.web.template style. This + provides renderers for "reload" and "refesh" and a self.monitor + attribute (which is an instance of IMonitor) + """ + refresh_time = timedelta(seconds=60) + + def __init__(self, monitor): + self.monitor = monitor + super(ReloadableMonitorElement, self).__init__() + + @renderer + def refresh(self, req, tag): + if self.monitor.is_finished(): + return u"" + tag.attributes[u"http-equiv"] = u"refresh" + tag.attributes[u"content"] = u"{}".format(self.refresh_time.seconds) + return tag + + @renderer + def reload(self, req, tag): + if self.monitor.is_finished(): + return u"" + reload_url = URL.from_text(u"{}".format(req.path)) + cancel_button = tags.form( + [ + tags.input(type=u"submit", value=u"Cancel"), + ], + action=reload_url.replace(query={u"t": u"cancel"}).to_uri().to_text(), + method=u"POST", + enctype=u"multipart/form-data", + ) + + return tag([ + u"Operation still running: ", + tags.a( + u"Reload", + href=reload_url.replace(query={u"output": u"html"}).to_uri().to_text(), + ), + cancel_button, + ]) + + +def _slashify_path(path): + """ + Converts a tuple from a 'manifest' path into a string with slashes + in it + """ + if not path: + return "" + return "/".join([p.encode("utf-8") for p in path]) + + +def _cap_to_link(root, path, cap): + """ + Turns a capability-string into a WebAPI link tag + + :param text root: the root piece of the URI + + :param text cap: the capability-string + + :returns: something suitable for `IRenderable`, specifically + either a valid local link (tags.a instance) to the capability + or an empty string. + """ + if cap: + root_url = URL.from_text(u"{}".format(root)) + cap_obj = from_string(cap) + if isinstance(cap_obj, (CHKFileURI, WriteableSSKFileURI, ReadonlySSKFileURI)): + uri_link = root_url.child( + u"file", + u"{}".format(urllib.quote(cap)), + u"{}".format(urllib.quote(path[-1])), + ) + else: + uri_link = root_url.child( + u"uri", + u"{}".format(urllib.quote(cap, safe="")), + ) + return tags.a(cap, href=uri_link.to_text()) + else: + return u"" + + +class ManifestElement(ReloadableMonitorElement): + loader = XMLFile(FilePath(__file__).sibling("manifest.xhtml")) + + def _si_abbrev(self): + si = self.monitor.origin_si + if not si: + return "" + return base32.b2a(si)[:6] + + @renderer + def title(self, req, tag): + return tag( + "Manifest of SI={}".format(self._si_abbrev()) + ) + + @renderer + def header(self, req, tag): + return tag( + "Manifest of SI={}".format(self._si_abbrev()) + ) + + @renderer + def items(self, req, tag): + manifest = self.monitor.get_status()["manifest"] + root = get_root(req) + rows = [ + { + "path": _slashify_path(path), + "cap": _cap_to_link(root, path, cap), + } + for path, cap in manifest + ] + return SlotsSequenceElement(tag, rows) + + +class ManifestResults(MultiFormatResource, ReloadMixin): # Control MultiFormatPage formatArgument = "output" @@ -954,21 +1212,19 @@ class ManifestResults(MultiFormatPage, ReloadMixin): self.client = client self.monitor = monitor - # The default format is HTML but the HTML renderer is just renderHTTP. - render_HTML = None - - def slashify_path(self, path): - if not path: - return "" - return "/".join([p.encode("utf-8") for p in path]) + def render_HTML(self, req): + return renderElement( + req, + ManifestElement(self.monitor) + ) def render_TEXT(self, req): req.setHeader("content-type", "text/plain") lines = [] is_finished = self.monitor.is_finished() lines.append("finished: " + {True: "yes", False: "no"}[is_finished]) - for (path, cap) in self.monitor.get_status()["manifest"]: - lines.append(self.slashify_path(path) + " " + cap) + for path, cap in self.monitor.get_status()["manifest"]: + lines.append(_slashify_path(path) + " " + cap) return "\n".join(lines) + "\n" def render_JSON(self, req): @@ -1002,37 +1258,6 @@ class ManifestResults(MultiFormatPage, ReloadMixin): # CPU. return json.dumps(status, indent=1) - def _si_abbrev(self): - si = self.monitor.origin_si - if not si: - return "" - return base32.b2a(si)[:6] - - def render_title(self, ctx): - return T.title["Manifest of SI=%s" % self._si_abbrev()] - - def render_header(self, ctx): - return T.p["Manifest of SI=%s" % self._si_abbrev()] - - def data_items(self, ctx, data): - return self.monitor.get_status()["manifest"] - - def render_row(self, ctx, path_cap): - path, cap = path_cap - ctx.fillSlots("path", self.slashify_path(path)) - root = get_root(ctx) - # TODO: we need a clean consistent way to get the type of a cap string - if cap: - if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"): - nameurl = urllib.quote(path[-1].encode("utf-8")) - uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap), - nameurl) - else: - uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe="")) - ctx.fillSlots("cap", T.a(href=uri_link)[cap]) - else: - ctx.fillSlots("cap", "") - return ctx.tag class DeepSizeResults(MultiFormatPage): # Control MultiFormatPage @@ -1062,24 +1287,30 @@ class DeepSizeResults(MultiFormatPage): } return json.dumps(status) -class DeepStatsResults(rend.Page): + +class DeepStatsResults(Resource, object): + """ + Renders the results of a 'deep-stats' operation on a directory + capability. + """ def __init__(self, client, monitor): self.client = client self.monitor = monitor - def renderHTTP(self, ctx): + def render(self, req): # JSON only - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + req.setHeader("content-type", "text/plain") s = self.monitor.get_status().copy() s["finished"] = self.monitor.is_finished() return json.dumps(s, indent=1) + @implementer(IPushProducer) class ManifestStreamer(dirnode.DeepStats): - def __init__(self, ctx, origin): + def __init__(self, req, origin): dirnode.DeepStats.__init__(self, origin) - self.req = IRequest(ctx) + self.req = req def setMonitor(self, monitor): self.monitor = monitor @@ -1134,9 +1365,9 @@ class ManifestStreamer(dirnode.DeepStats): @implementer(IPushProducer) class DeepCheckStreamer(dirnode.DeepStats): - def __init__(self, ctx, origin, verify, repair, add_lease): + def __init__(self, req, origin, verify, repair, add_lease): dirnode.DeepStats.__init__(self, origin) - self.req = IRequest(ctx) + self.req = req self.verify = verify self.repair = repair self.add_lease = add_lease @@ -1210,16 +1441,15 @@ class DeepCheckStreamer(dirnode.DeepStats): return "" -class UnknownNodeHandler(RenderMixin, rend.Page): +class UnknownNodeHandler(Resource, object): def __init__(self, client, node, parentnode=None, name=None): - rend.Page.__init__(self) + super(UnknownNodeHandler, self).__init__() assert node self.node = node self.parentnode = parentnode self.name = name - def render_GET(self, ctx): - req = IRequest(ctx) + def render_GET(self, req): t = get_arg(req, "t", "").strip() if t == "info": return MoreInfo(self.node) @@ -1229,13 +1459,13 @@ class UnknownNodeHandler(RenderMixin, rend.Page): d = self.parentnode.get_metadata_for(self.name) else: d = defer.succeed(None) - d.addCallback(lambda md: UnknownJSONMetadata(ctx, self.node, md, is_parent_known_immutable)) + d.addCallback(lambda md: UnknownJSONMetadata(req, self.node, md, is_parent_known_immutable)) return d raise WebError("GET unknown URI type: can only do t=info and t=json, not t=%s.\n" "Using a webapi server that supports a later version of Tahoe " "may help." % t) -def UnknownJSONMetadata(ctx, node, edge_metadata, is_parent_known_immutable): +def UnknownJSONMetadata(req, node, edge_metadata, is_parent_known_immutable): rw_uri = node.get_write_uri() ro_uri = node.get_readonly_uri() data = ("unknown", {}) @@ -1250,4 +1480,4 @@ def UnknownJSONMetadata(ctx, node, edge_metadata, is_parent_known_immutable): if edge_metadata is not None: data[1]['metadata'] = edge_metadata - return text_plain(json.dumps(data, indent=1) + "\n", ctx) + return text_plain(json.dumps(data, indent=1) + "\n", req) diff --git a/src/allmydata/web/directory.xhtml b/src/allmydata/web/directory.xhtml index ee94d5814..88058efa4 100644 --- a/src/allmydata/web/directory.xhtml +++ b/src/allmydata/web/directory.xhtml @@ -1,8 +1,8 @@ - + - + @@ -30,19 +30,19 @@
    -

    +

    -
    - - +
    +
    + @@ -51,25 +51,25 @@ - - - - - - - - + + + + + + + + - +
    Type Filename Size
    This directory is empty.
    This directory is empty.
    -
    +
    -
    +
    @@ -82,4 +82,4 @@
    - \ No newline at end of file + diff --git a/src/allmydata/web/download-status-timeline.xhtml b/src/allmydata/web/download-status-timeline.xhtml deleted file mode 100644 index 03fb629fc..000000000 --- a/src/allmydata/web/download-status-timeline.xhtml +++ /dev/null @@ -1,37 +0,0 @@ - - - AllMyData - Tahoe - File Download Status Timeline - - - - - - - - - - -

    File Download Status

    - -
      -
    • Started:
      • -
    • Storage Index: , - Total Size:
      • -
    • Progress: , - Status:
      • -
    - - -
    - - - - - -
    -
    - -
    Return to the Welcome Page
    - - - diff --git a/src/allmydata/web/download-status.xhtml b/src/allmydata/web/download-status.xhtml index 0c2455724..e5794ca79 100644 --- a/src/allmydata/web/download-status.xhtml +++ b/src/allmydata/web/download-status.xhtml @@ -16,7 +16,6 @@
  • Total Size:
  • Progress:
  • Status:
    • -
    • diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index e448e4b3b..1f0142df9 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -3,8 +3,12 @@ import json from twisted.web import http, static from twisted.internet import defer -from nevow import url, rend -from nevow.inevow import IRequest +from twisted.web.resource import ( + Resource, # note: Resource is an old-style class + ErrorPage, +) + +from nevow import url from allmydata.interfaces import ExistingChildError from allmydata.monitor import Monitor @@ -15,7 +19,7 @@ from allmydata.util import log, base32 from allmydata.util.encodingutil import quote_output from allmydata.blacklist import FileProhibited, ProhibitedNode -from allmydata.web.common import text_plain, WebError, RenderMixin, \ +from allmydata.web.common import text_plain, WebError, \ boolean_of_arg, get_arg, should_create_intermediate_directories, \ MyExceptionHandler, parse_replace_arg, parse_offset_arg, \ get_format, get_mutable_type, get_filenode_metadata @@ -23,7 +27,7 @@ from allmydata.web.check_results import CheckResultsRenderer, \ CheckAndRepairResultsRenderer, LiteralCheckResultsRenderer from allmydata.web.info import MoreInfo -class ReplaceMeMixin: +class ReplaceMeMixin(object): def replace_me_with_a_child(self, req, client, replace): # a new file is being uploaded in our place. file_format = get_format(req, "CHK") @@ -87,17 +91,16 @@ class ReplaceMeMixin: return d -class PlaceHolderNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): +class PlaceHolderNodeHandler(Resource, ReplaceMeMixin): def __init__(self, client, parentnode, name): - rend.Page.__init__(self) + super(PlaceHolderNodeHandler, self).__init__() self.client = client assert parentnode self.parentnode = parentnode self.name = name self.node = None - def render_PUT(self, ctx): - req = IRequest(ctx) + def render_PUT(self, req): t = get_arg(req, "t", "").strip() replace = parse_replace_arg(get_arg(req, "replace", "true")) @@ -112,8 +115,7 @@ class PlaceHolderNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): raise WebError("PUT to a file: bad t=%s" % t) - def render_POST(self, ctx): - req = IRequest(ctx) + def render_POST(self, req): t = get_arg(req, "t", "").strip() replace = boolean_of_arg(get_arg(req, "replace", "true")) if t == "upload": @@ -131,31 +133,36 @@ class PlaceHolderNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): when_done = get_arg(req, "when_done", None) if when_done: - d.addCallback(lambda res: url.URL.fromString(when_done)) + d.addCallback(lambda res: when_done) return d -class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): +class FileNodeHandler(Resource, ReplaceMeMixin, object): def __init__(self, client, node, parentnode=None, name=None): - rend.Page.__init__(self) + super(FileNodeHandler, self).__init__() self.client = client assert node self.node = node self.parentnode = parentnode self.name = name - def childFactory(self, ctx, name): - req = IRequest(ctx) + def getChild(self, name, req): if isinstance(self.node, ProhibitedNode): raise FileProhibited(self.node.reason) if should_create_intermediate_directories(req): - raise WebError("Cannot create directory %s, because its " - "parent is a file, not a directory" % quote_output(name, encoding='utf-8')) - raise WebError("Files have no children, certainly not named %s" - % quote_output(name, encoding='utf-8')) + return ErrorPage( + http.CONFLICT, + u"Cannot create directory %s, because its parent is a file, " + u"not a directory" % quote_output(name, encoding='utf-8'), + "no details" + ) + return ErrorPage( + http.BAD_REQUEST, + u"Files have no children named %s" % quote_output(name, encoding='utf-8'), + "no details", + ) - def render_GET(self, ctx): - req = IRequest(ctx) + def render_GET(self, req): t = get_arg(req, "t", "").strip() # t=info contains variable ophandles, so is not allowed an ETag. @@ -202,18 +209,17 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): self.parentnode.get_metadata_for(self.name)) else: d.addCallback(lambda ignored: None) - d.addCallback(lambda md: FileJSONMetadata(ctx, self.node, md)) + d.addCallback(lambda md: _file_json_metadata(req, self.node, md)) return d if t == "info": return MoreInfo(self.node) if t == "uri": - return FileURI(ctx, self.node) + return _file_uri(req, self.node) if t == "readonly-uri": - return FileReadOnlyURI(ctx, self.node) + return _file_read_only_uri(req, self.node) raise WebError("GET file: bad t=%s" % t) - def render_HEAD(self, ctx): - req = IRequest(ctx) + def render_HEAD(self, req): t = get_arg(req, "t", "").strip() if t: raise WebError("HEAD file: bad t=%s" % t) @@ -222,8 +228,7 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): d.addCallback(lambda dn: FileDownloader(dn, filename)) return d - def render_PUT(self, ctx): - req = IRequest(ctx) + def render_PUT(self, req): t = get_arg(req, "t", "").strip() replace = parse_replace_arg(get_arg(req, "replace", "true")) offset = parse_offset_arg(get_arg(req, "offset", None)) @@ -265,8 +270,7 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): raise WebError("PUT to a file: bad t=%s" % t) - def render_POST(self, ctx): - req = IRequest(ctx) + def render_POST(self, req): t = get_arg(req, "t", "").strip() replace = boolean_of_arg(get_arg(req, "replace", "true")) if t == "check": @@ -309,7 +313,7 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): d.addCallback(self._maybe_literal, CheckResultsRenderer) return d - def render_DELETE(self, ctx): + def render_DELETE(self, req): assert self.parentnode and self.name d = self.parentnode.delete(self.name) d.addCallback(lambda res: self.node.get_uri()) @@ -346,9 +350,9 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): return d -class FileDownloader(rend.Page): +class FileDownloader(Resource, object): def __init__(self, filenode, filename): - rend.Page.__init__(self) + super(FileDownloader, self).__init__() self.filenode = filenode self.filename = filename @@ -400,8 +404,7 @@ class FileDownloader(rend.Page): except ValueError: return None - def renderHTTP(self, ctx): - req = IRequest(ctx) + def render(self, req): gte = static.getTypeAndEncoding ctype, encoding = gte(self.filename, static.File.contentTypes, @@ -490,12 +493,12 @@ class FileDownloader(rend.Page): # We haven't written anything yet, so we can provide a # sensible error message. eh = MyExceptionHandler() - eh.renderHTTP_exception(ctx, f) + eh.renderHTTP_exception(req, f) d.addCallbacks(_finished, _error) return req.deferred -def FileJSONMetadata(ctx, filenode, edge_metadata): +def _file_json_metadata(req, filenode, edge_metadata): rw_uri = filenode.get_write_uri() ro_uri = filenode.get_readonly_uri() data = ("filenode", get_filenode_metadata(filenode)) @@ -509,16 +512,19 @@ def FileJSONMetadata(ctx, filenode, edge_metadata): if edge_metadata is not None: data[1]['metadata'] = edge_metadata - return text_plain(json.dumps(data, indent=1) + "\n", ctx) + return text_plain(json.dumps(data, indent=1) + "\n", req) -def FileURI(ctx, filenode): - return text_plain(filenode.get_uri(), ctx) -def FileReadOnlyURI(ctx, filenode): +def _file_uri(req, filenode): + return text_plain(filenode.get_uri(), req) + + +def _file_read_only_uri(req, filenode): if filenode.is_readonly(): - return text_plain(filenode.get_uri(), ctx) - return text_plain(filenode.get_readonly_uri(), ctx) + return text_plain(filenode.get_uri(), req) + return text_plain(filenode.get_readonly_uri(), req) + class FileNodeDownloadHandler(FileNodeHandler): - def childFactory(self, ctx, name): + def getChild(self, name, req): return FileNodeDownloadHandler(self.client, self.node, name=name) diff --git a/src/allmydata/web/helper.xhtml b/src/allmydata/web/helper.xhtml index 98223fe90..fab9fb8e4 100644 --- a/src/allmydata/web/helper.xhtml +++ b/src/allmydata/web/helper.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Helper Status @@ -7,25 +7,28 @@ -

    Helper Status

    +
    +

    Helper Status

    -

    Immutable Uploads

    -
      -
    • Active:
      • -
    • --
      • -
    • Bytes Fetched:
      • -
    • Incoming:
      • -
    • Encoding:
      • -
    • Bytes Encoded:
      • -
    • --
      • -
    • Total Requests:
      • -
        -
      • Already Present:
        • -
      • Need Upload:
        • -
      -
    +

    Immutable Uploads

    +
      +
    • Active:
      • +
    • --
      • +
    • Bytes Fetched:
      • +
    • Incoming:
      • +
    • Encoding:
      • +
    • Bytes Encoded:
      • +
    • --
      • +
    • Total Requests:
      • +
        +
      • Already Present:
        • +
      • Need Upload:
        • +
      +
    -
    Return to the Welcome Page
    +
    + +
    Return to the Welcome Page
    diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index ee1affaec..fa62ea23d 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -1,17 +1,47 @@ import os, urllib -from nevow import rend, tags as T -from nevow.inevow import IRequest +from twisted.python.filepath import FilePath +from twisted.web.template import tags as T, Element, renderElement, XMLFile, renderer from allmydata.util import base32 from allmydata.interfaces import IDirectoryNode, IFileNode, MDMF_VERSION -from allmydata.web.common import getxmlfile +from allmydata.web.common import MultiFormatResource from allmydata.mutable.common import UnrecoverableFileError # TODO: move -class MoreInfo(rend.Page): - addSlash = False - docFactory = getxmlfile("info.xhtml") + +class MoreInfo(MultiFormatResource): + """ + A ``Resource`` for describing more information about a node. + + :param node Node: The node to describe. + """ + + def __init__(self, node): + super(MoreInfo, self).__init__() + self.node = node + + def render_HTML(self, req): + """ + Render an HTML template describing this node. + """ + return renderElement(req, MoreInfoElement(self.node)) + + render_INFO = render_HTML + + +class MoreInfoElement(Element): + """ + An ``Element`` HTML template which can be flattened to describe this node. + + :param Node node: The node to describe. + """ + + loader = XMLFile(FilePath(__file__).sibling("info.xhtml")) + + def __init__(self, node): + super(MoreInfoElement, self).__init__() + self.original = node def abbrev(self, storage_index_or_none): if storage_index_or_none: @@ -38,27 +68,32 @@ class MoreInfo(rend.Page): return "immutable LIT file" return "unknown" - def render_title(self, ctx, data): + @renderer + def title(self, req, tag): node = self.original si = node.get_storage_index() t = "More Info for %s" % self.get_type() if si: t += " (SI=%s)" % self.abbrev(si) - return ctx.tag[t] + return tag(t) - def render_header(self, ctx, data): - return self.render_title(ctx, data) + @renderer + def header(self, req, tag): + return self.title(req, tag) - def render_type(self, ctx, data): - return ctx.tag[self.get_type()] + @renderer + def type(self, req, tag): + return tag(self.get_type()) - def render_si(self, ctx, data): + @renderer + def si(self, req, tag): si = self.original.get_storage_index() if not si: return "None" - return ctx.tag[base32.b2a(si)] + return tag(base32.b2a(si)) - def render_size(self, ctx, data): + @renderer + def size(self, req, tag): node = self.original d = node.get_current_size() def _no_size(size): @@ -70,67 +105,73 @@ class MoreInfo(rend.Page): f.trap(UnrecoverableFileError) return "?" d.addErrback(_handle_unrecoverable) - d.addCallback(lambda size: ctx.tag[size]) + d.addCallback(lambda size: tag(str(size))) return d - def render_directory_writecap(self, ctx, data): + @renderer + def directory_writecap(self, req, tag): node = self.original if not IDirectoryNode.providedBy(node): return "" if node.is_readonly(): return "" - return ctx.tag[node.get_uri()] + return tag(node.get_uri()) - def render_directory_readcap(self, ctx, data): + @renderer + def directory_readcap(self, req, tag): node = self.original if not IDirectoryNode.providedBy(node): return "" - return ctx.tag[node.get_readonly_uri()] + return tag(node.get_readonly_uri()) - def render_directory_verifycap(self, ctx, data): + @renderer + def directory_verifycap(self, req, tag): node = self.original if not IDirectoryNode.providedBy(node): return "" verifier = node.get_verify_cap() if verifier: - return ctx.tag[node.get_verify_cap().to_string()] + return tag(node.get_verify_cap().to_string()) return "" - def render_file_writecap(self, ctx, data): + @renderer + def file_writecap(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node write_uri = node.get_write_uri() if not write_uri: return "" - return ctx.tag[write_uri] + return tag(write_uri) - def render_file_readcap(self, ctx, data): + @renderer + def file_readcap(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node read_uri = node.get_readonly_uri() if not read_uri: return "" - return ctx.tag[read_uri] + return tag(read_uri) - def render_file_verifycap(self, ctx, data): + @renderer + def file_verifycap(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node verifier = node.get_verify_cap() if verifier: - return ctx.tag[node.get_verify_cap().to_string()] + return tag(node.get_verify_cap().to_string()) return "" - def get_root(self, ctx): - req = IRequest(ctx) + def get_root(self, req): # the addSlash=True gives us one extra (empty) segment depth = len(req.prepath) + len(req.postpath) - 1 link = "/".join([".."] * depth) return link - def render_raw_link(self, ctx, data): + @renderer + def raw_link(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): node = node._node @@ -138,147 +179,156 @@ class MoreInfo(rend.Page): pass else: return "" - root = self.get_root(ctx) + root = self.get_root(req) quoted_uri = urllib.quote(node.get_uri()) text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri) - return T.li["Raw data as ", T.a(href=text_plain_url)["text/plain"]] + return T.li("Raw data as ", T.a("text/plain", href=text_plain_url)) - def render_is_checkable(self, ctx, data): + @renderer + def is_checkable(self, req, tag): node = self.original si = node.get_storage_index() if si: - return ctx.tag + return tag # don't show checker button for LIT files return "" - def render_check_form(self, ctx, data): + @renderer + def check_form(self, req, tag): node = self.original quoted_uri = urllib.quote(node.get_uri()) - target = self.get_root(ctx) + "/uri/" + quoted_uri + target = self.get_root(req) + "/uri/" + quoted_uri if IDirectoryNode.providedBy(node): target += "/" check = T.form(action=target, method="post", - enctype="multipart/form-data")[ - T.fieldset[ + enctype="multipart/form-data")( + T.fieldset( T.input(type="hidden", name="t", value="check"), T.input(type="hidden", name="return_to", value="."), - T.legend(class_="freeform-form-label")["Check on this object"], - T.div[ + T.legend("Check on this object", class_="freeform-form-label"), + T.div( "Verify every bit? (EXPENSIVE):", T.input(type="checkbox", name="verify"), - ], - T.div["Repair any problems?: ", - T.input(type="checkbox", name="repair")], - T.div["Add/renew lease on all shares?: ", - T.input(type="checkbox", name="add-lease")], - T.div["Emit results in JSON format?: ", - T.input(type="checkbox", name="output", value="JSON")], + ), + T.div("Repair any problems?: ", + T.input(type="checkbox", name="repair")), + T.div("Add/renew lease on all shares?: ", + T.input(type="checkbox", name="add-lease")), + T.div("Emit results in JSON format?: ", + T.input(type="checkbox", name="output", value="JSON")), T.input(type="submit", value="Check"), - ]] - return ctx.tag[check] + )) + return tag(check) - def render_is_mutable_file(self, ctx, data): + @renderer + def is_mutable_file(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): return "" if (IFileNode.providedBy(node) and node.is_mutable() and not node.is_readonly()): - return ctx.tag + return tag return "" - def render_overwrite_form(self, ctx, data): + @renderer + def overwrite_form(self, req, tag): node = self.original - root = self.get_root(ctx) + root = self.get_root(req) action = "%s/uri/%s" % (root, urllib.quote(node.get_uri())) done_url = "%s/uri/%s?t=info" % (root, urllib.quote(node.get_uri())) overwrite = T.form(action=action, method="post", - enctype="multipart/form-data")[ - T.fieldset[ + enctype="multipart/form-data")( + T.fieldset( T.input(type="hidden", name="t", value="upload"), T.input(type='hidden', name='when_done', value=done_url), - T.legend(class_="freeform-form-label")["Overwrite"], + T.legend("Overwrite", class_="freeform-form-label"), "Upload new contents: ", T.input(type="file", name="file"), " ", T.input(type="submit", value="Replace Contents") - ]] - return ctx.tag[overwrite] + )) + return tag(overwrite) - def render_is_directory(self, ctx, data): + @renderer + def is_directory(self, req, tag): node = self.original if IDirectoryNode.providedBy(node): - return ctx.tag + return tag return "" - def render_deep_check_form(self, ctx, data): + @renderer + def deep_check_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) deep_check = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ + enctype="multipart/form-data")( + T.fieldset( T.input(type="hidden", name="t", value="start-deep-check"), T.input(type="hidden", name="return_to", value="."), - T.legend(class_="freeform-form-label")["Run a deep-check operation (EXPENSIVE)"], - T.div[ + T.legend("Run a deep-check operation (EXPENSIVE)", class_="freeform-form-label"), + T.div( "Verify every bit? (EVEN MORE EXPENSIVE):", T.input(type="checkbox", name="verify"), - ], - T.div["Repair any problems?: ", - T.input(type="checkbox", name="repair")], - T.div["Add/renew lease on all shares?: ", - T.input(type="checkbox", name="add-lease")], - T.div["Emit results in JSON format?: ", - T.input(type="checkbox", name="output", value="JSON")], + ), + T.div("Repair any problems?: ", + T.input(type="checkbox", name="repair")), + T.div("Add/renew lease on all shares?: ", + T.input(type="checkbox", name="add-lease")), + T.div("Emit results in JSON format?: ", + T.input(type="checkbox", name="output", value="JSON")), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Deep-Check"), - ]] - return ctx.tag[deep_check] + )) + return tag(deep_check) - def render_deep_size_form(self, ctx, data): + @renderer + def deep_size_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) deep_size = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ + enctype="multipart/form-data")( + T.fieldset( T.input(type="hidden", name="t", value="start-deep-size"), - T.legend(class_="freeform-form-label")["Run a deep-size operation (EXPENSIVE)"], + T.legend("Run a deep-size operation (EXPENSIVE)", class_="freeform-form-label"), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Deep-Size"), - ]] - return ctx.tag[deep_size] + )) + return tag(deep_size) - def render_deep_stats_form(self, ctx, data): + @renderer + def deep_stats_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) deep_stats = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ + enctype="multipart/form-data")( + T.fieldset( T.input(type="hidden", name="t", value="start-deep-stats"), - T.legend(class_="freeform-form-label")["Run a deep-stats operation (EXPENSIVE)"], + T.legend("Run a deep-stats operation (EXPENSIVE)", class_="freeform-form-label"), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Deep-Stats"), - ]] - return ctx.tag[deep_stats] + )) + return tag(deep_stats) - def render_manifest_form(self, ctx, data): + @renderer + def manifest_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) manifest = T.form(action=".", method="post", - enctype="multipart/form-data")[ - T.fieldset[ + enctype="multipart/form-data")( + T.fieldset( T.input(type="hidden", name="t", value="start-manifest"), - T.legend(class_="freeform-form-label")["Run a manifest operation (EXPENSIVE)"], - T.div["Output Format: ", + T.legend("Run a manifest operation (EXPENSIVE)", class_="freeform-form-label"), + T.div("Output Format: ", T.select(name="output") - [ T.option(value="html", selected="true")["HTML"], - T.option(value="text")["text"], - T.option(value="json")["JSON"], - ], - ], + ( T.option("HTML", value="html", selected="true"), + T.option("text", value="text"), + T.option("JSON", value="json"), + ), + ), T.input(type="hidden", name="ophandle", value=ophandle), T.input(type="submit", value="Manifest"), - ]] - return ctx.tag[manifest] + )) + return tag(manifest) # TODO: edge metadata diff --git a/src/allmydata/web/info.xhtml b/src/allmydata/web/info.xhtml index 376360e2f..d5ee0f9cb 100644 --- a/src/allmydata/web/info.xhtml +++ b/src/allmydata/web/info.xhtml @@ -1,67 +1,67 @@ - - + + <link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/icon.png" rel="shortcut icon" /> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> </head><body> -<h1 n:render="header"></h1> +<h1 t:render="header"></h1> <ul> - <li>Object Type: <span n:render="type" /></li> - <li>Storage Index: <tt n:render="si" /></li> - <li>Object Size: <span n:render="size" /></li> + <li>Object Type: <span t:render="type" /></li> + <li>Storage Index: <tt t:render="si" /></li> + <li>Object Size: <span t:render="size" /></li> <li>Access Caps (Tahoe-URIs): <table class="table-headings-left"> - <span n:render="is_directory"> + <span t:render="is_directory"> <tr> <th>Directory writecap</th> - <td><tt class="data-chars" n:render="directory_writecap" /></td> + <td><tt class="data-chars" t:render="directory_writecap" /></td> </tr> <tr> <th>Directory readcap</th> - <td><tt class="data-chars" n:render="directory_readcap" /></td> + <td><tt class="data-chars" t:render="directory_readcap" /></td> </tr> <tr> <th>Directory verifycap</th> - <td><tt class="data-chars" n:render="directory_verifycap" /></td> + <td><tt class="data-chars" t:render="directory_verifycap" /></td> </tr> </span> <tr> <th>File writecap</th> - <td><tt class="data-chars" n:render="file_writecap" /></td> + <td><tt class="data-chars" t:render="file_writecap" /></td> </tr> <tr> <th>File readcap</th> - <td><tt class="data-chars" n:render="file_readcap" /></td> + <td><tt class="data-chars" t:render="file_readcap" /></td> </tr> <tr> <th>File verifycap</th> - <td><tt class="data-chars" n:render="file_verifycap" /></td> + <td><tt class="data-chars" t:render="file_verifycap" /></td> </tr> </table></li> <li><a href="?t=json">JSON</a></li> - <li n:render="raw_link" /> + <li t:render="raw_link" /> </ul> -<div n:render="is_checkable"> +<div t:render="is_checkable"> <h2>Checker Operations</h2> - <div n:render="check_form" /> + <div t:render="check_form" /> </div> -<div n:render="is_mutable_file"> +<div t:render="is_mutable_file"> <h2>Mutable File Operations</h2> - <div n:render="overwrite_form" /> + <div t:render="overwrite_form" /> </div> -<div n:render="is_directory"> +<div t:render="is_directory"> <h2>Directory Operations</h2> - <div n:render="deep_check_form" /> - <div n:render="deep_size_form" /> - <div n:render="deep_stats_form" /> - <div n:render="manifest_form" /> + <div t:render="deep_check_form" /> + <div t:render="deep_size_form" /> + <div t:render="deep_stats_form" /> + <div t:render="manifest_form" /> </div> </body></html> diff --git a/src/allmydata/web/introducer.xhtml b/src/allmydata/web/introducer.xhtml index f64f7e092..d55d94082 100644 --- a/src/allmydata/web/introducer.xhtml +++ b/src/allmydata/web/introducer.xhtml @@ -1,4 +1,4 @@ -<html xmlns:n="http://nevow.com/ns/nevow/0.1"><head> +<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1"><head> <title>Tahoe-LAFS - Introducer Status @@ -10,23 +10,23 @@

    This Introducer

    - - - - +
    My nodeid:
    My versions:
    Tahoe-LAFS code imported from:
    + + +
    My nodeid:
    My versions:
    Tahoe-LAFS code imported from:
    -
    Announcement Summary:
    -
    Subscription Summary:
    +
    Announcement Summary:
    +
    Subscription Summary:

    Service Announcements

    - - +
    + @@ -34,23 +34,23 @@ - + - - - +
    +
    + + + - +
    Nickname
    ServerID
    		Version Service Name
    -
    -
    no peers!
    no peers!

    Subscribed Clients

    - - +
    + @@ -59,20 +59,20 @@ - + - - - - +
    +
    + + + + - +
    Nickname
    Tub ID
    		Version Subscribed To
    -
    -
    no subscribers!
    no subscribers!
    -

    Page rendered at

    +

    Page rendered at

    diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 349165e40..f57a5232a 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -1,35 +1,56 @@ import time, os -from nevow import rend -from nevow.static import File as nevow_File -from nevow.util import resource_filename +from pkg_resources import resource_filename +from twisted.web.template import Element, XMLFile, renderElement, renderer +from twisted.python.filepath import FilePath +from twisted.web import static import allmydata import json -from allmydata import get_package_versions_string +from allmydata.version_checks import get_package_versions_string from allmydata.util import idlib from allmydata.web.common import ( - getxmlfile, render_time, - MultiFormatPage, + MultiFormatResource, + SlotsSequenceElement, ) -class IntroducerRoot(MultiFormatPage): +class IntroducerRoot(MultiFormatResource): + """ + A ``Resource`` intended as the root resource for introducers. - addSlash = True - docFactory = getxmlfile("introducer.xhtml") - - child_operations = None + :param _IntroducerNode introducer_node: The introducer node to template + information about. + """ def __init__(self, introducer_node): + super(IntroducerRoot, self).__init__() self.introducer_node = introducer_node self.introducer_service = introducer_node.getServiceNamed("introducer") - rend.Page.__init__(self, introducer_node) + # necessary as a root Resource + self.putChild("", self) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, nevow_File(os.path.join(static_dir, filen))) + self.putChild(filen, static.File(os.path.join(static_dir, filen))) + + def _create_element(self): + """ + Create a ``IntroducerRootElement`` which can be flattened into an HTML + response. + """ + return IntroducerRootElement( + self.introducer_node, self.introducer_service) + + def render_HTML(self, req): + """ + Render an HTML template describing this introducer node. + """ + return renderElement(req, self._create_element()) def render_JSON(self, req): + """ + Render JSON describing this introducer node. + """ res = {} counts = {} @@ -37,7 +58,7 @@ class IntroducerRoot(MultiFormatPage): if s.service_name not in counts: counts[s.service_name] = 0 counts[s.service_name] += 1 - res["subscription_summary"] = counts + res[u"subscription_summary"] = counts announcement_summary = {} for ad in self.introducer_service.get_announcements(): @@ -45,21 +66,40 @@ class IntroducerRoot(MultiFormatPage): if service_name not in announcement_summary: announcement_summary[service_name] = 0 announcement_summary[service_name] += 1 - res["announcement_summary"] = announcement_summary + res[u"announcement_summary"] = announcement_summary - return json.dumps(res, indent=1) + "\n" + return json.dumps(res, indent=1) + b"\n" - # FIXME: This code is duplicated in root.py and introweb.py. - def data_rendered_at(self, ctx, data): - return render_time(time.time()) - def data_version(self, ctx, data): - return get_package_versions_string() - def data_import_path(self, ctx, data): - return str(allmydata).replace("/", "/ ") # XXX kludge for wrapping - def data_my_nodeid(self, ctx, data): - return idlib.nodeid_b2a(self.introducer_node.nodeid) - def render_announcement_summary(self, ctx, data): +class IntroducerRootElement(Element): + """ + An ``Element`` HTML template which can be flattened to describe this + introducer node. + + :param _IntroducerNode introducer_node: The introducer node to describe. + :param IntroducerService introducer_service: The introducer service created + by the node. + """ + + loader = XMLFile(FilePath(__file__).sibling("introducer.xhtml")) + + def __init__(self, introducer_node, introducer_service): + super(IntroducerRootElement, self).__init__() + self.introducer_node = introducer_node + self.introducer_service = introducer_service + self.node_data_dict = { + "my_nodeid": idlib.nodeid_b2a(self.introducer_node.nodeid), + "version": get_package_versions_string(), + "import_path": str(allmydata).replace("/", "/ "), # XXX kludge for wrapping + "rendered_at": render_time(time.time()), + } + + @renderer + def node_data(self, req, tag): + return tag.fillSlots(**self.node_data_dict) + + @renderer + def announcement_summary(self, req, tag): services = {} for ad in self.introducer_service.get_announcements(): if ad.service_name not in services: @@ -67,44 +107,43 @@ class IntroducerRoot(MultiFormatPage): services[ad.service_name] += 1 service_names = services.keys() service_names.sort() - return ", ".join(["%s: %d" % (service_name, services[service_name]) - for service_name in service_names]) + return u", ".join(u"{}: {}".format(service_name, services[service_name]) + for service_name in service_names) - def render_client_summary(self, ctx, data): + @renderer + def client_summary(self, req, tag): counts = {} for s in self.introducer_service.get_subscribers(): if s.service_name not in counts: counts[s.service_name] = 0 counts[s.service_name] += 1 - return ", ".join([ "%s: %d" % (name, counts[name]) - for name in sorted(counts.keys()) ] ) + return u", ".join(u"{}: {}".format(name, counts[name]) + for name in sorted(counts.keys())) - def data_services(self, ctx, data): + @renderer + def services(self, req, tag): services = self.introducer_service.get_announcements() services.sort(key=lambda ad: (ad.service_name, ad.nickname)) - return services + services = [{ + "serverid": ad.serverid, + "nickname": ad.nickname, + "connection-hints": + u"connection hints: " + u" ".join(ad.connection_hints), + "connected": u"?", + "announced": render_time(ad.when), + "version": ad.version, + "service_name": ad.service_name, + } for ad in services] + return SlotsSequenceElement(tag, services) - def render_service_row(self, ctx, ad): - ctx.fillSlots("serverid", ad.serverid) - ctx.fillSlots("nickname", ad.nickname) - ctx.fillSlots("connection-hints", - "connection hints: " + " ".join(ad.connection_hints)) - ctx.fillSlots("connected", "?") - when_s = render_time(ad.when) - ctx.fillSlots("announced", when_s) - ctx.fillSlots("version", ad.version) - ctx.fillSlots("service_name", ad.service_name) - return ctx.tag - - def data_subscribers(self, ctx, data): - return self.introducer_service.get_subscribers() - - def render_subscriber_row(self, ctx, s): - ctx.fillSlots("nickname", s.nickname) - ctx.fillSlots("tubid", s.tubid) - ctx.fillSlots("connected", s.remote_address) - since_s = render_time(s.when) - ctx.fillSlots("since", since_s) - ctx.fillSlots("version", s.version) - ctx.fillSlots("service_name", s.service_name) - return ctx.tag + @renderer + def subscribers(self, req, tag): + subscribers = [{ + "nickname": s.nickname, + "tubid": s.tubid, + "connected": s.remote_address, + "since": render_time(s.when), + "version": s.version, + "service_name": s.service_name, + } for s in self.introducer_service.get_subscribers()] + return SlotsSequenceElement(tag, subscribers) diff --git a/src/allmydata/web/magic_folder.py b/src/allmydata/web/magic_folder.py deleted file mode 100644 index 27d715d3a..000000000 --- a/src/allmydata/web/magic_folder.py +++ /dev/null @@ -1,52 +0,0 @@ -import json - -from allmydata.web.common import TokenOnlyWebApi, get_arg, WebError - - -class MagicFolderWebApi(TokenOnlyWebApi): - """ - I provide the web-based API for Magic Folder status etc. - """ - - def __init__(self, client): - TokenOnlyWebApi.__init__(self, client) - self.client = client - - def post_json(self, req): - req.setHeader("content-type", "application/json") - nick = get_arg(req, 'name', 'default') - - try: - magic_folder = self.client._magic_folders[nick] - except KeyError: - raise WebError( - "No such magic-folder '{}'".format(nick), - 404, - ) - - data = [] - for item in magic_folder.uploader.get_status(): - d = dict( - path=item.relpath_u, - status=item.status_history()[-1][0], - kind='upload', - ) - for (status, ts) in item.status_history(): - d[status + '_at'] = ts - d['percent_done'] = item.progress.progress - d['size'] = item.size - data.append(d) - - for item in magic_folder.downloader.get_status(): - d = dict( - path=item.relpath_u, - status=item.status_history()[-1][0], - kind='download', - ) - for (status, ts) in item.status_history(): - d[status + '_at'] = ts - d['percent_done'] = item.progress.progress - d['size'] = item.size - data.append(d) - - return json.dumps(data) diff --git a/src/allmydata/web/manifest.xhtml b/src/allmydata/web/manifest.xhtml index 6bf2b5536..c951ec68a 100644 --- a/src/allmydata/web/manifest.xhtml +++ b/src/allmydata/web/manifest.xhtml @@ -1,28 +1,28 @@ - + - + - + -

    +

    -

    +

    - - +
    + - - - + + + - +
    Path cap
    no items in the manifest!
    no items in the manifest!
    diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 21e43c9c4..93de2169e 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -1,6 +1,5 @@ import time -from zope.interface import implementer from nevow import rend, url, tags as T from nevow.inevow import IRequest from twisted.python.failure import Failure @@ -9,7 +8,7 @@ from twisted.web.http import NOT_FOUND from twisted.web.html import escape from twisted.application import service -from allmydata.web.common import IOpHandleTable, WebError, \ +from allmydata.web.common import WebError, \ get_root, get_arg, boolean_of_arg MINUTE = 60 @@ -18,8 +17,8 @@ DAY = 24*HOUR (MONITOR, RENDERER, WHEN_ADDED) = range(3) -@implementer(IOpHandleTable) class OphandleTable(rend.Page, service.Service): + name = "operations" UNCOLLECTED_HANDLE_LIFETIME = 4*DAY COLLECTED_HANDLE_LIFETIME = 1*DAY @@ -120,7 +119,7 @@ class OphandleTable(rend.Page, service.Service): self.timers.pop(ophandle, None) self.handles.pop(ophandle, None) -class ReloadMixin: +class ReloadMixin(object): REFRESH_TIME = 1*MINUTE def render_refresh(self, ctx, data): diff --git a/src/allmydata/web/rename-form.xhtml b/src/allmydata/web/rename-form.xhtml index cac8c2ac9..8f1ae9f04 100644 --- a/src/allmydata/web/rename-form.xhtml +++ b/src/allmydata/web/rename-form.xhtml @@ -1,6 +1,6 @@ - + - + @@ -8,43 +8,23 @@ -

    +

    Rename child - + Rename child: -
    +
    to name:
    -
    -
    - Move Child to Different Directory - - - - Move child:
    - to name:
    - in directory: - - - - -
    -
    -
    -
    -
    diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 4486c9530..d6fe6863d 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,24 +1,42 @@ -import time, os, json +import os +import time +import json +import urllib -from twisted.web import http -from nevow import rend, url, tags as T -from nevow.inevow import IRequest -from nevow.static import File as nevow_File # TODO: merge with static.File? -from nevow.util import resource_filename +from hyperlink import DecodedURL, URL +from pkg_resources import resource_filename +from twisted.web import ( + http, + resource, + static, +) +from twisted.web.util import redirectTo +from twisted.python.filepath import FilePath +from twisted.web.template import ( + Element, + XMLFile, + renderer, + renderElement, + tags, +) import allmydata # to display import path -from allmydata import get_package_versions_string +from allmydata.version_checks import get_package_versions_string from allmydata.util import log from allmydata.interfaces import IFileNode -from allmydata.web import filenode, directory, unlinked, status, operations -from allmydata.web import storage, magic_folder +from allmydata.web import ( + filenode, + directory, + unlinked, + status, +) +from allmydata.web import storage from allmydata.web.common import ( abbreviate_size, - getxmlfile, WebError, get_arg, - MultiFormatPage, - RenderMixin, + MultiFormatResource, + SlotsSequenceElement, get_format, get_mutable_type, render_time_delta, @@ -28,31 +46,53 @@ from allmydata.web.common import ( from allmydata.web.private import ( create_private_tree, ) +from allmydata import uri -class URIHandler(RenderMixin, rend.Page): - # I live at /uri . There are several operations defined on /uri itself, - # mostly involved with creation of unlinked files and directories. +class URIHandler(resource.Resource, object): + """ + I live at /uri . There are several operations defined on /uri itself, + mostly involved with creation of unlinked files and directories. + """ def __init__(self, client): - rend.Page.__init__(self, client) + super(URIHandler, self).__init__() self.client = client - def render_GET(self, ctx): - req = IRequest(ctx) - uri = get_arg(req, "uri", None) - if uri is None: + def render_GET(self, req): + """ + Historically, accessing this via "GET /uri?uri=" + was/is a feature -- which simply redirects to the more-common + "GET /uri/" with any other query args + preserved. New code should use "/uri/" + """ + uri_arg = req.args.get(b"uri", [None])[0] + if uri_arg is None: raise WebError("GET /uri requires uri=") - there = url.URL.fromContext(ctx) - there = there.clear("uri") - # I thought about escaping the childcap that we attach to the URL - # here, but it seems that nevow does that for us. - there = there.child(uri) - return there - def render_PUT(self, ctx): - req = IRequest(ctx) - # either "PUT /uri" to create an unlinked file, or - # "PUT /uri?t=mkdir" to create an unlinked directory + # shennanigans like putting "%2F" or just "/" itself, or ../ + # etc in the might be a vector for weirdness so we + # validate that this is a valid capability before proceeding. + cap = uri.from_string(uri_arg) + if isinstance(cap, uri.UnknownURI): + raise WebError("Invalid capability") + + # so, using URL.from_text(req.uri) isn't going to work because + # it seems Nevow was creating absolute URLs including + # host/port whereas req.uri is absolute (but lacks host/port) + redir_uri = URL.from_text(req.prePathURL().decode('utf8')) + redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8')) + # add back all the query args that AREN'T "?uri=" + for k, values in req.args.items(): + if k != b"uri": + for v in values: + redir_uri = redir_uri.add(k.decode('utf8'), v.decode('utf8')) + return redirectTo(redir_uri.to_text().encode('utf8'), req) + + def render_PUT(self, req): + """ + either "PUT /uri" to create an unlinked file, or + "PUT /uri?t=mkdir" to create an unlinked directory + """ t = get_arg(req, "t", "").strip() if t == "": file_format = get_format(req, "CHK") @@ -63,15 +103,18 @@ class URIHandler(RenderMixin, rend.Page): return unlinked.PUTUnlinkedCHK(req, self.client) if t == "mkdir": return unlinked.PUTUnlinkedCreateDirectory(req, self.client) - errmsg = ("/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, " - "and POST?t=mkdir") + errmsg = ( + "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, " + "and POST?t=mkdir" + ) raise WebError(errmsg, http.BAD_REQUEST) - def render_POST(self, ctx): - # "POST /uri?t=upload&file=newfile" to upload an - # unlinked file or "POST /uri?t=mkdir" to create a - # new directory - req = IRequest(ctx) + def render_POST(self, req): + """ + "POST /uri?t=upload&file=newfile" to upload an + unlinked file or "POST /uri?t=mkdir" to create a + new directory + """ t = get_arg(req, "t", "").strip() if t in ("", "upload"): file_format = get_format(req) @@ -92,25 +135,39 @@ class URIHandler(RenderMixin, rend.Page): "and POST?t=mkdir") raise WebError(errmsg, http.BAD_REQUEST) - def childFactory(self, ctx, name): - # 'name' is expected to be a URI + def getChild(self, name, req): + """ + Most requests look like /uri/ so this fetches the capability + and creates and appropriate handler (depending on the kind of + capability it was passed). + """ + # this is in case a URI like "/uri/?cap=" is + # passed -- we re-direct to the non-trailing-slash version so + # that there is just one valid URI for "uri" resource. + if not name: + u = DecodedURL.from_text(req.uri.decode('utf8')) + u = u.replace( + path=(s for s in u.path if s), # remove empty segments + ) + return redirectTo(u.to_uri().to_text().encode('utf8'), req) try: node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) except (TypeError, AssertionError): - raise WebError("'%s' is not a valid file- or directory- cap" - % name) + raise WebError( + "'{}' is not a valid file- or directory- cap".format(name) + ) -class FileHandler(rend.Page): + +class FileHandler(resource.Resource, object): # I handle /file/$FILECAP[/IGNORED] , which provides a URL from which a # file can be downloaded correctly by tools like "wget". def __init__(self, client): - rend.Page.__init__(self, client) + super(FileHandler, self).__init__() self.client = client - def childFactory(self, ctx, name): - req = IRequest(ctx) + def getChild(self, name, req): if req.method not in ("GET", "HEAD"): raise WebError("/file can only be used with GET or HEAD") # 'name' must be a file URI @@ -124,92 +181,86 @@ class FileHandler(rend.Page): raise WebError("'%s' is not a file-cap" % name) return filenode.FileNodeDownloadHandler(self.client, node) - def renderHTTP(self, ctx): + def render_GET(self, ctx): raise WebError("/file must be followed by a file-cap and a name", http.NOT_FOUND) -class IncidentReporter(RenderMixin, rend.Page): - def render_POST(self, ctx): - req = IRequest(ctx) +class IncidentReporter(MultiFormatResource): + """Handler for /report_incident POST request""" + + def render(self, req): + if req.method != "POST": + raise WebError("/report_incident can only be used with POST") + log.msg(format="User reports incident through web page: %(details)s", details=get_arg(req, "details", ""), level=log.WEIRD, umid="LkD9Pw") - req.setHeader("content-type", "text/plain") - return "An incident report has been saved to logs/incidents/ in the node directory." + req.setHeader("content-type", "text/plain; charset=UTF-8") + return b"An incident report has been saved to logs/incidents/ in the node directory." SPACE = u"\u00A0"*2 -class Root(MultiFormatPage): +class Root(MultiFormatResource): addSlash = True - docFactory = getxmlfile("welcome.xhtml") - - _connectedalts = { - "not-configured": "Not Configured", - "yes": "Connected", - "no": "Disconnected", - } def __init__(self, client, clock=None, now_fn=None): - rend.Page.__init__(self, client) - self.client = client - # If set, clock is a twisted.internet.task.Clock that the tests - # use to test ophandle expiration. - self.child_operations = operations.OphandleTable(clock) - self.now_fn = now_fn - try: - s = client.getServiceNamed("storage") - except KeyError: - s = None - self.child_storage = storage.StorageStatus(s, self.client.nickname) + """ + Render root page ("/") of the URI. - self.child_uri = URIHandler(client) - self.child_cap = URIHandler(client) + :client allmydata.client._Client: a stats provider. + :clock: unused here. + :now_fn: a function that returns current time. - # handler for "/magic_folder" URIs - self.child_magic_folder = magic_folder.MagicFolderWebApi(client) + """ + super(Root, self).__init__() + self._client = client + self._now_fn = now_fn + + self.putChild("uri", URIHandler(client)) + self.putChild("cap", URIHandler(client)) # Handler for everything beneath "/private", an area of the resource # hierarchy which is only accessible with the private per-node API # auth token. - self.child_private = create_private_tree(client.get_auth_token) + self.putChild("private", create_private_tree(client.get_auth_token)) - self.child_file = FileHandler(client) - self.child_named = FileHandler(client) - self.child_status = status.Status(client.get_history()) - self.child_statistics = status.Statistics(client.stats_provider) + self.putChild("file", FileHandler(client)) + self.putChild("named", FileHandler(client)) + self.putChild("status", status.Status(client.get_history())) + self.putChild("statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, nevow_File(os.path.join(static_dir, filen))) + self.putChild(filen, static.File(os.path.join(static_dir, filen))) - def child_helper_status(self, ctx): - # the Helper isn't attached until after the Tub starts, so this child - # needs to created on each request - return status.HelperStatus(self.client.helper) + self.putChild("report_incident", IncidentReporter()) - child_report_incident = IncidentReporter() - #child_server # let's reserve this for storage-server-over-HTTP - - # FIXME: This code is duplicated in root.py and introweb.py. - def data_rendered_at(self, ctx, data): - return render_time(time.time()) - def data_version(self, ctx, data): - return get_package_versions_string() - def data_import_path(self, ctx, data): - return str(allmydata) - def render_my_nodeid(self, ctx, data): - tubid_s = "TubID: "+self.client.get_long_tubid() - return T.td(title=tubid_s)[self.client.get_long_nodeid()] - - def data_my_nickname(self, ctx, data): - return self.client.nickname + def getChild(self, path, request): + if not path: + # Render "/" path. + return self + if path == "helper_status": + # the Helper isn't attached until after the Tub starts, so this child + # needs to created on each request + return status.HelperStatus(self._client.helper) + if path == "storage": + # Storage isn't initialized until after the web hierarchy is + # constructed so this child needs to be created later than + # `__init__`. + try: + storage_server = self._client.getServiceNamed("storage") + except KeyError: + storage_server = None + return storage.StorageStatus(storage_server, self._client.nickname) + def render_HTML(self, req): + return renderElement(req, RootElement(self._client, self._now_fn)) def render_JSON(self, req): req.setHeader("content-type", "application/json; charset=utf-8") - intro_summaries = [s.summary for s in self.client.introducer_connection_statuses()] - sb = self.client.get_storage_broker() + intro_summaries = [s.summary for s in self._client.introducer_connection_statuses()] + sb = self._client.get_storage_broker() servers = self._describe_known_servers(sb) result = { "introducers": { @@ -244,35 +295,54 @@ class Root(MultiFormatPage): return description +class RootElement(Element): - def data_magic_folders(self, ctx, data): - return self.client._magic_folders.keys() + loader = XMLFile(FilePath(__file__).sibling("welcome.xhtml")) - def render_magic_folder_row(self, ctx, data): - magic_folder = self.client._magic_folders[data] - (ok, messages) = magic_folder.get_public_status() - ctx.fillSlots("magic_folder_name", data) - if ok: - ctx.fillSlots("magic_folder_status", "yes") - ctx.fillSlots("magic_folder_status_alt", "working") - else: - ctx.fillSlots("magic_folder_status", "no") - ctx.fillSlots("magic_folder_status_alt", "not working") + def __init__(self, client, now_fn): + super(RootElement, self).__init__() + self._client = client + self._now_fn = now_fn - status = T.ul(class_="magic-folder-status") - for msg in messages: - status[T.li[str(msg)]] - return ctx.tag[status] + _connectedalts = { + "not-configured": "Not Configured", + "yes": "Connected", + "no": "Disconnected", + } - def render_magic_folder(self, ctx, data): - if not self.client._magic_folders: - return T.p() - return ctx.tag + @renderer + def my_nodeid(self, req, tag): + tubid_s = "TubID: "+self._client.get_long_tubid() + return tags.td(self._client.get_long_nodeid(), title=tubid_s) - def render_services(self, ctx, data): - ul = T.ul() + @renderer + def my_nickname(self, req, tag): + return tag(self._client.nickname) + + def _connected_introducers(self): + return len([1 for cs in self._client.introducer_connection_statuses() + if cs.connected]) + + @renderer + def connected_introducers(self, req, tag): + return tag(str(self._connected_introducers())) + + @renderer + def connected_to_at_least_one_introducer(self, req, tag): + if self._connected_introducers(): + return "yes" + return "no" + + @renderer + def connected_to_at_least_one_introducer_alt(self, req, tag): + state = self.connected_to_at_least_one_introducer(req, tag) + return self._connectedalts.get(state) + + @renderer + def services(self, req, tag): + ul = tags.ul() try: - ss = self.client.getServiceNamed("storage") + ss = self._client.getServiceNamed("storage") stats = ss.get_stats() if stats["storage_server.accepting_immutable_shares"]: msg = "accepting new shares" @@ -281,113 +351,60 @@ class Root(MultiFormatPage): available = stats.get("storage_server.disk_avail") if available is not None: msg += ", %s available" % abbreviate_size(available) - ul[T.li[T.a(href="storage")["Storage Server"], ": ", msg]] + ul(tags.li(tags.a("Storage Server", href="storage"), ": ", msg)) except KeyError: - ul[T.li["Not running storage server"]] + ul(tags.li("Not running storage server")) - if self.client.helper: - stats = self.client.helper.get_stats() + if self._client.helper: + stats = self._client.helper.get_stats() active_uploads = stats["chk_upload_helper.active_uploads"] - ul[T.li["Helper: %d active uploads" % (active_uploads,)]] + ul(tags.li("Helper: %d active uploads" % (active_uploads,))) else: - ul[T.li["Not running helper"]] + ul(tags.li("Not running helper")) - return ctx.tag[ul] + return tag(ul) - def data_introducer_description(self, ctx, data): - connected_count = self.data_connected_introducers( ctx, data ) + @renderer + def introducer_description(self, req, tag): + connected_count = self._connected_introducers() if connected_count == 0: - return "No introducers connected" + return tag("No introducers connected") elif connected_count == 1: - return "1 introducer connected" + return tag("1 introducer connected") else: - return "%s introducers connected" % (connected_count,) + return tag("%s introducers connected" % (connected_count,)) - def data_total_introducers(self, ctx, data): - return len(self.client.introducer_connection_statuses()) - - def data_connected_introducers(self, ctx, data): - return len([1 for cs in self.client.introducer_connection_statuses() - if cs.connected]) - - def data_connected_to_at_least_one_introducer(self, ctx, data): - if self.data_connected_introducers(ctx, data): - return "yes" - return "no" - - def data_connected_to_at_least_one_introducer_alt(self, ctx, data): - return self._connectedalts[self.data_connected_to_at_least_one_introducer(ctx, data)] + @renderer + def total_introducers(self, req, tag): + return tag(str(len(self._get_introducers()))) # In case we configure multiple introducers - def data_introducers(self, ctx, data): - return self.client.introducer_connection_statuses() + @renderer + def introducers(self, req, tag): + ix = self._get_introducers() + if not ix: + return tag("No introducers") + return tag - def _render_connection_status(self, ctx, cs): - connected = "yes" if cs.connected else "no" - ctx.fillSlots("service_connection_status", connected) - ctx.fillSlots("service_connection_status_alt", - self._connectedalts[connected]) + def _get_introducers(self): + return self._client.introducer_connection_statuses() - since = cs.last_connection_time - ctx.fillSlots("service_connection_status_rel_time", - render_time_delta(since, self.now_fn()) - if since is not None - else "N/A") - ctx.fillSlots("service_connection_status_abs_time", - render_time_attr(since) - if since is not None - else "N/A") - - last_received_data_time = cs.last_received_time - ctx.fillSlots("last_received_data_abs_time", - render_time_attr(last_received_data_time) - if last_received_data_time is not None - else "N/A") - ctx.fillSlots("last_received_data_rel_time", - render_time_delta(last_received_data_time, self.now_fn()) - if last_received_data_time is not None - else "N/A") - - others = cs.non_connected_statuses - if cs.connected: - ctx.fillSlots("summary", cs.summary) - if others: - details = "\n".join(["* %s: %s\n" % (which, others[which]) - for which in sorted(others)]) - ctx.fillSlots("details", "Other hints:\n" + details) - else: - ctx.fillSlots("details", "(no other hints)") - else: - details = T.ul() - for which in sorted(others): - details[T.li["%s: %s" % (which, others[which])]] - ctx.fillSlots("summary", [cs.summary, details]) - ctx.fillSlots("details", "") - - def render_introducers_row(self, ctx, cs): - self._render_connection_status(ctx, cs) - return ctx.tag - - def data_helper_furl_prefix(self, ctx, data): + @renderer + def helper_furl_prefix(self, req, tag): try: - uploader = self.client.getServiceNamed("uploader") + uploader = self._client.getServiceNamed("uploader") except KeyError: - return None + return tag("None") furl, connected = uploader.get_helper_info() if not furl: - return None + return tag("None") # trim off the secret swissnum (prefix, _, swissnum) = furl.rpartition("/") - return "%s/[censored]" % (prefix,) + return tag("%s/[censored]" % (prefix,)) - def data_helper_description(self, ctx, data): - if self.data_connected_to_helper(ctx, data) == "no": - return "Helper not connected" - return "Helper" - - def data_connected_to_helper(self, ctx, data): + def _connected_to_helper(self): try: - uploader = self.client.getServiceNamed("uploader") + uploader = self._client.getServiceNamed("uploader") except KeyError: return "no" # we don't even have an Uploader furl, connected = uploader.get_helper_info() @@ -398,123 +415,147 @@ class Root(MultiFormatPage): return "yes" return "no" - def data_connected_to_helper_alt(self, ctx, data): - return self._connectedalts[self.data_connected_to_helper(ctx, data)] + @renderer + def helper_description(self, req, tag): + if self._connected_to_helper() == "no": + return tag("Helper not connected") + return tag("Helper") - def data_known_storage_servers(self, ctx, data): - sb = self.client.get_storage_broker() - return len(sb.get_all_serverids()) + @renderer + def connected_to_helper(self, req, tag): + return tag(self._connected_to_helper()) - def data_connected_storage_servers(self, ctx, data): - sb = self.client.get_storage_broker() - return len(sb.get_connected_servers()) + @renderer + def connected_to_helper_alt(self, req, tag): + return tag(self._connectedalts.get(self._connected_to_helper())) - def data_services(self, ctx, data): - sb = self.client.get_storage_broker() + @renderer + def known_storage_servers(self, req, tag): + sb = self._client.get_storage_broker() + return tag(str(len(sb.get_all_serverids()))) + + @renderer + def connected_storage_servers(self, req, tag): + sb = self._client.get_storage_broker() + return tag(str(len(sb.get_connected_servers()))) + + @renderer + def services_table(self, req, tag): + rows = [ self._describe_server_and_connection(server) + for server in self._services() ] + return SlotsSequenceElement(tag, rows) + + @renderer + def introducers_table(self, req, tag): + rows = [ self._describe_connection_status(cs) + for cs in self._get_introducers() ] + return SlotsSequenceElement(tag, rows) + + def _services(self): + sb = self._client.get_storage_broker() return sorted(sb.get_known_servers(), key=lambda s: s.get_serverid()) - def render_service_row(self, ctx, server): - cs = server.get_connection_status() - self._render_connection_status(ctx, cs) + @staticmethod + def _describe_server(server): + """Return a dict containing server stats.""" + peerid = server.get_longname() + nickname = server.get_nickname() + version = server.get_announcement().get("my-version", "") - ctx.fillSlots("peerid", server.get_longname()) - ctx.fillSlots("nickname", server.get_nickname()) - - announcement = server.get_announcement() - version = announcement.get("my-version", "") - available_space = server.get_available_space() - if available_space is None: - available_space = "N/A" + space = server.get_available_space() + if space is not None: + available_space = abbreviate_size(space) else: - available_space = abbreviate_size(available_space) - ctx.fillSlots("version", version) - ctx.fillSlots("available_space", available_space) + available_space = "N/A" - return ctx.tag + return { + "peerid": peerid, + "nickname": nickname, + "version": version, + "available_space": available_space, + } - def render_download_form(self, ctx, data): - # this is a form where users can download files by URI - form = T.form(action="uri", method="get", - enctype="multipart/form-data")[ - T.fieldset[ - T.legend(class_="freeform-form-label")["Download a file"], - T.div["Tahoe-URI to download:"+SPACE, - T.input(type="text", name="uri")], - T.div["Filename to download as:"+SPACE, - T.input(type="text", name="filename")], - T.input(type="submit", value="Download!"), - ]] - return T.div[form] + def _describe_server_and_connection(self, server): + """Return a dict containing both server and connection stats.""" + srvstat = self._describe_server(server) + cs = server.get_connection_status() + constat = self._describe_connection_status(cs) + return dict(list(srvstat.items()) + list(constat.items())) - def render_view_form(self, ctx, data): - # this is a form where users can download files by URI, or jump to a - # named directory - form = T.form(action="uri", method="get", - enctype="multipart/form-data")[ - T.fieldset[ - T.legend(class_="freeform-form-label")["View a file or directory"], - "Tahoe-URI to view:"+SPACE, - T.input(type="text", name="uri"), SPACE*2, - T.input(type="submit", value="View!"), - ]] - return T.div[form] + def _describe_connection_status(self, cs): + """Return a dict containing some connection stats.""" + others = cs.non_connected_statuses - def render_upload_form(self, ctx, data): - # This is a form where users can upload unlinked files. - # Users can choose immutable, SDMF, or MDMF from a radio button. + if cs.connected: + summary = cs.summary + if others: + hints = "\n".join(["* %s: %s\n" % (which, others[which]) + for which in sorted(others)]) + details = "Other hints:\n" + hints + else: + details = "(no other hints)" + else: + details = tags.ul() + for which in sorted(others): + details(tags.li("%s: %s" % (which, others[which]))) + summary = [cs.summary, details] - upload_chk = T.input(type='radio', name='format', - value='chk', id='upload-chk', - checked='checked') - upload_sdmf = T.input(type='radio', name='format', - value='sdmf', id='upload-sdmf') - upload_mdmf = T.input(type='radio', name='format', - value='mdmf', id='upload-mdmf') + connected = "yes" if cs.connected else "no" + connected_alt = self._connectedalts[connected] - form = T.form(action="uri", method="post", - enctype="multipart/form-data")[ - T.fieldset[ - T.legend(class_="freeform-form-label")["Upload a file"], - T.div["Choose a file:"+SPACE, - T.input(type="file", name="file", class_="freeform-input-file")], - T.input(type="hidden", name="t", value="upload"), - T.div[upload_chk, T.label(for_="upload-chk") [" Immutable"], SPACE, - upload_sdmf, T.label(for_="upload-sdmf")[" SDMF"], SPACE, - upload_mdmf, T.label(for_="upload-mdmf")[" MDMF (experimental)"], SPACE*2, - T.input(type="submit", value="Upload!")], - ]] - return T.div[form] + since = cs.last_connection_time - def render_mkdir_form(self, ctx, data): - # This is a form where users can create new directories. - # Users can choose SDMF or MDMF from a radio button. + if since is not None: + service_connection_status_rel_time = render_time_delta(since, self._now_fn()) + service_connection_status_abs_time = render_time_attr(since) + else: + service_connection_status_rel_time = "N/A" + service_connection_status_abs_time = "N/A" - mkdir_sdmf = T.input(type='radio', name='format', - value='sdmf', id='mkdir-sdmf', - checked='checked') - mkdir_mdmf = T.input(type='radio', name='format', - value='mdmf', id='mkdir-mdmf') + last_received_data_time = cs.last_received_time - form = T.form(action="uri", method="post", - enctype="multipart/form-data")[ - T.fieldset[ - T.legend(class_="freeform-form-label")["Create a directory"], - mkdir_sdmf, T.label(for_='mkdir-sdmf')[" SDMF"], SPACE, - mkdir_mdmf, T.label(for_='mkdir-mdmf')[" MDMF (experimental)"], SPACE*2, - T.input(type="hidden", name="t", value="mkdir"), - T.input(type="hidden", name="redirect_to_result", value="true"), - T.input(type="submit", value="Create a directory"), - ]] - return T.div[form] + if last_received_data_time is not None: + last_received_data_abs_time = render_time_attr(last_received_data_time) + last_received_data_rel_time = render_time_delta(last_received_data_time, self._now_fn()) + else: + last_received_data_abs_time = "N/A" + last_received_data_rel_time = "N/A" - def render_incident_button(self, ctx, data): + return { + "summary": summary, + "details": details, + "service_connection_status": connected, + "service_connection_status_alt": connected_alt, + "service_connection_status_abs_time": service_connection_status_abs_time, + "service_connection_status_rel_time": service_connection_status_rel_time, + "last_received_data_abs_time": last_received_data_abs_time, + "last_received_data_rel_time": last_received_data_rel_time, + } + + @renderer + def incident_button(self, req, tag): # this button triggers a foolscap-logging "incident" - form = T.form(action="report_incident", method="post", - enctype="multipart/form-data")[ - T.fieldset[ - T.input(type="hidden", name="t", value="report-incident"), - "What went wrong?"+SPACE, - T.input(type="text", name="details"), SPACE, - T.input(type="submit", value=u"Save \u00BB"), - ]] - return T.div[form] + form = tags.form( + tags.fieldset( + tags.input(type="hidden", name="t", value="report-incident"), + "What went wrong?"+SPACE, + tags.input(type="text", name="details"), SPACE, + tags.input(type="submit", value=u"Save \u00BB"), + ), + action="report_incident", + method="post", + enctype="multipart/form-data" + ) + return tags.div(form) + + @renderer + def rendered_at(self, req, tag): + return tag(render_time(time.time())) + + @renderer + def version(self, req, tag): + return tag(get_package_versions_string()) + + @renderer + def import_path(self, req, tag): + return tag(str(allmydata)) diff --git a/src/allmydata/web/static/css/new-tahoe.css b/src/allmydata/web/static/css/new-tahoe.css index caf540c07..38bd662e1 100644 --- a/src/allmydata/web/static/css/new-tahoe.css +++ b/src/allmydata/web/static/css/new-tahoe.css @@ -53,11 +53,6 @@ body { .connection-status { } -.magic-folder-status { - clear: left; - margin-left: 40px; /* width of status-indicator + margins */ -} - .furl { font-size: 0.8em; word-wrap: break-word; diff --git a/src/allmydata/web/static/d3-2.4.6.js b/src/allmydata/web/static/d3-2.4.6.js deleted file mode 100644 index d24c2e4c8..000000000 --- a/src/allmydata/web/static/d3-2.4.6.js +++ /dev/null @@ -1,4150 +0,0 @@ -(function(){if (!Date.now) Date.now = function() { - return +new Date; -}; -try { - document.createElement("div").style.setProperty("opacity", 0, ""); -} catch (error) { - var d3_style_prototype = CSSStyleDeclaration.prototype, - d3_style_setProperty = d3_style_prototype.setProperty; - d3_style_prototype.setProperty = function(name, value, priority) { - d3_style_setProperty.call(this, name, value + "", priority); - }; -} -d3 = {version: "2.4.6"}; // semver -var d3_array = d3_arraySlice; // conversion for NodeLists - -function d3_arrayCopy(pseudoarray) { - var i = -1, n = pseudoarray.length, array = []; - while (++i < n) array.push(pseudoarray[i]); - return array; -} - -function d3_arraySlice(pseudoarray) { - return Array.prototype.slice.call(pseudoarray); -} - -try { - d3_array(document.documentElement.childNodes)[0].nodeType; -} catch(e) { - d3_array = d3_arrayCopy; -} - -var d3_arraySubclass = [].__proto__? - -// Until ECMAScript supports array subclassing, prototype injection works well. -function(array, prototype) { - array.__proto__ = prototype; -}: - -// And if your browser doesn't support __proto__, we'll use direct extension. -function(array, prototype) { - for (var property in prototype) array[property] = prototype[property]; -}; -function d3_this() { - return this; -} -d3.functor = function(v) { - return typeof v === "function" ? v : function() { return v; }; -}; -// A getter-setter method that preserves the appropriate `this` context. -d3.rebind = function(object, method) { - return function() { - var x = method.apply(object, arguments); - return arguments.length ? object : x; - }; -}; -d3.ascending = function(a, b) { - return a < b ? -1 : a > b ? 1 : a >= b ? 0 : NaN; -}; -d3.descending = function(a, b) { - return b < a ? -1 : b > a ? 1 : b >= a ? 0 : NaN; -}; -d3.mean = function(array, f) { - var n = array.length, - a, - m = 0, - i = -1, - j = 0; - if (arguments.length === 1) { - while (++i < n) if (d3_number(a = array[i])) m += (a - m) / ++j; - } else { - while (++i < n) if (d3_number(a = f.call(array, array[i], i))) m += (a - m) / ++j; - } - return j ? m : undefined; -}; -d3.median = function(array, f) { - if (arguments.length > 1) array = array.map(f); - array = array.filter(d3_number); - return array.length ? d3.quantile(array.sort(d3.ascending), .5) : undefined; -}; -d3.min = function(array, f) { - var i = -1, - n = array.length, - a, - b; - if (arguments.length === 1) { - while (++i < n && ((a = array[i]) == null || a != a)) a = undefined; - while (++i < n) if ((b = array[i]) != null && a > b) a = b; - } else { - while (++i < n && ((a = f.call(array, array[i], i)) == null || a != a)) a = undefined; - while (++i < n) if ((b = f.call(array, array[i], i)) != null && a > b) a = b; - } - return a; -}; -d3.max = function(array, f) { - var i = -1, - n = array.length, - a, - b; - if (arguments.length === 1) { - while (++i < n && ((a = array[i]) == null || a != a)) a = undefined; - while (++i < n) if ((b = array[i]) != null && b > a) a = b; - } else { - while (++i < n && ((a = f.call(array, array[i], i)) == null || a != a)) a = undefined; - while (++i < n) if ((b = f.call(array, array[i], i)) != null && b > a) a = b; - } - return a; -}; -function d3_number(x) { - return x != null && !isNaN(x); -} -d3.sum = function(array, f) { - var s = 0, - n = array.length, - a, - i = -1; - - if (arguments.length === 1) { - while (++i < n) if (!isNaN(a = +array[i])) s += a; - } else { - while (++i < n) if (!isNaN(a = +f.call(array, array[i], i))) s += a; - } - - return s; -}; -// R-7 per -d3.quantile = function(values, p) { - var H = (values.length - 1) * p + 1, - h = Math.floor(H), - v = values[h - 1], - e = H - h; - return e ? v + e * (values[h] - v) : v; -}; -d3.zip = function() { - if (!(n = arguments.length)) return []; - for (var i = -1, m = d3.min(arguments, d3_zipLength), zips = new Array(m); ++i < m;) { - for (var j = -1, n, zip = zips[i] = new Array(n); ++j < n;) { - zip[j] = arguments[j][i]; - } - } - return zips; -}; - -function d3_zipLength(d) { - return d.length; -} -// Locate the insertion point for x in a to maintain sorted order. The -// arguments lo and hi may be used to specify a subset of the array which should -// be considered; by default the entire array is used. If x is already present -// in a, the insertion point will be before (to the left of) any existing -// entries. The return value is suitable for use as the first argument to -// `array.splice` assuming that a is already sorted. -// -// The returned insertion point i partitions the array a into two halves so that -// all v < x for v in a[lo:i] for the left side and all v >= x for v in a[i:hi] -// for the right side. -d3.bisectLeft = function(a, x, lo, hi) { - if (arguments.length < 3) lo = 0; - if (arguments.length < 4) hi = a.length; - while (lo < hi) { - var mid = (lo + hi) >> 1; - if (a[mid] < x) lo = mid + 1; - else hi = mid; - } - return lo; -}; - -// Similar to bisectLeft, but returns an insertion point which comes after (to -// the right of) any existing entries of x in a. -// -// The returned insertion point i partitions the array into two halves so that -// all v <= x for v in a[lo:i] for the left side and all v > x for v in a[i:hi] -// for the right side. -d3.bisect = -d3.bisectRight = function(a, x, lo, hi) { - if (arguments.length < 3) lo = 0; - if (arguments.length < 4) hi = a.length; - while (lo < hi) { - var mid = (lo + hi) >> 1; - if (x < a[mid]) hi = mid; - else lo = mid + 1; - } - return lo; -}; -d3.first = function(array, f) { - var i = 0, - n = array.length, - a = array[0], - b; - if (arguments.length === 1) f = d3.ascending; - while (++i < n) { - if (f.call(array, a, b = array[i]) > 0) { - a = b; - } - } - return a; -}; -d3.last = function(array, f) { - var i = 0, - n = array.length, - a = array[0], - b; - if (arguments.length === 1) f = d3.ascending; - while (++i < n) { - if (f.call(array, a, b = array[i]) <= 0) { - a = b; - } - } - return a; -}; -d3.nest = function() { - var nest = {}, - keys = [], - sortKeys = [], - sortValues, - rollup; - - function map(array, depth) { - if (depth >= keys.length) return rollup - ? rollup.call(nest, array) : (sortValues - ? array.sort(sortValues) - : array); - - var i = -1, - n = array.length, - key = keys[depth++], - keyValue, - object, - o = {}; - - while (++i < n) { - if ((keyValue = key(object = array[i])) in o) { - o[keyValue].push(object); - } else { - o[keyValue] = [object]; - } - } - - for (keyValue in o) { - o[keyValue] = map(o[keyValue], depth); - } - - return o; - } - - function entries(map, depth) { - if (depth >= keys.length) return map; - - var a = [], - sortKey = sortKeys[depth++], - key; - - for (key in map) { - a.push({key: key, values: entries(map[key], depth)}); - } - - if (sortKey) a.sort(function(a, b) { - return sortKey(a.key, b.key); - }); - - return a; - } - - nest.map = function(array) { - return map(array, 0); - }; - - nest.entries = function(array) { - return entries(map(array, 0), 0); - }; - - nest.key = function(d) { - keys.push(d); - return nest; - }; - - // Specifies the order for the most-recently specified key. - // Note: only applies to entries. Map keys are unordered! - nest.sortKeys = function(order) { - sortKeys[keys.length - 1] = order; - return nest; - }; - - // Specifies the order for leaf values. - // Applies to both maps and entries array. - nest.sortValues = function(order) { - sortValues = order; - return nest; - }; - - nest.rollup = function(f) { - rollup = f; - return nest; - }; - - return nest; -}; -d3.keys = function(map) { - var keys = []; - for (var key in map) keys.push(key); - return keys; -}; -d3.values = function(map) { - var values = []; - for (var key in map) values.push(map[key]); - return values; -}; -d3.entries = function(map) { - var entries = []; - for (var key in map) entries.push({key: key, value: map[key]}); - return entries; -}; -d3.permute = function(array, indexes) { - var permutes = [], - i = -1, - n = indexes.length; - while (++i < n) permutes[i] = array[indexes[i]]; - return permutes; -}; -d3.merge = function(arrays) { - return Array.prototype.concat.apply([], arrays); -}; -d3.split = function(array, f) { - var arrays = [], - values = [], - value, - i = -1, - n = array.length; - if (arguments.length < 2) f = d3_splitter; - while (++i < n) { - if (f.call(values, value = array[i], i)) { - values = []; - } else { - if (!values.length) arrays.push(values); - values.push(value); - } - } - return arrays; -}; - -function d3_splitter(d) { - return d == null; -} -function d3_collapse(s) { - return s.replace(/(^\s+)|(\s+$)/g, "").replace(/\s+/g, " "); -} -/** - * @param {number} start - * @param {number=} stop - * @param {number=} step - */ -d3.range = function(start, stop, step) { - if (arguments.length < 3) { - step = 1; - if (arguments.length < 2) { - stop = start; - start = 0; - } - } - if ((stop - start) / step == Infinity) throw new Error("infinite range"); - var range = [], - i = -1, - j; - if (step < 0) while ((j = start + step * ++i) > stop) range.push(j); - else while ((j = start + step * ++i) < stop) range.push(j); - return range; -}; -d3.requote = function(s) { - return s.replace(d3_requote_re, "\\$&"); -}; - -var d3_requote_re = /[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g; -d3.round = function(x, n) { - return n - ? Math.round(x * Math.pow(10, n)) * Math.pow(10, -n) - : Math.round(x); -}; -d3.xhr = function(url, mime, callback) { - var req = new XMLHttpRequest; - if (arguments.length < 3) callback = mime; - else if (mime && req.overrideMimeType) req.overrideMimeType(mime); - req.open("GET", url, true); - req.onreadystatechange = function() { - if (req.readyState === 4) callback(req.status < 300 ? req : null); - }; - req.send(null); -}; -d3.text = function(url, mime, callback) { - function ready(req) { - callback(req && req.responseText); - } - if (arguments.length < 3) { - callback = mime; - mime = null; - } - d3.xhr(url, mime, ready); -}; -d3.json = function(url, callback) { - d3.text(url, "application/json", function(text) { - callback(text ? JSON.parse(text) : null); - }); -}; -d3.html = function(url, callback) { - d3.text(url, "text/html", function(text) { - if (text != null) { // Treat empty string as valid HTML. - var range = document.createRange(); - range.selectNode(document.body); - text = range.createContextualFragment(text); - } - callback(text); - }); -}; -d3.xml = function(url, mime, callback) { - function ready(req) { - callback(req && req.responseXML); - } - if (arguments.length < 3) { - callback = mime; - mime = null; - } - d3.xhr(url, mime, ready); -}; -d3.ns = { - - prefix: { - svg: "http://www.w3.org/2000/svg", - xhtml: "http://www.w3.org/1999/xhtml", - xlink: "http://www.w3.org/1999/xlink", - xml: "http://www.w3.org/XML/1998/namespace", - xmlns: "http://www.w3.org/2000/xmlns/" - }, - - qualify: function(name) { - var i = name.indexOf(":"); - return i < 0 ? name : { - space: d3.ns.prefix[name.substring(0, i)], - local: name.substring(i + 1) - }; - } - -}; -/** @param {...string} types */ -d3.dispatch = function(types) { - var dispatch = {}, - type; - for (var i = 0, n = arguments.length; i < n; i++) { - type = arguments[i]; - dispatch[type] = d3_dispatch(type); - } - return dispatch; -}; - -function d3_dispatch(type) { - var dispatch = {}, - listeners = []; - - dispatch.add = function(listener) { - for (var i = 0; i < listeners.length; i++) { - if (listeners[i].listener == listener) return dispatch; // already registered - } - listeners.push({listener: listener, on: true}); - return dispatch; - }; - - dispatch.remove = function(listener) { - for (var i = 0; i < listeners.length; i++) { - var l = listeners[i]; - if (l.listener == listener) { - l.on = false; - listeners = listeners.slice(0, i).concat(listeners.slice(i + 1)); - break; - } - } - return dispatch; - }; - - dispatch.dispatch = function() { - var ls = listeners; // defensive reference - for (var i = 0, n = ls.length; i < n; i++) { - var l = ls[i]; - if (l.on) l.listener.apply(this, arguments); - } - }; - - return dispatch; -}; -// TODO align -d3.format = function(specifier) { - var match = d3_format_re.exec(specifier), - fill = match[1] || " ", - sign = match[3] || "", - zfill = match[5], - width = +match[6], - comma = match[7], - precision = match[8], - type = match[9], - scale = 1, - suffix = "", - integer = false; - - if (precision) precision = +precision.substring(1); - - if (zfill) { - fill = "0"; // TODO align = "="; - if (comma) width -= Math.floor((width - 1) / 4); - } - - switch (type) { - case "n": comma = true; type = "g"; break; - case "%": scale = 100; suffix = "%"; type = "f"; break; - case "p": scale = 100; suffix = "%"; type = "r"; break; - case "d": integer = true; precision = 0; break; - case "s": scale = -1; type = "r"; break; - } - - // If no precision is specified for r, fallback to general notation. - if (type == "r" && !precision) type = "g"; - - type = d3_format_types[type] || d3_format_typeDefault; - - return function(value) { - - // Return the empty string for floats formatted as ints. - if (integer && (value % 1)) return ""; - - // Convert negative to positive, and record the sign prefix. - var negative = (value < 0) && (value = -value) ? "\u2212" : sign; - - // Apply the scale, computing it from the value's exponent for si format. - if (scale < 0) { - var prefix = d3.formatPrefix(value, precision); - value *= prefix.scale; - suffix = prefix.symbol; - } else { - value *= scale; - } - - // Convert to the desired precision. - value = type(value, precision); - - // If the fill character is 0, the sign and group is applied after the fill. - if (zfill) { - var length = value.length + negative.length; - if (length < width) value = new Array(width - length + 1).join(fill) + value; - if (comma) value = d3_format_group(value); - value = negative + value; - } - - // Otherwise (e.g., space-filling), the sign and group is applied before. - else { - if (comma) value = d3_format_group(value); - value = negative + value; - var length = value.length; - if (length < width) value = new Array(width - length + 1).join(fill) + value; - } - - return value + suffix; - }; -}; - -// [[fill]align][sign][#][0][width][,][.precision][type] -var d3_format_re = /(?:([^{])?([<>=^]))?([+\- ])?(#)?(0)?([0-9]+)?(,)?(\.[0-9]+)?([a-zA-Z%])?/; - -var d3_format_types = { - g: function(x, p) { return x.toPrecision(p); }, - e: function(x, p) { return x.toExponential(p); }, - f: function(x, p) { return x.toFixed(p); }, - r: function(x, p) { return d3.round(x, p = d3_format_precision(x, p)).toFixed(Math.max(0, Math.min(20, p))); } -}; - -function d3_format_precision(x, p) { - return p - (x ? 1 + Math.floor(Math.log(x + Math.pow(10, 1 + Math.floor(Math.log(x) / Math.LN10) - p)) / Math.LN10) : 1); -} - -function d3_format_typeDefault(x) { - return x + ""; -} - -// Apply comma grouping for thousands. -function d3_format_group(value) { - var i = value.lastIndexOf("."), - f = i >= 0 ? value.substring(i) : (i = value.length, ""), - t = []; - while (i > 0) t.push(value.substring(i -= 3, i + 3)); - return t.reverse().join(",") + f; -} -var d3_formatPrefixes = ["y","z","a","f","p","n","μ","m","","k","M","G","T","P","E","Z","Y"].map(d3_formatPrefix); - -d3.formatPrefix = function(value, precision) { - var i = 0; - if (value) { - if (value < 0) value *= -1; - if (precision) value = d3.round(value, d3_format_precision(value, precision)); - i = 1 + Math.floor(1e-12 + Math.log(value) / Math.LN10); - i = Math.max(-24, Math.min(24, Math.floor((i <= 0 ? i + 1 : i - 1) / 3) * 3)); - } - return d3_formatPrefixes[8 + i / 3]; -}; - -function d3_formatPrefix(d, i) { - return { - scale: Math.pow(10, (8 - i) * 3), - symbol: d - }; -} - -/* - * TERMS OF USE - EASING EQUATIONS - * - * Open source under the BSD License. - * - * Copyright 2001 Robert Penner - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * - Neither the name of the author nor the names of contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -var d3_ease_quad = d3_ease_poly(2), - d3_ease_cubic = d3_ease_poly(3); - -var d3_ease = { - linear: function() { return d3_ease_linear; }, - poly: d3_ease_poly, - quad: function() { return d3_ease_quad; }, - cubic: function() { return d3_ease_cubic; }, - sin: function() { return d3_ease_sin; }, - exp: function() { return d3_ease_exp; }, - circle: function() { return d3_ease_circle; }, - elastic: d3_ease_elastic, - back: d3_ease_back, - bounce: function() { return d3_ease_bounce; } -}; - -var d3_ease_mode = { - "in": function(f) { return f; }, - "out": d3_ease_reverse, - "in-out": d3_ease_reflect, - "out-in": function(f) { return d3_ease_reflect(d3_ease_reverse(f)); } -}; - -d3.ease = function(name) { - var i = name.indexOf("-"), - t = i >= 0 ? name.substring(0, i) : name, - m = i >= 0 ? name.substring(i + 1) : "in"; - return d3_ease_clamp(d3_ease_mode[m](d3_ease[t].apply(null, Array.prototype.slice.call(arguments, 1)))); -}; - -function d3_ease_clamp(f) { - return function(t) { - return t <= 0 ? 0 : t >= 1 ? 1 : f(t); - }; -} - -function d3_ease_reverse(f) { - return function(t) { - return 1 - f(1 - t); - }; -} - -function d3_ease_reflect(f) { - return function(t) { - return .5 * (t < .5 ? f(2 * t) : (2 - f(2 - 2 * t))); - }; -} - -function d3_ease_linear(t) { - return t; -} - -function d3_ease_poly(e) { - return function(t) { - return Math.pow(t, e); - } -} - -function d3_ease_sin(t) { - return 1 - Math.cos(t * Math.PI / 2); -} - -function d3_ease_exp(t) { - return Math.pow(2, 10 * (t - 1)); -} - -function d3_ease_circle(t) { - return 1 - Math.sqrt(1 - t * t); -} - -function d3_ease_elastic(a, p) { - var s; - if (arguments.length < 2) p = 0.45; - if (arguments.length < 1) { a = 1; s = p / 4; } - else s = p / (2 * Math.PI) * Math.asin(1 / a); - return function(t) { - return 1 + a * Math.pow(2, 10 * -t) * Math.sin((t - s) * 2 * Math.PI / p); - }; -} - -function d3_ease_back(s) { - if (!s) s = 1.70158; - return function(t) { - return t * t * ((s + 1) * t - s); - }; -} - -function d3_ease_bounce(t) { - return t < 1 / 2.75 ? 7.5625 * t * t - : t < 2 / 2.75 ? 7.5625 * (t -= 1.5 / 2.75) * t + .75 - : t < 2.5 / 2.75 ? 7.5625 * (t -= 2.25 / 2.75) * t + .9375 - : 7.5625 * (t -= 2.625 / 2.75) * t + .984375; -} -d3.event = null; -d3.interpolate = function(a, b) { - var i = d3.interpolators.length, f; - while (--i >= 0 && !(f = d3.interpolators[i](a, b))); - return f; -}; - -d3.interpolateNumber = function(a, b) { - b -= a; - return function(t) { return a + b * t; }; -}; - -d3.interpolateRound = function(a, b) { - b -= a; - return function(t) { return Math.round(a + b * t); }; -}; - -d3.interpolateString = function(a, b) { - var m, // current match - i, // current index - j, // current index (for coallescing) - s0 = 0, // start index of current string prefix - s1 = 0, // end index of current string prefix - s = [], // string constants and placeholders - q = [], // number interpolators - n, // q.length - o; - - // Reset our regular expression! - d3_interpolate_number.lastIndex = 0; - - // Find all numbers in b. - for (i = 0; m = d3_interpolate_number.exec(b); ++i) { - if (m.index) s.push(b.substring(s0, s1 = m.index)); - q.push({i: s.length, x: m[0]}); - s.push(null); - s0 = d3_interpolate_number.lastIndex; - } - if (s0 < b.length) s.push(b.substring(s0)); - - // Find all numbers in a. - for (i = 0, n = q.length; (m = d3_interpolate_number.exec(a)) && i < n; ++i) { - o = q[i]; - if (o.x == m[0]) { // The numbers match, so coallesce. - if (o.i) { - if (s[o.i + 1] == null) { // This match is followed by another number. - s[o.i - 1] += o.x; - s.splice(o.i, 1); - for (j = i + 1; j < n; ++j) q[j].i--; - } else { // This match is followed by a string, so coallesce twice. - s[o.i - 1] += o.x + s[o.i + 1]; - s.splice(o.i, 2); - for (j = i + 1; j < n; ++j) q[j].i -= 2; - } - } else { - if (s[o.i + 1] == null) { // This match is followed by another number. - s[o.i] = o.x; - } else { // This match is followed by a string, so coallesce twice. - s[o.i] = o.x + s[o.i + 1]; - s.splice(o.i + 1, 1); - for (j = i + 1; j < n; ++j) q[j].i--; - } - } - q.splice(i, 1); - n--; - i--; - } else { - o.x = d3.interpolateNumber(parseFloat(m[0]), parseFloat(o.x)); - } - } - - // Remove any numbers in b not found in a. - while (i < n) { - o = q.pop(); - if (s[o.i + 1] == null) { // This match is followed by another number. - s[o.i] = o.x; - } else { // This match is followed by a string, so coallesce twice. - s[o.i] = o.x + s[o.i + 1]; - s.splice(o.i + 1, 1); - } - n--; - } - - // Special optimization for only a single match. - if (s.length === 1) { - return s[0] == null ? q[0].x : function() { return b; }; - } - - // Otherwise, interpolate each of the numbers and rejoin the string. - return function(t) { - for (i = 0; i < n; ++i) s[(o = q[i]).i] = o.x(t); - return s.join(""); - }; -}; - -d3.interpolateRgb = function(a, b) { - a = d3.rgb(a); - b = d3.rgb(b); - var ar = a.r, - ag = a.g, - ab = a.b, - br = b.r - ar, - bg = b.g - ag, - bb = b.b - ab; - return function(t) { - return "#" - + d3_rgb_hex(Math.round(ar + br * t)) - + d3_rgb_hex(Math.round(ag + bg * t)) - + d3_rgb_hex(Math.round(ab + bb * t)); - }; -}; - -// interpolates HSL space, but outputs RGB string (for compatibility) -d3.interpolateHsl = function(a, b) { - a = d3.hsl(a); - b = d3.hsl(b); - var h0 = a.h, - s0 = a.s, - l0 = a.l, - h1 = b.h - h0, - s1 = b.s - s0, - l1 = b.l - l0; - return function(t) { - return d3_hsl_rgb(h0 + h1 * t, s0 + s1 * t, l0 + l1 * t).toString(); - }; -}; - -d3.interpolateArray = function(a, b) { - var x = [], - c = [], - na = a.length, - nb = b.length, - n0 = Math.min(a.length, b.length), - i; - for (i = 0; i < n0; ++i) x.push(d3.interpolate(a[i], b[i])); - for (; i < na; ++i) c[i] = a[i]; - for (; i < nb; ++i) c[i] = b[i]; - return function(t) { - for (i = 0; i < n0; ++i) c[i] = x[i](t); - return c; - }; -}; - -d3.interpolateObject = function(a, b) { - var i = {}, - c = {}, - k; - for (k in a) { - if (k in b) { - i[k] = d3_interpolateByName(k)(a[k], b[k]); - } else { - c[k] = a[k]; - } - } - for (k in b) { - if (!(k in a)) { - c[k] = b[k]; - } - } - return function(t) { - for (k in i) c[k] = i[k](t); - return c; - }; -} - -var d3_interpolate_number = /[-+]?(?:\d+\.\d+|\d+\.|\.\d+|\d+)(?:[eE][-]?\d+)?/g, - d3_interpolate_rgb = {background: 1, fill: 1, stroke: 1}; - -function d3_interpolateByName(n) { - return n in d3_interpolate_rgb || /\bcolor\b/.test(n) - ? d3.interpolateRgb - : d3.interpolate; -} - -d3.interpolators = [ - d3.interpolateObject, - function(a, b) { return (b instanceof Array) && d3.interpolateArray(a, b); }, - function(a, b) { return (typeof b === "string") && d3.interpolateString(String(a), b); }, - function(a, b) { return (typeof b === "string" ? b in d3_rgb_names || /^(#|rgb\(|hsl\()/.test(b) : b instanceof d3_Rgb || b instanceof d3_Hsl) && d3.interpolateRgb(String(a), b); }, - function(a, b) { return (typeof b === "number") && d3.interpolateNumber(+a, b); } -]; -function d3_uninterpolateNumber(a, b) { - b = b - (a = +a) ? 1 / (b - a) : 0; - return function(x) { return (x - a) * b; }; -} - -function d3_uninterpolateClamp(a, b) { - b = b - (a = +a) ? 1 / (b - a) : 0; - return function(x) { return Math.max(0, Math.min(1, (x - a) * b)); }; -} -d3.rgb = function(r, g, b) { - return arguments.length === 1 - ? (r instanceof d3_Rgb ? d3_rgb(r.r, r.g, r.b) - : d3_rgb_parse("" + r, d3_rgb, d3_hsl_rgb)) - : d3_rgb(~~r, ~~g, ~~b); -}; - -function d3_rgb(r, g, b) { - return new d3_Rgb(r, g, b); -} - -function d3_Rgb(r, g, b) { - this.r = r; - this.g = g; - this.b = b; -} - -d3_Rgb.prototype.brighter = function(k) { - k = Math.pow(0.7, arguments.length ? k : 1); - var r = this.r, - g = this.g, - b = this.b, - i = 30; - if (!r && !g && !b) return d3_rgb(i, i, i); - if (r && r < i) r = i; - if (g && g < i) g = i; - if (b && b < i) b = i; - return d3_rgb( - Math.min(255, Math.floor(r / k)), - Math.min(255, Math.floor(g / k)), - Math.min(255, Math.floor(b / k))); -}; - -d3_Rgb.prototype.darker = function(k) { - k = Math.pow(0.7, arguments.length ? k : 1); - return d3_rgb( - Math.floor(k * this.r), - Math.floor(k * this.g), - Math.floor(k * this.b)); -}; - -d3_Rgb.prototype.hsl = function() { - return d3_rgb_hsl(this.r, this.g, this.b); -}; - -d3_Rgb.prototype.toString = function() { - return "#" + d3_rgb_hex(this.r) + d3_rgb_hex(this.g) + d3_rgb_hex(this.b); -}; - -function d3_rgb_hex(v) { - return v < 0x10 - ? "0" + Math.max(0, v).toString(16) - : Math.min(255, v).toString(16); -} - -function d3_rgb_parse(format, rgb, hsl) { - var r = 0, // red channel; int in [0, 255] - g = 0, // green channel; int in [0, 255] - b = 0, // blue channel; int in [0, 255] - m1, // CSS color specification match - m2, // CSS color specification type (e.g., rgb) - name; - - /* Handle hsl, rgb. */ - m1 = /([a-z]+)\((.*)\)/i.exec(format); - if (m1) { - m2 = m1[2].split(","); - switch (m1[1]) { - case "hsl": { - return hsl( - parseFloat(m2[0]), // degrees - parseFloat(m2[1]) / 100, // percentage - parseFloat(m2[2]) / 100 // percentage - ); - } - case "rgb": { - return rgb( - d3_rgb_parseNumber(m2[0]), - d3_rgb_parseNumber(m2[1]), - d3_rgb_parseNumber(m2[2]) - ); - } - } - } - - /* Named colors. */ - if (name = d3_rgb_names[format]) return rgb(name.r, name.g, name.b); - - /* Hexadecimal colors: #rgb and #rrggbb. */ - if (format != null && format.charAt(0) === "#") { - if (format.length === 4) { - r = format.charAt(1); r += r; - g = format.charAt(2); g += g; - b = format.charAt(3); b += b; - } else if (format.length === 7) { - r = format.substring(1, 3); - g = format.substring(3, 5); - b = format.substring(5, 7); - } - r = parseInt(r, 16); - g = parseInt(g, 16); - b = parseInt(b, 16); - } - - return rgb(r, g, b); -} - -function d3_rgb_hsl(r, g, b) { - var min = Math.min(r /= 255, g /= 255, b /= 255), - max = Math.max(r, g, b), - d = max - min, - h, - s, - l = (max + min) / 2; - if (d) { - s = l < .5 ? d / (max + min) : d / (2 - max - min); - if (r == max) h = (g - b) / d + (g < b ? 6 : 0); - else if (g == max) h = (b - r) / d + 2; - else h = (r - g) / d + 4; - h *= 60; - } else { - s = h = 0; - } - return d3_hsl(h, s, l); -} - -function d3_rgb_parseNumber(c) { // either integer or percentage - var f = parseFloat(c); - return c.charAt(c.length - 1) === "%" ? Math.round(f * 2.55) : f; -} - -var d3_rgb_names = { - aliceblue: "#f0f8ff", - antiquewhite: "#faebd7", - aqua: "#00ffff", - aquamarine: "#7fffd4", - azure: "#f0ffff", - beige: "#f5f5dc", - bisque: "#ffe4c4", - black: "#000000", - blanchedalmond: "#ffebcd", - blue: "#0000ff", - blueviolet: "#8a2be2", - brown: "#a52a2a", - burlywood: "#deb887", - cadetblue: "#5f9ea0", - chartreuse: "#7fff00", - chocolate: "#d2691e", - coral: "#ff7f50", - cornflowerblue: "#6495ed", - cornsilk: "#fff8dc", - crimson: "#dc143c", - cyan: "#00ffff", - darkblue: "#00008b", - darkcyan: "#008b8b", - darkgoldenrod: "#b8860b", - darkgray: "#a9a9a9", - darkgreen: "#006400", - darkgrey: "#a9a9a9", - darkkhaki: "#bdb76b", - darkmagenta: "#8b008b", - darkolivegreen: "#556b2f", - darkorange: "#ff8c00", - darkorchid: "#9932cc", - darkred: "#8b0000", - darksalmon: "#e9967a", - darkseagreen: "#8fbc8f", - darkslateblue: "#483d8b", - darkslategray: "#2f4f4f", - darkslategrey: "#2f4f4f", - darkturquoise: "#00ced1", - darkviolet: "#9400d3", - deeppink: "#ff1493", - deepskyblue: "#00bfff", - dimgray: "#696969", - dimgrey: "#696969", - dodgerblue: "#1e90ff", - firebrick: "#b22222", - floralwhite: "#fffaf0", - forestgreen: "#228b22", - fuchsia: "#ff00ff", - gainsboro: "#dcdcdc", - ghostwhite: "#f8f8ff", - gold: "#ffd700", - goldenrod: "#daa520", - gray: "#808080", - green: "#008000", - greenyellow: "#adff2f", - grey: "#808080", - honeydew: "#f0fff0", - hotpink: "#ff69b4", - indianred: "#cd5c5c", - indigo: "#4b0082", - ivory: "#fffff0", - khaki: "#f0e68c", - lavender: "#e6e6fa", - lavenderblush: "#fff0f5", - lawngreen: "#7cfc00", - lemonchiffon: "#fffacd", - lightblue: "#add8e6", - lightcoral: "#f08080", - lightcyan: "#e0ffff", - lightgoldenrodyellow: "#fafad2", - lightgray: "#d3d3d3", - lightgreen: "#90ee90", - lightgrey: "#d3d3d3", - lightpink: "#ffb6c1", - lightsalmon: "#ffa07a", - lightseagreen: "#20b2aa", - lightskyblue: "#87cefa", - lightslategray: "#778899", - lightslategrey: "#778899", - lightsteelblue: "#b0c4de", - lightyellow: "#ffffe0", - lime: "#00ff00", - limegreen: "#32cd32", - linen: "#faf0e6", - magenta: "#ff00ff", - maroon: "#800000", - mediumaquamarine: "#66cdaa", - mediumblue: "#0000cd", - mediumorchid: "#ba55d3", - mediumpurple: "#9370db", - mediumseagreen: "#3cb371", - mediumslateblue: "#7b68ee", - mediumspringgreen: "#00fa9a", - mediumturquoise: "#48d1cc", - mediumvioletred: "#c71585", - midnightblue: "#191970", - mintcream: "#f5fffa", - mistyrose: "#ffe4e1", - moccasin: "#ffe4b5", - navajowhite: "#ffdead", - navy: "#000080", - oldlace: "#fdf5e6", - olive: "#808000", - olivedrab: "#6b8e23", - orange: "#ffa500", - orangered: "#ff4500", - orchid: "#da70d6", - palegoldenrod: "#eee8aa", - palegreen: "#98fb98", - paleturquoise: "#afeeee", - palevioletred: "#db7093", - papayawhip: "#ffefd5", - peachpuff: "#ffdab9", - peru: "#cd853f", - pink: "#ffc0cb", - plum: "#dda0dd", - powderblue: "#b0e0e6", - purple: "#800080", - red: "#ff0000", - rosybrown: "#bc8f8f", - royalblue: "#4169e1", - saddlebrown: "#8b4513", - salmon: "#fa8072", - sandybrown: "#f4a460", - seagreen: "#2e8b57", - seashell: "#fff5ee", - sienna: "#a0522d", - silver: "#c0c0c0", - skyblue: "#87ceeb", - slateblue: "#6a5acd", - slategray: "#708090", - slategrey: "#708090", - snow: "#fffafa", - springgreen: "#00ff7f", - steelblue: "#4682b4", - tan: "#d2b48c", - teal: "#008080", - thistle: "#d8bfd8", - tomato: "#ff6347", - turquoise: "#40e0d0", - violet: "#ee82ee", - wheat: "#f5deb3", - white: "#ffffff", - whitesmoke: "#f5f5f5", - yellow: "#ffff00", - yellowgreen: "#9acd32" -}; - -for (var d3_rgb_name in d3_rgb_names) { - d3_rgb_names[d3_rgb_name] = d3_rgb_parse( - d3_rgb_names[d3_rgb_name], - d3_rgb, - d3_hsl_rgb); -} -d3.hsl = function(h, s, l) { - return arguments.length === 1 - ? (h instanceof d3_Hsl ? d3_hsl(h.h, h.s, h.l) - : d3_rgb_parse("" + h, d3_rgb_hsl, d3_hsl)) - : d3_hsl(+h, +s, +l); -}; - -function d3_hsl(h, s, l) { - return new d3_Hsl(h, s, l); -} - -function d3_Hsl(h, s, l) { - this.h = h; - this.s = s; - this.l = l; -} - -d3_Hsl.prototype.brighter = function(k) { - k = Math.pow(0.7, arguments.length ? k : 1); - return d3_hsl(this.h, this.s, this.l / k); -}; - -d3_Hsl.prototype.darker = function(k) { - k = Math.pow(0.7, arguments.length ? k : 1); - return d3_hsl(this.h, this.s, k * this.l); -}; - -d3_Hsl.prototype.rgb = function() { - return d3_hsl_rgb(this.h, this.s, this.l); -}; - -d3_Hsl.prototype.toString = function() { - return this.rgb().toString(); -}; - -function d3_hsl_rgb(h, s, l) { - var m1, - m2; - - /* Some simple corrections for h, s and l. */ - h = h % 360; if (h < 0) h += 360; - s = s < 0 ? 0 : s > 1 ? 1 : s; - l = l < 0 ? 0 : l > 1 ? 1 : l; - - /* From FvD 13.37, CSS Color Module Level 3 */ - m2 = l <= .5 ? l * (1 + s) : l + s - l * s; - m1 = 2 * l - m2; - - function v(h) { - if (h > 360) h -= 360; - else if (h < 0) h += 360; - if (h < 60) return m1 + (m2 - m1) * h / 60; - if (h < 180) return m2; - if (h < 240) return m1 + (m2 - m1) * (240 - h) / 60; - return m1; - } - - function vv(h) { - return Math.round(v(h) * 255); - } - - return d3_rgb(vv(h + 120), vv(h), vv(h - 120)); -} -function d3_selection(groups) { - d3_arraySubclass(groups, d3_selectionPrototype); - return groups; -} - -var d3_select = function(s, n) { return n.querySelector(s); }, - d3_selectAll = function(s, n) { return n.querySelectorAll(s); }; - -// Prefer Sizzle, if available. -if (typeof Sizzle === "function") { - d3_select = function(s, n) { return Sizzle(s, n)[0]; }; - d3_selectAll = function(s, n) { return Sizzle.uniqueSort(Sizzle(s, n)); }; -} - -var d3_selectionPrototype = []; - -d3.selection = function() { - return d3_selectionRoot; -}; - -d3.selection.prototype = d3_selectionPrototype; -d3_selectionPrototype.select = function(selector) { - var subgroups = [], - subgroup, - subnode, - group, - node; - - if (typeof selector !== "function") selector = d3_selection_selector(selector); - - for (var j = -1, m = this.length; ++j < m;) { - subgroups.push(subgroup = []); - subgroup.parentNode = (group = this[j]).parentNode; - for (var i = -1, n = group.length; ++i < n;) { - if (node = group[i]) { - subgroup.push(subnode = selector.call(node, node.__data__, i)); - if (subnode && "__data__" in node) subnode.__data__ = node.__data__; - } else { - subgroup.push(null); - } - } - } - - return d3_selection(subgroups); -}; - -function d3_selection_selector(selector) { - return function() { - return d3_select(selector, this); - }; -} -d3_selectionPrototype.selectAll = function(selector) { - var subgroups = [], - subgroup, - node; - - if (typeof selector !== "function") selector = d3_selection_selectorAll(selector); - - for (var j = -1, m = this.length; ++j < m;) { - for (var group = this[j], i = -1, n = group.length; ++i < n;) { - if (node = group[i]) { - subgroups.push(subgroup = d3_array(selector.call(node, node.__data__, i))); - subgroup.parentNode = node; - } - } - } - - return d3_selection(subgroups); -}; - -function d3_selection_selectorAll(selector) { - return function() { - return d3_selectAll(selector, this); - }; -} -d3_selectionPrototype.attr = function(name, value) { - name = d3.ns.qualify(name); - - // If no value is specified, return the first value. - if (arguments.length < 2) { - var node = this.node(); - return name.local - ? node.getAttributeNS(name.space, name.local) - : node.getAttribute(name); - } - - function attrNull() { - this.removeAttribute(name); - } - - function attrNullNS() { - this.removeAttributeNS(name.space, name.local); - } - - function attrConstant() { - this.setAttribute(name, value); - } - - function attrConstantNS() { - this.setAttributeNS(name.space, name.local, value); - } - - function attrFunction() { - var x = value.apply(this, arguments); - if (x == null) this.removeAttribute(name); - else this.setAttribute(name, x); - } - - function attrFunctionNS() { - var x = value.apply(this, arguments); - if (x == null) this.removeAttributeNS(name.space, name.local); - else this.setAttributeNS(name.space, name.local, x); - } - - return this.each(value == null - ? (name.local ? attrNullNS : attrNull) : (typeof value === "function" - ? (name.local ? attrFunctionNS : attrFunction) - : (name.local ? attrConstantNS : attrConstant))); -}; -d3_selectionPrototype.classed = function(name, value) { - var names = name.split(d3_selection_classedWhitespace), - n = names.length, - i = -1; - if (arguments.length > 1) { - while (++i < n) d3_selection_classed.call(this, names[i], value); - return this; - } else { - while (++i < n) if (!d3_selection_classed.call(this, names[i])) return false; - return true; - } -}; - -var d3_selection_classedWhitespace = /\s+/g; - -function d3_selection_classed(name, value) { - var re = new RegExp("(^|\\s+)" + d3.requote(name) + "(\\s+|$)", "g"); - - // If no value is specified, return the first value. - if (arguments.length < 2) { - var node = this.node(); - if (c = node.classList) return c.contains(name); - var c = node.className; - re.lastIndex = 0; - return re.test(c.baseVal != null ? c.baseVal : c); - } - - function classedAdd() { - if (c = this.classList) return c.add(name); - var c = this.className, - cb = c.baseVal != null, - cv = cb ? c.baseVal : c; - re.lastIndex = 0; - if (!re.test(cv)) { - cv = d3_collapse(cv + " " + name); - if (cb) c.baseVal = cv; - else this.className = cv; - } - } - - function classedRemove() { - if (c = this.classList) return c.remove(name); - var c = this.className, - cb = c.baseVal != null, - cv = cb ? c.baseVal : c; - cv = d3_collapse(cv.replace(re, " ")); - if (cb) c.baseVal = cv; - else this.className = cv; - } - - function classedFunction() { - (value.apply(this, arguments) - ? classedAdd - : classedRemove).call(this); - } - - return this.each(typeof value === "function" - ? classedFunction : value - ? classedAdd - : classedRemove); -} -d3_selectionPrototype.style = function(name, value, priority) { - if (arguments.length < 3) priority = ""; - - // If no value is specified, return the first value. - if (arguments.length < 2) return window - .getComputedStyle(this.node(), null) - .getPropertyValue(name); - - function styleNull() { - this.style.removeProperty(name); - } - - function styleConstant() { - this.style.setProperty(name, value, priority); - } - - function styleFunction() { - var x = value.apply(this, arguments); - if (x == null) this.style.removeProperty(name); - else this.style.setProperty(name, x, priority); - } - - return this.each(value == null - ? styleNull : (typeof value === "function" - ? styleFunction : styleConstant)); -}; -d3_selectionPrototype.property = function(name, value) { - - // If no value is specified, return the first value. - if (arguments.length < 2) return this.node()[name]; - - function propertyNull() { - delete this[name]; - } - - function propertyConstant() { - this[name] = value; - } - - function propertyFunction() { - var x = value.apply(this, arguments); - if (x == null) delete this[name]; - else this[name] = x; - } - - return this.each(value == null - ? propertyNull : (typeof value === "function" - ? propertyFunction : propertyConstant)); -}; -d3_selectionPrototype.text = function(value) { - return arguments.length < 1 ? this.node().textContent - : (this.each(typeof value === "function" - ? function() { this.textContent = value.apply(this, arguments); } - : function() { this.textContent = value; })); -}; -d3_selectionPrototype.html = function(value) { - return arguments.length < 1 ? this.node().innerHTML - : (this.each(typeof value === "function" - ? function() { this.innerHTML = value.apply(this, arguments); } - : function() { this.innerHTML = value; })); -}; -// TODO append(node)? -// TODO append(function)? -d3_selectionPrototype.append = function(name) { - name = d3.ns.qualify(name); - - function append() { - return this.appendChild(document.createElement(name)); - } - - function appendNS() { - return this.appendChild(document.createElementNS(name.space, name.local)); - } - - return this.select(name.local ? appendNS : append); -}; -// TODO insert(node, function)? -// TODO insert(function, string)? -// TODO insert(function, function)? -d3_selectionPrototype.insert = function(name, before) { - name = d3.ns.qualify(name); - - function insert() { - return this.insertBefore( - document.createElement(name), - d3_select(before, this)); - } - - function insertNS() { - return this.insertBefore( - document.createElementNS(name.space, name.local), - d3_select(before, this)); - } - - return this.select(name.local ? insertNS : insert); -}; -// TODO remove(selector)? -// TODO remove(node)? -// TODO remove(function)? -d3_selectionPrototype.remove = function() { - return this.each(function() { - var parent = this.parentNode; - if (parent) parent.removeChild(this); - }); -}; -// TODO data(null) for clearing data? -d3_selectionPrototype.data = function(data, join) { - var enter = [], - update = [], - exit = []; - - function bind(group, groupData) { - var i, - n = group.length, - m = groupData.length, - n0 = Math.min(n, m), - n1 = Math.max(n, m), - updateNodes = [], - enterNodes = [], - exitNodes = [], - node, - nodeData; - - if (join) { - var nodeByKey = {}, - keys = [], - key, - j = groupData.length; - - for (i = -1; ++i < n;) { - key = join.call(node = group[i], node.__data__, i); - if (key in nodeByKey) { - exitNodes[j++] = node; // duplicate key - } else { - nodeByKey[key] = node; - } - keys.push(key); - } - - for (i = -1; ++i < m;) { - node = nodeByKey[key = join.call(groupData, nodeData = groupData[i], i)]; - if (node) { - node.__data__ = nodeData; - updateNodes[i] = node; - enterNodes[i] = exitNodes[i] = null; - } else { - enterNodes[i] = d3_selection_dataNode(nodeData); - updateNodes[i] = exitNodes[i] = null; - } - delete nodeByKey[key]; - } - - for (i = -1; ++i < n;) { - if (keys[i] in nodeByKey) { - exitNodes[i] = group[i]; - } - } - } else { - for (i = -1; ++i < n0;) { - node = group[i]; - nodeData = groupData[i]; - if (node) { - node.__data__ = nodeData; - updateNodes[i] = node; - enterNodes[i] = exitNodes[i] = null; - } else { - enterNodes[i] = d3_selection_dataNode(nodeData); - updateNodes[i] = exitNodes[i] = null; - } - } - for (; i < m; ++i) { - enterNodes[i] = d3_selection_dataNode(groupData[i]); - updateNodes[i] = exitNodes[i] = null; - } - for (; i < n1; ++i) { - exitNodes[i] = group[i]; - enterNodes[i] = updateNodes[i] = null; - } - } - - enterNodes.update - = updateNodes; - - enterNodes.parentNode - = updateNodes.parentNode - = exitNodes.parentNode - = group.parentNode; - - enter.push(enterNodes); - update.push(updateNodes); - exit.push(exitNodes); - } - - var i = -1, - n = this.length, - group; - if (typeof data === "function") { - while (++i < n) { - bind(group = this[i], data.call(group, group.parentNode.__data__, i)); - } - } else { - while (++i < n) { - bind(group = this[i], data); - } - } - - var selection = d3_selection(update); - selection.enter = function() { return d3_selection_enter(enter); }; - selection.exit = function() { return d3_selection(exit); }; - return selection; -}; - -function d3_selection_dataNode(data) { - return {__data__: data}; -} -// TODO preserve null elements to maintain index? -d3_selectionPrototype.filter = function(filter) { - var subgroups = [], - subgroup, - group, - node; - - for (var j = 0, m = this.length; j < m; j++) { - subgroups.push(subgroup = []); - subgroup.parentNode = (group = this[j]).parentNode; - for (var i = 0, n = group.length; i < n; i++) { - if ((node = group[i]) && filter.call(node, node.__data__, i)) { - subgroup.push(node); - } - } - } - - return d3_selection(subgroups); -}; -d3_selectionPrototype.map = function(map) { - return this.each(function() { - this.__data__ = map.apply(this, arguments); - }); -}; -d3_selectionPrototype.sort = function(comparator) { - comparator = d3_selection_sortComparator.apply(this, arguments); - for (var j = 0, m = this.length; j < m; j++) { - for (var group = this[j].sort(comparator), i = 1, n = group.length, prev = group[0]; i < n; i++) { - var node = group[i]; - if (node) { - if (prev) prev.parentNode.insertBefore(node, prev.nextSibling); - prev = node; - } - } - } - return this; -}; - -function d3_selection_sortComparator(comparator) { - if (!arguments.length) comparator = d3.ascending; - return function(a, b) { - return comparator(a && a.__data__, b && b.__data__); - }; -} -// type can be namespaced, e.g., "click.foo" -// listener can be null for removal -d3_selectionPrototype.on = function(type, listener, capture) { - if (arguments.length < 3) capture = false; - - // parse the type specifier - var name = "__on" + type, i = type.indexOf("."); - if (i > 0) type = type.substring(0, i); - - // if called with only one argument, return the current listener - if (arguments.length < 2) return (i = this.node()[name]) && i._; - - // remove the old event listener, and add the new event listener - return this.each(function(d, i) { - var node = this; - - if (node[name]) node.removeEventListener(type, node[name], capture); - if (listener) node.addEventListener(type, node[name] = l, capture); - - // wrapped event listener that preserves i - function l(e) { - var o = d3.event; // Events can be reentrant (e.g., focus). - d3.event = e; - try { - listener.call(node, node.__data__, i); - } finally { - d3.event = o; - } - } - - // stash the unwrapped listener for retrieval - l._ = listener; - }); -}; -d3_selectionPrototype.each = function(callback) { - for (var j = -1, m = this.length; ++j < m;) { - for (var group = this[j], i = -1, n = group.length; ++i < n;) { - var node = group[i]; - if (node) callback.call(node, node.__data__, i, j); - } - } - return this; -}; -// -// Note: assigning to the arguments array simultaneously changes the value of -// the corresponding argument! -// -// TODO The `this` argument probably shouldn't be the first argument to the -// callback, anyway, since it's redundant. However, that will require a major -// version bump due to backwards compatibility, so I'm not changing it right -// away. -// -d3_selectionPrototype.call = function(callback) { - callback.apply(this, (arguments[0] = this, arguments)); - return this; -}; -d3_selectionPrototype.empty = function() { - return !this.node(); -}; -d3_selectionPrototype.node = function(callback) { - for (var j = 0, m = this.length; j < m; j++) { - for (var group = this[j], i = 0, n = group.length; i < n; i++) { - var node = group[i]; - if (node) return node; - } - } - return null; -}; -d3_selectionPrototype.transition = function() { - var subgroups = [], - subgroup, - node; - - for (var j = -1, m = this.length; ++j < m;) { - subgroups.push(subgroup = []); - for (var group = this[j], i = -1, n = group.length; ++i < n;) { - subgroup.push((node = group[i]) ? {node: node, delay: 0, duration: 250} : null); - } - } - - return d3_transition(subgroups, d3_transitionInheritId || ++d3_transitionId, Date.now()); -}; -var d3_selectionRoot = d3_selection([[document]]); - -d3_selectionRoot[0].parentNode = document.documentElement; - -// TODO fast singleton implementation! -// TODO select(function) -d3.select = function(selector) { - return typeof selector === "string" - ? d3_selectionRoot.select(selector) - : d3_selection([[selector]]); // assume node -}; - -// TODO selectAll(function) -d3.selectAll = function(selector) { - return typeof selector === "string" - ? d3_selectionRoot.selectAll(selector) - : d3_selection([d3_array(selector)]); // assume node[] -}; -function d3_selection_enter(selection) { - d3_arraySubclass(selection, d3_selection_enterPrototype); - return selection; -} - -var d3_selection_enterPrototype = []; - -d3_selection_enterPrototype.append = d3_selectionPrototype.append; -d3_selection_enterPrototype.insert = d3_selectionPrototype.insert; -d3_selection_enterPrototype.empty = d3_selectionPrototype.empty; -d3_selection_enterPrototype.node = d3_selectionPrototype.node; -d3_selection_enterPrototype.select = function(selector) { - var subgroups = [], - subgroup, - subnode, - upgroup, - group, - node; - - for (var j = -1, m = this.length; ++j < m;) { - upgroup = (group = this[j]).update; - subgroups.push(subgroup = []); - subgroup.parentNode = group.parentNode; - for (var i = -1, n = group.length; ++i < n;) { - if (node = group[i]) { - subgroup.push(upgroup[i] = subnode = selector.call(group.parentNode, node.__data__, i)); - subnode.__data__ = node.__data__; - } else { - subgroup.push(null); - } - } - } - - return d3_selection(subgroups); -}; -function d3_transition(groups, id, time) { - d3_arraySubclass(groups, d3_transitionPrototype); - - var tweens = {}, - event = d3.dispatch("start", "end"), - ease = d3_transitionEase; - - groups.id = id; - - groups.time = time; - - groups.tween = function(name, tween) { - if (arguments.length < 2) return tweens[name]; - if (tween == null) delete tweens[name]; - else tweens[name] = tween; - return groups; - }; - - groups.ease = function(value) { - if (!arguments.length) return ease; - ease = typeof value === "function" ? value : d3.ease.apply(d3, arguments); - return groups; - }; - - groups.each = function(type, listener) { - if (arguments.length < 2) return d3_transition_each.call(groups, type); - event[type].add(listener); - return groups; - }; - - d3.timer(function(elapsed) { - groups.each(function(d, i, j) { - var tweened = [], - node = this, - delay = groups[j][i].delay, - duration = groups[j][i].duration, - lock = node.__transition__ || (node.__transition__ = {active: 0, count: 0}); - - ++lock.count; - - delay <= elapsed ? start(elapsed) : d3.timer(start, delay, time); - - function start(elapsed) { - if (lock.active > id) return stop(); - lock.active = id; - - for (var tween in tweens) { - if (tween = tweens[tween].call(node, d, i)) { - tweened.push(tween); - } - } - - event.start.dispatch.call(node, d, i); - if (!tick(elapsed)) d3.timer(tick, 0, time); - return 1; - } - - function tick(elapsed) { - if (lock.active !== id) return stop(); - - var t = (elapsed - delay) / duration, - e = ease(t), - n = tweened.length; - - while (n > 0) { - tweened[--n].call(node, e); - } - - if (t >= 1) { - stop(); - d3_transitionInheritId = id; - event.end.dispatch.call(node, d, i); - d3_transitionInheritId = 0; - return 1; - } - } - - function stop() { - if (!--lock.count) delete node.__transition__; - return 1; - } - }); - return 1; - }, 0, time); - - return groups; -} - -var d3_transitionRemove = {}; - -function d3_transitionNull(d, i, a) { - return a != "" && d3_transitionRemove; -} - -function d3_transitionTween(b) { - - function transitionFunction(d, i, a) { - var v = b.call(this, d, i); - return v == null - ? a != "" && d3_transitionRemove - : a != v && d3.interpolate(a, v); - } - - function transitionString(d, i, a) { - return a != b && d3.interpolate(a, b); - } - - return typeof b === "function" ? transitionFunction - : b == null ? d3_transitionNull - : (b += "", transitionString); -} - -var d3_transitionPrototype = [], - d3_transitionId = 0, - d3_transitionInheritId = 0, - d3_transitionEase = d3.ease("cubic-in-out"); - -d3_transitionPrototype.call = d3_selectionPrototype.call; - -d3.transition = function() { - return d3_selectionRoot.transition(); -}; - -d3.transition.prototype = d3_transitionPrototype; -d3_transitionPrototype.select = function(selector) { - var subgroups = [], - subgroup, - subnode, - node; - - if (typeof selector !== "function") selector = d3_selection_selector(selector); - - for (var j = -1, m = this.length; ++j < m;) { - subgroups.push(subgroup = []); - for (var group = this[j], i = -1, n = group.length; ++i < n;) { - if ((node = group[i]) && (subnode = selector.call(node.node, node.node.__data__, i))) { - if ("__data__" in node.node) subnode.__data__ = node.node.__data__; - subgroup.push({node: subnode, delay: node.delay, duration: node.duration}); - } else { - subgroup.push(null); - } - } - } - - return d3_transition(subgroups, this.id, this.time).ease(this.ease()); -}; -d3_transitionPrototype.selectAll = function(selector) { - var subgroups = [], - subgroup, - subnodes, - node; - - if (typeof selector !== "function") selector = d3_selection_selectorAll(selector); - - for (var j = -1, m = this.length; ++j < m;) { - for (var group = this[j], i = -1, n = group.length; ++i < n;) { - if (node = group[i]) { - subnodes = selector.call(node.node, node.node.__data__, i); - subgroups.push(subgroup = []); - for (var k = -1, o = subnodes.length; ++k < o;) { - subgroup.push({node: subnodes[k], delay: node.delay, duration: node.duration}); - } - } - } - } - - return d3_transition(subgroups, this.id, this.time).ease(this.ease()); -}; -d3_transitionPrototype.attr = function(name, value) { - return this.attrTween(name, d3_transitionTween(value)); -}; - -d3_transitionPrototype.attrTween = function(nameNS, tween) { - var name = d3.ns.qualify(nameNS); - - function attrTween(d, i) { - var f = tween.call(this, d, i, this.getAttribute(name)); - return f === d3_transitionRemove - ? (this.removeAttribute(name), null) - : f && function(t) { this.setAttribute(name, f(t)); }; - } - - function attrTweenNS(d, i) { - var f = tween.call(this, d, i, this.getAttributeNS(name.space, name.local)); - return f === d3_transitionRemove - ? (this.removeAttributeNS(name.space, name.local), null) - : f && function(t) { this.setAttributeNS(name.space, name.local, f(t)); }; - } - - return this.tween("attr." + nameNS, name.local ? attrTweenNS : attrTween); -}; -d3_transitionPrototype.style = function(name, value, priority) { - if (arguments.length < 3) priority = ""; - return this.styleTween(name, d3_transitionTween(value), priority); -}; - -d3_transitionPrototype.styleTween = function(name, tween, priority) { - if (arguments.length < 3) priority = ""; - return this.tween("style." + name, function(d, i) { - var f = tween.call(this, d, i, window.getComputedStyle(this, null).getPropertyValue(name)); - return f === d3_transitionRemove - ? (this.style.removeProperty(name), null) - : f && function(t) { this.style.setProperty(name, f(t), priority); }; - }); -}; -d3_transitionPrototype.text = function(value) { - return this.tween("text", function(d, i) { - this.textContent = typeof value === "function" - ? value.call(this, d, i) - : value; - }); -}; -d3_transitionPrototype.remove = function() { - return this.each("end", function() { - var p; - if (!this.__transition__ && (p = this.parentNode)) p.removeChild(this); - }); -}; -d3_transitionPrototype.delay = function(value) { - var groups = this; - return groups.each(typeof value === "function" - ? function(d, i, j) { groups[j][i].delay = +value.apply(this, arguments); } - : (value = +value, function(d, i, j) { groups[j][i].delay = value; })); -}; -d3_transitionPrototype.duration = function(value) { - var groups = this; - return groups.each(typeof value === "function" - ? function(d, i, j) { groups[j][i].duration = +value.apply(this, arguments); } - : (value = +value, function(d, i, j) { groups[j][i].duration = value; })); -}; -function d3_transition_each(callback) { - for (var j = 0, m = this.length; j < m; j++) { - for (var group = this[j], i = 0, n = group.length; i < n; i++) { - var node = group[i]; - if (node) callback.call(node = node.node, node.__data__, i, j); - } - } - return this; -} -d3_transitionPrototype.transition = function() { - return this.select(d3_this); -}; -var d3_timer_queue = null, - d3_timer_interval, // is an interval (or frame) active? - d3_timer_timeout; // is a timeout active? - -// The timer will continue to fire until callback returns true. -d3.timer = function(callback, delay, then) { - var found = false, - t0, - t1 = d3_timer_queue; - - if (arguments.length < 3) { - if (arguments.length < 2) delay = 0; - else if (!isFinite(delay)) return; - then = Date.now(); - } - - // See if the callback's already in the queue. - while (t1) { - if (t1.callback === callback) { - t1.then = then; - t1.delay = delay; - found = true; - break; - } - t0 = t1; - t1 = t1.next; - } - - // Otherwise, add the callback to the queue. - if (!found) d3_timer_queue = { - callback: callback, - then: then, - delay: delay, - next: d3_timer_queue - }; - - // Start animatin'! - if (!d3_timer_interval) { - d3_timer_timeout = clearTimeout(d3_timer_timeout); - d3_timer_interval = 1; - d3_timer_frame(d3_timer_step); - } -} - -function d3_timer_step() { - var elapsed, - now = Date.now(), - t1 = d3_timer_queue; - - while (t1) { - elapsed = now - t1.then; - if (elapsed >= t1.delay) t1.flush = t1.callback(elapsed); - t1 = t1.next; - } - - var delay = d3_timer_flush() - now; - if (delay > 24) { - if (isFinite(delay)) { - clearTimeout(d3_timer_timeout); - d3_timer_timeout = setTimeout(d3_timer_step, delay); - } - d3_timer_interval = 0; - } else { - d3_timer_interval = 1; - d3_timer_frame(d3_timer_step); - } -} - -d3.timer.flush = function() { - var elapsed, - now = Date.now(), - t1 = d3_timer_queue; - - while (t1) { - elapsed = now - t1.then; - if (!t1.delay) t1.flush = t1.callback(elapsed); - t1 = t1.next; - } - - d3_timer_flush(); -}; - -// Flush after callbacks, to avoid concurrent queue modification. -function d3_timer_flush() { - var t0 = null, - t1 = d3_timer_queue, - then = Infinity; - while (t1) { - if (t1.flush) { - t1 = t0 ? t0.next = t1.next : d3_timer_queue = t1.next; - } else { - then = Math.min(then, t1.then + t1.delay); - t1 = (t0 = t1).next; - } - } - return then; -} - -var d3_timer_frame = window.requestAnimationFrame - || window.webkitRequestAnimationFrame - || window.mozRequestAnimationFrame - || window.oRequestAnimationFrame - || window.msRequestAnimationFrame - || function(callback) { setTimeout(callback, 17); }; -function d3_noop() {} -d3.scale = {}; - -function d3_scaleExtent(domain) { - var start = domain[0], stop = domain[domain.length - 1]; - return start < stop ? [start, stop] : [stop, start]; -} -function d3_scale_nice(domain, nice) { - var i0 = 0, - i1 = domain.length - 1, - x0 = domain[i0], - x1 = domain[i1], - dx; - - if (x1 < x0) { - dx = i0; i0 = i1; i1 = dx; - dx = x0; x0 = x1; x1 = dx; - } - - if (dx = x1 - x0) { - nice = nice(dx); - domain[i0] = nice.floor(x0); - domain[i1] = nice.ceil(x1); - } - - return domain; -} - -function d3_scale_niceDefault() { - return Math; -} -d3.scale.linear = function() { - return d3_scale_linear([0, 1], [0, 1], d3.interpolate, false); -}; - -function d3_scale_linear(domain, range, interpolate, clamp) { - var output, - input; - - function rescale() { - var linear = domain.length == 2 ? d3_scale_bilinear : d3_scale_polylinear, - uninterpolate = clamp ? d3_uninterpolateClamp : d3_uninterpolateNumber; - output = linear(domain, range, uninterpolate, interpolate); - input = linear(range, domain, uninterpolate, d3.interpolate); - return scale; - } - - function scale(x) { - return output(x); - } - - // Note: requires range is coercible to number! - scale.invert = function(y) { - return input(y); - }; - - scale.domain = function(x) { - if (!arguments.length) return domain; - domain = x.map(Number); - return rescale(); - }; - - scale.range = function(x) { - if (!arguments.length) return range; - range = x; - return rescale(); - }; - - scale.rangeRound = function(x) { - return scale.range(x).interpolate(d3.interpolateRound); - }; - - scale.clamp = function(x) { - if (!arguments.length) return clamp; - clamp = x; - return rescale(); - }; - - scale.interpolate = function(x) { - if (!arguments.length) return interpolate; - interpolate = x; - return rescale(); - }; - - scale.ticks = function(m) { - return d3_scale_linearTicks(domain, m); - }; - - scale.tickFormat = function(m) { - return d3_scale_linearTickFormat(domain, m); - }; - - scale.nice = function() { - d3_scale_nice(domain, d3_scale_linearNice); - return rescale(); - }; - - scale.copy = function() { - return d3_scale_linear(domain, range, interpolate, clamp); - }; - - return rescale(); -}; - -function d3_scale_linearRebind(scale, linear) { - scale.range = d3.rebind(scale, linear.range); - scale.rangeRound = d3.rebind(scale, linear.rangeRound); - scale.interpolate = d3.rebind(scale, linear.interpolate); - scale.clamp = d3.rebind(scale, linear.clamp); - return scale; -} - -function d3_scale_linearNice(dx) { - dx = Math.pow(10, Math.round(Math.log(dx) / Math.LN10) - 1); - return { - floor: function(x) { return Math.floor(x / dx) * dx; }, - ceil: function(x) { return Math.ceil(x / dx) * dx; } - }; -} - -// TODO Dates? Ugh. -function d3_scale_linearTickRange(domain, m) { - var extent = d3_scaleExtent(domain), - span = extent[1] - extent[0], - step = Math.pow(10, Math.floor(Math.log(span / m) / Math.LN10)), - err = m / span * step; - - // Filter ticks to get closer to the desired count. - if (err <= .15) step *= 10; - else if (err <= .35) step *= 5; - else if (err <= .75) step *= 2; - - // Round start and stop values to step interval. - extent[0] = Math.ceil(extent[0] / step) * step; - extent[1] = Math.floor(extent[1] / step) * step + step * .5; // inclusive - extent[2] = step; - return extent; -} - -function d3_scale_linearTicks(domain, m) { - return d3.range.apply(d3, d3_scale_linearTickRange(domain, m)); -} - -function d3_scale_linearTickFormat(domain, m) { - return d3.format(",." + Math.max(0, -Math.floor(Math.log(d3_scale_linearTickRange(domain, m)[2]) / Math.LN10 + .01)) + "f"); -} -function d3_scale_bilinear(domain, range, uninterpolate, interpolate) { - var u = uninterpolate(domain[0], domain[1]), - i = interpolate(range[0], range[1]); - return function(x) { - return i(u(x)); - }; -} -function d3_scale_polylinear(domain, range, uninterpolate, interpolate) { - var u = [], - i = [], - j = 0, - n = domain.length; - - while (++j < n) { - u.push(uninterpolate(domain[j - 1], domain[j])); - i.push(interpolate(range[j - 1], range[j])); - } - - return function(x) { - var j = d3.bisect(domain, x, 1, domain.length - 1) - 1; - return i[j](u[j](x)); - }; -} -d3.scale.log = function() { - return d3_scale_log(d3.scale.linear(), d3_scale_logp); -}; - -function d3_scale_log(linear, log) { - var pow = log.pow; - - function scale(x) { - return linear(log(x)); - } - - scale.invert = function(x) { - return pow(linear.invert(x)); - }; - - scale.domain = function(x) { - if (!arguments.length) return linear.domain().map(pow); - log = x[0] < 0 ? d3_scale_logn : d3_scale_logp; - pow = log.pow; - linear.domain(x.map(log)); - return scale; - }; - - scale.nice = function() { - linear.domain(d3_scale_nice(linear.domain(), d3_scale_niceDefault)); - return scale; - }; - - scale.ticks = function() { - var extent = d3_scaleExtent(linear.domain()), - ticks = []; - if (extent.every(isFinite)) { - var i = Math.floor(extent[0]), - j = Math.ceil(extent[1]), - u = Math.round(pow(extent[0])), - v = Math.round(pow(extent[1])); - if (log === d3_scale_logn) { - ticks.push(pow(i)); - for (; i++ < j;) for (var k = 9; k > 0; k--) ticks.push(pow(i) * k); - } else { - for (; i < j; i++) for (var k = 1; k < 10; k++) ticks.push(pow(i) * k); - ticks.push(pow(i)); - } - for (i = 0; ticks[i] < u; i++) {} // strip small values - for (j = ticks.length; ticks[j - 1] > v; j--) {} // strip big values - ticks = ticks.slice(i, j); - } - return ticks; - }; - - scale.tickFormat = function(n, format) { - if (arguments.length < 2) format = d3_scale_logFormat; - if (arguments.length < 1) return format; - var k = n / scale.ticks().length, - f = log === d3_scale_logn ? (e = -1e-15, Math.floor) : (e = 1e-15, Math.ceil), - e; - return function(d) { - return d / pow(f(log(d) + e)) < k ? format(d) : ""; - }; - }; - - scale.copy = function() { - return d3_scale_log(linear.copy(), log); - }; - - return d3_scale_linearRebind(scale, linear); -}; - -var d3_scale_logFormat = d3.format("e"); - -function d3_scale_logp(x) { - return Math.log(x) / Math.LN10; -} - -function d3_scale_logn(x) { - return -Math.log(-x) / Math.LN10; -} - -d3_scale_logp.pow = function(x) { - return Math.pow(10, x); -}; - -d3_scale_logn.pow = function(x) { - return -Math.pow(10, -x); -}; -d3.scale.pow = function() { - return d3_scale_pow(d3.scale.linear(), 1); -}; - -function d3_scale_pow(linear, exponent) { - var powp = d3_scale_powPow(exponent), - powb = d3_scale_powPow(1 / exponent); - - function scale(x) { - return linear(powp(x)); - } - - scale.invert = function(x) { - return powb(linear.invert(x)); - }; - - scale.domain = function(x) { - if (!arguments.length) return linear.domain().map(powb); - linear.domain(x.map(powp)); - return scale; - }; - - scale.ticks = function(m) { - return d3_scale_linearTicks(scale.domain(), m); - }; - - scale.tickFormat = function(m) { - return d3_scale_linearTickFormat(scale.domain(), m); - }; - - scale.nice = function() { - return scale.domain(d3_scale_nice(scale.domain(), d3_scale_linearNice)); - }; - - scale.exponent = function(x) { - if (!arguments.length) return exponent; - var domain = scale.domain(); - powp = d3_scale_powPow(exponent = x); - powb = d3_scale_powPow(1 / exponent); - return scale.domain(domain); - }; - - scale.copy = function() { - return d3_scale_pow(linear.copy(), exponent); - }; - - return d3_scale_linearRebind(scale, linear); -}; - -function d3_scale_powPow(e) { - return function(x) { - return x < 0 ? -Math.pow(-x, e) : Math.pow(x, e); - }; -} -d3.scale.sqrt = function() { - return d3.scale.pow().exponent(.5); -}; -d3.scale.ordinal = function() { - return d3_scale_ordinal([], {t: "range", x: []}); -}; - -function d3_scale_ordinal(domain, ranger) { - var index, - range, - rangeBand; - - function scale(x) { - return range[((index[x] || (index[x] = domain.push(x))) - 1) % range.length]; - } - - scale.domain = function(x) { - if (!arguments.length) return domain; - domain = []; - index = {}; - var i = -1, n = x.length, xi; - while (++i < n) if (!index[xi = x[i]]) index[xi] = domain.push(xi); - return scale[ranger.t](ranger.x, ranger.p); - }; - - scale.range = function(x) { - if (!arguments.length) return range; - range = x; - rangeBand = 0; - ranger = {t: "range", x: x}; - return scale; - }; - - scale.rangePoints = function(x, padding) { - if (arguments.length < 2) padding = 0; - var start = x[0], - stop = x[1], - step = (stop - start) / (domain.length - 1 + padding); - range = domain.length < 2 ? [(start + stop) / 2] : d3.range(start + step * padding / 2, stop + step / 2, step); - rangeBand = 0; - ranger = {t: "rangePoints", x: x, p: padding}; - return scale; - }; - - scale.rangeBands = function(x, padding) { - if (arguments.length < 2) padding = 0; - var start = x[0], - stop = x[1], - step = (stop - start) / (domain.length + padding); - range = d3.range(start + step * padding, stop, step); - rangeBand = step * (1 - padding); - ranger = {t: "rangeBands", x: x, p: padding}; - return scale; - }; - - scale.rangeRoundBands = function(x, padding) { - if (arguments.length < 2) padding = 0; - var start = x[0], - stop = x[1], - step = Math.floor((stop - start) / (domain.length + padding)), - err = stop - start - (domain.length - padding) * step; - range = d3.range(start + Math.round(err / 2), stop, step); - rangeBand = Math.round(step * (1 - padding)); - ranger = {t: "rangeRoundBands", x: x, p: padding}; - return scale; - }; - - scale.rangeBand = function() { - return rangeBand; - }; - - scale.copy = function() { - return d3_scale_ordinal(domain, ranger); - }; - - return scale.domain(domain); -}; -/* - * This product includes color specifications and designs developed by Cynthia - * Brewer (http://colorbrewer.org/). See lib/colorbrewer for more information. - */ - -d3.scale.category10 = function() { - return d3.scale.ordinal().range(d3_category10); -}; - -d3.scale.category20 = function() { - return d3.scale.ordinal().range(d3_category20); -}; - -d3.scale.category20b = function() { - return d3.scale.ordinal().range(d3_category20b); -}; - -d3.scale.category20c = function() { - return d3.scale.ordinal().range(d3_category20c); -}; - -var d3_category10 = [ - "#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", - "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf" -]; - -var d3_category20 = [ - "#1f77b4", "#aec7e8", - "#ff7f0e", "#ffbb78", - "#2ca02c", "#98df8a", - "#d62728", "#ff9896", - "#9467bd", "#c5b0d5", - "#8c564b", "#c49c94", - "#e377c2", "#f7b6d2", - "#7f7f7f", "#c7c7c7", - "#bcbd22", "#dbdb8d", - "#17becf", "#9edae5" -]; - -var d3_category20b = [ - "#393b79", "#5254a3", "#6b6ecf", "#9c9ede", - "#637939", "#8ca252", "#b5cf6b", "#cedb9c", - "#8c6d31", "#bd9e39", "#e7ba52", "#e7cb94", - "#843c39", "#ad494a", "#d6616b", "#e7969c", - "#7b4173", "#a55194", "#ce6dbd", "#de9ed6" -]; - -var d3_category20c = [ - "#3182bd", "#6baed6", "#9ecae1", "#c6dbef", - "#e6550d", "#fd8d3c", "#fdae6b", "#fdd0a2", - "#31a354", "#74c476", "#a1d99b", "#c7e9c0", - "#756bb1", "#9e9ac8", "#bcbddc", "#dadaeb", - "#636363", "#969696", "#bdbdbd", "#d9d9d9" -]; -d3.scale.quantile = function() { - return d3_scale_quantile([], []); -}; - -function d3_scale_quantile(domain, range) { - var thresholds; - - function rescale() { - var k = 0, - n = domain.length, - q = range.length; - thresholds = []; - while (++k < q) thresholds[k - 1] = d3.quantile(domain, k / q); - return scale; - } - - function scale(x) { - if (isNaN(x = +x)) return NaN; - return range[d3.bisect(thresholds, x)]; - } - - scale.domain = function(x) { - if (!arguments.length) return domain; - domain = x.filter(function(d) { return !isNaN(d); }).sort(d3.ascending); - return rescale(); - }; - - scale.range = function(x) { - if (!arguments.length) return range; - range = x; - return rescale(); - }; - - scale.quantiles = function() { - return thresholds; - }; - - scale.copy = function() { - return d3_scale_quantile(domain, range); // copy on write! - }; - - return rescale(); -}; -d3.scale.quantize = function() { - return d3_scale_quantize(0, 1, [0, 1]); -}; - -function d3_scale_quantize(x0, x1, range) { - var kx, i; - - function scale(x) { - return range[Math.max(0, Math.min(i, Math.floor(kx * (x - x0))))]; - } - - function rescale() { - kx = range.length / (x1 - x0); - i = range.length - 1; - return scale; - } - - scale.domain = function(x) { - if (!arguments.length) return [x0, x1]; - x0 = +x[0]; - x1 = +x[x.length - 1]; - return rescale(); - }; - - scale.range = function(x) { - if (!arguments.length) return range; - range = x; - return rescale(); - }; - - scale.copy = function() { - return d3_scale_quantize(x0, x1, range); // copy on write - }; - - return rescale(); -}; -d3.svg = {}; -d3.svg.arc = function() { - var innerRadius = d3_svg_arcInnerRadius, - outerRadius = d3_svg_arcOuterRadius, - startAngle = d3_svg_arcStartAngle, - endAngle = d3_svg_arcEndAngle; - - function arc() { - var r0 = innerRadius.apply(this, arguments), - r1 = outerRadius.apply(this, arguments), - a0 = startAngle.apply(this, arguments) + d3_svg_arcOffset, - a1 = endAngle.apply(this, arguments) + d3_svg_arcOffset, - da = (a1 < a0 && (da = a0, a0 = a1, a1 = da), a1 - a0), - df = da < Math.PI ? "0" : "1", - c0 = Math.cos(a0), - s0 = Math.sin(a0), - c1 = Math.cos(a1), - s1 = Math.sin(a1); - return da >= d3_svg_arcMax - ? (r0 - ? "M0," + r1 - + "A" + r1 + "," + r1 + " 0 1,1 0," + (-r1) - + "A" + r1 + "," + r1 + " 0 1,1 0," + r1 - + "M0," + r0 - + "A" + r0 + "," + r0 + " 0 1,0 0," + (-r0) - + "A" + r0 + "," + r0 + " 0 1,0 0," + r0 - + "Z" - : "M0," + r1 - + "A" + r1 + "," + r1 + " 0 1,1 0," + (-r1) - + "A" + r1 + "," + r1 + " 0 1,1 0," + r1 - + "Z") - : (r0 - ? "M" + r1 * c0 + "," + r1 * s0 - + "A" + r1 + "," + r1 + " 0 " + df + ",1 " + r1 * c1 + "," + r1 * s1 - + "L" + r0 * c1 + "," + r0 * s1 - + "A" + r0 + "," + r0 + " 0 " + df + ",0 " + r0 * c0 + "," + r0 * s0 - + "Z" - : "M" + r1 * c0 + "," + r1 * s0 - + "A" + r1 + "," + r1 + " 0 " + df + ",1 " + r1 * c1 + "," + r1 * s1 - + "L0,0" - + "Z"); - } - - arc.innerRadius = function(v) { - if (!arguments.length) return innerRadius; - innerRadius = d3.functor(v); - return arc; - }; - - arc.outerRadius = function(v) { - if (!arguments.length) return outerRadius; - outerRadius = d3.functor(v); - return arc; - }; - - arc.startAngle = function(v) { - if (!arguments.length) return startAngle; - startAngle = d3.functor(v); - return arc; - }; - - arc.endAngle = function(v) { - if (!arguments.length) return endAngle; - endAngle = d3.functor(v); - return arc; - }; - - arc.centroid = function() { - var r = (innerRadius.apply(this, arguments) - + outerRadius.apply(this, arguments)) / 2, - a = (startAngle.apply(this, arguments) - + endAngle.apply(this, arguments)) / 2 + d3_svg_arcOffset; - return [Math.cos(a) * r, Math.sin(a) * r]; - }; - - return arc; -}; - -var d3_svg_arcOffset = -Math.PI / 2, - d3_svg_arcMax = 2 * Math.PI - 1e-6; - -function d3_svg_arcInnerRadius(d) { - return d.innerRadius; -} - -function d3_svg_arcOuterRadius(d) { - return d.outerRadius; -} - -function d3_svg_arcStartAngle(d) { - return d.startAngle; -} - -function d3_svg_arcEndAngle(d) { - return d.endAngle; -} -function d3_svg_line(projection) { - var x = d3_svg_lineX, - y = d3_svg_lineY, - interpolate = "linear", - interpolator = d3_svg_lineInterpolators[interpolate], - tension = .7; - - function line(d) { - return d.length < 1 ? null : "M" + interpolator(projection(d3_svg_linePoints(this, d, x, y)), tension); - } - - line.x = function(v) { - if (!arguments.length) return x; - x = v; - return line; - }; - - line.y = function(v) { - if (!arguments.length) return y; - y = v; - return line; - }; - - line.interpolate = function(v) { - if (!arguments.length) return interpolate; - interpolator = d3_svg_lineInterpolators[interpolate = v]; - return line; - }; - - line.tension = function(v) { - if (!arguments.length) return tension; - tension = v; - return line; - }; - - return line; -} - -d3.svg.line = function() { - return d3_svg_line(Object); -}; - -// Converts the specified array of data into an array of points -// (x-y tuples), by evaluating the specified `x` and `y` functions on each -// data point. The `this` context of the evaluated functions is the specified -// "self" object; each function is passed the current datum and index. -function d3_svg_linePoints(self, d, x, y) { - var points = [], - i = -1, - n = d.length, - fx = typeof x === "function", - fy = typeof y === "function", - value; - if (fx && fy) { - while (++i < n) points.push([ - x.call(self, value = d[i], i), - y.call(self, value, i) - ]); - } else if (fx) { - while (++i < n) points.push([x.call(self, d[i], i), y]); - } else if (fy) { - while (++i < n) points.push([x, y.call(self, d[i], i)]); - } else { - while (++i < n) points.push([x, y]); - } - return points; -} - -// The default `x` property, which references d[0]. -function d3_svg_lineX(d) { - return d[0]; -} - -// The default `y` property, which references d[1]. -function d3_svg_lineY(d) { - return d[1]; -} - -// The various interpolators supported by the `line` class. -var d3_svg_lineInterpolators = { - "linear": d3_svg_lineLinear, - "step-before": d3_svg_lineStepBefore, - "step-after": d3_svg_lineStepAfter, - "basis": d3_svg_lineBasis, - "basis-open": d3_svg_lineBasisOpen, - "basis-closed": d3_svg_lineBasisClosed, - "bundle": d3_svg_lineBundle, - "cardinal": d3_svg_lineCardinal, - "cardinal-open": d3_svg_lineCardinalOpen, - "cardinal-closed": d3_svg_lineCardinalClosed, - "monotone": d3_svg_lineMonotone -}; - -// Linear interpolation; generates "L" commands. -function d3_svg_lineLinear(points) { - var i = 0, - n = points.length, - p = points[0], - path = [p[0], ",", p[1]]; - while (++i < n) path.push("L", (p = points[i])[0], ",", p[1]); - return path.join(""); -} - -// Step interpolation; generates "H" and "V" commands. -function d3_svg_lineStepBefore(points) { - var i = 0, - n = points.length, - p = points[0], - path = [p[0], ",", p[1]]; - while (++i < n) path.push("V", (p = points[i])[1], "H", p[0]); - return path.join(""); -} - -// Step interpolation; generates "H" and "V" commands. -function d3_svg_lineStepAfter(points) { - var i = 0, - n = points.length, - p = points[0], - path = [p[0], ",", p[1]]; - while (++i < n) path.push("H", (p = points[i])[0], "V", p[1]); - return path.join(""); -} - -// Open cardinal spline interpolation; generates "C" commands. -function d3_svg_lineCardinalOpen(points, tension) { - return points.length < 4 - ? d3_svg_lineLinear(points) - : points[1] + d3_svg_lineHermite(points.slice(1, points.length - 1), - d3_svg_lineCardinalTangents(points, tension)); -} - -// Closed cardinal spline interpolation; generates "C" commands. -function d3_svg_lineCardinalClosed(points, tension) { - return points.length < 3 - ? d3_svg_lineLinear(points) - : points[0] + d3_svg_lineHermite((points.push(points[0]), points), - d3_svg_lineCardinalTangents([points[points.length - 2]] - .concat(points, [points[1]]), tension)); -} - -// Cardinal spline interpolation; generates "C" commands. -function d3_svg_lineCardinal(points, tension, closed) { - return points.length < 3 - ? d3_svg_lineLinear(points) - : points[0] + d3_svg_lineHermite(points, - d3_svg_lineCardinalTangents(points, tension)); -} - -// Hermite spline construction; generates "C" commands. -function d3_svg_lineHermite(points, tangents) { - if (tangents.length < 1 - || (points.length != tangents.length - && points.length != tangents.length + 2)) { - return d3_svg_lineLinear(points); - } - - var quad = points.length != tangents.length, - path = "", - p0 = points[0], - p = points[1], - t0 = tangents[0], - t = t0, - pi = 1; - - if (quad) { - path += "Q" + (p[0] - t0[0] * 2 / 3) + "," + (p[1] - t0[1] * 2 / 3) - + "," + p[0] + "," + p[1]; - p0 = points[1]; - pi = 2; - } - - if (tangents.length > 1) { - t = tangents[1]; - p = points[pi]; - pi++; - path += "C" + (p0[0] + t0[0]) + "," + (p0[1] + t0[1]) - + "," + (p[0] - t[0]) + "," + (p[1] - t[1]) - + "," + p[0] + "," + p[1]; - for (var i = 2; i < tangents.length; i++, pi++) { - p = points[pi]; - t = tangents[i]; - path += "S" + (p[0] - t[0]) + "," + (p[1] - t[1]) - + "," + p[0] + "," + p[1]; - } - } - - if (quad) { - var lp = points[pi]; - path += "Q" + (p[0] + t[0] * 2 / 3) + "," + (p[1] + t[1] * 2 / 3) - + "," + lp[0] + "," + lp[1]; - } - - return path; -} - -// Generates tangents for a cardinal spline. -function d3_svg_lineCardinalTangents(points, tension) { - var tangents = [], - a = (1 - tension) / 2, - p0, - p1 = points[0], - p2 = points[1], - i = 1, - n = points.length; - while (++i < n) { - p0 = p1; - p1 = p2; - p2 = points[i]; - tangents.push([a * (p2[0] - p0[0]), a * (p2[1] - p0[1])]); - } - return tangents; -} - -// B-spline interpolation; generates "C" commands. -function d3_svg_lineBasis(points) { - if (points.length < 3) return d3_svg_lineLinear(points); - var i = 1, - n = points.length, - pi = points[0], - x0 = pi[0], - y0 = pi[1], - px = [x0, x0, x0, (pi = points[1])[0]], - py = [y0, y0, y0, pi[1]], - path = [x0, ",", y0]; - d3_svg_lineBasisBezier(path, px, py); - while (++i < n) { - pi = points[i]; - px.shift(); px.push(pi[0]); - py.shift(); py.push(pi[1]); - d3_svg_lineBasisBezier(path, px, py); - } - i = -1; - while (++i < 2) { - px.shift(); px.push(pi[0]); - py.shift(); py.push(pi[1]); - d3_svg_lineBasisBezier(path, px, py); - } - return path.join(""); -} - -// Open B-spline interpolation; generates "C" commands. -function d3_svg_lineBasisOpen(points) { - if (points.length < 4) return d3_svg_lineLinear(points); - var path = [], - i = -1, - n = points.length, - pi, - px = [0], - py = [0]; - while (++i < 3) { - pi = points[i]; - px.push(pi[0]); - py.push(pi[1]); - } - path.push(d3_svg_lineDot4(d3_svg_lineBasisBezier3, px) - + "," + d3_svg_lineDot4(d3_svg_lineBasisBezier3, py)); - --i; while (++i < n) { - pi = points[i]; - px.shift(); px.push(pi[0]); - py.shift(); py.push(pi[1]); - d3_svg_lineBasisBezier(path, px, py); - } - return path.join(""); -} - -// Closed B-spline interpolation; generates "C" commands. -function d3_svg_lineBasisClosed(points) { - var path, - i = -1, - n = points.length, - m = n + 4, - pi, - px = [], - py = []; - while (++i < 4) { - pi = points[i % n]; - px.push(pi[0]); - py.push(pi[1]); - } - path = [ - d3_svg_lineDot4(d3_svg_lineBasisBezier3, px), ",", - d3_svg_lineDot4(d3_svg_lineBasisBezier3, py) - ]; - --i; while (++i < m) { - pi = points[i % n]; - px.shift(); px.push(pi[0]); - py.shift(); py.push(pi[1]); - d3_svg_lineBasisBezier(path, px, py); - } - return path.join(""); -} - -function d3_svg_lineBundle(points, tension) { - var n = points.length - 1, - x0 = points[0][0], - y0 = points[0][1], - dx = points[n][0] - x0, - dy = points[n][1] - y0, - i = -1, - p, - t; - while (++i <= n) { - p = points[i]; - t = i / n; - p[0] = tension * p[0] + (1 - tension) * (x0 + t * dx); - p[1] = tension * p[1] + (1 - tension) * (y0 + t * dy); - } - return d3_svg_lineBasis(points); -} - -// Returns the dot product of the given four-element vectors. -function d3_svg_lineDot4(a, b) { - return a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3]; -} - -// Matrix to transform basis (b-spline) control points to bezier -// control points. Derived from FvD 11.2.8. -var d3_svg_lineBasisBezier1 = [0, 2/3, 1/3, 0], - d3_svg_lineBasisBezier2 = [0, 1/3, 2/3, 0], - d3_svg_lineBasisBezier3 = [0, 1/6, 2/3, 1/6]; - -// Pushes a "C" Bézier curve onto the specified path array, given the -// two specified four-element arrays which define the control points. -function d3_svg_lineBasisBezier(path, x, y) { - path.push( - "C", d3_svg_lineDot4(d3_svg_lineBasisBezier1, x), - ",", d3_svg_lineDot4(d3_svg_lineBasisBezier1, y), - ",", d3_svg_lineDot4(d3_svg_lineBasisBezier2, x), - ",", d3_svg_lineDot4(d3_svg_lineBasisBezier2, y), - ",", d3_svg_lineDot4(d3_svg_lineBasisBezier3, x), - ",", d3_svg_lineDot4(d3_svg_lineBasisBezier3, y)); -} - -// Computes the slope from points p0 to p1. -function d3_svg_lineSlope(p0, p1) { - return (p1[1] - p0[1]) / (p1[0] - p0[0]); -} - -// Compute three-point differences for the given points. -// http://en.wikipedia.org/wiki/Cubic_Hermite_spline#Finite_difference -function d3_svg_lineFiniteDifferences(points) { - var i = 0, - j = points.length - 1, - m = [], - p0 = points[0], - p1 = points[1], - d = m[0] = d3_svg_lineSlope(p0, p1); - while (++i < j) { - m[i] = d + (d = d3_svg_lineSlope(p0 = p1, p1 = points[i + 1])); - } - m[i] = d; - return m; -} - -// Interpolates the given points using Fritsch-Carlson Monotone cubic Hermite -// interpolation. Returns an array of tangent vectors. For details, see -// http://en.wikipedia.org/wiki/Monotone_cubic_interpolation -function d3_svg_lineMonotoneTangents(points) { - var tangents = [], - d, - a, - b, - s, - m = d3_svg_lineFiniteDifferences(points), - i = -1, - j = points.length - 1; - - // The first two steps are done by computing finite-differences: - // 1. Compute the slopes of the secant lines between successive points. - // 2. Initialize the tangents at every point as the average of the secants. - - // Then, for each segment… - while (++i < j) { - d = d3_svg_lineSlope(points[i], points[i + 1]); - - // 3. If two successive yk = y{k + 1} are equal (i.e., d is zero), then set - // mk = m{k + 1} = 0 as the spline connecting these points must be flat to - // preserve monotonicity. Ignore step 4 and 5 for those k. - - if (Math.abs(d) < 1e-6) { - m[i] = m[i + 1] = 0; - } else { - // 4. Let ak = mk / dk and bk = m{k + 1} / dk. - a = m[i] / d; - b = m[i + 1] / d; - - // 5. Prevent overshoot and ensure monotonicity by restricting the - // magnitude of vector to a circle of radius 3. - s = a * a + b * b; - if (s > 9) { - s = d * 3 / Math.sqrt(s); - m[i] = s * a; - m[i + 1] = s * b; - } - } - } - - // Compute the normalized tangent vector from the slopes. Note that if x is - // not monotonic, it's possible that the slope will be infinite, so we protect - // against NaN by setting the coordinate to zero. - i = -1; while (++i <= j) { - s = (points[Math.min(j, i + 1)][0] - points[Math.max(0, i - 1)][0]) - / (6 * (1 + m[i] * m[i])); - tangents.push([s || 0, m[i] * s || 0]); - } - - return tangents; -} - -function d3_svg_lineMonotone(points) { - return points.length < 3 - ? d3_svg_lineLinear(points) - : points[0] + - d3_svg_lineHermite(points, d3_svg_lineMonotoneTangents(points)); -} -d3.svg.line.radial = function() { - var line = d3_svg_line(d3_svg_lineRadial); - line.radius = line.x, delete line.x; - line.angle = line.y, delete line.y; - return line; -}; - -function d3_svg_lineRadial(points) { - var point, - i = -1, - n = points.length, - r, - a; - while (++i < n) { - point = points[i]; - r = point[0]; - a = point[1] + d3_svg_arcOffset; - point[0] = r * Math.cos(a); - point[1] = r * Math.sin(a); - } - return points; -} -function d3_svg_area(projection) { - var x0 = d3_svg_lineX, - x1 = d3_svg_lineX, - y0 = 0, - y1 = d3_svg_lineY, - interpolate, - i0, - i1, - tension = .7; - - function area(d) { - if (d.length < 1) return null; - var points0 = d3_svg_linePoints(this, d, x0, y0), - points1 = d3_svg_linePoints(this, d, x0 === x1 ? d3_svg_areaX(points0) : x1, y0 === y1 ? d3_svg_areaY(points0) : y1); - return "M" + i0(projection(points1), tension) - + "L" + i1(projection(points0.reverse()), tension) - + "Z"; - } - - area.x = function(x) { - if (!arguments.length) return x1; - x0 = x1 = x; - return area; - }; - - area.x0 = function(x) { - if (!arguments.length) return x0; - x0 = x; - return area; - }; - - area.x1 = function(x) { - if (!arguments.length) return x1; - x1 = x; - return area; - }; - - area.y = function(y) { - if (!arguments.length) return y1; - y0 = y1 = y; - return area; - }; - - area.y0 = function(y) { - if (!arguments.length) return y0; - y0 = y; - return area; - }; - - area.y1 = function(y) { - if (!arguments.length) return y1; - y1 = y; - return area; - }; - - area.interpolate = function(x) { - if (!arguments.length) return interpolate; - i0 = d3_svg_lineInterpolators[interpolate = x]; - i1 = i0.reverse || i0; - return area; - }; - - area.tension = function(x) { - if (!arguments.length) return tension; - tension = x; - return area; - }; - - return area.interpolate("linear"); -} - -d3_svg_lineStepBefore.reverse = d3_svg_lineStepAfter; -d3_svg_lineStepAfter.reverse = d3_svg_lineStepBefore; - -d3.svg.area = function() { - return d3_svg_area(Object); -}; - -function d3_svg_areaX(points) { - return function(d, i) { - return points[i][0]; - }; -} - -function d3_svg_areaY(points) { - return function(d, i) { - return points[i][1]; - }; -} -d3.svg.area.radial = function() { - var area = d3_svg_area(d3_svg_lineRadial); - area.radius = area.x, delete area.x; - area.innerRadius = area.x0, delete area.x0; - area.outerRadius = area.x1, delete area.x1; - area.angle = area.y, delete area.y; - area.startAngle = area.y0, delete area.y0; - area.endAngle = area.y1, delete area.y1; - return area; -}; -d3.svg.chord = function() { - var source = d3_svg_chordSource, - target = d3_svg_chordTarget, - radius = d3_svg_chordRadius, - startAngle = d3_svg_arcStartAngle, - endAngle = d3_svg_arcEndAngle; - - // TODO Allow control point to be customized. - - function chord(d, i) { - var s = subgroup(this, source, d, i), - t = subgroup(this, target, d, i); - return "M" + s.p0 - + arc(s.r, s.p1) + (equals(s, t) - ? curve(s.r, s.p1, s.r, s.p0) - : curve(s.r, s.p1, t.r, t.p0) - + arc(t.r, t.p1) - + curve(t.r, t.p1, s.r, s.p0)) - + "Z"; - } - - function subgroup(self, f, d, i) { - var subgroup = f.call(self, d, i), - r = radius.call(self, subgroup, i), - a0 = startAngle.call(self, subgroup, i) + d3_svg_arcOffset, - a1 = endAngle.call(self, subgroup, i) + d3_svg_arcOffset; - return { - r: r, - a0: a0, - a1: a1, - p0: [r * Math.cos(a0), r * Math.sin(a0)], - p1: [r * Math.cos(a1), r * Math.sin(a1)] - }; - } - - function equals(a, b) { - return a.a0 == b.a0 && a.a1 == b.a1; - } - - function arc(r, p) { - return "A" + r + "," + r + " 0 0,1 " + p; - } - - function curve(r0, p0, r1, p1) { - return "Q 0,0 " + p1; - } - - chord.radius = function(v) { - if (!arguments.length) return radius; - radius = d3.functor(v); - return chord; - }; - - chord.source = function(v) { - if (!arguments.length) return source; - source = d3.functor(v); - return chord; - }; - - chord.target = function(v) { - if (!arguments.length) return target; - target = d3.functor(v); - return chord; - }; - - chord.startAngle = function(v) { - if (!arguments.length) return startAngle; - startAngle = d3.functor(v); - return chord; - }; - - chord.endAngle = function(v) { - if (!arguments.length) return endAngle; - endAngle = d3.functor(v); - return chord; - }; - - return chord; -}; - -function d3_svg_chordSource(d) { - return d.source; -} - -function d3_svg_chordTarget(d) { - return d.target; -} - -function d3_svg_chordRadius(d) { - return d.radius; -} - -function d3_svg_chordStartAngle(d) { - return d.startAngle; -} - -function d3_svg_chordEndAngle(d) { - return d.endAngle; -} -d3.svg.diagonal = function() { - var source = d3_svg_chordSource, - target = d3_svg_chordTarget, - projection = d3_svg_diagonalProjection; - - function diagonal(d, i) { - var p0 = source.call(this, d, i), - p3 = target.call(this, d, i), - m = (p0.y + p3.y) / 2, - p = [p0, {x: p0.x, y: m}, {x: p3.x, y: m}, p3]; - p = p.map(projection); - return "M" + p[0] + "C" + p[1] + " " + p[2] + " " + p[3]; - } - - diagonal.source = function(x) { - if (!arguments.length) return source; - source = d3.functor(x); - return diagonal; - }; - - diagonal.target = function(x) { - if (!arguments.length) return target; - target = d3.functor(x); - return diagonal; - }; - - diagonal.projection = function(x) { - if (!arguments.length) return projection; - projection = x; - return diagonal; - }; - - return diagonal; -}; - -function d3_svg_diagonalProjection(d) { - return [d.x, d.y]; -} -d3.svg.diagonal.radial = function() { - var diagonal = d3.svg.diagonal(), - projection = d3_svg_diagonalProjection, - projection_ = diagonal.projection; - - diagonal.projection = function(x) { - return arguments.length - ? projection_(d3_svg_diagonalRadialProjection(projection = x)) - : projection; - }; - - return diagonal; -}; - -function d3_svg_diagonalRadialProjection(projection) { - return function() { - var d = projection.apply(this, arguments), - r = d[0], - a = d[1] + d3_svg_arcOffset; - return [r * Math.cos(a), r * Math.sin(a)]; - }; -} -d3.svg.mouse = function(container) { - return d3_svg_mousePoint(container, d3.event); -}; - -// https://bugs.webkit.org/show_bug.cgi?id=44083 -var d3_mouse_bug44083 = /WebKit/.test(navigator.userAgent) ? -1 : 0; - -function d3_svg_mousePoint(container, e) { - var point = (container.ownerSVGElement || container).createSVGPoint(); - if ((d3_mouse_bug44083 < 0) && (window.scrollX || window.scrollY)) { - var svg = d3.select(document.body) - .append("svg:svg") - .style("position", "absolute") - .style("top", 0) - .style("left", 0); - var ctm = svg[0][0].getScreenCTM(); - d3_mouse_bug44083 = !(ctm.f || ctm.e); - svg.remove(); - } - if (d3_mouse_bug44083) { - point.x = e.pageX; - point.y = e.pageY; - } else { - point.x = e.clientX; - point.y = e.clientY; - } - point = point.matrixTransform(container.getScreenCTM().inverse()); - return [point.x, point.y]; -}; -d3.svg.touches = function(container) { - var touches = d3.event.touches; - return touches ? d3_array(touches).map(function(touch) { - var point = d3_svg_mousePoint(container, touch); - point.identifier = touch.identifier; - return point; - }) : []; -}; -d3.svg.symbol = function() { - var type = d3_svg_symbolType, - size = d3_svg_symbolSize; - - function symbol(d, i) { - return (d3_svg_symbols[type.call(this, d, i)] - || d3_svg_symbols.circle) - (size.call(this, d, i)); - } - - symbol.type = function(x) { - if (!arguments.length) return type; - type = d3.functor(x); - return symbol; - }; - - // size of symbol in square pixels - symbol.size = function(x) { - if (!arguments.length) return size; - size = d3.functor(x); - return symbol; - }; - - return symbol; -}; - -function d3_svg_symbolSize() { - return 64; -} - -function d3_svg_symbolType() { - return "circle"; -} - -// TODO cross-diagonal? -var d3_svg_symbols = { - "circle": function(size) { - var r = Math.sqrt(size / Math.PI); - return "M0," + r - + "A" + r + "," + r + " 0 1,1 0," + (-r) - + "A" + r + "," + r + " 0 1,1 0," + r - + "Z"; - }, - "cross": function(size) { - var r = Math.sqrt(size / 5) / 2; - return "M" + -3 * r + "," + -r - + "H" + -r - + "V" + -3 * r - + "H" + r - + "V" + -r - + "H" + 3 * r - + "V" + r - + "H" + r - + "V" + 3 * r - + "H" + -r - + "V" + r - + "H" + -3 * r - + "Z"; - }, - "diamond": function(size) { - var ry = Math.sqrt(size / (2 * d3_svg_symbolTan30)), - rx = ry * d3_svg_symbolTan30; - return "M0," + -ry - + "L" + rx + ",0" - + " 0," + ry - + " " + -rx + ",0" - + "Z"; - }, - "square": function(size) { - var r = Math.sqrt(size) / 2; - return "M" + -r + "," + -r - + "L" + r + "," + -r - + " " + r + "," + r - + " " + -r + "," + r - + "Z"; - }, - "triangle-down": function(size) { - var rx = Math.sqrt(size / d3_svg_symbolSqrt3), - ry = rx * d3_svg_symbolSqrt3 / 2; - return "M0," + ry - + "L" + rx +"," + -ry - + " " + -rx + "," + -ry - + "Z"; - }, - "triangle-up": function(size) { - var rx = Math.sqrt(size / d3_svg_symbolSqrt3), - ry = rx * d3_svg_symbolSqrt3 / 2; - return "M0," + -ry - + "L" + rx +"," + ry - + " " + -rx + "," + ry - + "Z"; - } -}; - -d3.svg.symbolTypes = d3.keys(d3_svg_symbols); - -var d3_svg_symbolSqrt3 = Math.sqrt(3), - d3_svg_symbolTan30 = Math.tan(30 * Math.PI / 180); -d3.svg.axis = function() { - var scale = d3.scale.linear(), - orient = "bottom", - tickMajorSize = 6, - tickMinorSize = 6, - tickEndSize = 6, - tickPadding = 3, - tickArguments_ = [10], - tickFormat_, - tickSubdivide = 0; - - function axis(selection) { - selection.each(function(d, i, j) { - var g = d3.select(this); - - // If selection is a transition, create subtransitions. - var transition = selection.delay ? function(o) { - var id = d3_transitionInheritId; - try { - d3_transitionInheritId = selection.id; - return o.transition() - .delay(selection[j][i].delay) - .duration(selection[j][i].duration) - .ease(selection.ease()); - } finally { - d3_transitionInheritId = id; - } - } : Object; - - // Ticks. - var ticks = scale.ticks.apply(scale, tickArguments_), - tickFormat = tickFormat_ == null ? scale.tickFormat.apply(scale, tickArguments_) : tickFormat_; - - // Minor ticks. - var subticks = d3_svg_axisSubdivide(scale, ticks, tickSubdivide), - subtick = g.selectAll(".minor").data(subticks, String), - subtickEnter = subtick.enter().insert("svg:line", "g").attr("class", "tick minor").style("opacity", 1e-6), - subtickExit = transition(subtick.exit()).style("opacity", 1e-6).remove(), - subtickUpdate = transition(subtick).style("opacity", 1); - - // Major ticks. - var tick = g.selectAll("g").data(ticks, String), - tickEnter = tick.enter().insert("svg:g", "path").style("opacity", 1e-6), - tickExit = transition(tick.exit()).style("opacity", 1e-6).remove(), - tickUpdate = transition(tick).style("opacity", 1), - tickTransform; - - // Domain. - var range = d3_scaleExtent(scale.range()), - path = g.selectAll(".domain").data([0]), - pathEnter = path.enter().append("svg:path").attr("class", "domain"), - pathUpdate = transition(path); - - // Stash the new scale and grab the old scale. - var scale0 = this.__chart__ || scale; - this.__chart__ = scale.copy(); - - tickEnter.append("svg:line").attr("class", "tick"); - tickEnter.append("svg:text"); - tickUpdate.select("text").text(tickFormat); - - switch (orient) { - case "bottom": { - tickTransform = d3_svg_axisX; - subtickUpdate.attr("x2", 0).attr("y2", tickMinorSize); - tickUpdate.select("line").attr("x2", 0).attr("y2", tickMajorSize); - tickUpdate.select("text").attr("x", 0).attr("y", Math.max(tickMajorSize, 0) + tickPadding).attr("dy", ".71em").attr("text-anchor", "middle"); - pathUpdate.attr("d", "M" + range[0] + "," + tickEndSize + "V0H" + range[1] + "V" + tickEndSize); - break; - } - case "top": { - tickTransform = d3_svg_axisX; - subtickUpdate.attr("x2", 0).attr("y2", -tickMinorSize); - tickUpdate.select("line").attr("x2", 0).attr("y2", -tickMajorSize); - tickUpdate.select("text").attr("x", 0).attr("y", -(Math.max(tickMajorSize, 0) + tickPadding)).attr("dy", "0em").attr("text-anchor", "middle"); - pathUpdate.attr("d", "M" + range[0] + "," + -tickEndSize + "V0H" + range[1] + "V" + -tickEndSize); - break; - } - case "left": { - tickTransform = d3_svg_axisY; - subtickUpdate.attr("x2", -tickMinorSize).attr("y2", 0); - tickUpdate.select("line").attr("x2", -tickMajorSize).attr("y2", 0); - tickUpdate.select("text").attr("x", -(Math.max(tickMajorSize, 0) + tickPadding)).attr("y", 0).attr("dy", ".32em").attr("text-anchor", "end"); - pathUpdate.attr("d", "M" + -tickEndSize + "," + range[0] + "H0V" + range[1] + "H" + -tickEndSize); - break; - } - case "right": { - tickTransform = d3_svg_axisY; - subtickUpdate.attr("x2", tickMinorSize).attr("y2", 0); - tickUpdate.select("line").attr("x2", tickMajorSize).attr("y2", 0); - tickUpdate.select("text").attr("x", Math.max(tickMajorSize, 0) + tickPadding).attr("y", 0).attr("dy", ".32em").attr("text-anchor", "start"); - pathUpdate.attr("d", "M" + tickEndSize + "," + range[0] + "H0V" + range[1] + "H" + tickEndSize); - break; - } - } - - tickEnter.call(tickTransform, scale0); - tickUpdate.call(tickTransform, scale); - tickExit.call(tickTransform, scale); - - subtickEnter.call(tickTransform, scale0); - subtickUpdate.call(tickTransform, scale); - subtickExit.call(tickTransform, scale); - }); - } - - axis.scale = function(x) { - if (!arguments.length) return scale; - scale = x; - return axis; - }; - - axis.orient = function(x) { - if (!arguments.length) return orient; - orient = x; - return axis; - }; - - axis.ticks = function() { - if (!arguments.length) return tickArguments_; - tickArguments_ = arguments; - return axis; - }; - - axis.tickFormat = function(x) { - if (!arguments.length) return tickFormat_; - tickFormat_ = x; - return axis; - }; - - axis.tickSize = function(x, y, z) { - if (!arguments.length) return tickMajorSize; - var n = arguments.length - 1; - tickMajorSize = +x; - tickMinorSize = n > 1 ? +y : tickMajorSize; - tickEndSize = n > 0 ? +arguments[n] : tickMajorSize; - return axis; - }; - - axis.tickPadding = function(x) { - if (!arguments.length) return tickPadding; - tickPadding = +x; - return axis; - }; - - axis.tickSubdivide = function(x) { - if (!arguments.length) return tickSubdivide; - tickSubdivide = +x; - return axis; - }; - - return axis; -}; - -function d3_svg_axisX(selection, x) { - selection.attr("transform", function(d) { return "translate(" + x(d) + ",0)"; }); -} - -function d3_svg_axisY(selection, y) { - selection.attr("transform", function(d) { return "translate(0," + y(d) + ")"; }); -} - -function d3_svg_axisSubdivide(scale, ticks, m) { - subticks = []; - if (m && ticks.length > 1) { - var extent = d3_scaleExtent(scale.domain()), - subticks, - i = -1, - n = ticks.length, - d = (ticks[1] - ticks[0]) / ++m, - j, - v; - while (++i < n) { - for (j = m; --j > 0;) { - if ((v = +ticks[i] - j * d) >= extent[0]) { - subticks.push(v); - } - } - } - for (--i, j = 0; ++j < m && (v = +ticks[i] + j * d) < extent[1];) { - subticks.push(v); - } - } - return subticks; -} -d3.behavior = {}; -d3.behavior.drag = function() { - var event = d3.dispatch("drag", "dragstart", "dragend"); - - function drag() { - this - .on("mousedown.drag", mousedown) - .on("touchstart.drag", mousedown); - - d3.select(window) - .on("mousemove.drag", d3_behavior_dragMove) - .on("touchmove.drag", d3_behavior_dragMove) - .on("mouseup.drag", d3_behavior_dragUp, true) - .on("touchend.drag", d3_behavior_dragUp, true) - .on("click.drag", d3_behavior_dragClick, true); - } - - // snapshot the local context for subsequent dispatch - function start() { - d3_behavior_dragEvent = event; - d3_behavior_dragEventTarget = d3.event.target; - d3_behavior_dragOffset = d3_behavior_dragPoint((d3_behavior_dragTarget = this).parentNode); - d3_behavior_dragMoved = 0; - d3_behavior_dragArguments = arguments; - } - - function mousedown() { - start.apply(this, arguments); - d3_behavior_dragDispatch("dragstart"); - } - - drag.on = function(type, listener) { - event[type].add(listener); - return drag; - }; - - return drag; -}; - -var d3_behavior_dragEvent, - d3_behavior_dragEventTarget, - d3_behavior_dragTarget, - d3_behavior_dragArguments, - d3_behavior_dragOffset, - d3_behavior_dragMoved, - d3_behavior_dragStopClick; - -function d3_behavior_dragDispatch(type) { - var o = d3.event, p = d3_behavior_dragTarget.parentNode, dx = 0, dy = 0; - - if (p) { - p = d3_behavior_dragPoint(p); - dx = p[0] - d3_behavior_dragOffset[0]; - dy = p[1] - d3_behavior_dragOffset[1]; - d3_behavior_dragOffset = p; - d3_behavior_dragMoved |= dx | dy; - } - - try { - d3.event = {dx: dx, dy: dy}; - d3_behavior_dragEvent[type].dispatch.apply(d3_behavior_dragTarget, d3_behavior_dragArguments); - } finally { - d3.event = o; - } - - o.preventDefault(); -} - -function d3_behavior_dragPoint(container) { - return d3.event.touches - ? d3.svg.touches(container)[0] - : d3.svg.mouse(container); -} - -function d3_behavior_dragMove() { - if (!d3_behavior_dragTarget) return; - var parent = d3_behavior_dragTarget.parentNode; - - // O NOES! The drag element was removed from the DOM. - if (!parent) return d3_behavior_dragUp(); - - d3_behavior_dragDispatch("drag"); - d3_behavior_dragCancel(); -} - -function d3_behavior_dragUp() { - if (!d3_behavior_dragTarget) return; - d3_behavior_dragDispatch("dragend"); - d3_behavior_dragTarget = null; - - // If the node was moved, prevent the mouseup from propagating. - // Also prevent the subsequent click from propagating (e.g., for anchors). - if (d3_behavior_dragMoved && d3_behavior_dragEventTarget === d3.event.target) { - d3_behavior_dragStopClick = true; - d3_behavior_dragCancel(); - } -} - -function d3_behavior_dragClick() { - if (d3_behavior_dragStopClick && d3_behavior_dragEventTarget === d3.event.target) { - d3_behavior_dragCancel(); - d3_behavior_dragStopClick = false; - d3_behavior_dragEventTarget = null; - } -} - -function d3_behavior_dragCancel() { - d3.event.stopPropagation(); - d3.event.preventDefault(); -} -// TODO unbind zoom behavior? -// TODO unbind listener? -d3.behavior.zoom = function() { - var xyz = [0, 0, 0], - event = d3.dispatch("zoom"); - - function zoom() { - this - .on("mousedown.zoom", mousedown) - .on("mousewheel.zoom", mousewheel) - .on("DOMMouseScroll.zoom", mousewheel) - .on("dblclick.zoom", dblclick) - .on("touchstart.zoom", touchstart); - - d3.select(window) - .on("mousemove.zoom", d3_behavior_zoomMousemove) - .on("mouseup.zoom", d3_behavior_zoomMouseup) - .on("touchmove.zoom", d3_behavior_zoomTouchmove) - .on("touchend.zoom", d3_behavior_zoomTouchup) - .on("click.zoom", d3_behavior_zoomClick, true); - } - - // snapshot the local context for subsequent dispatch - function start() { - d3_behavior_zoomXyz = xyz; - d3_behavior_zoomDispatch = event.zoom.dispatch; - d3_behavior_zoomEventTarget = d3.event.target; - d3_behavior_zoomTarget = this; - d3_behavior_zoomArguments = arguments; - } - - function mousedown() { - start.apply(this, arguments); - d3_behavior_zoomPanning = d3_behavior_zoomLocation(d3.svg.mouse(d3_behavior_zoomTarget)); - d3_behavior_zoomMoved = false; - d3.event.preventDefault(); - window.focus(); - } - - // store starting mouse location - function mousewheel() { - start.apply(this, arguments); - if (!d3_behavior_zoomZooming) d3_behavior_zoomZooming = d3_behavior_zoomLocation(d3.svg.mouse(d3_behavior_zoomTarget)); - d3_behavior_zoomTo(d3_behavior_zoomDelta() + xyz[2], d3.svg.mouse(d3_behavior_zoomTarget), d3_behavior_zoomZooming); - } - - function dblclick() { - start.apply(this, arguments); - var mouse = d3.svg.mouse(d3_behavior_zoomTarget); - d3_behavior_zoomTo(d3.event.shiftKey ? Math.ceil(xyz[2] - 1) : Math.floor(xyz[2] + 1), mouse, d3_behavior_zoomLocation(mouse)); - } - - // doubletap detection - function touchstart() { - start.apply(this, arguments); - var touches = d3_behavior_zoomTouchup(), - touch, - now = Date.now(); - if ((touches.length === 1) && (now - d3_behavior_zoomLast < 300)) { - d3_behavior_zoomTo(1 + Math.floor(xyz[2]), touch = touches[0], d3_behavior_zoomLocations[touch.identifier]); - } - d3_behavior_zoomLast = now; - } - - zoom.on = function(type, listener) { - event[type].add(listener); - return zoom; - }; - - return zoom; -}; - -var d3_behavior_zoomDiv, - d3_behavior_zoomPanning, - d3_behavior_zoomZooming, - d3_behavior_zoomLocations = {}, // identifier -> location - d3_behavior_zoomLast = 0, - d3_behavior_zoomXyz, - d3_behavior_zoomDispatch, - d3_behavior_zoomEventTarget, - d3_behavior_zoomTarget, - d3_behavior_zoomArguments, - d3_behavior_zoomMoved, - d3_behavior_zoomStopClick; - -function d3_behavior_zoomLocation(point) { - return [ - point[0] - d3_behavior_zoomXyz[0], - point[1] - d3_behavior_zoomXyz[1], - d3_behavior_zoomXyz[2] - ]; -} - -// detect the pixels that would be scrolled by this wheel event -function d3_behavior_zoomDelta() { - - // mousewheel events are totally broken! - // https://bugs.webkit.org/show_bug.cgi?id=40441 - // not only that, but Chrome and Safari differ in re. to acceleration! - if (!d3_behavior_zoomDiv) { - d3_behavior_zoomDiv = d3.select("body").append("div") - .style("visibility", "hidden") - .style("top", 0) - .style("height", 0) - .style("width", 0) - .style("overflow-y", "scroll") - .append("div") - .style("height", "2000px") - .node().parentNode; - } - - var e = d3.event, delta; - try { - d3_behavior_zoomDiv.scrollTop = 1000; - d3_behavior_zoomDiv.dispatchEvent(e); - delta = 1000 - d3_behavior_zoomDiv.scrollTop; - } catch (error) { - delta = e.wheelDelta || (-e.detail * 5); - } - - return delta * .005; -} - -// Note: Since we don't rotate, it's possible for the touches to become -// slightly detached from their original positions. Thus, we recompute the -// touch points on touchend as well as touchstart! -function d3_behavior_zoomTouchup() { - var touches = d3.svg.touches(d3_behavior_zoomTarget), - i = -1, - n = touches.length, - touch; - while (++i < n) d3_behavior_zoomLocations[(touch = touches[i]).identifier] = d3_behavior_zoomLocation(touch); - return touches; -} - -function d3_behavior_zoomTouchmove() { - var touches = d3.svg.touches(d3_behavior_zoomTarget); - switch (touches.length) { - - // single-touch pan - case 1: { - var touch = touches[0]; - d3_behavior_zoomTo(d3_behavior_zoomXyz[2], touch, d3_behavior_zoomLocations[touch.identifier]); - break; - } - - // double-touch pan + zoom - case 2: { - var p0 = touches[0], - p1 = touches[1], - p2 = [(p0[0] + p1[0]) / 2, (p0[1] + p1[1]) / 2], - l0 = d3_behavior_zoomLocations[p0.identifier], - l1 = d3_behavior_zoomLocations[p1.identifier], - l2 = [(l0[0] + l1[0]) / 2, (l0[1] + l1[1]) / 2, l0[2]]; - d3_behavior_zoomTo(Math.log(d3.event.scale) / Math.LN2 + l0[2], p2, l2); - break; - } - } -} - -function d3_behavior_zoomMousemove() { - d3_behavior_zoomZooming = null; - if (d3_behavior_zoomPanning) { - d3_behavior_zoomMoved = true; - d3_behavior_zoomTo(d3_behavior_zoomXyz[2], d3.svg.mouse(d3_behavior_zoomTarget), d3_behavior_zoomPanning); - } -} - -function d3_behavior_zoomMouseup() { - if (d3_behavior_zoomPanning) { - if (d3_behavior_zoomMoved && d3_behavior_zoomEventTarget === d3.event.target) { - d3_behavior_zoomStopClick = true; - } - d3_behavior_zoomMousemove(); - d3_behavior_zoomPanning = null; - } -} - -function d3_behavior_zoomClick() { - if (d3_behavior_zoomStopClick && d3_behavior_zoomEventTarget === d3.event.target) { - d3.event.stopPropagation(); - d3.event.preventDefault(); - d3_behavior_zoomStopClick = false; - d3_behavior_zoomEventTarget = null; - } -} - -function d3_behavior_zoomTo(z, x0, x1) { - var K = Math.pow(2, (d3_behavior_zoomXyz[2] = z) - x1[2]), - x = d3_behavior_zoomXyz[0] = x0[0] - K * x1[0], - y = d3_behavior_zoomXyz[1] = x0[1] - K * x1[1], - o = d3.event, // Events can be reentrant (e.g., focus). - k = Math.pow(2, z); - - d3.event = { - scale: k, - translate: [x, y], - transform: function(sx, sy) { - if (sx) transform(sx, x); - if (sy) transform(sy, y); - } - }; - - function transform(scale, o) { - var domain = scale.__domain || (scale.__domain = scale.domain()), - range = scale.range().map(function(v) { return (v - o) / k; }); - scale.domain(domain).domain(range.map(scale.invert)); - } - - try { - d3_behavior_zoomDispatch.apply(d3_behavior_zoomTarget, d3_behavior_zoomArguments); - } finally { - d3.event = o; - } - - o.preventDefault(); -} -})(); diff --git a/src/allmydata/web/static/d3-2.4.6.time.js b/src/allmydata/web/static/d3-2.4.6.time.js deleted file mode 100644 index e1c0831c8..000000000 --- a/src/allmydata/web/static/d3-2.4.6.time.js +++ /dev/null @@ -1,692 +0,0 @@ -(function(){d3.time = {}; - -var d3_time = Date; -d3.time.format = function(template) { - var n = template.length; - - function format(date) { - var string = [], - i = -1, - j = 0, - c, - f; - while (++i < n) { - if (template.charCodeAt(i) == 37) { - string.push( - template.substring(j, i), - (f = d3_time_formats[c = template.charAt(++i)]) - ? f(date) : c); - j = i + 1; - } - } - string.push(template.substring(j, i)); - return string.join(""); - } - - format.parse = function(string) { - var date = new d3_time(1900, 0, 1), - i = d3_time_parse(date, template, string, 0); - if (i != string.length) return null; - if (date.hour12) { - var hours = date.getHours() % 12; - date.setHours(date.hour12pm ? hours + 12 : hours); - } - delete date.hour12; - delete date.hour12pm; - return date; - }; - - format.toString = function() { - return template; - }; - - return format; -}; - -function d3_time_parse(date, template, string, j) { - var c, - p, - i = 0, - n = template.length, - m = string.length; - while (i < n) { - if (j >= m) return -1; - c = template.charCodeAt(i++); - if (c == 37) { - p = d3_time_parsers[template.charAt(i++)]; - if (!p || ((j = p(date, string, j)) < 0)) return -1; - } else if (c != string.charCodeAt(j++)) { - return -1; - } - } - return j; -} - -var d3_time_zfill2 = d3.format("02d"), - d3_time_zfill3 = d3.format("03d"), - d3_time_zfill4 = d3.format("04d"), - d3_time_sfill2 = d3.format("2d"); - -var d3_time_formats = { - a: function(d) { return d3_time_weekdays[d.getDay()].substring(0, 3); }, - A: function(d) { return d3_time_weekdays[d.getDay()]; }, - b: function(d) { return d3_time_months[d.getMonth()].substring(0, 3); }, - B: function(d) { return d3_time_months[d.getMonth()]; }, - c: d3.time.format("%a %b %e %H:%M:%S %Y"), - d: function(d) { return d3_time_zfill2(d.getDate()); }, - e: function(d) { return d3_time_sfill2(d.getDate()); }, - H: function(d) { return d3_time_zfill2(d.getHours()); }, - I: function(d) { return d3_time_zfill2(d.getHours() % 12 || 12); }, - j: d3_time_dayOfYear, - L: function(d) { return d3_time_zfill3(d.getMilliseconds()); }, - m: function(d) { return d3_time_zfill2(d.getMonth() + 1); }, - M: function(d) { return d3_time_zfill2(d.getMinutes()); }, - p: function(d) { return d.getHours() >= 12 ? "PM" : "AM"; }, - S: function(d) { return d3_time_zfill2(d.getSeconds()); }, - U: d3_time_weekNumberSunday, - w: function(d) { return d.getDay(); }, - W: d3_time_weekNumberMonday, - x: d3.time.format("%m/%d/%y"), - X: d3.time.format("%H:%M:%S"), - y: function(d) { return d3_time_zfill2(d.getFullYear() % 100); }, - Y: function(d) { return d3_time_zfill4(d.getFullYear() % 10000); }, - Z: d3_time_zone, - "%": function(d) { return "%"; } -}; - -var d3_time_parsers = { - a: d3_time_parseWeekdayAbbrev, - A: d3_time_parseWeekday, - b: d3_time_parseMonthAbbrev, - B: d3_time_parseMonth, - c: d3_time_parseLocaleFull, - d: d3_time_parseDay, - e: d3_time_parseDay, - H: d3_time_parseHour24, - I: d3_time_parseHour12, - // j: function(d, s, i) { /*TODO day of year [001,366] */ return i; }, - L: d3_time_parseMilliseconds, - m: d3_time_parseMonthNumber, - M: d3_time_parseMinutes, - p: d3_time_parseAmPm, - S: d3_time_parseSeconds, - // U: function(d, s, i) { /*TODO week number (sunday) [00,53] */ return i; }, - // w: function(d, s, i) { /*TODO weekday [0,6] */ return i; }, - // W: function(d, s, i) { /*TODO week number (monday) [00,53] */ return i; }, - x: d3_time_parseLocaleDate, - X: d3_time_parseLocaleTime, - y: d3_time_parseYear, - Y: d3_time_parseFullYear - // , - // Z: function(d, s, i) { /*TODO time zone */ return i; }, - // "%": function(d, s, i) { /*TODO literal % */ return i; } -}; - -// Note: weekday is validated, but does not set the date. -function d3_time_parseWeekdayAbbrev(date, string, i) { - return string.substring(i, i += 3).toLowerCase() in d3_time_weekdayAbbrevLookup ? i : -1; -} - -var d3_time_weekdayAbbrevLookup = { - sun: 3, - mon: 3, - tue: 3, - wed: 3, - thu: 3, - fri: 3, - sat: 3 -}; - -// Note: weekday is validated, but does not set the date. -function d3_time_parseWeekday(date, string, i) { - d3_time_weekdayRe.lastIndex = 0; - var n = d3_time_weekdayRe.exec(string.substring(i, i + 10)); - return n ? i += n[0].length : -1; -} - -var d3_time_weekdayRe = /^(?:Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday)/ig; - -var d3_time_weekdays = [ - "Sunday", - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday", - "Saturday" -]; - -function d3_time_parseMonthAbbrev(date, string, i) { - var n = d3_time_monthAbbrevLookup[string.substring(i, i += 3).toLowerCase()]; - return n == null ? -1 : (date.setMonth(n), i); -} - -var d3_time_monthAbbrevLookup = { - jan: 0, - feb: 1, - mar: 2, - apr: 3, - may: 4, - jun: 5, - jul: 6, - aug: 7, - sep: 8, - oct: 9, - nov: 10, - dec: 11 -}; - -function d3_time_parseMonth(date, string, i) { - d3_time_monthRe.lastIndex = 0; - var n = d3_time_monthRe.exec(string.substring(i, i + 12)); - return n ? (date.setMonth(d3_time_monthLookup[n[0].toLowerCase()]), i += n[0].length) : -1; -} - -var d3_time_monthRe = /^(?:January|February|March|April|May|June|July|August|September|October|November|December)/ig; - -var d3_time_monthLookup = { - january: 0, - february: 1, - march: 2, - april: 3, - may: 4, - june: 5, - july: 6, - august: 7, - september: 8, - october: 9, - november: 10, - december: 11 -}; - -var d3_time_months = [ - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December" -]; - -function d3_time_parseLocaleFull(date, string, i) { - return d3_time_parse(date, d3_time_formats.c.toString(), string, i); -} - -function d3_time_parseLocaleDate(date, string, i) { - return d3_time_parse(date, d3_time_formats.x.toString(), string, i); -} - -function d3_time_parseLocaleTime(date, string, i) { - return d3_time_parse(date, d3_time_formats.X.toString(), string, i); -} - -function d3_time_parseFullYear(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 4)); - return n ? (date.setFullYear(n[0]), i += n[0].length) : -1; -} - -function d3_time_parseYear(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 2)); - return n ? (date.setFullYear(d3_time_century() + +n[0]), i += n[0].length) : -1; -} - -function d3_time_century() { - return ~~(new Date().getFullYear() / 1000) * 1000; -} - -function d3_time_parseMonthNumber(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 2)); - return n ? (date.setMonth(n[0] - 1), i += n[0].length) : -1; -} - -function d3_time_parseDay(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 2)); - return n ? (date.setDate(+n[0]), i += n[0].length) : -1; -} - -// Note: we don't validate that the hour is in the range [0,23]. -function d3_time_parseHour24(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 2)); - return n ? (date.setHours(+n[0]), i += n[0].length) : -1; -} - -// Note: we don't validate that the hour is in the range [1,12]. -function d3_time_parseHour12(date, string, i) { - date.hour12 = true; - return d3_time_parseHour24(date, string, i); -} - -function d3_time_parseMinutes(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 2)); - return n ? (date.setMinutes(+n[0]), i += n[0].length) : -1; -} - -function d3_time_parseSeconds(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 2)); - return n ? (date.setSeconds(+n[0]), i += n[0].length) : -1; -} - -function d3_time_parseMilliseconds(date, string, i) { - d3_time_numberRe.lastIndex = 0; - var n = d3_time_numberRe.exec(string.substring(i, i + 3)); - return n ? (date.setMilliseconds(+n[0]), i += n[0].length) : -1; -} - -// Note: we don't look at the next directive. -var d3_time_numberRe = /\s*\d+/; - -function d3_time_parseAmPm(date, string, i) { - var n = d3_time_amPmLookup[string.substring(i, i += 2).toLowerCase()]; - return n == null ? -1 : (date.hour12pm = n, i); -} - -var d3_time_amPmLookup = { - am: 0, - pm: 1 -}; - -function d3_time_year(d) { - return new d3_time(d.getFullYear(), 0, 1); -} - -function d3_time_daysElapsed(d0, d1) { - return ~~((d1 - d0) / 864e5 - (d1.getTimezoneOffset() - d0.getTimezoneOffset()) / 1440); -} - -function d3_time_dayOfYear(d) { - return d3_time_zfill3(1 + d3_time_daysElapsed(d3_time_year(d), d)); -} - -function d3_time_weekNumberSunday(d) { - var d0 = d3_time_year(d); - return d3_time_zfill2(~~((d3_time_daysElapsed(d0, d) + d0.getDay()) / 7)); -} - -function d3_time_weekNumberMonday(d) { - var d0 = d3_time_year(d); - return d3_time_zfill2(~~((d3_time_daysElapsed(d0, d) + (d0.getDay() + 6) % 7) / 7)); -} - -// TODO table of time zone offset names? -function d3_time_zone(d) { - var z = d.getTimezoneOffset(), - zs = z > 0 ? "-" : "+", - zh = ~~(Math.abs(z) / 60), - zm = Math.abs(z) % 60; - return zs + d3_time_zfill2(zh) + d3_time_zfill2(zm); -} -d3.time.format.utc = function(template) { - var local = d3.time.format(template); - - function format(date) { - try { - d3_time = d3_time_format_utc; - var utc = new d3_time(); - utc._ = date; - return local(utc); - } finally { - d3_time = Date; - } - } - - format.parse = function(string) { - try { - d3_time = d3_time_format_utc; - var date = local.parse(string); - return date && date._; - } finally { - d3_time = Date; - } - }; - - format.toString = local.toString; - - return format; -}; - -function d3_time_format_utc() { - this._ = new Date(Date.UTC.apply(this, arguments)); -} - -d3_time_format_utc.prototype = { - getDate: function() { return this._.getUTCDate(); }, - getDay: function() { return this._.getUTCDay(); }, - getFullYear: function() { return this._.getUTCFullYear(); }, - getHours: function() { return this._.getUTCHours(); }, - getMilliseconds: function() { return this._.getUTCMilliseconds(); }, - getMinutes: function() { return this._.getUTCMinutes(); }, - getMonth: function() { return this._.getUTCMonth(); }, - getSeconds: function() { return this._.getUTCSeconds(); }, - getTimezoneOffset: function() { return 0; }, - valueOf: function() { return this._.getTime(); }, - setDate: function(x) { this._.setUTCDate(x); }, - setDay: function(x) { this._.setUTCDay(x); }, - setFullYear: function(x) { this._.setUTCFullYear(x); }, - setHours: function(x) { this._.setUTCHours(x); }, - setMilliseconds: function(x) { this._.setUTCMilliseconds(x); }, - setMinutes: function(x) { this._.setUTCMinutes(x); }, - setMonth: function(x) { this._.setUTCMonth(x); }, - setSeconds: function(x) { this._.setUTCSeconds(x); } -}; -var d3_time_formatIso = d3.time.format.utc("%Y-%m-%dT%H:%M:%S.%LZ"); - -d3.time.format.iso = Date.prototype.toISOString ? d3_time_formatIsoNative : d3_time_formatIso; - -function d3_time_formatIsoNative(date) { - return date.toISOString(); -} - -d3_time_formatIsoNative.parse = function(string) { - return new Date(string); -}; - -d3_time_formatIsoNative.toString = d3_time_formatIso.toString; -function d3_time_range(floor, step, number) { - return function(t0, t1, dt) { - var time = floor(t0), times = []; - if (time < t0) step(time); - if (dt > 1) { - while (time < t1) { - var date = new Date(+time); - if (!(number(date) % dt)) times.push(date); - step(time); - } - } else { - while (time < t1) times.push(new Date(+time)), step(time); - } - return times; - }; -} -d3.time.second = function(date) { - return new Date(~~(date / 1e3) * 1e3); -}; - -d3.time.second.utc = d3.time.second; -d3.time.seconds = d3_time_range(d3.time.second, function(date) { - date.setTime(date.getTime() + 1e3); -}, function(date) { - return date.getSeconds(); -}); - -d3.time.seconds.utc = d3.time.seconds; -d3.time.minute = function(date) { - return new Date(~~(date / 6e4) * 6e4); -}; - -d3.time.minute.utc = d3.time.minute;d3.time.minutes = d3_time_range(d3.time.minute, d3_time_minutesStep, function(date) { - return date.getMinutes(); -}); - -d3.time.minutes.utc = d3_time_range(d3.time.minute, d3_time_minutesStep, function(date) { - return date.getUTCMinutes(); -}); - -function d3_time_minutesStep(date) { - date.setTime(date.getTime() + 6e4); // assumes no leap seconds -} -d3.time.hour = function(date) { - var offset = date.getTimezoneOffset() / 60; - return new Date((~~(date / 36e5 - offset) + offset) * 36e5); -}; - -d3.time.hour.utc = function(date) { - return new Date(~~(date / 36e5) * 36e5); -}; -d3.time.hours = d3_time_range(d3.time.hour, d3_time_hoursStep, function(date) { - return date.getHours(); -}); - -d3.time.hours.utc = d3_time_range(d3.time.hour.utc, d3_time_hoursStep, function(date) { - return date.getUTCHours(); -}); - -function d3_time_hoursStep(date) { - date.setTime(date.getTime() + 36e5); -} -d3.time.day = function(date) { - return new Date(date.getFullYear(), date.getMonth(), date.getDate()); -}; - -d3.time.day.utc = function(date) { - return new Date(~~(date / 864e5) * 864e5); -}; -d3.time.days = d3_time_range(d3.time.day, function(date) { - date.setDate(date.getDate() + 1); -}, function(date) { - return date.getDate() - 1; -}); - -d3.time.days.utc = d3_time_range(d3.time.day.utc, function(date) { - date.setUTCDate(date.getUTCDate() + 1); -}, function(date) { - return date.getUTCDate() - 1; -}); -d3.time.week = function(date) { - (date = d3.time.day(date)).setDate(date.getDate() - date.getDay()); - return date; -}; - -d3.time.week.utc = function(date) { - (date = d3.time.day.utc(date)).setUTCDate(date.getUTCDate() - date.getUTCDay()); - return date; -}; -d3.time.weeks = d3_time_range(d3.time.week, function(date) { - date.setDate(date.getDate() + 7); -}, function(date) { - return ~~((date - new Date(date.getFullYear(), 0, 1)) / 6048e5); -}); - -d3.time.weeks.utc = d3_time_range(d3.time.week.utc, function(date) { - date.setUTCDate(date.getUTCDate() + 7); -}, function(date) { - return ~~((date - Date.UTC(date.getUTCFullYear(), 0, 1)) / 6048e5); -}); -d3.time.month = function(date) { - return new Date(date.getFullYear(), date.getMonth(), 1); -}; - -d3.time.month.utc = function(date) { - return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1)); -}; -d3.time.months = d3_time_range(d3.time.month, function(date) { - date.setMonth(date.getMonth() + 1); -}, function(date) { - return date.getMonth(); -}); - -d3.time.months.utc = d3_time_range(d3.time.month.utc, function(date) { - date.setUTCMonth(date.getUTCMonth() + 1); -}, function(date) { - return date.getUTCMonth(); -}); -d3.time.year = function(date) { - return new Date(date.getFullYear(), 0, 1); -}; - -d3.time.year.utc = function(date) { - return new Date(Date.UTC(date.getUTCFullYear(), 0, 1)); -}; -d3.time.years = d3_time_range(d3.time.year, function(date) { - date.setFullYear(date.getFullYear() + 1); -}, function(date) { - return date.getFullYear(); -}); - -d3.time.years.utc = d3_time_range(d3.time.year.utc, function(date) { - date.setUTCFullYear(date.getUTCFullYear() + 1); -}, function(date) { - return date.getUTCFullYear(); -}); -// TODO nice -function d3_time_scale(linear, methods, format) { - - function scale(x) { - return linear(x); - } - - scale.invert = function(x) { - return d3_time_scaleDate(linear.invert(x)); - }; - - scale.domain = function(x) { - if (!arguments.length) return linear.domain().map(d3_time_scaleDate); - linear.domain(x); - return scale; - }; - - scale.ticks = function(m, k) { - var extent = d3_time_scaleExtent(scale.domain()); - if (typeof m !== "function") { - var span = extent[1] - extent[0], - target = span / m, - i = d3.bisect(d3_time_scaleSteps, target, 1, d3_time_scaleSteps.length - 1); - if (Math.log(target / d3_time_scaleSteps[i - 1]) < Math.log(d3_time_scaleSteps[i] / target)) --i; - m = methods[i]; - k = m[1]; - m = m[0]; - } - return m(extent[0], extent[1], k); - }; - - scale.tickFormat = function() { - return format; - }; - - scale.copy = function() { - return d3_time_scale(linear.copy(), methods, format); - }; - - // TOOD expose d3_scale_linear_rebind? - scale.range = d3.rebind(scale, linear.range); - scale.rangeRound = d3.rebind(scale, linear.rangeRound); - scale.interpolate = d3.rebind(scale, linear.interpolate); - scale.clamp = d3.rebind(scale, linear.clamp); - - return scale; -} - -// TODO expose d3_scaleExtent? -function d3_time_scaleExtent(domain) { - var start = domain[0], stop = domain[domain.length - 1]; - return start < stop ? [start, stop] : [stop, start]; -} - -function d3_time_scaleDate(t) { - return new Date(t); -} - -function d3_time_scaleFormat(formats) { - return function(date) { - var i = formats.length - 1, f = formats[i]; - while (!f[1](date)) f = formats[--i]; - return f[0](date); - }; -} - -var d3_time_scaleSteps = [ - 1e3, // 1-second - 5e3, // 5-second - 15e3, // 15-second - 3e4, // 30-second - 6e4, // 1-minute - 3e5, // 5-minute - 9e5, // 15-minute - 18e5, // 30-minute - 36e5, // 1-hour - 108e5, // 3-hour - 216e5, // 6-hour - 432e5, // 12-hour - 864e5, // 1-day - 1728e5, // 2-day - 6048e5, // 1-week - 1728e6, // 1-month - 7776e6, // 3-month - 31536e6 // 1-year -]; - -var d3_time_scaleLocalMethods = [ - [d3.time.seconds, 1], - [d3.time.seconds, 5], - [d3.time.seconds, 15], - [d3.time.seconds, 30], - [d3.time.minutes, 1], - [d3.time.minutes, 5], - [d3.time.minutes, 15], - [d3.time.minutes, 30], - [d3.time.hours, 1], - [d3.time.hours, 3], - [d3.time.hours, 6], - [d3.time.hours, 12], - [d3.time.days, 1], - [d3.time.days, 2], - [d3.time.weeks, 1], - [d3.time.months, 1], - [d3.time.months, 3], - [d3.time.years, 1] -]; - -var d3_time_scaleLocalFormats = [ - [d3.time.format("%Y"), function(d) { return true; }], - [d3.time.format("%B"), function(d) { return d.getMonth(); }], - [d3.time.format("%b %d"), function(d) { return d.getDate() != 1; }], - [d3.time.format("%a %d"), function(d) { return d.getDay() && d.getDate() != 1; }], - [d3.time.format("%I %p"), function(d) { return d.getHours(); }], - [d3.time.format("%I:%M"), function(d) { return d.getMinutes(); }], - [d3.time.format(":%S"), function(d) { return d.getSeconds() || d.getMilliseconds(); }] -]; - -var d3_time_scaleLocalFormat = d3_time_scaleFormat(d3_time_scaleLocalFormats); - -d3.time.scale = function() { - return d3_time_scale(d3.scale.linear(), d3_time_scaleLocalMethods, d3_time_scaleLocalFormat); -}; -var d3_time_scaleUTCMethods = [ - [d3.time.seconds.utc, 1], - [d3.time.seconds.utc, 5], - [d3.time.seconds.utc, 15], - [d3.time.seconds.utc, 30], - [d3.time.minutes.utc, 1], - [d3.time.minutes.utc, 5], - [d3.time.minutes.utc, 15], - [d3.time.minutes.utc, 30], - [d3.time.hours.utc, 1], - [d3.time.hours.utc, 3], - [d3.time.hours.utc, 6], - [d3.time.hours.utc, 12], - [d3.time.days.utc, 1], - [d3.time.days.utc, 2], - [d3.time.weeks.utc, 1], - [d3.time.months.utc, 1], - [d3.time.months.utc, 3], - [d3.time.years.utc, 1] -]; - -var d3_time_scaleUTCFormats = [ - [d3.time.format.utc("%Y"), function(d) { return true; }], - [d3.time.format.utc("%B"), function(d) { return d.getUTCMonth(); }], - [d3.time.format.utc("%b %d"), function(d) { return d.getUTCDate() != 1; }], - [d3.time.format.utc("%a %d"), function(d) { return d.getUTCDay() && d.getUTCDate() != 1; }], - [d3.time.format.utc("%I %p"), function(d) { return d.getUTCHours(); }], - [d3.time.format.utc("%I:%M"), function(d) { return d.getUTCMinutes(); }], - [d3.time.format.utc(":%S"), function(d) { return d.getUTCSeconds() || d.getUTCMilliseconds(); }] -]; - -var d3_time_scaleUTCFormat = d3_time_scaleFormat(d3_time_scaleUTCFormats); - -d3.time.scale.utc = function() { - return d3_time_scale(d3.scale.linear(), d3_time_scaleUTCMethods, d3_time_scaleUTCFormat); -}; -})(); diff --git a/src/allmydata/web/static/d3-LICENSE.txt b/src/allmydata/web/static/d3-LICENSE.txt deleted file mode 100644 index f154b09bb..000000000 --- a/src/allmydata/web/static/d3-LICENSE.txt +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) 2010, Michael Bostock -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* The name Michael Bostock may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/allmydata/web/static/download_status_timeline.js b/src/allmydata/web/static/download_status_timeline.js deleted file mode 100644 index efd3ee375..000000000 --- a/src/allmydata/web/static/download_status_timeline.js +++ /dev/null @@ -1,478 +0,0 @@ - -var globals = {}; - -function onDataReceived(data) { - //console.log("got data, now rendering"); - var show_misc = false; - //delete data.misc; - var timeline = d3.select("#timeline"); - var w = Number(timeline.style("width").slice(0,-2)); - // the SVG fills the width of the whole div, but it will extend - // as far vertically as necessary (depends upon the data) - var chart = timeline.append("svg:svg") - .attr("id", "outer_chart") - .attr("width", w) - .attr("pointer-events", "all") - .append("svg:g") - .call(d3.behavior.zoom().on("zoom", pan_and_zoom)) - ; - // this "backboard" rect lets us catch mouse events anywhere in the - // chart, even between the bars. Without it, we only see events on solid - // objects like bars and text, but not in the gaps between. - chart.append("svg:rect") - .attr("id", "outer_rect") - .attr("width", w).attr("height", 200).attr("fill", "none"); - - // but the stuff we put inside it should have some room - w = w-50; - - // at this point we assume the data is fixed, but the zoom/pan is not - - // create the static things (those which don't exist or not exist based - // upon the timeline data). Their locations will be adjusted later, - // during redraw, when we know the x+y coordinates - chart.append("svg:text") - .attr("class", "dyhb-label") - //.attr("x", "20px").attr("y", y) - .attr("text-anchor", "start") // anchor at top-left - .attr("dy", ".71em") - .attr("fill", "black") - .text("DYHB requests"); - - chart.append("svg:text") - .attr("class", "read-label") - //.attr("x", "20px").attr("y", y) - .attr("text-anchor", "start") // anchor at top-left - .attr("dy", ".71em") - .attr("fill", "black") - .text("read() requests"); - chart.append("svg:text") - .attr("class", "segment-label") - //.attr("x", "20px").attr("y", y) - .attr("text-anchor", "start") // anchor at top-left - .attr("dy", ".71em") - .attr("fill", "black") - .text("segment() requests"); - chart.append("svg:text") - .attr("class", "block-label") - //.attr("x", "20px").attr("y", y) - .attr("text-anchor", "start") // anchor at top-left - .attr("dy", ".71em") - .attr("fill", "black") - .text("block() requests"); - chart.append("svg:text") - .attr("class", "seconds-label") - //.attr("x", w/2).attr("y", y + 35) - .attr("text-anchor", "middle") - .attr("fill", "black") - .text("seconds"); - - - function reltime(t) {return t-data.bounds.min;} - var last = data.bounds.max - data.bounds.min; - //last = reltime(d3.max(data.dyhb, function(d){return d.finish_time;})); - last = last * 1.05; - // long downloads are likely to have too much info, start small - if (last > 10.0) - last = 10.0; - // d3.time.scale() has no support for ms or us. - var xOFF = d3.time.scale().domain([data.bounds.min, data.bounds.max]) - .range([0,w]); - var x = d3.scale.linear().domain([-last*0.05, last]) - .range([0,w]); - function tx(d) { return "translate(" +x(d) + ",0)"; } - function left(d) { return x(reltime(d.start_time)); } - function right(d) { - return d.finish_time ? x(reltime(d.finish_time)) : "1px"; - } - function width(d) { - return d.finish_time ? x(reltime(d.finish_time))-x(reltime(d.start_time)) : "1px"; - } - function halfwidth(d) { - if (d.finish_time) - return (x(reltime(d.finish_time))-x(reltime(d.start_time)))/2; - return "1px"; - } - function middle(d) { - if (d.finish_time) - return (x(reltime(d.start_time))+x(reltime(d.finish_time)))/2; - else - return x(reltime(d.start_time)) + 1; - } - function color(d) { return data.server_info[d.serverid].color; } - function servername(d) { return data.server_info[d.serverid].short; } - function timeformat(duration) { - // TODO: trim to microseconds, maybe humanize - return duration; - } - - function zoomin() { - //console.log("zoom in!"); - //console.log(x.domain()); - var old = x.domain(); - var quarterwidth = (old[1] - old[0])/4; - x.domain([old[0]+quarterwidth, old[1]-quarterwidth]); - //console.log(x.domain()); - redraw(); - //d3.event.preventDefault(); - } - - function zoomout() { - //console.log("zoom out!"); - var old = x.domain(); - var halfwidth = (old[1] - old[0])/2; - x.domain([old[0]-halfwidth, old[1]+halfwidth]); - redraw(); - } - - function pan_and_zoom() { - //console.log("P", x.domain()); - if (d3.event) d3.event.transform(x); - //console.log("p", x.domain()); - redraw(); - } - - function clip() { - var clipped = {}; - var min = data.bounds.min + x.domain()[0]; - var max = data.bounds.min + x.domain()[1]; - function inside(d) { - var finish_time = d.finish_time || d.start_time; - if (Math.max(d.start_time, min) < Math.min(finish_time, max)) - return true; - return false; - } - clipped.dyhb = data.dyhb.filter(inside); - clipped.read = data.read.filter(inside); - clipped.segment = data.segment.filter(inside); - clipped.block = data.block.filter(inside); - if (show_misc && data.misc) - clipped.misc = data.misc.filter(inside); - return clipped; - } - - function redraw() { - // at this point zoom/pan must be fixed - var clipped = clip(data); - - var y = 0; - //chart.select(".dyhb-label") - // .attr("x", x(0))//"20px") - // .attr("y", y); - y += 20; - - // DYHB section - var dyhb_y = d3.scale.ordinal() - .domain(d3.range(data.dyhb.length)) - .rangeBands([y, y+data.dyhb.length*20], .2); - y += data.dyhb.length*20; - var dyhb = chart.selectAll("g.dyhb") // one per row - .data(clipped.dyhb, function(d) { return d.start_time; }) - .attr("transform", function(d,i) { - return "translate("+x(reltime(d.start_time))+"," - +dyhb_y(i)+")"; - }); - var new_dyhb = dyhb.enter().append("svg:g") - .attr("class", "dyhb") - .attr("transform", function(d,i) { - return "translate("+x(reltime(d.start_time))+"," - +dyhb_y(i)+")"; - }) - ; - dyhb.exit().remove(); - dyhb.select("rect") - .attr("width", width) - ; - new_dyhb.append("svg:rect") - .attr("width", width) - .attr("height", dyhb_y.rangeBand()) - .attr("stroke", "black") - .attr("fill", color) - .attr("title", function(d){return "shnums: "+d.response_shnums;}) - ; - new_dyhb.append("svg:text") - .attr("text-anchor", "end") - .attr("fill", "#444") - .attr("x", "-0.3em") // for some reason dx doesn't work - .attr("dy", "1.0em") - .attr("font-size", "12px") - .text(servername) - ; - dyhb.select(".rightbox") - .attr("transform", function(d) {return "translate("+width(d) - +",0)";}); - var dyhb_rightboxes = new_dyhb.append("svg:g") - .attr("class", "rightbox") - .attr("transform", function(d) {return "translate("+width(d) - +",0)";}) - ; - dyhb_rightboxes.append("svg:text") - .attr("text-anchor", "start") - .attr("y", dyhb_y.rangeBand()) - .attr("dx", "0.5em") - .attr("dy", "-0.4em") - .attr("fill", "#444") - .attr("font-size", "14px") - .text(function (d) {return "shnums: "+d.response_shnums;}) - ; - - // read() requests - chart.select(".read-label") - .attr("x", "20px") - .attr("y", y); - y += 20; - var read = chart.selectAll("g.read") - .data(clipped.read, function(d) { return d.start_time; }) - .attr("transform", function(d) { - return "translate("+left(d)+","+(y+30*d.row)+")"; }); - read.select("rect") - .attr("width", width); - read.select("text") - .attr("x", halfwidth); - var new_read = read.enter().append("svg:g") - .attr("class", "read") - .attr("transform", function(d) { - return "translate("+left(d)+","+(y+30*d.row)+")"; }) - ; - read.exit().remove(); - y += 30*(1+d3.max(data.read, function(d){return d.row;})); - new_read.append("svg:rect") - .attr("width", width) - .attr("height", 20) - .attr("fill", "red") - .attr("stroke", "black") - .attr("title", function(d) - {return "read(start="+d.start+", len="+d.length+") -> " - +d.bytes_returned+" bytes";}) - ; - new_read.append("svg:text") - .attr("x", halfwidth) - .attr("dy", "0.9em") - .attr("fill", "black") - .text(function(d) {return "["+d.start+":+"+d.length+"]";}) - ; - - // segment requests - chart.select(".segment-label") - .attr("x", "20px") - .attr("y", y); - y += 20; - var segment = chart.selectAll("g.segment") - .data(clipped.segment, function(d) { return d.start_time; }) - .attr("transform", function(d) { - return "translate("+left(d)+","+(y+30*d.row)+")"; }); - segment.select("rect") - .attr("width", width); - segment.select("text") - .attr("x", halfwidth); - var new_segment = segment.enter().append("svg:g") - .attr("class", "segment") - .attr("transform", function(d) { - return "translate("+left(d)+","+(y+30*d.row)+")"; }) - ; - segment.exit().remove(); - y += 30*(1+d3.max(data.segment, function(d){return d.row;})); - new_segment.append("svg:rect") - .attr("width", width) - .attr("height", 20) - .attr("fill", function(d){return d.success ? "#cfc" : "#fcc";}) - .attr("stroke", "black") - .attr("stroke-width", 1) - .attr("title", function(d) { - return "seg"+d.segment_number+" ["+d.segment_start - +":+"+d.segment_length+"] (took " - +timeformat(d.finish_time-d.start_time)+")";}) - ; - new_segment.append("svg:text") - .attr("x", halfwidth) - .attr("text-anchor", "middle") - .attr("dy", "0.9em") - .attr("fill", "black") - .text(function(d) {return d.segment_number;}) - ; - - var shnum_colors = d3.scale.category10(); - - if (show_misc && "misc" in clipped) { - // misc requests - var misc_label = chart.select("text.misc-label"); - if (!misc_label.node()) { - chart.append("svg:text") - .attr("class", "misc-label"); - misc_label = chart.select("text.misc-label"); - } - misc_label - .attr("text-anchor", "start") // anchor at top-left - .attr("dy", ".71em") - .attr("fill", "black") - .text("misc requests") - .attr("x", "20px") - .attr("y", y) - ; - y += 20; - var misc = chart.selectAll("g.misc") - .data(clipped.misc, function(d) { return d.start_time; }) - .attr("transform", function(d) { - return "translate("+left(d)+","+(y+30*d.row)+")"; }); - misc.select("rect") - .attr("width", width); - misc.select("text") - .attr("x", halfwidth); - var new_misc = misc.enter().append("svg:g") - .attr("class", "misc") - .attr("transform", function(d) { - return "translate("+left(d)+","+(y+30*d.row)+")"; }) - ; - misc.exit().remove(); - y += 30*(1+d3.max(data.misc, function(d){return d.row;})); - new_misc.append("svg:rect") - .attr("width", width) - .attr("height", 20) - .attr("fill", "white") - .attr("stroke", "black") - .attr("stroke-width", 1) - .attr("title", function(d) { - return d.what+" (took " - +timeformat(d.finish_time-d.start_time)+")";}) - ; - new_misc.append("svg:text") - .attr("x", halfwidth) - .attr("text-anchor", "middle") - .attr("dy", "0.9em") - .attr("fill", "black") - .text(function(d) {return d.what;}) - ; - } else { - chart.select("text.misc-label").remove(); - chart.selectAll("g.misc").remove(); - } - // block requests - chart.select(".block-label") - .attr("x", "20px") - .attr("y", y); - y += 20; - var block_row_to_y = {}; - var dummy = function() { - var row_y=y; - for (var group=0; group < data.block_rownums.length; group++) { - for (var row=0; row < data.block_rownums[group]; row++) { - block_row_to_y[group+"-"+row] = row_y; - row_y += 12; y += 12; - } - row_y += 5; y += 5; - } - }(); - var blocks = chart.selectAll("g.block") - .data(clipped.block, function(d) { return d.start_time; }) - .attr("transform", function(d) { - var ry = block_row_to_y[d.row[0]+"-"+d.row[1]]; - return "translate("+left(d)+"," +ry+")"; }); - blocks.select("rect") - .attr("width", width); - blocks.select("text") - .attr("x", halfwidth); - var new_blocks = blocks.enter().append("svg:g") - .attr("class", "block") - .attr("transform", function(d) { - var ry = block_row_to_y[d.row[0]+"-"+d.row[1]]; - return "translate("+left(d)+"," +ry+")"; }) - ; - blocks.exit().remove(); - // everything appended to blocks starts at the top-left of the - // correct per-rect location - new_blocks.append("svg:rect") - .attr("width", width) - .attr("y", function(d) {return (d.response_length > 100) ? 0:3;}) - .attr("height", - function(d) {return (d.response_length > 100) ? 10:4;}) - .attr("fill", color) - .attr("stroke", function(d){return shnum_colors(d.shnum);}) - .attr("stroke-width", 1) - .attr("title", function(d){ - return "sh"+d.shnum+"-on-"+servername(d) - +" ["+d.start+":+"+d.length+"] -> " - +d.response_length;}) - ; - new_blocks.append("svg:text") - .attr("x", halfwidth) - .attr("dy", "0.9em") - .attr("fill", "black") - .attr("font-size", "8px") - .attr("text-anchor", "middle") - .text(function(d) {return "sh"+d.shnum;}) - ; - - var num = d3.format(".4g"); - - // horizontal scale markers: vertical lines at rational timestamps - var rules = chart.selectAll("g.rule") - .data(x.ticks(10)) - .attr("transform", tx); - rules.select("text").text(x.tickFormat(10)); - - var newrules = rules.enter().insert("svg:g") - .attr("class", "rule") - .attr("transform", tx) - ; - - newrules.append("svg:line") - .attr("class", "rule-tick") - .attr("stroke", "black"); - chart.selectAll("line.rule-tick") - .attr("y1", y) - .attr("y2", y + 6); - newrules.append("svg:line") - .attr("class", "rule-red") - .attr("stroke", "red") - .attr("stroke-opacity", .3); - chart.selectAll("line.rule-red") - .attr("y1", 0) - .attr("y2", y); - newrules.append("svg:text") - .attr("class", "rule-text") - .attr("dy", ".71em") - .attr("text-anchor", "middle") - .attr("fill", "black") - .text(x.tickFormat(10)); - chart.selectAll("text.rule-text") - .attr("y", y + 9); - rules.exit().remove(); - chart.select(".seconds-label") - .attr("x", w/2) - .attr("y", y + 35); - y += 45; - - d3.select("#outer_chart").attr("height", y); - d3.select("#outer_rect").attr("height", y); - d3.select("#zoom").attr("transform", "translate("+(w-10)+","+10+")"); - } - globals.x = x; - globals.redraw = redraw; - - d3.select("#zoom_in_button").on("click", zoomin); - d3.select("#zoom_out_button").on("click", zoomout); - d3.select("#toggle_misc_button").on("click", - function() { - show_misc = !show_misc; - redraw(); - }); - d3.select("#reset_button").on("click", - function() { - x.domain([-last*0.05, last]).range([0,w]); - redraw(); - }); - - redraw(); -} - -$(function() { - var datafile = "event_json"; - if (location.hash) - datafile = location.hash.slice(1); - //console.log("fetching data from "+datafile); - $.ajax({url: datafile, - method: 'GET', - dataType: 'json', - success: onDataReceived }); -}); - diff --git a/src/allmydata/web/static/jquery-1.11.0.js b/src/allmydata/web/static/jquery-1.11.0.js deleted file mode 100644 index 3c88fa8b7..000000000 --- a/src/allmydata/web/static/jquery-1.11.0.js +++ /dev/null @@ -1,10337 +0,0 @@ -/*! - * jQuery JavaScript Library v1.11.0 - * http://jquery.com/ - * - * Includes Sizzle.js - * http://sizzlejs.com/ - * - * Copyright 2005, 2014 jQuery Foundation, Inc. and other contributors - * Released under the MIT license - * http://jquery.org/license - * - * Date: 2014-01-23T21:02Z - */ - -(function( global, factory ) { - - if ( typeof module === "object" && typeof module.exports === "object" ) { - // For CommonJS and CommonJS-like environments where a proper window is present, - // execute the factory and get jQuery - // For environments that do not inherently posses a window with a document - // (such as Node.js), expose a jQuery-making factory as module.exports - // This accentuates the need for the creation of a real window - // e.g. var jQuery = require("jquery")(window); - // See ticket #14549 for more info - module.exports = global.document ? - factory( global, true ) : - function( w ) { - if ( !w.document ) { - throw new Error( "jQuery requires a window with a document" ); - } - return factory( w ); - }; - } else { - factory( global ); - } - -// Pass this if window is not defined yet -}(typeof window !== "undefined" ? window : this, function( window, noGlobal ) { - -// Can't do this because several apps including ASP.NET trace -// the stack via arguments.caller.callee and Firefox dies if -// you try to trace through "use strict" call chains. (#13335) -// Support: Firefox 18+ -// - -var deletedIds = []; - -var slice = deletedIds.slice; - -var concat = deletedIds.concat; - -var push = deletedIds.push; - -var indexOf = deletedIds.indexOf; - -var class2type = {}; - -var toString = class2type.toString; - -var hasOwn = class2type.hasOwnProperty; - -var trim = "".trim; - -var support = {}; - - - -var - version = "1.11.0", - - // Define a local copy of jQuery - jQuery = function( selector, context ) { - // The jQuery object is actually just the init constructor 'enhanced' - // Need init if jQuery is called (just allow error to be thrown if not included) - return new jQuery.fn.init( selector, context ); - }, - - // Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE) - rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, - - // Matches dashed string for camelizing - rmsPrefix = /^-ms-/, - rdashAlpha = /-([\da-z])/gi, - - // Used by jQuery.camelCase as callback to replace() - fcamelCase = function( all, letter ) { - return letter.toUpperCase(); - }; - -jQuery.fn = jQuery.prototype = { - // The current version of jQuery being used - jquery: version, - - constructor: jQuery, - - // Start with an empty selector - selector: "", - - // The default length of a jQuery object is 0 - length: 0, - - toArray: function() { - return slice.call( this ); - }, - - // Get the Nth element in the matched element set OR - // Get the whole matched element set as a clean array - get: function( num ) { - return num != null ? - - // Return a 'clean' array - ( num < 0 ? this[ num + this.length ] : this[ num ] ) : - - // Return just the object - slice.call( this ); - }, - - // Take an array of elements and push it onto the stack - // (returning the new matched element set) - pushStack: function( elems ) { - - // Build a new jQuery matched element set - var ret = jQuery.merge( this.constructor(), elems ); - - // Add the old object onto the stack (as a reference) - ret.prevObject = this; - ret.context = this.context; - - // Return the newly-formed element set - return ret; - }, - - // Execute a callback for every element in the matched set. - // (You can seed the arguments with an array of args, but this is - // only used internally.) - each: function( callback, args ) { - return jQuery.each( this, callback, args ); - }, - - map: function( callback ) { - return this.pushStack( jQuery.map(this, function( elem, i ) { - return callback.call( elem, i, elem ); - })); - }, - - slice: function() { - return this.pushStack( slice.apply( this, arguments ) ); - }, - - first: function() { - return this.eq( 0 ); - }, - - last: function() { - return this.eq( -1 ); - }, - - eq: function( i ) { - var len = this.length, - j = +i + ( i < 0 ? len : 0 ); - return this.pushStack( j >= 0 && j < len ? [ this[j] ] : [] ); - }, - - end: function() { - return this.prevObject || this.constructor(null); - }, - - // For internal use only. - // Behaves like an Array's method, not like a jQuery method. - push: push, - sort: deletedIds.sort, - splice: deletedIds.splice -}; - -jQuery.extend = jQuery.fn.extend = function() { - var src, copyIsArray, copy, name, options, clone, - target = arguments[0] || {}, - i = 1, - length = arguments.length, - deep = false; - - // Handle a deep copy situation - if ( typeof target === "boolean" ) { - deep = target; - - // skip the boolean and the target - target = arguments[ i ] || {}; - i++; - } - - // Handle case when target is a string or something (possible in deep copy) - if ( typeof target !== "object" && !jQuery.isFunction(target) ) { - target = {}; - } - - // extend jQuery itself if only one argument is passed - if ( i === length ) { - target = this; - i--; - } - - for ( ; i < length; i++ ) { - // Only deal with non-null/undefined values - if ( (options = arguments[ i ]) != null ) { - // Extend the base object - for ( name in options ) { - src = target[ name ]; - copy = options[ name ]; - - // Prevent never-ending loop - if ( target === copy ) { - continue; - } - - // Recurse if we're merging plain objects or arrays - if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { - if ( copyIsArray ) { - copyIsArray = false; - clone = src && jQuery.isArray(src) ? src : []; - - } else { - clone = src && jQuery.isPlainObject(src) ? src : {}; - } - - // Never move original objects, clone them - target[ name ] = jQuery.extend( deep, clone, copy ); - - // Don't bring in undefined values - } else if ( copy !== undefined ) { - target[ name ] = copy; - } - } - } - } - - // Return the modified object - return target; -}; - -jQuery.extend({ - // Unique for each copy of jQuery on the page - expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), - - // Assume jQuery is ready without the ready module - isReady: true, - - error: function( msg ) { - throw new Error( msg ); - }, - - noop: function() {}, - - // See test/unit/core.js for details concerning isFunction. - // Since version 1.3, DOM methods and functions like alert - // aren't supported. They return false on IE (#2968). - isFunction: function( obj ) { - return jQuery.type(obj) === "function"; - }, - - isArray: Array.isArray || function( obj ) { - return jQuery.type(obj) === "array"; - }, - - isWindow: function( obj ) { - /* jshint eqeqeq: false */ - return obj != null && obj == obj.window; - }, - - isNumeric: function( obj ) { - // parseFloat NaNs numeric-cast false positives (null|true|false|"") - // ...but misinterprets leading-number strings, particularly hex literals ("0x...") - // subtraction forces infinities to NaN - return obj - parseFloat( obj ) >= 0; - }, - - isEmptyObject: function( obj ) { - var name; - for ( name in obj ) { - return false; - } - return true; - }, - - isPlainObject: function( obj ) { - var key; - - // Must be an Object. - // Because of IE, we also have to check the presence of the constructor property. - // Make sure that DOM nodes and window objects don't pass through, as well - if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { - return false; - } - - try { - // Not own constructor property must be Object - if ( obj.constructor && - !hasOwn.call(obj, "constructor") && - !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { - return false; - } - } catch ( e ) { - // IE8,9 Will throw exceptions on certain host objects #9897 - return false; - } - - // Support: IE<9 - // Handle iteration over inherited properties before own properties. - if ( support.ownLast ) { - for ( key in obj ) { - return hasOwn.call( obj, key ); - } - } - - // Own properties are enumerated firstly, so to speed up, - // if last one is own, then all properties are own. - for ( key in obj ) {} - - return key === undefined || hasOwn.call( obj, key ); - }, - - type: function( obj ) { - if ( obj == null ) { - return obj + ""; - } - return typeof obj === "object" || typeof obj === "function" ? - class2type[ toString.call(obj) ] || "object" : - typeof obj; - }, - - // Evaluates a script in a global context - // Workarounds based on findings by Jim Driscoll - // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context - globalEval: function( data ) { - if ( data && jQuery.trim( data ) ) { - // We use execScript on Internet Explorer - // We use an anonymous function so that context is window - // rather than jQuery in Firefox - ( window.execScript || function( data ) { - window[ "eval" ].call( window, data ); - } )( data ); - } - }, - - // Convert dashed to camelCase; used by the css and data modules - // Microsoft forgot to hump their vendor prefix (#9572) - camelCase: function( string ) { - return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); - }, - - nodeName: function( elem, name ) { - return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); - }, - - // args is for internal usage only - each: function( obj, callback, args ) { - var value, - i = 0, - length = obj.length, - isArray = isArraylike( obj ); - - if ( args ) { - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback.apply( obj[ i ], args ); - - if ( value === false ) { - break; - } - } - } else { - for ( i in obj ) { - value = callback.apply( obj[ i ], args ); - - if ( value === false ) { - break; - } - } - } - - // A special, fast, case for the most common use of each - } else { - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback.call( obj[ i ], i, obj[ i ] ); - - if ( value === false ) { - break; - } - } - } else { - for ( i in obj ) { - value = callback.call( obj[ i ], i, obj[ i ] ); - - if ( value === false ) { - break; - } - } - } - } - - return obj; - }, - - // Use native String.trim function wherever possible - trim: trim && !trim.call("\uFEFF\xA0") ? - function( text ) { - return text == null ? - "" : - trim.call( text ); - } : - - // Otherwise use our own trimming functionality - function( text ) { - return text == null ? - "" : - ( text + "" ).replace( rtrim, "" ); - }, - - // results is for internal usage only - makeArray: function( arr, results ) { - var ret = results || []; - - if ( arr != null ) { - if ( isArraylike( Object(arr) ) ) { - jQuery.merge( ret, - typeof arr === "string" ? - [ arr ] : arr - ); - } else { - push.call( ret, arr ); - } - } - - return ret; - }, - - inArray: function( elem, arr, i ) { - var len; - - if ( arr ) { - if ( indexOf ) { - return indexOf.call( arr, elem, i ); - } - - len = arr.length; - i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; - - for ( ; i < len; i++ ) { - // Skip accessing in sparse arrays - if ( i in arr && arr[ i ] === elem ) { - return i; - } - } - } - - return -1; - }, - - merge: function( first, second ) { - var len = +second.length, - j = 0, - i = first.length; - - while ( j < len ) { - first[ i++ ] = second[ j++ ]; - } - - // Support: IE<9 - // Workaround casting of .length to NaN on otherwise arraylike objects (e.g., NodeLists) - if ( len !== len ) { - while ( second[j] !== undefined ) { - first[ i++ ] = second[ j++ ]; - } - } - - first.length = i; - - return first; - }, - - grep: function( elems, callback, invert ) { - var callbackInverse, - matches = [], - i = 0, - length = elems.length, - callbackExpect = !invert; - - // Go through the array, only saving the items - // that pass the validator function - for ( ; i < length; i++ ) { - callbackInverse = !callback( elems[ i ], i ); - if ( callbackInverse !== callbackExpect ) { - matches.push( elems[ i ] ); - } - } - - return matches; - }, - - // arg is for internal usage only - map: function( elems, callback, arg ) { - var value, - i = 0, - length = elems.length, - isArray = isArraylike( elems ), - ret = []; - - // Go through the array, translating each of the items to their new values - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback( elems[ i ], i, arg ); - - if ( value != null ) { - ret.push( value ); - } - } - - // Go through every key on the object, - } else { - for ( i in elems ) { - value = callback( elems[ i ], i, arg ); - - if ( value != null ) { - ret.push( value ); - } - } - } - - // Flatten any nested arrays - return concat.apply( [], ret ); - }, - - // A global GUID counter for objects - guid: 1, - - // Bind a function to a context, optionally partially applying any - // arguments. - proxy: function( fn, context ) { - var args, proxy, tmp; - - if ( typeof context === "string" ) { - tmp = fn[ context ]; - context = fn; - fn = tmp; - } - - // Quick check to determine if target is callable, in the spec - // this throws a TypeError, but we will just return undefined. - if ( !jQuery.isFunction( fn ) ) { - return undefined; - } - - // Simulated bind - args = slice.call( arguments, 2 ); - proxy = function() { - return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); - }; - - // Set the guid of unique handler to the same of original handler, so it can be removed - proxy.guid = fn.guid = fn.guid || jQuery.guid++; - - return proxy; - }, - - now: function() { - return +( new Date() ); - }, - - // jQuery.support is not used in Core but other projects attach their - // properties to it so it needs to exist. - support: support -}); - -// Populate the class2type map -jQuery.each("Boolean Number String Function Array Date RegExp Object Error".split(" "), function(i, name) { - class2type[ "[object " + name + "]" ] = name.toLowerCase(); -}); - -function isArraylike( obj ) { - var length = obj.length, - type = jQuery.type( obj ); - - if ( type === "function" || jQuery.isWindow( obj ) ) { - return false; - } - - if ( obj.nodeType === 1 && length ) { - return true; - } - - return type === "array" || length === 0 || - typeof length === "number" && length > 0 && ( length - 1 ) in obj; -} -var Sizzle = -/*! - * Sizzle CSS Selector Engine v1.10.16 - * http://sizzlejs.com/ - * - * Copyright 2013 jQuery Foundation, Inc. and other contributors - * Released under the MIT license - * http://jquery.org/license - * - * Date: 2014-01-13 - */ -(function( window ) { - -var i, - support, - Expr, - getText, - isXML, - compile, - outermostContext, - sortInput, - hasDuplicate, - - // Local document vars - setDocument, - document, - docElem, - documentIsHTML, - rbuggyQSA, - rbuggyMatches, - matches, - contains, - - // Instance-specific data - expando = "sizzle" + -(new Date()), - preferredDoc = window.document, - dirruns = 0, - done = 0, - classCache = createCache(), - tokenCache = createCache(), - compilerCache = createCache(), - sortOrder = function( a, b ) { - if ( a === b ) { - hasDuplicate = true; - } - return 0; - }, - - // General-purpose constants - strundefined = typeof undefined, - MAX_NEGATIVE = 1 << 31, - - // Instance methods - hasOwn = ({}).hasOwnProperty, - arr = [], - pop = arr.pop, - push_native = arr.push, - push = arr.push, - slice = arr.slice, - // Use a stripped-down indexOf if we can't use a native one - indexOf = arr.indexOf || function( elem ) { - var i = 0, - len = this.length; - for ( ; i < len; i++ ) { - if ( this[i] === elem ) { - return i; - } - } - return -1; - }, - - booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", - - // Regular expressions - - // Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace - whitespace = "[\\x20\\t\\r\\n\\f]", - // http://www.w3.org/TR/css3-syntax/#characters - characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+", - - // Loosely modeled on CSS identifier characters - // An unquoted value should be a CSS identifier http://www.w3.org/TR/css3-selectors/#attribute-selectors - // Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier - identifier = characterEncoding.replace( "w", "w#" ), - - // Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors - attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace + - "*(?:([*^$|!~]?=)" + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]", - - // Prefer arguments quoted, - // then not containing pseudos/brackets, - // then attribute selectors/non-parenthetical expressions, - // then anything else - // These preferences are here to reduce the number of selectors - // needing tokenize in the PSEUDO preFilter - pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)", - - // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter - rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), - - rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), - rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), - - rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), - - rpseudo = new RegExp( pseudos ), - ridentifier = new RegExp( "^" + identifier + "$" ), - - matchExpr = { - "ID": new RegExp( "^#(" + characterEncoding + ")" ), - "CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ), - "TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ), - "ATTR": new RegExp( "^" + attributes ), - "PSEUDO": new RegExp( "^" + pseudos ), - "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + - "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + - "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), - "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), - // For use in libraries implementing .is() - // We use this for POS matching in `select` - "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + - whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) - }, - - rinputs = /^(?:input|select|textarea|button)$/i, - rheader = /^h\d$/i, - - rnative = /^[^{]+\{\s*\[native \w/, - - // Easily-parseable/retrievable ID or TAG or CLASS selectors - rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, - - rsibling = /[+~]/, - rescape = /'|\\/g, - - // CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters - runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), - funescape = function( _, escaped, escapedWhitespace ) { - var high = "0x" + escaped - 0x10000; - // NaN means non-codepoint - // Support: Firefox - // Workaround erroneous numeric interpretation of +"0x" - return high !== high || escapedWhitespace ? - escaped : - high < 0 ? - // BMP codepoint - String.fromCharCode( high + 0x10000 ) : - // Supplemental Plane codepoint (surrogate pair) - String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); - }; - -// Optimize for push.apply( _, NodeList ) -try { - push.apply( - (arr = slice.call( preferredDoc.childNodes )), - preferredDoc.childNodes - ); - // Support: Android<4.0 - // Detect silently failing push.apply - arr[ preferredDoc.childNodes.length ].nodeType; -} catch ( e ) { - push = { apply: arr.length ? - - // Leverage slice if possible - function( target, els ) { - push_native.apply( target, slice.call(els) ); - } : - - // Support: IE<9 - // Otherwise append directly - function( target, els ) { - var j = target.length, - i = 0; - // Can't trust NodeList.length - while ( (target[j++] = els[i++]) ) {} - target.length = j - 1; - } - }; -} - -function Sizzle( selector, context, results, seed ) { - var match, elem, m, nodeType, - // QSA vars - i, groups, old, nid, newContext, newSelector; - - if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { - setDocument( context ); - } - - context = context || document; - results = results || []; - - if ( !selector || typeof selector !== "string" ) { - return results; - } - - if ( (nodeType = context.nodeType) !== 1 && nodeType !== 9 ) { - return []; - } - - if ( documentIsHTML && !seed ) { - - // Shortcuts - if ( (match = rquickExpr.exec( selector )) ) { - // Speed-up: Sizzle("#ID") - if ( (m = match[1]) ) { - if ( nodeType === 9 ) { - elem = context.getElementById( m ); - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document (jQuery #6963) - if ( elem && elem.parentNode ) { - // Handle the case where IE, Opera, and Webkit return items - // by name instead of ID - if ( elem.id === m ) { - results.push( elem ); - return results; - } - } else { - return results; - } - } else { - // Context is not a document - if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) && - contains( context, elem ) && elem.id === m ) { - results.push( elem ); - return results; - } - } - - // Speed-up: Sizzle("TAG") - } else if ( match[2] ) { - push.apply( results, context.getElementsByTagName( selector ) ); - return results; - - // Speed-up: Sizzle(".CLASS") - } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { - push.apply( results, context.getElementsByClassName( m ) ); - return results; - } - } - - // QSA path - if ( support.qsa && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { - nid = old = expando; - newContext = context; - newSelector = nodeType === 9 && selector; - - // qSA works strangely on Element-rooted queries - // We can work around this by specifying an extra ID on the root - // and working up from there (Thanks to Andrew Dupont for the technique) - // IE 8 doesn't work on object elements - if ( nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { - groups = tokenize( selector ); - - if ( (old = context.getAttribute("id")) ) { - nid = old.replace( rescape, "\\$&" ); - } else { - context.setAttribute( "id", nid ); - } - nid = "[id='" + nid + "'] "; - - i = groups.length; - while ( i-- ) { - groups[i] = nid + toSelector( groups[i] ); - } - newContext = rsibling.test( selector ) && testContext( context.parentNode ) || context; - newSelector = groups.join(","); - } - - if ( newSelector ) { - try { - push.apply( results, - newContext.querySelectorAll( newSelector ) - ); - return results; - } catch(qsaError) { - } finally { - if ( !old ) { - context.removeAttribute("id"); - } - } - } - } - } - - // All others - return select( selector.replace( rtrim, "$1" ), context, results, seed ); -} - -/** - * Create key-value caches of limited size - * @returns {Function(string, Object)} Returns the Object data after storing it on itself with - * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) - * deleting the oldest entry - */ -function createCache() { - var keys = []; - - function cache( key, value ) { - // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) - if ( keys.push( key + " " ) > Expr.cacheLength ) { - // Only keep the most recent entries - delete cache[ keys.shift() ]; - } - return (cache[ key + " " ] = value); - } - return cache; -} - -/** - * Mark a function for special use by Sizzle - * @param {Function} fn The function to mark - */ -function markFunction( fn ) { - fn[ expando ] = true; - return fn; -} - -/** - * Support testing using an element - * @param {Function} fn Passed the created div and expects a boolean result - */ -function assert( fn ) { - var div = document.createElement("div"); - - try { - return !!fn( div ); - } catch (e) { - return false; - } finally { - // Remove from its parent by default - if ( div.parentNode ) { - div.parentNode.removeChild( div ); - } - // release memory in IE - div = null; - } -} - -/** - * Adds the same handler for all of the specified attrs - * @param {String} attrs Pipe-separated list of attributes - * @param {Function} handler The method that will be applied - */ -function addHandle( attrs, handler ) { - var arr = attrs.split("|"), - i = attrs.length; - - while ( i-- ) { - Expr.attrHandle[ arr[i] ] = handler; - } -} - -/** - * Checks document order of two siblings - * @param {Element} a - * @param {Element} b - * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b - */ -function siblingCheck( a, b ) { - var cur = b && a, - diff = cur && a.nodeType === 1 && b.nodeType === 1 && - ( ~b.sourceIndex || MAX_NEGATIVE ) - - ( ~a.sourceIndex || MAX_NEGATIVE ); - - // Use IE sourceIndex if available on both nodes - if ( diff ) { - return diff; - } - - // Check if b follows a - if ( cur ) { - while ( (cur = cur.nextSibling) ) { - if ( cur === b ) { - return -1; - } - } - } - - return a ? 1 : -1; -} - -/** - * Returns a function to use in pseudos for input types - * @param {String} type - */ -function createInputPseudo( type ) { - return function( elem ) { - var name = elem.nodeName.toLowerCase(); - return name === "input" && elem.type === type; - }; -} - -/** - * Returns a function to use in pseudos for buttons - * @param {String} type - */ -function createButtonPseudo( type ) { - return function( elem ) { - var name = elem.nodeName.toLowerCase(); - return (name === "input" || name === "button") && elem.type === type; - }; -} - -/** - * Returns a function to use in pseudos for positionals - * @param {Function} fn - */ -function createPositionalPseudo( fn ) { - return markFunction(function( argument ) { - argument = +argument; - return markFunction(function( seed, matches ) { - var j, - matchIndexes = fn( [], seed.length, argument ), - i = matchIndexes.length; - - // Match elements found at the specified indexes - while ( i-- ) { - if ( seed[ (j = matchIndexes[i]) ] ) { - seed[j] = !(matches[j] = seed[j]); - } - } - }); - }); -} - -/** - * Checks a node for validity as a Sizzle context - * @param {Element|Object=} context - * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value - */ -function testContext( context ) { - return context && typeof context.getElementsByTagName !== strundefined && context; -} - -// Expose support vars for convenience -support = Sizzle.support = {}; - -/** - * Detects XML nodes - * @param {Element|Object} elem An element or a document - * @returns {Boolean} True iff elem is a non-HTML XML node - */ -isXML = Sizzle.isXML = function( elem ) { - // documentElement is verified for cases where it doesn't yet exist - // (such as loading iframes in IE - #4833) - var documentElement = elem && (elem.ownerDocument || elem).documentElement; - return documentElement ? documentElement.nodeName !== "HTML" : false; -}; - -/** - * Sets document-related variables once based on the current document - * @param {Element|Object} [doc] An element or document object to use to set the document - * @returns {Object} Returns the current document - */ -setDocument = Sizzle.setDocument = function( node ) { - var hasCompare, - doc = node ? node.ownerDocument || node : preferredDoc, - parent = doc.defaultView; - - // If no document and documentElement is available, return - if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { - return document; - } - - // Set our document - document = doc; - docElem = doc.documentElement; - - // Support tests - documentIsHTML = !isXML( doc ); - - // Support: IE>8 - // If iframe document is assigned to "document" variable and if iframe has been reloaded, - // IE will throw "permission denied" error when accessing "document" variable, see jQuery #13936 - // IE6-8 do not support the defaultView property so parent will be undefined - if ( parent && parent !== parent.top ) { - // IE11 does not have attachEvent, so all must suffer - if ( parent.addEventListener ) { - parent.addEventListener( "unload", function() { - setDocument(); - }, false ); - } else if ( parent.attachEvent ) { - parent.attachEvent( "onunload", function() { - setDocument(); - }); - } - } - - /* Attributes - ---------------------------------------------------------------------- */ - - // Support: IE<8 - // Verify that getAttribute really returns attributes and not properties (excepting IE8 booleans) - support.attributes = assert(function( div ) { - div.className = "i"; - return !div.getAttribute("className"); - }); - - /* getElement(s)By* - ---------------------------------------------------------------------- */ - - // Check if getElementsByTagName("*") returns only elements - support.getElementsByTagName = assert(function( div ) { - div.appendChild( doc.createComment("") ); - return !div.getElementsByTagName("*").length; - }); - - // Check if getElementsByClassName can be trusted - support.getElementsByClassName = rnative.test( doc.getElementsByClassName ) && assert(function( div ) { - div.innerHTML = "
    "; - - // Support: Safari<4 - // Catch class over-caching - div.firstChild.className = "i"; - // Support: Opera<10 - // Catch gEBCN failure to find non-leading classes - return div.getElementsByClassName("i").length === 2; - }); - - // Support: IE<10 - // Check if getElementById returns elements by name - // The broken getElementById methods don't pick up programatically-set names, - // so use a roundabout getElementsByName test - support.getById = assert(function( div ) { - docElem.appendChild( div ).id = expando; - return !doc.getElementsByName || !doc.getElementsByName( expando ).length; - }); - - // ID find and filter - if ( support.getById ) { - Expr.find["ID"] = function( id, context ) { - if ( typeof context.getElementById !== strundefined && documentIsHTML ) { - var m = context.getElementById( id ); - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - return m && m.parentNode ? [m] : []; - } - }; - Expr.filter["ID"] = function( id ) { - var attrId = id.replace( runescape, funescape ); - return function( elem ) { - return elem.getAttribute("id") === attrId; - }; - }; - } else { - // Support: IE6/7 - // getElementById is not reliable as a find shortcut - delete Expr.find["ID"]; - - Expr.filter["ID"] = function( id ) { - var attrId = id.replace( runescape, funescape ); - return function( elem ) { - var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id"); - return node && node.value === attrId; - }; - }; - } - - // Tag - Expr.find["TAG"] = support.getElementsByTagName ? - function( tag, context ) { - if ( typeof context.getElementsByTagName !== strundefined ) { - return context.getElementsByTagName( tag ); - } - } : - function( tag, context ) { - var elem, - tmp = [], - i = 0, - results = context.getElementsByTagName( tag ); - - // Filter out possible comments - if ( tag === "*" ) { - while ( (elem = results[i++]) ) { - if ( elem.nodeType === 1 ) { - tmp.push( elem ); - } - } - - return tmp; - } - return results; - }; - - // Class - Expr.find["CLASS"] = support.getElementsByClassName && function( className, context ) { - if ( typeof context.getElementsByClassName !== strundefined && documentIsHTML ) { - return context.getElementsByClassName( className ); - } - }; - - /* QSA/matchesSelector - ---------------------------------------------------------------------- */ - - // QSA and matchesSelector support - - // matchesSelector(:active) reports false when true (IE9/Opera 11.5) - rbuggyMatches = []; - - // qSa(:focus) reports false when true (Chrome 21) - // We allow this because of a bug in IE8/9 that throws an error - // whenever `document.activeElement` is accessed on an iframe - // So, we allow :focus to pass through QSA all the time to avoid the IE error - // See http://bugs.jquery.com/ticket/13378 - rbuggyQSA = []; - - if ( (support.qsa = rnative.test( doc.querySelectorAll )) ) { - // Build QSA regex - // Regex strategy adopted from Diego Perini - assert(function( div ) { - // Select is set to empty string on purpose - // This is to test IE's treatment of not explicitly - // setting a boolean content attribute, - // since its presence should be enough - // http://bugs.jquery.com/ticket/12359 - div.innerHTML = ""; - - // Support: IE8, Opera 10-12 - // Nothing should be selected when empty strings follow ^= or $= or *= - if ( div.querySelectorAll("[t^='']").length ) { - rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); - } - - // Support: IE8 - // Boolean attributes and "value" are not treated correctly - if ( !div.querySelectorAll("[selected]").length ) { - rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); - } - - // Webkit/Opera - :checked should return selected option elements - // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked - // IE8 throws error here and will not see later tests - if ( !div.querySelectorAll(":checked").length ) { - rbuggyQSA.push(":checked"); - } - }); - - assert(function( div ) { - // Support: Windows 8 Native Apps - // The type and name attributes are restricted during .innerHTML assignment - var input = doc.createElement("input"); - input.setAttribute( "type", "hidden" ); - div.appendChild( input ).setAttribute( "name", "D" ); - - // Support: IE8 - // Enforce case-sensitivity of name attribute - if ( div.querySelectorAll("[name=d]").length ) { - rbuggyQSA.push( "name" + whitespace + "*[*^$|!~]?=" ); - } - - // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) - // IE8 throws error here and will not see later tests - if ( !div.querySelectorAll(":enabled").length ) { - rbuggyQSA.push( ":enabled", ":disabled" ); - } - - // Opera 10-11 does not throw on post-comma invalid pseudos - div.querySelectorAll("*,:x"); - rbuggyQSA.push(",.*:"); - }); - } - - if ( (support.matchesSelector = rnative.test( (matches = docElem.webkitMatchesSelector || - docElem.mozMatchesSelector || - docElem.oMatchesSelector || - docElem.msMatchesSelector) )) ) { - - assert(function( div ) { - // Check to see if it's possible to do matchesSelector - // on a disconnected node (IE 9) - support.disconnectedMatch = matches.call( div, "div" ); - - // This should fail with an exception - // Gecko does not error, returns false instead - matches.call( div, "[s!='']:x" ); - rbuggyMatches.push( "!=", pseudos ); - }); - } - - rbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join("|") ); - rbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join("|") ); - - /* Contains - ---------------------------------------------------------------------- */ - hasCompare = rnative.test( docElem.compareDocumentPosition ); - - // Element contains another - // Purposefully does not implement inclusive descendent - // As in, an element does not contain itself - contains = hasCompare || rnative.test( docElem.contains ) ? - function( a, b ) { - var adown = a.nodeType === 9 ? a.documentElement : a, - bup = b && b.parentNode; - return a === bup || !!( bup && bup.nodeType === 1 && ( - adown.contains ? - adown.contains( bup ) : - a.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16 - )); - } : - function( a, b ) { - if ( b ) { - while ( (b = b.parentNode) ) { - if ( b === a ) { - return true; - } - } - } - return false; - }; - - /* Sorting - ---------------------------------------------------------------------- */ - - // Document order sorting - sortOrder = hasCompare ? - function( a, b ) { - - // Flag for duplicate removal - if ( a === b ) { - hasDuplicate = true; - return 0; - } - - // Sort on method existence if only one input has compareDocumentPosition - var compare = !a.compareDocumentPosition - !b.compareDocumentPosition; - if ( compare ) { - return compare; - } - - // Calculate position if both inputs belong to the same document - compare = ( a.ownerDocument || a ) === ( b.ownerDocument || b ) ? - a.compareDocumentPosition( b ) : - - // Otherwise we know they are disconnected - 1; - - // Disconnected nodes - if ( compare & 1 || - (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { - - // Choose the first element that is related to our preferred document - if ( a === doc || a.ownerDocument === preferredDoc && contains(preferredDoc, a) ) { - return -1; - } - if ( b === doc || b.ownerDocument === preferredDoc && contains(preferredDoc, b) ) { - return 1; - } - - // Maintain original order - return sortInput ? - ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : - 0; - } - - return compare & 4 ? -1 : 1; - } : - function( a, b ) { - // Exit early if the nodes are identical - if ( a === b ) { - hasDuplicate = true; - return 0; - } - - var cur, - i = 0, - aup = a.parentNode, - bup = b.parentNode, - ap = [ a ], - bp = [ b ]; - - // Parentless nodes are either documents or disconnected - if ( !aup || !bup ) { - return a === doc ? -1 : - b === doc ? 1 : - aup ? -1 : - bup ? 1 : - sortInput ? - ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : - 0; - - // If the nodes are siblings, we can do a quick check - } else if ( aup === bup ) { - return siblingCheck( a, b ); - } - - // Otherwise we need full lists of their ancestors for comparison - cur = a; - while ( (cur = cur.parentNode) ) { - ap.unshift( cur ); - } - cur = b; - while ( (cur = cur.parentNode) ) { - bp.unshift( cur ); - } - - // Walk down the tree looking for a discrepancy - while ( ap[i] === bp[i] ) { - i++; - } - - return i ? - // Do a sibling check if the nodes have a common ancestor - siblingCheck( ap[i], bp[i] ) : - - // Otherwise nodes in our document sort first - ap[i] === preferredDoc ? -1 : - bp[i] === preferredDoc ? 1 : - 0; - }; - - return doc; -}; - -Sizzle.matches = function( expr, elements ) { - return Sizzle( expr, null, null, elements ); -}; - -Sizzle.matchesSelector = function( elem, expr ) { - // Set document vars if needed - if ( ( elem.ownerDocument || elem ) !== document ) { - setDocument( elem ); - } - - // Make sure that attribute selectors are quoted - expr = expr.replace( rattributeQuotes, "='$1']" ); - - if ( support.matchesSelector && documentIsHTML && - ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && - ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { - - try { - var ret = matches.call( elem, expr ); - - // IE 9's matchesSelector returns false on disconnected nodes - if ( ret || support.disconnectedMatch || - // As well, disconnected nodes are said to be in a document - // fragment in IE 9 - elem.document && elem.document.nodeType !== 11 ) { - return ret; - } - } catch(e) {} - } - - return Sizzle( expr, document, null, [elem] ).length > 0; -}; - -Sizzle.contains = function( context, elem ) { - // Set document vars if needed - if ( ( context.ownerDocument || context ) !== document ) { - setDocument( context ); - } - return contains( context, elem ); -}; - -Sizzle.attr = function( elem, name ) { - // Set document vars if needed - if ( ( elem.ownerDocument || elem ) !== document ) { - setDocument( elem ); - } - - var fn = Expr.attrHandle[ name.toLowerCase() ], - // Don't get fooled by Object.prototype properties (jQuery #13807) - val = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ? - fn( elem, name, !documentIsHTML ) : - undefined; - - return val !== undefined ? - val : - support.attributes || !documentIsHTML ? - elem.getAttribute( name ) : - (val = elem.getAttributeNode(name)) && val.specified ? - val.value : - null; -}; - -Sizzle.error = function( msg ) { - throw new Error( "Syntax error, unrecognized expression: " + msg ); -}; - -/** - * Document sorting and removing duplicates - * @param {ArrayLike} results - */ -Sizzle.uniqueSort = function( results ) { - var elem, - duplicates = [], - j = 0, - i = 0; - - // Unless we *know* we can detect duplicates, assume their presence - hasDuplicate = !support.detectDuplicates; - sortInput = !support.sortStable && results.slice( 0 ); - results.sort( sortOrder ); - - if ( hasDuplicate ) { - while ( (elem = results[i++]) ) { - if ( elem === results[ i ] ) { - j = duplicates.push( i ); - } - } - while ( j-- ) { - results.splice( duplicates[ j ], 1 ); - } - } - - // Clear input after sorting to release objects - // See https://github.com/jquery/sizzle/pull/225 - sortInput = null; - - return results; -}; - -/** - * Utility function for retrieving the text value of an array of DOM nodes - * @param {Array|Element} elem - */ -getText = Sizzle.getText = function( elem ) { - var node, - ret = "", - i = 0, - nodeType = elem.nodeType; - - if ( !nodeType ) { - // If no nodeType, this is expected to be an array - while ( (node = elem[i++]) ) { - // Do not traverse comment nodes - ret += getText( node ); - } - } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { - // Use textContent for elements - // innerText usage removed for consistency of new lines (jQuery #11153) - if ( typeof elem.textContent === "string" ) { - return elem.textContent; - } else { - // Traverse its children - for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { - ret += getText( elem ); - } - } - } else if ( nodeType === 3 || nodeType === 4 ) { - return elem.nodeValue; - } - // Do not include comment or processing instruction nodes - - return ret; -}; - -Expr = Sizzle.selectors = { - - // Can be adjusted by the user - cacheLength: 50, - - createPseudo: markFunction, - - match: matchExpr, - - attrHandle: {}, - - find: {}, - - relative: { - ">": { dir: "parentNode", first: true }, - " ": { dir: "parentNode" }, - "+": { dir: "previousSibling", first: true }, - "~": { dir: "previousSibling" } - }, - - preFilter: { - "ATTR": function( match ) { - match[1] = match[1].replace( runescape, funescape ); - - // Move the given value to match[3] whether quoted or unquoted - match[3] = ( match[4] || match[5] || "" ).replace( runescape, funescape ); - - if ( match[2] === "~=" ) { - match[3] = " " + match[3] + " "; - } - - return match.slice( 0, 4 ); - }, - - "CHILD": function( match ) { - /* matches from matchExpr["CHILD"] - 1 type (only|nth|...) - 2 what (child|of-type) - 3 argument (even|odd|\d*|\d*n([+-]\d+)?|...) - 4 xn-component of xn+y argument ([+-]?\d*n|) - 5 sign of xn-component - 6 x of xn-component - 7 sign of y-component - 8 y of y-component - */ - match[1] = match[1].toLowerCase(); - - if ( match[1].slice( 0, 3 ) === "nth" ) { - // nth-* requires argument - if ( !match[3] ) { - Sizzle.error( match[0] ); - } - - // numeric x and y parameters for Expr.filter.CHILD - // remember that false/true cast respectively to 0/1 - match[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === "even" || match[3] === "odd" ) ); - match[5] = +( ( match[7] + match[8] ) || match[3] === "odd" ); - - // other types prohibit arguments - } else if ( match[3] ) { - Sizzle.error( match[0] ); - } - - return match; - }, - - "PSEUDO": function( match ) { - var excess, - unquoted = !match[5] && match[2]; - - if ( matchExpr["CHILD"].test( match[0] ) ) { - return null; - } - - // Accept quoted arguments as-is - if ( match[3] && match[4] !== undefined ) { - match[2] = match[4]; - - // Strip excess characters from unquoted arguments - } else if ( unquoted && rpseudo.test( unquoted ) && - // Get excess from tokenize (recursively) - (excess = tokenize( unquoted, true )) && - // advance to the next closing parenthesis - (excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) { - - // excess is a negative index - match[0] = match[0].slice( 0, excess ); - match[2] = unquoted.slice( 0, excess ); - } - - // Return only captures needed by the pseudo filter method (type and argument) - return match.slice( 0, 3 ); - } - }, - - filter: { - - "TAG": function( nodeNameSelector ) { - var nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase(); - return nodeNameSelector === "*" ? - function() { return true; } : - function( elem ) { - return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; - }; - }, - - "CLASS": function( className ) { - var pattern = classCache[ className + " " ]; - - return pattern || - (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && - classCache( className, function( elem ) { - return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== strundefined && elem.getAttribute("class") || "" ); - }); - }, - - "ATTR": function( name, operator, check ) { - return function( elem ) { - var result = Sizzle.attr( elem, name ); - - if ( result == null ) { - return operator === "!="; - } - if ( !operator ) { - return true; - } - - result += ""; - - return operator === "=" ? result === check : - operator === "!=" ? result !== check : - operator === "^=" ? check && result.indexOf( check ) === 0 : - operator === "*=" ? check && result.indexOf( check ) > -1 : - operator === "$=" ? check && result.slice( -check.length ) === check : - operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 : - operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : - false; - }; - }, - - "CHILD": function( type, what, argument, first, last ) { - var simple = type.slice( 0, 3 ) !== "nth", - forward = type.slice( -4 ) !== "last", - ofType = what === "of-type"; - - return first === 1 && last === 0 ? - - // Shortcut for :nth-*(n) - function( elem ) { - return !!elem.parentNode; - } : - - function( elem, context, xml ) { - var cache, outerCache, node, diff, nodeIndex, start, - dir = simple !== forward ? "nextSibling" : "previousSibling", - parent = elem.parentNode, - name = ofType && elem.nodeName.toLowerCase(), - useCache = !xml && !ofType; - - if ( parent ) { - - // :(first|last|only)-(child|of-type) - if ( simple ) { - while ( dir ) { - node = elem; - while ( (node = node[ dir ]) ) { - if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { - return false; - } - } - // Reverse direction for :only-* (if we haven't yet done so) - start = dir = type === "only" && !start && "nextSibling"; - } - return true; - } - - start = [ forward ? parent.firstChild : parent.lastChild ]; - - // non-xml :nth-child(...) stores cache data on `parent` - if ( forward && useCache ) { - // Seek `elem` from a previously-cached index - outerCache = parent[ expando ] || (parent[ expando ] = {}); - cache = outerCache[ type ] || []; - nodeIndex = cache[0] === dirruns && cache[1]; - diff = cache[0] === dirruns && cache[2]; - node = nodeIndex && parent.childNodes[ nodeIndex ]; - - while ( (node = ++nodeIndex && node && node[ dir ] || - - // Fallback to seeking `elem` from the start - (diff = nodeIndex = 0) || start.pop()) ) { - - // When found, cache indexes on `parent` and break - if ( node.nodeType === 1 && ++diff && node === elem ) { - outerCache[ type ] = [ dirruns, nodeIndex, diff ]; - break; - } - } - - // Use previously-cached element index if available - } else if ( useCache && (cache = (elem[ expando ] || (elem[ expando ] = {}))[ type ]) && cache[0] === dirruns ) { - diff = cache[1]; - - // xml :nth-child(...) or :nth-last-child(...) or :nth(-last)?-of-type(...) - } else { - // Use the same loop as above to seek `elem` from the start - while ( (node = ++nodeIndex && node && node[ dir ] || - (diff = nodeIndex = 0) || start.pop()) ) { - - if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { - // Cache the index of each encountered element - if ( useCache ) { - (node[ expando ] || (node[ expando ] = {}))[ type ] = [ dirruns, diff ]; - } - - if ( node === elem ) { - break; - } - } - } - } - - // Incorporate the offset, then check against cycle size - diff -= last; - return diff === first || ( diff % first === 0 && diff / first >= 0 ); - } - }; - }, - - "PSEUDO": function( pseudo, argument ) { - // pseudo-class names are case-insensitive - // http://www.w3.org/TR/selectors/#pseudo-classes - // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters - // Remember that setFilters inherits from pseudos - var args, - fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || - Sizzle.error( "unsupported pseudo: " + pseudo ); - - // The user may use createPseudo to indicate that - // arguments are needed to create the filter function - // just as Sizzle does - if ( fn[ expando ] ) { - return fn( argument ); - } - - // But maintain support for old signatures - if ( fn.length > 1 ) { - args = [ pseudo, pseudo, "", argument ]; - return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? - markFunction(function( seed, matches ) { - var idx, - matched = fn( seed, argument ), - i = matched.length; - while ( i-- ) { - idx = indexOf.call( seed, matched[i] ); - seed[ idx ] = !( matches[ idx ] = matched[i] ); - } - }) : - function( elem ) { - return fn( elem, 0, args ); - }; - } - - return fn; - } - }, - - pseudos: { - // Potentially complex pseudos - "not": markFunction(function( selector ) { - // Trim the selector passed to compile - // to avoid treating leading and trailing - // spaces as combinators - var input = [], - results = [], - matcher = compile( selector.replace( rtrim, "$1" ) ); - - return matcher[ expando ] ? - markFunction(function( seed, matches, context, xml ) { - var elem, - unmatched = matcher( seed, null, xml, [] ), - i = seed.length; - - // Match elements unmatched by `matcher` - while ( i-- ) { - if ( (elem = unmatched[i]) ) { - seed[i] = !(matches[i] = elem); - } - } - }) : - function( elem, context, xml ) { - input[0] = elem; - matcher( input, null, xml, results ); - return !results.pop(); - }; - }), - - "has": markFunction(function( selector ) { - return function( elem ) { - return Sizzle( selector, elem ).length > 0; - }; - }), - - "contains": markFunction(function( text ) { - return function( elem ) { - return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; - }; - }), - - // "Whether an element is represented by a :lang() selector - // is based solely on the element's language value - // being equal to the identifier C, - // or beginning with the identifier C immediately followed by "-". - // The matching of C against the element's language value is performed case-insensitively. - // The identifier C does not have to be a valid language name." - // http://www.w3.org/TR/selectors/#lang-pseudo - "lang": markFunction( function( lang ) { - // lang value must be a valid identifier - if ( !ridentifier.test(lang || "") ) { - Sizzle.error( "unsupported lang: " + lang ); - } - lang = lang.replace( runescape, funescape ).toLowerCase(); - return function( elem ) { - var elemLang; - do { - if ( (elemLang = documentIsHTML ? - elem.lang : - elem.getAttribute("xml:lang") || elem.getAttribute("lang")) ) { - - elemLang = elemLang.toLowerCase(); - return elemLang === lang || elemLang.indexOf( lang + "-" ) === 0; - } - } while ( (elem = elem.parentNode) && elem.nodeType === 1 ); - return false; - }; - }), - - // Miscellaneous - "target": function( elem ) { - var hash = window.location && window.location.hash; - return hash && hash.slice( 1 ) === elem.id; - }, - - "root": function( elem ) { - return elem === docElem; - }, - - "focus": function( elem ) { - return elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex); - }, - - // Boolean properties - "enabled": function( elem ) { - return elem.disabled === false; - }, - - "disabled": function( elem ) { - return elem.disabled === true; - }, - - "checked": function( elem ) { - // In CSS3, :checked should return both checked and selected elements - // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked - var nodeName = elem.nodeName.toLowerCase(); - return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected); - }, - - "selected": function( elem ) { - // Accessing this property makes selected-by-default - // options in Safari work properly - if ( elem.parentNode ) { - elem.parentNode.selectedIndex; - } - - return elem.selected === true; - }, - - // Contents - "empty": function( elem ) { - // http://www.w3.org/TR/selectors/#empty-pseudo - // :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5), - // but not by others (comment: 8; processing instruction: 7; etc.) - // nodeType < 6 works because attributes (2) do not appear as children - for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { - if ( elem.nodeType < 6 ) { - return false; - } - } - return true; - }, - - "parent": function( elem ) { - return !Expr.pseudos["empty"]( elem ); - }, - - // Element/input types - "header": function( elem ) { - return rheader.test( elem.nodeName ); - }, - - "input": function( elem ) { - return rinputs.test( elem.nodeName ); - }, - - "button": function( elem ) { - var name = elem.nodeName.toLowerCase(); - return name === "input" && elem.type === "button" || name === "button"; - }, - - "text": function( elem ) { - var attr; - return elem.nodeName.toLowerCase() === "input" && - elem.type === "text" && - - // Support: IE<8 - // New HTML5 attribute values (e.g., "search") appear with elem.type === "text" - ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === "text" ); - }, - - // Position-in-collection - "first": createPositionalPseudo(function() { - return [ 0 ]; - }), - - "last": createPositionalPseudo(function( matchIndexes, length ) { - return [ length - 1 ]; - }), - - "eq": createPositionalPseudo(function( matchIndexes, length, argument ) { - return [ argument < 0 ? argument + length : argument ]; - }), - - "even": createPositionalPseudo(function( matchIndexes, length ) { - var i = 0; - for ( ; i < length; i += 2 ) { - matchIndexes.push( i ); - } - return matchIndexes; - }), - - "odd": createPositionalPseudo(function( matchIndexes, length ) { - var i = 1; - for ( ; i < length; i += 2 ) { - matchIndexes.push( i ); - } - return matchIndexes; - }), - - "lt": createPositionalPseudo(function( matchIndexes, length, argument ) { - var i = argument < 0 ? argument + length : argument; - for ( ; --i >= 0; ) { - matchIndexes.push( i ); - } - return matchIndexes; - }), - - "gt": createPositionalPseudo(function( matchIndexes, length, argument ) { - var i = argument < 0 ? argument + length : argument; - for ( ; ++i < length; ) { - matchIndexes.push( i ); - } - return matchIndexes; - }) - } -}; - -Expr.pseudos["nth"] = Expr.pseudos["eq"]; - -// Add button/input type pseudos -for ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) { - Expr.pseudos[ i ] = createInputPseudo( i ); -} -for ( i in { submit: true, reset: true } ) { - Expr.pseudos[ i ] = createButtonPseudo( i ); -} - -// Easy API for creating new setFilters -function setFilters() {} -setFilters.prototype = Expr.filters = Expr.pseudos; -Expr.setFilters = new setFilters(); - -function tokenize( selector, parseOnly ) { - var matched, match, tokens, type, - soFar, groups, preFilters, - cached = tokenCache[ selector + " " ]; - - if ( cached ) { - return parseOnly ? 0 : cached.slice( 0 ); - } - - soFar = selector; - groups = []; - preFilters = Expr.preFilter; - - while ( soFar ) { - - // Comma and first run - if ( !matched || (match = rcomma.exec( soFar )) ) { - if ( match ) { - // Don't consume trailing commas as valid - soFar = soFar.slice( match[0].length ) || soFar; - } - groups.push( (tokens = []) ); - } - - matched = false; - - // Combinators - if ( (match = rcombinators.exec( soFar )) ) { - matched = match.shift(); - tokens.push({ - value: matched, - // Cast descendant combinators to space - type: match[0].replace( rtrim, " " ) - }); - soFar = soFar.slice( matched.length ); - } - - // Filters - for ( type in Expr.filter ) { - if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] || - (match = preFilters[ type ]( match ))) ) { - matched = match.shift(); - tokens.push({ - value: matched, - type: type, - matches: match - }); - soFar = soFar.slice( matched.length ); - } - } - - if ( !matched ) { - break; - } - } - - // Return the length of the invalid excess - // if we're just parsing - // Otherwise, throw an error or return tokens - return parseOnly ? - soFar.length : - soFar ? - Sizzle.error( selector ) : - // Cache the tokens - tokenCache( selector, groups ).slice( 0 ); -} - -function toSelector( tokens ) { - var i = 0, - len = tokens.length, - selector = ""; - for ( ; i < len; i++ ) { - selector += tokens[i].value; - } - return selector; -} - -function addCombinator( matcher, combinator, base ) { - var dir = combinator.dir, - checkNonElements = base && dir === "parentNode", - doneName = done++; - - return combinator.first ? - // Check against closest ancestor/preceding element - function( elem, context, xml ) { - while ( (elem = elem[ dir ]) ) { - if ( elem.nodeType === 1 || checkNonElements ) { - return matcher( elem, context, xml ); - } - } - } : - - // Check against all ancestor/preceding elements - function( elem, context, xml ) { - var oldCache, outerCache, - newCache = [ dirruns, doneName ]; - - // We can't set arbitrary data on XML nodes, so they don't benefit from dir caching - if ( xml ) { - while ( (elem = elem[ dir ]) ) { - if ( elem.nodeType === 1 || checkNonElements ) { - if ( matcher( elem, context, xml ) ) { - return true; - } - } - } - } else { - while ( (elem = elem[ dir ]) ) { - if ( elem.nodeType === 1 || checkNonElements ) { - outerCache = elem[ expando ] || (elem[ expando ] = {}); - if ( (oldCache = outerCache[ dir ]) && - oldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) { - - // Assign to newCache so results back-propagate to previous elements - return (newCache[ 2 ] = oldCache[ 2 ]); - } else { - // Reuse newcache so results back-propagate to previous elements - outerCache[ dir ] = newCache; - - // A match means we're done; a fail means we have to keep checking - if ( (newCache[ 2 ] = matcher( elem, context, xml )) ) { - return true; - } - } - } - } - } - }; -} - -function elementMatcher( matchers ) { - return matchers.length > 1 ? - function( elem, context, xml ) { - var i = matchers.length; - while ( i-- ) { - if ( !matchers[i]( elem, context, xml ) ) { - return false; - } - } - return true; - } : - matchers[0]; -} - -function condense( unmatched, map, filter, context, xml ) { - var elem, - newUnmatched = [], - i = 0, - len = unmatched.length, - mapped = map != null; - - for ( ; i < len; i++ ) { - if ( (elem = unmatched[i]) ) { - if ( !filter || filter( elem, context, xml ) ) { - newUnmatched.push( elem ); - if ( mapped ) { - map.push( i ); - } - } - } - } - - return newUnmatched; -} - -function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { - if ( postFilter && !postFilter[ expando ] ) { - postFilter = setMatcher( postFilter ); - } - if ( postFinder && !postFinder[ expando ] ) { - postFinder = setMatcher( postFinder, postSelector ); - } - return markFunction(function( seed, results, context, xml ) { - var temp, i, elem, - preMap = [], - postMap = [], - preexisting = results.length, - - // Get initial elements from seed or context - elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), - - // Prefilter to get matcher input, preserving a map for seed-results synchronization - matcherIn = preFilter && ( seed || !selector ) ? - condense( elems, preMap, preFilter, context, xml ) : - elems, - - matcherOut = matcher ? - // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, - postFinder || ( seed ? preFilter : preexisting || postFilter ) ? - - // ...intermediate processing is necessary - [] : - - // ...otherwise use results directly - results : - matcherIn; - - // Find primary matches - if ( matcher ) { - matcher( matcherIn, matcherOut, context, xml ); - } - - // Apply postFilter - if ( postFilter ) { - temp = condense( matcherOut, postMap ); - postFilter( temp, [], context, xml ); - - // Un-match failing elements by moving them back to matcherIn - i = temp.length; - while ( i-- ) { - if ( (elem = temp[i]) ) { - matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem); - } - } - } - - if ( seed ) { - if ( postFinder || preFilter ) { - if ( postFinder ) { - // Get the final matcherOut by condensing this intermediate into postFinder contexts - temp = []; - i = matcherOut.length; - while ( i-- ) { - if ( (elem = matcherOut[i]) ) { - // Restore matcherIn since elem is not yet a final match - temp.push( (matcherIn[i] = elem) ); - } - } - postFinder( null, (matcherOut = []), temp, xml ); - } - - // Move matched elements from seed to results to keep them synchronized - i = matcherOut.length; - while ( i-- ) { - if ( (elem = matcherOut[i]) && - (temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) { - - seed[temp] = !(results[temp] = elem); - } - } - } - - // Add elements to results, through postFinder if defined - } else { - matcherOut = condense( - matcherOut === results ? - matcherOut.splice( preexisting, matcherOut.length ) : - matcherOut - ); - if ( postFinder ) { - postFinder( null, results, matcherOut, xml ); - } else { - push.apply( results, matcherOut ); - } - } - }); -} - -function matcherFromTokens( tokens ) { - var checkContext, matcher, j, - len = tokens.length, - leadingRelative = Expr.relative[ tokens[0].type ], - implicitRelative = leadingRelative || Expr.relative[" "], - i = leadingRelative ? 1 : 0, - - // The foundational matcher ensures that elements are reachable from top-level context(s) - matchContext = addCombinator( function( elem ) { - return elem === checkContext; - }, implicitRelative, true ), - matchAnyContext = addCombinator( function( elem ) { - return indexOf.call( checkContext, elem ) > -1; - }, implicitRelative, true ), - matchers = [ function( elem, context, xml ) { - return ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( - (checkContext = context).nodeType ? - matchContext( elem, context, xml ) : - matchAnyContext( elem, context, xml ) ); - } ]; - - for ( ; i < len; i++ ) { - if ( (matcher = Expr.relative[ tokens[i].type ]) ) { - matchers = [ addCombinator(elementMatcher( matchers ), matcher) ]; - } else { - matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches ); - - // Return special upon seeing a positional matcher - if ( matcher[ expando ] ) { - // Find the next relative operator (if any) for proper handling - j = ++i; - for ( ; j < len; j++ ) { - if ( Expr.relative[ tokens[j].type ] ) { - break; - } - } - return setMatcher( - i > 1 && elementMatcher( matchers ), - i > 1 && toSelector( - // If the preceding token was a descendant combinator, insert an implicit any-element `*` - tokens.slice( 0, i - 1 ).concat({ value: tokens[ i - 2 ].type === " " ? "*" : "" }) - ).replace( rtrim, "$1" ), - matcher, - i < j && matcherFromTokens( tokens.slice( i, j ) ), - j < len && matcherFromTokens( (tokens = tokens.slice( j )) ), - j < len && toSelector( tokens ) - ); - } - matchers.push( matcher ); - } - } - - return elementMatcher( matchers ); -} - -function matcherFromGroupMatchers( elementMatchers, setMatchers ) { - var bySet = setMatchers.length > 0, - byElement = elementMatchers.length > 0, - superMatcher = function( seed, context, xml, results, outermost ) { - var elem, j, matcher, - matchedCount = 0, - i = "0", - unmatched = seed && [], - setMatched = [], - contextBackup = outermostContext, - // We must always have either seed elements or outermost context - elems = seed || byElement && Expr.find["TAG"]( "*", outermost ), - // Use integer dirruns iff this is the outermost matcher - dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1), - len = elems.length; - - if ( outermost ) { - outermostContext = context !== document && context; - } - - // Add elements passing elementMatchers directly to results - // Keep `i` a string if there are no elements so `matchedCount` will be "00" below - // Support: IE<9, Safari - // Tolerate NodeList properties (IE: "length"; Safari: ) matching elements by id - for ( ; i !== len && (elem = elems[i]) != null; i++ ) { - if ( byElement && elem ) { - j = 0; - while ( (matcher = elementMatchers[j++]) ) { - if ( matcher( elem, context, xml ) ) { - results.push( elem ); - break; - } - } - if ( outermost ) { - dirruns = dirrunsUnique; - } - } - - // Track unmatched elements for set filters - if ( bySet ) { - // They will have gone through all possible matchers - if ( (elem = !matcher && elem) ) { - matchedCount--; - } - - // Lengthen the array for every element, matched or not - if ( seed ) { - unmatched.push( elem ); - } - } - } - - // Apply set filters to unmatched elements - matchedCount += i; - if ( bySet && i !== matchedCount ) { - j = 0; - while ( (matcher = setMatchers[j++]) ) { - matcher( unmatched, setMatched, context, xml ); - } - - if ( seed ) { - // Reintegrate element matches to eliminate the need for sorting - if ( matchedCount > 0 ) { - while ( i-- ) { - if ( !(unmatched[i] || setMatched[i]) ) { - setMatched[i] = pop.call( results ); - } - } - } - - // Discard index placeholder values to get only actual matches - setMatched = condense( setMatched ); - } - - // Add matches to results - push.apply( results, setMatched ); - - // Seedless set matches succeeding multiple successful matchers stipulate sorting - if ( outermost && !seed && setMatched.length > 0 && - ( matchedCount + setMatchers.length ) > 1 ) { - - Sizzle.uniqueSort( results ); - } - } - - // Override manipulation of globals by nested matchers - if ( outermost ) { - dirruns = dirrunsUnique; - outermostContext = contextBackup; - } - - return unmatched; - }; - - return bySet ? - markFunction( superMatcher ) : - superMatcher; -} - -compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { - var i, - setMatchers = [], - elementMatchers = [], - cached = compilerCache[ selector + " " ]; - - if ( !cached ) { - // Generate a function of recursive functions that can be used to check each element - if ( !group ) { - group = tokenize( selector ); - } - i = group.length; - while ( i-- ) { - cached = matcherFromTokens( group[i] ); - if ( cached[ expando ] ) { - setMatchers.push( cached ); - } else { - elementMatchers.push( cached ); - } - } - - // Cache the compiled function - cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); - } - return cached; -}; - -function multipleContexts( selector, contexts, results ) { - var i = 0, - len = contexts.length; - for ( ; i < len; i++ ) { - Sizzle( selector, contexts[i], results ); - } - return results; -} - -function select( selector, context, results, seed ) { - var i, tokens, token, type, find, - match = tokenize( selector ); - - if ( !seed ) { - // Try to minimize operations if there is only one group - if ( match.length === 1 ) { - - // Take a shortcut and set the context if the root selector is an ID - tokens = match[0] = match[0].slice( 0 ); - if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && - support.getById && context.nodeType === 9 && documentIsHTML && - Expr.relative[ tokens[1].type ] ) { - - context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; - if ( !context ) { - return results; - } - selector = selector.slice( tokens.shift().value.length ); - } - - // Fetch a seed set for right-to-left matching - i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; - while ( i-- ) { - token = tokens[i]; - - // Abort if we hit a combinator - if ( Expr.relative[ (type = token.type) ] ) { - break; - } - if ( (find = Expr.find[ type ]) ) { - // Search, expanding context for leading sibling combinators - if ( (seed = find( - token.matches[0].replace( runescape, funescape ), - rsibling.test( tokens[0].type ) && testContext( context.parentNode ) || context - )) ) { - - // If seed is empty or no tokens remain, we can return early - tokens.splice( i, 1 ); - selector = seed.length && toSelector( tokens ); - if ( !selector ) { - push.apply( results, seed ); - return results; - } - - break; - } - } - } - } - } - - // Compile and execute a filtering function - // Provide `match` to avoid retokenization if we modified the selector above - compile( selector, match )( - seed, - context, - !documentIsHTML, - results, - rsibling.test( selector ) && testContext( context.parentNode ) || context - ); - return results; -} - -// One-time assignments - -// Sort stability -support.sortStable = expando.split("").sort( sortOrder ).join("") === expando; - -// Support: Chrome<14 -// Always assume duplicates if they aren't passed to the comparison function -support.detectDuplicates = !!hasDuplicate; - -// Initialize against the default document -setDocument(); - -// Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) -// Detached nodes confoundingly follow *each other* -support.sortDetached = assert(function( div1 ) { - // Should return 1, but returns 4 (following) - return div1.compareDocumentPosition( document.createElement("div") ) & 1; -}); - -// Support: IE<8 -// Prevent attribute/property "interpolation" -// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx -if ( !assert(function( div ) { - div.innerHTML = ""; - return div.firstChild.getAttribute("href") === "#" ; -}) ) { - addHandle( "type|href|height|width", function( elem, name, isXML ) { - if ( !isXML ) { - return elem.getAttribute( name, name.toLowerCase() === "type" ? 1 : 2 ); - } - }); -} - -// Support: IE<9 -// Use defaultValue in place of getAttribute("value") -if ( !support.attributes || !assert(function( div ) { - div.innerHTML = ""; - div.firstChild.setAttribute( "value", "" ); - return div.firstChild.getAttribute( "value" ) === ""; -}) ) { - addHandle( "value", function( elem, name, isXML ) { - if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { - return elem.defaultValue; - } - }); -} - -// Support: IE<9 -// Use getAttributeNode to fetch booleans when getAttribute lies -if ( !assert(function( div ) { - return div.getAttribute("disabled") == null; -}) ) { - addHandle( booleans, function( elem, name, isXML ) { - var val; - if ( !isXML ) { - return elem[ name ] === true ? name.toLowerCase() : - (val = elem.getAttributeNode( name )) && val.specified ? - val.value : - null; - } - }); -} - -return Sizzle; - -})( window ); - - - -jQuery.find = Sizzle; -jQuery.expr = Sizzle.selectors; -jQuery.expr[":"] = jQuery.expr.pseudos; -jQuery.unique = Sizzle.uniqueSort; -jQuery.text = Sizzle.getText; -jQuery.isXMLDoc = Sizzle.isXML; -jQuery.contains = Sizzle.contains; - - - -var rneedsContext = jQuery.expr.match.needsContext; - -var rsingleTag = (/^<(\w+)\s*\/?>(?:<\/\1>|)$/); - - - -var risSimple = /^.[^:#\[\.,]*$/; - -// Implement the identical functionality for filter and not -function winnow( elements, qualifier, not ) { - if ( jQuery.isFunction( qualifier ) ) { - return jQuery.grep( elements, function( elem, i ) { - /* jshint -W018 */ - return !!qualifier.call( elem, i, elem ) !== not; - }); - - } - - if ( qualifier.nodeType ) { - return jQuery.grep( elements, function( elem ) { - return ( elem === qualifier ) !== not; - }); - - } - - if ( typeof qualifier === "string" ) { - if ( risSimple.test( qualifier ) ) { - return jQuery.filter( qualifier, elements, not ); - } - - qualifier = jQuery.filter( qualifier, elements ); - } - - return jQuery.grep( elements, function( elem ) { - return ( jQuery.inArray( elem, qualifier ) >= 0 ) !== not; - }); -} - -jQuery.filter = function( expr, elems, not ) { - var elem = elems[ 0 ]; - - if ( not ) { - expr = ":not(" + expr + ")"; - } - - return elems.length === 1 && elem.nodeType === 1 ? - jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] : - jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { - return elem.nodeType === 1; - })); -}; - -jQuery.fn.extend({ - find: function( selector ) { - var i, - ret = [], - self = this, - len = self.length; - - if ( typeof selector !== "string" ) { - return this.pushStack( jQuery( selector ).filter(function() { - for ( i = 0; i < len; i++ ) { - if ( jQuery.contains( self[ i ], this ) ) { - return true; - } - } - }) ); - } - - for ( i = 0; i < len; i++ ) { - jQuery.find( selector, self[ i ], ret ); - } - - // Needed because $( selector, context ) becomes $( context ).find( selector ) - ret = this.pushStack( len > 1 ? jQuery.unique( ret ) : ret ); - ret.selector = this.selector ? this.selector + " " + selector : selector; - return ret; - }, - filter: function( selector ) { - return this.pushStack( winnow(this, selector || [], false) ); - }, - not: function( selector ) { - return this.pushStack( winnow(this, selector || [], true) ); - }, - is: function( selector ) { - return !!winnow( - this, - - // If this is a positional/relative selector, check membership in the returned set - // so $("p:first").is("p:last") won't return true for a doc with two "p". - typeof selector === "string" && rneedsContext.test( selector ) ? - jQuery( selector ) : - selector || [], - false - ).length; - } -}); - - -// Initialize a jQuery object - - -// A central reference to the root jQuery(document) -var rootjQuery, - - // Use the correct document accordingly with window argument (sandbox) - document = window.document, - - // A simple way to check for HTML strings - // Prioritize #id over to avoid XSS via location.hash (#9521) - // Strict HTML recognition (#11290: must start with <) - rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/, - - init = jQuery.fn.init = function( selector, context ) { - var match, elem; - - // HANDLE: $(""), $(null), $(undefined), $(false) - if ( !selector ) { - return this; - } - - // Handle HTML strings - if ( typeof selector === "string" ) { - if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { - // Assume that strings that start and end with <> are HTML and skip the regex check - match = [ null, selector, null ]; - - } else { - match = rquickExpr.exec( selector ); - } - - // Match html or make sure no context is specified for #id - if ( match && (match[1] || !context) ) { - - // HANDLE: $(html) -> $(array) - if ( match[1] ) { - context = context instanceof jQuery ? context[0] : context; - - // scripts is true for back-compat - // Intentionally let the error be thrown if parseHTML is not present - jQuery.merge( this, jQuery.parseHTML( - match[1], - context && context.nodeType ? context.ownerDocument || context : document, - true - ) ); - - // HANDLE: $(html, props) - if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) { - for ( match in context ) { - // Properties of context are called as methods if possible - if ( jQuery.isFunction( this[ match ] ) ) { - this[ match ]( context[ match ] ); - - // ...and otherwise set as attributes - } else { - this.attr( match, context[ match ] ); - } - } - } - - return this; - - // HANDLE: $(#id) - } else { - elem = document.getElementById( match[2] ); - - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - if ( elem && elem.parentNode ) { - // Handle the case where IE and Opera return items - // by name instead of ID - if ( elem.id !== match[2] ) { - return rootjQuery.find( selector ); - } - - // Otherwise, we inject the element directly into the jQuery object - this.length = 1; - this[0] = elem; - } - - this.context = document; - this.selector = selector; - return this; - } - - // HANDLE: $(expr, $(...)) - } else if ( !context || context.jquery ) { - return ( context || rootjQuery ).find( selector ); - - // HANDLE: $(expr, context) - // (which is just equivalent to: $(context).find(expr) - } else { - return this.constructor( context ).find( selector ); - } - - // HANDLE: $(DOMElement) - } else if ( selector.nodeType ) { - this.context = this[0] = selector; - this.length = 1; - return this; - - // HANDLE: $(function) - // Shortcut for document ready - } else if ( jQuery.isFunction( selector ) ) { - return typeof rootjQuery.ready !== "undefined" ? - rootjQuery.ready( selector ) : - // Execute immediately if ready is not present - selector( jQuery ); - } - - if ( selector.selector !== undefined ) { - this.selector = selector.selector; - this.context = selector.context; - } - - return jQuery.makeArray( selector, this ); - }; - -// Give the init function the jQuery prototype for later instantiation -init.prototype = jQuery.fn; - -// Initialize central reference -rootjQuery = jQuery( document ); - - -var rparentsprev = /^(?:parents|prev(?:Until|All))/, - // methods guaranteed to produce a unique set when starting from a unique set - guaranteedUnique = { - children: true, - contents: true, - next: true, - prev: true - }; - -jQuery.extend({ - dir: function( elem, dir, until ) { - var matched = [], - cur = elem[ dir ]; - - while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { - if ( cur.nodeType === 1 ) { - matched.push( cur ); - } - cur = cur[dir]; - } - return matched; - }, - - sibling: function( n, elem ) { - var r = []; - - for ( ; n; n = n.nextSibling ) { - if ( n.nodeType === 1 && n !== elem ) { - r.push( n ); - } - } - - return r; - } -}); - -jQuery.fn.extend({ - has: function( target ) { - var i, - targets = jQuery( target, this ), - len = targets.length; - - return this.filter(function() { - for ( i = 0; i < len; i++ ) { - if ( jQuery.contains( this, targets[i] ) ) { - return true; - } - } - }); - }, - - closest: function( selectors, context ) { - var cur, - i = 0, - l = this.length, - matched = [], - pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? - jQuery( selectors, context || this.context ) : - 0; - - for ( ; i < l; i++ ) { - for ( cur = this[i]; cur && cur !== context; cur = cur.parentNode ) { - // Always skip document fragments - if ( cur.nodeType < 11 && (pos ? - pos.index(cur) > -1 : - - // Don't pass non-elements to Sizzle - cur.nodeType === 1 && - jQuery.find.matchesSelector(cur, selectors)) ) { - - matched.push( cur ); - break; - } - } - } - - return this.pushStack( matched.length > 1 ? jQuery.unique( matched ) : matched ); - }, - - // Determine the position of an element within - // the matched set of elements - index: function( elem ) { - - // No argument, return index in parent - if ( !elem ) { - return ( this[0] && this[0].parentNode ) ? this.first().prevAll().length : -1; - } - - // index in selector - if ( typeof elem === "string" ) { - return jQuery.inArray( this[0], jQuery( elem ) ); - } - - // Locate the position of the desired element - return jQuery.inArray( - // If it receives a jQuery object, the first element is used - elem.jquery ? elem[0] : elem, this ); - }, - - add: function( selector, context ) { - return this.pushStack( - jQuery.unique( - jQuery.merge( this.get(), jQuery( selector, context ) ) - ) - ); - }, - - addBack: function( selector ) { - return this.add( selector == null ? - this.prevObject : this.prevObject.filter(selector) - ); - } -}); - -function sibling( cur, dir ) { - do { - cur = cur[ dir ]; - } while ( cur && cur.nodeType !== 1 ); - - return cur; -} - -jQuery.each({ - parent: function( elem ) { - var parent = elem.parentNode; - return parent && parent.nodeType !== 11 ? parent : null; - }, - parents: function( elem ) { - return jQuery.dir( elem, "parentNode" ); - }, - parentsUntil: function( elem, i, until ) { - return jQuery.dir( elem, "parentNode", until ); - }, - next: function( elem ) { - return sibling( elem, "nextSibling" ); - }, - prev: function( elem ) { - return sibling( elem, "previousSibling" ); - }, - nextAll: function( elem ) { - return jQuery.dir( elem, "nextSibling" ); - }, - prevAll: function( elem ) { - return jQuery.dir( elem, "previousSibling" ); - }, - nextUntil: function( elem, i, until ) { - return jQuery.dir( elem, "nextSibling", until ); - }, - prevUntil: function( elem, i, until ) { - return jQuery.dir( elem, "previousSibling", until ); - }, - siblings: function( elem ) { - return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem ); - }, - children: function( elem ) { - return jQuery.sibling( elem.firstChild ); - }, - contents: function( elem ) { - return jQuery.nodeName( elem, "iframe" ) ? - elem.contentDocument || elem.contentWindow.document : - jQuery.merge( [], elem.childNodes ); - } -}, function( name, fn ) { - jQuery.fn[ name ] = function( until, selector ) { - var ret = jQuery.map( this, fn, until ); - - if ( name.slice( -5 ) !== "Until" ) { - selector = until; - } - - if ( selector && typeof selector === "string" ) { - ret = jQuery.filter( selector, ret ); - } - - if ( this.length > 1 ) { - // Remove duplicates - if ( !guaranteedUnique[ name ] ) { - ret = jQuery.unique( ret ); - } - - // Reverse order for parents* and prev-derivatives - if ( rparentsprev.test( name ) ) { - ret = ret.reverse(); - } - } - - return this.pushStack( ret ); - }; -}); -var rnotwhite = (/\S+/g); - - - -// String to Object options format cache -var optionsCache = {}; - -// Convert String-formatted options into Object-formatted ones and store in cache -function createOptions( options ) { - var object = optionsCache[ options ] = {}; - jQuery.each( options.match( rnotwhite ) || [], function( _, flag ) { - object[ flag ] = true; - }); - return object; -} - -/* - * Create a callback list using the following parameters: - * - * options: an optional list of space-separated options that will change how - * the callback list behaves or a more traditional option object - * - * By default a callback list will act like an event callback list and can be - * "fired" multiple times. - * - * Possible options: - * - * once: will ensure the callback list can only be fired once (like a Deferred) - * - * memory: will keep track of previous values and will call any callback added - * after the list has been fired right away with the latest "memorized" - * values (like a Deferred) - * - * unique: will ensure a callback can only be added once (no duplicate in the list) - * - * stopOnFalse: interrupt callings when a callback returns false - * - */ -jQuery.Callbacks = function( options ) { - - // Convert options from String-formatted to Object-formatted if needed - // (we check in cache first) - options = typeof options === "string" ? - ( optionsCache[ options ] || createOptions( options ) ) : - jQuery.extend( {}, options ); - - var // Flag to know if list is currently firing - firing, - // Last fire value (for non-forgettable lists) - memory, - // Flag to know if list was already fired - fired, - // End of the loop when firing - firingLength, - // Index of currently firing callback (modified by remove if needed) - firingIndex, - // First callback to fire (used internally by add and fireWith) - firingStart, - // Actual callback list - list = [], - // Stack of fire calls for repeatable lists - stack = !options.once && [], - // Fire callbacks - fire = function( data ) { - memory = options.memory && data; - fired = true; - firingIndex = firingStart || 0; - firingStart = 0; - firingLength = list.length; - firing = true; - for ( ; list && firingIndex < firingLength; firingIndex++ ) { - if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) { - memory = false; // To prevent further calls using add - break; - } - } - firing = false; - if ( list ) { - if ( stack ) { - if ( stack.length ) { - fire( stack.shift() ); - } - } else if ( memory ) { - list = []; - } else { - self.disable(); - } - } - }, - // Actual Callbacks object - self = { - // Add a callback or a collection of callbacks to the list - add: function() { - if ( list ) { - // First, we save the current length - var start = list.length; - (function add( args ) { - jQuery.each( args, function( _, arg ) { - var type = jQuery.type( arg ); - if ( type === "function" ) { - if ( !options.unique || !self.has( arg ) ) { - list.push( arg ); - } - } else if ( arg && arg.length && type !== "string" ) { - // Inspect recursively - add( arg ); - } - }); - })( arguments ); - // Do we need to add the callbacks to the - // current firing batch? - if ( firing ) { - firingLength = list.length; - // With memory, if we're not firing then - // we should call right away - } else if ( memory ) { - firingStart = start; - fire( memory ); - } - } - return this; - }, - // Remove a callback from the list - remove: function() { - if ( list ) { - jQuery.each( arguments, function( _, arg ) { - var index; - while ( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { - list.splice( index, 1 ); - // Handle firing indexes - if ( firing ) { - if ( index <= firingLength ) { - firingLength--; - } - if ( index <= firingIndex ) { - firingIndex--; - } - } - } - }); - } - return this; - }, - // Check if a given callback is in the list. - // If no argument is given, return whether or not list has callbacks attached. - has: function( fn ) { - return fn ? jQuery.inArray( fn, list ) > -1 : !!( list && list.length ); - }, - // Remove all callbacks from the list - empty: function() { - list = []; - firingLength = 0; - return this; - }, - // Have the list do nothing anymore - disable: function() { - list = stack = memory = undefined; - return this; - }, - // Is it disabled? - disabled: function() { - return !list; - }, - // Lock the list in its current state - lock: function() { - stack = undefined; - if ( !memory ) { - self.disable(); - } - return this; - }, - // Is it locked? - locked: function() { - return !stack; - }, - // Call all callbacks with the given context and arguments - fireWith: function( context, args ) { - if ( list && ( !fired || stack ) ) { - args = args || []; - args = [ context, args.slice ? args.slice() : args ]; - if ( firing ) { - stack.push( args ); - } else { - fire( args ); - } - } - return this; - }, - // Call all the callbacks with the given arguments - fire: function() { - self.fireWith( this, arguments ); - return this; - }, - // To know if the callbacks have already been called at least once - fired: function() { - return !!fired; - } - }; - - return self; -}; - - -jQuery.extend({ - - Deferred: function( func ) { - var tuples = [ - // action, add listener, listener list, final state - [ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ], - [ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ], - [ "notify", "progress", jQuery.Callbacks("memory") ] - ], - state = "pending", - promise = { - state: function() { - return state; - }, - always: function() { - deferred.done( arguments ).fail( arguments ); - return this; - }, - then: function( /* fnDone, fnFail, fnProgress */ ) { - var fns = arguments; - return jQuery.Deferred(function( newDefer ) { - jQuery.each( tuples, function( i, tuple ) { - var fn = jQuery.isFunction( fns[ i ] ) && fns[ i ]; - // deferred[ done | fail | progress ] for forwarding actions to newDefer - deferred[ tuple[1] ](function() { - var returned = fn && fn.apply( this, arguments ); - if ( returned && jQuery.isFunction( returned.promise ) ) { - returned.promise() - .done( newDefer.resolve ) - .fail( newDefer.reject ) - .progress( newDefer.notify ); - } else { - newDefer[ tuple[ 0 ] + "With" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments ); - } - }); - }); - fns = null; - }).promise(); - }, - // Get a promise for this deferred - // If obj is provided, the promise aspect is added to the object - promise: function( obj ) { - return obj != null ? jQuery.extend( obj, promise ) : promise; - } - }, - deferred = {}; - - // Keep pipe for back-compat - promise.pipe = promise.then; - - // Add list-specific methods - jQuery.each( tuples, function( i, tuple ) { - var list = tuple[ 2 ], - stateString = tuple[ 3 ]; - - // promise[ done | fail | progress ] = list.add - promise[ tuple[1] ] = list.add; - - // Handle state - if ( stateString ) { - list.add(function() { - // state = [ resolved | rejected ] - state = stateString; - - // [ reject_list | resolve_list ].disable; progress_list.lock - }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); - } - - // deferred[ resolve | reject | notify ] - deferred[ tuple[0] ] = function() { - deferred[ tuple[0] + "With" ]( this === deferred ? promise : this, arguments ); - return this; - }; - deferred[ tuple[0] + "With" ] = list.fireWith; - }); - - // Make the deferred a promise - promise.promise( deferred ); - - // Call given func if any - if ( func ) { - func.call( deferred, deferred ); - } - - // All done! - return deferred; - }, - - // Deferred helper - when: function( subordinate /* , ..., subordinateN */ ) { - var i = 0, - resolveValues = slice.call( arguments ), - length = resolveValues.length, - - // the count of uncompleted subordinates - remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, - - // the master Deferred. If resolveValues consist of only a single Deferred, just use that. - deferred = remaining === 1 ? subordinate : jQuery.Deferred(), - - // Update function for both resolve and progress values - updateFunc = function( i, contexts, values ) { - return function( value ) { - contexts[ i ] = this; - values[ i ] = arguments.length > 1 ? slice.call( arguments ) : value; - if ( values === progressValues ) { - deferred.notifyWith( contexts, values ); - - } else if ( !(--remaining) ) { - deferred.resolveWith( contexts, values ); - } - }; - }, - - progressValues, progressContexts, resolveContexts; - - // add listeners to Deferred subordinates; treat others as resolved - if ( length > 1 ) { - progressValues = new Array( length ); - progressContexts = new Array( length ); - resolveContexts = new Array( length ); - for ( ; i < length; i++ ) { - if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { - resolveValues[ i ].promise() - .done( updateFunc( i, resolveContexts, resolveValues ) ) - .fail( deferred.reject ) - .progress( updateFunc( i, progressContexts, progressValues ) ); - } else { - --remaining; - } - } - } - - // if we're not waiting on anything, resolve the master - if ( !remaining ) { - deferred.resolveWith( resolveContexts, resolveValues ); - } - - return deferred.promise(); - } -}); - - -// The deferred used on DOM ready -var readyList; - -jQuery.fn.ready = function( fn ) { - // Add the callback - jQuery.ready.promise().done( fn ); - - return this; -}; - -jQuery.extend({ - // Is the DOM ready to be used? Set to true once it occurs. - isReady: false, - - // A counter to track how many items to wait for before - // the ready event fires. See #6781 - readyWait: 1, - - // Hold (or release) the ready event - holdReady: function( hold ) { - if ( hold ) { - jQuery.readyWait++; - } else { - jQuery.ready( true ); - } - }, - - // Handle when the DOM is ready - ready: function( wait ) { - - // Abort if there are pending holds or we're already ready - if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { - return; - } - - // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). - if ( !document.body ) { - return setTimeout( jQuery.ready ); - } - - // Remember that the DOM is ready - jQuery.isReady = true; - - // If a normal DOM Ready event fired, decrement, and wait if need be - if ( wait !== true && --jQuery.readyWait > 0 ) { - return; - } - - // If there are functions bound, to execute - readyList.resolveWith( document, [ jQuery ] ); - - // Trigger any bound ready events - if ( jQuery.fn.trigger ) { - jQuery( document ).trigger("ready").off("ready"); - } - } -}); - -/** - * Clean-up method for dom ready events - */ -function detach() { - if ( document.addEventListener ) { - document.removeEventListener( "DOMContentLoaded", completed, false ); - window.removeEventListener( "load", completed, false ); - - } else { - document.detachEvent( "onreadystatechange", completed ); - window.detachEvent( "onload", completed ); - } -} - -/** - * The ready event handler and self cleanup method - */ -function completed() { - // readyState === "complete" is good enough for us to call the dom ready in oldIE - if ( document.addEventListener || event.type === "load" || document.readyState === "complete" ) { - detach(); - jQuery.ready(); - } -} - -jQuery.ready.promise = function( obj ) { - if ( !readyList ) { - - readyList = jQuery.Deferred(); - - // Catch cases where $(document).ready() is called after the browser event has already occurred. - // we once tried to use readyState "interactive" here, but it caused issues like the one - // discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15 - if ( document.readyState === "complete" ) { - // Handle it asynchronously to allow scripts the opportunity to delay ready - setTimeout( jQuery.ready ); - - // Standards-based browsers support DOMContentLoaded - } else if ( document.addEventListener ) { - // Use the handy event callback - document.addEventListener( "DOMContentLoaded", completed, false ); - - // A fallback to window.onload, that will always work - window.addEventListener( "load", completed, false ); - - // If IE event model is used - } else { - // Ensure firing before onload, maybe late but safe also for iframes - document.attachEvent( "onreadystatechange", completed ); - - // A fallback to window.onload, that will always work - window.attachEvent( "onload", completed ); - - // If IE and not a frame - // continually check to see if the document is ready - var top = false; - - try { - top = window.frameElement == null && document.documentElement; - } catch(e) {} - - if ( top && top.doScroll ) { - (function doScrollCheck() { - if ( !jQuery.isReady ) { - - try { - // Use the trick by Diego Perini - // http://javascript.nwbox.com/IEContentLoaded/ - top.doScroll("left"); - } catch(e) { - return setTimeout( doScrollCheck, 50 ); - } - - // detach all dom ready events - detach(); - - // and execute any waiting functions - jQuery.ready(); - } - })(); - } - } - } - return readyList.promise( obj ); -}; - - -var strundefined = typeof undefined; - - - -// Support: IE<9 -// Iteration over object's inherited properties before its own -var i; -for ( i in jQuery( support ) ) { - break; -} -support.ownLast = i !== "0"; - -// Note: most support tests are defined in their respective modules. -// false until the test is run -support.inlineBlockNeedsLayout = false; - -jQuery(function() { - // We need to execute this one support test ASAP because we need to know - // if body.style.zoom needs to be set. - - var container, div, - body = document.getElementsByTagName("body")[0]; - - if ( !body ) { - // Return for frameset docs that don't have a body - return; - } - - // Setup - container = document.createElement( "div" ); - container.style.cssText = "border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px"; - - div = document.createElement( "div" ); - body.appendChild( container ).appendChild( div ); - - if ( typeof div.style.zoom !== strundefined ) { - // Support: IE<8 - // Check if natively block-level elements act like inline-block - // elements when setting their display to 'inline' and giving - // them layout - div.style.cssText = "border:0;margin:0;width:1px;padding:1px;display:inline;zoom:1"; - - if ( (support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 )) ) { - // Prevent IE 6 from affecting layout for positioned elements #11048 - // Prevent IE from shrinking the body in IE 7 mode #12869 - // Support: IE<8 - body.style.zoom = 1; - } - } - - body.removeChild( container ); - - // Null elements to avoid leaks in IE - container = div = null; -}); - - - - -(function() { - var div = document.createElement( "div" ); - - // Execute the test only if not already executed in another module. - if (support.deleteExpando == null) { - // Support: IE<9 - support.deleteExpando = true; - try { - delete div.test; - } catch( e ) { - support.deleteExpando = false; - } - } - - // Null elements to avoid leaks in IE. - div = null; -})(); - - -/** - * Determines whether an object can have data - */ -jQuery.acceptData = function( elem ) { - var noData = jQuery.noData[ (elem.nodeName + " ").toLowerCase() ], - nodeType = +elem.nodeType || 1; - - // Do not set data on non-element DOM nodes because it will not be cleared (#8335). - return nodeType !== 1 && nodeType !== 9 ? - false : - - // Nodes accept data unless otherwise specified; rejection can be conditional - !noData || noData !== true && elem.getAttribute("classid") === noData; -}; - - -var rbrace = /^(?:\{[\w\W]*\}|\[[\w\W]*\])$/, - rmultiDash = /([A-Z])/g; - -function dataAttr( elem, key, data ) { - // If nothing was found internally, try to fetch any - // data from the HTML5 data-* attribute - if ( data === undefined && elem.nodeType === 1 ) { - - var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); - - data = elem.getAttribute( name ); - - if ( typeof data === "string" ) { - try { - data = data === "true" ? true : - data === "false" ? false : - data === "null" ? null : - // Only convert to a number if it doesn't change the string - +data + "" === data ? +data : - rbrace.test( data ) ? jQuery.parseJSON( data ) : - data; - } catch( e ) {} - - // Make sure we set the data so it isn't changed later - jQuery.data( elem, key, data ); - - } else { - data = undefined; - } - } - - return data; -} - -// checks a cache object for emptiness -function isEmptyDataObject( obj ) { - var name; - for ( name in obj ) { - - // if the public data object is empty, the private is still empty - if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { - continue; - } - if ( name !== "toJSON" ) { - return false; - } - } - - return true; -} - -function internalData( elem, name, data, pvt /* Internal Use Only */ ) { - if ( !jQuery.acceptData( elem ) ) { - return; - } - - var ret, thisCache, - internalKey = jQuery.expando, - - // We have to handle DOM nodes and JS objects differently because IE6-7 - // can't GC object references properly across the DOM-JS boundary - isNode = elem.nodeType, - - // Only DOM nodes need the global jQuery cache; JS object data is - // attached directly to the object so GC can occur automatically - cache = isNode ? jQuery.cache : elem, - - // Only defining an ID for JS objects if its cache already exists allows - // the code to shortcut on the same path as a DOM node with no cache - id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey; - - // Avoid doing any more work than we need to when trying to get data on an - // object that has no data at all - if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && data === undefined && typeof name === "string" ) { - return; - } - - if ( !id ) { - // Only DOM nodes need a new unique ID for each element since their data - // ends up in the global cache - if ( isNode ) { - id = elem[ internalKey ] = deletedIds.pop() || jQuery.guid++; - } else { - id = internalKey; - } - } - - if ( !cache[ id ] ) { - // Avoid exposing jQuery metadata on plain JS objects when the object - // is serialized using JSON.stringify - cache[ id ] = isNode ? {} : { toJSON: jQuery.noop }; - } - - // An object can be passed to jQuery.data instead of a key/value pair; this gets - // shallow copied over onto the existing cache - if ( typeof name === "object" || typeof name === "function" ) { - if ( pvt ) { - cache[ id ] = jQuery.extend( cache[ id ], name ); - } else { - cache[ id ].data = jQuery.extend( cache[ id ].data, name ); - } - } - - thisCache = cache[ id ]; - - // jQuery data() is stored in a separate object inside the object's internal data - // cache in order to avoid key collisions between internal data and user-defined - // data. - if ( !pvt ) { - if ( !thisCache.data ) { - thisCache.data = {}; - } - - thisCache = thisCache.data; - } - - if ( data !== undefined ) { - thisCache[ jQuery.camelCase( name ) ] = data; - } - - // Check for both converted-to-camel and non-converted data property names - // If a data property was specified - if ( typeof name === "string" ) { - - // First Try to find as-is property data - ret = thisCache[ name ]; - - // Test for null|undefined property data - if ( ret == null ) { - - // Try to find the camelCased property - ret = thisCache[ jQuery.camelCase( name ) ]; - } - } else { - ret = thisCache; - } - - return ret; -} - -function internalRemoveData( elem, name, pvt ) { - if ( !jQuery.acceptData( elem ) ) { - return; - } - - var thisCache, i, - isNode = elem.nodeType, - - // See jQuery.data for more information - cache = isNode ? jQuery.cache : elem, - id = isNode ? elem[ jQuery.expando ] : jQuery.expando; - - // If there is already no cache entry for this object, there is no - // purpose in continuing - if ( !cache[ id ] ) { - return; - } - - if ( name ) { - - thisCache = pvt ? cache[ id ] : cache[ id ].data; - - if ( thisCache ) { - - // Support array or space separated string names for data keys - if ( !jQuery.isArray( name ) ) { - - // try the string as a key before any manipulation - if ( name in thisCache ) { - name = [ name ]; - } else { - - // split the camel cased version by spaces unless a key with the spaces exists - name = jQuery.camelCase( name ); - if ( name in thisCache ) { - name = [ name ]; - } else { - name = name.split(" "); - } - } - } else { - // If "name" is an array of keys... - // When data is initially created, via ("key", "val") signature, - // keys will be converted to camelCase. - // Since there is no way to tell _how_ a key was added, remove - // both plain key and camelCase key. #12786 - // This will only penalize the array argument path. - name = name.concat( jQuery.map( name, jQuery.camelCase ) ); - } - - i = name.length; - while ( i-- ) { - delete thisCache[ name[i] ]; - } - - // If there is no data left in the cache, we want to continue - // and let the cache object itself get destroyed - if ( pvt ? !isEmptyDataObject(thisCache) : !jQuery.isEmptyObject(thisCache) ) { - return; - } - } - } - - // See jQuery.data for more information - if ( !pvt ) { - delete cache[ id ].data; - - // Don't destroy the parent cache unless the internal data object - // had been the only thing left in it - if ( !isEmptyDataObject( cache[ id ] ) ) { - return; - } - } - - // Destroy the cache - if ( isNode ) { - jQuery.cleanData( [ elem ], true ); - - // Use delete when supported for expandos or `cache` is not a window per isWindow (#10080) - /* jshint eqeqeq: false */ - } else if ( support.deleteExpando || cache != cache.window ) { - /* jshint eqeqeq: true */ - delete cache[ id ]; - - // When all else fails, null - } else { - cache[ id ] = null; - } -} - -jQuery.extend({ - cache: {}, - - // The following elements (space-suffixed to avoid Object.prototype collisions) - // throw uncatchable exceptions if you attempt to set expando properties - noData: { - "applet ": true, - "embed ": true, - // ...but Flash objects (which have this classid) *can* handle expandos - "object ": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" - }, - - hasData: function( elem ) { - elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; - return !!elem && !isEmptyDataObject( elem ); - }, - - data: function( elem, name, data ) { - return internalData( elem, name, data ); - }, - - removeData: function( elem, name ) { - return internalRemoveData( elem, name ); - }, - - // For internal use only. - _data: function( elem, name, data ) { - return internalData( elem, name, data, true ); - }, - - _removeData: function( elem, name ) { - return internalRemoveData( elem, name, true ); - } -}); - -jQuery.fn.extend({ - data: function( key, value ) { - var i, name, data, - elem = this[0], - attrs = elem && elem.attributes; - - // Special expections of .data basically thwart jQuery.access, - // so implement the relevant behavior ourselves - - // Gets all values - if ( key === undefined ) { - if ( this.length ) { - data = jQuery.data( elem ); - - if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) { - i = attrs.length; - while ( i-- ) { - name = attrs[i].name; - - if ( name.indexOf("data-") === 0 ) { - name = jQuery.camelCase( name.slice(5) ); - - dataAttr( elem, name, data[ name ] ); - } - } - jQuery._data( elem, "parsedAttrs", true ); - } - } - - return data; - } - - // Sets multiple values - if ( typeof key === "object" ) { - return this.each(function() { - jQuery.data( this, key ); - }); - } - - return arguments.length > 1 ? - - // Sets one value - this.each(function() { - jQuery.data( this, key, value ); - }) : - - // Gets one value - // Try to fetch any internally stored data first - elem ? dataAttr( elem, key, jQuery.data( elem, key ) ) : undefined; - }, - - removeData: function( key ) { - return this.each(function() { - jQuery.removeData( this, key ); - }); - } -}); - - -jQuery.extend({ - queue: function( elem, type, data ) { - var queue; - - if ( elem ) { - type = ( type || "fx" ) + "queue"; - queue = jQuery._data( elem, type ); - - // Speed up dequeue by getting out quickly if this is just a lookup - if ( data ) { - if ( !queue || jQuery.isArray(data) ) { - queue = jQuery._data( elem, type, jQuery.makeArray(data) ); - } else { - queue.push( data ); - } - } - return queue || []; - } - }, - - dequeue: function( elem, type ) { - type = type || "fx"; - - var queue = jQuery.queue( elem, type ), - startLength = queue.length, - fn = queue.shift(), - hooks = jQuery._queueHooks( elem, type ), - next = function() { - jQuery.dequeue( elem, type ); - }; - - // If the fx queue is dequeued, always remove the progress sentinel - if ( fn === "inprogress" ) { - fn = queue.shift(); - startLength--; - } - - if ( fn ) { - - // Add a progress sentinel to prevent the fx queue from being - // automatically dequeued - if ( type === "fx" ) { - queue.unshift( "inprogress" ); - } - - // clear up the last queue stop function - delete hooks.stop; - fn.call( elem, next, hooks ); - } - - if ( !startLength && hooks ) { - hooks.empty.fire(); - } - }, - - // not intended for public consumption - generates a queueHooks object, or returns the current one - _queueHooks: function( elem, type ) { - var key = type + "queueHooks"; - return jQuery._data( elem, key ) || jQuery._data( elem, key, { - empty: jQuery.Callbacks("once memory").add(function() { - jQuery._removeData( elem, type + "queue" ); - jQuery._removeData( elem, key ); - }) - }); - } -}); - -jQuery.fn.extend({ - queue: function( type, data ) { - var setter = 2; - - if ( typeof type !== "string" ) { - data = type; - type = "fx"; - setter--; - } - - if ( arguments.length < setter ) { - return jQuery.queue( this[0], type ); - } - - return data === undefined ? - this : - this.each(function() { - var queue = jQuery.queue( this, type, data ); - - // ensure a hooks for this queue - jQuery._queueHooks( this, type ); - - if ( type === "fx" && queue[0] !== "inprogress" ) { - jQuery.dequeue( this, type ); - } - }); - }, - dequeue: function( type ) { - return this.each(function() { - jQuery.dequeue( this, type ); - }); - }, - clearQueue: function( type ) { - return this.queue( type || "fx", [] ); - }, - // Get a promise resolved when queues of a certain type - // are emptied (fx is the type by default) - promise: function( type, obj ) { - var tmp, - count = 1, - defer = jQuery.Deferred(), - elements = this, - i = this.length, - resolve = function() { - if ( !( --count ) ) { - defer.resolveWith( elements, [ elements ] ); - } - }; - - if ( typeof type !== "string" ) { - obj = type; - type = undefined; - } - type = type || "fx"; - - while ( i-- ) { - tmp = jQuery._data( elements[ i ], type + "queueHooks" ); - if ( tmp && tmp.empty ) { - count++; - tmp.empty.add( resolve ); - } - } - resolve(); - return defer.promise( obj ); - } -}); -var pnum = (/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/).source; - -var cssExpand = [ "Top", "Right", "Bottom", "Left" ]; - -var isHidden = function( elem, el ) { - // isHidden might be called from jQuery#filter function; - // in that case, element will be second argument - elem = el || elem; - return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); - }; - - - -// Multifunctional method to get and set values of a collection -// The value/s can optionally be executed if it's a function -var access = jQuery.access = function( elems, fn, key, value, chainable, emptyGet, raw ) { - var i = 0, - length = elems.length, - bulk = key == null; - - // Sets many values - if ( jQuery.type( key ) === "object" ) { - chainable = true; - for ( i in key ) { - jQuery.access( elems, fn, i, key[i], true, emptyGet, raw ); - } - - // Sets one value - } else if ( value !== undefined ) { - chainable = true; - - if ( !jQuery.isFunction( value ) ) { - raw = true; - } - - if ( bulk ) { - // Bulk operations run against the entire set - if ( raw ) { - fn.call( elems, value ); - fn = null; - - // ...except when executing function values - } else { - bulk = fn; - fn = function( elem, key, value ) { - return bulk.call( jQuery( elem ), value ); - }; - } - } - - if ( fn ) { - for ( ; i < length; i++ ) { - fn( elems[i], key, raw ? value : value.call( elems[i], i, fn( elems[i], key ) ) ); - } - } - } - - return chainable ? - elems : - - // Gets - bulk ? - fn.call( elems ) : - length ? fn( elems[0], key ) : emptyGet; -}; -var rcheckableType = (/^(?:checkbox|radio)$/i); - - - -(function() { - var fragment = document.createDocumentFragment(), - div = document.createElement("div"), - input = document.createElement("input"); - - // Setup - div.setAttribute( "className", "t" ); - div.innerHTML = "
    a"; - - // IE strips leading whitespace when .innerHTML is used - support.leadingWhitespace = div.firstChild.nodeType === 3; - - // Make sure that tbody elements aren't automatically inserted - // IE will insert them into empty tables - support.tbody = !div.getElementsByTagName( "tbody" ).length; - - // Make sure that link elements get serialized correctly by innerHTML - // This requires a wrapper element in IE - support.htmlSerialize = !!div.getElementsByTagName( "link" ).length; - - // Makes sure cloning an html5 element does not cause problems - // Where outerHTML is undefined, this still works - support.html5Clone = - document.createElement( "nav" ).cloneNode( true ).outerHTML !== "<:nav>"; - - // Check if a disconnected checkbox will retain its checked - // value of true after appended to the DOM (IE6/7) - input.type = "checkbox"; - input.checked = true; - fragment.appendChild( input ); - support.appendChecked = input.checked; - - // Make sure textarea (and checkbox) defaultValue is properly cloned - // Support: IE6-IE11+ - div.innerHTML = ""; - support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; - - // #11217 - WebKit loses check when the name is after the checked attribute - fragment.appendChild( div ); - div.innerHTML = ""; - - // Support: Safari 5.1, iOS 5.1, Android 4.x, Android 2.3 - // old WebKit doesn't clone checked state correctly in fragments - support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; - - // Support: IE<9 - // Opera does not clone events (and typeof div.attachEvent === undefined). - // IE9-10 clones events bound via attachEvent, but they don't trigger with .click() - support.noCloneEvent = true; - if ( div.attachEvent ) { - div.attachEvent( "onclick", function() { - support.noCloneEvent = false; - }); - - div.cloneNode( true ).click(); - } - - // Execute the test only if not already executed in another module. - if (support.deleteExpando == null) { - // Support: IE<9 - support.deleteExpando = true; - try { - delete div.test; - } catch( e ) { - support.deleteExpando = false; - } - } - - // Null elements to avoid leaks in IE. - fragment = div = input = null; -})(); - - -(function() { - var i, eventName, - div = document.createElement( "div" ); - - // Support: IE<9 (lack submit/change bubble), Firefox 23+ (lack focusin event) - for ( i in { submit: true, change: true, focusin: true }) { - eventName = "on" + i; - - if ( !(support[ i + "Bubbles" ] = eventName in window) ) { - // Beware of CSP restrictions (https://developer.mozilla.org/en/Security/CSP) - div.setAttribute( eventName, "t" ); - support[ i + "Bubbles" ] = div.attributes[ eventName ].expando === false; - } - } - - // Null elements to avoid leaks in IE. - div = null; -})(); - - -var rformElems = /^(?:input|select|textarea)$/i, - rkeyEvent = /^key/, - rmouseEvent = /^(?:mouse|contextmenu)|click/, - rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, - rtypenamespace = /^([^.]*)(?:\.(.+)|)$/; - -function returnTrue() { - return true; -} - -function returnFalse() { - return false; -} - -function safeActiveElement() { - try { - return document.activeElement; - } catch ( err ) { } -} - -/* - * Helper functions for managing events -- not part of the public interface. - * Props to Dean Edwards' addEvent library for many of the ideas. - */ -jQuery.event = { - - global: {}, - - add: function( elem, types, handler, data, selector ) { - var tmp, events, t, handleObjIn, - special, eventHandle, handleObj, - handlers, type, namespaces, origType, - elemData = jQuery._data( elem ); - - // Don't attach events to noData or text/comment nodes (but allow plain objects) - if ( !elemData ) { - return; - } - - // Caller can pass in an object of custom data in lieu of the handler - if ( handler.handler ) { - handleObjIn = handler; - handler = handleObjIn.handler; - selector = handleObjIn.selector; - } - - // Make sure that the handler has a unique ID, used to find/remove it later - if ( !handler.guid ) { - handler.guid = jQuery.guid++; - } - - // Init the element's event structure and main handler, if this is the first - if ( !(events = elemData.events) ) { - events = elemData.events = {}; - } - if ( !(eventHandle = elemData.handle) ) { - eventHandle = elemData.handle = function( e ) { - // Discard the second event of a jQuery.event.trigger() and - // when an event is called after a page has unloaded - return typeof jQuery !== strundefined && (!e || jQuery.event.triggered !== e.type) ? - jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : - undefined; - }; - // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events - eventHandle.elem = elem; - } - - // Handle multiple events separated by a space - types = ( types || "" ).match( rnotwhite ) || [ "" ]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[t] ) || []; - type = origType = tmp[1]; - namespaces = ( tmp[2] || "" ).split( "." ).sort(); - - // There *must* be a type, no attaching namespace-only handlers - if ( !type ) { - continue; - } - - // If event changes its type, use the special event handlers for the changed type - special = jQuery.event.special[ type ] || {}; - - // If selector defined, determine special event api type, otherwise given type - type = ( selector ? special.delegateType : special.bindType ) || type; - - // Update special based on newly reset type - special = jQuery.event.special[ type ] || {}; - - // handleObj is passed to all event handlers - handleObj = jQuery.extend({ - type: type, - origType: origType, - data: data, - handler: handler, - guid: handler.guid, - selector: selector, - needsContext: selector && jQuery.expr.match.needsContext.test( selector ), - namespace: namespaces.join(".") - }, handleObjIn ); - - // Init the event handler queue if we're the first - if ( !(handlers = events[ type ]) ) { - handlers = events[ type ] = []; - handlers.delegateCount = 0; - - // Only use addEventListener/attachEvent if the special events handler returns false - if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { - // Bind the global event handler to the element - if ( elem.addEventListener ) { - elem.addEventListener( type, eventHandle, false ); - - } else if ( elem.attachEvent ) { - elem.attachEvent( "on" + type, eventHandle ); - } - } - } - - if ( special.add ) { - special.add.call( elem, handleObj ); - - if ( !handleObj.handler.guid ) { - handleObj.handler.guid = handler.guid; - } - } - - // Add to the element's handler list, delegates in front - if ( selector ) { - handlers.splice( handlers.delegateCount++, 0, handleObj ); - } else { - handlers.push( handleObj ); - } - - // Keep track of which events have ever been used, for event optimization - jQuery.event.global[ type ] = true; - } - - // Nullify elem to prevent memory leaks in IE - elem = null; - }, - - // Detach an event or set of events from an element - remove: function( elem, types, handler, selector, mappedTypes ) { - var j, handleObj, tmp, - origCount, t, events, - special, handlers, type, - namespaces, origType, - elemData = jQuery.hasData( elem ) && jQuery._data( elem ); - - if ( !elemData || !(events = elemData.events) ) { - return; - } - - // Once for each type.namespace in types; type may be omitted - types = ( types || "" ).match( rnotwhite ) || [ "" ]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[t] ) || []; - type = origType = tmp[1]; - namespaces = ( tmp[2] || "" ).split( "." ).sort(); - - // Unbind all events (on this namespace, if provided) for the element - if ( !type ) { - for ( type in events ) { - jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); - } - continue; - } - - special = jQuery.event.special[ type ] || {}; - type = ( selector ? special.delegateType : special.bindType ) || type; - handlers = events[ type ] || []; - tmp = tmp[2] && new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ); - - // Remove matching events - origCount = j = handlers.length; - while ( j-- ) { - handleObj = handlers[ j ]; - - if ( ( mappedTypes || origType === handleObj.origType ) && - ( !handler || handler.guid === handleObj.guid ) && - ( !tmp || tmp.test( handleObj.namespace ) ) && - ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { - handlers.splice( j, 1 ); - - if ( handleObj.selector ) { - handlers.delegateCount--; - } - if ( special.remove ) { - special.remove.call( elem, handleObj ); - } - } - } - - // Remove generic event handler if we removed something and no more handlers exist - // (avoids potential for endless recursion during removal of special event handlers) - if ( origCount && !handlers.length ) { - if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { - jQuery.removeEvent( elem, type, elemData.handle ); - } - - delete events[ type ]; - } - } - - // Remove the expando if it's no longer used - if ( jQuery.isEmptyObject( events ) ) { - delete elemData.handle; - - // removeData also checks for emptiness and clears the expando if empty - // so use it instead of delete - jQuery._removeData( elem, "events" ); - } - }, - - trigger: function( event, data, elem, onlyHandlers ) { - var handle, ontype, cur, - bubbleType, special, tmp, i, - eventPath = [ elem || document ], - type = hasOwn.call( event, "type" ) ? event.type : event, - namespaces = hasOwn.call( event, "namespace" ) ? event.namespace.split(".") : []; - - cur = tmp = elem = elem || document; - - // Don't do events on text and comment nodes - if ( elem.nodeType === 3 || elem.nodeType === 8 ) { - return; - } - - // focus/blur morphs to focusin/out; ensure we're not firing them right now - if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { - return; - } - - if ( type.indexOf(".") >= 0 ) { - // Namespaced trigger; create a regexp to match event type in handle() - namespaces = type.split("."); - type = namespaces.shift(); - namespaces.sort(); - } - ontype = type.indexOf(":") < 0 && "on" + type; - - // Caller can pass in a jQuery.Event object, Object, or just an event type string - event = event[ jQuery.expando ] ? - event : - new jQuery.Event( type, typeof event === "object" && event ); - - // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) - event.isTrigger = onlyHandlers ? 2 : 3; - event.namespace = namespaces.join("."); - event.namespace_re = event.namespace ? - new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ) : - null; - - // Clean up the event in case it is being reused - event.result = undefined; - if ( !event.target ) { - event.target = elem; - } - - // Clone any incoming data and prepend the event, creating the handler arg list - data = data == null ? - [ event ] : - jQuery.makeArray( data, [ event ] ); - - // Allow special events to draw outside the lines - special = jQuery.event.special[ type ] || {}; - if ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) { - return; - } - - // Determine event propagation path in advance, per W3C events spec (#9951) - // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) - if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { - - bubbleType = special.delegateType || type; - if ( !rfocusMorph.test( bubbleType + type ) ) { - cur = cur.parentNode; - } - for ( ; cur; cur = cur.parentNode ) { - eventPath.push( cur ); - tmp = cur; - } - - // Only add window if we got to document (e.g., not plain obj or detached DOM) - if ( tmp === (elem.ownerDocument || document) ) { - eventPath.push( tmp.defaultView || tmp.parentWindow || window ); - } - } - - // Fire handlers on the event path - i = 0; - while ( (cur = eventPath[i++]) && !event.isPropagationStopped() ) { - - event.type = i > 1 ? - bubbleType : - special.bindType || type; - - // jQuery handler - handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); - if ( handle ) { - handle.apply( cur, data ); - } - - // Native handler - handle = ontype && cur[ ontype ]; - if ( handle && handle.apply && jQuery.acceptData( cur ) ) { - event.result = handle.apply( cur, data ); - if ( event.result === false ) { - event.preventDefault(); - } - } - } - event.type = type; - - // If nobody prevented the default action, do it now - if ( !onlyHandlers && !event.isDefaultPrevented() ) { - - if ( (!special._default || special._default.apply( eventPath.pop(), data ) === false) && - jQuery.acceptData( elem ) ) { - - // Call a native DOM method on the target with the same name name as the event. - // Can't use an .isFunction() check here because IE6/7 fails that test. - // Don't do default actions on window, that's where global variables be (#6170) - if ( ontype && elem[ type ] && !jQuery.isWindow( elem ) ) { - - // Don't re-trigger an onFOO event when we call its FOO() method - tmp = elem[ ontype ]; - - if ( tmp ) { - elem[ ontype ] = null; - } - - // Prevent re-triggering of the same event, since we already bubbled it above - jQuery.event.triggered = type; - try { - elem[ type ](); - } catch ( e ) { - // IE<9 dies on focus/blur to hidden element (#1486,#12518) - // only reproducible on winXP IE8 native, not IE9 in IE8 mode - } - jQuery.event.triggered = undefined; - - if ( tmp ) { - elem[ ontype ] = tmp; - } - } - } - } - - return event.result; - }, - - dispatch: function( event ) { - - // Make a writable jQuery.Event from the native event object - event = jQuery.event.fix( event ); - - var i, ret, handleObj, matched, j, - handlerQueue = [], - args = slice.call( arguments ), - handlers = ( jQuery._data( this, "events" ) || {} )[ event.type ] || [], - special = jQuery.event.special[ event.type ] || {}; - - // Use the fix-ed jQuery.Event rather than the (read-only) native event - args[0] = event; - event.delegateTarget = this; - - // Call the preDispatch hook for the mapped type, and let it bail if desired - if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { - return; - } - - // Determine handlers - handlerQueue = jQuery.event.handlers.call( this, event, handlers ); - - // Run delegates first; they may want to stop propagation beneath us - i = 0; - while ( (matched = handlerQueue[ i++ ]) && !event.isPropagationStopped() ) { - event.currentTarget = matched.elem; - - j = 0; - while ( (handleObj = matched.handlers[ j++ ]) && !event.isImmediatePropagationStopped() ) { - - // Triggered event must either 1) have no namespace, or - // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). - if ( !event.namespace_re || event.namespace_re.test( handleObj.namespace ) ) { - - event.handleObj = handleObj; - event.data = handleObj.data; - - ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) - .apply( matched.elem, args ); - - if ( ret !== undefined ) { - if ( (event.result = ret) === false ) { - event.preventDefault(); - event.stopPropagation(); - } - } - } - } - } - - // Call the postDispatch hook for the mapped type - if ( special.postDispatch ) { - special.postDispatch.call( this, event ); - } - - return event.result; - }, - - handlers: function( event, handlers ) { - var sel, handleObj, matches, i, - handlerQueue = [], - delegateCount = handlers.delegateCount, - cur = event.target; - - // Find delegate handlers - // Black-hole SVG instance trees (#13180) - // Avoid non-left-click bubbling in Firefox (#3861) - if ( delegateCount && cur.nodeType && (!event.button || event.type !== "click") ) { - - /* jshint eqeqeq: false */ - for ( ; cur != this; cur = cur.parentNode || this ) { - /* jshint eqeqeq: true */ - - // Don't check non-elements (#13208) - // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) - if ( cur.nodeType === 1 && (cur.disabled !== true || event.type !== "click") ) { - matches = []; - for ( i = 0; i < delegateCount; i++ ) { - handleObj = handlers[ i ]; - - // Don't conflict with Object.prototype properties (#13203) - sel = handleObj.selector + " "; - - if ( matches[ sel ] === undefined ) { - matches[ sel ] = handleObj.needsContext ? - jQuery( sel, this ).index( cur ) >= 0 : - jQuery.find( sel, this, null, [ cur ] ).length; - } - if ( matches[ sel ] ) { - matches.push( handleObj ); - } - } - if ( matches.length ) { - handlerQueue.push({ elem: cur, handlers: matches }); - } - } - } - } - - // Add the remaining (directly-bound) handlers - if ( delegateCount < handlers.length ) { - handlerQueue.push({ elem: this, handlers: handlers.slice( delegateCount ) }); - } - - return handlerQueue; - }, - - fix: function( event ) { - if ( event[ jQuery.expando ] ) { - return event; - } - - // Create a writable copy of the event object and normalize some properties - var i, prop, copy, - type = event.type, - originalEvent = event, - fixHook = this.fixHooks[ type ]; - - if ( !fixHook ) { - this.fixHooks[ type ] = fixHook = - rmouseEvent.test( type ) ? this.mouseHooks : - rkeyEvent.test( type ) ? this.keyHooks : - {}; - } - copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; - - event = new jQuery.Event( originalEvent ); - - i = copy.length; - while ( i-- ) { - prop = copy[ i ]; - event[ prop ] = originalEvent[ prop ]; - } - - // Support: IE<9 - // Fix target property (#1925) - if ( !event.target ) { - event.target = originalEvent.srcElement || document; - } - - // Support: Chrome 23+, Safari? - // Target should not be a text node (#504, #13143) - if ( event.target.nodeType === 3 ) { - event.target = event.target.parentNode; - } - - // Support: IE<9 - // For mouse/key events, metaKey==false if it's undefined (#3368, #11328) - event.metaKey = !!event.metaKey; - - return fixHook.filter ? fixHook.filter( event, originalEvent ) : event; - }, - - // Includes some event props shared by KeyEvent and MouseEvent - props: "altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), - - fixHooks: {}, - - keyHooks: { - props: "char charCode key keyCode".split(" "), - filter: function( event, original ) { - - // Add which for key events - if ( event.which == null ) { - event.which = original.charCode != null ? original.charCode : original.keyCode; - } - - return event; - } - }, - - mouseHooks: { - props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), - filter: function( event, original ) { - var body, eventDoc, doc, - button = original.button, - fromElement = original.fromElement; - - // Calculate pageX/Y if missing and clientX/Y available - if ( event.pageX == null && original.clientX != null ) { - eventDoc = event.target.ownerDocument || document; - doc = eventDoc.documentElement; - body = eventDoc.body; - - event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); - event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); - } - - // Add relatedTarget, if necessary - if ( !event.relatedTarget && fromElement ) { - event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; - } - - // Add which for click: 1 === left; 2 === middle; 3 === right - // Note: button is not normalized, so don't use it - if ( !event.which && button !== undefined ) { - event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); - } - - return event; - } - }, - - special: { - load: { - // Prevent triggered image.load events from bubbling to window.load - noBubble: true - }, - focus: { - // Fire native event if possible so blur/focus sequence is correct - trigger: function() { - if ( this !== safeActiveElement() && this.focus ) { - try { - this.focus(); - return false; - } catch ( e ) { - // Support: IE<9 - // If we error on focus to hidden element (#1486, #12518), - // let .trigger() run the handlers - } - } - }, - delegateType: "focusin" - }, - blur: { - trigger: function() { - if ( this === safeActiveElement() && this.blur ) { - this.blur(); - return false; - } - }, - delegateType: "focusout" - }, - click: { - // For checkbox, fire native event so checked state will be right - trigger: function() { - if ( jQuery.nodeName( this, "input" ) && this.type === "checkbox" && this.click ) { - this.click(); - return false; - } - }, - - // For cross-browser consistency, don't fire native .click() on links - _default: function( event ) { - return jQuery.nodeName( event.target, "a" ); - } - }, - - beforeunload: { - postDispatch: function( event ) { - - // Even when returnValue equals to undefined Firefox will still show alert - if ( event.result !== undefined ) { - event.originalEvent.returnValue = event.result; - } - } - } - }, - - simulate: function( type, elem, event, bubble ) { - // Piggyback on a donor event to simulate a different one. - // Fake originalEvent to avoid donor's stopPropagation, but if the - // simulated event prevents default then we do the same on the donor. - var e = jQuery.extend( - new jQuery.Event(), - event, - { - type: type, - isSimulated: true, - originalEvent: {} - } - ); - if ( bubble ) { - jQuery.event.trigger( e, null, elem ); - } else { - jQuery.event.dispatch.call( elem, e ); - } - if ( e.isDefaultPrevented() ) { - event.preventDefault(); - } - } -}; - -jQuery.removeEvent = document.removeEventListener ? - function( elem, type, handle ) { - if ( elem.removeEventListener ) { - elem.removeEventListener( type, handle, false ); - } - } : - function( elem, type, handle ) { - var name = "on" + type; - - if ( elem.detachEvent ) { - - // #8545, #7054, preventing memory leaks for custom events in IE6-8 - // detachEvent needed property on element, by name of that event, to properly expose it to GC - if ( typeof elem[ name ] === strundefined ) { - elem[ name ] = null; - } - - elem.detachEvent( name, handle ); - } - }; - -jQuery.Event = function( src, props ) { - // Allow instantiation without the 'new' keyword - if ( !(this instanceof jQuery.Event) ) { - return new jQuery.Event( src, props ); - } - - // Event object - if ( src && src.type ) { - this.originalEvent = src; - this.type = src.type; - - // Events bubbling up the document may have been marked as prevented - // by a handler lower down the tree; reflect the correct value. - this.isDefaultPrevented = src.defaultPrevented || - src.defaultPrevented === undefined && ( - // Support: IE < 9 - src.returnValue === false || - // Support: Android < 4.0 - src.getPreventDefault && src.getPreventDefault() ) ? - returnTrue : - returnFalse; - - // Event type - } else { - this.type = src; - } - - // Put explicitly provided properties onto the event object - if ( props ) { - jQuery.extend( this, props ); - } - - // Create a timestamp if incoming event doesn't have one - this.timeStamp = src && src.timeStamp || jQuery.now(); - - // Mark it as fixed - this[ jQuery.expando ] = true; -}; - -// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding -// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html -jQuery.Event.prototype = { - isDefaultPrevented: returnFalse, - isPropagationStopped: returnFalse, - isImmediatePropagationStopped: returnFalse, - - preventDefault: function() { - var e = this.originalEvent; - - this.isDefaultPrevented = returnTrue; - if ( !e ) { - return; - } - - // If preventDefault exists, run it on the original event - if ( e.preventDefault ) { - e.preventDefault(); - - // Support: IE - // Otherwise set the returnValue property of the original event to false - } else { - e.returnValue = false; - } - }, - stopPropagation: function() { - var e = this.originalEvent; - - this.isPropagationStopped = returnTrue; - if ( !e ) { - return; - } - // If stopPropagation exists, run it on the original event - if ( e.stopPropagation ) { - e.stopPropagation(); - } - - // Support: IE - // Set the cancelBubble property of the original event to true - e.cancelBubble = true; - }, - stopImmediatePropagation: function() { - this.isImmediatePropagationStopped = returnTrue; - this.stopPropagation(); - } -}; - -// Create mouseenter/leave events using mouseover/out and event-time checks -jQuery.each({ - mouseenter: "mouseover", - mouseleave: "mouseout" -}, function( orig, fix ) { - jQuery.event.special[ orig ] = { - delegateType: fix, - bindType: fix, - - handle: function( event ) { - var ret, - target = this, - related = event.relatedTarget, - handleObj = event.handleObj; - - // For mousenter/leave call the handler if related is outside the target. - // NB: No relatedTarget if the mouse left/entered the browser window - if ( !related || (related !== target && !jQuery.contains( target, related )) ) { - event.type = handleObj.origType; - ret = handleObj.handler.apply( this, arguments ); - event.type = fix; - } - return ret; - } - }; -}); - -// IE submit delegation -if ( !support.submitBubbles ) { - - jQuery.event.special.submit = { - setup: function() { - // Only need this for delegated form submit events - if ( jQuery.nodeName( this, "form" ) ) { - return false; - } - - // Lazy-add a submit handler when a descendant form may potentially be submitted - jQuery.event.add( this, "click._submit keypress._submit", function( e ) { - // Node name check avoids a VML-related crash in IE (#9807) - var elem = e.target, - form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; - if ( form && !jQuery._data( form, "submitBubbles" ) ) { - jQuery.event.add( form, "submit._submit", function( event ) { - event._submit_bubble = true; - }); - jQuery._data( form, "submitBubbles", true ); - } - }); - // return undefined since we don't need an event listener - }, - - postDispatch: function( event ) { - // If form was submitted by the user, bubble the event up the tree - if ( event._submit_bubble ) { - delete event._submit_bubble; - if ( this.parentNode && !event.isTrigger ) { - jQuery.event.simulate( "submit", this.parentNode, event, true ); - } - } - }, - - teardown: function() { - // Only need this for delegated form submit events - if ( jQuery.nodeName( this, "form" ) ) { - return false; - } - - // Remove delegated handlers; cleanData eventually reaps submit handlers attached above - jQuery.event.remove( this, "._submit" ); - } - }; -} - -// IE change delegation and checkbox/radio fix -if ( !support.changeBubbles ) { - - jQuery.event.special.change = { - - setup: function() { - - if ( rformElems.test( this.nodeName ) ) { - // IE doesn't fire change on a check/radio until blur; trigger it on click - // after a propertychange. Eat the blur-change in special.change.handle. - // This still fires onchange a second time for check/radio after blur. - if ( this.type === "checkbox" || this.type === "radio" ) { - jQuery.event.add( this, "propertychange._change", function( event ) { - if ( event.originalEvent.propertyName === "checked" ) { - this._just_changed = true; - } - }); - jQuery.event.add( this, "click._change", function( event ) { - if ( this._just_changed && !event.isTrigger ) { - this._just_changed = false; - } - // Allow triggered, simulated change events (#11500) - jQuery.event.simulate( "change", this, event, true ); - }); - } - return false; - } - // Delegated event; lazy-add a change handler on descendant inputs - jQuery.event.add( this, "beforeactivate._change", function( e ) { - var elem = e.target; - - if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "changeBubbles" ) ) { - jQuery.event.add( elem, "change._change", function( event ) { - if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { - jQuery.event.simulate( "change", this.parentNode, event, true ); - } - }); - jQuery._data( elem, "changeBubbles", true ); - } - }); - }, - - handle: function( event ) { - var elem = event.target; - - // Swallow native change events from checkbox/radio, we already triggered them above - if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { - return event.handleObj.handler.apply( this, arguments ); - } - }, - - teardown: function() { - jQuery.event.remove( this, "._change" ); - - return !rformElems.test( this.nodeName ); - } - }; -} - -// Create "bubbling" focus and blur events -if ( !support.focusinBubbles ) { - jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { - - // Attach a single capturing handler on the document while someone wants focusin/focusout - var handler = function( event ) { - jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); - }; - - jQuery.event.special[ fix ] = { - setup: function() { - var doc = this.ownerDocument || this, - attaches = jQuery._data( doc, fix ); - - if ( !attaches ) { - doc.addEventListener( orig, handler, true ); - } - jQuery._data( doc, fix, ( attaches || 0 ) + 1 ); - }, - teardown: function() { - var doc = this.ownerDocument || this, - attaches = jQuery._data( doc, fix ) - 1; - - if ( !attaches ) { - doc.removeEventListener( orig, handler, true ); - jQuery._removeData( doc, fix ); - } else { - jQuery._data( doc, fix, attaches ); - } - } - }; - }); -} - -jQuery.fn.extend({ - - on: function( types, selector, data, fn, /*INTERNAL*/ one ) { - var type, origFn; - - // Types can be a map of types/handlers - if ( typeof types === "object" ) { - // ( types-Object, selector, data ) - if ( typeof selector !== "string" ) { - // ( types-Object, data ) - data = data || selector; - selector = undefined; - } - for ( type in types ) { - this.on( type, selector, data, types[ type ], one ); - } - return this; - } - - if ( data == null && fn == null ) { - // ( types, fn ) - fn = selector; - data = selector = undefined; - } else if ( fn == null ) { - if ( typeof selector === "string" ) { - // ( types, selector, fn ) - fn = data; - data = undefined; - } else { - // ( types, data, fn ) - fn = data; - data = selector; - selector = undefined; - } - } - if ( fn === false ) { - fn = returnFalse; - } else if ( !fn ) { - return this; - } - - if ( one === 1 ) { - origFn = fn; - fn = function( event ) { - // Can use an empty set, since event contains the info - jQuery().off( event ); - return origFn.apply( this, arguments ); - }; - // Use same guid so caller can remove using origFn - fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); - } - return this.each( function() { - jQuery.event.add( this, types, fn, data, selector ); - }); - }, - one: function( types, selector, data, fn ) { - return this.on( types, selector, data, fn, 1 ); - }, - off: function( types, selector, fn ) { - var handleObj, type; - if ( types && types.preventDefault && types.handleObj ) { - // ( event ) dispatched jQuery.Event - handleObj = types.handleObj; - jQuery( types.delegateTarget ).off( - handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, - handleObj.selector, - handleObj.handler - ); - return this; - } - if ( typeof types === "object" ) { - // ( types-object [, selector] ) - for ( type in types ) { - this.off( type, selector, types[ type ] ); - } - return this; - } - if ( selector === false || typeof selector === "function" ) { - // ( types [, fn] ) - fn = selector; - selector = undefined; - } - if ( fn === false ) { - fn = returnFalse; - } - return this.each(function() { - jQuery.event.remove( this, types, fn, selector ); - }); - }, - - trigger: function( type, data ) { - return this.each(function() { - jQuery.event.trigger( type, data, this ); - }); - }, - triggerHandler: function( type, data ) { - var elem = this[0]; - if ( elem ) { - return jQuery.event.trigger( type, data, elem, true ); - } - } -}); - - -function createSafeFragment( document ) { - var list = nodeNames.split( "|" ), - safeFrag = document.createDocumentFragment(); - - if ( safeFrag.createElement ) { - while ( list.length ) { - safeFrag.createElement( - list.pop() - ); - } - } - return safeFrag; -} - -var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" + - "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", - rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g, - rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"), - rleadingWhitespace = /^\s+/, - rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi, - rtagName = /<([\w:]+)/, - rtbody = /\s*$/g, - - // We have to close these tags to support XHTML (#13200) - wrapMap = { - option: [ 1, "" ], - legend: [ 1, "
    ", "
    " ], - area: [ 1, "", "" ], - param: [ 1, "", "" ], - thead: [ 1, "", "
    " ], - tr: [ 2, "", "
    " ], - col: [ 2, "", "
    " ], - td: [ 3, "", "
    " ], - - // IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags, - // unless wrapped in a div with non-breaking characters in front of it. - _default: support.htmlSerialize ? [ 0, "", "" ] : [ 1, "X
    ", "
    " ] - }, - safeFragment = createSafeFragment( document ), - fragmentDiv = safeFragment.appendChild( document.createElement("div") ); - -wrapMap.optgroup = wrapMap.option; -wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; -wrapMap.th = wrapMap.td; - -function getAll( context, tag ) { - var elems, elem, - i = 0, - found = typeof context.getElementsByTagName !== strundefined ? context.getElementsByTagName( tag || "*" ) : - typeof context.querySelectorAll !== strundefined ? context.querySelectorAll( tag || "*" ) : - undefined; - - if ( !found ) { - for ( found = [], elems = context.childNodes || context; (elem = elems[i]) != null; i++ ) { - if ( !tag || jQuery.nodeName( elem, tag ) ) { - found.push( elem ); - } else { - jQuery.merge( found, getAll( elem, tag ) ); - } - } - } - - return tag === undefined || tag && jQuery.nodeName( context, tag ) ? - jQuery.merge( [ context ], found ) : - found; -} - -// Used in buildFragment, fixes the defaultChecked property -function fixDefaultChecked( elem ) { - if ( rcheckableType.test( elem.type ) ) { - elem.defaultChecked = elem.checked; - } -} - -// Support: IE<8 -// Manipulating tables requires a tbody -function manipulationTarget( elem, content ) { - return jQuery.nodeName( elem, "table" ) && - jQuery.nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ? - - elem.getElementsByTagName("tbody")[0] || - elem.appendChild( elem.ownerDocument.createElement("tbody") ) : - elem; -} - -// Replace/restore the type attribute of script elements for safe DOM manipulation -function disableScript( elem ) { - elem.type = (jQuery.find.attr( elem, "type" ) !== null) + "/" + elem.type; - return elem; -} -function restoreScript( elem ) { - var match = rscriptTypeMasked.exec( elem.type ); - if ( match ) { - elem.type = match[1]; - } else { - elem.removeAttribute("type"); - } - return elem; -} - -// Mark scripts as having already been evaluated -function setGlobalEval( elems, refElements ) { - var elem, - i = 0; - for ( ; (elem = elems[i]) != null; i++ ) { - jQuery._data( elem, "globalEval", !refElements || jQuery._data( refElements[i], "globalEval" ) ); - } -} - -function cloneCopyEvent( src, dest ) { - - if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { - return; - } - - var type, i, l, - oldData = jQuery._data( src ), - curData = jQuery._data( dest, oldData ), - events = oldData.events; - - if ( events ) { - delete curData.handle; - curData.events = {}; - - for ( type in events ) { - for ( i = 0, l = events[ type ].length; i < l; i++ ) { - jQuery.event.add( dest, type, events[ type ][ i ] ); - } - } - } - - // make the cloned public data object a copy from the original - if ( curData.data ) { - curData.data = jQuery.extend( {}, curData.data ); - } -} - -function fixCloneNodeIssues( src, dest ) { - var nodeName, e, data; - - // We do not need to do anything for non-Elements - if ( dest.nodeType !== 1 ) { - return; - } - - nodeName = dest.nodeName.toLowerCase(); - - // IE6-8 copies events bound via attachEvent when using cloneNode. - if ( !support.noCloneEvent && dest[ jQuery.expando ] ) { - data = jQuery._data( dest ); - - for ( e in data.events ) { - jQuery.removeEvent( dest, e, data.handle ); - } - - // Event data gets referenced instead of copied if the expando gets copied too - dest.removeAttribute( jQuery.expando ); - } - - // IE blanks contents when cloning scripts, and tries to evaluate newly-set text - if ( nodeName === "script" && dest.text !== src.text ) { - disableScript( dest ).text = src.text; - restoreScript( dest ); - - // IE6-10 improperly clones children of object elements using classid. - // IE10 throws NoModificationAllowedError if parent is null, #12132. - } else if ( nodeName === "object" ) { - if ( dest.parentNode ) { - dest.outerHTML = src.outerHTML; - } - - // This path appears unavoidable for IE9. When cloning an object - // element in IE9, the outerHTML strategy above is not sufficient. - // If the src has innerHTML and the destination does not, - // copy the src.innerHTML into the dest.innerHTML. #10324 - if ( support.html5Clone && ( src.innerHTML && !jQuery.trim(dest.innerHTML) ) ) { - dest.innerHTML = src.innerHTML; - } - - } else if ( nodeName === "input" && rcheckableType.test( src.type ) ) { - // IE6-8 fails to persist the checked state of a cloned checkbox - // or radio button. Worse, IE6-7 fail to give the cloned element - // a checked appearance if the defaultChecked value isn't also set - - dest.defaultChecked = dest.checked = src.checked; - - // IE6-7 get confused and end up setting the value of a cloned - // checkbox/radio button to an empty string instead of "on" - if ( dest.value !== src.value ) { - dest.value = src.value; - } - - // IE6-8 fails to return the selected option to the default selected - // state when cloning options - } else if ( nodeName === "option" ) { - dest.defaultSelected = dest.selected = src.defaultSelected; - - // IE6-8 fails to set the defaultValue to the correct value when - // cloning other types of input fields - } else if ( nodeName === "input" || nodeName === "textarea" ) { - dest.defaultValue = src.defaultValue; - } -} - -jQuery.extend({ - clone: function( elem, dataAndEvents, deepDataAndEvents ) { - var destElements, node, clone, i, srcElements, - inPage = jQuery.contains( elem.ownerDocument, elem ); - - if ( support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) { - clone = elem.cloneNode( true ); - - // IE<=8 does not properly clone detached, unknown element nodes - } else { - fragmentDiv.innerHTML = elem.outerHTML; - fragmentDiv.removeChild( clone = fragmentDiv.firstChild ); - } - - if ( (!support.noCloneEvent || !support.noCloneChecked) && - (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { - - // We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2 - destElements = getAll( clone ); - srcElements = getAll( elem ); - - // Fix all IE cloning issues - for ( i = 0; (node = srcElements[i]) != null; ++i ) { - // Ensure that the destination node is not null; Fixes #9587 - if ( destElements[i] ) { - fixCloneNodeIssues( node, destElements[i] ); - } - } - } - - // Copy the events from the original to the clone - if ( dataAndEvents ) { - if ( deepDataAndEvents ) { - srcElements = srcElements || getAll( elem ); - destElements = destElements || getAll( clone ); - - for ( i = 0; (node = srcElements[i]) != null; i++ ) { - cloneCopyEvent( node, destElements[i] ); - } - } else { - cloneCopyEvent( elem, clone ); - } - } - - // Preserve script evaluation history - destElements = getAll( clone, "script" ); - if ( destElements.length > 0 ) { - setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); - } - - destElements = srcElements = node = null; - - // Return the cloned set - return clone; - }, - - buildFragment: function( elems, context, scripts, selection ) { - var j, elem, contains, - tmp, tag, tbody, wrap, - l = elems.length, - - // Ensure a safe fragment - safe = createSafeFragment( context ), - - nodes = [], - i = 0; - - for ( ; i < l; i++ ) { - elem = elems[ i ]; - - if ( elem || elem === 0 ) { - - // Add nodes directly - if ( jQuery.type( elem ) === "object" ) { - jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); - - // Convert non-html into a text node - } else if ( !rhtml.test( elem ) ) { - nodes.push( context.createTextNode( elem ) ); - - // Convert html into DOM nodes - } else { - tmp = tmp || safe.appendChild( context.createElement("div") ); - - // Deserialize a standard representation - tag = (rtagName.exec( elem ) || [ "", "" ])[ 1 ].toLowerCase(); - wrap = wrapMap[ tag ] || wrapMap._default; - - tmp.innerHTML = wrap[1] + elem.replace( rxhtmlTag, "<$1>" ) + wrap[2]; - - // Descend through wrappers to the right content - j = wrap[0]; - while ( j-- ) { - tmp = tmp.lastChild; - } - - // Manually add leading whitespace removed by IE - if ( !support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { - nodes.push( context.createTextNode( rleadingWhitespace.exec( elem )[0] ) ); - } - - // Remove IE's autoinserted from table fragments - if ( !support.tbody ) { - - // String was a , *may* have spurious - elem = tag === "table" && !rtbody.test( elem ) ? - tmp.firstChild : - - // String was a bare or - wrap[1] === "
    " && !rtbody.test( elem ) ? - tmp : - 0; - - j = elem && elem.childNodes.length; - while ( j-- ) { - if ( jQuery.nodeName( (tbody = elem.childNodes[j]), "tbody" ) && !tbody.childNodes.length ) { - elem.removeChild( tbody ); - } - } - } - - jQuery.merge( nodes, tmp.childNodes ); - - // Fix #12392 for WebKit and IE > 9 - tmp.textContent = ""; - - // Fix #12392 for oldIE - while ( tmp.firstChild ) { - tmp.removeChild( tmp.firstChild ); - } - - // Remember the top-level container for proper cleanup - tmp = safe.lastChild; - } - } - } - - // Fix #11356: Clear elements from fragment - if ( tmp ) { - safe.removeChild( tmp ); - } - - // Reset defaultChecked for any radios and checkboxes - // about to be appended to the DOM in IE 6/7 (#8060) - if ( !support.appendChecked ) { - jQuery.grep( getAll( nodes, "input" ), fixDefaultChecked ); - } - - i = 0; - while ( (elem = nodes[ i++ ]) ) { - - // #4087 - If origin and destination elements are the same, and this is - // that element, do not do anything - if ( selection && jQuery.inArray( elem, selection ) !== -1 ) { - continue; - } - - contains = jQuery.contains( elem.ownerDocument, elem ); - - // Append to fragment - tmp = getAll( safe.appendChild( elem ), "script" ); - - // Preserve script evaluation history - if ( contains ) { - setGlobalEval( tmp ); - } - - // Capture executables - if ( scripts ) { - j = 0; - while ( (elem = tmp[ j++ ]) ) { - if ( rscriptType.test( elem.type || "" ) ) { - scripts.push( elem ); - } - } - } - } - - tmp = null; - - return safe; - }, - - cleanData: function( elems, /* internal */ acceptData ) { - var elem, type, id, data, - i = 0, - internalKey = jQuery.expando, - cache = jQuery.cache, - deleteExpando = support.deleteExpando, - special = jQuery.event.special; - - for ( ; (elem = elems[i]) != null; i++ ) { - if ( acceptData || jQuery.acceptData( elem ) ) { - - id = elem[ internalKey ]; - data = id && cache[ id ]; - - if ( data ) { - if ( data.events ) { - for ( type in data.events ) { - if ( special[ type ] ) { - jQuery.event.remove( elem, type ); - - // This is a shortcut to avoid jQuery.event.remove's overhead - } else { - jQuery.removeEvent( elem, type, data.handle ); - } - } - } - - // Remove cache only if it was not already removed by jQuery.event.remove - if ( cache[ id ] ) { - - delete cache[ id ]; - - // IE does not allow us to delete expando properties from nodes, - // nor does it have a removeAttribute function on Document nodes; - // we must handle all of these cases - if ( deleteExpando ) { - delete elem[ internalKey ]; - - } else if ( typeof elem.removeAttribute !== strundefined ) { - elem.removeAttribute( internalKey ); - - } else { - elem[ internalKey ] = null; - } - - deletedIds.push( id ); - } - } - } - } - } -}); - -jQuery.fn.extend({ - text: function( value ) { - return access( this, function( value ) { - return value === undefined ? - jQuery.text( this ) : - this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) ); - }, null, value, arguments.length ); - }, - - append: function() { - return this.domManip( arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.appendChild( elem ); - } - }); - }, - - prepend: function() { - return this.domManip( arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.insertBefore( elem, target.firstChild ); - } - }); - }, - - before: function() { - return this.domManip( arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this ); - } - }); - }, - - after: function() { - return this.domManip( arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this.nextSibling ); - } - }); - }, - - remove: function( selector, keepData /* Internal Use Only */ ) { - var elem, - elems = selector ? jQuery.filter( selector, this ) : this, - i = 0; - - for ( ; (elem = elems[i]) != null; i++ ) { - - if ( !keepData && elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem ) ); - } - - if ( elem.parentNode ) { - if ( keepData && jQuery.contains( elem.ownerDocument, elem ) ) { - setGlobalEval( getAll( elem, "script" ) ); - } - elem.parentNode.removeChild( elem ); - } - } - - return this; - }, - - empty: function() { - var elem, - i = 0; - - for ( ; (elem = this[i]) != null; i++ ) { - // Remove element nodes and prevent memory leaks - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - } - - // Remove any remaining nodes - while ( elem.firstChild ) { - elem.removeChild( elem.firstChild ); - } - - // If this is a select, ensure that it displays empty (#12336) - // Support: IE<9 - if ( elem.options && jQuery.nodeName( elem, "select" ) ) { - elem.options.length = 0; - } - } - - return this; - }, - - clone: function( dataAndEvents, deepDataAndEvents ) { - dataAndEvents = dataAndEvents == null ? false : dataAndEvents; - deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; - - return this.map(function() { - return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); - }); - }, - - html: function( value ) { - return access( this, function( value ) { - var elem = this[ 0 ] || {}, - i = 0, - l = this.length; - - if ( value === undefined ) { - return elem.nodeType === 1 ? - elem.innerHTML.replace( rinlinejQuery, "" ) : - undefined; - } - - // See if we can take a shortcut and just use innerHTML - if ( typeof value === "string" && !rnoInnerhtml.test( value ) && - ( support.htmlSerialize || !rnoshimcache.test( value ) ) && - ( support.leadingWhitespace || !rleadingWhitespace.test( value ) ) && - !wrapMap[ (rtagName.exec( value ) || [ "", "" ])[ 1 ].toLowerCase() ] ) { - - value = value.replace( rxhtmlTag, "<$1>" ); - - try { - for (; i < l; i++ ) { - // Remove element nodes and prevent memory leaks - elem = this[i] || {}; - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - elem.innerHTML = value; - } - } - - elem = 0; - - // If using innerHTML throws an exception, use the fallback method - } catch(e) {} - } - - if ( elem ) { - this.empty().append( value ); - } - }, null, value, arguments.length ); - }, - - replaceWith: function() { - var arg = arguments[ 0 ]; - - // Make the changes, replacing each context element with the new content - this.domManip( arguments, function( elem ) { - arg = this.parentNode; - - jQuery.cleanData( getAll( this ) ); - - if ( arg ) { - arg.replaceChild( elem, this ); - } - }); - - // Force removal if there was no new content (e.g., from empty arguments) - return arg && (arg.length || arg.nodeType) ? this : this.remove(); - }, - - detach: function( selector ) { - return this.remove( selector, true ); - }, - - domManip: function( args, callback ) { - - // Flatten any nested arrays - args = concat.apply( [], args ); - - var first, node, hasScripts, - scripts, doc, fragment, - i = 0, - l = this.length, - set = this, - iNoClone = l - 1, - value = args[0], - isFunction = jQuery.isFunction( value ); - - // We can't cloneNode fragments that contain checked, in WebKit - if ( isFunction || - ( l > 1 && typeof value === "string" && - !support.checkClone && rchecked.test( value ) ) ) { - return this.each(function( index ) { - var self = set.eq( index ); - if ( isFunction ) { - args[0] = value.call( this, index, self.html() ); - } - self.domManip( args, callback ); - }); - } - - if ( l ) { - fragment = jQuery.buildFragment( args, this[ 0 ].ownerDocument, false, this ); - first = fragment.firstChild; - - if ( fragment.childNodes.length === 1 ) { - fragment = first; - } - - if ( first ) { - scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); - hasScripts = scripts.length; - - // Use the original fragment for the last item instead of the first because it can end up - // being emptied incorrectly in certain situations (#8070). - for ( ; i < l; i++ ) { - node = fragment; - - if ( i !== iNoClone ) { - node = jQuery.clone( node, true, true ); - - // Keep references to cloned scripts for later restoration - if ( hasScripts ) { - jQuery.merge( scripts, getAll( node, "script" ) ); - } - } - - callback.call( this[i], node, i ); - } - - if ( hasScripts ) { - doc = scripts[ scripts.length - 1 ].ownerDocument; - - // Reenable scripts - jQuery.map( scripts, restoreScript ); - - // Evaluate executable scripts on first document insertion - for ( i = 0; i < hasScripts; i++ ) { - node = scripts[ i ]; - if ( rscriptType.test( node.type || "" ) && - !jQuery._data( node, "globalEval" ) && jQuery.contains( doc, node ) ) { - - if ( node.src ) { - // Optional AJAX dependency, but won't run scripts if not present - if ( jQuery._evalUrl ) { - jQuery._evalUrl( node.src ); - } - } else { - jQuery.globalEval( ( node.text || node.textContent || node.innerHTML || "" ).replace( rcleanScript, "" ) ); - } - } - } - } - - // Fix #11809: Avoid leaking memory - fragment = first = null; - } - } - - return this; - } -}); - -jQuery.each({ - appendTo: "append", - prependTo: "prepend", - insertBefore: "before", - insertAfter: "after", - replaceAll: "replaceWith" -}, function( name, original ) { - jQuery.fn[ name ] = function( selector ) { - var elems, - i = 0, - ret = [], - insert = jQuery( selector ), - last = insert.length - 1; - - for ( ; i <= last; i++ ) { - elems = i === last ? this : this.clone(true); - jQuery( insert[i] )[ original ]( elems ); - - // Modern browsers can apply jQuery collections as arrays, but oldIE needs a .get() - push.apply( ret, elems.get() ); - } - - return this.pushStack( ret ); - }; -}); - - -var iframe, - elemdisplay = {}; - -/** - * Retrieve the actual display of a element - * @param {String} name nodeName of the element - * @param {Object} doc Document object - */ -// Called only from within defaultDisplay -function actualDisplay( name, doc ) { - var elem = jQuery( doc.createElement( name ) ).appendTo( doc.body ), - - // getDefaultComputedStyle might be reliably used only on attached element - display = window.getDefaultComputedStyle ? - - // Use of this method is a temporary fix (more like optmization) until something better comes along, - // since it was removed from specification and supported only in FF - window.getDefaultComputedStyle( elem[ 0 ] ).display : jQuery.css( elem[ 0 ], "display" ); - - // We don't have any data stored on the element, - // so use "detach" method as fast way to get rid of the element - elem.detach(); - - return display; -} - -/** - * Try to determine the default display value of an element - * @param {String} nodeName - */ -function defaultDisplay( nodeName ) { - var doc = document, - display = elemdisplay[ nodeName ]; - - if ( !display ) { - display = actualDisplay( nodeName, doc ); - - // If the simple way fails, read from inside an iframe - if ( display === "none" || !display ) { - - // Use the already-created iframe if possible - iframe = (iframe || jQuery( "