diff --git a/.circleci/config.yml b/.circleci/config.yml index e75a6ba0f..9f7381f33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,10 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT + # The default trial args include --rterrors which is incompatible with + # this reporter on Python 3. So drop that and just specify the + # reporter. + TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" TAHOE_LAFS_TOX_ENVIRONMENT: "py36" diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 8894b9691..bf1b6ee8e 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -65,7 +65,7 @@ TIMEOUT="timeout --kill-after 1m 15m" # Send the output directly to a file because transporting the binary subunit2 # via tox and then scraping it out is hideous and failure prone. export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" -export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors" +export TAHOE_LAFS_TRIAL_ARGS="${TAHOE_LAFS_TRIAL_ARGS:---reporter=subunitv2-file --rterrors}" export PIP_NO_INDEX="1" if [ "${ALLOWED_FAILURE}" = "yes" ]; then diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing deleted file mode 100644 index 78e9e29ae..000000000 --- a/misc/python3/ratchet-passing +++ /dev/null @@ -1,236 +0,0 @@ -allmydata.test.mutable.test_exceptions.Exceptions.test_repr -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_1s -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_25s -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_day -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_future_5_minutes -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_hours -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_month -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year -allmydata.test.test_abbreviate.Abbreviate.test_parse_space -allmydata.test.test_abbreviate.Abbreviate.test_space -allmydata.test.test_abbreviate.Abbreviate.test_time -allmydata.test.test_backupdb.BackupDB.test_basic -allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2 -allmydata.test.test_backupdb.BackupDB.test_wrong_version -allmydata.test.test_base32.Base32.test_a2b -allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons -allmydata.test.test_base32.Base32.test_b2a -allmydata.test.test_base32.Base32.test_b2a_or_none -allmydata.test.test_base62.Base62.test_ende_0x00 -allmydata.test.test_base62.Base62.test_ende_0x000000 -allmydata.test.test_base62.Base62.test_ende_0x01 -allmydata.test.test_base62.Base62.test_ende_0x0100 -allmydata.test.test_base62.Base62.test_ende_0x010000 -allmydata.test.test_base62.Base62.test_ende_longrandstr -allmydata.test.test_base62.Base62.test_ende_randstr -allmydata.test.test_base62.Base62.test_known_values -allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars -allmydata.test.test_base62.Base62.test_odd_sizes -allmydata.test.test_base62.Base62.test_roundtrip -allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes -allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes -allmydata.test.test_crypto.TestEd25519.test_key_serialization -allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey -allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes -allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes -allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes -allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey -allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input -allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input -allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input -allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input -allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair -allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair -allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes -allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size -allmydata.test.test_crypto.TestRegression.test_iv_not_bytes -allmydata.test.test_crypto.TestRegression.test_key_incorrect_size -allmydata.test.test_crypto.TestRegression.test_old_start_up_test -allmydata.test.test_crypto.TestRsa.test_keys -allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey -allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey -allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad -allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string -allmydata.test.test_crypto.TestUtil.test_remove_prefix_good -allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial -allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero -allmydata.test.test_deferredutil.DeferredUtilTests.test_failure -allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results -allmydata.test.test_deferredutil.DeferredUtilTests.test_success -allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls -allmydata.test.test_dictutil.DictUtil.test_auxdict -allmydata.test.test_dictutil.DictUtil.test_dict_of_sets -allmydata.test.test_encodingutil.EncodingUtilErrors.test_argv_to_unicode -allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding -allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding_not_from_stdout -allmydata.test.test_encodingutil.EncodingUtilErrors.test_no_unicode_normalization -allmydata.test.test_encodingutil.EncodingUtilErrors.test_unicode_to_output -allmydata.test.test_encodingutil.FilePaths.test_extend_filepath -allmydata.test.test_encodingutil.FilePaths.test_to_filepath -allmydata.test.test_encodingutil.FilePaths.test_unicode_from_filepath -allmydata.test.test_encodingutil.FilePaths.test_unicode_segments_from -allmydata.test.test_encodingutil.MacOSXLeopard.test_argv_to_unicode -allmydata.test.test_encodingutil.MacOSXLeopard.test_listdir_unicode -allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_platform_py3 -allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_argv_py3 -allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_output -allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_url -allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_argv_to_unicode -allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_listdir_unicode -allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_platform_py3 -allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_argv_py3 -allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_output -allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_url -allmydata.test.test_encodingutil.OpenBSD.test_argv_to_unicode -allmydata.test.test_encodingutil.OpenBSD.test_listdir_unicode -allmydata.test.test_encodingutil.OpenBSD.test_unicode_platform_py3 -allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_argv_py3 -allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_output -allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_url -allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_ascii -allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_default -allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_latin1 -allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_utf8 -allmydata.test.test_encodingutil.QuotePaths.test_quote_filepath -allmydata.test.test_encodingutil.QuotePaths.test_quote_path -allmydata.test.test_encodingutil.StdlibUnicode.test_mkdir_open_exists_abspath_listdir_expanduser -allmydata.test.test_encodingutil.TestToFromStr.test_from_utf8_or_none -allmydata.test.test_encodingutil.TestToFromStr.test_to_str -allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_argv_to_unicode -allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_listdir_unicode -allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_platform_py3 -allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_argv_py3 -allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_output -allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_url -allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_argv_to_unicode -allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_listdir_unicode -allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_platform_py3 -allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_argv_py3 -allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_output -allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_url -allmydata.test.test_encodingutil.Windows.test_argv_to_unicode -allmydata.test.test_encodingutil.Windows.test_unicode_platform_py3 -allmydata.test.test_encodingutil.Windows.test_unicode_to_argv_py3 -allmydata.test.test_encodingutil.Windows.test_unicode_to_output -allmydata.test.test_encodingutil.Windows.test_unicode_to_url -allmydata.test.test_happiness.Happiness.test_100 -allmydata.test.test_happiness.Happiness.test_calc_happy -allmydata.test.test_happiness.Happiness.test_everything_broken -allmydata.test.test_happiness.Happiness.test_hypothesis0 -allmydata.test.test_happiness.Happiness.test_hypothesis_0 -allmydata.test.test_happiness.Happiness.test_hypothesis_1 -allmydata.test.test_happiness.Happiness.test_placement_1 -allmydata.test.test_happiness.Happiness.test_placement_simple -allmydata.test.test_happiness.Happiness.test_redistribute -allmydata.test.test_happiness.Happiness.test_unhappy -allmydata.test.test_happiness.HappinessUtils.test_residual_0 -allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph -allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph -allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy -allmydata.test.test_happiness.PlacementTests.test_more_hypothesis -allmydata.test.test_hashtree.Complete.test_create -allmydata.test.test_hashtree.Complete.test_dump -allmydata.test.test_hashtree.Complete.test_needed_hashes -allmydata.test.test_hashtree.Incomplete.test_check -allmydata.test.test_hashtree.Incomplete.test_create -allmydata.test.test_hashtree.Incomplete.test_depth_of -allmydata.test.test_hashtree.Incomplete.test_large -allmydata.test.test_hashtree.Incomplete.test_needed_hashes -allmydata.test.test_hashutil.HashUtilTests.test_chk -allmydata.test.test_hashutil.HashUtilTests.test_hashers -allmydata.test.test_hashutil.HashUtilTests.test_known_answers -allmydata.test.test_hashutil.HashUtilTests.test_random_key -allmydata.test.test_hashutil.HashUtilTests.test_sha256d -allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated -allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare -allmydata.test.test_humanreadable.HumanReadable.test_repr -allmydata.test.test_iputil.GcUtil.test_gc_after_allocations -allmydata.test.test_iputil.GcUtil.test_release_delays_gc -allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for -allmydata.test.test_iputil.ListAddresses.test_list_async -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route -allmydata.test.test_iputil.ListenOnUsed.test_random_port -allmydata.test.test_iputil.ListenOnUsed.test_specific_port -allmydata.test.test_log.Log.test_default_facility -allmydata.test.test_log.Log.test_err -allmydata.test.test_log.Log.test_grandparent_id -allmydata.test.test_log.Log.test_no_prefix -allmydata.test.test_log.Log.test_numming -allmydata.test.test_log.Log.test_parent_id -allmydata.test.test_log.Log.test_with_bytes_prefix -allmydata.test.test_log.Log.test_with_prefix -allmydata.test.test_netstring.Netstring.test_encode -allmydata.test.test_netstring.Netstring.test_extra -allmydata.test.test_netstring.Netstring.test_nested -allmydata.test.test_netstring.Netstring.test_split -allmydata.test.test_observer.Observer.test_lazy_oneshot -allmydata.test.test_observer.Observer.test_observerlist -allmydata.test.test_observer.Observer.test_oneshot -allmydata.test.test_observer.Observer.test_oneshot_fireagain -allmydata.test.test_pipeline.Pipeline.test_basic -allmydata.test.test_pipeline.Pipeline.test_errors -allmydata.test.test_pipeline.Pipeline.test_errors2 -allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting -allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct -allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist -allmydata.test.test_spans.ByteSpans.test_basic -allmydata.test.test_spans.ByteSpans.test_large -allmydata.test.test_spans.ByteSpans.test_math -allmydata.test.test_spans.ByteSpans.test_overlap -allmydata.test.test_spans.ByteSpans.test_random -allmydata.test.test_spans.StringSpans.test_basic -allmydata.test.test_spans.StringSpans.test_random -allmydata.test.test_spans.StringSpans.test_test -allmydata.test.test_statistics.Statistics.test_binomial_coeff -allmydata.test.test_statistics.Statistics.test_binomial_distribution_pmf -allmydata.test.test_statistics.Statistics.test_convolve -allmydata.test.test_statistics.Statistics.test_find_k -allmydata.test.test_statistics.Statistics.test_pr_backup_file_loss -allmydata.test.test_statistics.Statistics.test_pr_file_loss -allmydata.test.test_statistics.Statistics.test_repair_cost -allmydata.test.test_statistics.Statistics.test_repair_count_pmf -allmydata.test.test_statistics.Statistics.test_survival_pmf -allmydata.test.test_time_format.TimeFormat.test_epoch -allmydata.test.test_time_format.TimeFormat.test_epoch_in_London -allmydata.test.test_time_format.TimeFormat.test_format_delta -allmydata.test.test_time_format.TimeFormat.test_format_time -allmydata.test.test_time_format.TimeFormat.test_format_time_y2038 -allmydata.test.test_time_format.TimeFormat.test_iso_utc -allmydata.test.test_time_format.TimeFormat.test_parse_date -allmydata.test.test_time_format.TimeFormat.test_parse_duration -allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode -allmydata.test.test_util.FileUtil.test_create_long_path -allmydata.test.test_util.FileUtil.test_disk_stats -allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative -allmydata.test.test_util.FileUtil.test_du -allmydata.test.test_util.FileUtil.test_encrypted_tempfile -allmydata.test.test_util.FileUtil.test_get_pathinfo -allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink -allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode -allmydata.test.test_util.FileUtil.test_remove_if_possible -allmydata.test.test_util.FileUtil.test_rename -allmydata.test.test_util.FileUtil.test_rename_no_overwrite -allmydata.test.test_util.FileUtil.test_replace_file -allmydata.test.test_util.FileUtil.test_rm_dir -allmydata.test.test_util.FileUtil.test_windows_expanduser_win7 -allmydata.test.test_util.FileUtil.test_windows_expanduser_xp -allmydata.test.test_util.FileUtil.test_write_atomically -allmydata.test.test_util.IDLib.test_nodeid_b2a -allmydata.test.test_util.Math.test_round_sigfigs -allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True -allmydata.test.test_util.PollMixinTests.test_PollMixin_True -allmydata.test.test_util.PollMixinTests.test_timeout -allmydata.test.test_util.YAML.test_convert -allmydata.test.test_version.CheckRequirement.test_cross_check -allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions -allmydata.test.test_version.CheckRequirement.test_extract_openssl_version -allmydata.test.test_version.CheckRequirement.test_packages_from_pkg_resources -allmydata.test.test_version.T.test_report_import_error -allmydata.test.test_version.VersionTestCase.test_basic_versions -allmydata.test.test_version.VersionTestCase.test_comparison -allmydata.test.test_version.VersionTestCase.test_from_parts -allmydata.test.test_version.VersionTestCase.test_irrational_versions -allmydata.test.test_version.VersionTestCase.test_suggest_normalized_version diff --git a/misc/python3/ratchet.py b/misc/python3/ratchet.py deleted file mode 100755 index cb672cf67..000000000 --- a/misc/python3/ratchet.py +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -'''Ratchet up passing tests, or ratchet down failing tests. - -Usage: - - ratchet.py <"up" or "down"> - -This script helps when you expect a large test suite to fail spectactularly in -some environment, and you want to gradually improve the situation with minimal -impact to forward development of the same codebase for other environments. The -initial and primary usecase is porting from Python 2 to Python 3. - -The idea is to emit JUnit XML from your test runner, and then invoke ratchet.py -to consume this XML output and operate on a so-called "tracking" file. When -ratcheting up passing tests, the tracking file will contain a list of tests, -one per line, that passed. When ratching down, the tracking file contains a -list of failing tests. On each subsequent run, ratchet.py will compare the -prior results in the tracking file with the new results in the XML, and will -report on both welcome and unwelcome changes. It will modify the tracking file -in the case of welcome changes, and therein lies the ratcheting. - -The exit codes are: - - 0 - no changes observed - 1 - changes observed, whether welcome or unwelcome - 2 - invocation error - -If does not exist, you'll get a FileNotFoundError: - - >>> _test('up', None, None) # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - FileNotFoundError: ... - -If does not exist, that's fine: - - >>> _test('up', '1', None) - Some tests not required to pass did: - c0.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Eep! 0 test(s) were required to pass, but instead 1 did. 🐭 - -Same if you're ratcheting down: - - >>> _test('down', '1', None) - All and only tests expected to fail did. 💃 - -If the test run has the same output as last time, it's all good: - - >>> _test('up', '01001110', '01001110') - All and only tests required to pass did. 💃 - - >>> _test('down', '01001110', '10110001') - All and only tests expected to fail did. 💃 - -If there's a welcome change, that's noted: - - >>> _test('up', '0101', '0100') - Some tests not required to pass did: - c3.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Eep! 1 test(s) were required to pass, but instead 2 did. 🐭 - - >>> _test('down', '0011', '1110') - Some tests expected to fail didn't: - c2.t - Conveniently, they have been removed from `` for you. Perhaps commit that? - Eep! 3 test(s) were expected to fail, but instead 2 did. 🐭 - -And if there is an unwelcome change, that is noted as well: - - >>> _test('up', '1101', '1111') - Some tests required to pass didn't: - c2.t - Eep! 4 test(s) were required to pass, but instead 3 did. 🐭 - - >>> _test('down', '0000', '1101') - Some tests not expected to fail did: - c2.t - Eep! 3 test(s) were expected to fail, but instead 4 did. 🐭 - -And if there are both welcome and unwelcome changes, they are both noted: - - >>> _test('up', '1101', '1011') - Some tests not required to pass did: - c1.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Some tests required to pass didn't: - c2.t - Eep! 3 test(s) were required to pass, but instead 3 did. 🐭 - - >>> _test('down', '0100', '1100') - Some tests not expected to fail did: - c2.t - c3.t - Some tests expected to fail didn't: - c1.t - Conveniently, they have been removed from `` for you. Perhaps commit that? - Eep! 2 test(s) were expected to fail, but instead 3 did. 🐭 - - -To test ratchet.py itself: - - python3 -m doctest ratchet.py - -''' -from __future__ import absolute_import, division, print_function, unicode_literals - -import io -import os -import re -import sys -import tempfile -import xml.etree.ElementTree as Etree - - -class JUnitXMLFile(object): - '''Represent a file containing test results in JUnit XML format. - - >>> eg = _mktemp_junitxml('0100111') - >>> results = JUnitXMLFile(eg.name).parse() - >>> results.failed - ['c0.t', 'c2.t', 'c3.t'] - >>> results.passed - ['c1.t', 'c4.t', 'c5.t', 'c6.t'] - - ''' - - def __init__(self, filepath): - self.filepath = filepath - self.failed = [] - self.failed_aggregates = {} - self.stderr_output = [] - self.passed = [] - self._tree = None - - def parse(self): - if self._tree: - raise RuntimeError('already parsed') - self._tree = Etree.parse(self.filepath) - for testcase in self._tree.findall('testcase'): - self.process_testcase(testcase) - return self - - def process_testcase(self, case): - key = self.case_key(case) - - # look at children but throw away stderr output - nonpassing = [c for c in case if not c.tag == 'system-err'] - n = len(nonpassing) - if n > 1: - raise RuntimeError(f'multiple results for {key}: {nonpassing}') - elif n == 1: - result = nonpassing.pop() - self.failed.append(key) - message = result.get('message') - self.failed_aggregates.setdefault(message, []).append(key) - else: - self.passed.append(key) - - @staticmethod - def case_key(case): - return f'{case.get("classname")}.{case.get("name")}' - - def report(self, details=False): - for k, v in sorted( - self.failed_aggregates.items(), - key = lambda i: len(i[1]), - reverse=True): - print(f'# {k}') - for t in v: - print(f' - {t}') - - -def load_previous_results(txt): - try: - previous_results = open(txt).read() - except FileNotFoundError: - previous_results = '' - parsed = set() - for line in previous_results.splitlines(): - if not line or line.startswith('#'): - continue - parsed.add(line) - return parsed - - -def print_tests(tests): - for test in sorted(tests): - print(' ', test) - - -def ratchet_up_passing(tracking_path, tests): - try: - old = set(open(tracking_path, 'r')) - except FileNotFoundError: - old = set() - new = set(t + '\n' for t in tests) - merged = sorted(old | new) - open(tracking_path, 'w+').writelines(merged) - - -def ratchet_down_failing(tracking_path, tests): - new = set(t + '\n' for t in tests) - open(tracking_path, 'w+').writelines(sorted(new)) - - -def main(direction, junitxml_path, tracking_path): - '''Takes a string indicating which direction to ratchet, "up" or "down," - and two paths, one to test-runner output in JUnit XML format, the other to - a file tracking test results (one test case dotted name per line). Walk the - former looking for the latter, and react appropriately. - - >>> inp = _mktemp_junitxml('0100111') - >>> out = _mktemp_tracking('0000000') - >>> _test_main('up', inp.name, out.name) - Some tests not required to pass did: - c1.t - c4.t - c5.t - c6.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Eep! 0 test(s) were required to pass, but instead 4 did. 🐭 - - ''' - - results = JUnitXMLFile(junitxml_path).parse() - - if tracking_path == '...': - # Shortcut to aid in debugging XML parsing issues. - results.report() - return - - previous = load_previous_results(tracking_path) - current = set(results.passed if direction == 'up' else results.failed) - - subjunctive = {'up': 'required to pass', 'down': 'expected to fail'}[direction] - ratchet = None - - too_many = current - previous - if too_many: - print(f'Some tests not {subjunctive} did:') - print_tests(too_many) - if direction == 'up': - # Too many passing tests is good -- let's do more of those! - ratchet_up_passing(tracking_path, current) - print(f'Conveniently, they have been added to `{tracking_path}` for you. Perhaps commit that?') - - not_enough = previous - current - if not_enough: - print(f'Some tests {subjunctive} didn\'t:') - print_tests(not_enough) - if direction == 'down': - # Not enough failing tests is good -- let's do more of those! - ratchet_down_failing(tracking_path, current) - print(f'Conveniently, they have been removed from `{tracking_path}` for you. Perhaps commit that?') - - if too_many or not_enough: - print(f'Eep! {len(previous)} test(s) were {subjunctive}, but instead {len(current)} did. 🐭') - return 1 - - print(f'All and only tests {subjunctive} did. 💃') - return 0 - - -# When called as an executable ... - -if __name__ == '__main__': - try: - direction, junitxml_path, tracking_path = sys.argv[1:4] - if direction not in ('up', 'down'): - raise ValueError - except ValueError: - doc = '\n'.join(__doc__.splitlines()[:6]) - doc = re.sub(' ratchet.py', f' {sys.argv[0]}', doc) - print(doc, file=sys.stderr) - exit_code = 2 - else: - exit_code = main(direction, junitxml_path, tracking_path) - sys.exit(exit_code) - - -# Helpers for when called under doctest ... - -def _test(*a): - return _test_main(*_mk(*a)) - - -def _test_main(direction, junitxml, tracking): - '''Takes a string 'up' or 'down' and paths to (or open file objects for) - the JUnit XML and tracking files to use for this test run. Captures and - emits stdout (slightly modified) for inspection via doctest.''' - junitxml_path = junitxml.name if hasattr(junitxml, 'name') else junitxml - tracking_path = tracking.name if hasattr(tracking, 'name') else tracking - - old_stdout = sys.stdout - sys.stdout = io.StringIO() - try: - main(direction, junitxml_path, tracking_path) - finally: - sys.stdout.seek(0) - out = sys.stdout.read() - out = re.sub('`.*?`', '``', out).strip() - sys.stdout = old_stdout - print(out) - - -class _PotentialFile(object): - '''Represent a file that we are able to create but which doesn't exist yet, - and which, if we create it, will be automatically torn down when the test - run is over.''' - - def __init__(self, filename): - self.d = tempfile.TemporaryDirectory() - self.name = os.path.join(self.d.name, filename) - - -def _mk(direction, spec_junitxml, spec_tracking): - '''Takes a string 'up' or 'down' and two bit strings specifying the state - of the JUnit XML results file and the tracking file to set up for this test - case. Returns the direction (unharmed) and two file-ish objects. - - If a spec string is None the corresponding return value will be a - _PotentialFile object, which has a .name attribute (like a true file - object) that points to a file that does not exist, but could. - - The reason not to simply return the path in all cases is that the file - objects are actually temporary file objects that destroy the underlying - file when they go out of scope, and we want to keep the underlying file - around until the end of the test run.''' - - if None not in(spec_junitxml, spec_tracking): - if len(spec_junitxml) != len(spec_tracking): - raise ValueError('if both given, must be the same length: `{spec_junitxml}` and `{spec_tracking}`') - if spec_junitxml is None: - junitxml_fp = _PotentialFile('results.xml') - else: - junitxml_fp = _mktemp_junitxml(spec_junitxml) - if spec_tracking is None: - tracking_fp = _PotentialFile('tracking') - else: - tracking_fp = _mktemp_tracking(spec_tracking) - return direction, junitxml_fp, tracking_fp - - -def _mktemp_junitxml(spec): - '''Test helper to generate a raw JUnit XML file. - - >>> fp = _mktemp_junitxml('00101') - >>> open(fp.name).read()[:11] - '' - - ''' - fp = tempfile.NamedTemporaryFile() - fp.write(b'') - - passed = '''\ - -''' - failed = '''\ - -Traceback (most recent call last): - File "/foo/bar/baz/buz.py", line 1, in <module> -NameError: name 'heck' is not defined - - -''' - - i = 0 - for c in spec: - if c == '0': - out = failed - elif c == '1': - out = passed - else: - raise ValueError(f'bad c: `{c}`') - fp.write(out.format(i=i).encode('utf8')) - i += 1 - - fp.write(b'') - fp.flush() - return fp - - -def _mktemp_tracking(spec): - '''Test helper to prefabricate a tracking file. - - >>> fp = _mktemp_tracking('01101') - >>> print(open(fp.name).read()[:-1]) - c1.t - c2.t - c4.t - - ''' - fp = tempfile.NamedTemporaryFile() - - i = 0 - for c in spec: - if c == '0': - pass - elif c == '1': - fp.write(f'c{i}.t\n'.encode('utf8')) - else: - raise ValueError(f'bad c: `{c}`') - i += 1 - - fp.flush() - return fp diff --git a/newsfragments/3316.minor b/newsfragments/3316.minor new file mode 100644 index 000000000..9457b486e --- /dev/null +++ b/newsfragments/3316.minor @@ -0,0 +1 @@ +Port checker result pages' rendering from nevow to twisted web templates. diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor index e69de29bb..8b1378917 100644 --- a/newsfragments/3372.minor +++ b/newsfragments/3372.minor @@ -0,0 +1 @@ + diff --git a/src/allmydata/test/python3_tests.py b/src/allmydata/test/python3_tests.py new file mode 100644 index 000000000..9326caa51 --- /dev/null +++ b/src/allmydata/test/python3_tests.py @@ -0,0 +1,37 @@ +""" +This module defines the subset of the full test suite which is expected to +pass on Python 3 in a way which makes that suite discoverable by trial. + +This module has been ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from twisted.python.reflect import ( + namedModule, +) +from twisted.trial.runner import ( + TestLoader, +) +from twisted.trial.unittest import ( + TestSuite, +) + +from allmydata.util._python3 import ( + PORTED_TEST_MODULES, +) + +def testSuite(): + loader = TestLoader() + return TestSuite(list( + loader.loadModule(namedModule(module)) + for module + in PORTED_TEST_MODULES + )) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 5eed6f21f..2296194f0 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -1,10 +1,25 @@ import json import os.path, shutil + +from bs4 import BeautifulSoup + from twisted.trial import unittest from twisted.internet import defer + +from nevow.inevow import IRequest +from zope.interface import implementer +from twisted.web.server import Request +from twisted.web.test.requesthelper import DummyChannel +from twisted.web.template import flattenString + from allmydata import check_results, uri from allmydata import uri as tahoe_uri +from allmydata.interfaces import ( + IServer, + ICheckResults, + ICheckAndRepairResults, +) from allmydata.util import base32 from allmydata.web import check_results as web_check_results from allmydata.storage_client import StorageFarmBroker, NativeStorageServer @@ -12,18 +27,115 @@ from allmydata.storage.server import storage_index_to_dir from allmydata.monitor import Monitor from allmydata.test.no_network import GridTestMixin from allmydata.immutable.upload import Data -from allmydata.test.common_web import WebRenderingMixin from allmydata.mutable.publish import MutableData from .common import ( EMPTY_CLIENT_CONFIG, ) +from .web.common import ( + assert_soup_has_favicon, + assert_soup_has_tag_with_content, +) + class FakeClient(object): def get_storage_broker(self): return self.storage_broker -class WebResultsRendering(unittest.TestCase, WebRenderingMixin): +@implementer(IRequest) +class TestRequest(Request, object): + """ + A minimal Request class to use in tests. + + XXX: We have to have this class because `common.get_arg()` expects + a `nevow.inevow.IRequest`, which `twisted.web.server.Request` + isn't. The request needs to have `args`, `fields`, `prepath`, and + `postpath` properties so that `allmydata.web.common.get_arg()` + won't complain. + """ + def __init__(self, args=None, fields=None): + super(TestRequest, self).__init__(DummyChannel()) + self.args = args or {} + self.fields = fields or {} + self.prepath = [b""] + self.postpath = [b""] + + +@implementer(IServer) +class FakeServer(object): + + def get_name(self): + return "fake name" + + def get_longname(self): + return "fake longname" + + def get_nickname(self): + return "fake nickname" + + +@implementer(ICheckResults) +class FakeCheckResults(object): + + def __init__(self, si=None, + healthy=False, recoverable=False, + summary="fake summary"): + self._storage_index = si + self._is_healthy = healthy + self._is_recoverable = recoverable + self._summary = summary + + def get_storage_index(self): + return self._storage_index + + def get_storage_index_string(self): + return base32.b2a_or_none(self._storage_index) + + def is_healthy(self): + return self._is_healthy + + def is_recoverable(self): + return self._is_recoverable + + def get_summary(self): + return self._summary + + def get_corrupt_shares(self): + # returns a list of (IServer, storage_index, sharenum) + return [(FakeServer(), "", 0)] + + +@implementer(ICheckAndRepairResults) +class FakeCheckAndRepairResults(object): + + def __init__(self, si=None, + repair_attempted=False, + repair_success=False): + self._storage_index = si + self._repair_attempted = repair_attempted + self._repair_success = repair_success + + def get_storage_index(self): + return self._storage_index + + def get_pre_repair_results(self): + return FakeCheckResults() + + def get_post_repair_results(self): + return FakeCheckResults() + + def get_repair_attempted(self): + return self._repair_attempted + + def get_repair_successful(self): + return self._repair_success + + +class WebResultsRendering(unittest.TestCase): + + @staticmethod + def remove_tags(html): + return BeautifulSoup(html).get_text(separator=" ") def create_fake_client(self): sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) @@ -51,34 +163,31 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): c.storage_broker = sb return c - def render_json(self, page): - d = self.render1(page, args={"output": ["json"]}) - return d + def render_json(self, resource): + return resource.render(TestRequest(args={"output": ["json"]})) + + def render_element(self, element, args=None): + d = flattenString(TestRequest(args), element) + return unittest.TestCase().successResultOf(d) def test_literal(self): + lcr = web_check_results.LiteralCheckResultsRendererElement() + + html = self.render_element(lcr) + self.failUnlessIn("Literal files are always healthy", html) + + html = self.render_element(lcr, args={"return_to": ["FOOURL"]}) + self.failUnlessIn("Literal files are always healthy", html) + self.failUnlessIn('Return to file.', html) + c = self.create_fake_client() lcr = web_check_results.LiteralCheckResultsRenderer(c) - d = self.render1(lcr) - def _check(html): - s = self.remove_tags(html) - self.failUnlessIn("Literal files are always healthy", s) - d.addCallback(_check) - d.addCallback(lambda ignored: - self.render1(lcr, args={"return_to": ["FOOURL"]})) - def _check_return_to(html): - s = self.remove_tags(html) - self.failUnlessIn("Literal files are always healthy", s) - self.failUnlessIn('Return to file.', - html) - d.addCallback(_check_return_to) - d.addCallback(lambda ignored: self.render_json(lcr)) - def _check_json(js): - j = json.loads(js) - self.failUnlessEqual(j["storage-index"], "") - self.failUnlessEqual(j["results"]["healthy"], True) - d.addCallback(_check_json) - return d + js = self.render_json(lcr) + j = json.loads(js) + self.failUnlessEqual(j["storage-index"], "") + self.failUnlessEqual(j["results"]["healthy"], True) + def test_check(self): c = self.create_fake_client() @@ -108,8 +217,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): healthy=True, recoverable=True, summary="groovy", **data) - w = web_check_results.CheckResultsRenderer(c, cr) - html = self.render2(w) + w = web_check_results.CheckResultsRendererElement(c, cr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Healthy : groovy", s) @@ -120,14 +229,14 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): self.failUnlessIn("Wrong Shares: 0", s) self.failUnlessIn("Recoverable Versions: 1", s) self.failUnlessIn("Unrecoverable Versions: 0", s) - self.failUnlessIn("Good Shares (sorted in share order): Share ID Nickname Node ID shareid1 peer-0 00000000 peer-f ffffffff", s) + self.failUnlessIn("Good Shares (sorted in share order): Share ID Nickname Node ID shareid1 peer-0 00000000 peer-f ffffffff", s) cr = check_results.CheckResults(u, u.get_storage_index(), healthy=False, recoverable=True, summary="ungroovy", **data) - w = web_check_results.CheckResultsRenderer(c, cr) - html = self.render2(w) + w = web_check_results.CheckResultsRendererElement(c, cr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Healthy! : ungroovy", s) @@ -138,22 +247,23 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): healthy=False, recoverable=False, summary="rather dead", **data) - w = web_check_results.CheckResultsRenderer(c, cr) - html = self.render2(w) + w = web_check_results.CheckResultsRendererElement(c, cr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Recoverable! : rather dead", s) - self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s) + self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s) - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Recoverable! : rather dead", s) - html = self.render2(w, args={"return_to": ["FOOURL"]}) + html = self.render_element(w, args={"return_to": ["FOOURL"]}) self.failUnlessIn('Return to file/directory.', html) + w = web_check_results.CheckResultsRenderer(c, cr) d = self.render_json(w) def _check_json(jdata): j = json.loads(jdata) @@ -178,15 +288,15 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): 'recoverable': False, } self.failUnlessEqual(j["results"], expected) - d.addCallback(_check_json) - d.addCallback(lambda ignored: self.render1(w)) + _check_json(d) + + w = web_check_results.CheckResultsRendererElement(c, cr) + d = self.render_element(w) def _check(html): s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) self.failUnlessIn("Not Recoverable! : rather dead", s) - d.addCallback(_check) - return d - + _check(html) def test_check_and_repair(self): c = self.create_fake_client() @@ -244,8 +354,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): crr.post_repair_results = post_cr crr.repair_attempted = False - w = web_check_results.CheckAndRepairResultsRenderer(c, crr) - html = self.render2(w) + w = web_check_results.CheckAndRepairResultsRendererElement(c, crr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -256,7 +366,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): crr.repair_attempted = True crr.repair_successful = True - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -271,7 +381,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): summary="better", **data) crr.post_repair_results = post_cr - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -286,7 +396,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): summary="worse", **data) crr.post_repair_results = post_cr - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -294,24 +404,218 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): self.failUnlessIn("Repair unsuccessful", s) self.failUnlessIn("Post-Repair Checker Results:", s) - d = self.render_json(w) - def _got_json(data): - j = json.loads(data) - self.failUnlessEqual(j["repair-attempted"], True) - self.failUnlessEqual(j["storage-index"], - "2k6avpjga3dho3zsjo6nnkt7n4") - self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing") - self.failUnlessEqual(j["post-repair-results"]["summary"], "worse") - d.addCallback(_got_json) + w = web_check_results.CheckAndRepairResultsRenderer(c, crr) + j = json.loads(self.render_json(w)) + self.failUnlessEqual(j["repair-attempted"], True) + self.failUnlessEqual(j["storage-index"], + "2k6avpjga3dho3zsjo6nnkt7n4") + self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing") + self.failUnlessEqual(j["post-repair-results"]["summary"], "worse") + + w = web_check_results.CheckAndRepairResultsRenderer(c, None) + j = json.loads(self.render_json(w)) + self.failUnlessEqual(j["repair-attempted"], False) + self.failUnlessEqual(j["storage-index"], "") + + + def test_deep_check_renderer(self): + status = check_results.DeepCheckResults("fake-root-si") + status.add_check( + FakeCheckResults("", False, False), + (u"fake", u"unhealthy", u"unrecoverable") + ) + status.add_check( + FakeCheckResults("", True, True), + (u"fake", u"healthy", u"recoverable") + ) + status.add_check( + FakeCheckResults("", True, False), + (u"fake", u"healthy", u"unrecoverable") + ) + status.add_check( + FakeCheckResults("", False, True), + (u"fake", u"unhealthy", u"recoverable") + ) + + monitor = Monitor() + monitor.set_status(status) + + elem = web_check_results.DeepCheckResultsRendererElement(monitor) + doc = self.render_element(elem) + soup = BeautifulSoup(doc, 'html5lib') + + assert_soup_has_favicon(self, soup) + + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Deep Check Results" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h1", + "Deep-Check Results for root SI=" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Checked: 4" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Healthy: 2" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unhealthy: 2" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unrecoverable: 2" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Corrupt Shares: 4" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Files/Directories That Had Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"fake/unhealthy/recoverable: fake summary" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"fake/unhealthy/unrecoverable: fake summary" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Servers on which corrupt shares were found" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Corrupt Shares" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"All Results" + ) + + def test_deep_check_and_repair_renderer(self): + status = check_results.DeepCheckAndRepairResults("") + + status.add_check_and_repair( + FakeCheckAndRepairResults("attempted/success", True, True), + (u"attempted", u"success") + ) + status.add_check_and_repair( + FakeCheckAndRepairResults("attempted/failure", True, False), + (u"attempted", u"failure") + ) + status.add_check_and_repair( + FakeCheckAndRepairResults("unattempted/failure", False, False), + (u"unattempted", u"failure") + ) + + monitor = Monitor() + monitor.set_status(status) + + elem = web_check_results.DeepCheckAndRepairResultsRendererElement(monitor) + doc = self.render_element(elem) + soup = BeautifulSoup(doc, 'html5lib') + + assert_soup_has_favicon(self, soup) + + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Deep Check Results" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h1", + u"Deep-Check-And-Repair Results for root SI=" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Checked: 3" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Healthy (before repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unhealthy (before repair): 3" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Corrupt Shares (before repair): 3" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Repairs Attempted: 2" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Repairs Successful: 1" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + "Repairs Unsuccessful: 1" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Healthy (after repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unhealthy (after repair): 3" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Corrupt Shares (after repair): 3" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Files/Directories That Had Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Files/Directories That Still Have Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Servers on which corrupt shares were found" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Remaining Corrupt Shares" + ) - w2 = web_check_results.CheckAndRepairResultsRenderer(c, None) - d.addCallback(lambda ignored: self.render_json(w2)) - def _got_lit_results(data): - j = json.loads(data) - self.failUnlessEqual(j["repair-attempted"], False) - self.failUnlessEqual(j["storage-index"], "") - d.addCallback(_got_lit_results) - return d class BalancingAct(GridTestMixin, unittest.TestCase): # test for #1115 regarding the 'count-good-share-hosts' metric diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 871cdeb26..1f568ad8d 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -5,8 +5,6 @@ unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8') unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8') unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8') -FAVICON_MARKUP = '' - def assert_soup_has_favicon(testcase, soup): """ diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 2b953b82c..73c354567 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -21,7 +21,12 @@ from allmydata.mutable import publish from .. import common_util as testutil from ..common import WebErrorMixin, ShouldFailMixin from ..no_network import GridTestMixin -from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP +from .common import ( + assert_soup_has_favicon, + unknown_immcap, + unknown_rocap, + unknown_rwcap, +) DIR_HTML_TAG = '' @@ -92,7 +97,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _got_html_good(res): self.failUnlessIn("Healthy", res) self.failIfIn("Not Healthy", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) + d.addCallback(_got_html_good) d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere") def _got_html_good_return_to(res): @@ -235,7 +242,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("Healthy", res) self.failIfIn("Not Healthy", res) self.failUnlessIn("No repair necessary", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) + d.addCallback(_got_html_good) d.addCallback(self.CHECK, "sick", "t=check&repair=true") diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index b6f3ba3c4..b4d604ed4 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -54,6 +54,9 @@ from .common import ( assert_soup_has_tag_with_attributes, assert_soup_has_tag_with_content, assert_soup_has_tag_with_attributes_and_content, + unknown_rwcap, + unknown_rocap, + unknown_immcap, ) from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION @@ -65,7 +68,6 @@ from ..common_web import ( Error, ) from allmydata.client import _Client, SecretHolder -from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP # create a fake uploader/downloader, and a couple of fake dirnodes, then # create a webserver that works against them @@ -3262,13 +3264,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi res = yield self.get_operation_results(None, "123", "html") self.failUnlessIn("Objects Checked: 11", res) self.failUnlessIn("Objects Healthy: 11", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) res = yield self.GET("/operations/123/") # should be the same as without the slash self.failUnlessIn("Objects Checked: 11", res) self.failUnlessIn("Objects Healthy: 11", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) yield self.shouldFail2(error.Error, "one", "404 Not Found", "No detailed results for SI bogus", @@ -3318,7 +3322,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessIn("Objects Unhealthy (after repair): 0", res) self.failUnlessIn("Corrupt Shares (after repair): 0", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) d.addCallback(_check_html) return d diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 3799cdeb7..f85d7d28e 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,6 +1,15 @@ """ Track the port to Python 3. +The two easiest ways to run the part of the test suite which is expected to +pass on Python 3 are:: + + $ tox -e py36 + +and:: + + $ trial allmydata.test.python3_tests + This module has been ported to Python 3. """ @@ -73,7 +82,3 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.test_version", ] - -if __name__ == '__main__': - from subprocess import check_call - check_call(["trial"] + PORTED_TEST_MODULES) diff --git a/src/allmydata/web/check-and-repair-results.xhtml b/src/allmydata/web/check-and-repair-results.xhtml index 34eb31787..d2c9c3a47 100644 --- a/src/allmydata/web/check-and-repair-results.xhtml +++ b/src/allmydata/web/check-and-repair-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Check Results @@ -7,17 +7,17 @@ -

File Check-And-Repair Results for SI=

+

File Check-And-Repair Results for SI=

-
+
-
+
-
+
-
+
-
+
diff --git a/src/allmydata/web/check-results.xhtml b/src/allmydata/web/check-results.xhtml index 5367552fc..2c9f5c283 100644 --- a/src/allmydata/web/check-results.xhtml +++ b/src/allmydata/web/check-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Check Results @@ -7,17 +7,17 @@ -

File Check Results for SI=

+

File Check Results for SI=

- +
-
+
-
+
-
+
diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 7a9badad4..500ac15a7 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -1,12 +1,35 @@ import time import json -from nevow import rend, inevow, tags as T -from twisted.web import http, html -from allmydata.web.common import getxmlfile, get_arg, get_root, WebError + +from twisted.web import ( + http, + html, +) +from twisted.python.filepath import FilePath +from twisted.web.template import ( + Element, + XMLFile, + renderer, + renderElement, + tags, +) +from allmydata.web.common import ( + get_arg, + get_root, + WebError, + MultiFormatResource, + SlotsSequenceElement, +) from allmydata.web.operations import ReloadMixin -from allmydata.interfaces import ICheckAndRepairResults, ICheckResults -from allmydata.util import base32, dictutil +from allmydata.interfaces import ( + ICheckAndRepairResults, + ICheckResults, +) +from allmydata.util import ( + base32, + dictutil, +) def json_check_counts(r): @@ -64,53 +87,64 @@ def json_check_and_repair_results(r): return data class ResultsBase(object): - # self.client must point to the Client, so we can get nicknames and + # self._client must point to the Client, so we can get nicknames and # determine the permuted peer order def _join_pathstring(self, path): + """ + :param tuple path: a path represented by a tuple, such as + ``(u'some', u'dir', u'file')``. + + :return: a string joined by path separaters, such as + ``u'some/dir/file'``. + """ if path: pathstring = "/".join(self._html(path)) else: pathstring = "" return pathstring - def _render_results(self, ctx, cr): + def _render_results(self, req, cr): assert ICheckResults(cr) - c = self.client + c = self._client sb = c.get_storage_broker() r = [] def add(name, value): - r.append(T.li[name + ": ", value]) + r.append(tags.li(name + ": ", value)) + + add("Report", tags.pre("\n".join(self._html(cr.get_report())))) - add("Report", T.pre["\n".join(self._html(cr.get_report()))]) add("Share Counts", "need %d-of-%d, have %d" % (cr.get_encoding_needed(), cr.get_encoding_expected(), cr.get_share_counter_good())) - add("Happiness Level", cr.get_happiness()) - add("Hosts with good shares", cr.get_host_counter_good_shares()) + add("Happiness Level", str(cr.get_happiness())) + add("Hosts with good shares", str(cr.get_host_counter_good_shares())) if cr.get_corrupt_shares(): badsharemap = [] for (s, si, shnum) in cr.get_corrupt_shares(): - d = T.tr[T.td["sh#%d" % shnum], - T.td[T.div(class_="nickname")[s.get_nickname()], - T.div(class_="nodeid")[T.tt[s.get_name()]]], - ] + d = tags.tr(tags.td("sh#%d" % shnum), + tags.td(tags.div(s.get_nickname(), class_="nickname"), + tags.div(tags.tt(s.get_name()), class_="nodeid")),) badsharemap.append(d) - add("Corrupt shares", T.table()[ - T.tr[T.th["Share ID"], - T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], - badsharemap]) + add("Corrupt shares", + tags.table( + tags.tr(tags.th("Share ID"), + tags.th((tags.div("Nickname"), tags.div("Node ID", class_="nodeid")), class_="nickname-and-peerid")), + badsharemap)) else: add("Corrupt shares", "none") - add("Wrong Shares", cr.get_share_counter_wrong()) + add("Wrong Shares", str(cr.get_share_counter_wrong())) sharemap_data = [] shares_on_server = dictutil.DictOfSets() - # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined. + # FIXME: The two tables below contain nickname-and-nodeid + # table column markup which is duplicated with each other, + # introducer.xhtml, and deep-check-results.xhtml. All of these + # (and any other presentations of nickname-and-nodeid) should be combined. for shareid in sorted(cr.get_sharemap().keys()): servers = sorted(cr.get_sharemap()[shareid], @@ -119,19 +153,20 @@ class ResultsBase(object): shares_on_server.add(s, shareid) shareid_s = "" if i == 0: - shareid_s = shareid - d = T.tr[T.td[shareid_s], - T.td[T.div(class_="nickname")[s.get_nickname()], - T.div(class_="nodeid")[T.tt[s.get_name()]]] - ] + shareid_s = str(shareid) + d = tags.tr(tags.td(shareid_s), + tags.td(tags.div(s.get_nickname(), class_="nickname"), + tags.div(tags.tt(s.get_name()), class_="nodeid"))) sharemap_data.append(d) + add("Good Shares (sorted in share order)", - T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], - sharemap_data]) + tags.table(tags.tr(tags.th("Share ID"), + tags.th(tags.div("Nickname"), + tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid")), + sharemap_data)) - - add("Recoverable Versions", cr.get_version_counter_recoverable()) - add("Unrecoverable Versions", cr.get_version_counter_unrecoverable()) + add("Recoverable Versions", str(cr.get_version_counter_recoverable())) + add("Unrecoverable Versions", str(cr.get_version_counter_unrecoverable())) # this table is sorted by permuted order permuted_servers = [s @@ -144,20 +179,23 @@ class ResultsBase(object): for s in permuted_servers: shareids = list(shares_on_server.get(s, [])) shareids.reverse() - shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ] - d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()], - T.div(class_="nodeid")[T.tt[s.get_name()]]], - T.td[shareids_s], - ] + shareids_s = [tags.tt(str(shareid), " ") for shareid in sorted(shareids)] + + d = tags.tr(tags.td(tags.div(s.get_nickname(), class_="nickname"), + tags.div(tags.tt(s.get_name()), class_="nodeid")), + tags.td(shareids_s), ) servermap.append(d) num_shares_left -= len(shareids) if not num_shares_left: break - add("Share Balancing (servers in permuted order)", - T.table()[T.tr[T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]], T.th["Share IDs"]], - servermap]) - return T.ul[r] + add("Share Balancing (servers in permuted order)", + tags.table(tags.tr(tags.th(tags.div("Nickname"), + tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid"), + tags.th("Share IDs")), + servermap)) + + return tags.ul(r) def _html(self, s): if isinstance(s, (str, unicode)): @@ -165,91 +203,114 @@ class ResultsBase(object): assert isinstance(s, (list, tuple)) return [html.escape(w) for w in s] - def want_json(self, ctx): - output = get_arg(inevow.IRequest(ctx), "output", "").lower() - if output.lower() == "json": - return True - return False - - def _render_si_link(self, ctx, storage_index): + def _render_si_link(self, req, storage_index): si_s = base32.b2a(storage_index) - req = inevow.IRequest(ctx) ophandle = req.prepath[-1] - target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s) - output = get_arg(ctx, "output") + target = "%s/operations/%s/%s" % (get_root(req), ophandle, si_s) + output = get_arg(req, "output") if output: target = target + "?output=%s" % output - return T.a(href=target)[si_s] + return tags.a(si_s, href=target) -class LiteralCheckResultsRenderer(rend.Page, ResultsBase): - docFactory = getxmlfile("literal-check-results.xhtml") + +class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): + + formatArgument = "output" def __init__(self, client): - self.client = client - rend.Page.__init__(self, client) + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + """ + super(LiteralCheckResultsRenderer, self).__init__() + self._client = client - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + def render_HTML(self, req): + return renderElement(req, LiteralCheckResultsRendererElement()) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") data = json_check_results(None) return json.dumps(data, indent=1) + "\n" - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + +class LiteralCheckResultsRendererElement(Element): + + loader = XMLFile(FilePath(__file__).sibling("literal-check-results.xhtml")) + + def __init__(self): + super(LiteralCheckResultsRendererElement, self).__init__() + + @renderer + def return_to(self, req, tag): return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file."]] + return tags.div(tags.a("Return to file.", href=return_to)) return "" + class CheckerBase(object): - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + @renderer + def storage_index(self, req, tag): + return self._results.get_storage_index_string() - def render_storage_index(self, ctx, data): - return self.r.get_storage_index_string() - - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def return_to(self, req, tag): return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file/directory."]] + return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" -class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase): - docFactory = getxmlfile("check-results.xhtml") + +class CheckResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, results): - self.client = client - self.r = ICheckResults(results) - rend.Page.__init__(self, results) + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.interfaces.ICheckResults results: results of check/vefify operation. + """ + super(CheckResultsRenderer, self).__init__() + self._client = client + self._results = ICheckResults(results) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") - data = json_check_results(self.r) + def render_HTML(self, req): + return renderElement(req, CheckResultsRendererElement(self._client, self._results)) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") + data = json_check_results(self._results) return json.dumps(data, indent=1) + "\n" - def render_summary(self, ctx, data): + +class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): + + loader = XMLFile(FilePath(__file__).sibling("check-results.xhtml")) + + def __init__(self, client, results): + super(CheckResultsRendererElement, self).__init__() + self._client = client + self._results = results + + @renderer + def summary(self, req, tag): results = [] - if data.is_healthy(): + if self._results.is_healthy(): results.append("Healthy") - elif data.is_recoverable(): + elif self._results.is_recoverable(): results.append("Not Healthy!") else: results.append("Not Recoverable!") results.append(" : ") - results.append(self._html(data.get_summary())) - return ctx.tag[results] + results.append(self._html(self._results.get_summary())) + return tag(results) - def render_repair(self, ctx, data): - if data.is_healthy(): + @renderer + def repair(self, req, tag): + if self._results.is_healthy(): return "" + #repair = T.form(action=".", method="post", # enctype="multipart/form-data")[ # T.fieldset[ @@ -258,30 +319,52 @@ class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase): # T.input(type="submit", value="Repair"), # ]] #return ctx.tag[repair] + return "" # repair button disabled until we make it work correctly, # see #622 for details - def render_results(self, ctx, data): - cr = self._render_results(ctx, data) - return ctx.tag[cr] + @renderer + def results(self, req, tag): + cr = self._render_results(req, self._results) + return tag(cr) -class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase): - docFactory = getxmlfile("check-and-repair-results.xhtml") +class CheckAndRepairResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, results): - self.client = client - self.r = None + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.interfaces.ICheckResults results: check/verify results. + """ + super(CheckAndRepairResultsRenderer, self).__init__() + self._client = client + self._results = None if results: - self.r = ICheckAndRepairResults(results) - rend.Page.__init__(self, results) + self._results = ICheckAndRepairResults(results) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") - data = json_check_and_repair_results(self.r) + def render_HTML(self, req): + elem = CheckAndRepairResultsRendererElement(self._client, self._results) + return renderElement(req, elem) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") + data = json_check_and_repair_results(self._results) return json.dumps(data, indent=1) + "\n" - def render_summary(self, ctx, data): - cr = data.get_post_repair_results() + +class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): + + loader = XMLFile(FilePath(__file__).sibling("check-and-repair-results.xhtml")) + + def __init__(self, client, results): + super(CheckAndRepairResultsRendererElement, self).__init__() + self._client = client + self._results = results + + @renderer + def summary(self, req, tag): + cr = self._results.get_post_repair_results() results = [] if cr.is_healthy(): results.append("Healthy") @@ -291,35 +374,44 @@ class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase): results.append("Not Recoverable!") results.append(" : ") results.append(self._html(cr.get_summary())) - return ctx.tag[results] + return tag(results) - def render_repair_results(self, ctx, data): - if data.get_repair_attempted(): - if data.get_repair_successful(): - return ctx.tag["Repair successful"] + @renderer + def repair_results(self, req, tag): + if self._results.get_repair_attempted(): + if self._results.get_repair_successful(): + return tag("Repair successful") else: - return ctx.tag["Repair unsuccessful"] - return ctx.tag["No repair necessary"] + return tag("Repair unsuccessful") + return tag("No repair necessary") - def render_post_repair_results(self, ctx, data): - cr = self._render_results(ctx, data.get_post_repair_results()) - return ctx.tag[T.div["Post-Repair Checker Results:"], cr] + @renderer + def post_repair_results(self, req, tag): + cr = self._render_results(req, self._results.get_post_repair_results()) + return tag(tags.div("Post-Repair Checker Results:"), cr) - def render_maybe_pre_repair_results(self, ctx, data): - if data.get_repair_attempted(): - cr = self._render_results(ctx, data.get_pre_repair_results()) - return ctx.tag[T.div["Pre-Repair Checker Results:"], cr] + @renderer + def maybe_pre_repair_results(self, req, tag): + if self._results.get_repair_attempted(): + cr = self._render_results(req, self._results.get_pre_repair_results()) + return tag(tags.div("Pre-Repair Checker Results:"), cr) return "" -class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): - docFactory = getxmlfile("deep-check-results.xhtml") +class DeepCheckResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, monitor): - self.client = client + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. + """ + super(DeepCheckResultsRenderer, self).__init__() + self._client = client self.monitor = monitor - def childFactory(self, ctx, name): + def getChild(self, name, req): if not name: return self # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information @@ -327,19 +419,18 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): si = base32.a2b(name) r = self.monitor.get_status() try: - return CheckResultsRenderer(self.client, + return CheckResultsRenderer(self._client, r.get_results_for_storage_index(si)) except KeyError: raise WebError("No detailed results for SI %s" % html.escape(name), http.NOT_FOUND) - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + def render_HTML(self, req): + elem = DeepCheckResultsRendererElement(self.monitor) + return renderElement(req, elem) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") data = {} data["finished"] = self.monitor.is_finished() res = self.monitor.get_status() @@ -361,116 +452,170 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): data["stats"] = res.get_stats() return json.dumps(data, indent=1) + "\n" - def render_root_storage_index(self, ctx, data): + +class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): + + loader = XMLFile(FilePath(__file__).sibling("deep-check-results.xhtml")) + + def __init__(self, monitor): + super(DeepCheckResultsRendererElement, self).__init__() + self.monitor = monitor + + @renderer + def root_storage_index(self, req, tag): + if not self.monitor.get_status(): + return "" return self.monitor.get_status().get_root_storage_index_string() - def data_objects_checked(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-checked"] - def data_objects_healthy(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-healthy"] - def data_objects_unhealthy(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-unhealthy"] - def data_objects_unrecoverable(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-unrecoverable"] + def _get_monitor_counter(self, name): + if not self.monitor.get_status(): + return "" + return str(self.monitor.get_status().get_counters().get(name)) - def data_count_corrupt_shares(self, ctx, data): - return self.monitor.get_status().get_counters()["count-corrupt-shares"] + @renderer + def objects_checked(self, req, tag): + return self._get_monitor_counter("count-objects-checked") - def render_problems_p(self, ctx, data): - c = self.monitor.get_status().get_counters() - if c["count-objects-unhealthy"]: - return ctx.tag + @renderer + def objects_healthy(self, req, tag): + return self._get_monitor_counter("count-objects-healthy") + + @renderer + def objects_unhealthy(self, req, tag): + return self._get_monitor_counter("count-objects-unhealthy") + + @renderer + def objects_unrecoverable(self, req, tag): + return self._get_monitor_counter("count-objects-unrecoverable") + + @renderer + def count_corrupt_shares(self, req, tag): + return self._get_monitor_counter("count-corrupt-shares") + + @renderer + def problems_p(self, req, tag): + if self._get_monitor_counter("count-objects-unhealthy"): + return tag return "" - def data_problems(self, ctx, data): + @renderer + def problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() + problems = [] + for path in sorted(all_objects.keys()): cr = all_objects[path] assert ICheckResults.providedBy(cr) if not cr.is_healthy(): - yield path, cr + summary_text = "" + summary = cr.get_summary() + if summary: + summary_text = ": " + summary + summary_text += " [SI: %s]" % cr.get_storage_index_string() + problems.append({ + # Not sure self._join_pathstring(path) is the + # right thing to use here. + "problem": self._join_pathstring(path) + self._html(summary_text), + }) - def render_problem(self, ctx, data): - path, cr = data - summary_text = "" - summary = cr.get_summary() - if summary: - summary_text = ": " + summary - summary_text += " [SI: %s]" % cr.get_storage_index_string() - return ctx.tag[self._join_pathstring(path), self._html(summary_text)] + return SlotsSequenceElement(tag, problems) - - def render_servers_with_corrupt_shares_p(self, ctx, data): - if self.monitor.get_status().get_counters()["count-corrupt-shares"]: - return ctx.tag + @renderer + def servers_with_corrupt_shares_p(self, req, tag): + if self._get_monitor_counter("count-corrupt-shares"): + return tag return "" - def data_servers_with_corrupt_shares(self, ctx, data): + @renderer + def servers_with_corrupt_shares(self, req, tag): servers = [s for (s, storage_index, sharenum) in self.monitor.get_status().get_corrupt_shares()] servers.sort(key=lambda s: s.get_longname()) - return servers - def render_server_problem(self, ctx, server): - data = [server.get_name()] - nickname = server.get_nickname() - if nickname: - data.append(" (%s)" % self._html(nickname)) - return ctx.tag[data] + problems = [] + for server in servers: + name = [server.get_name()] + nickname = server.get_nickname() + if nickname: + name.append(" (%s)" % self._html(nickname)) + problems.append({"problem": name}) - def render_corrupt_shares_p(self, ctx, data): - if self.monitor.get_status().get_counters()["count-corrupt-shares"]: - return ctx.tag + return SlotsSequenceElement(tag, problems) + + @renderer + def corrupt_shares_p(self, req, tag): + if self._get_monitor_counter("count-corrupt-shares"): + return tag return "" - def data_corrupt_shares(self, ctx, data): - return self.monitor.get_status().get_corrupt_shares() - def render_share_problem(self, ctx, data): - server, storage_index, sharenum = data - nickname = server.get_nickname() - ctx.fillSlots("serverid", server.get_name()) - if nickname: - ctx.fillSlots("nickname", self._html(nickname)) - ctx.fillSlots("si", self._render_si_link(ctx, storage_index)) - ctx.fillSlots("shnum", str(sharenum)) - return ctx.tag - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def corrupt_shares(self, req, tag): + shares = self.monitor.get_status().get_corrupt_shares() + problems = [] + + for share in shares: + server, storage_index, sharenum = share + nickname = server.get_nickname() + problem = { + "serverid": server.get_name(), + "nickname": self._html(nickname), + "si": self._render_si_link(req, storage_index), + "shnum": str(sharenum), + } + problems.append(problem) + + return SlotsSequenceElement(tag, problems) + + @renderer + def return_to(self, req, tag): return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file/directory."]] + return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" - def data_all_objects(self, ctx, data): - r = self.monitor.get_status().get_all_results() - for path in sorted(r.keys()): - yield (path, r[path]) + @renderer + def all_objects(self, req, tag): + results = self.monitor.get_status().get_all_results() + objects = [] - def render_object(self, ctx, data): - path, r = data - ctx.fillSlots("path", self._join_pathstring(path)) - ctx.fillSlots("healthy", str(r.is_healthy())) - ctx.fillSlots("recoverable", str(r.is_recoverable())) - storage_index = r.get_storage_index() - ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index)) - ctx.fillSlots("summary", self._html(r.get_summary())) - return ctx.tag + for path in sorted(results.keys()): + result = results.get(path) + storage_index = result.get_storage_index() + object = { + "path": self._join_pathstring(path), + "healthy": str(result.is_healthy()), + "recoverable": str(result.is_recoverable()), + "storage_index": self._render_si_link(req, storage_index), + "summary": self._html(result.get_summary()), + } + objects.append(object) - def render_runtime(self, ctx, data): - req = inevow.IRequest(ctx) - runtime = time.time() - req.processing_started_timestamp - return ctx.tag["runtime: %s seconds" % runtime] + return SlotsSequenceElement(tag, objects) -class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): - docFactory = getxmlfile("deep-check-and-repair-results.xhtml") + @renderer + def runtime(self, req, tag): + runtime = 'unknown' + if hasattr(req, 'processing_started_timestamp'): + runtime = time.time() - req.processing_started_timestamp + return tag("runtime: %s seconds" % runtime) + + +class DeepCheckAndRepairResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, monitor): - self.client = client + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. + """ + super(DeepCheckAndRepairResultsRenderer, self).__init__() + self._client = client self.monitor = monitor - def childFactory(self, ctx, name): + def getChild(self, name, req): if not name: return self # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information @@ -479,18 +624,17 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): s = self.monitor.get_status() try: results = s.get_results_for_storage_index(si) - return CheckAndRepairResultsRenderer(self.client, results) + return CheckAndRepairResultsRenderer(self._client, results) except KeyError: raise WebError("No detailed results for SI %s" % html.escape(name), http.NOT_FOUND) - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + def render_HTML(self, req): + elem = DeepCheckAndRepairResultsRendererElement(self.monitor) + return renderElement(req, elem) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") res = self.monitor.get_status() data = {} data["finished"] = self.monitor.is_finished() @@ -531,119 +675,132 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): data["stats"] = res.get_stats() return json.dumps(data, indent=1) + "\n" - def render_root_storage_index(self, ctx, data): - return self.monitor.get_status().get_root_storage_index_string() - def data_objects_checked(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-checked"] +class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): + """ + The page generated here has several elements common to "deep check + results" page; hence the code reuse. + """ - def data_objects_healthy(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"] - def data_objects_unhealthy(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"] - def data_corrupt_shares(self, ctx, data): - return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"] + loader = XMLFile(FilePath(__file__).sibling("deep-check-and-repair-results.xhtml")) - def data_repairs_attempted(self, ctx, data): - return self.monitor.get_status().get_counters()["count-repairs-attempted"] - def data_repairs_successful(self, ctx, data): - return self.monitor.get_status().get_counters()["count-repairs-successful"] - def data_repairs_unsuccessful(self, ctx, data): - return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"] + def __init__(self, monitor): + super(DeepCheckAndRepairResultsRendererElement, self).__init__(monitor) + self.monitor = monitor - def data_objects_healthy_post(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"] - def data_objects_unhealthy_post(self, ctx, data): - return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"] - def data_corrupt_shares_post(self, ctx, data): - return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"] + @renderer + def objects_healthy(self, req, tag): + return self._get_monitor_counter("count-objects-healthy-pre-repair") - def render_pre_repair_problems_p(self, ctx, data): - c = self.monitor.get_status().get_counters() - if c["count-objects-unhealthy-pre-repair"]: - return ctx.tag + @renderer + def objects_unhealthy(self, req, tag): + return self._get_monitor_counter("count-objects-unhealthy-pre-repair") + + @renderer + def corrupt_shares(self, req, tag): + return self._get_monitor_counter("count-corrupt-shares-pre-repair") + + @renderer + def repairs_attempted(self, req, tag): + return self._get_monitor_counter("count-repairs-attempted") + + @renderer + def repairs_successful(self, req, tag): + return self._get_monitor_counter("count-repairs-successful") + + @renderer + def repairs_unsuccessful(self, req, tag): + return self._get_monitor_counter("count-repairs-unsuccessful") + + @renderer + def objects_healthy_post(self, req, tag): + return self._get_monitor_counter("count-objects-healthy-post-repair") + + @renderer + def objects_unhealthy_post(self, req, tag): + return self._get_monitor_counter("count-objects-unhealthy-post-repair") + + @renderer + def corrupt_shares_post(self, req, tag): + return self._get_monitor_counter("count-corrupt-shares-post-repair") + + @renderer + def pre_repair_problems_p(self, req, tag): + if self._get_monitor_counter("count-objects-unhealthy-pre-repair"): + return tag return "" - def data_pre_repair_problems(self, ctx, data): + @renderer + def pre_repair_problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() + problems = [] + for path in sorted(all_objects.keys()): r = all_objects[path] assert ICheckAndRepairResults.providedBy(r) cr = r.get_pre_repair_results() if not cr.is_healthy(): - yield path, cr + problem = self._join_pathstring(path), ": ", self._html(cr.get_summary()) + problems.append({"problem": problem}) - def render_problem(self, ctx, data): - path, cr = data - return ctx.tag[self._join_pathstring(path), ": ", - self._html(cr.get_summary())] + return SlotsSequenceElement(tag, problems) - def render_post_repair_problems_p(self, ctx, data): - c = self.monitor.get_status().get_counters() - if (c["count-objects-unhealthy-post-repair"] - or c["count-corrupt-shares-post-repair"]): - return ctx.tag + @renderer + def post_repair_problems_p(self, req, tag): + if (self._get_monitor_counter("count-objects-unhealthy-post-repair") + or self._get_monitor_counter("count-corrupt-shares-post-repair")): + return tag return "" - def data_post_repair_problems(self, ctx, data): + @renderer + def post_repair_problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() + problems = [] + for path in sorted(all_objects.keys()): r = all_objects[path] assert ICheckAndRepairResults.providedBy(r) cr = r.get_post_repair_results() if not cr.is_healthy(): - yield path, cr + problem = self._join_pathstring(path), ": ", self._html(cr.get_summary()) + problems.append({"problem": problem}) - def render_servers_with_corrupt_shares_p(self, ctx, data): - if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]: - return ctx.tag - return "" - def data_servers_with_corrupt_shares(self, ctx, data): - return [] # TODO - def render_server_problem(self, ctx, data): - pass + return SlotsSequenceElement(tag, problems) - - def render_remaining_corrupt_shares_p(self, ctx, data): - if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]: - return ctx.tag - return "" - def data_post_repair_corrupt_shares(self, ctx, data): - return [] # TODO - - def render_share_problem(self, ctx, data): - pass - - - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) - return_to = get_arg(req, "return_to", None) - if return_to: - return T.div[T.a(href=return_to)["Return to file/directory."]] + @renderer + def remaining_corrupt_shares_p(self, req, tag): + if self._get_monitor_counter("count-corrupt-shares-post-repair"): + return tag return "" - def data_all_objects(self, ctx, data): - r = self.monitor.get_status().get_all_results() - for path in sorted(r.keys()): - yield (path, r[path]) + @renderer + def post_repair_corrupt_shares(self, req, tag): + # TODO: this was not implemented before porting to + # twisted.web.template; leaving it as such. + # + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3371 + corrupt = [{"share":"unimplemented"}] + return SlotsSequenceElement(tag, corrupt) - def render_object(self, ctx, data): - path, r = data - ctx.fillSlots("path", self._join_pathstring(path)) - ctx.fillSlots("healthy_pre_repair", - str(r.get_pre_repair_results().is_healthy())) - ctx.fillSlots("recoverable_pre_repair", - str(r.get_pre_repair_results().is_recoverable())) - ctx.fillSlots("healthy_post_repair", - str(r.get_post_repair_results().is_healthy())) - storage_index = r.get_storage_index() - ctx.fillSlots("storage_index", - self._render_si_link(ctx, storage_index)) - ctx.fillSlots("summary", - self._html(r.get_pre_repair_results().get_summary())) - return ctx.tag + @renderer + def all_objects(self, req, tag): + results = {} + if self.monitor.get_status(): + results = self.monitor.get_status().get_all_results() + objects = [] + + for path in sorted(results.keys()): + result = results[path] + storage_index = result.get_storage_index() + obj = { + "path": self._join_pathstring(path), + "healthy_pre_repair": str(result.get_pre_repair_results().is_healthy()), + "recoverable_pre_repair": str(result.get_pre_repair_results().is_recoverable()), + "healthy_post_repair": str(result.get_post_repair_results().is_healthy()), + "storage_index": self._render_si_link(req, storage_index), + "summary": self._html(result.get_pre_repair_results().get_summary()), + } + objects.append(obj) + + return SlotsSequenceElement(tag, objects) - def render_runtime(self, ctx, data): - req = inevow.IRequest(ctx) - runtime = time.time() - req.processing_started_timestamp - return ctx.tag["runtime: %s seconds" % runtime] diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index a7db34837..be7f35b1d 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -1,95 +1,106 @@ - + Tahoe-LAFS - Deep Check Results - +

Deep-Check-And-Repair Results for root - SI=

+ SI= -

+

Counters:

    -
  • Objects Checked:
  • +
  • Objects Checked:
  • -
  • Objects Healthy (before repair):
  • -
  • Objects Unhealthy (before repair):
  • -
  • Corrupt Shares (before repair):
  • +
  • Objects Healthy (before repair):
  • +
  • Objects Unhealthy (before repair):
  • +
  • Corrupt Shares (before repair):
  • -
  • Repairs Attempted:
  • -
  • Repairs Successful:
  • -
  • Repairs Unsuccessful:
  • +
  • Repairs Attempted:
  • +
  • Repairs Successful:
  • +
  • Repairs Unsuccessful:
  • -
  • Objects Healthy (after repair):
  • -
  • Objects Unhealthy (after repair):
  • -
  • Corrupt Shares (after repair):
  • +
  • Objects Healthy (after repair):
  • +
  • Objects Unhealthy (after repair):
  • +
  • Corrupt Shares (after repair):
-
+

Files/Directories That Had Problems:

-
    -
  • -
  • None
  • +
      +
    • + +
    • +
    • None
-
+

Files/Directories That Still Have Problems:

-
    -
  • -
  • None
  • +
      +
    • + +
    • +
    • None
-
+

Servers on which corrupt shares were found

-
    -
  • -
  • None
  • +
      +
    • + +
    • +
    • None
-
+

Remaining Corrupt Shares

These shares need to be manually inspected and removed.

-
    -
  • -
  • None
  • +
      +
    • + +
    • +
    • None
-
+
- - - - - - - - +
Relative PathHealthy Pre-RepairRecoverable Pre-RepairHealthy Post-RepairStorage IndexSummary
+ + + + + + + - - - - - - - + + + + + + + + + +
Relative PathHealthy Pre-RepairRecoverable Pre-RepairHealthy Post-RepairStorage IndexSummary
Nothing to report yet.
-
+
diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index cb2330b7d..7fb324050 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -1,87 +1,93 @@ - + Tahoe-LAFS - Deep Check Results - + -

Deep-Check Results for root SI=

+

Deep-Check Results for root SI=

-

+

Counters:

    -
  • Objects Checked:
  • -
  • Objects Healthy:
  • -
  • Objects Unhealthy:
  • -
  • Objects Unrecoverable:
  • -
  • Corrupt Shares:
  • - +
  • Objects Checked:
  • +
  • Objects Healthy:
  • +
  • Objects Unhealthy:
  • +
  • Objects Unrecoverable:
  • +
  • Corrupt Shares:
-
+

Files/Directories That Had Problems:

-
    -
  • -
  • None
  • +
      +
    • + +
    • +
    • None
-
+

Servers on which corrupt shares were found

-
    -
  • -
  • None
  • +
      +
    • + +
    • +
    • None
-
+

Corrupt Shares

If repair fails, these shares need to be manually inspected and removed.

- - - - - - +
ServerServer NicknameStorage IndexShare Number
+ + + + + - - - - - + + + + +
ServerServer NicknameStorage IndexShare Number
-
+

All Results

- - - - - - - +
Relative PathHealthyRecoverableStorage IndexSummary
+ + + + + + - - - - - - + + + + + + + + +
Relative PathHealthyRecoverableStorage IndexSummary
Nothing to report yet.
-
+
diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index fa62ea23d..7eccfea9e 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -261,7 +261,7 @@ class MoreInfoElement(Element): @renderer def deep_check_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - deep_check = T.form(action=".", method="post", + deep_check = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-check"), @@ -287,7 +287,7 @@ class MoreInfoElement(Element): @renderer def deep_size_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - deep_size = T.form(action=".", method="post", + deep_size = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-size"), @@ -300,7 +300,7 @@ class MoreInfoElement(Element): @renderer def deep_stats_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - deep_stats = T.form(action=".", method="post", + deep_stats = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-stats"), @@ -313,7 +313,7 @@ class MoreInfoElement(Element): @renderer def manifest_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - manifest = T.form(action=".", method="post", + manifest = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-manifest"), diff --git a/src/allmydata/web/literal-check-results.xhtml b/src/allmydata/web/literal-check-results.xhtml index 95b4b0cf7..f82ddf8d3 100644 --- a/src/allmydata/web/literal-check-results.xhtml +++ b/src/allmydata/web/literal-check-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Check Results @@ -11,7 +11,7 @@
Literal files are always healthy: their data is contained in the URI
-
+
diff --git a/tox.ini b/tox.ini index 0376ab28e..98ca90c39 100644 --- a/tox.ini +++ b/tox.ini @@ -49,9 +49,8 @@ commands = tahoe --version [testenv:py36] -# On macOS, git inside of ratchet.sh needs $HOME. -passenv = {[testenv]passenv} HOME -commands = {toxinidir}/misc/python3/ratchet.sh +commands = + trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests} [testenv:integration] setenv =