Merge remote-tracking branch 'origin/master' into 3834.renew_secret_comparison_helper

.readthedocs.yaml

@@ -0,0 +1,5 @@
+version: 2
+
+python:
+   install:
+   - requirements: docs/requirements.txt

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+docutils<0.18  # https://github.com/sphinx-doc/sphinx/issues/9788
+recommonmark
+sphinx_rtd_theme

integration/test_tor.py

@@ -35,6 +35,9 @@ from allmydata.test.common import (
 if sys.platform.startswith('win'):
     pytest.skip('Skipping Tor tests on Windows', allow_module_level=True)
 
+if PY2:
+    pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True)
+
 @pytest_twisted.inlineCallbacks
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)

newsfragments/3837.other

@@ -0,0 +1 @@
+Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies.

src/allmydata/scripts/debug.py

@@ -746,6 +746,13 @@ def _describe_mutable_share(abs_sharefile, f, now, si_s, out):
 
     if share_type == "SDMF":
         f.seek(m.DATA_OFFSET)
+
+        # Read at least the mutable header length, if possible.  If there's
+        # less data than that in the share, don't try to read more (we won't
+        # be able to unpack the header in this case but we surely don't want
+        # to try to unpack bytes *following* the data section as if they were
+        # header data).  Rather than 2000 we could use HEADER_LENGTH from
+        # allmydata/mutable/layout.py, probably.
         data = f.read(min(data_length, 2000))
 
         try:
@@ -810,8 +817,8 @@ def _describe_immutable_share(abs_sharefile, now, si_s, out):
     sf = ShareFile(abs_sharefile)
     bp = ImmediateReadBucketProxy(sf)
 
-    expiration_time = min( [lease.get_expiration_time()
-                            for lease in sf.get_leases()] )
+    expiration_time = min(lease.get_expiration_time()
+                          for lease in sf.get_leases())
     expiration = max(0, expiration_time - now)
 
     UEB_data = call(bp.get_uri_extension)
@@ -934,8 +941,9 @@ def corrupt_share(options):
         if MutableShareFile.is_valid_header(prefix):
             # mutable
             m = MutableShareFile(fn)
-            f = open(fn, "rb")
+            with open(fn, "rb") as f:
                 f.seek(m.DATA_OFFSET)
+                # Read enough data to get a mutable header to unpack.
                 data = f.read(2000)
             # make sure this slot contains an SMDF share
             assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported"

src/allmydata/test/common_system.py

@@ -672,11 +672,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         """
         iv_dir = self.getdir("introducer")
         if not os.path.isdir(iv_dir):
-            _, port_endpoint = self.port_assigner.assign(reactor)
+            _, web_port_endpoint = self.port_assigner.assign(reactor)
+            main_location_hint, main_port_endpoint = self.port_assigner.assign(reactor)
             introducer_config = (
                 u"[node]\n"
                 u"nickname = introducer \N{BLACK SMILING FACE}\n" +
-                u"web.port = {}\n".format(port_endpoint)
+                u"web.port = {}\n".format(web_port_endpoint) +
+                u"tub.port = {}\n".format(main_port_endpoint) +
+                u"tub.location = {}\n".format(main_location_hint)
             ).encode("utf-8")
 
             fileutil.make_dirs(iv_dir)

src/allmydata/test/no_network.py

@@ -25,6 +25,11 @@ if PY2:
 from past.builtins import unicode
 from six import ensure_text
 
+try:
+    from typing import Dict, Callable
+except ImportError:
+    pass
+
 import os
 from base64 import b32encode
 from functools import (
@@ -479,6 +484,18 @@ class GridTestMixin(object):
 
     def set_up_grid(self, num_clients=1, num_servers=10,
                     client_config_hooks={}, oneshare=False):
+        """
+        Create a Tahoe-LAFS storage grid.
+
+        :param num_clients: See ``NoNetworkGrid``
+        :param num_servers: See `NoNetworkGrid``
+        :param client_config_hooks: See ``NoNetworkGrid``
+
+        :param bool oneshare: If ``True`` then the first client node is
+            configured with ``n == k == happy == 1``.
+
+        :return: ``None``
+        """
         # self.basedir must be set
         port_assigner = SameProcessStreamEndpointAssigner()
         port_assigner.setUp()
@@ -557,6 +574,15 @@ class GridTestMixin(object):
         return sorted(shares)
 
     def copy_shares(self, uri):
+        # type: (bytes) -> Dict[bytes, bytes]
+        """
+        Read all of the share files for the given capability from the storage area
+        of the storage servers created by ``set_up_grid``.
+
+        :param bytes uri: A Tahoe-LAFS data capability.
+
+        :return: A ``dict`` mapping share file names to share file contents.
+        """
         shares = {}
         for (shnum, serverid, sharefile) in self.find_uri_shares(uri):
             with open(sharefile, "rb") as f:
@@ -601,10 +627,15 @@ class GridTestMixin(object):
                     f.write(corruptdata)
 
     def corrupt_all_shares(self, uri, corruptor, debug=False):
+        # type: (bytes, Callable[[bytes, bool], bytes], bool) -> None
+        """
+        Apply ``corruptor`` to the contents of all share files associated with a
+        given capability and replace the share file contents with its result.
+        """
         for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri):
             with open(i_sharefile, "rb") as f:
                 sharedata = f.read()
-            corruptdata = corruptor(sharedata, debug=debug)
+            corruptdata = corruptor(sharedata, debug)
             with open(i_sharefile, "wb") as f:
                 f.write(corruptdata)
 

src/allmydata/test/test_download.py

@@ -14,6 +14,11 @@ if PY2:
 # a previous run. This asserts that the current code is capable of decoding
 # shares from a previous version.
 
+try:
+    from typing import Any
+except ImportError:
+    pass
+
 import six
 import os
 from twisted.trial import unittest
@@ -951,12 +956,52 @@ class Corruption(_Base, unittest.TestCase):
         self.corrupt_shares_numbered(imm_uri, [2], _corruptor)
 
     def _corrupt_set(self, ign, imm_uri, which, newvalue):
+        # type: (Any, bytes, int, int) -> None
+        """
+        Replace a single byte share file number 2 for the given capability with a
+        new byte.
+
+        :param imm_uri: Corrupt share number 2 belonging to this capability.
+        :param which: The byte position to replace.
+        :param newvalue: The new byte value to set in the share.
+        """
         log.msg("corrupt %d" % which)
         def _corruptor(s, debug=False):
             return s[:which] + bchr(newvalue) + s[which+1:]
         self.corrupt_shares_numbered(imm_uri, [2], _corruptor)
 
     def test_each_byte(self):
+        """
+        Test share selection behavior of the downloader in the face of certain
+        kinds of data corruption.
+
+        1. upload a small share to the no-network grid
+        2. read all of the resulting share files out of the no-network storage servers
+        3. for each of
+
+           a. each byte of the share file version field
+           b. each byte of the immutable share version field
+           c. each byte of the immutable share data offset field
+           d. the most significant byte of the block_shares offset field
+           e. one of the bytes of one of the merkle trees
+           f. one of the bytes of the share hashes list
+
+           i. flip the least significant bit in all of the the share files
+           ii. perform the download/check/restore process
+
+        4. add 2 ** 24 to the share file version number
+        5. perform the download/check/restore process
+
+        6. add 2 ** 24 to the share version number
+        7. perform the download/check/restore process
+
+        The download/check/restore process is:
+
+        1. attempt to download the data
+        2. assert that the recovered plaintext is correct
+        3. assert that only the "correct" share numbers were used to reconstruct the plaintext
+        4. restore all of the share files to their pristine condition
+        """
         # Setting catalog_detection=True performs an exhaustive test of the
         # Downloader's response to corruption in the lsb of each byte of the
         # 2070-byte share, with two goals: make sure we tolerate all forms of
@@ -1145,8 +1190,18 @@ class Corruption(_Base, unittest.TestCase):
         return d
 
     def _corrupt_flip_all(self, ign, imm_uri, which):
+        # type: (Any, bytes, int) -> None
+        """
+        Flip the least significant bit at a given byte position in all share files
+        for the given capability.
+        """
         def _corruptor(s, debug=False):
-            return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:]
+            # type: (bytes, bool) -> bytes
+            before_corruption = s[:which]
+            after_corruption = s[which+1:]
+            original_byte = s[which:which+1]
+            corrupt_byte = bchr(ord(original_byte) ^ 0x01)
+            return b"".join([before_corruption, corrupt_byte, after_corruption])
         self.corrupt_all_shares(imm_uri, _corruptor)
 
 class DownloadV2(_Base, unittest.TestCase):

tox.ini

@@ -217,13 +217,8 @@ commands =
 # your web browser.
 
 [testenv:docs]
-# we pin docutils because of https://sourceforge.net/p/docutils/bugs/301/
-# which asserts when it reads links to .svg files (e.g. about.rst)
 deps =
-     sphinx
-     docutils==0.12
-     recommonmark
-     sphinx_rtd_theme
+     -r docs/requirements.txt
 # normal install is not needed for docs, and slows things down
 skip_install = True
 commands =