Merge remote-tracking branch 'origin/master' into 2995.remove-unnecessary-key-copying

.appveyor.yml

@@ -48,9 +48,9 @@ test_script:
   # Put your test command here.
   # Note that you must use the environment variable %PYTHON% to refer to
   # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python evrsion you want to use on PATH.
+  # to put the Python version you want to use on PATH.
   - |
-    %PYTHON%\Scripts\tox.exe -e py
+    %PYTHON%\Scripts\tox.exe -e coverage
 
 after_test:
   # This builds the main tahoe wheel, and wheels for all dependencies.
@@ -63,6 +63,10 @@ after_test:
     rd /s /q _trial_temp
     %PYTHON%\python.exe setup.py bdist_wheel
     %PYTHON%\python.exe -m pip wheel -w dist .
+  - |
+    %PYTHON%\python.exe -m pip install codecov coverage
+    %PYTHON%\python.exe -m coverage xml -o coverage.xml -i
+    %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
 
 artifacts:
   # bdist_wheel puts your built wheel in the dist directory

.github/CONTRIBUTING.rst

@@ -0,0 +1,20 @@
+.. -*- coding: utf-8 -*-
+
+.. This document is rendered on the GitHub PR creation page to guide
+   contributors.  It is also rendered into the overall documentation.
+
+Contributing to Tahoe-LAFS
+==========================
+
+As an open source project,
+Tahoe-LAFS welcomes contributions of many forms.
+
+Examples of contributions include:
+
+* `Code patches <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Patches>`_
+* `Documentation improvements <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Doc>`_
+* `Bug reports <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/HowToReportABug>`_
+* `Patch reviews <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/PatchReviewProcess>`_
+
+Before authoring or reviewing a patch,
+please familiarize yourself with the `coding standard <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`_.

docs/index.rst

@@ -24,6 +24,7 @@ Contents:
    frontends/download-status
 
    known_issues
+   ../.github/CONTRIBUTING
 
    servers
    helper
@@ -64,4 +65,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

integration/test_magic_folder.py

@@ -395,3 +395,67 @@ def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_fur
         assert exists(join(magic_folder, "its_a_file"))
         assert not exists(join(magic_folder, "its_a_file.backup"))
         time.sleep(1)
+
+
+@pytest_twisted.inlineCallbacks
+def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder):
+    """
+    Alice can add new files to a magic folder while Bob is offline.  When Bob
+    comes back online his copy is updated to reflect the new files.
+    """
+    alice_magic_dir, bob_magic_dir = magic_folder
+    alice_node_dir = join(temp_dir, "alice")
+    bob_node_dir = join(temp_dir, "bob")
+
+    # Take Bob offline.
+    yield util.cli(reactor, bob_node_dir, "stop")
+
+    # Create a couple files in Alice's local directory.
+    some_files = list(
+        (name * 3) + ".added-while-offline"
+        for name
+        in "xyz"
+    )
+    for name in some_files:
+        with open(join(alice_magic_dir, name), "w") as f:
+            f.write(name + " some content")
+
+    good = False
+    for i in range(15):
+        status = yield util.magic_folder_cli(reactor, alice_node_dir, "status")
+        good = status.count(".added-while-offline (36 B): good, version=0") == len(some_files) * 2
+        if good:
+            # We saw each file as having a local good state and a remote good
+            # state.  That means we're ready to involve Bob.
+            break
+        else:
+            time.sleep(1.0)
+
+    assert good, (
+        "Timed out waiting for good Alice state.  Last status:\n{}".format(status)
+    )
+
+    # Start Bob up again
+    magic_text = 'Completed initial Magic Folder scan successfully'
+    yield util._run_node(reactor, bob_node_dir, request, magic_text)
+
+    yield util.await_files_exist(
+        list(
+            join(bob_magic_dir, name)
+            for name
+            in some_files
+        ),
+        await_all=True,
+    )
+    # Let it settle.  It would be nicer to have a readable status output we
+    # could query.  Parsing the current text format is more than I want to
+    # deal with right now.
+    time.sleep(1.0)
+    conflict_files = list(name + ".conflict" for name in some_files)
+    assert all(
+        list(
+            not exists(join(bob_magic_dir, name))
+            for name
+            in conflict_files
+        ),
+    )

integration/util.py

@@ -303,7 +303,7 @@ def await_files_exist(paths, timeout=15, await_all=False):
     an Exception is raised
     """
     start_time = time.time()
-    while time.time() - start_time < 15.0:
+    while time.time() - start_time < timeout:
         print("  waiting for: {}".format(' '.join(paths)))
         found = [p for p in paths if exists(p)]
         print("found: {}".format(found))
@@ -329,3 +329,19 @@ def await_file_vanishes(path, timeout=10):
             return
         time.sleep(1)
     raise FileShouldVanishException(path, timeout)
+
+
+def cli(reactor, node_dir, *argv):
+    proto = _CollectOutputProtocol()
+    reactor.spawnProcess(
+        proto,
+        sys.executable,
+        [
+            sys.executable, '-m', 'allmydata.scripts.runner',
+            '--node-directory', node_dir,
+        ] + list(argv),
+    )
+    return proto.done
+
+def magic_folder_cli(reactor, node_dir, *argv):
+    return cli(reactor, node_dir, "magic-folder", *argv)

newsfragments/2965.bugfix

@@ -0,0 +1 @@
+Magic-Folders now creates spurious conflict files in fewer cases.  In particular, if files are added to the folder while a client is offline, that client will not create conflict files for all those new files when it starts up.

newsfragments/3003.other

@@ -0,0 +1 @@
+The contributor guidelines are now linked from the GitHub pull request creation page.

src/allmydata/frontends/magic_folder.py

@@ -988,8 +988,17 @@ class QueueMixin(HookMixin):
         with action.context():
             d = DeferredContext(defer.Deferred())
 
-            # adds items to our deque
+            # During startup we scanned the collective for items to download.
-            d.addCallback(lambda ignored: self._perform_scan())
+            # If we found work to do, we do not need to perform another scan
+            # here.  More importantly, the logic for determining which items
+            # to download is *not correct* in the case where two scans are
+            # performed with no intermediate emptying of the work queue.
+            # Therefore, skip the scan any time there is queued work.  The
+            # only time we expect there to be any, though, is on the first
+            # time through this loop.
+            if not self._deque:
+                # adds items to our deque
+                d.addCallback(lambda ignored: self._perform_scan())
 
             # process anything in our queue
             d.addCallback(lambda ignored: self._process_deque())
@@ -1747,7 +1756,7 @@ class Downloader(QueueMixin, WriteFileMixin):
                         "Last tried at %s" % self.nice_current_time(),
                     )
                     write_traceback()
-                    yield task.deferLater(self._clock, self._poll_interval, lambda: None)
+                    yield task.deferLater(self._clock, self._scan_delay(), lambda: None)
 
     def nice_current_time(self):
         return format_time(datetime.fromtimestamp(self._clock.seconds()).timetuple())
@@ -1891,6 +1900,7 @@ class Downloader(QueueMixin, WriteFileMixin):
 
     @eliotutil.log_call_deferred(SCAN_REMOTE_COLLECTIVE.action_type)
     def _scan_remote_collective(self, scan_self=False):
+        precondition(not self._deque, "Items in _deque invalidate should_download logic")
         scan_batch = {}  # path -> [(filenode, metadata)]
         d = DeferredContext(self._collective_dirnode.list())
         def scan_collective(dirmap):

src/allmydata/test/test_magic_folder.py

@@ -1084,50 +1084,54 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
         # now, we ONLY want to do the scan, not a full iteration of
         # the process loop. So we do just the scan part "by hand" in
         # Bob's downloader
-        yield self.bob_magicfolder.downloader._perform_scan()
+        with start_action(action_type=u"test:perform-scan"):
-        # while we're delving into internals, I guess we might as well
+            yield self.bob_magicfolder.downloader._perform_scan()
-        # confirm that we did queue up an item to download
+            # while we're delving into internals, I guess we might as well
-        self.assertEqual(1, len(self.bob_magicfolder.downloader._deque))
+            # confirm that we did queue up an item to download
+            self.assertEqual(1, len(self.bob_magicfolder.downloader._deque))
 
-        # break all the servers so the download fails. the count is 2
+        # break all the servers so the download fails.  count=1 because we
-        # because the "full iteration" will do a scan (downloading the
+        # only want the download attempted by _process_deque to fail.  After
-        # metadata file) and then process the deque (trying to
+        # that, we want it to work again.
-        # download the item we queued up already)
         for server_id in self.g.get_all_serverids():
-            self.g.break_server(server_id, count=2)
+            self.g.break_server(server_id, count=1)
 
-        # now let bob try to do the download
+        # now let bob try to do the download.  Reach in and call
-        yield iterate(self.bob_magicfolder)
+        # _process_deque directly because we are already half-way through a
+        # logical iteration thanks to the _perform_scan call above.
+        with start_action(action_type=u"test:process-deque"):
+            yield self.bob_magicfolder.downloader._process_deque()
 
-        self.eliot_logger.flushTracebacks(UnrecoverableFileError)
+            self.eliot_logger.flushTracebacks(UnrecoverableFileError)
-        logged = self.eliot_logger.flushTracebacks(NoSharesError)
+            logged = self.eliot_logger.flushTracebacks(NoSharesError)
-        self.assertEqual(
+            self.assertEqual(
-            1,
+                1,
-            len(logged),
+                len(logged),
-            "Got other than expected single NoSharesError: {}".format(logged),
+                "Got other than expected single NoSharesError: {}".format(logged),
-        )
+            )
 
-        # ...however Bob shouldn't have downloaded anything
+            # ...however Bob shouldn't have downloaded anything
-        self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
+            self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
-        # bob should *not* have downloaded anything, as we failed all the servers
+            # bob should *not* have downloaded anything, as we failed all the servers
-        self.failUnlessReallyEqual(
+            self.failUnlessReallyEqual(
-            self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
+                self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
-            0
+                0
-        )
+            )
-        self.failUnlessReallyEqual(
+            self.failUnlessReallyEqual(
-            self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
+                self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
-            1
+                1
-        )
+            )
 
-        # now we let Bob try again
+        with start_action(action_type=u"test:iterate"):
-        yield iterate(self.bob_magicfolder)
+            # now we let Bob try again
+            yield iterate(self.bob_magicfolder)
 
-        # ...and he should have succeeded
+            # ...and he should have succeeded
-        self.failUnlessReallyEqual(
+            self.failUnlessReallyEqual(
-            self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
+                self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
-            1
+                1
-        )
+            )
-        yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0)
+            yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 0)
 
     @inline_callbacks
     def test_conflict_local_change_fresh(self):