mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-02-20 17:52:50 +00:00
fix 2880
- remember upload URI so we don't re-download - account for empty files in "should_download" - fix 'conflicted' asserts in tests
This commit is contained in:
meejah
parent
5b93e193c3
commit
87ad3cb8e9
@ -114,7 +114,7 @@ def test_bob_creates_alice_deletes_bob_restores(magic_folder):
|
||||
# .. she previously deleted it? does that really make sense)
|
||||
|
||||
util.await_file_contents(
|
||||
join(alice_dir, "boom.conflict"),
|
||||
join(alice_dir, "boom"),
|
||||
"bob wrote this again, because reasons",
|
||||
)
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@ from allmydata.util.time_format import format_time
|
||||
from allmydata.immutable.upload import FileName, Data
|
||||
from allmydata import magicfolderdb, magicpath
|
||||
|
||||
|
||||
IN_EXCL_UNLINK = 0x04000000L
|
||||
|
||||
def get_inotify_module():
|
||||
@ -422,9 +423,9 @@ class QueueMixin(HookMixin):
|
||||
self._log(" done: %r" % proc)
|
||||
except Exception as e:
|
||||
log.err("processing '%r' failed: %s" % (item, e))
|
||||
proc = None # actually in old _lazy_tail way, proc would be Failure
|
||||
# XXX can we just get rid of the hooks now?
|
||||
yield self._call_hook(proc, 'processed')
|
||||
proc = Failure()
|
||||
|
||||
self._call_hook(proc, 'processed')
|
||||
|
||||
def _get_relpath(self, filepath):
|
||||
self._log("_get_relpath(%r)" % (filepath,))
|
||||
@ -720,7 +721,10 @@ class Uploader(QueueMixin):
|
||||
|
||||
def _add_db_entry(filenode):
|
||||
filecap = filenode.get_uri()
|
||||
last_downloaded_uri = metadata.get('last_downloaded_uri', None)
|
||||
# if we're uploading a file, we want to set
|
||||
# last_downloaded_uri to the filecap so that we don't
|
||||
# immediately re-download it when we start up next
|
||||
last_downloaded_uri = metadata.get('last_downloaded_uri', filecap)
|
||||
self._db.did_upload_version(relpath_u, new_version, filecap,
|
||||
last_downloaded_uri, last_downloaded_timestamp,
|
||||
pathinfo)
|
||||
@ -792,7 +796,10 @@ class Uploader(QueueMixin):
|
||||
|
||||
def _add_db_entry(filenode):
|
||||
filecap = filenode.get_uri()
|
||||
last_downloaded_uri = metadata.get('last_downloaded_uri', None)
|
||||
# if we're uploading a file, we want to set
|
||||
# last_downloaded_uri to the filecap so that we don't
|
||||
# immediately re-download it when we start up next
|
||||
last_downloaded_uri = metadata.get('last_downloaded_uri', filecap)
|
||||
self._db.did_upload_version(relpath_u, new_version, filecap,
|
||||
last_downloaded_uri, last_downloaded_timestamp,
|
||||
pathinfo)
|
||||
@ -903,6 +910,21 @@ class WriteFileMixin(object):
|
||||
return abspath_u
|
||||
|
||||
|
||||
def _is_empty_filecap(client, cap):
|
||||
"""
|
||||
Internal helper.
|
||||
|
||||
:param cap: a capability URI
|
||||
|
||||
:returns: True if "cap" represents an empty file
|
||||
"""
|
||||
node = client.create_node_from_uri(
|
||||
None,
|
||||
cap.encode('ascii'),
|
||||
)
|
||||
return (not node.get_size())
|
||||
|
||||
|
||||
class DownloadItem(QueuedItem):
|
||||
"""
|
||||
Represents a single item in the _deque of the Downloader
|
||||
@ -982,7 +1004,9 @@ class Downloader(QueueMixin, WriteFileMixin):
|
||||
self._log("version %r" % (db_entry.version,))
|
||||
if db_entry.version < remote_version:
|
||||
return True
|
||||
if db_entry.last_downloaded_uri != remote_uri:
|
||||
if db_entry.last_downloaded_uri is None and _is_empty_filecap(self._client, remote_uri):
|
||||
pass
|
||||
elif db_entry.last_downloaded_uri != remote_uri:
|
||||
return True
|
||||
return False
|
||||
|
||||
@ -1144,7 +1168,8 @@ class Downloader(QueueMixin, WriteFileMixin):
|
||||
def do_update_db(written_abspath_u):
|
||||
filecap = item.file_node.get_uri()
|
||||
last_uploaded_uri = item.metadata.get('last_uploaded_uri', None)
|
||||
self._log("DOUPDATEDB %r" % written_abspath_u)
|
||||
if not item.file_node.get_size():
|
||||
filecap = None # ^ is an empty file
|
||||
last_downloaded_uri = filecap
|
||||
last_downloaded_timestamp = now
|
||||
written_pathinfo = get_pathinfo(written_abspath_u)
|
||||
@ -1178,8 +1203,9 @@ class Downloader(QueueMixin, WriteFileMixin):
|
||||
if db_entry:
|
||||
if dmd_last_downloaded_uri is not None and db_entry.last_downloaded_uri is not None:
|
||||
if dmd_last_downloaded_uri != db_entry.last_downloaded_uri:
|
||||
is_conflict = True
|
||||
self._count('objects_conflicted')
|
||||
if not _is_empty_filecap(self._client, dmd_last_downloaded_uri):
|
||||
is_conflict = True
|
||||
self._count('objects_conflicted')
|
||||
elif dmd_last_uploaded_uri is not None and dmd_last_uploaded_uri != db_entry.last_uploaded_uri:
|
||||
is_conflict = True
|
||||
self._count('objects_conflicted')
|
||||
|
||||
@ -865,6 +865,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
1
|
||||
)
|
||||
self.failUnless(os.path.exists(bob_fname))
|
||||
self.failUnless(not os.path.exists(bob_fname + '.backup'))
|
||||
self.failUnless(not os.path.exists(bob_fname + '.conflict'))
|
||||
|
||||
# now alice deletes it (alice should upload, bob download)
|
||||
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
|
||||
@ -881,9 +883,12 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
|
||||
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
|
||||
yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
|
||||
self.failIf(os.path.exists(bob_fname))
|
||||
self.assertFalse(os.path.exists(bob_fname))
|
||||
self.assertTrue(os.path.exists(bob_fname + '.backup'))
|
||||
self.assertFalse(os.path.exists(bob_fname + '.conflict'))
|
||||
|
||||
# now alice restores the file (with new contents)
|
||||
os.unlink(bob_fname + '.backup')
|
||||
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
|
||||
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
|
||||
yield self.alice_fileops.write(alice_fname, 'alice wuz here\n')
|
||||
@ -1080,13 +1085,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
|
||||
|
||||
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
|
||||
d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
|
||||
def Alice_conflicts_with_Bobs_last_downloaded_uri():
|
||||
if _debug: print "Alice conflicts with Bob\n"
|
||||
@ -1105,10 +1110,10 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
|
||||
d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
|
||||
|
||||
@ -1124,7 +1129,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
|
||||
d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file2", 0))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
|
||||
|
||||
def advance(ign):
|
||||
@ -1151,13 +1156,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
|
||||
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
|
||||
# d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
|
||||
@ -1168,13 +1173,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
bob_clock.advance(6)
|
||||
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
|
||||
## d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
|
||||
@ -1197,13 +1202,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
|
||||
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
|
||||
|
||||
@ -1215,8 +1220,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(foo)
|
||||
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
|
||||
|
||||
# prepare to perform another conflict test
|
||||
@ -1231,8 +1236,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def Bob_to_rewrite_file3():
|
||||
@ -1245,13 +1250,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
|
||||
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
|
||||
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
|
||||
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ def _with_log(op, res):
|
||||
"""
|
||||
try:
|
||||
op(res)
|
||||
except defer.AlreadyCalledError, e:
|
||||
except defer.AlreadyCalledError as e:
|
||||
log.err(e, op=repr(op), level=log.WEIRD)
|
||||
|
||||
def eventually_callback(d):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user