ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-22 02:16:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'pr313'

Browse Source
This commit is contained in:
Brian Warner 2016-08-18 18:03:38 -07:00
commit 9979a7dca3
3 changed files with 171 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/allmydata/frontends/magic_folder.py
View File

@ -27,7 +27,6 @@ from allmydata.util.time_format import format_time
from allmydata.immutable.upload import FileName, Data
from allmydata import magicfolderdb, magicpath
defer.setDebugging(True)
IN_EXCL_UNLINK = 0x04000000L
def get_inotify_module():
@ -109,11 +108,14 @@ class MagicFolder(service.MultiService):
self.uploader.start_uploading() # synchronous, returns None
return self.downloader.start_downloading()
@defer.inlineCallbacks
def finish(self):
d = self.uploader.stop()
d2 = self.downloader.stop()
d.addCallback(lambda ign: d2)
return d
# must stop these concurrently so that the clock.advance()s
# work correctly in the tests. Also, it's arguably
# most-correct.
d0 = self.downloader.stop()
d1 = self.uploader.stop()
yield defer.DeferredList([d0, d1])
def remove_service(self):
return service.MultiService.disownServiceParent(self)
@ -133,6 +135,7 @@ class QueueMixin(HookMixin):
'processed': None,
'started': None,
'iteration': None,
'inotify': None,
}
self.started_d = self.set_hook('started')
@ -371,6 +374,7 @@ class Uploader(QueueMixin):
def stop(self):
self._log("stop")
self._stopped = True
self._notifier.stopReading()
self._count('dirs_monitored', -1)
self.periodic_callid.cancel()
@ -378,7 +382,6 @@ class Uploader(QueueMixin):
d = self._notifier.wait_until_stopped()
else:
d = defer.succeed(None)
self._stopped = True
# wait for processing loop to actually exit
d.addCallback(lambda ign: self._processing)
return d
@ -467,6 +470,7 @@ class Uploader(QueueMixin):
return
self._add_pending(relpath_u)
self._call_hook(path, 'inotify')
def _process(self, item):
# Uploader

12
src/allmydata/test/cli/test_magic_folder.py
View File

@ -126,8 +126,16 @@ class MagicFolderCLITestMixin(CLITestMixin, GridTestMixin):
def init_magicfolder(self, client_num, upload_dircap, collective_dircap, local_magic_dir, clock):
dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.get_clientdir(i=client_num))
magicfolder = MagicFolder(self.get_client(client_num), upload_dircap, collective_dircap, local_magic_dir,
dbfile, 0077, pending_delay=0.2, clock=clock)
magicfolder = MagicFolder(
client=self.get_client(client_num),
upload_dircap=upload_dircap,
collective_dircap=collective_dircap,
local_path_u=local_magic_dir,
dbfile=dbfile,
umask=0o077,
pending_delay=0.2,
clock=clock,
)
magicfolder.downloader._turn_delay = 0
magicfolder.setServiceParent(self.get_client(client_num))

269
src/allmydata/test/test_magic_folder.py
View File

@ -130,6 +130,63 @@ def iterate(magic):
yield iterate_downloader(magic)
class FileOperationsHelper(object):
"""
This abstracts all file operations we might do in magic-folder unit-tests.
This is so we can correctly wait for inotify events to 'actually'
propagate. For the mock tests this is easy, since we're sending
them sychronously. For the Real tests we have to wait for the
actual inotify thing.
We could write this as a mixin instead; might fit existing style better?
"""
def __init__(self, uploader, inject_events=False):
self._uploader = uploader
self._inotify = fake_inotify # fixme?
self._fake_inotify = inject_events
def move(self, from_path_u, to_path_u):
from_fname = from_path_u
to_fname = to_path_u
d = self._uploader.set_hook('inotify')
os.rename(from_fname, to_fname)
self._maybe_notify(to_fname, self._inotify.IN_MOVED_TO)
# hmm? we weren't faking IN_MOVED_FROM previously .. but seems like we should have been?
# self._uploader._notifier.event(to_filepath(from_fname), self._inotify.IN_MOVED_FROM)
return d
def write(self, path_u, contents):
fname = path_u
d = self._uploader.set_hook('inotify')
with open(fname, "wb") as f:
f.write(contents)
self._maybe_notify(fname, self._inotify.IN_CLOSE_WRITE)
return d
def mkdir(self, path_u):
fname = path_u
d = self._uploader.set_hook('inotify')
os.mkdir(fname)
self._maybe_notify(fname, self._inotify.IN_CREATE | self._inotify.IN_ISDIR)
return d
def delete(self, path_u):
fname = path_u
d = self._uploader.set_hook('inotify')
os.unlink(fname)
self._maybe_notify(fname, self._inotify.IN_DELETE)
return d
def _maybe_notify(self, fname, mask):
if self._fake_inotify:
self._uploader._notifier.event(to_filepath(fname), self._inotify.IN_DELETE)
class CheckerMixin(object):
"""
Factored out of one of the many test classes.
@ -153,25 +210,18 @@ class CheckerMixin(object):
previously_disappeared = self._get_count('uploader.objects_disappeared')
path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
path = to_filepath(path_u)
if directory:
os.mkdir(path_u)
event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
yield self.fileops.mkdir(path_u)
else:
# We don't use FilePath.setContent() here because it creates a temporary file that
# is renamed into place, which causes events that the test is not expecting.
f = open(path_u, "wb")
try:
f.write(data)
finally:
f.close()
yield self.fileops.write(path_u, data)
yield iterate(self.magicfolder)
if temporary:
os.unlink(path_u)
yield self.notify(path, self.inotify.IN_DELETE, flush=False)
event_mask = self.inotify.IN_CLOSE_WRITE
yield iterate(self.magicfolder)
yield self.fileops.delete(path_u)
yield self.notify(path, event_mask)
yield iterate(self.magicfolder)
encoded_name_u = magicpath.path2magic(name_u)
@ -222,6 +272,7 @@ class CheckerMixin(object):
class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin, CheckerMixin):
inject_inotify = False
def setUp(self):
# super(MagicFolderAliceBobTestMixin, self).setUp() # XXX huh, why isn't this working?
@ -260,6 +311,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
self.alice_magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
self.alice_collective_dircap,
self.alice_magic_dir, self.alice_clock)
self.alice_fileops = FileOperationsHelper(self.alice_magicfolder.uploader, self.inject_inotify)
d0 = self.alice_magicfolder.uploader.set_hook('iteration')
d1 = self.alice_magicfolder.downloader.set_hook('iteration')
self.alice_clock.advance(self.alice_magicfolder.uploader.scan_interval + 1)
@ -283,6 +335,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
self.bob_magicfolder = self.init_magicfolder(1, self.bob_upload_dircap,
self.bob_collective_dircap,
self.bob_magic_dir, self.bob_clock)
self.bob_fileops = FileOperationsHelper(self.bob_magicfolder.uploader, self.inject_inotify)
d0 = self.bob_magicfolder.uploader.set_hook('iteration')
d1 = self.bob_magicfolder.downloader.set_hook('iteration')
self.bob_clock.advance(self.alice_magicfolder.uploader.scan_interval + 1)
@ -292,28 +345,31 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(get_Bob_magicfolder)
return d
@defer.inlineCallbacks
def tearDown(self):
d = GridTestMixin.tearDown(self)
d.addCallback(lambda ign: self.alice_magicfolder.finish())
d.addCallback(lambda ign: self.bob_magicfolder.finish())
yield GridTestMixin.tearDown(self)
d0 = self.alice_magicfolder.finish()
d1 = self.bob_magicfolder.finish()
for mf in [self.alice_magicfolder, self.bob_magicfolder]:
for loader in [mf.uploader, mf.downloader]:
loader._clock.advance(loader.scan_interval + 1)
# XXX double-check: are self.mktemp() dirs blown away automagically?
return d
yield d0
yield d1
@defer.inlineCallbacks
def test_alice_delete_bob_restore(self):
alice_fname = os.path.join(self.alice_magic_dir, 'blam')
bob_fname = os.path.join(self.bob_magic_dir, 'blam')
alice_up = self.alice_magicfolder.uploader.set_hook('processed')
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate(self.alice_magicfolder) # for windows
# alice uploads
yield iterate_uploader(self.alice_magicfolder)
yield alice_up
yield alice_proc
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0)
yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
@ -335,11 +391,12 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
1
)
yield iterate(self.bob_magicfolder) # for windows
# now bob deletes it (bob should upload, alice download)
bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
os.unlink(bob_fname)
yield self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
yield self.bob_fileops.delete(bob_fname)
yield iterate(self.bob_magicfolder) # for windows
yield iterate_uploader(self.bob_magicfolder)
yield bob_proc
@ -354,11 +411,15 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
# not *entirely* sure why we need to iterate Alice for the
# real test here. But, we do.
yield iterate(self.alice_magicfolder)
# now alice restores it (alice should upload, bob download)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'new contents\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'new contents\n')
yield iterate(self.alice_magicfolder) # for windows
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc
@ -384,8 +445,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate(self.alice_magicfolder) # for windows
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc # alice uploads
@ -393,6 +454,9 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
yield iterate_downloader(self.bob_magicfolder)
yield bob_proc # bob downloads
yield iterate(self.alice_magicfolder) # for windows
yield iterate(self.bob_magicfolder) # for windows
# check the state (XXX I had to switch the versions to 0; is that really right? why?)
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0)
yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
@ -410,11 +474,10 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now bob deletes it (bob should upload, alice download)
bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
os.unlink(bob_fname)
yield self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
yield self.bob_fileops.delete(bob_fname)
# just after notifying bob, we also delete alice's,
# covering the 'except' flow in _rename_deleted_file()
os.unlink(alice_fname)
yield self.alice_fileops.delete(alice_fname)
yield iterate_uploader(self.bob_magicfolder)
yield bob_proc
@ -435,9 +498,9 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
bob_fname = os.path.join(self.bob_magic_dir, 'blam')
# alice creates a file, bob downloads it
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate(self.alice_magicfolder)
yield iterate(self.alice_magicfolder)
yield iterate(self.bob_magicfolder)
@ -455,10 +518,11 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
1
)
yield iterate(self.bob_magicfolder)
# now bob updates it (bob should upload, alice download)
fileutil.write(bob_fname, 'bob wuz here\n')
yield self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
yield self.bob_fileops.write(bob_fname, 'bob wuz here\n')
yield iterate(self.bob_magicfolder)
yield iterate(self.bob_magicfolder)
yield iterate(self.alice_magicfolder)
@ -474,8 +538,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# bob_fname = os.path.join(self.bob_magic_dir, 'blam')
# Alice creates a file
fileutil.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)]))
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)]))
yield iterate(self.alice_magicfolder)
# check alice created the file
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0)
@ -531,8 +594,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate(self.alice_magicfolder) # for windows
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc # alice uploads
@ -558,8 +621,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now alice deletes it (alice should upload, bob download)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
os.unlink(alice_fname)
yield self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
yield self.alice_fileops.delete(alice_fname)
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc
@ -576,8 +638,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now alice restores the file (with new contents)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'alice wuz here\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'alice wuz here\n')
yield iterate(self.alice_magicfolder) # for windows
yield iterate_uploader(self.alice_magicfolder)
yield iterate_downloader(self.alice_magicfolder) # why?
@ -644,9 +706,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
def Alice_to_write_a_file():
if _debug: print "Alice writes a file\n\n\n\n\n"
self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
yield task.deferLater(reactor, 5, lambda: None)
fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
yield self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
yield iterate(self.alice_magicfolder)
d.addCallback(_wait_for, Alice_to_write_a_file)
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
@ -669,19 +730,19 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
@defer.inlineCallbacks
def Alice_to_delete_file():
if _debug: print "Alice deletes the file!\n\n\n\n"
yield task.deferLater(reactor, 5, lambda: None)
os.unlink(self.file_path)
yield self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
yield self.alice_fileops.delete(self.file_path)
yield iterate(self.alice_magicfolder)
yield iterate(self.bob_magicfolder)
d.addCallback(_wait_for, Alice_to_delete_file)
@defer.inlineCallbacks
def notify_bob_moved(ign):
# WARNING: this is just directly notifying for the mock
# tests, because in the Real* tests the .backup file will
# me moved into place (from the original)
p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
fileutil.write((p + u'.backup'), "meow, meow meow. meow? meow meow! meow.")
yield self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
yield self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
if self.bob_fileops._fake_inotify:
self.bob_magicfolder.uploader._notifier.event(to_filepath(p + u'.backup'), fake_inotify.IN_MOVED_TO)
yield iterate(self.bob_magicfolder)
d.addCallback(notify_bob_moved)
@ -699,11 +760,15 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.bob_magicfolder))
@defer.inlineCallbacks
def Alice_to_rewrite_file():
if _debug: print "Alice rewrites file\n"
self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(
self.file_path,
"Alice suddenly sees the white rabbit running into the forest.",
)
yield iterate(self.alice_magicfolder)
d.addCallback(_wait_for, Alice_to_rewrite_file)
d.addCallback(lambda ign: iterate(self.bob_magicfolder))
@ -750,13 +815,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
# d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 2, magic=self.bob_magicfolder))
@defer.inlineCallbacks
def Bob_to_rewrite_file():
if _debug: print "Bob rewrites file\n"
self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
if _debug: print "---- bob's file is %r" % (self.file_path,)
fileutil.write(self.file_path, "No white rabbit to be found.")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
d.addCallback(lambda ign: task.deferLater(reactor, 5, lambda: None))
yield self.bob_fileops.write(self.file_path, "No white rabbit to be found.")
yield iterate(self.bob_magicfolder)
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
@ -802,16 +867,11 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# prepare to perform another conflict test
@defer.inlineCallbacks
def Alice_to_write_file2():
# uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
if _debug: print "Alice writes a file2\n"
yield task.deferLater(reactor, 5, lambda: None)
self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
fileutil.write(self.file_path, "something")
d = self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
d = self.alice_fileops.write(self.file_path, "something")
self.bob_clock.advance(4)
yield d
# yield uploaded_d
d.addCallback(lambda ign: task.deferLater(reactor, 5, lambda: None))
d.addCallback(_wait_for, Alice_to_write_file2)
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file2", 0))
@ -833,12 +893,13 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(advance)
d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file2", 0))
@defer.inlineCallbacks
def Bob_to_rewrite_file2():
if _debug: print "Bob rewrites file2\n"
self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
if _debug: print "---- bob's file is %r" % (self.file_path,)
fileutil.write(self.file_path, "roger roger. what vector?")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
yield self.bob_fileops.write(self.file_path, "roger roger. what vector?")
yield iterate(self.bob_magicfolder)
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
@ -911,11 +972,12 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
# prepare to perform another conflict test
@defer.inlineCallbacks
def Alice_to_write_file3():
if _debug: print "Alice writes a file\n"
self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
fileutil.write(self.file_path, "something")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(self.file_path, "something")
yield iterate(self.alice_magicfolder)
d.addCallback(_wait_for, Alice_to_write_file3)
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
@ -924,12 +986,14 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
@defer.inlineCallbacks
def Bob_to_rewrite_file3():
if _debug: print "Bob rewrites file3\n"
self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
if _debug: print "---- bob's file is %r" % (self.file_path,)
fileutil.write(self.file_path, "roger roger")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
yield iterate(self.bob_magicfolder)
yield self.bob_fileops.write(self.file_path, "roger roger")
yield iterate(self.bob_magicfolder)
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
@ -989,6 +1053,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
def _wait_until_started(self, ign):
#print "_wait_until_started"
self.magicfolder = self.get_client().getServiceNamed('magic-folder')
self.fileops = FileOperationsHelper(self.magicfolder.uploader, self.inject_inotify)
self.up_clock = task.Clock()
self.down_clock = task.Clock()
self.magicfolder.uploader._clock = self.up_clock
@ -1087,8 +1152,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
@defer.inlineCallbacks
def _check_move_empty_tree(res):
self.mkdir_nonascii(empty_tree_dir)
os.rename(empty_tree_dir, new_empty_tree_dir)
yield self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
yield self.fileops.move(empty_tree_dir, new_empty_tree_dir)
yield iterate(self.magicfolder)
d.addCallback(_check_move_empty_tree)
@ -1103,8 +1167,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
self.mkdir_nonascii(small_tree_dir)
what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
fileutil.write(what_path, "say when")
os.rename(small_tree_dir, new_small_tree_dir)
yield self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
yield self.fileops.move(small_tree_dir, new_small_tree_dir)
yield iterate(self.magicfolder)
# when we add the dir, we queue a scan of it; so we want
# the upload to "go" as well requiring 1 more iteration
@ -1120,9 +1183,9 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
@defer.inlineCallbacks
def _check_moved_tree_is_watched(res):
another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
fileutil.write(another_path, "file")
yield self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(another_path, "file")
yield iterate(self.magicfolder)
yield iterate(self.magicfolder) # windows; why?
d.addCallback(_check_moved_tree_is_watched)
d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
@ -1147,9 +1210,9 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
@defer.inlineCallbacks
def create_test_file(filename):
test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
fileutil.write(test_file, "meow %s" % filename)
yield self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(test_file, "meow %s" % filename)
yield iterate(self.magicfolder)
yield iterate(self.magicfolder) # windows; why?
d.addCallback(lambda ign: create_test_file(u"what1"))
d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
@ -1175,16 +1238,15 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
def test_delete(self):
# setup: create a file 'foo'
path = os.path.join(self.local_dir, u'foo')
fileutil.write(path, 'foo\n')
yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(path, 'foo\n')
yield iterate_uploader(self.magicfolder)
yield iterate_uploader(self.magicfolder) # req'd for windows; not sure why?
self.assertTrue(os.path.exists(path))
node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
# the test: delete the file (and do fake notifies)
os.unlink(path)
yield self.notify(to_filepath(path), self.inotify.IN_DELETE)
yield self.fileops.delete(path)
yield iterate_uploader(self.magicfolder)
self.assertFalse(os.path.exists(path))
@ -1199,14 +1261,13 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
def test_delete_and_restore(self):
# setup: create a file
path = os.path.join(self.local_dir, u'foo')
fileutil.write(path, 'foo\n')
yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(path, 'foo\n')
yield iterate_uploader(self.magicfolder)
yield iterate_uploader(self.magicfolder) # req'd for windows; why?
self.assertTrue(os.path.exists(path))
# ...and delete the file
os.unlink(path)
yield self.notify(to_filepath(path), self.inotify.IN_DELETE)
yield self.fileops.delete(path)
yield iterate_uploader(self.magicfolder)
self.assertFalse(os.path.exists(path))
@ -1217,8 +1278,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
# restore the file, with different contents
path = os.path.join(self.local_dir, u'foo')
fileutil.write(path, 'bar\n')
yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(path, 'bar\n')
yield iterate_uploader(self.magicfolder)
# ensure we still have a DB entry, and that the version is 2
@ -1251,23 +1311,18 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
class MockTestAliceBob(MagicFolderAliceBobTestMixin, unittest.TestCase):
inject_inotify = True
def setUp(self):
d = super(MockTestAliceBob, self).setUp()
self.inotify = fake_inotify
self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
return d
def notify(self, path, mask, magic=None, flush=True):
if magic is None:
magic = self.magicfolder
magic.uploader._notifier.event(path, mask)
# no flush for the mock test.
return task.deferLater(reactor, 0.1, lambda: None)
class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
"""This can run on any platform, and even if twisted.internet.inotify can't be imported."""
inject_inotify = True
def setUp(self):
d = super(MockTest, self).setUp()
@ -1275,13 +1330,6 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
return d
def notify(self, path, mask, magic=None, flush=True):
if magic is None:
magic = self.magicfolder
magic.uploader._notifier.event(path, mask)
# no flush for the mock test.
return task.deferLater(reactor, 0.1, lambda: None)
def test_errors(self):
self.set_up_grid(oneshare=True)
@ -1425,38 +1473,23 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
class RealTest(SingleMagicFolderTestMixin, unittest.TestCase):
"""This is skipped unless both Twisted and the platform support inotify."""
inject_inotify = False
def setUp(self):
d = super(RealTest, self).setUp()
self.inotify = magic_folder.get_inotify_module()
return d
def notify(self, path, mask, magic=None, flush=True):
# Writing to the filesystem causes the notification.
# Actually, there's no way to know when the actual
# notification will occur, and anyway we're not waiting for
# them in any case...so we'll just fudge it and hope 100ms is enough.
delay = 0.1 if sys.platform == "win32" else 0.1
return task.deferLater(reactor, delay, lambda: None)
class RealTestAliceBob(MagicFolderAliceBobTestMixin, unittest.TestCase):
"""This is skipped unless both Twisted and the platform support inotify."""
inject_inotify = False
def setUp(self):
d = super(RealTestAliceBob, self).setUp()
self.inotify = magic_folder.get_inotify_module()
return d
# XXX flush doesn't do anything (anymore?)
def notify(self, path, mask, magic=None, flush=True):
# Writing to the filesystem causes the notification.
# Actually, there's no way to know when the actual
# notification will occur, and anyway we're not waiting for
# them in any case...so we'll just fudge it and hope 100ms is enough.
delay = 0.1 if sys.platform == "win32" else 0.1
return task.deferLater(reactor, delay, lambda: None)
try:
magic_folder.get_inotify_module()