From 054efe055c094378e5b49b36c934682995e524d1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 16 Aug 2016 16:29:37 -0600 Subject: [PATCH] Fix file-operations/inotify testing This re-factors the magic-folder tests to abstract the whole "do a file operation" so we can properly send fake (or wait for real) inotify events to the uploader/downloader. This speeds up the tests quite a bit and makes test_alice_bob reasonable again (at about 1.5s instead of over 30s). --- src/allmydata/frontends/magic_folder.py | 16 +- src/allmydata/test/cli/test_magic_folder.py | 12 +- src/allmydata/test/test_magic_folder.py | 239 ++++++++++---------- 3 files changed, 141 insertions(+), 126 deletions(-) diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index bf1a1aace..9da16b6e4 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -27,7 +27,6 @@ from allmydata.util.time_format import format_time from allmydata.immutable.upload import FileName, Data from allmydata import magicfolderdb, magicpath -defer.setDebugging(True) IN_EXCL_UNLINK = 0x04000000L def get_inotify_module(): @@ -109,11 +108,14 @@ class MagicFolder(service.MultiService): self.uploader.start_uploading() # synchronous, returns None return self.downloader.start_downloading() + @defer.inlineCallbacks def finish(self): - d = self.uploader.stop() - d2 = self.downloader.stop() - d.addCallback(lambda ign: d2) - return d + # must stop these concurrently so that the clock.advance()s + # work correctly in the tests. Also, it's arguably + # most-correct. + d0 = self.downloader.stop() + d1 = self.uploader.stop() + yield defer.DeferredList([d0, d1]) def remove_service(self): return service.MultiService.disownServiceParent(self) @@ -133,6 +135,7 @@ class QueueMixin(HookMixin): 'processed': None, 'started': None, 'iteration': None, + 'inotify': None, } self.started_d = self.set_hook('started') @@ -371,6 +374,7 @@ class Uploader(QueueMixin): def stop(self): self._log("stop") + self._stopped = True self._notifier.stopReading() self._count('dirs_monitored', -1) self.periodic_callid.cancel() @@ -378,7 +382,6 @@ class Uploader(QueueMixin): d = self._notifier.wait_until_stopped() else: d = defer.succeed(None) - self._stopped = True # wait for processing loop to actually exit d.addCallback(lambda ign: self._processing) return d @@ -467,6 +470,7 @@ class Uploader(QueueMixin): return self._add_pending(relpath_u) + self._call_hook(path, 'inotify') def _process(self, item): # Uploader diff --git a/src/allmydata/test/cli/test_magic_folder.py b/src/allmydata/test/cli/test_magic_folder.py index 71024ca0c..a69b763f4 100644 --- a/src/allmydata/test/cli/test_magic_folder.py +++ b/src/allmydata/test/cli/test_magic_folder.py @@ -126,8 +126,16 @@ class MagicFolderCLITestMixin(CLITestMixin, GridTestMixin): def init_magicfolder(self, client_num, upload_dircap, collective_dircap, local_magic_dir, clock): dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.get_clientdir(i=client_num)) - magicfolder = MagicFolder(self.get_client(client_num), upload_dircap, collective_dircap, local_magic_dir, - dbfile, 0077, pending_delay=0.2, clock=clock) + magicfolder = MagicFolder( + client=self.get_client(client_num), + upload_dircap=upload_dircap, + collective_dircap=collective_dircap, + local_path_u=local_magic_dir, + dbfile=dbfile, + umask=0o077, + pending_delay=0.2, + clock=clock, + ) magicfolder.downloader._turn_delay = 0 magicfolder.setServiceParent(self.get_client(client_num)) diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py index 91f8ae485..b20196285 100644 --- a/src/allmydata/test/test_magic_folder.py +++ b/src/allmydata/test/test_magic_folder.py @@ -130,6 +130,63 @@ def iterate(magic): yield iterate_downloader(magic) +class FileOperationsHelper(object): + """ + This abstracts all file operations we might do in magic-folder unit-tests. + + This is so we can correctly wait for inotify events to 'actually' + propagate. For the mock tests this is easy, since we're sending + them sychronously. For the Real tests we have to wait for the + actual inotify thing. + + We could write this as a mixin instead; might fit existing style better? + """ + + def __init__(self, uploader, inject_events=False): + self._uploader = uploader + self._inotify = fake_inotify # fixme? + self._fake_inotify = inject_events + + def move(self, from_path_u, to_path_u): + from_fname = from_path_u + to_fname = to_path_u + d = self._uploader.set_hook('inotify') + os.rename(from_fname, to_fname) + + self._maybe_notify(to_fname, self._inotify.IN_MOVED_TO) + # hmm? we weren't faking IN_MOVED_FROM previously .. but seems like we should have been? + # self._uploader._notifier.event(to_filepath(from_fname), self._inotify.IN_MOVED_FROM) + return d + + def write(self, path_u, contents): + fname = path_u + d = self._uploader.set_hook('inotify') + with open(fname, "wb") as f: + f.write(contents) + + self._maybe_notify(fname, self._inotify.IN_CLOSE_WRITE) + return d + + def mkdir(self, path_u): + fname = path_u + d = self._uploader.set_hook('inotify') + os.mkdir(fname) + self._maybe_notify(fname, self._inotify.IN_CREATE | self._inotify.IN_ISDIR) + return d + + def delete(self, path_u): + fname = path_u + d = self._uploader.set_hook('inotify') + os.unlink(fname) + + self._maybe_notify(fname, self._inotify.IN_DELETE) + return d + + def _maybe_notify(self, fname, mask): + if self._fake_inotify: + self._uploader._notifier.event(to_filepath(fname), self._inotify.IN_DELETE) + + class CheckerMixin(object): """ Factored out of one of the many test classes. @@ -153,25 +210,17 @@ class CheckerMixin(object): previously_disappeared = self._get_count('uploader.objects_disappeared') path_u = abspath_expanduser_unicode(name_u, base=self.local_dir) - path = to_filepath(path_u) if directory: - os.mkdir(path_u) - event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR + yield self.fileops.mkdir(path_u) else: # We don't use FilePath.setContent() here because it creates a temporary file that # is renamed into place, which causes events that the test is not expecting. - f = open(path_u, "wb") - try: - f.write(data) - finally: - f.close() + yield self.fileops.write(path_u, data) if temporary: - os.unlink(path_u) - yield self.notify(path, self.inotify.IN_DELETE, flush=False) - event_mask = self.inotify.IN_CLOSE_WRITE + yield iterate(self.magicfolder) + yield self.fileops.delete(path_u) - yield self.notify(path, event_mask) yield iterate(self.magicfolder) encoded_name_u = magicpath.path2magic(name_u) @@ -222,6 +271,7 @@ class CheckerMixin(object): class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin, CheckerMixin): + inject_inotify = False def setUp(self): # super(MagicFolderAliceBobTestMixin, self).setUp() # XXX huh, why isn't this working? @@ -260,6 +310,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea self.alice_magicfolder = self.init_magicfolder(0, self.alice_upload_dircap, self.alice_collective_dircap, self.alice_magic_dir, self.alice_clock) + self.alice_fileops = FileOperationsHelper(self.alice_magicfolder.uploader, self.inject_inotify) d0 = self.alice_magicfolder.uploader.set_hook('iteration') d1 = self.alice_magicfolder.downloader.set_hook('iteration') self.alice_clock.advance(self.alice_magicfolder.uploader.scan_interval + 1) @@ -283,6 +334,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea self.bob_magicfolder = self.init_magicfolder(1, self.bob_upload_dircap, self.bob_collective_dircap, self.bob_magic_dir, self.bob_clock) + self.bob_fileops = FileOperationsHelper(self.bob_magicfolder.uploader, self.inject_inotify) d0 = self.bob_magicfolder.uploader.set_hook('iteration') d1 = self.bob_magicfolder.downloader.set_hook('iteration') self.bob_clock.advance(self.alice_magicfolder.uploader.scan_interval + 1) @@ -292,28 +344,30 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea d.addCallback(get_Bob_magicfolder) return d + @defer.inlineCallbacks def tearDown(self): - d = GridTestMixin.tearDown(self) - d.addCallback(lambda ign: self.alice_magicfolder.finish()) - d.addCallback(lambda ign: self.bob_magicfolder.finish()) + yield GridTestMixin.tearDown(self) + d0 = self.alice_magicfolder.finish() + d1 = self.bob_magicfolder.finish() + for mf in [self.alice_magicfolder, self.bob_magicfolder]: for loader in [mf.uploader, mf.downloader]: loader._clock.advance(loader.scan_interval + 1) - # XXX double-check: are self.mktemp() dirs blown away automagically? - return d + + yield d0 + yield d1 @defer.inlineCallbacks def test_alice_delete_bob_restore(self): alice_fname = os.path.join(self.alice_magic_dir, 'blam') bob_fname = os.path.join(self.bob_magic_dir, 'blam') - alice_up = self.alice_magicfolder.uploader.set_hook('processed') - fileutil.write(alice_fname, 'contents0\n') - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + alice_proc = self.alice_magicfolder.uploader.set_hook('processed') + yield self.alice_fileops.write(alice_fname, 'contents0\n') # alice uploads yield iterate_uploader(self.alice_magicfolder) - yield alice_up + yield alice_proc yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0) @@ -338,8 +392,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea # now bob deletes it (bob should upload, alice download) bob_proc = self.bob_magicfolder.uploader.set_hook('processed') alice_proc = self.alice_magicfolder.downloader.set_hook('processed') - os.unlink(bob_fname) - yield self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder) + yield self.bob_fileops.delete(bob_fname) yield iterate_uploader(self.bob_magicfolder) yield bob_proc @@ -354,11 +407,14 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1) yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1) + # not *entirely* sure why we need to iterate Alice for the + # real test here. But, we do. + yield iterate(self.alice_magicfolder) + # now alice restores it (alice should upload, bob download) alice_proc = self.alice_magicfolder.uploader.set_hook('processed') bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - fileutil.write(alice_fname, 'new contents\n') - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(alice_fname, 'new contents\n') yield iterate_uploader(self.alice_magicfolder) yield alice_proc @@ -384,8 +440,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea alice_proc = self.alice_magicfolder.uploader.set_hook('processed') bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - fileutil.write(alice_fname, 'contents0\n') - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(alice_fname, 'contents0\n') yield iterate_uploader(self.alice_magicfolder) yield alice_proc # alice uploads @@ -410,11 +465,10 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea # now bob deletes it (bob should upload, alice download) bob_proc = self.bob_magicfolder.uploader.set_hook('processed') alice_proc = self.alice_magicfolder.downloader.set_hook('processed') - os.unlink(bob_fname) - yield self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder) + yield self.bob_fileops.delete(bob_fname) # just after notifying bob, we also delete alice's, # covering the 'except' flow in _rename_deleted_file() - os.unlink(alice_fname) + yield self.alice_fileops.delete(alice_fname) yield iterate_uploader(self.bob_magicfolder) yield bob_proc @@ -435,8 +489,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea bob_fname = os.path.join(self.bob_magic_dir, 'blam') # alice creates a file, bob downloads it - fileutil.write(alice_fname, 'contents0\n') - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(alice_fname, 'contents0\n') yield iterate(self.alice_magicfolder) yield iterate(self.bob_magicfolder) @@ -456,8 +509,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea ) # now bob updates it (bob should upload, alice download) - fileutil.write(bob_fname, 'bob wuz here\n') - yield self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder) + yield self.bob_fileops.write(bob_fname, 'bob wuz here\n') yield iterate(self.bob_magicfolder) yield iterate(self.alice_magicfolder) @@ -474,8 +526,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea # bob_fname = os.path.join(self.bob_magic_dir, 'blam') # Alice creates a file - fileutil.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)])) - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)])) yield iterate(self.alice_magicfolder) # check alice created the file yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0) @@ -531,8 +582,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea alice_proc = self.alice_magicfolder.uploader.set_hook('processed') bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - fileutil.write(alice_fname, 'contents0\n') - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(alice_fname, 'contents0\n') yield iterate_uploader(self.alice_magicfolder) yield alice_proc # alice uploads @@ -558,8 +608,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea # now alice deletes it (alice should upload, bob download) alice_proc = self.alice_magicfolder.uploader.set_hook('processed') bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - os.unlink(alice_fname) - yield self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder) + yield self.alice_fileops.delete(alice_fname) yield iterate_uploader(self.alice_magicfolder) yield alice_proc @@ -576,8 +625,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea # now alice restores the file (with new contents) alice_proc = self.alice_magicfolder.uploader.set_hook('processed') bob_proc = self.bob_magicfolder.downloader.set_hook('processed') - fileutil.write(alice_fname, 'alice wuz here\n') - yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(alice_fname, 'alice wuz here\n') yield iterate_uploader(self.alice_magicfolder) yield iterate_downloader(self.alice_magicfolder) # why? @@ -644,9 +692,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea def Alice_to_write_a_file(): if _debug: print "Alice writes a file\n\n\n\n\n" self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u) - yield task.deferLater(reactor, 5, lambda: None) - fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.") - yield self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + yield self.alice_fileops.write(self.file_path, "meow, meow meow. meow? meow meow! meow.") d.addCallback(_wait_for, Alice_to_write_a_file) d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0)) @@ -669,19 +715,19 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea @defer.inlineCallbacks def Alice_to_delete_file(): if _debug: print "Alice deletes the file!\n\n\n\n" - yield task.deferLater(reactor, 5, lambda: None) - os.unlink(self.file_path) - yield self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder) + yield self.alice_fileops.delete(self.file_path) yield iterate(self.alice_magicfolder) yield iterate(self.bob_magicfolder) d.addCallback(_wait_for, Alice_to_delete_file) @defer.inlineCallbacks def notify_bob_moved(ign): + # WARNING: this is just directly notifying for the mock + # tests, because in the Real* tests the .backup file will + # me moved into place (from the original) p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u) - fileutil.write((p + u'.backup'), "meow, meow meow. meow? meow meow! meow.") - yield self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False) - yield self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder) + if self.bob_fileops._fake_inotify: + self.bob_magicfolder.uploader._notifier.event(to_filepath(p + u'.backup'), fake_inotify.IN_MOVED_TO) yield iterate(self.bob_magicfolder) d.addCallback(notify_bob_moved) @@ -702,8 +748,10 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea def Alice_to_rewrite_file(): if _debug: print "Alice rewrites file\n" self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u) - fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.") - return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + return self.alice_fileops.write( + self.file_path, + "Alice suddenly sees the white rabbit running into the forest.", + ) d.addCallback(_wait_for, Alice_to_rewrite_file) d.addCallback(lambda ign: iterate(self.bob_magicfolder)) @@ -754,9 +802,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea if _debug: print "Bob rewrites file\n" self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u) if _debug: print "---- bob's file is %r" % (self.file_path,) - fileutil.write(self.file_path, "No white rabbit to be found.") - return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder) - d.addCallback(lambda ign: task.deferLater(reactor, 5, lambda: None)) + return self.bob_fileops.write(self.file_path, "No white rabbit to be found.") d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False)) d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3)) @@ -802,16 +848,11 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea # prepare to perform another conflict test @defer.inlineCallbacks def Alice_to_write_file2(): -# uploaded_d = self.bob_magicfolder.uploader.set_hook('processed') if _debug: print "Alice writes a file2\n" - yield task.deferLater(reactor, 5, lambda: None) self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u) - fileutil.write(self.file_path, "something") - d = self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + d = self.alice_fileops.write(self.file_path, "something") self.bob_clock.advance(4) yield d -# yield uploaded_d - d.addCallback(lambda ign: task.deferLater(reactor, 5, lambda: None)) d.addCallback(_wait_for, Alice_to_write_file2) d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0)) d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file2", 0)) @@ -837,8 +878,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea if _debug: print "Bob rewrites file2\n" self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u) if _debug: print "---- bob's file is %r" % (self.file_path,) - fileutil.write(self.file_path, "roger roger. what vector?") - return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder) + return self.bob_fileops.write(self.file_path, "roger roger. what vector?") d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False)) d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1)) d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5)) @@ -914,8 +954,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea def Alice_to_write_file3(): if _debug: print "Alice writes a file\n" self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u) - fileutil.write(self.file_path, "something") - return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder) + return self.alice_fileops.write(self.file_path, "something") d.addCallback(_wait_for, Alice_to_write_file3) d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0)) d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder)) @@ -928,8 +967,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea if _debug: print "Bob rewrites file3\n" self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u) if _debug: print "---- bob's file is %r" % (self.file_path,) - fileutil.write(self.file_path, "roger roger") - return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder) + return self.bob_fileops.write(self.file_path, "roger roger") d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False)) d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1)) d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7)) @@ -989,6 +1027,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall def _wait_until_started(self, ign): #print "_wait_until_started" self.magicfolder = self.get_client().getServiceNamed('magic-folder') + self.fileops = FileOperationsHelper(self.magicfolder.uploader, self.inject_inotify) self.up_clock = task.Clock() self.down_clock = task.Clock() self.magicfolder.uploader._clock = self.up_clock @@ -1087,8 +1126,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall @defer.inlineCallbacks def _check_move_empty_tree(res): self.mkdir_nonascii(empty_tree_dir) - os.rename(empty_tree_dir, new_empty_tree_dir) - yield self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO) + yield self.fileops.move(empty_tree_dir, new_empty_tree_dir) yield iterate(self.magicfolder) d.addCallback(_check_move_empty_tree) @@ -1103,8 +1141,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall self.mkdir_nonascii(small_tree_dir) what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir) fileutil.write(what_path, "say when") - os.rename(small_tree_dir, new_small_tree_dir) - yield self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO) + yield self.fileops.move(small_tree_dir, new_small_tree_dir) yield iterate(self.magicfolder) # when we add the dir, we queue a scan of it; so we want # the upload to "go" as well requiring 1 more iteration @@ -1120,8 +1157,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall @defer.inlineCallbacks def _check_moved_tree_is_watched(res): another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir) - fileutil.write(another_path, "file") - yield self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE) + yield self.fileops.write(another_path, "file") yield iterate(self.magicfolder) d.addCallback(_check_moved_tree_is_watched) @@ -1147,8 +1183,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall @defer.inlineCallbacks def create_test_file(filename): test_file = abspath_expanduser_unicode(filename, base=self.local_dir) - fileutil.write(test_file, "meow %s" % filename) - yield self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE) + yield self.fileops.write(test_file, "meow %s" % filename) yield iterate(self.magicfolder) d.addCallback(lambda ign: create_test_file(u"what1")) @@ -1175,16 +1210,14 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall def test_delete(self): # setup: create a file 'foo' path = os.path.join(self.local_dir, u'foo') - fileutil.write(path, 'foo\n') - yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE) + yield self.fileops.write(path, 'foo\n') yield iterate_uploader(self.magicfolder) self.assertTrue(os.path.exists(path)) node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo') self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,)) # the test: delete the file (and do fake notifies) - os.unlink(path) - yield self.notify(to_filepath(path), self.inotify.IN_DELETE) + yield self.fileops.delete(path) yield iterate_uploader(self.magicfolder) self.assertFalse(os.path.exists(path)) @@ -1199,14 +1232,12 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall def test_delete_and_restore(self): # setup: create a file path = os.path.join(self.local_dir, u'foo') - fileutil.write(path, 'foo\n') - yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE) + yield self.fileops.write(path, 'foo\n') yield iterate_uploader(self.magicfolder) self.assertTrue(os.path.exists(path)) # ...and delete the file - os.unlink(path) - yield self.notify(to_filepath(path), self.inotify.IN_DELETE) + yield self.fileops.delete(path) yield iterate_uploader(self.magicfolder) self.assertFalse(os.path.exists(path)) @@ -1217,8 +1248,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall # restore the file, with different contents path = os.path.join(self.local_dir, u'foo') - fileutil.write(path, 'bar\n') - yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE) + yield self.fileops.write(path, 'bar\n') yield iterate_uploader(self.magicfolder) # ensure we still have a DB entry, and that the version is 2 @@ -1251,23 +1281,18 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall class MockTestAliceBob(MagicFolderAliceBobTestMixin, unittest.TestCase): + inject_inotify = True + def setUp(self): d = super(MockTestAliceBob, self).setUp() self.inotify = fake_inotify self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify) return d - def notify(self, path, mask, magic=None, flush=True): - if magic is None: - magic = self.magicfolder - magic.uploader._notifier.event(path, mask) - # no flush for the mock test. - return task.deferLater(reactor, 0.1, lambda: None) - - class MockTest(SingleMagicFolderTestMixin, unittest.TestCase): """This can run on any platform, and even if twisted.internet.inotify can't be imported.""" + inject_inotify = True def setUp(self): d = super(MockTest, self).setUp() @@ -1275,13 +1300,6 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase): self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify) return d - def notify(self, path, mask, magic=None, flush=True): - if magic is None: - magic = self.magicfolder - magic.uploader._notifier.event(path, mask) - # no flush for the mock test. - return task.deferLater(reactor, 0.1, lambda: None) - def test_errors(self): self.set_up_grid(oneshare=True) @@ -1425,38 +1443,23 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase): class RealTest(SingleMagicFolderTestMixin, unittest.TestCase): """This is skipped unless both Twisted and the platform support inotify.""" + inject_inotify = False def setUp(self): d = super(RealTest, self).setUp() self.inotify = magic_folder.get_inotify_module() return d - def notify(self, path, mask, magic=None, flush=True): - # Writing to the filesystem causes the notification. - # Actually, there's no way to know when the actual - # notification will occur, and anyway we're not waiting for - # them in any case...so we'll just fudge it and hope 100ms is enough. - delay = 0.1 if sys.platform == "win32" else 0.1 - return task.deferLater(reactor, delay, lambda: None) - class RealTestAliceBob(MagicFolderAliceBobTestMixin, unittest.TestCase): """This is skipped unless both Twisted and the platform support inotify.""" + inject_inotify = False def setUp(self): d = super(RealTestAliceBob, self).setUp() self.inotify = magic_folder.get_inotify_module() return d - # XXX flush doesn't do anything (anymore?) - def notify(self, path, mask, magic=None, flush=True): - # Writing to the filesystem causes the notification. - # Actually, there's no way to know when the actual - # notification will occur, and anyway we're not waiting for - # them in any case...so we'll just fudge it and hope 100ms is enough. - delay = 0.1 if sys.platform == "win32" else 0.1 - return task.deferLater(reactor, delay, lambda: None) - try: magic_folder.get_inotify_module()