From b949082f8ee21ee4e0f378ba7d80bdfcf199c295 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Tue, 29 Mar 2016 17:03:24 +0100 Subject: [PATCH] Change type of mtime and ctime stored in magicfolderdb to integer nanoseconds. Signed-off-by: Daira Hopwood --- src/allmydata/frontends/magic_folder.py | 16 +++++----- src/allmydata/magicfolderdb.py | 31 ++++++++++++-------- src/allmydata/test/test_magic_folder.py | 22 +++++++++----- src/allmydata/util/fileutil.py | 39 +++++++++++++------------ 4 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index 25138d025..9618f252e 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -47,18 +47,20 @@ def get_inotify_module(): def is_new_file(pathinfo, db_entry): if db_entry is None: + print "is_new_file: True because db_entry is None" return True if not pathinfo.exists and db_entry.size is None: - print("NOT because", pathinfo.exists, db_entry.size) + print("is_new_file: False because", pathinfo.exists, db_entry.size) return False - print("NOT because", pathinfo.size, pathinfo.ctime, pathinfo.mtime, - db_entry.size, db_entry.ctime, db_entry.mtime, - ((pathinfo.size, pathinfo.ctime, pathinfo.mtime) != - (db_entry.size, db_entry.ctime, db_entry.mtime))) - return ((pathinfo.size, pathinfo.ctime, pathinfo.mtime) != - (db_entry.size, db_entry.ctime, db_entry.mtime)) + result = ((pathinfo.size, pathinfo.ctime_ns, pathinfo.mtime_ns) != + (db_entry.size, db_entry.ctime_ns, db_entry.mtime_ns)) + + print("is_new_file:", result, "because", + pathinfo.size, pathinfo.ctime_ns, pathinfo.mtime_ns, + db_entry.size, db_entry.ctime_ns, db_entry.mtime_ns) + return result class MagicFolder(service.MultiService): diff --git a/src/allmydata/magicfolderdb.py b/src/allmydata/magicfolderdb.py index 5ad6e7444..c8d7a5589 100644 --- a/src/allmydata/magicfolderdb.py +++ b/src/allmydata/magicfolderdb.py @@ -14,11 +14,10 @@ CREATE TABLE version CREATE TABLE local_files ( - path VARCHAR(1024) PRIMARY KEY, -- UTF-8 filename relative to local magic folder dir - -- note that size is before mtime and ctime here, but after in function parameters - size INTEGER, -- ST_SIZE, or NULL if the file has been deleted - mtime NUMBER, -- ST_MTIME - ctime NUMBER, -- ST_CTIME + path VARCHAR(1024) PRIMARY KEY, -- UTF-8 filename relative to local magic folder dir + size INTEGER, -- ST_SIZE, or NULL if the file has been deleted + mtime_ns INTEGER, -- ST_MTIME in nanoseconds + ctime_ns INTEGER, -- ST_CTIME in nanoseconds version INTEGER, last_uploaded_uri VARCHAR(256), -- URI:CHK:... last_downloaded_uri VARCHAR(256), -- URI:CHK:... @@ -43,7 +42,8 @@ def get_magicfolderdb(dbfile, stderr=sys.stderr, print >>stderr, e return None -PathEntry = namedtuple('PathEntry', 'size mtime ctime version last_uploaded_uri last_downloaded_uri last_downloaded_timestamp') +PathEntry = namedtuple('PathEntry', 'size mtime_ns ctime_ns version last_uploaded_uri' + 'last_downloaded_uri last_downloaded_timestamp') class MagicFolderDB(object): VERSION = 1 @@ -62,7 +62,8 @@ class MagicFolderDB(object): if there is no such entry. """ c = self.cursor - c.execute("SELECT size, mtime, ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp" + c.execute("SELECT size, mtime_ns, ctime_ns, version, last_uploaded_uri," + " last_downloaded_uri, last_downloaded_timestamp" " FROM local_files" " WHERE path=?", (relpath_u,)) @@ -71,8 +72,9 @@ class MagicFolderDB(object): print "no dbentry for %r" % (relpath_u,) return None else: - (size, mtime, ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp) = row - return PathEntry(size=size, mtime=mtime, ctime=ctime, version=version, + (size, mtime_ns, ctime_ns, version, last_uploaded_uri, + last_downloaded_uri, last_downloaded_timestamp) = row + return PathEntry(size=size, mtime_ns=mtime_ns, ctime_ns=ctime_ns, version=version, last_uploaded_uri=last_uploaded_uri, last_downloaded_uri=last_downloaded_uri, last_downloaded_timestamp=last_downloaded_timestamp) @@ -91,12 +93,17 @@ class MagicFolderDB(object): try: print "insert" self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?,?,?)", - (relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp)) + (relpath_u, pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns, + version, last_uploaded_uri, last_downloaded_uri, + last_downloaded_timestamp)) except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): print "err... update" self.cursor.execute("UPDATE local_files" - " SET size=?, mtime=?, ctime=?, version=?, last_uploaded_uri=?, last_downloaded_uri=?, last_downloaded_timestamp=?" + " SET size=?, mtime_ns=?, ctime_ns=?, version=?, last_uploaded_uri=?," + " last_downloaded_uri=?, last_downloaded_timestamp=?" " WHERE path=?", - (pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, relpath_u)) + (pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns, version, + last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, + relpath_u)) self.connection.commit() print "committed" diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py index f08b8d26c..4092f8f61 100644 --- a/src/allmydata/test/test_magic_folder.py +++ b/src/allmydata/test/test_magic_folder.py @@ -1,5 +1,5 @@ -import os, sys +import os, sys, time import shutil, simplejson if False: @@ -1001,16 +1001,16 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall relpath1 = u"myFile1" pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False, - exists=True, size=1, mtime=123, ctime=456) + exists=True, size=1, mtime_ns=123, ctime_ns=456) db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo) c = db.cursor - c.execute("SELECT size, mtime, ctime" + c.execute("SELECT size, mtime_ns, ctime_ns" " FROM local_files" " WHERE path=?", (relpath1,)) row = c.fetchone() - self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime)) + self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns)) # Second test uses magic_folder.is_new_file instead of SQL query directly # to confirm the previous upload entry in the db. @@ -1023,7 +1023,8 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry)) different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False, - exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime) + exists=True, size=0, mtime_ns=pathinfo.mtime_ns, + ctime_ns=pathinfo.ctime_ns) self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry)) def _test_magicfolder_start_service(self): @@ -1314,7 +1315,8 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase): fileutil.write(local_file, "foo") # if is_conflict is False, then the .conflict file shouldn't exist. - writefile._write_downloaded_file(workdir, local_file, "bar", False, None) + now = time.time() + writefile._write_downloaded_file(workdir, local_file, "bar", False, now=now) conflicted_path = local_file + u".conflict" self.failIf(os.path.exists(conflicted_path)) @@ -1326,9 +1328,15 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase): # .tmp file shouldn't exist self.failIf(os.path.exists(local_file + u".tmp")) - # .. and the original file should have the new content + # The original file should have the new content self.failUnlessEqual(fileutil.read(local_file), "bar") + # .. and approximately the correct timestamp. + pathinfo = fileutil.get_pathinfo(local_file) + error_ns = pathinfo.mtime_ns - fileutil.seconds_to_ns(now - WriteFileMixin.FUDGE_SECONDS) + permitted_error_ns = fileutil.seconds_to_ns(WriteFileMixin.FUDGE_SECONDS)/4 + self.failUnless(abs(error_ns) < permitted_error_ns, (error_ns, permitted_error_ns)) + # now a test for conflicted case writefile._write_downloaded_file(workdir, local_file, "bar", True, None) self.failUnless(os.path.exists(conflicted_path)) diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 8551796c7..f328af7b0 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -695,30 +695,33 @@ else: except EnvironmentError: reraise(ConflictError) -PathInfo = namedtuple('PathInfo', 'isdir isfile islink exists size mtime ctime') +PathInfo = namedtuple('PathInfo', 'isdir isfile islink exists size mtime_ns ctime_ns') -def get_pathinfo(path_u, now=None): +def seconds_to_ns(t): + return int(t * 1000000000) + +def get_pathinfo(path_u, now_ns=None): try: statinfo = os.lstat(path_u) mode = statinfo.st_mode - return PathInfo(isdir =stat.S_ISDIR(mode), - isfile=stat.S_ISREG(mode), - islink=stat.S_ISLNK(mode), - exists=True, - size =statinfo.st_size, - mtime =statinfo.st_mtime, - ctime =statinfo.st_ctime, + return PathInfo(isdir =stat.S_ISDIR(mode), + isfile =stat.S_ISREG(mode), + islink =stat.S_ISLNK(mode), + exists =True, + size =statinfo.st_size, + mtime_ns=seconds_to_ns(statinfo.st_mtime), + ctime_ns=seconds_to_ns(statinfo.st_ctime), ) except OSError as e: if e.errno == ENOENT: - if now is None: - now = time.time() - return PathInfo(isdir =False, - isfile=False, - islink=False, - exists=False, - size =None, - mtime =now, - ctime =now, + if now_ns is None: + now_ns = seconds_to_ns(time.time()) + return PathInfo(isdir =False, + isfile =False, + islink =False, + exists =False, + size =None, + mtime_ns=now_ns, + ctime_ns=now_ns, ) raise