ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-12 07:52:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Change type of mtime and ctime stored in magicfolderdb to integer nanoseconds.

Browse Source 
Signed-off-by: Daira Hopwood <daira@jacaranda.org>
This commit is contained in:
Daira Hopwood 2016-03-29 17:03:24 +01:00 committed by Brian Warner
parent 7c85de813f
commit b949082f8e
4 changed files with 64 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/allmydata/frontends/magic_folder.py
View File

@ -47,18 +47,20 @@ def get_inotify_module():
def is_new_file(pathinfo, db_entry): def is_new_file(pathinfo, db_entry):
if db_entry is None: if db_entry is None:
print "is_new_file: True because db_entry is None"
return True return True
if not pathinfo.exists and db_entry.size is None: if not pathinfo.exists and db_entry.size is None:
print("NOT because", pathinfo.exists, db_entry.size) print("is_new_file: False because", pathinfo.exists, db_entry.size)
return False return False
print("NOT because", pathinfo.size, pathinfo.ctime, pathinfo.mtime, result = ((pathinfo.size, pathinfo.ctime_ns, pathinfo.mtime_ns) !=
db_entry.size, db_entry.ctime, db_entry.mtime, (db_entry.size, db_entry.ctime_ns, db_entry.mtime_ns))
((pathinfo.size, pathinfo.ctime, pathinfo.mtime) !=
(db_entry.size, db_entry.ctime, db_entry.mtime))) print("is_new_file:", result, "because",
return ((pathinfo.size, pathinfo.ctime, pathinfo.mtime) != pathinfo.size, pathinfo.ctime_ns, pathinfo.mtime_ns,
(db_entry.size, db_entry.ctime, db_entry.mtime)) db_entry.size, db_entry.ctime_ns, db_entry.mtime_ns)
return result
class MagicFolder(service.MultiService): class MagicFolder(service.MultiService):

27
src/allmydata/magicfolderdb.py
View File

@ -15,10 +15,9 @@ CREATE TABLE version
CREATE TABLE local_files CREATE TABLE local_files
( (
path VARCHAR(1024) PRIMARY KEY, -- UTF-8 filename relative to local magic folder dir path VARCHAR(1024) PRIMARY KEY, -- UTF-8 filename relative to local magic folder dir
-- note that size is before mtime and ctime here, but after in function parameters
size INTEGER, -- ST_SIZE, or NULL if the file has been deleted size INTEGER, -- ST_SIZE, or NULL if the file has been deleted
mtime NUMBER, -- ST_MTIME mtime_ns INTEGER, -- ST_MTIME in nanoseconds
ctime NUMBER, -- ST_CTIME ctime_ns INTEGER, -- ST_CTIME in nanoseconds
version INTEGER, version INTEGER,
last_uploaded_uri VARCHAR(256), -- URI:CHK:... last_uploaded_uri VARCHAR(256), -- URI:CHK:...
last_downloaded_uri VARCHAR(256), -- URI:CHK:... last_downloaded_uri VARCHAR(256), -- URI:CHK:...
@ -43,7 +42,8 @@ def get_magicfolderdb(dbfile, stderr=sys.stderr,
print >>stderr, e print >>stderr, e
return None return None
PathEntry = namedtuple('PathEntry', 'size mtime ctime version last_uploaded_uri last_downloaded_uri last_downloaded_timestamp') PathEntry = namedtuple('PathEntry', 'size mtime_ns ctime_ns version last_uploaded_uri'
'last_downloaded_uri last_downloaded_timestamp')
class MagicFolderDB(object): class MagicFolderDB(object):
VERSION = 1 VERSION = 1
@ -62,7 +62,8 @@ class MagicFolderDB(object):
if there is no such entry. if there is no such entry.
""" """
c = self.cursor c = self.cursor
c.execute("SELECT size, mtime, ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp" c.execute("SELECT size, mtime_ns, ctime_ns, version, last_uploaded_uri,"
" last_downloaded_uri, last_downloaded_timestamp"
" FROM local_files" " FROM local_files"
" WHERE path=?", " WHERE path=?",
(relpath_u,)) (relpath_u,))
@ -71,8 +72,9 @@ class MagicFolderDB(object):
print "no dbentry for %r" % (relpath_u,) print "no dbentry for %r" % (relpath_u,)
return None return None
else: else:
(size, mtime, ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp) = row (size, mtime_ns, ctime_ns, version, last_uploaded_uri,
return PathEntry(size=size, mtime=mtime, ctime=ctime, version=version, last_downloaded_uri, last_downloaded_timestamp) = row
return PathEntry(size=size, mtime_ns=mtime_ns, ctime_ns=ctime_ns, version=version,
last_uploaded_uri=last_uploaded_uri, last_uploaded_uri=last_uploaded_uri,
last_downloaded_uri=last_downloaded_uri, last_downloaded_uri=last_downloaded_uri,
last_downloaded_timestamp=last_downloaded_timestamp) last_downloaded_timestamp=last_downloaded_timestamp)
@ -91,12 +93,17 @@ class MagicFolderDB(object):
try: try:
print "insert" print "insert"
self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?,?,?)", self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?,?,?)",
(relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp)) (relpath_u, pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns,
version, last_uploaded_uri, last_downloaded_uri,
last_downloaded_timestamp))
except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
print "err... update" print "err... update"
self.cursor.execute("UPDATE local_files" self.cursor.execute("UPDATE local_files"
" SET size=?, mtime=?, ctime=?, version=?, last_uploaded_uri=?, last_downloaded_uri=?, last_downloaded_timestamp=?" " SET size=?, mtime_ns=?, ctime_ns=?, version=?, last_uploaded_uri=?,"
" last_downloaded_uri=?, last_downloaded_timestamp=?"
" WHERE path=?", " WHERE path=?",
(pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, relpath_u)) (pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns, version,
last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp,
relpath_u))
self.connection.commit() self.connection.commit()
print "committed" print "committed"

22
src/allmydata/test/test_magic_folder.py
View File

@ -1,5 +1,5 @@
import os, sys import os, sys, time
import shutil, simplejson import shutil, simplejson
if False: if False:
@ -1001,16 +1001,16 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
relpath1 = u"myFile1" relpath1 = u"myFile1"
pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False, pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
exists=True, size=1, mtime=123, ctime=456) exists=True, size=1, mtime_ns=123, ctime_ns=456)
db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo) db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
c = db.cursor c = db.cursor
c.execute("SELECT size, mtime, ctime" c.execute("SELECT size, mtime_ns, ctime_ns"
" FROM local_files" " FROM local_files"
" WHERE path=?", " WHERE path=?",
(relpath1,)) (relpath1,))
row = c.fetchone() row = c.fetchone()
self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime)) self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime_ns, pathinfo.ctime_ns))
# Second test uses magic_folder.is_new_file instead of SQL query directly # Second test uses magic_folder.is_new_file instead of SQL query directly
# to confirm the previous upload entry in the db. # to confirm the previous upload entry in the db.
@ -1023,7 +1023,8 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry)) self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False, different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime) exists=True, size=0, mtime_ns=pathinfo.mtime_ns,
ctime_ns=pathinfo.ctime_ns)
self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry)) self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
def _test_magicfolder_start_service(self): def _test_magicfolder_start_service(self):
@ -1314,7 +1315,8 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
fileutil.write(local_file, "foo") fileutil.write(local_file, "foo")
# if is_conflict is False, then the .conflict file shouldn't exist. # if is_conflict is False, then the .conflict file shouldn't exist.
writefile._write_downloaded_file(workdir, local_file, "bar", False, None) now = time.time()
writefile._write_downloaded_file(workdir, local_file, "bar", False, now=now)
conflicted_path = local_file + u".conflict" conflicted_path = local_file + u".conflict"
self.failIf(os.path.exists(conflicted_path)) self.failIf(os.path.exists(conflicted_path))
@ -1326,9 +1328,15 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
# .tmp file shouldn't exist # .tmp file shouldn't exist
self.failIf(os.path.exists(local_file + u".tmp")) self.failIf(os.path.exists(local_file + u".tmp"))
# .. and the original file should have the new content # The original file should have the new content
self.failUnlessEqual(fileutil.read(local_file), "bar") self.failUnlessEqual(fileutil.read(local_file), "bar")
# .. and approximately the correct timestamp.
pathinfo = fileutil.get_pathinfo(local_file)
error_ns = pathinfo.mtime_ns - fileutil.seconds_to_ns(now - WriteFileMixin.FUDGE_SECONDS)
permitted_error_ns = fileutil.seconds_to_ns(WriteFileMixin.FUDGE_SECONDS)/4
self.failUnless(abs(error_ns) < permitted_error_ns, (error_ns, permitted_error_ns))
# now a test for conflicted case # now a test for conflicted case
writefile._write_downloaded_file(workdir, local_file, "bar", True, None) writefile._write_downloaded_file(workdir, local_file, "bar", True, None)
self.failUnless(os.path.exists(conflicted_path)) self.failUnless(os.path.exists(conflicted_path))

31
src/allmydata/util/fileutil.py
View File

@ -695,30 +695,33 @@ else:
except EnvironmentError: except EnvironmentError:
reraise(ConflictError) reraise(ConflictError)
PathInfo = namedtuple('PathInfo', 'isdir isfile islink exists size mtime ctime') PathInfo = namedtuple('PathInfo', 'isdir isfile islink exists size mtime_ns ctime_ns')
def get_pathinfo(path_u, now=None): def seconds_to_ns(t):
return int(t * 1000000000)
def get_pathinfo(path_u, now_ns=None):
try: try:
statinfo = os.lstat(path_u) statinfo = os.lstat(path_u)
mode = statinfo.st_mode mode = statinfo.st_mode
return PathInfo(isdir =stat.S_ISDIR(mode), return PathInfo(isdir =stat.S_ISDIR(mode),
isfile=stat.S_ISREG(mode), isfile =stat.S_ISREG(mode),
islink=stat.S_ISLNK(mode), islink =stat.S_ISLNK(mode),
exists=True, exists =True,
size =statinfo.st_size, size =statinfo.st_size,
mtime =statinfo.st_mtime, mtime_ns=seconds_to_ns(statinfo.st_mtime),
ctime =statinfo.st_ctime, ctime_ns=seconds_to_ns(statinfo.st_ctime),
) )
except OSError as e: except OSError as e:
if e.errno == ENOENT: if e.errno == ENOENT:
if now is None: if now_ns is None:
now = time.time() now_ns = seconds_to_ns(time.time())
return PathInfo(isdir =False, return PathInfo(isdir =False,
isfile=False, isfile =False,
islink=False, islink =False,
exists=False, exists =False,
size =None, size =None,
mtime =now, mtime_ns=now_ns,
ctime =now, ctime_ns=now_ns,
) )
raise raise