ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-05-31 14:50:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

WIP: Refactoring to get db fields in one query.

Browse Source 
Signed-off-by: Daira Hopwood <daira@jacaranda.org>
This commit is contained in:
Daira Hopwood 2015-10-29 20:43:44 +00:00 committed by Brian Warner
parent bca4c87e47
commit 9d59ef493f
2 changed files with 47 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/allmydata/frontends/magic_folder.py
View File

@ -329,17 +329,15 @@ class Uploader(QueueMixin):
# FIXME merge this with the 'isfile' case. # FIXME merge this with the 'isfile' case.
self._log("notified object %s disappeared (this is normal)" % quote_filepath(fp)) self._log("notified object %s disappeared (this is normal)" % quote_filepath(fp))
self._count('objects_disappeared') self._count('objects_disappeared')
if not self._db.check_file_db_exists(relpath_u):
db_entry = self._db.get_db_entry(relpath_u)
if db_entry is None:
return None return None
last_downloaded_timestamp = now last_downloaded_timestamp = now # is this correct?
last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u)
current_version = self._db.get_local_file_version(relpath_u) if self._db.is_new_file(pathinfo, relpath_u):
if current_version is None: new_version = db_entry.version + 1
new_version = 0
elif self._db.is_new_file(pathinfo, relpath_u):
new_version = current_version + 1
else: else:
self._log("Not uploading %r" % (relpath_u,)) self._log("Not uploading %r" % (relpath_u,))
self._count('objects_not_uploaded') self._count('objects_not_uploaded')
@ -348,8 +346,8 @@ class Uploader(QueueMixin):
metadata = { 'version': new_version, metadata = { 'version': new_version,
'deleted': True, 'deleted': True,
'last_downloaded_timestamp': last_downloaded_timestamp } 'last_downloaded_timestamp': last_downloaded_timestamp }
if last_downloaded_uri is not None: if db_entry.last_downloaded_uri is not None:
metadata['last_downloaded_uri'] = last_downloaded_uri metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri
empty_uploadable = Data("", self._client.convergence) empty_uploadable = Data("", self._client.convergence)
d2 = self._upload_dirnode.add_file(encoded_path_u, empty_uploadable, d2 = self._upload_dirnode.add_file(encoded_path_u, empty_uploadable,
@ -357,8 +355,10 @@ class Uploader(QueueMixin):
def _add_db_entry(filenode): def _add_db_entry(filenode):
filecap = filenode.get_uri() filecap = filenode.get_uri()
last_downloaded_uri = metadata.get('last_downloaded_uri', None)
self._db.did_upload_version(relpath_u, new_version, filecap, self._db.did_upload_version(relpath_u, new_version, filecap,
last_downloaded_uri, last_downloaded_timestamp, pathinfo) last_downloaded_uri, last_downloaded_timestamp,
pathinfo)
self._count('files_uploaded') self._count('files_uploaded')
d2.addCallback(_add_db_entry) d2.addCallback(_add_db_entry)
return d2 return d2
@ -383,14 +383,14 @@ class Uploader(QueueMixin):
upload_d.addCallback(lambda ign: self._scan(relpath_u)) upload_d.addCallback(lambda ign: self._scan(relpath_u))
return upload_d return upload_d
elif pathinfo.isfile: elif pathinfo.isfile:
last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u) db_entry = self._db.get_db_entry(relpath_u)
last_downloaded_timestamp = now last_downloaded_timestamp = now
current_version = self._db.get_local_file_version(relpath_u) if db_entry is None:
if current_version is None:
new_version = 0 new_version = 0
elif self._db.is_new_file(pathinfo, relpath_u): elif self._db.is_new_file(pathinfo, relpath_u):
new_version = current_version + 1 new_version = db_entry.version + 1
else: else:
self._log("Not uploading %r" % (relpath_u,)) self._log("Not uploading %r" % (relpath_u,))
self._count('objects_not_uploaded') self._count('objects_not_uploaded')
@ -398,8 +398,8 @@ class Uploader(QueueMixin):
metadata = { 'version': new_version, metadata = { 'version': new_version,
'last_downloaded_timestamp': last_downloaded_timestamp } 'last_downloaded_timestamp': last_downloaded_timestamp }
if last_downloaded_uri is not None: if db_entry is not None and db_entry.last_downloaded_uri is not None:
metadata['last_downloaded_uri'] = last_downloaded_uri metadata['last_downloaded_uri'] = db_entry.last_downloaded_uri
uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) uploadable = FileName(unicode_from_filepath(fp), self._client.convergence)
d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable,
@ -409,7 +409,8 @@ class Uploader(QueueMixin):
filecap = filenode.get_uri() filecap = filenode.get_uri()
last_downloaded_uri = metadata.get('last_downloaded_uri', None) last_downloaded_uri = metadata.get('last_downloaded_uri', None)
self._db.did_upload_version(relpath_u, new_version, filecap, self._db.did_upload_version(relpath_u, new_version, filecap,
last_downloaded_uri, last_downloaded_timestamp, pathinfo) last_downloaded_uri, last_downloaded_timestamp,
pathinfo)
self._count('files_uploaded') self._count('files_uploaded')
d2.addCallback(_add_db_entry) d2.addCallback(_add_db_entry)
return d2 return d2
@ -557,9 +558,11 @@ class Downloader(QueueMixin, WriteFileMixin):
self._log("nope") self._log("nope")
return False return False
self._log("yep") self._log("yep")
v = self._db.get_local_file_version(relpath_u) db_entry = self._db.get_db_entry(relpath_u)
self._log("v = %r" % (v,)) if db_entry is None:
return (v is None or v < remote_version) return True
self._log("version %r" % (db_entry.version,))
return (db_entry.version < remote_version)
def _get_local_latest(self, relpath_u): def _get_local_latest(self, relpath_u):
""" """
@ -569,7 +572,8 @@ class Downloader(QueueMixin, WriteFileMixin):
""" """
if not self._get_filepath(relpath_u).exists(): if not self._get_filepath(relpath_u).exists():
return None return None
return self._db.get_local_file_version(relpath_u) db_entry = self._db.get_db_entry(relpath_u)
return None if db_entry is None else db_entry.version
def _get_collective_latest_file(self, filename): def _get_collective_latest_file(self, filename):
""" """
@ -704,20 +708,19 @@ class Downloader(QueueMixin, WriteFileMixin):
d.addCallback(fail) d.addCallback(fail)
else: else:
is_conflict = False is_conflict = False
if self._db.check_file_db_exists(relpath_u): db_entry = self._db.get_db_entry(relpath_u)
if db_entry:
dmd_last_downloaded_uri = metadata.get('last_downloaded_uri', None) dmd_last_downloaded_uri = metadata.get('last_downloaded_uri', None)
local_last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u)
print "metadata %r" % (metadata,) print "metadata %r" % (metadata,)
print "<<<<--- if %r != %r" % (dmd_last_downloaded_uri, local_last_downloaded_uri) print "<<<<--- if %r != %r" % (dmd_last_downloaded_uri, db_entry.last_downloaded_uri)
if dmd_last_downloaded_uri is not None and local_last_downloaded_uri is not None: if dmd_last_downloaded_uri is not None and db_entry.last_downloaded_uri is not None:
if dmd_last_downloaded_uri != local_last_downloaded_uri: if dmd_last_downloaded_uri != db_entry.last_downloaded_uri:
is_conflict = True is_conflict = True
self._count('objects_conflicted') self._count('objects_conflicted')
else: else:
dmd_last_uploaded_uri = metadata.get('last_uploaded_uri', None) dmd_last_uploaded_uri = metadata.get('last_uploaded_uri', None)
local_last_uploaded_uri = self._db.get_last_uploaded_uri(relpath_u) print ">>>> if %r != %r" % (dmd_last_uploaded_uri, db_entry.last_uploaded_uri)
print ">>>> if %r != %r" % (dmd_last_uploaded_uri, local_last_uploaded_uri) if dmd_last_uploaded_uri is not None and dmd_last_uploaded_uri != db_entry.last_uploaded_uri:
if dmd_last_uploaded_uri is not None and dmd_last_uploaded_uri != local_last_uploaded_uri:
is_conflict = True is_conflict = True
self._count('objects_conflicted') self._count('objects_conflicted')
else: else:

69
src/allmydata/magicfolderdb.py
View File

@ -1,5 +1,6 @@
import sys import sys
from collections import namedtuple
from allmydata.util.dbutil import get_db, DBError from allmydata.util.dbutil import get_db, DBError
@ -42,6 +43,7 @@ def get_magicfolderdb(dbfile, stderr=sys.stderr,
print >>stderr, e print >>stderr, e
return None return None
PathEntry = namedtuple('PathEntry', 'size mtime ctime version last_uploaded_uri last_downloaded_uri last_downloaded_timestamp')
class MagicFolderDB(object): class MagicFolderDB(object):
VERSION = 1 VERSION = 1
@ -51,20 +53,25 @@ class MagicFolderDB(object):
self.connection = connection self.connection = connection
self.cursor = connection.cursor() self.cursor = connection.cursor()
def check_file_db_exists(self, path): def get_db_entry(self, relpath_u):
"""I will tell you if a given file has an entry in my database or not """
by returning True or False. Retrieve the entry in the database for a given path, or return None
if there is no such entry.
""" """
c = self.cursor c = self.cursor
c.execute("SELECT size,mtime,ctime" c.execute("SELECT size, mtime, ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp"
" FROM local_files" " FROM local_files"
" WHERE path=?", " WHERE path=?",
(path,)) (relpath_u,))
row = self.cursor.fetchone() row = self.cursor.fetchone()
if not row: if not row:
return False return None
else: else:
return True (size, mtime, ctime, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp) = row
return PathEntry(size=size, mtime=mtime, ctime=ctime, version=version,
last_uploaded_uri=last_uploaded_uri,
last_downloaded_uri=last_downloaded_uri,
last_downloaded_timestamp=last_downloaded_timestamp)
def get_all_relpaths(self): def get_all_relpaths(self):
""" """
@ -75,54 +82,6 @@ class MagicFolderDB(object):
rows = self.cursor.fetchall() rows = self.cursor.fetchall()
return set([r[0] for r in rows]) return set([r[0] for r in rows])
def get_last_downloaded_uri(self, relpath_u):
"""
Return the last downloaded uri recorded in the magic folder db.
If none are found then return None.
"""
c = self.cursor
c.execute("SELECT last_downloaded_uri"
" FROM local_files"
" WHERE path=?",
(relpath_u,))
row = self.cursor.fetchone()
if not row:
return None
else:
return row[0]
def get_last_uploaded_uri(self, relpath_u):
"""
Return the last downloaded uri recorded in the magic folder db.
If none are found then return None.
"""
c = self.cursor
c.execute("SELECT last_uploaded_uri"
" FROM local_files"
" WHERE path=?",
(relpath_u,))
row = self.cursor.fetchone()
if not row:
return None
else:
return row[0]
def get_local_file_version(self, relpath_u):
"""
Return the version of a local file tracked by our magic folder db.
If no db entry is found then return None.
"""
c = self.cursor
c.execute("SELECT version"
" FROM local_files"
" WHERE path=?",
(relpath_u,))
row = self.cursor.fetchone()
if not row:
return None
else:
return row[0]
def did_upload_version(self, relpath_u, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, pathinfo): def did_upload_version(self, relpath_u, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, pathinfo):
print "%r.did_upload_version(%r, %r, %r, %r, %r, %r)" % (self, relpath_u, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, pathinfo) print "%r.did_upload_version(%r, %r, %r, %r, %r, %r)" % (self, relpath_u, version, last_uploaded_uri, last_downloaded_uri, last_downloaded_timestamp, pathinfo)
try: try: