Simplifications resulting from requiring Python 2.5 and therefore being able to use sqlite3 from the standard library. This also drops sqlite3 from the set of versions and paths we report.

src/allmydata/__init__.py

@@ -195,15 +195,11 @@ def get_package_versions_and_locations():
                 trace_info = (etype, str(emsg), ([None] + traceback.extract_tb(etrace))[-1])
                 packages.append( (pkgname, (None, None, trace_info)) )
             else:
-                if 'sqlite' in pkgname:
+                comment = None
-                    packages.append( (pkgname, (get_version(module, 'version'), package_dir(module.__file__),
+                if pkgname == 'setuptools' and hasattr(module, '_distribute'):
-                                               'sqlite %s' % (get_version(module, 'sqlite_version'),))) )
+                    # distribute does not report its version in any module variables
-                else:
+                    comment = 'distribute'
-                    comment = None
+                packages.append( (pkgname, (get_version(module, '__version__'), package_dir(module.__file__), comment)) )
-                    if pkgname == 'setuptools' and hasattr(module, '_distribute'):
-                        # distribute does not report its version in any module variables
-                        comment = 'distribute'
-                    packages.append( (pkgname, (get_version(module, '__version__'), package_dir(module.__file__), comment)) )
         elif pkgname == 'python':
             packages.append( (pkgname, (platform.python_version(), sys.executable, None)) )
         elif pkgname == 'platform':
@@ -278,7 +274,7 @@ def cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list):
     """This function returns a list of errors due to any failed cross-checks."""
 
     errors = []
-    not_pkg_resourceable = set(['sqlite3', 'python', 'platform', __appname__.lower()])
+    not_pkg_resourceable = set(['python', 'platform', __appname__.lower()])
     not_import_versionable = set(['zope.interface', 'mock', 'pyasn1'])
     ignorable = set(['argparse', 'pyutil', 'zbase32', 'distribute', 'twisted-web', 'twisted-core'])
 

src/allmydata/_auto_deps.py

@@ -80,17 +80,6 @@ package_imports = [
 def require_more():
     import sys
 
-    # Sqlite comes built into Python >= 2.5, and is provided by the "pysqlite"
-    # distribution for Python 2.4.
-    try:
-        import sqlite3
-        sqlite3 # hush pyflakes
-        package_imports.append(('sqlite3', 'sqlite3'))
-    except ImportError:
-        # pysqlite v2.0.5 was shipped in Ubuntu 6.06 LTS "dapper" and Nexenta NCP 1.
-        install_requires.append("pysqlite >= 2.0.5")
-        package_imports.append(('pysqlite', 'pysqlite2.dbapi2'))
-
     # Don't try to get the version number of setuptools in frozen builds, because
     # that triggers 'site' processing that causes failures. Note that frozen
     # builds still (unfortunately) import pkg_resources in .tac files, so the

src/allmydata/scripts/backupdb.py

@@ -1,11 +1,6 @@
 
-# the backupdb is only available if sqlite3 is available. Python-2.5.x and
-# beyond include sqlite3 in the standard library. For python-2.4, the
-# "pysqlite2" "package" (or "module") (which, despite the confusing name, uses
-# sqlite3, and which, confusingly, comes in the "pysqlite" "distribution" (or
-# "package")) must be installed. On debian, install python-pysqlite2
-
 import os.path, sys, time, random, stat
+
 from allmydata.util.netstring import netstring
 from allmydata.util.hashutil import backupdb_dirhash
 from allmydata.util import base32
@@ -68,19 +63,12 @@ def get_backupdb(dbfile, stderr=sys.stderr,
                  create_version=(SCHEMA_v2, 2), just_create=False):
     # open or create the given backupdb file. The parent directory must
     # exist.
-    try:
+    import sqlite3
-        import sqlite3
-        sqlite = sqlite3 # pyflakes whines about 'import sqlite3 as sqlite' ..
-    except ImportError:
-        from pysqlite2 import dbapi2
-        sqlite = dbapi2 # .. when this clause does it too
-        # This import should never fail, because setuptools requires that the
-        # "pysqlite" distribution is present at start time (if on Python < 2.5).
 
     must_create = not os.path.exists(dbfile)
     try:
-        db = sqlite.connect(dbfile)
+        db = sqlite3.connect(dbfile)
-    except (EnvironmentError, sqlite.OperationalError), e:
+    except (EnvironmentError, sqlite3.OperationalError), e:
         print >>stderr, "Unable to create/open backupdb file %s: %s" % (dbfile, e)
         return None
 
@@ -94,7 +82,7 @@ def get_backupdb(dbfile, stderr=sys.stderr,
     try:
         c.execute("SELECT version FROM version")
         version = c.fetchone()[0]
-    except sqlite.DatabaseError, e:
+    except sqlite3.DatabaseError, e:
         # this indicates that the file is not a compatible database format.
         # Perhaps it was created with an old version, or it might be junk.
         print >>stderr, "backupdb file is unusable: %s" % e
@@ -108,7 +96,7 @@ def get_backupdb(dbfile, stderr=sys.stderr,
         db.commit()
         version = 2
     if version == 2:
-        return BackupDB_v2(sqlite, db)
+        return BackupDB_v2(sqlite3, db)
     print >>stderr, "Unable to handle backupdb version %s" % version
     return None
 
@@ -263,7 +251,7 @@ class BackupDB_v2:
             c.execute("INSERT INTO caps (filecap) VALUES (?)", (filecap,))
         except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
             # sqlite3 on sid gives IntegrityError
-            # pysqlite2 on dapper gives OperationalError
+            # pysqlite2 (which we don't use, so maybe no longer relevant) on dapper gives OperationalError
             pass
         c.execute("SELECT fileid FROM caps WHERE filecap=?", (filecap,))
         foundrow = c.fetchone()

src/allmydata/test/test_backupdb.py

@@ -9,12 +9,10 @@ from allmydata.util.assertutil import precondition
 from allmydata.scripts import backupdb
 
 class BackupDB(unittest.TestCase):
-    def create_or_skip(self, dbfile):
+    def create(self, dbfile):
         stderr = StringIO()
         bdb = backupdb.get_backupdb(dbfile, stderr=stderr)
-        if not bdb:
+        self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,))
-            if "I was unable to import a python sqlite library" in stderr.getvalue():
-                raise unittest.SkipTest("sqlite unavailable, skipping test")
         return bdb
 
     def skip_if_cannot_represent_filename(self, u):
@@ -31,8 +29,7 @@ class BackupDB(unittest.TestCase):
         self.basedir = basedir = os.path.join("backupdb", "create")
         fileutil.make_dirs(basedir)
         dbfile = os.path.join(basedir, "dbfile")
-        bdb = self.create_or_skip(dbfile)
+        bdb = self.create(dbfile)
-        self.failUnless(bdb)
         self.failUnlessEqual(bdb.VERSION, 2)
 
     def test_upgrade_v1_v2(self):
@@ -43,13 +40,9 @@ class BackupDB(unittest.TestCase):
         created = backupdb.get_backupdb(dbfile, stderr=stderr,
                                         create_version=(backupdb.SCHEMA_v1, 1),
                                         just_create=True)
-        if not created:
+        self.failUnless(created, "unable to create v1 backupdb")
-            if "I was unable to import a python sqlite library" in stderr.getvalue():
-                raise unittest.SkipTest("sqlite unavailable, skipping test")
-            self.fail("unable to create v1 backupdb")
         # now we should have a v1 database on disk
-        bdb = self.create_or_skip(dbfile)
+        bdb = self.create(dbfile)
-        self.failUnless(bdb)
         self.failUnlessEqual(bdb.VERSION, 2)
 
     def test_fail(self):
@@ -65,12 +58,8 @@ class BackupDB(unittest.TestCase):
                                     stderr_f)
         self.failUnlessEqual(bdb, None)
         stderr = stderr_f.getvalue()
-        if "I was unable to import a python sqlite library" in stderr:
+        self.failUnlessIn("backupdb file is unusable", stderr)
-            pass
+        self.failUnlessIn("file is encrypted or is not a database", stderr)
-        else:
-            self.failUnless("backupdb file is unusable" in stderr, stderr)
-            self.failUnless("file is encrypted or is not a database" in stderr,
-                            stderr)
 
         # put a directory in the way, to exercise a different error path
         where = os.path.join(basedir, "roadblock-dir")
@@ -79,12 +68,8 @@ class BackupDB(unittest.TestCase):
         bdb = backupdb.get_backupdb(where, stderr_f)
         self.failUnlessEqual(bdb, None)
         stderr = stderr_f.getvalue()
-        if "I was unable to import a python sqlite library" in stderr:
+        self.failUnlessIn("Unable to create/open backupdb file %s" % (where,), stderr)
-            pass
+        self.failUnlessIn("unable to open database file", stderr)
-        else:
-            self.failUnless(("Unable to create/open backupdb file %s" % where)
-                            in stderr, stderr)
-            self.failUnless("unable to open database file" in stderr, stderr)
 
 
     def writeto(self, filename, data):
@@ -98,8 +83,7 @@ class BackupDB(unittest.TestCase):
         self.basedir = basedir = os.path.join("backupdb", "check")
         fileutil.make_dirs(basedir)
         dbfile = os.path.join(basedir, "dbfile")
-        bdb = self.create_or_skip(dbfile)
+        bdb = self.create(dbfile)
-        self.failUnless(bdb)
 
         foo_fn = self.writeto("foo.txt", "foo.txt")
         blah_fn = self.writeto("bar/blah.txt", "blah.txt")
@@ -164,7 +148,7 @@ class BackupDB(unittest.TestCase):
         fileutil.make_dirs(basedir)
 
         where = os.path.join(basedir, "tooold.db")
-        bdb = self.create_or_skip(where)
+        bdb = self.create(where)
         # reach into the DB and make it old
         bdb.cursor.execute("UPDATE version SET version=0")
         bdb.connection.commit()
@@ -182,8 +166,7 @@ class BackupDB(unittest.TestCase):
         self.basedir = basedir = os.path.join("backupdb", "directory")
         fileutil.make_dirs(basedir)
         dbfile = os.path.join(basedir, "dbfile")
-        bdb = self.create_or_skip(dbfile)
+        bdb = self.create(dbfile)
-        self.failUnless(bdb)
 
         contents = {u"file1": "URI:CHK:blah1",
                     u"file2": "URI:CHK:blah2",
@@ -245,8 +228,7 @@ class BackupDB(unittest.TestCase):
         self.basedir = basedir = os.path.join("backupdb", "unicode")
         fileutil.make_dirs(basedir)
         dbfile = os.path.join(basedir, "dbfile")
-        bdb = self.create_or_skip(dbfile)
+        bdb = self.create(dbfile)
-        self.failUnless(bdb)
 
         self.writeto(u"f\u00f6\u00f6.txt", "foo.txt")
         files = [fn for fn in listdir_unicode(unicode(basedir)) if fn.endswith(".txt")]

src/allmydata/test/test_cli.py

@@ -2461,8 +2461,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         # is the backupdb available? If so, we test that a second backup does
         # not create new directories.
         hush = StringIO()
-        have_bdb = backupdb.get_backupdb(os.path.join(self.basedir, "dbtest"),
+        bdb = backupdb.get_backupdb(os.path.join(self.basedir, "dbtest"),
-                                         hush)
+                                    hush)
+        self.failUnless(bdb)
 
         # create a small local directory with a couple of files
         source = os.path.join(self.basedir, "home")
@@ -2481,13 +2482,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
 
         d = self.do_cli("create-alias", "tahoe")
 
-        if not have_bdb:
-            d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:backups"))
-            def _should_complain((rc, out, err)):
-                self.failUnless("I was unable to import a python sqlite library" in err, err)
-            d.addCallback(_should_complain)
-            d.addCallback(self.stall, 1.1) # make sure the backups get distinct timestamps
-
         d.addCallback(lambda res: do_backup())
         def _check0((rc, out, err)):
             self.failUnlessReallyEqual(err, "")
@@ -2548,61 +2542,56 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
             # available
             self.failUnlessReallyEqual(err, "")
             self.failUnlessReallyEqual(rc, 0)
-            if have_bdb:
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
-                fu, fr, fs, dc, dr, ds = self.count_output(out)
+            # foo.txt, bar.txt, blah.txt
-                # foo.txt, bar.txt, blah.txt
+            self.failUnlessReallyEqual(fu, 0)
-                self.failUnlessReallyEqual(fu, 0)
+            self.failUnlessReallyEqual(fr, 3)
-                self.failUnlessReallyEqual(fr, 3)
+            self.failUnlessReallyEqual(fs, 0)
-                self.failUnlessReallyEqual(fs, 0)
+            # empty, home, home/parent, home/parent/subdir
-                # empty, home, home/parent, home/parent/subdir
+            self.failUnlessReallyEqual(dc, 0)
-                self.failUnlessReallyEqual(dc, 0)
+            self.failUnlessReallyEqual(dr, 4)
-                self.failUnlessReallyEqual(dr, 4)
+            self.failUnlessReallyEqual(ds, 0)
-                self.failUnlessReallyEqual(ds, 0)
         d.addCallback(_check4a)
 
-        if have_bdb:
+        # sneak into the backupdb, crank back the "last checked"
-            # sneak into the backupdb, crank back the "last checked"
+        # timestamp to force a check on all files
-            # timestamp to force a check on all files
+        def _reset_last_checked(res):
-            def _reset_last_checked(res):
+            dbfile = os.path.join(self.get_clientdir(),
-                dbfile = os.path.join(self.get_clientdir(),
+                                  "private", "backupdb.sqlite")
-                                      "private", "backupdb.sqlite")
+            self.failUnless(os.path.exists(dbfile), dbfile)
-                self.failUnless(os.path.exists(dbfile), dbfile)
+            bdb = backupdb.get_backupdb(dbfile)
-                bdb = backupdb.get_backupdb(dbfile)
+            bdb.cursor.execute("UPDATE last_upload SET last_checked=0")
-                bdb.cursor.execute("UPDATE last_upload SET last_checked=0")
+            bdb.cursor.execute("UPDATE directories SET last_checked=0")
-                bdb.cursor.execute("UPDATE directories SET last_checked=0")
+            bdb.connection.commit()
-                bdb.connection.commit()
 
-            d.addCallback(_reset_last_checked)
+        d.addCallback(_reset_last_checked)
 
-            d.addCallback(self.stall, 1.1)
+        d.addCallback(self.stall, 1.1)
-            d.addCallback(lambda res: do_backup(verbose=True))
+        d.addCallback(lambda res: do_backup(verbose=True))
-            def _check4b((rc, out, err)):
+        def _check4b((rc, out, err)):
-                # we should check all files, and re-use all of them. None of
+            # we should check all files, and re-use all of them. None of
-                # the directories should have been changed, so we should
+            # the directories should have been changed, so we should
-                # re-use all of them too.
+            # re-use all of them too.
-                self.failUnlessReallyEqual(err, "")
+            self.failUnlessReallyEqual(err, "")
-                self.failUnlessReallyEqual(rc, 0)
+            self.failUnlessReallyEqual(rc, 0)
-                fu, fr, fs, dc, dr, ds = self.count_output(out)
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
-                fchecked, dchecked = self.count_output2(out)
+            fchecked, dchecked = self.count_output2(out)
-                self.failUnlessReallyEqual(fchecked, 3)
+            self.failUnlessReallyEqual(fchecked, 3)
-                self.failUnlessReallyEqual(fu, 0)
+            self.failUnlessReallyEqual(fu, 0)
-                self.failUnlessReallyEqual(fr, 3)
+            self.failUnlessReallyEqual(fr, 3)
-                self.failUnlessReallyEqual(fs, 0)
+            self.failUnlessReallyEqual(fs, 0)
-                self.failUnlessReallyEqual(dchecked, 4)
+            self.failUnlessReallyEqual(dchecked, 4)
-                self.failUnlessReallyEqual(dc, 0)
+            self.failUnlessReallyEqual(dc, 0)
-                self.failUnlessReallyEqual(dr, 4)
+            self.failUnlessReallyEqual(dr, 4)
-                self.failUnlessReallyEqual(ds, 0)
+            self.failUnlessReallyEqual(ds, 0)
-            d.addCallback(_check4b)
+        d.addCallback(_check4b)
 
         d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
         def _check5((rc, out, err)):
             self.failUnlessReallyEqual(err, "")
             self.failUnlessReallyEqual(rc, 0)
             self.new_archives = out.split()
-            expected_new = 2
+            self.failUnlessReallyEqual(len(self.new_archives), 3, out)
-            if have_bdb:
-                expected_new += 1
-            self.failUnlessReallyEqual(len(self.new_archives), expected_new, out)
             # the original backup should still be the oldest (i.e. sorts
             # alphabetically towards the beginning)
             self.failUnlessReallyEqual(sorted(self.new_archives)[0],
@@ -2627,27 +2616,23 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
             # and upload the rest. None of the directories can be reused.
             self.failUnlessReallyEqual(err, "")
             self.failUnlessReallyEqual(rc, 0)
-            if have_bdb:
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
-                fu, fr, fs, dc, dr, ds = self.count_output(out)
+            # new foo.txt, surprise file, subfile, empty
-                # new foo.txt, surprise file, subfile, empty
+            self.failUnlessReallyEqual(fu, 4)
-                self.failUnlessReallyEqual(fu, 4)
+            # old bar.txt
-                # old bar.txt
+            self.failUnlessReallyEqual(fr, 1)
-                self.failUnlessReallyEqual(fr, 1)
+            self.failUnlessReallyEqual(fs, 0)
-                self.failUnlessReallyEqual(fs, 0)
+            # home, parent, subdir, blah.txt, surprisedir
-                # home, parent, subdir, blah.txt, surprisedir
+            self.failUnlessReallyEqual(dc, 5)
-                self.failUnlessReallyEqual(dc, 5)
+            self.failUnlessReallyEqual(dr, 0)
-                self.failUnlessReallyEqual(dr, 0)
+            self.failUnlessReallyEqual(ds, 0)
-                self.failUnlessReallyEqual(ds, 0)
         d.addCallback(_check5a)
         d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
         def _check6((rc, out, err)):
             self.failUnlessReallyEqual(err, "")
             self.failUnlessReallyEqual(rc, 0)
             self.new_archives = out.split()
-            expected_new = 3
+            self.failUnlessReallyEqual(len(self.new_archives), 4)
-            if have_bdb:
-                expected_new += 1
-            self.failUnlessReallyEqual(len(self.new_archives), expected_new)
             self.failUnlessReallyEqual(sorted(self.new_archives)[0],
                                  self.old_archives[0])
         d.addCallback(_check6)

src/allmydata/test/test_version.py

@@ -58,7 +58,7 @@ class CheckRequirement(unittest.TestCase):
         res = cross_check({}, [])
         self.failUnlessEqual(res, [])
 
-        res = cross_check({}, [("sqlite3", ("1.0", "", "blah"))])
+        res = cross_check({}, [("allmydata-tahoe", ("1.0", "", "blah"))])
         self.failUnlessEqual(res, [])
 
         res = cross_check({"foo": ("unparseable", "")}, [])