util/cachedir.py: add a cache-directory manager class, which expires+deletes unused files after a while

src/allmydata/test/test_util.py

@@ -8,7 +8,7 @@ from twisted.python import failure
 
 from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
 from allmydata.util import assertutil, fileutil, deferredutil
-from allmydata.util import limiter, time_format, pollmixin
+from allmydata.util import limiter, time_format, pollmixin, cachedir
 
 class Base32(unittest.TestCase):
     def test_b2a_matches_Pythons(self):
@@ -541,3 +541,66 @@ class TimeFormat(unittest.TestCase):
         self.failUnless("not a complete ISO8601 timestamp" in str(e))
         s = time_format.iso_utc_time_to_localseconds("1970-01-01_00:00:01.500")
         self.failUnlessEqual(s, 1.5)
+
+class CacheDir(unittest.TestCase):
+    def test_basic(self):
+        basedir = "test_util/CacheDir/test_basic"
+
+        def _failIfExists(name):
+            absfn = os.path.join(basedir, name)
+            self.failIf(os.path.exists(absfn),
+                        "%s exists but it shouldn't" % absfn)
+
+        def _failUnlessExists(name):
+            absfn = os.path.join(basedir, name)
+            self.failUnless(os.path.exists(absfn),
+                            "%s doesn't exist but it should" % absfn)
+
+        cdm = cachedir.CacheDirectoryManager(basedir)
+        a = cdm.get_file("a")
+        b = cdm.get_file("b")
+        c = cdm.get_file("c")
+        f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f
+        f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f
+        f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f
+
+        _failUnlessExists("a")
+        _failUnlessExists("b")
+        _failUnlessExists("c")
+
+        cdm.check()
+
+        _failUnlessExists("a")
+        _failUnlessExists("b")
+        _failUnlessExists("c")
+
+        del a
+        # this file won't be deleted yet, because it isn't old enough
+        cdm.check()
+        _failUnlessExists("a")
+        _failUnlessExists("b")
+        _failUnlessExists("c")
+
+        # we change the definition of "old" to make everything old
+        cdm.old = -10
+
+        cdm.check()
+        _failIfExists("a")
+        _failUnlessExists("b")
+        _failUnlessExists("c")
+
+        cdm.old = 60*60
+
+        del b
+
+        cdm.check()
+        _failIfExists("a")
+        _failUnlessExists("b")
+        _failUnlessExists("c")
+
+        b2 = cdm.get_file("b")
+
+        cdm.check()
+        _failIfExists("a")
+        _failUnlessExists("b")
+        _failUnlessExists("c")

src/allmydata/util/cachedir.py

@@ -0,0 +1,43 @@
+
+import os.path, stat, weakref, time
+from twisted.application import service, internet
+from allmydata.util import fileutil
+
+HOUR = 60*60
+
+class CacheDirectoryManager(service.MultiService):
+    def __init__(self, basedir, pollinterval=1*HOUR, old=1*HOUR):
+        service.MultiService.__init__(self)
+        self.basedir = basedir
+        fileutil.make_dirs(basedir)
+        self.old = old
+        self.files = weakref.WeakValueDictionary()
+
+        t = internet.TimerService(pollinterval, self.check)
+        t.setServiceParent(self)
+
+    def get_file(self, key):
+        assert isinstance(key, str) # used as filename
+        absfn = os.path.join(self.basedir, key)
+        if os.path.exists(absfn):
+            os.utime(absfn, None)
+        cf = CacheFile(absfn)
+        self.files[key] = cf
+        return cf
+
+    def check(self):
+        now = time.time()
+        for fn in os.listdir(self.basedir):
+            if fn in self.files:
+                continue
+            absfn = os.path.join(self.basedir, fn)
+            mtime = os.stat(absfn)[stat.ST_MTIME]
+            if now - mtime > self.old:
+                os.remove(absfn)
+
+class CacheFile:
+    def __init__(self, absfn):
+        self.filename = absfn
+
+    def get_filename(self):
+        return self.filename