tests: add support for figleaf code-coverage gathering

.darcs-boringfile

@@ -46,3 +46,6 @@
 ^(dapper|sid)/debian/files$
 ^build($|/)
 ^build-stamp$
+^\.figleaf$
+^coverage-html($|/)
+^twisted/plugins/dropin\.cache$

Makefile

@@ -15,6 +15,11 @@ run-client3:
 test:
 	trial allmydata
 
+test-figleaf:
+	trial --reporter=bwverbose-figleaf allmydata
+	figleaf2html -d coverage-html -x allmydata/test/figleaf.excludes
+# after doing test-figleaf, point your browser at coverage-html/index.html
+
 create_dirs:
 	mkdir -p queen-basedir
 	mkdir -p client-basedir

allmydata/test/figleaf.excludes

@@ -0,0 +1,2 @@
+^/home/warner/stuff/python/twisted/Twisted/
+^/var/lib

allmydata/test/trial_figleaf.py

@@ -0,0 +1,127 @@
+
+"""A Trial IReporter plugin that gathers figleaf code-coverage information.
+
+Once this plugin is installed, trial can be invoked with one of two new
+--reporter options:
+
+  trial --reporter=verbose-figleaf ARGS
+  trial --reporter-bwverbose-figleaf ARGS
+
+Once such a test run has finished, there will be a .figleaf file in the
+top-level directory. This file can be turned into a directory of .html files
+(with index.html as the starting point) by running:
+
+ figleaf2html -d OUTPUTDIR [-x EXCLUDEFILE]
+
+Figleaf thinks of everyting in terms of absolute filenames rather than
+modules. The EXCLUDEFILE may be necessary to keep it from providing reports
+on non-Code-Under-Test files that live in unusual locations. In particular,
+if you use extra PYTHONPATH arguments to point at some alternate version of
+an upstream library (like Twisted), or if something like debian's
+python-support puts symlinks to .py files in sys.path but not the .py files
+themselves, figleaf will present coverage information on both of these. The
+EXCLUDEFILE option might help to inhibit these.
+
+Other figleaf problems:
+
+ the annotated code files are written to BASENAME(file).html, which results
+ in collisions between similarly-named source files.
+
+ The line-wise coverage information isn't quite right. Blank lines are
+ counted as unreached code, lambdas aren't quite right, and some multiline
+ comments (docstrings?) aren't quite right.
+
+"""
+
+# TODO: pull some of figleaf into our tree so we can customize it more
+# easily.
+
+from twisted.trial.reporter import TreeReporter, VerboseTextReporter
+
+# These plugins are registered via twisted/plugins/allmydata_trial.py . See
+# the notes there for an explanation of how that works.
+
+
+
+# Reporters don't really get told about the suite starting and stopping.
+
+# The Reporter class is imported before the test classes are.
+
+# The test classes are imported before the Reporter is created. To get
+# control earlier than that requires modifying twisted/scripts/trial.py .
+
+# Then Reporter.__init__ is called.
+
+# Then tests run, calling things like write() and addSuccess(). Each test is
+# framed by a startTest/stopTest call.
+
+# Then the results are emitted, calling things like printErrors,
+# printSummary, and wasSuccessful.
+
+# So for code-coverage (not including import), start in __init__ and finish
+# in printSummary. To include import, we have to start in our own import and
+# finish in printSummary.
+
+import figleaf
+figleaf.start()
+
+class FigleafReporter(TreeReporter):
+    def __init__(self, *args, **kwargs):
+        TreeReporter.__init__(self, *args, **kwargs)
+
+    def printSummary(self):
+        figleaf.stop()
+        figleaf.write_coverage(".figleaf")
+        print "Figleaf results written to .figleaf"
+        return TreeReporter.printSummary(self)
+
+class FigleafTextReporter(VerboseTextReporter):
+    def __init__(self, *args, **kwargs):
+        VerboseTextReporter.__init__(self, *args, **kwargs)
+
+    def printSummary(self):
+        figleaf.stop()
+        figleaf.write_coverage(".figleaf")
+        print "Figleaf results written to .figleaf"
+        return VerboseTextReporter.printSummary(self)
+
+class not_FigleafReporter(object):
+    # this class, used as a reporter on a fully-passing test suite, doesn't
+    # trigger exceptions. So it is a guide to what methods are invoked on a
+    # Reporter.
+    def __init__(self, *args, **kwargs):
+        print "FIGLEAF HERE"
+        self.r = TreeReporter(*args, **kwargs)
+        self.shouldStop = self.r.shouldStop
+        self.separator = self.r.separator
+        self.testsRun = self.r.testsRun
+        self._starting2 = False
+
+    def write(self, *args):
+        if not self._starting2:
+            self._starting2 = True
+            print "FIRST WRITE"
+        return self.r.write(*args)
+
+    def startTest(self, *args, **kwargs):
+        return self.r.startTest(*args, **kwargs)
+
+    def stopTest(self, *args, **kwargs):
+        return self.r.stopTest(*args, **kwargs)
+
+    def addSuccess(self, *args, **kwargs):
+        return self.r.addSuccess(*args, **kwargs)
+
+    def printErrors(self, *args, **kwargs):
+        return self.r.printErrors(*args, **kwargs)
+
+    def writeln(self, *args, **kwargs):
+        return self.r.writeln(*args, **kwargs)
+
+    def printSummary(self, *args, **kwargs):
+        print "PRINT SUMMARY"
+        return self.r.printSummary(*args, **kwargs)
+
+    def wasSuccessful(self, *args, **kwargs):
+        return self.r.wasSuccessful(*args, **kwargs)
+

twisted/plugins/allmydata_trial.py

@@ -0,0 +1,48 @@
+#! /usr/bin/python
+
+from zope.interface import implements
+from twisted.trial.itrial import IReporter
+from twisted.plugin import IPlugin
+
+# register a plugin that can create our FigleafReporter. The reporter itself
+# lives in a separate place
+
+# note that this .py file is *not* in a package: there is no __init__.py in
+# our parent directory. This is important, because otherwise ours would fight
+# with Twisted's. When trial looks for plugins, it merely executes all the
+# *.py files it finds in and twisted/plugins/ subdirectories of anything on
+# sys.path . The namespace that results from executing these .py files is
+# examined for instances which provide both IPlugin and the target interface
+# (in this case, trial is looking for IReporter instances). Each such
+# instance tells the application how to create a plugin by naming the module
+# and class that should be instantiated.
+
+# When installing our package via setup.py, arrange for this file to be
+# installed to the system-wide twisted/plugins/ directory.
+
+class _Reporter(object):
+    implements(IPlugin, IReporter)
+
+    def __init__(self, name, module, description, longOpt, shortOpt, klass):
+        self.name = name
+        self.module = module
+        self.description = description
+        self.longOpt = longOpt
+        self.shortOpt = shortOpt
+        self.klass = klass
+
+
+fig = _Reporter("Figleaf Code-Coverage Reporter",
+                "allmydata.test.trial_figleaf",
+                description="verbose color output (with figleaf coverage)",
+                longOpt="verbose-figleaf",
+                shortOpt="f",
+                klass="FigleafReporter")
+
+bwfig = _Reporter("Figleaf Code-Coverage Reporter (colorless)",
+                  "allmydata.test.trial_figleaf",
+                  description="Colorless verbose output (with figleaf coverage)",
+                  longOpt="bwverbose-figleaf",
+                  shortOpt=None,
+                  klass="FigleafTextReporter")
+