improve run-deprecations script

Rewrote in Twisted, which lets us read/scan/print all log lines in
realtime. The output is now correctly interleaved (as well as
maintaining the stdout-vs-stderr of each message). The renamed
--warnings= logfile records all relevant lines from *both* stdout and
stderr (i.e. any that includes "DeprecationWarning"), which handles a
change (perhaps in recent Twisteds?) that emits these warnings on stdout
instead of stderr.

misc/build_helpers/run-deprecations.py

@@ -1,4 +1,5 @@
-import sys, os, subprocess
+import sys, os, io
+from twisted.internet import reactor, protocol, task, defer
 from twisted.python.procutils import which
 from twisted.python import usage
 
@@ -8,7 +9,7 @@ from twisted.python import usage
 
 class Options(usage.Options):
     optParameters = [
-        ["stderr", None, None, "file to write stderr into at end of test run"],
+        ["warnings", None, None, "file to write warnings into at end of test run"],
         ]
 
     def parseArgs(self, command, *args):
@@ -16,13 +17,26 @@ class Options(usage.Options):
         self["args"] = list(args)
 
     description = """Run as:
-PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--stderr=STDERRFILE]  COMMAND ARGS..
+PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--warnings=STDERRFILE]  COMMAND ARGS..
 """
 
+class RunPP(protocol.ProcessProtocol):
+    def outReceived(self, data):
+        self.stdout.write(data)
+        sys.stdout.write(data)
+    def errReceived(self, data):
+        self.stderr.write(data)
+        sys.stderr.write(data)
+    def processEnded(self, reason):
+        signal = reason.value.signal
+        rc = reason.value.exitCode
+        self.d.callback((signal, rc))
+
+@defer.inlineCallbacks
+def run_command(main):
     config = Options()
     config.parseOptions()
 
-
     command = config["command"]
     if "/" in command:
         # don't search
@@ -38,32 +52,38 @@ pw = os.environ.get("PYTHONWARNINGS")
     DDW = "default::DeprecationWarning"
     if pw != DDW:
         print "note: $PYTHONWARNINGS is '%s', not the expected %s" % (pw, DDW)
-
-print "note: stderr is being captured, and will be emitted at the end"
         sys.stdout.flush()
 
-# stdout goes directly to the parent, so test progress can be watched in real
-# time. But subprocess.Popen() doesn't give us any good way of seeing it
-p = subprocess.Popen([exe] + config["args"], stderr=subprocess.PIPE)
-stderr = p.communicate()[1]
-rc = p.returncode
-count = 0
+    pp = RunPP()
+    pp.d = defer.Deferred()
+    pp.stdout = io.BytesIO()
+    pp.stderr = io.BytesIO()
+    reactor.spawnProcess(pp, exe, [exe] + config["args"], env=None)
+    (signal, rc) = yield pp.d
 
-if config["stderr"]:
-    with open(config["stderr"], "wb") as f:
-        print >>f, stderr,
+    warnings = []
 
-if stderr:
-    print >>sys.stderr, "--"
-    print >>sys.stderr, "Captured stderr follows:"
-    for line in stderr.splitlines():
+    pp.stdout.seek(0)
+    for line in pp.stdout.readlines():
         if "DeprecationWarning" in line:
-            count += 1
-        print >>sys.stderr, line
-    print >>sys.stderr, "--"
+            warnings.append(line) # includes newline
 
-if count:
-    print "ERROR: %d deprecation warnings found" % count
+    pp.stderr.seek(0)
+    for line in pp.stderr.readlines():
+        if "DeprecationWarning" in line:
+            warnings.append(line)
+
+    if warnings:
+        if config["warnings"]:
+            with open(config["warnings"], "wb") as f:
+                print >>f, "".join(warnings)
+        print "ERROR: %d deprecation warnings found" % len(warnings)
         sys.exit(1)
+
     print "no deprecation warnings"
+    if signal:
+        sys.exit(signal)
     sys.exit(rc)
+
+
+task.react(run_command)

tox.ini

@@ -18,7 +18,7 @@ passenv = USERPROFILE HOMEDRIVE HOMEPATH
 setenv =
          PYTHONWARNINGS=default::DeprecationWarning
 commands =
-         python misc/build_helpers/run-deprecations.py --stderr=_trial_temp/stderr.log trial --rterrors {posargs:allmydata}
+         python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial --rterrors {posargs:allmydata}
 
 [testenv:upcoming-deprecations]
 basepython=python2.7
@@ -29,7 +29,7 @@ deps =
      git+https://github.com/twisted/twisted
      git+https://github.com/warner/foolscap
 commands =
-         python misc/build_helpers/run-deprecations.py --stderr=_trial_temp/stderr.log trial --rterrors {posargs:allmydata}
+         python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial --rterrors {posargs:allmydata}
 
 [testenv:checkmemory]
 commands =