improve run-deprecations script

Rewrote in Twisted, which lets us read/scan/print all log lines in
realtime. The output is now correctly interleaved (as well as
maintaining the stdout-vs-stderr of each message). The renamed
--warnings= logfile records all relevant lines from *both* stdout and
stderr (i.e. any that includes "DeprecationWarning"), which handles a
change (perhaps in recent Twisteds?) that emits these warnings on stdout
instead of stderr.

misc/build_helpers/run-deprecations.py

@@ -1,4 +1,5 @@
-import sys, os, subprocess
+import sys, os, io
+from twisted.internet import reactor, protocol, task, defer
 from twisted.python.procutils import which
 from twisted.python import usage
 
@@ -8,7 +9,7 @@ from twisted.python import usage
 
 class Options(usage.Options):
     optParameters = [
-        ["stderr", None, None, "file to write stderr into at end of test run"],
+        ["warnings", None, None, "file to write warnings into at end of test run"],
         ]
 
     def parseArgs(self, command, *args):
@@ -16,54 +17,73 @@ class Options(usage.Options):
         self["args"] = list(args)
 
     description = """Run as:
-PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--stderr=STDERRFILE]  COMMAND ARGS..
+PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--warnings=STDERRFILE]  COMMAND ARGS..
 """
 
-config = Options()
+class RunPP(protocol.ProcessProtocol):
-config.parseOptions()
+    def outReceived(self, data):
+        self.stdout.write(data)
+        sys.stdout.write(data)
+    def errReceived(self, data):
+        self.stderr.write(data)
+        sys.stderr.write(data)
+    def processEnded(self, reason):
+        signal = reason.value.signal
+        rc = reason.value.exitCode
+        self.d.callback((signal, rc))
 
+@defer.inlineCallbacks
+def run_command(main):
+    config = Options()
+    config.parseOptions()
 
-command = config["command"]
+    command = config["command"]
-if "/" in command:
+    if "/" in command:
-    # don't search
+        # don't search
-    exe = command
+        exe = command
-else:
+    else:
-    executables = which(command)
+        executables = which(command)
-    if not executables:
+        if not executables:
-        raise ValueError("unable to find '%s' in PATH (%s)" %
+            raise ValueError("unable to find '%s' in PATH (%s)" %
-                         (command, os.environ.get("PATH")))
+                             (command, os.environ.get("PATH")))
-    exe = executables[0]
+        exe = executables[0]
 
-pw = os.environ.get("PYTHONWARNINGS")
+    pw = os.environ.get("PYTHONWARNINGS")
-DDW = "default::DeprecationWarning"
+    DDW = "default::DeprecationWarning"
-if pw != DDW:
+    if pw != DDW:
-    print "note: $PYTHONWARNINGS is '%s', not the expected %s" % (pw, DDW)
+        print "note: $PYTHONWARNINGS is '%s', not the expected %s" % (pw, DDW)
+        sys.stdout.flush()
 
-print "note: stderr is being captured, and will be emitted at the end"
+    pp = RunPP()
-sys.stdout.flush()
+    pp.d = defer.Deferred()
+    pp.stdout = io.BytesIO()
+    pp.stderr = io.BytesIO()
+    reactor.spawnProcess(pp, exe, [exe] + config["args"], env=None)
+    (signal, rc) = yield pp.d
 
-# stdout goes directly to the parent, so test progress can be watched in real
+    warnings = []
-# time. But subprocess.Popen() doesn't give us any good way of seeing it
-p = subprocess.Popen([exe] + config["args"], stderr=subprocess.PIPE)
-stderr = p.communicate()[1]
-rc = p.returncode
-count = 0
 
-if config["stderr"]:
+    pp.stdout.seek(0)
-    with open(config["stderr"], "wb") as f:
+    for line in pp.stdout.readlines():
-        print >>f, stderr,
-
-if stderr:
-    print >>sys.stderr, "--"
-    print >>sys.stderr, "Captured stderr follows:"
-    for line in stderr.splitlines():
         if "DeprecationWarning" in line:
-            count += 1
+            warnings.append(line) # includes newline
-        print >>sys.stderr, line
-    print >>sys.stderr, "--"
 
-if count:
+    pp.stderr.seek(0)
-    print "ERROR: %d deprecation warnings found" % count
+    for line in pp.stderr.readlines():
-    sys.exit(1)
+        if "DeprecationWarning" in line:
-print "no deprecation warnings"
+            warnings.append(line)
-sys.exit(rc)
+
+    if warnings:
+        if config["warnings"]:
+            with open(config["warnings"], "wb") as f:
+                print >>f, "".join(warnings)
+        print "ERROR: %d deprecation warnings found" % len(warnings)
+        sys.exit(1)
+
+    print "no deprecation warnings"
+    if signal:
+        sys.exit(signal)
+    sys.exit(rc)
+
+
+task.react(run_command)

tox.ini

@@ -18,7 +18,7 @@ passenv = USERPROFILE HOMEDRIVE HOMEPATH
 setenv =
          PYTHONWARNINGS=default::DeprecationWarning
 commands =
-         python misc/build_helpers/run-deprecations.py --stderr=_trial_temp/stderr.log trial --rterrors {posargs:allmydata}
+         python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial --rterrors {posargs:allmydata}
 
 [testenv:upcoming-deprecations]
 basepython=python2.7
@@ -29,7 +29,7 @@ deps =
      git+https://github.com/twisted/twisted
      git+https://github.com/warner/foolscap
 commands =
-         python misc/build_helpers/run-deprecations.py --stderr=_trial_temp/stderr.log trial --rterrors {posargs:allmydata}
+         python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial --rterrors {posargs:allmydata}
 
 [testenv:checkmemory]
 commands =