From 9d20de3db998859c096c6f214b7d1c5443aa098a Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 6 Apr 2016 11:07:06 -1000 Subject: [PATCH] improve run-deprecations script Rewrote in Twisted, which lets us read/scan/print all log lines in realtime. The output is now correctly interleaved (as well as maintaining the stdout-vs-stderr of each message). The renamed --warnings= logfile records all relevant lines from *both* stdout and stderr (i.e. any that includes "DeprecationWarning"), which handles a change (perhaps in recent Twisteds?) that emits these warnings on stdout instead of stderr. --- misc/build_helpers/run-deprecations.py | 106 +++++++++++++++---------- tox.ini | 4 +- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index 19d3ca031..5df3797a4 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -1,4 +1,5 @@ -import sys, os, subprocess +import sys, os, io +from twisted.internet import reactor, protocol, task, defer from twisted.python.procutils import which from twisted.python import usage @@ -8,7 +9,7 @@ from twisted.python import usage class Options(usage.Options): optParameters = [ - ["stderr", None, None, "file to write stderr into at end of test run"], + ["warnings", None, None, "file to write warnings into at end of test run"], ] def parseArgs(self, command, *args): @@ -16,54 +17,73 @@ class Options(usage.Options): self["args"] = list(args) description = """Run as: -PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--stderr=STDERRFILE] COMMAND ARGS.. +PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--warnings=STDERRFILE] COMMAND ARGS.. """ -config = Options() -config.parseOptions() +class RunPP(protocol.ProcessProtocol): + def outReceived(self, data): + self.stdout.write(data) + sys.stdout.write(data) + def errReceived(self, data): + self.stderr.write(data) + sys.stderr.write(data) + def processEnded(self, reason): + signal = reason.value.signal + rc = reason.value.exitCode + self.d.callback((signal, rc)) +@defer.inlineCallbacks +def run_command(main): + config = Options() + config.parseOptions() -command = config["command"] -if "/" in command: - # don't search - exe = command -else: - executables = which(command) - if not executables: - raise ValueError("unable to find '%s' in PATH (%s)" % - (command, os.environ.get("PATH"))) - exe = executables[0] + command = config["command"] + if "/" in command: + # don't search + exe = command + else: + executables = which(command) + if not executables: + raise ValueError("unable to find '%s' in PATH (%s)" % + (command, os.environ.get("PATH"))) + exe = executables[0] -pw = os.environ.get("PYTHONWARNINGS") -DDW = "default::DeprecationWarning" -if pw != DDW: - print "note: $PYTHONWARNINGS is '%s', not the expected %s" % (pw, DDW) + pw = os.environ.get("PYTHONWARNINGS") + DDW = "default::DeprecationWarning" + if pw != DDW: + print "note: $PYTHONWARNINGS is '%s', not the expected %s" % (pw, DDW) + sys.stdout.flush() -print "note: stderr is being captured, and will be emitted at the end" -sys.stdout.flush() + pp = RunPP() + pp.d = defer.Deferred() + pp.stdout = io.BytesIO() + pp.stderr = io.BytesIO() + reactor.spawnProcess(pp, exe, [exe] + config["args"], env=None) + (signal, rc) = yield pp.d -# stdout goes directly to the parent, so test progress can be watched in real -# time. But subprocess.Popen() doesn't give us any good way of seeing it -p = subprocess.Popen([exe] + config["args"], stderr=subprocess.PIPE) -stderr = p.communicate()[1] -rc = p.returncode -count = 0 + warnings = [] -if config["stderr"]: - with open(config["stderr"], "wb") as f: - print >>f, stderr, - -if stderr: - print >>sys.stderr, "--" - print >>sys.stderr, "Captured stderr follows:" - for line in stderr.splitlines(): + pp.stdout.seek(0) + for line in pp.stdout.readlines(): if "DeprecationWarning" in line: - count += 1 - print >>sys.stderr, line - print >>sys.stderr, "--" + warnings.append(line) # includes newline -if count: - print "ERROR: %d deprecation warnings found" % count - sys.exit(1) -print "no deprecation warnings" -sys.exit(rc) + pp.stderr.seek(0) + for line in pp.stderr.readlines(): + if "DeprecationWarning" in line: + warnings.append(line) + + if warnings: + if config["warnings"]: + with open(config["warnings"], "wb") as f: + print >>f, "".join(warnings) + print "ERROR: %d deprecation warnings found" % len(warnings) + sys.exit(1) + + print "no deprecation warnings" + if signal: + sys.exit(signal) + sys.exit(rc) + + +task.react(run_command) diff --git a/tox.ini b/tox.ini index f1df4acc3..fb3b34d17 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ passenv = USERPROFILE HOMEDRIVE HOMEPATH setenv = PYTHONWARNINGS=default::DeprecationWarning commands = - python misc/build_helpers/run-deprecations.py --stderr=_trial_temp/stderr.log trial --rterrors {posargs:allmydata} + python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial --rterrors {posargs:allmydata} [testenv:upcoming-deprecations] basepython=python2.7 @@ -29,7 +29,7 @@ deps = git+https://github.com/twisted/twisted git+https://github.com/warner/foolscap commands = - python misc/build_helpers/run-deprecations.py --stderr=_trial_temp/stderr.log trial --rterrors {posargs:allmydata} + python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial --rterrors {posargs:allmydata} [testenv:checkmemory] commands =