Attempt to avoid the hang condition

The Python 2.7 subprocess module does not promise thread safety.

src/allmydata/test/_twisted_9607.py

@@ -0,0 +1,74 @@
+"""
+A copy of the implementation of Twisted's ``getProcessOutputAndValue``
+with the fix for Twisted #9607 (support for stdinBytes) patched in.
+"""
+
+from __future__ import (
+    division,
+    absolute_import,
+    print_function,
+    unicode_literals,
+)
+
+from io import BytesIO
+
+from twisted.internet import protocol, defer
+
+
+class _EverythingGetter(protocol.ProcessProtocol):
+
+    def __init__(self, deferred, stdinBytes=None):
+        self.deferred = deferred
+        self.outBuf = BytesIO()
+        self.errBuf = BytesIO()
+        self.outReceived = self.outBuf.write
+        self.errReceived = self.errBuf.write
+        self.stdinBytes = stdinBytes
+
+    def connectionMade(self):
+        if self.stdinBytes is not None:
+            self.transport.writeToChild(0, self.stdinBytes)
+            # The only compelling reason not to _always_ close stdin here is
+            # backwards compatibility.
+            self.transport.closeStdin()
+
+    def processEnded(self, reason):
+        out = self.outBuf.getvalue()
+        err = self.errBuf.getvalue()
+        e = reason.value
+        code = e.exitCode
+        if e.signal:
+            self.deferred.errback((out, err, e.signal))
+        else:
+            self.deferred.callback((out, err, code))
+
+
+
+def _callProtocolWithDeferred(protocol, executable, args, env, path,
+                              reactor=None, protoArgs=()):
+    if reactor is None:
+        from twisted.internet import reactor
+
+    d = defer.Deferred()
+    p = protocol(d, *protoArgs)
+    reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path)
+    return d
+
+
+
+def getProcessOutputAndValue(executable, args=(), env={}, path=None,
+                             reactor=None, stdinBytes=None):
+    """Spawn a process and returns a Deferred that will be called back with
+    its output (from stdout and stderr) and it's exit code as (out, err, code)
+    If a signal is raised, the Deferred will errback with the stdout and
+    stderr up to that point, along with the signal, as (out, err, signalNum)
+    """
+    return _callProtocolWithDeferred(
+        _EverythingGetter,
+        executable,
+        args,
+        env,
+        path,
+        reactor,
+        protoArgs=(stdinBytes,),
+    )

src/allmydata/test/test_runner.py

@@ -1,9 +1,13 @@
-import os.path, re, sys, subprocess
+
+from __future__ import (
+    absolute_import,
+)
+
+import os.path, re, sys
 
 from twisted.trial import unittest
 
 from twisted.python import usage, runtime
-from twisted.internet import threads
 from twisted.internet.defer import inlineCallbacks, returnValue
 
 from allmydata.util import fileutil, pollmixin
@@ -15,6 +19,9 @@ import allmydata
 from allmydata import __appname__
 from .common_util import parse_cli, run_cli
 
+from ._twisted_9607 import (
+    getProcessOutputAndValue,
+)
 
 timeout = 240
 
@@ -52,18 +59,19 @@ class RunBinTahoeMixin:
         returnValue(tahoe_pieces[-1].strip("()"))
 
     def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
-        command = [sys.executable] + python_options + ["-m", "allmydata.scripts.runner"] + args
+        command = sys.executable
+        argv = python_options + ["-m", "allmydata.scripts.runner"] + args
 
-        if stdin is None:
+        if env is None:
-            stdin_stream = None
+            env = os.environ
-        else:
-            stdin_stream = subprocess.PIPE
 
-        def _run():
+        d = getProcessOutputAndValue(command, argv, env, stdinBytes=stdin)
-            p = subprocess.Popen(command, stdin=stdin_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+        def fix_signal(result):
-            (out, err) = p.communicate(stdin)
+            # Mirror subprocess.Popen.returncode structure
-            return (out, err, p.returncode)
+            (out, err, signal) = result
-        return threads.deferToThread(_run)
+            return (out, err, -signal)
+        d.addErrback(fix_signal)
+        return d
 
 
 class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin):