Changes to Tahoe needed to work with new zetuptoolz (that does not use .exe wrappers on Windows), and to support Unicode arguments and stdout/stderr -- v5

bin/tahoe-script.template

@@ -5,12 +5,17 @@ import errno, sys, os, subprocess
 where = os.path.realpath(sys.argv[0])
 base = os.path.dirname(os.path.dirname(where))
 
+if sys.platform == "win32":
+    installed_tahoe = os.path.join(os.path.dirname(sys.executable), 'Scripts', 'tahoe.pyscript')
+else:
+    installed_tahoe = "/usr/bin/tahoe"
+
 whoami = '''\
-I am a "bin/tahoe" executable who is only for the convenience of running
+I am a "bin%stahoe" executable who is only for the convenience of running
 Tahoe from its source distribution -- I work only when invoked as the "tahoe"
 script that lives in the "bin/" subdirectory of a Tahoe source code
 distribution, and only if you have already run "make".
-'''
+''' % (os.path.sep,)
 
 # look for Tahoe.home .
 homemarker = os.path.join(base, "Tahoe.home")
@@ -19,9 +24,9 @@ if not os.path.exists(homemarker):
     print '''\
 I just tried to run and found that I am not living in such a directory, so I
 am stopping now. To run Tahoe after it has been is installed, please execute
-my brother, also named "tahoe", who gets installed into the appropriate place
+my brother, who gets installed into the appropriate place for executables
-for executables when you run "make install" (perhaps as /usr/bin/tahoe).
+when you run "make install" (perhaps as "%s").
-'''
+''' % (installed_tahoe,)
     sys.exit(1)
 
 # we've found our home. Put the tahoe support/lib etc. in our PYTHONPATH.
@@ -41,30 +46,50 @@ else:
     pp = supportdir
 os.environ["PYTHONPATH"] = pp
 
-# find the location of the tahoe executable.
+# find commandline args and the location of the tahoe executable.
-bin_dir = "bin"
 if sys.platform == "win32":
-    bin_dir = "Scripts"
+    import re
-executable = os.path.join(base, "support", bin_dir, "tahoe")
+    from ctypes import WINFUNCTYPE, POINTER, byref, c_wchar_p, c_int, windll
+
+    GetCommandLineW = WINFUNCTYPE(c_wchar_p)(("GetCommandLineW", windll.kernel32))
+    CommandLineToArgvW = WINFUNCTYPE(POINTER(c_wchar_p), c_wchar_p, POINTER(c_int)) \
+                            (("CommandLineToArgvW", windll.shell32))
+
+    argc = c_int(0)
+    argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
+
+    # See src/allmydata/scripts/runner.py for the corresponding unmangler.
+    # Note that this doesn't escape \x7F. If it did, test_unicode_arguments_and_output
+    # in test_runner.py wouldn't work.
+    def mangle(s):
+        return str(re.sub(ur'[^\x20-\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s))
+
+    argv = [mangle(argv_unicode[i]) for i in xrange(1, argc.value)]
+    local_tahoe = "Scripts\\tahoe.pyscript"
+else:
+    argv = sys.argv
+    local_tahoe = "bin/tahoe"
+
+script = os.path.join(base, "support", local_tahoe)
 
 try:
-    res = subprocess.call([executable] + sys.argv[1:], env=os.environ)
+    res = subprocess.call([sys.executable, script] + argv[1:], env=os.environ)
 except (OSError, IOError), le:
     if le.args[0] == errno.ENOENT:
         print whoami
         print '''\
-I just tried to run and could not find my brother, named
+I just tried to run and could not find my brother at
-"../support/bin/tahoe". To run Tahoe when it is installed, please execute my
+"%s". To run Tahoe when it is installed, please execute my
-brother, also named "tahoe", who gets installed into the appropriate place
+brother, who gets installed into the appropriate place for executables
-for executables when you run "make install" (perhaps as /usr/bin/tahoe).
+when you run "make install" (perhaps as "%s").
-'''
+''' % (script, installed_tahoe)
         raise
 except Exception, le:
     print whoami
     print '''\
-I just tried to invoke my brother, named "../support/bin/tahoe" and got an
+I just tried to invoke my brother at "%s"
-exception.
+and got an exception.
-'''
+''' % (script,)
     raise
 else:
     sys.exit(res)

setup.py

@@ -235,55 +235,47 @@ class MakeExecutable(Command):
     def run(self):
         bin_tahoe_template = os.path.join("bin", "tahoe-script.template")
 
-        # Create the 'tahoe-script.py' file under the 'bin' directory. The
+        if sys.platform == 'win32':
-        # 'tahoe-script.py' file is exactly the same as the
+            # 'tahoe' script is needed for cygwin
-        # 'tahoe-script.template' script except that the shebang line is
+            script_names = ["tahoe.pyscript", "tahoe"]
-        # rewritten to use our sys.executable for the interpreter. On
+        else:
-        # Windows, create a tahoe.exe will execute it. On non-Windows, make a
+            script_names = ["tahoe"]
-        # symlink to it from 'tahoe'. The tahoe.exe will be copied from the
+
-        # setuptools egg's cli.exe and this will work from a zip-safe and
+        # Create the tahoe script file under the 'bin' directory. This
-        # non-zip-safe setuptools egg.
+        # file is exactly the same as the 'tahoe-script.template' script
+        # except that the shebang line is rewritten to use our sys.executable
+        # for the interpreter.
         f = open(bin_tahoe_template, "rU")
         script_lines = f.readlines()
         f.close()
-        script_lines[0] = "#!%s\n" % sys.executable
+        script_lines[0] = '#!%s\n' % (sys.executable,)
-        tahoe_script = os.path.join("bin", "tahoe-script.py")
+        for script_name in script_names:
-        f = open(tahoe_script, "w")
+            tahoe_script = os.path.join("bin", script_name)
+            try:
+                os.remove(tahoe_script)
+            except Exception:
+                if os.path.exists(tahoe_script):
+                   raise
+            f = open(tahoe_script, "wb")
             for line in script_lines:
                 f.write(line)
             f.close()
-        if sys.platform == "win32":
-            from pkg_resources import require
-            setuptools_egg = require("setuptools")[0].location
-            if os.path.isfile(setuptools_egg):
-                z = zipfile.ZipFile(setuptools_egg, 'r')
-                for filename in z.namelist():
-                    if 'cli.exe' in filename:
-                        cli_exe = z.read(filename)
-            else:
-                cli_exe = os.path.join(setuptools_egg, 'setuptools', 'cli.exe')
-            tahoe_exe = os.path.join("bin", "tahoe.exe")
-            if os.path.isfile(setuptools_egg):
-                f = open(tahoe_exe, 'wb')
-                f.write(cli_exe)
-                f.close()
-            else:
-                shutil.copy(cli_exe, tahoe_exe)
-        else:
-            try:
-                os.remove(os.path.join('bin', 'tahoe'))
-            except:
-                # okay, probably it was already gone
-                pass
-            os.symlink('tahoe-script.py', os.path.join('bin', 'tahoe'))
 
-        # chmod +x bin/tahoe-script.py
+            # chmod +x
             old_mode = stat.S_IMODE(os.stat(tahoe_script)[stat.ST_MODE])
             new_mode = old_mode | (stat.S_IXUSR | stat.S_IRUSR |
                                    stat.S_IXGRP | stat.S_IRGRP |
                                    stat.S_IXOTH | stat.S_IROTH )
             os.chmod(tahoe_script, new_mode)
 
+        old_tahoe_exe = os.path.join("bin", "tahoe.exe")
+        try:
+            os.remove(old_tahoe_exe)
+        except Exception:
+            if os.path.exists(old_tahoe_exe):
+                raise
+
+
 class MySdist(sdist.sdist):
     """ A hook in the sdist command so that we can determine whether this the
     tarball should be 'SUMO' or not, i.e. whether or not to include the

src/allmydata/scripts/runner.py

@@ -10,6 +10,7 @@ import allmydata
 pkg_resources.require(allmydata.__appname__)
 from allmydata.scripts.common import BaseOptions
 from allmydata.scripts import debug, create_node, startstop_node, cli, keygen, stats_gatherer
+from allmydata.util.encodingutil import quote_output, get_argv_encoding
 
 def GROUP(s):
     # Usage.parseOptions compares argv[1] against command[0], so it will
@@ -19,7 +20,7 @@ def GROUP(s):
 
 
 class Options(BaseOptions, usage.Options):
-    synopsis = "Usage:  tahoe <command> [command options]"
+    synopsis = "\nUsage:  tahoe <command> [command options]"
     subCommands = ( GROUP("Administration")
                     +   create_node.subCommands
                     +   keygen.subCommands
@@ -42,9 +43,13 @@ class Options(BaseOptions, usage.Options):
 
 def runner(argv,
            run_by_human=True,
-           stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr,
+           stdin=None, stdout=None, stderr=None,
            install_node_control=True, additional_commands=None):
 
+    stdin  = stdin  or sys.stdin
+    stdout = stdout or sys.stdout
+    stderr = stderr or sys.stderr
+
     config = Options()
     if install_node_control:
         config.subCommands.extend(startstop_node.subCommands)
@@ -63,8 +68,12 @@ def runner(argv,
         c = config
         while hasattr(c, 'subOptions'):
             c = c.subOptions
-        print str(c)
+        print >>stdout, str(c)
-        print "%s:  %s" % (sys.argv[0], e)
+        try:
+            msg = e.args[0].decode(get_argv_encoding())
+        except Exception:
+            msg = repr(e)
+        print >>stdout, "%s:  %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False))
         return 1
 
     command = config.subCommand
@@ -99,6 +108,11 @@ def runner(argv,
 
     return rc
 
+
 def run(install_node_control=True):
-    rc = runner(sys.argv[1:])
+    if sys.platform == "win32":
+        from allmydata.windows.fixups import initialize
+        initialize()
+
+    rc = runner(sys.argv[1:], install_node_control=install_node_control)
     sys.exit(rc)

src/allmydata/test/__init__.py

@@ -24,3 +24,8 @@ def disable_foolscap_incidents():
 
 # we disable incident reporting for all unit tests.
 disable_foolscap_incidents()
+
+import sys
+if sys.platform == "win32":
+    from allmydata.windows.fixups import initialize
+    initialize()

src/allmydata/test/test_encodingutil.py

@@ -22,16 +22,22 @@ if __name__ == "__main__":
         print "Usage: %s lumi<e-grave>re" % sys.argv[0]
         sys.exit(1)
 
+    if sys.platform == "win32":
+        try:
+            from allmydata.windows.fixups import initialize
+        except ImportError:
+            print "set PYTHONPATH to the src directory"
+            sys.exit(1)
+        initialize()
+
     print
     print "class MyWeirdOS(EncodingUtil, unittest.TestCase):"
     print "    uname = '%s'" % ' '.join(platform.uname())
-    if sys.platform != "win32":
     print "    argv = %s" % repr(sys.argv[1])
     print "    platform = '%s'" % sys.platform
     print "    filesystem_encoding = '%s'" % sys.getfilesystemencoding()
     print "    output_encoding = '%s'" % sys.stdout.encoding
-    print "    argv_encoding = '%s'" % (sys.platform == "win32" and 'ascii' or sys.stdout.encoding)
+    print "    argv_encoding = '%s'" % sys.stdout.encoding
-
     try:
         tmpdir = tempfile.mkdtemp()
         for fname in TEST_FILENAMES:
@@ -56,6 +62,7 @@ from mock import patch
 import os, sys, locale
 
 from allmydata.test.common_util import ReallyEqualMixin
+from allmydata.util import encodingutil
 from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \
     unicode_to_output, quote_output, unicode_platform, listdir_unicode, \
     FilenameEncodingError, get_output_encoding, get_filesystem_encoding, _reload
@@ -64,8 +71,6 @@ from allmydata.dirnode import normalize
 from twisted.python import usage
 
 class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
-    def tearDown(self):
-        _reload()
 
     @patch('sys.stdout')
     def test_get_output_encoding(self, mock_stdout):
@@ -78,10 +83,15 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(get_output_encoding(), 'utf-8')
 
         mock_stdout.encoding = 'koi8-r'
+        expected = sys.platform == "win32" and 'utf-8' or 'koi8-r'
         _reload()
-        self.failUnlessReallyEqual(get_output_encoding(), 'koi8-r')
+        self.failUnlessReallyEqual(get_output_encoding(), expected)
 
         mock_stdout.encoding = 'nonexistent_encoding'
+        if sys.platform == "win32":
+            _reload()
+            self.failUnlessReallyEqual(get_output_encoding(), 'utf-8')
+        else:
             self.failUnlessRaises(AssertionError, _reload)
 
     @patch('locale.getpreferredencoding')
@@ -94,12 +104,13 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
         old_stdout = sys.stdout
         sys.stdout = DummyStdout()
         try:
+            expected = sys.platform == "win32" and 'utf-8' or 'koi8-r'
             _reload()
-            self.failUnlessReallyEqual(get_output_encoding(), 'koi8-r')
+            self.failUnlessReallyEqual(get_output_encoding(), expected)
 
             sys.stdout.encoding = None
             _reload()
-            self.failUnlessReallyEqual(get_output_encoding(), 'koi8-r')
+            self.failUnlessReallyEqual(get_output_encoding(), expected)
 
             mock_locale_getpreferredencoding.return_value = None
             _reload()
@@ -107,20 +118,14 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
         finally:
             sys.stdout = old_stdout
 
-    @patch('sys.stdout')
+    def test_argv_to_unicode(self):
-    def test_argv_to_unicode(self, mock):
+        encodingutil.output_encoding = 'utf-8'
-        mock.encoding = 'utf-8'
-        _reload()
-
         self.failUnlessRaises(usage.UsageError,
                               argv_to_unicode,
                               lumiere_nfc.encode('latin1'))
 
-    @patch('sys.stdout')
+    def test_unicode_to_output(self):
-    def test_unicode_to_output(self, mock):
+        encodingutil.output_encoding = 'koi8-r'
-        # Encoding koi8-r cannot represent e-grave
-        mock.encoding = 'koi8-r'
-        _reload()
         self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, lumiere_nfc)
 
     @patch('os.listdir')
@@ -171,9 +176,9 @@ class EncodingUtilNonUnicodePlatform(unittest.TestCase):
                               listdir_unicode,
                               u'/' + lumiere_nfc)
 
+
 class EncodingUtil(ReallyEqualMixin):
     def setUp(self):
-        # Mock sys.platform because unicode_platform() uses it
         self.original_platform = sys.platform
         sys.platform = self.platform
 
@@ -197,12 +202,12 @@ class EncodingUtil(ReallyEqualMixin):
 
     @patch('sys.stdout')
     def test_unicode_to_output(self, mock):
-        if 'output' not in dir(self):
+        if 'argv' not in dir(self):
             return
 
         mock.encoding = self.output_encoding
         _reload()
-        self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.output)
+        self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.argv)
 
     def test_unicode_platform(self):
         matrix = {
@@ -287,14 +292,18 @@ class StdlibUnicode(unittest.TestCase):
 
 
 class QuoteOutput(ReallyEqualMixin, unittest.TestCase):
+    def tearDown(self):
+        _reload()
+
     def _check(self, inp, out, enc, optional_quotes):
         out2 = out
         if optional_quotes:
             out2 = out2[1:-1]
         self.failUnlessReallyEqual(quote_output(inp, encoding=enc), out)
         self.failUnlessReallyEqual(quote_output(inp, encoding=enc, quotemarks=False), out2)
-        if out[0:2] != 'b"':
+        if out[0:2] == 'b"':
-            if isinstance(inp, str):
+            pass
+        elif isinstance(inp, str):
             self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc), out)
             self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quotemarks=False), out2)
         else:
@@ -368,24 +377,19 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase):
         check(u"\"\u2621", u"'\"\u2621'")
         check(u"\u2621\"", u"'\u2621\"'", True)
 
-    @patch('sys.stdout')
+    def test_quote_output_default(self):
-    def test_quote_output_mock(self, mock_stdout):
+        encodingutil.output_encoding = 'ascii'
-        mock_stdout.encoding = 'ascii'
-        _reload()
         self.test_quote_output_ascii(None)
 
-        mock_stdout.encoding = 'latin1'
+        encodingutil.output_encoding = 'latin1'
-        _reload()
         self.test_quote_output_latin1(None)
 
-        mock_stdout.encoding = 'utf-8'
+        encodingutil.output_encoding = 'utf-8'
-        _reload()
         self.test_quote_output_utf8(None)
 
 
 class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
-    output = 'lumi\xc3\xa8re'
     argv = 'lumi\xc3\xa8re'
     platform = 'linux2'
     filesystem_encoding = 'UTF-8'
@@ -395,7 +399,6 @@ class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase):
 
 class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
-    output = 'lumi\xe8re'
     argv = 'lumi\xe8re'
     platform = 'linux2'
     filesystem_encoding = 'ISO-8859-1'
@@ -403,37 +406,18 @@ class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase):
     argv_encoding = 'ISO-8859-1'
     dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3']
 
-class WindowsXP(EncodingUtil, unittest.TestCase):
+class Windows(EncodingUtil, unittest.TestCase):
     uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
-    output = 'lumi\x8are'
+    argv = 'lumi\xc3\xa8re'
     platform = 'win32'
     filesystem_encoding = 'mbcs'
-    output_encoding = 'cp850'
+    output_encoding = 'utf-8'
-    argv_encoding = 'ascii'
+    argv_encoding = 'utf-8'
-    dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
-
-class WindowsXP_UTF8(EncodingUtil, unittest.TestCase):
-    uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
-    output = 'lumi\xc3\xa8re'
-    platform = 'win32'
-    filesystem_encoding = 'mbcs'
-    output_encoding = 'cp65001'
-    argv_encoding = 'ascii'
-    dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
-
-class WindowsVista(EncodingUtil, unittest.TestCase):
-    uname = 'Windows Vista 6.0.6000 x86 x86 Family 6 Model 15 Stepping 11, GenuineIntel'
-    output = 'lumi\x8are'
-    platform = 'win32'
-    filesystem_encoding = 'mbcs'
-    output_encoding = 'cp850'
-    argv_encoding = 'ascii'
     dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
 class MacOSXLeopard(EncodingUtil, unittest.TestCase):
     uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
     output = 'lumi\xc3\xa8re'
-    argv = 'lumi\xc3\xa8re'
     platform = 'darwin'
     filesystem_encoding = 'utf-8'
     output_encoding = 'UTF-8'

src/allmydata/test/test_runner.py

@@ -7,6 +7,7 @@ from twisted.internet import utils
 import os.path, re, sys
 from cStringIO import StringIO
 from allmydata.util import fileutil, pollmixin
+from allmydata.util.encodingutil import unicode_to_argv, unicode_to_output
 from allmydata.scripts import runner
 
 from allmydata.test import common_util
@@ -47,6 +48,24 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase, SkipMixin):
         d.addCallback(_cb)
         return d
 
+    def test_unicode_arguments_and_output(self):
+        self.skip_if_cannot_run_bintahoe()
+
+        tricky = u"\u2621"
+        try:
+            tricky_arg = unicode_to_argv(tricky, mangle=True)
+            tricky_out = unicode_to_output(tricky)
+        except UnicodeEncodeError:
+            raise unittest.SkipTest("A non-ASCII argument/output could not be encoded on this platform.")
+
+        d = utils.getProcessOutputAndValue(bintahoe, args=[tricky_arg], env=os.environ)
+        def _cb(res):
+            out, err, rc_or_sig = res
+            self.failUnlessEqual(rc_or_sig, 1, str((out, err, rc_or_sig)))
+            self.failUnlessIn("Unknown command: "+tricky_out, out)
+        d.addCallback(_cb)
+        return d
+
     def test_version_no_noise(self):
         self.skip_if_cannot_run_bintahoe()
         import pkg_resources

src/allmydata/util/encodingutil.py

@@ -13,7 +13,7 @@ from allmydata.util import log
 from allmydata.util.fileutil import abspath_expanduser_unicode
 
 
-def _canonical_encoding(encoding):
+def canonical_encoding(encoding):
     if encoding is None:
         log.msg("Warning: falling back to UTF-8 encoding.", level=log.WEIRD)
         encoding = 'utf-8'
@@ -23,6 +23,9 @@ def _canonical_encoding(encoding):
     elif encoding == "us-ascii" or encoding == "646" or encoding == "ansi_x3.4-1968":
         encoding = 'ascii'
 
+    return encoding
+
+def check_encoding(encoding):
     # sometimes Python returns an encoding name that it doesn't support for conversion
     # fail early if this happens
     try:
@@ -30,8 +33,6 @@ def _canonical_encoding(encoding):
     except (LookupError, AttributeError):
         raise AssertionError("The character encoding '%s' is not supported for conversion." % (encoding,))
 
-    return encoding
-
 filesystem_encoding = None
 output_encoding = None
 argv_encoding = None
@@ -40,8 +41,14 @@ is_unicode_platform = False
 def _reload():
     global filesystem_encoding, output_encoding, argv_encoding, is_unicode_platform
 
-    filesystem_encoding = _canonical_encoding(sys.getfilesystemencoding())
+    filesystem_encoding = canonical_encoding(sys.getfilesystemencoding())
+    check_encoding(filesystem_encoding)
 
+    if sys.platform == 'win32':
+        # On Windows we install UTF-8 stream wrappers for sys.stdout and
+        # sys.stderr, and reencode the arguments as UTF-8 (see scripts/runner.py).
+        output_encoding = 'utf-8'
+    else:
         outenc = None
         if hasattr(sys.stdout, 'encoding'):
             outenc = sys.stdout.encoding
@@ -50,13 +57,11 @@ def _reload():
                 outenc = locale.getpreferredencoding()
             except Exception:
                 pass  # work around <http://bugs.python.org/issue1443504>
-    output_encoding = _canonical_encoding(outenc)
+        output_encoding = canonical_encoding(outenc)
 
-    if sys.platform == 'win32':
+    check_encoding(output_encoding)
-        # Unicode arguments are not supported on Windows yet; see #565 and #1074.
-        argv_encoding = 'ascii'
-    else:
     argv_encoding = output_encoding
+
     is_unicode_platform = sys.platform in ["win32", "darwin"]
 
 _reload()