Merge branch '2912-unicode-error'

closes ticket:2912

Also adds a new travis builder (with LANG=C) to exercise the same unicode
problems that several of our buildbot workers see.

.travis.yml

@@ -1,14 +1,8 @@
 dist: trusty
 sudo: required
-os:
- - "linux"
- - "osx"
 cache: pip
 before_cache:
   - rm -f $HOME/.cache/pip/log/debug.log
-python:
-  - "2.7"
-  - "pypy"
 before_install:
   - sh -c set
   - git config --global user.email "travis-tahoe@tahoe-lafs.org"
@@ -38,6 +32,11 @@ notifications:
       - "%{repository}#%{build_number} [%{branch}: %{commit} by %{author}] %{message}"
       - "Changes: %{compare_url} | Details: %{build_url}"
 matrix:
-  allow_failures:
-    - python: "pypy"
+  include:
+  - os: linux
+    env: LANG=en_US.UTF-8
+  - os: linux
+    env: LANG=C
+  - os: osx
+    env: LANG=en_US.UTF-8
   fast_finish: true

src/allmydata/test/cli/test_alias.py

@@ -4,22 +4,14 @@ from mock import patch
 from twisted.trial import unittest
 from twisted.internet.defer import inlineCallbacks
 
-from allmydata.util.encodingutil import unicode_to_argv, get_io_encoding
+from allmydata.util.encodingutil import unicode_to_argv
 from allmydata.scripts.common import get_aliases
 from allmydata.test.no_network import GridTestMixin
 from .common import CLITestMixin
-
+from ..common_util import skip_if_cannot_represent_argv
 
 # see also test_create_alias
 
-def skip_unless_unicode():
-    try:
-        etudes_arg = u"\u00E9tudes".encode(get_io_encoding())
-        del etudes_arg
-    except UnicodeEncodeError:
-        raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
-
-
 class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
 
     @inlineCallbacks
@@ -55,7 +47,7 @@ class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
         this
         """
         self.basedir = "cli/ListAlias/test_list_unicode_mismatch_json"
-        skip_unless_unicode()
+        skip_if_cannot_represent_argv(u"tahoe\u263A")
         self.set_up_grid(oneshare=True)
 
         rc, stdout, stderr = yield self.do_cli(
@@ -87,7 +79,7 @@ class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
     @inlineCallbacks
     def test_list_unicode_mismatch(self):
         self.basedir = "cli/ListAlias/test_list_unicode_mismatch"
-        skip_unless_unicode()
+        skip_if_cannot_represent_argv(u"tahoe\u263A")
         self.set_up_grid(oneshare=True)
 
         rc, stdout, stderr = yield self.do_cli(

src/allmydata/test/common_util.py

@@ -22,6 +22,13 @@ def skip_if_cannot_represent_filename(u):
         except UnicodeEncodeError:
             raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
 
+def skip_if_cannot_represent_argv(u):
+    precondition(isinstance(u, unicode))
+    try:
+        u.encode(get_io_encoding())
+    except UnicodeEncodeError:
+        raise unittest.SkipTest("A non-ASCII argv could not be encoded on this platform.")
+
 def run_cli(verb, *args, **kwargs):
     precondition(not [True for arg in args if not isinstance(arg, str)],
                  "arguments to do_cli must be strs -- convert using unicode_to_argv", args=args)

src/allmydata/test/test_client.py

@@ -11,7 +11,7 @@ from allmydata.node import OldConfigError, OldConfigOptionError, UnescapedHashEr
 from allmydata.frontends.auth import NeedRootcapLookupScheme
 from allmydata import client
 from allmydata.storage_client import StorageFarmBroker
-from allmydata.util import base32, fileutil
+from allmydata.util import base32, fileutil, encodingutil
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.interfaces import IFilesystemNode, IFileNode, \
      IImmutableFileNode, IMutableFileNode, IDirectoryNode
@@ -320,8 +320,11 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
         # don't want a literal absolute path like /myowndir which we won't
         # have write permission to.  So construct an absolute path that we
         # should be able to write to.
+        base = u"\N{SNOWMAN}"
+        if encodingutil.filesystem_encoding != "utf-8":
+            base = u"melted_snowman"
         expected_path = abspath_expanduser_unicode(
-            u"client.Basic.test_absolute_storage_dir_myowndir/\N{SNOWMAN}"
+            u"client.Basic.test_absolute_storage_dir_myowndir/" + base
         )
         config_path = expected_path.encode("utf-8")
         self._storage_dir_test(