Merge remote-tracking branch 'origin/3358.fileutil-to-python-3' into 3376.encodingutil-python-3

misc/python3/ratchet-passing

@@ -9,6 +9,9 @@ allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year
 allmydata.test.test_abbreviate.Abbreviate.test_parse_space
 allmydata.test.test_abbreviate.Abbreviate.test_space
 allmydata.test.test_abbreviate.Abbreviate.test_time
+allmydata.test.test_backupdb.BackupDB.test_basic
+allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2
+allmydata.test.test_backupdb.BackupDB.test_wrong_version
 allmydata.test.test_base32.Base32.test_a2b
 allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons
 allmydata.test.test_base32.Base32.test_b2a
@@ -145,6 +148,32 @@ allmydata.test.test_time_format.TimeFormat.test_format_time_y2038
 allmydata.test.test_time_format.TimeFormat.test_iso_utc
 allmydata.test.test_time_format.TimeFormat.test_parse_date
 allmydata.test.test_time_format.TimeFormat.test_parse_duration
+allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode
+allmydata.test.test_util.FileUtil.test_create_long_path
+allmydata.test.test_util.FileUtil.test_disk_stats
+allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative
+allmydata.test.test_util.FileUtil.test_du
+allmydata.test.test_util.FileUtil.test_encrypted_tempfile
+allmydata.test.test_util.FileUtil.test_get_pathinfo
+allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink
+allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode
+allmydata.test.test_util.FileUtil.test_remove_if_possible
+allmydata.test.test_util.FileUtil.test_rename
+allmydata.test.test_util.FileUtil.test_rename_no_overwrite
+allmydata.test.test_util.FileUtil.test_replace_file
+allmydata.test.test_util.FileUtil.test_rm_dir
+allmydata.test.test_util.FileUtil.test_windows_expanduser_win7
+allmydata.test.test_util.FileUtil.test_windows_expanduser_xp
+allmydata.test.test_util.FileUtil.test_write_atomically
+allmydata.test.test_util.IDLib.test_nodeid_b2a
+allmydata.test.test_util.Limiter.test_errors
+allmydata.test.test_util.Limiter.test_limiter
+allmydata.test.test_util.Log.test_err
+allmydata.test.test_util.Math.test_round_sigfigs
+allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True
+allmydata.test.test_util.PollMixinTests.test_PollMixin_True
+allmydata.test.test_util.PollMixinTests.test_timeout
+allmydata.test.test_util.YAML.test_convert
 allmydata.test.test_version.CheckRequirement.test_cross_check
 allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions
 allmydata.test.test_version.CheckRequirement.test_extract_openssl_version

src/allmydata/client.py

@@ -741,7 +741,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         private_key_str = self.config.get_or_create_private_config("node.privkey", _make_key)
         private_key, public_key = ed25519.signing_keypair_from_string(private_key_str)
         public_key_str = ed25519.string_from_verifying_key(public_key)
-        self.config.write_config_file("node.pubkey", public_key_str + "\n")
+        self.config.write_config_file("node.pubkey", public_key_str + "\n", "w")
         self._node_private_key = private_key
         self._node_public_key = public_key
 

src/allmydata/test/common_util.py

@@ -10,10 +10,13 @@ from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
-                                         get_io_encoding)
-from ..scripts import runner
-from .common_py3 import SignalMixin
 
+                                         get_io_encoding)
+from future.utils import PY2
+if PY2: # XXX this is a hack that makes some tests pass on Python3, remove
+        # in the future
+    from ..scripts import runner
+from .common_py3 import SignalMixin
 
 def skip_if_cannot_represent_filename(u):
     precondition(isinstance(u, unicode))

src/allmydata/test/test_util.py

@@ -1,5 +1,15 @@
-from __future__ import print_function
+"""
+Ported to Python3.
+"""
 
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 import six
 import os, time, sys
 import yaml
@@ -19,7 +29,7 @@ if six.PY3:
 
 class IDLib(unittest.TestCase):
     def test_nodeid_b2a(self):
-        self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
+        self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32)
 
 
 class MyList(list):
@@ -85,10 +95,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
         basedir = "util/FileUtil/test_write_atomically"
         fileutil.make_dirs(basedir)
         fn = os.path.join(basedir, "here")
-        fileutil.write_atomically(fn, "one")
+        fileutil.write_atomically(fn, b"one", "b")
-        self.failUnlessEqual(fileutil.read(fn), "one")
+        self.failUnlessEqual(fileutil.read(fn), b"one")
-        fileutil.write_atomically(fn, "two", mode="") # non-binary
+        fileutil.write_atomically(fn, u"two", mode="") # non-binary
-        self.failUnlessEqual(fileutil.read(fn), "two")
+        self.failUnlessEqual(fileutil.read(fn), b"two")
 
     def test_rename(self):
         basedir = "util/FileUtil/test_rename"
@@ -111,20 +121,20 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
         self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
 
         # when only dest exists
-        fileutil.write(dest_path,   "dest")
+        fileutil.write(dest_path,   b"dest")
         self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
-        self.failUnlessEqual(fileutil.read(dest_path),   "dest")
+        self.failUnlessEqual(fileutil.read(dest_path),   b"dest")
 
         # when both exist
-        fileutil.write(source_path, "source")
+        fileutil.write(source_path, b"source")
         self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
-        self.failUnlessEqual(fileutil.read(source_path), "source")
+        self.failUnlessEqual(fileutil.read(source_path), b"source")
-        self.failUnlessEqual(fileutil.read(dest_path),   "dest")
+        self.failUnlessEqual(fileutil.read(dest_path),   b"dest")
 
         # when only source exists
         os.remove(dest_path)
         fileutil.rename_no_overwrite(source_path, dest_path)
-        self.failUnlessEqual(fileutil.read(dest_path), "source")
+        self.failUnlessEqual(fileutil.read(dest_path), b"source")
         self.failIf(os.path.exists(source_path))
 
     def test_replace_file(self):
@@ -138,21 +148,21 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
         self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path)
 
         # when only replaced exists
-        fileutil.write(replaced_path,    "foo")
+        fileutil.write(replaced_path,   b"foo")
         self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path)
-        self.failUnlessEqual(fileutil.read(replaced_path), "foo")
+        self.failUnlessEqual(fileutil.read(replaced_path), b"foo")
 
         # when both replaced and replacement exist
-        fileutil.write(replacement_path, "bar")
+        fileutil.write(replacement_path, b"bar")
         fileutil.replace_file(replaced_path, replacement_path)
-        self.failUnlessEqual(fileutil.read(replaced_path), "bar")
+        self.failUnlessEqual(fileutil.read(replaced_path), b"bar")
         self.failIf(os.path.exists(replacement_path))
 
         # when only replacement exists
         os.remove(replaced_path)
-        fileutil.write(replacement_path, "bar")
+        fileutil.write(replacement_path, b"bar")
         fileutil.replace_file(replaced_path, replacement_path)
-        self.failUnlessEqual(fileutil.read(replaced_path), "bar")
+        self.failUnlessEqual(fileutil.read(replaced_path), b"bar")
         self.failIf(os.path.exists(replacement_path))
 
     def test_du(self):
@@ -170,13 +180,15 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
         self.failUnlessEqual(10+11+12+13, used)
 
     def test_abspath_expanduser_unicode(self):
-        self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring")
+        self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring")
 
-        saved_cwd = os.path.normpath(os.getcwdu())
+        saved_cwd = os.path.normpath(os.getcwd())
+        if PY2:
+            saved_cwd = saved_cwd.decode("utf8")
         abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
         abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False)
-        self.failUnless(isinstance(saved_cwd, unicode), saved_cwd)
+        self.failUnless(isinstance(saved_cwd, str), saved_cwd)
-        self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd)
+        self.failUnless(isinstance(abspath_cwd, str), abspath_cwd)
         if sys.platform == "win32":
             self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd))
         else:
@@ -237,10 +249,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
                 os.chdir(cwd)
                 for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'):
                     uabspath = fileutil.abspath_expanduser_unicode(upath)
-                    self.failUnless(isinstance(uabspath, unicode), uabspath)
+                    self.failUnless(isinstance(uabspath, str), uabspath)
 
                     uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False)
-                    self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong)
+                    self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong)
             finally:
                 os.chdir(saved_cwd)
 
@@ -293,9 +305,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
             fileutil.remove(long_path)
         self.addCleanup(_cleanup)
 
-        fileutil.write(long_path, "test")
+        fileutil.write(long_path, b"test")
         self.failUnless(os.path.exists(long_path))
-        self.failUnlessEqual(fileutil.read(long_path), "test")
+        self.failUnlessEqual(fileutil.read(long_path), b"test")
         _cleanup()
         self.failIf(os.path.exists(long_path))
 
@@ -353,7 +365,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
 
         # create a file
         f = os.path.join(basedir, "1.txt")
-        fileutil.write(f, "a"*10)
+        fileutil.write(f, b"a"*10)
         fileinfo = fileutil.get_pathinfo(f)
         self.failUnlessTrue(fileinfo.isfile)
         self.failUnlessTrue(fileinfo.exists)
@@ -381,7 +393,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
         fileutil.make_dirs(basedir)
 
         f = os.path.join(basedir, "1.txt")
-        fileutil.write(f, "a"*10)
+        fileutil.write(f, b"a"*10)
 
         # create a symlink pointing to 1.txt
         slname = os.path.join(basedir, "linkto1.txt")
@@ -394,7 +406,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
 
     def test_encrypted_tempfile(self):
         f = EncryptedTemporaryFile()
-        f.write("foobar")
+        f.write(b"foobar")
         f.close()
 
 
@@ -409,7 +421,7 @@ class PollMixinTests(unittest.TestCase):
 
     def test_PollMixin_False_then_True(self):
         i = iter([False, True])
-        d = self.pm.poll(check_f=i.next,
+        d = self.pm.poll(check_f=lambda: next(i),
                          pollinterval=0.1)
         return d
 
@@ -454,6 +466,6 @@ class YAML(unittest.TestCase):
     def test_convert(self):
         data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])
         back = yamlutil.safe_load(data)
-        self.failUnlessEqual(type(back[0]), unicode)
+        self.assertIsInstance(back[0], str)
-        self.failUnlessEqual(type(back[1]), unicode)
+        self.assertIsInstance(back[1], str)
-        self.failUnlessEqual(type(back[2]), unicode)
+        self.assertIsInstance(back[2], str)

src/allmydata/util/_python3.py

@@ -30,6 +30,7 @@ PORTED_MODULES = [
     "allmydata.util.base32",
     "allmydata.util.base62",
     "allmydata.util.deferredutil",
+    "allmydata.util.fileutil",
     "allmydata.util.dictutil",
     "allmydata.util.gcutil",
     "allmydata.util.hashutil",
@@ -67,10 +68,10 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
     "allmydata.test.test_time_format",
+    "allmydata.test.test_util",
     "allmydata.test.test_version",
 ]
 
-
 if __name__ == '__main__':
     from subprocess import check_call
     check_call(["trial"] + PORTED_TEST_MODULES)

src/allmydata/util/fileutil.py

@@ -1,9 +1,19 @@
-from __future__ import print_function
-
 """
+Ported to Python3.
+
 Futz with files like a pro.
 """
 
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # open is not here because we want to use native strings on Py2
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import sys, os, stat, tempfile, time, binascii
 import six
 from collections import namedtuple
@@ -253,6 +263,9 @@ def move_into_place(source, dest):
     os.rename(source, dest)
 
 def write_atomically(target, contents, mode="b"):
+    assert (
+        isinstance(contents, bytes) and "b" in mode or
+        isinstance(contents, str) and "t" in mode or mode == ""), (type(contents), mode)
     with open(target+".tmp", "w"+mode) as f:
         f.write(contents)
     move_into_place(target+".tmp", target)
@@ -277,7 +290,7 @@ def put_file(path, inf):
             outf.write(data)
 
 def precondition_abspath(path):
-    if not isinstance(path, unicode):
+    if not isinstance(path, str):
         raise AssertionError("an abspath must be a Unicode string")
 
     if sys.platform == "win32":
@@ -309,7 +322,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True):
     abspath_expanduser_unicode.
     On Windows, the result will be a long path unless long_path is given as False.
     """
-    if not isinstance(path, unicode):
+    if not isinstance(path, str):
         raise AssertionError("paths must be Unicode strings")
     if base is not None and long_path:
         precondition_abspath(base)
@@ -330,7 +343,10 @@ def abspath_expanduser_unicode(path, base=None, long_path=True):
 
     if not os.path.isabs(path):
         if base is None:
-            path = os.path.join(os.getcwdu(), path)
+            cwd = os.getcwd()
+            if PY2:
+                cwd = cwd.decode('utf8')
+            path = os.path.join(cwd, path)
         else:
             path = os.path.join(base, path)
 
@@ -415,7 +431,7 @@ ERROR_ENVVAR_NOT_FOUND = 203
 def windows_getenv(name):
     # Based on <http://stackoverflow.com/questions/2608200/problems-with-umlauts-in-python-appdata-environvent-variable/2608368#2608368>,
     # with improved error handling. Returns None if there is no enivronment variable of the given name.
-    if not isinstance(name, unicode):
+    if not isinstance(name, str):
         raise AssertionError("name must be Unicode")
 
     n = GetEnvironmentVariableW(name, None, 0)

src/allmydata/util/statistics.py

@@ -20,10 +20,9 @@ from future.utils import PY2
 if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from functools import reduce
-
 from allmydata.util.mathutil import round_sigfigs
 import math
+from functools import reduce
 import sys
 
 def pr_file_loss(p_list, k):