diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 1e11722cb..1c2d1eda8 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -9,6 +9,9 @@ allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year allmydata.test.test_abbreviate.Abbreviate.test_parse_space allmydata.test.test_abbreviate.Abbreviate.test_space allmydata.test.test_abbreviate.Abbreviate.test_time +allmydata.test.test_backupdb.BackupDB.test_basic +allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2 +allmydata.test.test_backupdb.BackupDB.test_wrong_version allmydata.test.test_base32.Base32.test_a2b allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons allmydata.test.test_base32.Base32.test_b2a @@ -145,6 +148,29 @@ allmydata.test.test_time_format.TimeFormat.test_format_time_y2038 allmydata.test.test_time_format.TimeFormat.test_iso_utc allmydata.test.test_time_format.TimeFormat.test_parse_date allmydata.test.test_time_format.TimeFormat.test_parse_duration +allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode +allmydata.test.test_util.FileUtil.test_create_long_path +allmydata.test.test_util.FileUtil.test_disk_stats +allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative +allmydata.test.test_util.FileUtil.test_du +allmydata.test.test_util.FileUtil.test_encrypted_tempfile +allmydata.test.test_util.FileUtil.test_get_pathinfo +allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink +allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode +allmydata.test.test_util.FileUtil.test_remove_if_possible +allmydata.test.test_util.FileUtil.test_rename +allmydata.test.test_util.FileUtil.test_rename_no_overwrite +allmydata.test.test_util.FileUtil.test_replace_file +allmydata.test.test_util.FileUtil.test_rm_dir +allmydata.test.test_util.FileUtil.test_windows_expanduser_win7 +allmydata.test.test_util.FileUtil.test_windows_expanduser_xp +allmydata.test.test_util.FileUtil.test_write_atomically +allmydata.test.test_util.IDLib.test_nodeid_b2a +allmydata.test.test_util.Math.test_round_sigfigs +allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True +allmydata.test.test_util.PollMixinTests.test_PollMixin_True +allmydata.test.test_util.PollMixinTests.test_timeout +allmydata.test.test_util.YAML.test_convert allmydata.test.test_version.CheckRequirement.test_cross_check allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions allmydata.test.test_version.CheckRequirement.test_extract_openssl_version diff --git a/newsfragments/3358.minor b/newsfragments/3358.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 5e8bf0d6d..f731c3163 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -741,7 +741,7 @@ class _Client(node.Node, pollmixin.PollMixin): private_key_str = self.config.get_or_create_private_config("node.privkey", _make_key) private_key, public_key = ed25519.signing_keypair_from_string(private_key_str) public_key_str = ed25519.string_from_verifying_key(public_key) - self.config.write_config_file("node.pubkey", public_key_str + "\n") + self.config.write_config_file("node.pubkey", public_key_str + "\n", "w") self._node_private_key = private_key self._node_public_key = public_key diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 261d39af3..4e25e897e 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -10,10 +10,13 @@ from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, - get_io_encoding) -from ..scripts import runner -from .common_py3 import SignalMixin + get_io_encoding) +from future.utils import PY2 +if PY2: # XXX this is a hack that makes some tests pass on Python3, remove + # in the future + from ..scripts import runner +from .common_py3 import SignalMixin def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 6030f2c49..f1f2b1c66 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,5 +1,15 @@ -from __future__ import print_function +""" +Ported to Python3. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six import os, time, sys import yaml @@ -19,7 +29,7 @@ if six.PY3: class IDLib(unittest.TestCase): def test_nodeid_b2a(self): - self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) + self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32) class MyList(list): @@ -85,10 +95,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): basedir = "util/FileUtil/test_write_atomically" fileutil.make_dirs(basedir) fn = os.path.join(basedir, "here") - fileutil.write_atomically(fn, "one") - self.failUnlessEqual(fileutil.read(fn), "one") - fileutil.write_atomically(fn, "two", mode="") # non-binary - self.failUnlessEqual(fileutil.read(fn), "two") + fileutil.write_atomically(fn, b"one", "b") + self.failUnlessEqual(fileutil.read(fn), b"one") + fileutil.write_atomically(fn, u"two", mode="") # non-binary + self.failUnlessEqual(fileutil.read(fn), b"two") def test_rename(self): basedir = "util/FileUtil/test_rename" @@ -111,20 +121,20 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path) # when only dest exists - fileutil.write(dest_path, "dest") + fileutil.write(dest_path, b"dest") self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path) - self.failUnlessEqual(fileutil.read(dest_path), "dest") + self.failUnlessEqual(fileutil.read(dest_path), b"dest") # when both exist - fileutil.write(source_path, "source") + fileutil.write(source_path, b"source") self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path) - self.failUnlessEqual(fileutil.read(source_path), "source") - self.failUnlessEqual(fileutil.read(dest_path), "dest") + self.failUnlessEqual(fileutil.read(source_path), b"source") + self.failUnlessEqual(fileutil.read(dest_path), b"dest") # when only source exists os.remove(dest_path) fileutil.rename_no_overwrite(source_path, dest_path) - self.failUnlessEqual(fileutil.read(dest_path), "source") + self.failUnlessEqual(fileutil.read(dest_path), b"source") self.failIf(os.path.exists(source_path)) def test_replace_file(self): @@ -138,21 +148,21 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path) # when only replaced exists - fileutil.write(replaced_path, "foo") + fileutil.write(replaced_path, b"foo") self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path) - self.failUnlessEqual(fileutil.read(replaced_path), "foo") + self.failUnlessEqual(fileutil.read(replaced_path), b"foo") # when both replaced and replacement exist - fileutil.write(replacement_path, "bar") + fileutil.write(replacement_path, b"bar") fileutil.replace_file(replaced_path, replacement_path) - self.failUnlessEqual(fileutil.read(replaced_path), "bar") + self.failUnlessEqual(fileutil.read(replaced_path), b"bar") self.failIf(os.path.exists(replacement_path)) # when only replacement exists os.remove(replaced_path) - fileutil.write(replacement_path, "bar") + fileutil.write(replacement_path, b"bar") fileutil.replace_file(replaced_path, replacement_path) - self.failUnlessEqual(fileutil.read(replaced_path), "bar") + self.failUnlessEqual(fileutil.read(replaced_path), b"bar") self.failIf(os.path.exists(replacement_path)) def test_du(self): @@ -170,13 +180,15 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessEqual(10+11+12+13, used) def test_abspath_expanduser_unicode(self): - self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring") + self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring") - saved_cwd = os.path.normpath(os.getcwdu()) + saved_cwd = os.path.normpath(os.getcwd()) + if PY2: + saved_cwd = saved_cwd.decode("utf8") abspath_cwd = fileutil.abspath_expanduser_unicode(u".") abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False) - self.failUnless(isinstance(saved_cwd, unicode), saved_cwd) - self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd) + self.failUnless(isinstance(saved_cwd, str), saved_cwd) + self.failUnless(isinstance(abspath_cwd, str), abspath_cwd) if sys.platform == "win32": self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd)) else: @@ -237,10 +249,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): os.chdir(cwd) for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'): uabspath = fileutil.abspath_expanduser_unicode(upath) - self.failUnless(isinstance(uabspath, unicode), uabspath) + self.failUnless(isinstance(uabspath, str), uabspath) uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False) - self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong) + self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong) finally: os.chdir(saved_cwd) @@ -293,9 +305,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): fileutil.remove(long_path) self.addCleanup(_cleanup) - fileutil.write(long_path, "test") + fileutil.write(long_path, b"test") self.failUnless(os.path.exists(long_path)) - self.failUnlessEqual(fileutil.read(long_path), "test") + self.failUnlessEqual(fileutil.read(long_path), b"test") _cleanup() self.failIf(os.path.exists(long_path)) @@ -353,7 +365,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): # create a file f = os.path.join(basedir, "1.txt") - fileutil.write(f, "a"*10) + fileutil.write(f, b"a"*10) fileinfo = fileutil.get_pathinfo(f) self.failUnlessTrue(fileinfo.isfile) self.failUnlessTrue(fileinfo.exists) @@ -381,7 +393,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): fileutil.make_dirs(basedir) f = os.path.join(basedir, "1.txt") - fileutil.write(f, "a"*10) + fileutil.write(f, b"a"*10) # create a symlink pointing to 1.txt slname = os.path.join(basedir, "linkto1.txt") @@ -394,7 +406,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): def test_encrypted_tempfile(self): f = EncryptedTemporaryFile() - f.write("foobar") + f.write(b"foobar") f.close() @@ -409,7 +421,7 @@ class PollMixinTests(unittest.TestCase): def test_PollMixin_False_then_True(self): i = iter([False, True]) - d = self.pm.poll(check_f=i.next, + d = self.pm.poll(check_f=lambda: next(i), pollinterval=0.1) return d @@ -454,6 +466,6 @@ class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) back = yamlutil.safe_load(data) - self.failUnlessEqual(type(back[0]), unicode) - self.failUnlessEqual(type(back[1]), unicode) - self.failUnlessEqual(type(back[2]), unicode) + self.assertIsInstance(back[0], str) + self.assertIsInstance(back[1], str) + self.assertIsInstance(back[2], str) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2838fe8ea..a1750ca18 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -30,6 +30,7 @@ PORTED_MODULES = [ "allmydata.util.base32", "allmydata.util.base62", "allmydata.util.deferredutil", + "allmydata.util.fileutil", "allmydata.util.dictutil", "allmydata.util.gcutil", "allmydata.util.hashutil", @@ -67,10 +68,10 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_spans", "allmydata.test.test_statistics", "allmydata.test.test_time_format", + "allmydata.test.test_util", "allmydata.test.test_version", ] - if __name__ == '__main__': from subprocess import check_call check_call(["trial"] + PORTED_TEST_MODULES) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index b31fd1205..a6a2b97a5 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -140,7 +140,7 @@ def to_str(s): return s.encode('utf-8') def from_utf8_or_none(s): - precondition(isinstance(s, (NoneType, str)), s) + precondition(isinstance(s, str) or s is None, s) if s is None: return s return s.decode('utf-8') diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 36946ff02..693cd1d63 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -1,9 +1,19 @@ -from __future__ import print_function - """ +Ported to Python3. + Futz with files like a pro. """ +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # open is not here because we want to use native strings on Py2 + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys, os, stat, tempfile, time, binascii import six from collections import namedtuple @@ -253,6 +263,9 @@ def move_into_place(source, dest): os.rename(source, dest) def write_atomically(target, contents, mode="b"): + assert ( + isinstance(contents, bytes) and "b" in mode or + isinstance(contents, str) and "t" in mode or mode == ""), (type(contents), mode) with open(target+".tmp", "w"+mode) as f: f.write(contents) move_into_place(target+".tmp", target) @@ -277,7 +290,7 @@ def put_file(path, inf): outf.write(data) def precondition_abspath(path): - if not isinstance(path, unicode): + if not isinstance(path, str): raise AssertionError("an abspath must be a Unicode string") if sys.platform == "win32": @@ -309,7 +322,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): abspath_expanduser_unicode. On Windows, the result will be a long path unless long_path is given as False. """ - if not isinstance(path, unicode): + if not isinstance(path, str): raise AssertionError("paths must be Unicode strings") if base is not None and long_path: precondition_abspath(base) @@ -330,7 +343,10 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): if not os.path.isabs(path): if base is None: - path = os.path.join(os.getcwdu(), path) + cwd = os.getcwd() + if PY2: + cwd = cwd.decode('utf8') + path = os.path.join(cwd, path) else: path = os.path.join(base, path) @@ -415,7 +431,7 @@ ERROR_ENVVAR_NOT_FOUND = 203 def windows_getenv(name): # Based on , # with improved error handling. Returns None if there is no enivronment variable of the given name. - if not isinstance(name, unicode): + if not isinstance(name, str): raise AssertionError("name must be Unicode") n = GetEnvironmentVariableW(name, None, 0) diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index 3da5da70a..a517751d6 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -20,10 +20,9 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from functools import reduce - from allmydata.util.mathutil import round_sigfigs import math +from functools import reduce import sys def pr_file_loss(p_list, k):