ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-11 23:42:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3358.fileutil-to-python-3' into 3376.encodingutil-python-3

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2020-08-12 10:15:45 -04:00
commit 5dde21f966
8 changed files with 106 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
misc/python3/ratchet-passing
View File

@ -9,6 +9,9 @@ allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year
allmydata.test.test_abbreviate.Abbreviate.test_parse_space
allmydata.test.test_abbreviate.Abbreviate.test_space
allmydata.test.test_abbreviate.Abbreviate.test_time
allmydata.test.test_backupdb.BackupDB.test_basic
allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2
allmydata.test.test_backupdb.BackupDB.test_wrong_version
allmydata.test.test_base32.Base32.test_a2b
allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons
allmydata.test.test_base32.Base32.test_b2a
@ -145,6 +148,32 @@ allmydata.test.test_time_format.TimeFormat.test_format_time_y2038
allmydata.test.test_time_format.TimeFormat.test_iso_utc
allmydata.test.test_time_format.TimeFormat.test_parse_date
allmydata.test.test_time_format.TimeFormat.test_parse_duration
allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode
allmydata.test.test_util.FileUtil.test_create_long_path
allmydata.test.test_util.FileUtil.test_disk_stats
allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative
allmydata.test.test_util.FileUtil.test_du
allmydata.test.test_util.FileUtil.test_encrypted_tempfile
allmydata.test.test_util.FileUtil.test_get_pathinfo
allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink
allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode
allmydata.test.test_util.FileUtil.test_remove_if_possible
allmydata.test.test_util.FileUtil.test_rename
allmydata.test.test_util.FileUtil.test_rename_no_overwrite
allmydata.test.test_util.FileUtil.test_replace_file
allmydata.test.test_util.FileUtil.test_rm_dir
allmydata.test.test_util.FileUtil.test_windows_expanduser_win7
allmydata.test.test_util.FileUtil.test_windows_expanduser_xp
allmydata.test.test_util.FileUtil.test_write_atomically
allmydata.test.test_util.IDLib.test_nodeid_b2a
allmydata.test.test_util.Limiter.test_errors
allmydata.test.test_util.Limiter.test_limiter
allmydata.test.test_util.Log.test_err
allmydata.test.test_util.Math.test_round_sigfigs
allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True
allmydata.test.test_util.PollMixinTests.test_PollMixin_True
allmydata.test.test_util.PollMixinTests.test_timeout
allmydata.test.test_util.YAML.test_convert
allmydata.test.test_version.CheckRequirement.test_cross_check
allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions
allmydata.test.test_version.CheckRequirement.test_extract_openssl_version

0
newsfragments/3358.minor Normal file
View File

2
src/allmydata/client.py
View File

@ -741,7 +741,7 @@ class _Client(node.Node, pollmixin.PollMixin):
private_key_str = self.config.get_or_create_private_config("node.privkey", _make_key)
private_key, public_key = ed25519.signing_keypair_from_string(private_key_str)
public_key_str = ed25519.string_from_verifying_key(public_key)
self.config.write_config_file("node.pubkey", public_key_str + "\n")
self.config.write_config_file("node.pubkey", public_key_str + "\n", "w")
self._node_private_key = private_key
self._node_public_key = public_key

9
src/allmydata/test/common_util.py
View File

@ -10,10 +10,13 @@ from twisted.trial import unittest
from ..util.assertutil import precondition
from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
get_io_encoding)
from ..scripts import runner
from .common_py3 import SignalMixin
get_io_encoding)
from future.utils import PY2
if PY2: # XXX this is a hack that makes some tests pass on Python3, remove
# in the future
from ..scripts import runner
from .common_py3 import SignalMixin
def skip_if_cannot_represent_filename(u):
precondition(isinstance(u, unicode))

78
src/allmydata/test/test_util.py
View File

@ -1,5 +1,15 @@
from __future__ import print_function
"""
Ported to Python3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import six
import os, time, sys
import yaml
@ -19,7 +29,7 @@ if six.PY3:
class IDLib(unittest.TestCase):
def test_nodeid_b2a(self):
self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32)
class MyList(list):
@ -85,10 +95,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
basedir = "util/FileUtil/test_write_atomically"
fileutil.make_dirs(basedir)
fn = os.path.join(basedir, "here")
fileutil.write_atomically(fn, "one")
self.failUnlessEqual(fileutil.read(fn), "one")
fileutil.write_atomically(fn, "two", mode="") # non-binary
self.failUnlessEqual(fileutil.read(fn), "two")
fileutil.write_atomically(fn, b"one", "b")
self.failUnlessEqual(fileutil.read(fn), b"one")
fileutil.write_atomically(fn, u"two", mode="") # non-binary
self.failUnlessEqual(fileutil.read(fn), b"two")
def test_rename(self):
basedir = "util/FileUtil/test_rename"
@ -111,20 +121,20 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
# when only dest exists
fileutil.write(dest_path, "dest")
fileutil.write(dest_path, b"dest")
self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
self.failUnlessEqual(fileutil.read(dest_path), "dest")
self.failUnlessEqual(fileutil.read(dest_path), b"dest")
# when both exist
fileutil.write(source_path, "source")
fileutil.write(source_path, b"source")
self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
self.failUnlessEqual(fileutil.read(source_path), "source")
self.failUnlessEqual(fileutil.read(dest_path), "dest")
self.failUnlessEqual(fileutil.read(source_path), b"source")
self.failUnlessEqual(fileutil.read(dest_path), b"dest")
# when only source exists
os.remove(dest_path)
fileutil.rename_no_overwrite(source_path, dest_path)
self.failUnlessEqual(fileutil.read(dest_path), "source")
self.failUnlessEqual(fileutil.read(dest_path), b"source")
self.failIf(os.path.exists(source_path))
def test_replace_file(self):
@ -138,21 +148,21 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path)
# when only replaced exists
fileutil.write(replaced_path, "foo")
fileutil.write(replaced_path, b"foo")
self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path)
self.failUnlessEqual(fileutil.read(replaced_path), "foo")
self.failUnlessEqual(fileutil.read(replaced_path), b"foo")
# when both replaced and replacement exist
fileutil.write(replacement_path, "bar")
fileutil.write(replacement_path, b"bar")
fileutil.replace_file(replaced_path, replacement_path)
self.failUnlessEqual(fileutil.read(replaced_path), "bar")
self.failUnlessEqual(fileutil.read(replaced_path), b"bar")
self.failIf(os.path.exists(replacement_path))
# when only replacement exists
os.remove(replaced_path)
fileutil.write(replacement_path, "bar")
fileutil.write(replacement_path, b"bar")
fileutil.replace_file(replaced_path, replacement_path)
self.failUnlessEqual(fileutil.read(replaced_path), "bar")
self.failUnlessEqual(fileutil.read(replaced_path), b"bar")
self.failIf(os.path.exists(replacement_path))
def test_du(self):
@ -170,13 +180,15 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
self.failUnlessEqual(10+11+12+13, used)
def test_abspath_expanduser_unicode(self):
self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring")
self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring")
saved_cwd = os.path.normpath(os.getcwdu())
saved_cwd = os.path.normpath(os.getcwd())
if PY2:
saved_cwd = saved_cwd.decode("utf8")
abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False)
self.failUnless(isinstance(saved_cwd, unicode), saved_cwd)
self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd)
self.failUnless(isinstance(saved_cwd, str), saved_cwd)
self.failUnless(isinstance(abspath_cwd, str), abspath_cwd)
if sys.platform == "win32":
self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd))
else:
@ -237,10 +249,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
os.chdir(cwd)
for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'):
uabspath = fileutil.abspath_expanduser_unicode(upath)
self.failUnless(isinstance(uabspath, unicode), uabspath)
self.failUnless(isinstance(uabspath, str), uabspath)
uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False)
self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong)
self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong)
finally:
os.chdir(saved_cwd)
@ -293,9 +305,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
fileutil.remove(long_path)
self.addCleanup(_cleanup)
fileutil.write(long_path, "test")
fileutil.write(long_path, b"test")
self.failUnless(os.path.exists(long_path))
self.failUnlessEqual(fileutil.read(long_path), "test")
self.failUnlessEqual(fileutil.read(long_path), b"test")
_cleanup()
self.failIf(os.path.exists(long_path))
@ -353,7 +365,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
# create a file
f = os.path.join(basedir, "1.txt")
fileutil.write(f, "a"*10)
fileutil.write(f, b"a"*10)
fileinfo = fileutil.get_pathinfo(f)
self.failUnlessTrue(fileinfo.isfile)
self.failUnlessTrue(fileinfo.exists)
@ -381,7 +393,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
fileutil.make_dirs(basedir)
f = os.path.join(basedir, "1.txt")
fileutil.write(f, "a"*10)
fileutil.write(f, b"a"*10)
# create a symlink pointing to 1.txt
slname = os.path.join(basedir, "linkto1.txt")
@ -394,7 +406,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
def test_encrypted_tempfile(self):
f = EncryptedTemporaryFile()
f.write("foobar")
f.write(b"foobar")
f.close()
@ -409,7 +421,7 @@ class PollMixinTests(unittest.TestCase):
def test_PollMixin_False_then_True(self):
i = iter([False, True])
d = self.pm.poll(check_f=i.next,
d = self.pm.poll(check_f=lambda: next(i),
pollinterval=0.1)
return d
@ -454,6 +466,6 @@ class YAML(unittest.TestCase):
def test_convert(self):
data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])
back = yamlutil.safe_load(data)
self.failUnlessEqual(type(back[0]), unicode)
self.failUnlessEqual(type(back[1]), unicode)
self.failUnlessEqual(type(back[2]), unicode)
self.assertIsInstance(back[0], str)
self.assertIsInstance(back[1], str)
self.assertIsInstance(back[2], str)

3
src/allmydata/util/_python3.py
View File

@ -30,6 +30,7 @@ PORTED_MODULES = [
"allmydata.util.base32",
"allmydata.util.base62",
"allmydata.util.deferredutil",
"allmydata.util.fileutil",
"allmydata.util.dictutil",
"allmydata.util.gcutil",
"allmydata.util.hashutil",
@ -67,10 +68,10 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_spans",
"allmydata.test.test_statistics",
"allmydata.test.test_time_format",
"allmydata.test.test_util",
"allmydata.test.test_version",
]
if __name__ == '__main__':
from subprocess import check_call
check_call(["trial"] + PORTED_TEST_MODULES)

28
src/allmydata/util/fileutil.py
View File

@ -1,9 +1,19 @@
from __future__ import print_function
"""
Ported to Python3.
Futz with files like a pro.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# open is not here because we want to use native strings on Py2
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys, os, stat, tempfile, time, binascii
import six
from collections import namedtuple
@ -253,6 +263,9 @@ def move_into_place(source, dest):
os.rename(source, dest)
def write_atomically(target, contents, mode="b"):
assert (
isinstance(contents, bytes) and "b" in mode or
isinstance(contents, str) and "t" in mode or mode == ""), (type(contents), mode)
with open(target+".tmp", "w"+mode) as f:
f.write(contents)
move_into_place(target+".tmp", target)
@ -277,7 +290,7 @@ def put_file(path, inf):
outf.write(data)
def precondition_abspath(path):
if not isinstance(path, unicode):
if not isinstance(path, str):
raise AssertionError("an abspath must be a Unicode string")
if sys.platform == "win32":
@ -309,7 +322,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True):
abspath_expanduser_unicode.
On Windows, the result will be a long path unless long_path is given as False.
"""
if not isinstance(path, unicode):
if not isinstance(path, str):
raise AssertionError("paths must be Unicode strings")
if base is not None and long_path:
precondition_abspath(base)
@ -330,7 +343,10 @@ def abspath_expanduser_unicode(path, base=None, long_path=True):
if not os.path.isabs(path):
if base is None:
path = os.path.join(os.getcwdu(), path)
cwd = os.getcwd()
if PY2:
cwd = cwd.decode('utf8')
path = os.path.join(cwd, path)
else:
path = os.path.join(base, path)
@ -415,7 +431,7 @@ ERROR_ENVVAR_NOT_FOUND = 203
def windows_getenv(name):
# Based on <http://stackoverflow.com/questions/2608200/problems-with-umlauts-in-python-appdata-environvent-variable/2608368#2608368>,
# with improved error handling. Returns None if there is no enivronment variable of the given name.
if not isinstance(name, unicode):
if not isinstance(name, str):
raise AssertionError("name must be Unicode")
n = GetEnvironmentVariableW(name, None, 0)

3
src/allmydata/util/statistics.py
View File

@ -20,10 +20,9 @@ from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from functools import reduce
from allmydata.util.mathutil import round_sigfigs
import math
from functools import reduce
import sys
def pr_file_loss(p_list, k):