ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-31 16:36:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #755 from tahoe-lafs/3356.iputil-python-3

Browse Source 
Port iputil to Python 3

Fixes ticket:3356
This commit is contained in:
Itamar Turner-Trauring 2020-07-28 14:18:55 -04:00 committed by GitHub
commit afd6ab3e65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 155 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
misc/python3/ratchet-passing
View File

@ -44,6 +44,14 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d
allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated
allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare
allmydata.test.test_humanreadable.HumanReadable.test_repr
allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for
allmydata.test.test_iputil.ListAddresses.test_list_async
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route
allmydata.test.test_iputil.ListenOnUsed.test_random_port
allmydata.test.test_iputil.ListenOnUsed.test_specific_port
allmydata.test.test_netstring.Netstring.test_encode
allmydata.test.test_netstring.Netstring.test_extra
allmydata.test.test_netstring.Netstring.test_nested

0
newsfragments/3356.minor Normal file
View File

1
nix/tahoe-lafs.nix
View File

@ -59,6 +59,7 @@ python.pkgs.buildPythonPackage rec {
fixtures
beautifulsoup4
html5lib
tenacity
];
checkPhase = ''

1
setup.py
View File

@ -390,6 +390,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
"beautifulsoup4",
"html5lib",
"junitxml",
"tenacity",
] + tor_requires + i2p_requires,
"tor": tor_requires,
"i2p": i2p_requires,

25
src/allmydata/test/common_py3.py
View File

@ -15,6 +15,9 @@ if PY2:
import os
import time
import signal
from twisted.internet import reactor
class TimezoneMixin(object):
@ -40,3 +43,25 @@ class TimezoneMixin(object):
def have_working_tzset(self):
return hasattr(time, 'tzset')
class SignalMixin(object):
# This class is necessary for any code which wants to use Processes
# outside the usual reactor.run() environment. It is copied from
# Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
# something rather different.
sigchldHandler = None
def setUp(self):
# make sure SIGCHLD handler is installed, as it should be on
# reactor.run(). problem is reactor may not have been run when this
# test runs.
if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
self.sigchldHandler = signal.signal(signal.SIGCHLD,
reactor._handleSigchld)
return super(SignalMixin, self).setUp()
def tearDown(self):
if self.sigchldHandler:
signal.signal(signal.SIGCHLD, self.sigchldHandler)
return super(SignalMixin, self).tearDown()

25
src/allmydata/test/common_util.py
View File

@ -1,6 +1,6 @@
from __future__ import print_function
import os, signal
import os
from random import randrange
from six.moves import StringIO
@ -12,6 +12,8 @@ from ..util.assertutil import precondition
from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
get_io_encoding)
from ..scripts import runner
from .common_py3 import SignalMixin
def skip_if_cannot_represent_filename(u):
precondition(isinstance(u, unicode))
@ -88,27 +90,6 @@ class ReallyEqualMixin(object):
self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg))
class SignalMixin(object):
# This class is necessary for any code which wants to use Processes
# outside the usual reactor.run() environment. It is copied from
# Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
# something rather different.
sigchldHandler = None
def setUp(self):
# make sure SIGCHLD handler is installed, as it should be on
# reactor.run(). problem is reactor may not have been run when this
# test runs.
if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
self.sigchldHandler = signal.signal(signal.SIGCHLD,
reactor._handleSigchld)
return super(SignalMixin, self).setUp()
def tearDown(self):
if self.sigchldHandler:
signal.signal(signal.SIGCHLD, self.sigchldHandler)
return super(SignalMixin, self).tearDown()
class StallMixin(object):
def stall(self, res=None, delay=1):
d = defer.Deferred()

78
src/allmydata/test/test_iputil.py
View File

@ -1,16 +1,36 @@
"""
Tests for allmydata.util.iputil.
import re, errno, subprocess, os
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, native_str
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401
import re, errno, subprocess, os, socket
from twisted.trial import unittest
from tenacity import retry, stop_after_attempt
from foolscap.api import Tub
from allmydata.util import iputil
import allmydata.test.common_util as testutil
import allmydata.test.common_py3 as testutil
from allmydata.util.namespace import Namespace
DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$")
DOTTED_QUAD_RE=re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$")
MOCK_IPADDR_OUTPUT = """\
# Mock output from subprocesses should be bytes, that's what happens on both
# Python 2 and Python 3:
MOCK_IPADDR_OUTPUT = b"""\
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN \n\
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
@ -28,7 +48,7 @@ MOCK_IPADDR_OUTPUT = """\
valid_lft forever preferred_lft forever
"""
MOCK_IFCONFIG_OUTPUT = """\
MOCK_IFCONFIG_OUTPUT = b"""\
eth1 Link encap:Ethernet HWaddr d4:3d:7e:01:b4:3e \n\
inet addr:192.168.0.6 Bcast:192.168.0.255 Mask:255.255.255.0
inet6 addr: fe80::d63d:7eff:fe01:b43e/64 Scope:Link
@ -59,7 +79,7 @@ wlan0 Link encap:Ethernet HWaddr 90:f6:52:27:15:0a \n\
"""
# This is actually from a VirtualBox VM running XP.
MOCK_ROUTE_OUTPUT = """\
MOCK_ROUTE_OUTPUT = b"""\
===========================================================================
Interface List
0x1 ........................... MS TCP Loopback interface
@ -98,6 +118,11 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase):
def test_get_local_ip_for(self):
addr = iputil.get_local_ip_for('127.0.0.1')
self.failUnless(DOTTED_QUAD_RE.match(addr))
# Bytes can be taken as input:
bytes_addr = iputil.get_local_ip_for(b'127.0.0.1')
self.assertEqual(addr, bytes_addr)
# The output is a native string:
self.assertIsInstance(addr, native_str)
def test_list_async(self):
d = iputil.get_local_addresses_async()
@ -162,3 +187,44 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase):
def test_list_async_mock_cygwin(self):
self.patch(iputil, 'platform', "cygwin")
return self._test_list_async_mock(None, None, CYGWIN_TEST_ADDRESSES)
class ListenOnUsed(unittest.TestCase):
"""Tests for listenOnUnused."""
def create_tub(self, basedir):
os.makedirs(basedir)
tubfile = os.path.join(basedir, "tub.pem")
tub = Tub(certFile=tubfile)
tub.setOption("expose-remote-exception-types", False)
tub.startService()
self.addCleanup(tub.stopService)
return tub
@retry(stop=stop_after_attempt(7))
def test_random_port(self):
"""A random port is selected if none is given."""
tub = self.create_tub("utils/ListenOnUsed/test_randomport")
self.assertEqual(len(tub.getListeners()), 0)
portnum = iputil.listenOnUnused(tub)
# We can connect to this port:
s = socket.socket()
s.connect(("127.0.0.1", portnum))
s.close()
self.assertEqual(len(tub.getListeners()), 1)
# Listen on another port:
tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2")
portnum2 = iputil.listenOnUnused(tub2)
self.assertNotEqual(portnum, portnum2)
@retry(stop=stop_after_attempt(7))
def test_specific_port(self):
"""The given port is used."""
tub = self.create_tub("utils/ListenOnUsed/test_givenport")
s = socket.socket()
s.bind(("127.0.0.1", 0))
port = s.getsockname()[1]
s.close()
port2 = iputil.listenOnUnused(tub, port)
self.assertEqual(port, port2)

2
src/allmydata/util/_python3.py
View File

@ -23,6 +23,7 @@ PORTED_MODULES = [
"allmydata.util.deferredutil",
"allmydata.util.hashutil",
"allmydata.util.humanreadable",
"allmydata.util.iputil",
"allmydata.util.mathutil",
"allmydata.util.namespace",
"allmydata.util.netstring",
@ -43,6 +44,7 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_hashtree",
"allmydata.test.test_hashutil",
"allmydata.test.test_humanreadable",
"allmydata.test.test_iputil",
"allmydata.test.test_netstring",
"allmydata.test.test_observer",
"allmydata.test.test_pipeline",

3
src/allmydata/util/encodingutil.py
View File

@ -4,7 +4,6 @@ unicode and back.
"""
import sys, os, re, locale
from types import NoneType
from allmydata.util.assertutil import precondition, _assert
from twisted.python import usage
@ -12,6 +11,8 @@ from twisted.python.filepath import FilePath
from allmydata.util import log
from allmydata.util.fileutil import abspath_expanduser_unicode
NoneType = type(None)
def canonical_encoding(encoding):
if encoding is None:

4
src/allmydata/util/fileutil.py
View File

@ -4,7 +4,7 @@ from __future__ import print_function
Futz with files like a pro.
"""
import sys, exceptions, os, stat, tempfile, time, binascii
import sys, os, stat, tempfile, time, binascii
import six
from collections import namedtuple
from errno import ENOENT
@ -190,7 +190,7 @@ def make_dirs(dirname, mode=0o777):
if not os.path.isdir(dirname):
if tx:
raise tx
raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning...
raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning...
def rm_dir(dirname):
"""

55
src/allmydata/util/iputil.py
View File

@ -1,4 +1,18 @@
# from the Python Standard Library
"""
Utilities for getting IP addresses.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, native_str
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401
import os, re, socket, subprocess, errno
from sys import platform
@ -88,13 +102,18 @@ except ImportError:
increase_rlimits = _increase_rlimits
def get_local_addresses_sync():
return _synchronously_find_addresses_via_config()
"""
Return a list of IPv4 addresses (as dotted-quad native strings) that are
currently configured on this host, sorted in descending order of how likely
we think they are to work.
"""
return [native_str(a) for a in _synchronously_find_addresses_via_config()]
def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
"""
Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad
strings) that are currently configured on this host, sorted in descending
order of how likely we think they are to work.
native strings) that are currently configured on this host, sorted in
descending order of how likely we think they are to work.
@param target: we want to learn an IP address they could try using to
connect to us; The default value is fine, but it might help if you
@ -117,13 +136,13 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
addresses.append(addr)
return addresses
d.addCallback(_collect)
d.addCallback(lambda addresses: [native_str(s) for s in addresses])
return d
def get_local_ip_for(target):
"""Find out what our IP address is for use by a given target.
@return: the IP address as a dotted-quad string which could be used by
@return: the IP address as a dotted-quad native string which could be used
to connect to us. It might work for them, it might not. If
there is no suitable address (perhaps we don't currently have an
externally-visible interface), this will return None.
@ -162,7 +181,7 @@ def get_local_ip_for(target):
except (socket.error, CannotListenError):
# no route to that host
localip = None
return localip
return native_str(localip)
# Wow, I'm really amazed at home much mileage we've gotten out of calling
@ -171,11 +190,11 @@ def get_local_ip_for(target):
# ... thus wrote Greg Smith in time immemorial...
# Also, the Win32 APIs for this are really klunky and error-prone. --Daira
_win32_re = re.compile(r'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S)
_win32_re = re.compile(br'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S)
_win32_commands = (('route.exe', ('print',), _win32_re),)
# These work in most Unices.
_addr_re = re.compile(r'^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S)
_addr_re = re.compile(br'^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S)
_unix_commands = (('/bin/ip', ('addr',), _addr_re),
('/sbin/ip', ('addr',), _addr_re),
('/sbin/ifconfig', ('-a',), _addr_re),
@ -209,10 +228,13 @@ def _synchronously_find_addresses_via_config():
else:
exes_to_try = which(pathtotool)
subprocess_error = getattr(
subprocess, "SubprocessError", subprocess.CalledProcessError
)
for exe in exes_to_try:
try:
addresses = _query(exe, args, regex)
except Exception:
except (IOError, OSError, ValueError, subprocess_error):
addresses = []
if addresses:
return addresses
@ -222,9 +244,9 @@ def _synchronously_find_addresses_via_config():
def _query(path, args, regex):
if not os.path.isfile(path):
return []
env = {'LANG': 'en_US.UTF-8'}
env = {native_str('LANG'): native_str('en_US.UTF-8')}
TRIES = 5
for trial in xrange(TRIES):
for trial in range(TRIES):
try:
p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
(output, err) = p.communicate()
@ -235,13 +257,13 @@ def _query(path, args, regex):
raise
addresses = []
outputsplit = output.split('\n')
outputsplit = output.split(b'\n')
for outline in outputsplit:
m = regex.match(outline)
if m:
addr = m.group('address')
if addr not in addresses:
addresses.append(addr)
addresses.append(addr.decode("utf-8"))
return addresses
@ -304,7 +326,7 @@ def _foolscapEndpointForPortNumber(portnum):
# approach is error prone for the reasons described on
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2787
portnum = allocate_tcp_port()
return (portnum, "tcp:%d" % (portnum,))
return (portnum, native_str("tcp:%d" % (portnum,)))
@implementer(IStreamServerEndpoint)
@ -353,7 +375,7 @@ def listenOnUnused(tub, portnum=None):
"""
portnum, endpoint = _foolscapEndpointForPortNumber(portnum)
tub.listenOn(endpoint)
tub.setLocation("localhost:%d" % (portnum,))
tub.setLocation(native_str("localhost:%d" % (portnum,)))
return portnum
@ -362,4 +384,5 @@ __all__ = ["allocate_tcp_port",
"get_local_addresses_sync",
"get_local_addresses_async",
"get_local_ip_for",
"listenOnUnused",
]