From 135ade02b159a8bde5c9083797d10720baf5c2f8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 24 Jul 2020 13:47:56 -0400
Subject: [PATCH 01/21] Start porting.
---
src/allmydata/test/test_iputil.py | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index ebb77c3c8..7f26a53fb 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -10,7 +10,9 @@ from allmydata.util.namespace import Namespace
DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$")
-MOCK_IPADDR_OUTPUT = """\
+# Mock output from subprocesses should be bytes, that's what happens on both
+# Python 2 and Python 3:
+MOCK_IPADDR_OUTPUT = b"""\
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN \n\
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
@@ -28,7 +30,7 @@ MOCK_IPADDR_OUTPUT = """\
valid_lft forever preferred_lft forever
"""
-MOCK_IFCONFIG_OUTPUT = """\
+MOCK_IFCONFIG_OUTPUT = b"""\
eth1 Link encap:Ethernet HWaddr d4:3d:7e:01:b4:3e \n\
inet addr:192.168.0.6 Bcast:192.168.0.255 Mask:255.255.255.0
inet6 addr: fe80::d63d:7eff:fe01:b43e/64 Scope:Link
@@ -59,7 +61,7 @@ wlan0 Link encap:Ethernet HWaddr 90:f6:52:27:15:0a \n\
"""
# This is actually from a VirtualBox VM running XP.
-MOCK_ROUTE_OUTPUT = """\
+MOCK_ROUTE_OUTPUT = b"""\
===========================================================================
Interface List
0x1 ........................... MS TCP Loopback interface
@@ -98,6 +100,11 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase):
def test_get_local_ip_for(self):
addr = iputil.get_local_ip_for('127.0.0.1')
self.failUnless(DOTTED_QUAD_RE.match(addr))
+ # Bytes can be taken as input:
+ bytes_addr = iputil.get_local_ip_for(b'127.0.0.1')
+ self.assertEqual(addr, bytes_addr)
+ # The output is a native string:
+ self.assertIsInstance(addr, str)
def test_list_async(self):
d = iputil.get_local_addresses_async()
From 45a891114b033306dcbb66b3310d5d601970f628 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 24 Jul 2020 13:48:11 -0400
Subject: [PATCH 02/21] Fix some things that prevent Python 3 imports.
---
src/allmydata/util/encodingutil.py | 3 ++-
src/allmydata/util/fileutil.py | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py
index 568267b35..b31fd1205 100644
--- a/src/allmydata/util/encodingutil.py
+++ b/src/allmydata/util/encodingutil.py
@@ -4,7 +4,6 @@ unicode and back.
"""
import sys, os, re, locale
-from types import NoneType
from allmydata.util.assertutil import precondition, _assert
from twisted.python import usage
@@ -12,6 +11,8 @@ from twisted.python.filepath import FilePath
from allmydata.util import log
from allmydata.util.fileutil import abspath_expanduser_unicode
+NoneType = type(None)
+
def canonical_encoding(encoding):
if encoding is None:
diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py
index 269a8a356..36946ff02 100644
--- a/src/allmydata/util/fileutil.py
+++ b/src/allmydata/util/fileutil.py
@@ -4,7 +4,7 @@ from __future__ import print_function
Futz with files like a pro.
"""
-import sys, exceptions, os, stat, tempfile, time, binascii
+import sys, os, stat, tempfile, time, binascii
import six
from collections import namedtuple
from errno import ENOENT
@@ -190,7 +190,7 @@ def make_dirs(dirname, mode=0o777):
if not os.path.isdir(dirname):
if tx:
raise tx
- raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning...
+ raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning...
def rm_dir(dirname):
"""
From bf134019797b2fadc8f92405720b1956e4b2fbc5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 24 Jul 2020 14:58:08 -0400
Subject: [PATCH 03/21] Move SignalMixin to common_py3.py.
---
src/allmydata/test/common_py3.py | 25 +++++++++++++++++++++++++
src/allmydata/test/common_util.py | 23 ++---------------------
src/allmydata/test/test_iputil.py | 2 +-
3 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py
index e6303d2f2..97745e293 100644
--- a/src/allmydata/test/common_py3.py
+++ b/src/allmydata/test/common_py3.py
@@ -15,6 +15,9 @@ if PY2:
import os
import time
+import signal
+
+from twisted.internet import reactor
class TimezoneMixin(object):
@@ -40,3 +43,25 @@ class TimezoneMixin(object):
def have_working_tzset(self):
return hasattr(time, 'tzset')
+
+
+class SignalMixin(object):
+ # This class is necessary for any code which wants to use Processes
+ # outside the usual reactor.run() environment. It is copied from
+ # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
+ # something rather different.
+ sigchldHandler = None
+
+ def setUp(self):
+ # make sure SIGCHLD handler is installed, as it should be on
+ # reactor.run(). problem is reactor may not have been run when this
+ # test runs.
+ if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
+ self.sigchldHandler = signal.signal(signal.SIGCHLD,
+ reactor._handleSigchld)
+ return super(SignalMixin, self).setUp()
+
+ def tearDown(self):
+ if self.sigchldHandler:
+ signal.signal(signal.SIGCHLD, self.sigchldHandler)
+ return super(SignalMixin, self).tearDown()
diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 996052692..83f37a11d 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -12,6 +12,8 @@ from ..util.assertutil import precondition
from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
get_io_encoding)
from ..scripts import runner
+from .common_py3 import SignalMixin
+
def skip_if_cannot_represent_filename(u):
precondition(isinstance(u, unicode))
@@ -88,27 +90,6 @@ class ReallyEqualMixin(object):
self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg))
-class SignalMixin(object):
- # This class is necessary for any code which wants to use Processes
- # outside the usual reactor.run() environment. It is copied from
- # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
- # something rather different.
- sigchldHandler = None
-
- def setUp(self):
- # make sure SIGCHLD handler is installed, as it should be on
- # reactor.run(). problem is reactor may not have been run when this
- # test runs.
- if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
- self.sigchldHandler = signal.signal(signal.SIGCHLD,
- reactor._handleSigchld)
- return super(SignalMixin, self).setUp()
-
- def tearDown(self):
- if self.sigchldHandler:
- signal.signal(signal.SIGCHLD, self.sigchldHandler)
- return super(SignalMixin, self).tearDown()
-
class StallMixin(object):
def stall(self, res=None, delay=1):
d = defer.Deferred()
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index 7f26a53fb..0c0992430 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -4,7 +4,7 @@ import re, errno, subprocess, os
from twisted.trial import unittest
from allmydata.util import iputil
-import allmydata.test.common_util as testutil
+import allmydata.test.common_py3 as testutil
from allmydata.util.namespace import Namespace
From 1f34e6298790c6ff9db53743a3a0b441ffaa1313 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 11:14:01 -0400
Subject: [PATCH 04/21] Port test module to Python 3.
---
src/allmydata/test/test_iputil.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index 0c0992430..1113e3540 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -1,3 +1,11 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, native_str
+if PY2:
+ from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401
import re, errno, subprocess, os
@@ -8,7 +16,7 @@ import allmydata.test.common_py3 as testutil
from allmydata.util.namespace import Namespace
-DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$")
+DOTTED_QUAD_RE=re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$")
# Mock output from subprocesses should be bytes, that's what happens on both
# Python 2 and Python 3:
@@ -104,7 +112,7 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase):
bytes_addr = iputil.get_local_ip_for(b'127.0.0.1')
self.assertEqual(addr, bytes_addr)
# The output is a native string:
- self.assertIsInstance(addr, str)
+ self.assertIsInstance(addr, native_str)
def test_list_async(self):
d = iputil.get_local_addresses_async()
From fce7221481fd582a904bdb16ea3b81b57dfac970 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 11:42:20 -0400
Subject: [PATCH 05/21] Some steps to manually port to Python 3.
---
src/allmydata/util/iputil.py | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 57a7e790a..71dded28f 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -171,11 +171,11 @@ def get_local_ip_for(target):
# ... thus wrote Greg Smith in time immemorial...
# Also, the Win32 APIs for this are really klunky and error-prone. --Daira
-_win32_re = re.compile(r'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S)
+_win32_re = re.compile(br'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S)
_win32_commands = (('route.exe', ('print',), _win32_re),)
# These work in most Unices.
-_addr_re = re.compile(r'^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S)
+_addr_re = re.compile(br'^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S)
_unix_commands = (('/bin/ip', ('addr',), _addr_re),
('/sbin/ip', ('addr',), _addr_re),
('/sbin/ifconfig', ('-a',), _addr_re),
@@ -209,10 +209,13 @@ def _synchronously_find_addresses_via_config():
else:
exes_to_try = which(pathtotool)
+ subprocess_error = getattr(
+ subprocess, "SubprocessError", subprocess.CalledProcessError
+ )
for exe in exes_to_try:
try:
addresses = _query(exe, args, regex)
- except Exception:
+ except (IOError, OSError, ValueError, subprocess_error):
addresses = []
if addresses:
return addresses
@@ -224,7 +227,7 @@ def _query(path, args, regex):
return []
env = {'LANG': 'en_US.UTF-8'}
TRIES = 5
- for trial in xrange(TRIES):
+ for trial in range(TRIES):
try:
p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
(output, err) = p.communicate()
@@ -235,13 +238,13 @@ def _query(path, args, regex):
raise
addresses = []
- outputsplit = output.split('\n')
+ outputsplit = output.split(b'\n')
for outline in outputsplit:
m = regex.match(outline)
if m:
addr = m.group('address')
if addr not in addresses:
- addresses.append(addr)
+ addresses.append(addr.decode("utf-8"))
return addresses
From 436c2e77e422370c81ccdb650816ca8cd48ecca0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 11:44:58 -0400
Subject: [PATCH 06/21] Automated port to Python 3.
---
src/allmydata/util/iputil.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 71dded28f..4623dbba6 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -1,4 +1,12 @@
-# from the Python Standard Library
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, native_str
+if PY2:
+ from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401
+
import os, re, socket, subprocess, errno
from sys import platform
From 9919d2c9a7fcb2887541d887d94daf2db8b08275 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 11:46:03 -0400
Subject: [PATCH 07/21] Record port status.
---
src/allmydata/test/test_iputil.py | 6 ++++++
src/allmydata/util/_python3.py | 2 ++
src/allmydata/util/iputil.py | 6 ++++++
3 files changed, 14 insertions(+)
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index 1113e3540..3fac776c2 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -1,3 +1,9 @@
+"""
+Tests for allmydata.util.iputil.
+
+Ported to Python 3.
+"""
+
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index ff6916a72..d939d08b3 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -23,6 +23,7 @@ PORTED_MODULES = [
"allmydata.util.deferredutil",
"allmydata.util.hashutil",
"allmydata.util.humanreadable",
+ "allmydata.util.iputil",
"allmydata.util.mathutil",
"allmydata.util.namespace",
"allmydata.util.netstring",
@@ -42,6 +43,7 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_hashtree",
"allmydata.test.test_hashutil",
"allmydata.test.test_humanreadable",
+ "allmydata.test.test_iputil",
"allmydata.test.test_netstring",
"allmydata.test.test_observer",
"allmydata.test.test_pipeline",
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 4623dbba6..947430e35 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -1,3 +1,9 @@
+"""
+Utilities for getting IP addresses.
+
+Ported to Python 3.
+"""
+
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
From f9bda5bbd3975746dda392b636b80eb372d56370 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 11:46:22 -0400
Subject: [PATCH 08/21] News file.
---
newsfragments/3356.minor | 0
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 newsfragments/3356.minor
diff --git a/newsfragments/3356.minor b/newsfragments/3356.minor
new file mode 100644
index 000000000..e69de29bb
From 817355d17e98fffe6bb5ee9a6e8cc03e6081de93 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 13:06:41 -0400
Subject: [PATCH 09/21] Minimal testing for listenOnUnused.
---
src/allmydata/test/test_iputil.py | 33 ++++++++++++++++++++++++++++++-
src/allmydata/util/iputil.py | 1 +
2 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index 3fac776c2..a058da299 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -13,10 +13,12 @@ from future.utils import PY2, native_str
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401
-import re, errno, subprocess, os
+import re, errno, subprocess, os, socket
from twisted.trial import unittest
+from foolscap.api import Tub
+
from allmydata.util import iputil
import allmydata.test.common_py3 as testutil
from allmydata.util.namespace import Namespace
@@ -183,3 +185,32 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase):
def test_list_async_mock_cygwin(self):
self.patch(iputil, 'platform', "cygwin")
return self._test_list_async_mock(None, None, CYGWIN_TEST_ADDRESSES)
+
+
+class ListenOnUsed(unittest.TestCase):
+ """Tests for listenOnUnused."""
+
+ def create_tub(self, basedir):
+ os.makedirs(basedir)
+ tubfile = os.path.join(basedir, "tub.pem")
+ tub = Tub(certFile=tubfile)
+ tub.setOption("expose-remote-exception-types", False)
+ tub.startService()
+ self.addCleanup(tub.stopService)
+ return tub
+
+ def test_random_port(self):
+ """A random port is selected if none is given."""
+ tub = self.create_tub("utils/ListenOnUsed/test_randomport")
+ self.assertEqual(len(tub.getListeners()), 0)
+ portnum = iputil.listenOnUnused(tub)
+ # We can connect to this port:
+ s = socket.socket()
+ s.connect(("127.0.0.1", portnum))
+ s.close()
+ self.assertEqual(len(tub.getListeners()), 1)
+
+ # Listen on another port:
+ tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2")
+ portnum2 = iputil.listenOnUnused(tub2)
+ self.assertNotEqual(portnum, portnum2)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 947430e35..6a9639280 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -379,4 +379,5 @@ __all__ = ["allocate_tcp_port",
"get_local_addresses_sync",
"get_local_addresses_async",
"get_local_ip_for",
+ "listenOnUnused",
]
From 0071c1a48d0fa7ade416216456ff9f1c5446ef14 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 13:08:25 -0400
Subject: [PATCH 10/21] More passing tests.
---
misc/python3/ratchet-passing | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index 6832b7fa7..bbe594998 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -44,6 +44,13 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d
allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated
allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare
allmydata.test.test_humanreadable.HumanReadable.test_repr
+allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for
+allmydata.test.test_iputil.ListAddresses.test_list_async
+allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin
+allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig
+allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr
+allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route
+allmydata.test.test_iputil.ListenOnUsed.test_random_port
allmydata.test.test_netstring.Netstring.test_encode
allmydata.test.test_netstring.Netstring.test_extra
allmydata.test.test_netstring.Netstring.test_nested
From d91b4f0e40ef6359213c73927e0102b8ce338f53 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 15:12:09 -0400
Subject: [PATCH 11/21] Fix some tests and flake issues.
---
src/allmydata/test/common_util.py | 2 +-
src/allmydata/util/iputil.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 83f37a11d..261d39af3 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -1,6 +1,6 @@
from __future__ import print_function
-import os, signal
+import os
from random import randrange
from six.moves import StringIO
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 6a9639280..940f215db 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -370,7 +370,7 @@ def listenOnUnused(tub, portnum=None):
"""
portnum, endpoint = _foolscapEndpointForPortNumber(portnum)
tub.listenOn(endpoint)
- tub.setLocation("localhost:%d" % (portnum,))
+ tub.setLocation(native_str("localhost:%d" % (portnum,)))
return portnum
From ce8a57580846d659eb680c1f9efa429cabe3402d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 15:16:25 -0400
Subject: [PATCH 12/21] Test for specific ports.
---
src/allmydata/test/test_iputil.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index a058da299..42677af2f 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -214,3 +214,13 @@ class ListenOnUsed(unittest.TestCase):
tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2")
portnum2 = iputil.listenOnUnused(tub2)
self.assertNotEqual(portnum, portnum2)
+
+ def test_specific_port(self):
+ """The given port is used."""
+ tub = self.create_tub("utils/ListenOnUsed/test_givenport")
+ s = socket.socket()
+ s.bind(("127.0.0.1", 0))
+ port = s.getsockname()[1]
+ s.close()
+ port2 = iputil.listenOnUnused(tub, port)
+ self.assertEqual(port, port2)
From 74fe9ccf56a36902ff884bc298b4a85d4d4fc545 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 15:16:34 -0400
Subject: [PATCH 13/21] Foolscap expects a native string.
---
src/allmydata/util/iputil.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 940f215db..af982e908 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -321,7 +321,7 @@ def _foolscapEndpointForPortNumber(portnum):
# approach is error prone for the reasons described on
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2787
portnum = allocate_tcp_port()
- return (portnum, "tcp:%d" % (portnum,))
+ return (portnum, native_str("tcp:%d" % (portnum,)))
@implementer(IStreamServerEndpoint)
From d8bf811b92acaef6510e3e7503fb8424c2f57d5b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 15:20:35 -0400
Subject: [PATCH 14/21] Native string all the things.
---
src/allmydata/util/iputil.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index af982e908..921c481ff 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -102,7 +102,7 @@ except ImportError:
increase_rlimits = _increase_rlimits
def get_local_addresses_sync():
- return _synchronously_find_addresses_via_config()
+ return [native_str(a) for a in _synchronously_find_addresses_via_config()]
def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
"""
@@ -131,7 +131,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
addresses.append(addr)
return addresses
d.addCallback(_collect)
-
+ d.addCallback(lambda addresses: [native_str(s) for s in addresses])
return d
def get_local_ip_for(target):
@@ -176,7 +176,7 @@ def get_local_ip_for(target):
except (socket.error, CannotListenError):
# no route to that host
localip = None
- return localip
+ return native_str(localip)
# Wow, I'm really amazed at home much mileage we've gotten out of calling
From 94d489ff46883279f2994c03ca316f0c82f490d5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 16:01:12 -0400
Subject: [PATCH 15/21] Fix trailing whitespace.
---
src/allmydata/util/iputil.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 921c481ff..ac5367cfc 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -131,7 +131,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
addresses.append(addr)
return addresses
d.addCallback(_collect)
- d.addCallback(lambda addresses: [native_str(s) for s in addresses])
+ d.addCallback(lambda addresses: [native_str(s) for s in addresses])
return d
def get_local_ip_for(target):
From cbbe260a4eafef224f3bf06f599030a463f0c7c0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 27 Jul 2020 16:19:36 -0400
Subject: [PATCH 16/21] Windows doesn't like Unicode strings in os.environ in
Python 2.7.
---
src/allmydata/util/iputil.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index ac5367cfc..36f4bb79b 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -239,7 +239,7 @@ def _synchronously_find_addresses_via_config():
def _query(path, args, regex):
if not os.path.isfile(path):
return []
- env = {'LANG': 'en_US.UTF-8'}
+ env = {native_str('LANG'): native_str('en_US.UTF-8')}
TRIES = 5
for trial in range(TRIES):
try:
From 4fdcd06fbdb4ce1e2dc7d3ea10ccccb70ca5eea2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 28 Jul 2020 10:48:17 -0400
Subject: [PATCH 17/21] Another passing test for the ratchet.
---
misc/python3/ratchet-passing | 1 +
1 file changed, 1 insertion(+)
diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bbe594998..68f1a968b 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -51,6 +51,7 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route
allmydata.test.test_iputil.ListenOnUsed.test_random_port
+allmydata.test.test_iputil.ListenOnUsed.test_specific_port
allmydata.test.test_netstring.Netstring.test_encode
allmydata.test.test_netstring.Netstring.test_extra
allmydata.test.test_netstring.Netstring.test_nested
From 4dae5d867c8dd44d9ceb68b6732030f00b391c6b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 28 Jul 2020 11:08:38 -0400
Subject: [PATCH 18/21] Add retry logic, listenOnUnused is potentially flaky.
---
setup.py | 1 +
src/allmydata/test/test_iputil.py | 4 ++++
2 files changed, 5 insertions(+)
diff --git a/setup.py b/setup.py
index 9bb385cb4..a35023b8b 100644
--- a/setup.py
+++ b/setup.py
@@ -390,6 +390,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
"beautifulsoup4",
"html5lib",
"junitxml",
+ "tenacity",
] + tor_requires + i2p_requires,
"tor": tor_requires,
"i2p": i2p_requires,
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index 42677af2f..c6caed7f9 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -17,6 +17,8 @@ import re, errno, subprocess, os, socket
from twisted.trial import unittest
+from tenacity import retry, stop_after_attempt
+
from foolscap.api import Tub
from allmydata.util import iputil
@@ -199,6 +201,7 @@ class ListenOnUsed(unittest.TestCase):
self.addCleanup(tub.stopService)
return tub
+ @retry(stop=stop_after_attempt(7))
def test_random_port(self):
"""A random port is selected if none is given."""
tub = self.create_tub("utils/ListenOnUsed/test_randomport")
@@ -215,6 +218,7 @@ class ListenOnUsed(unittest.TestCase):
portnum2 = iputil.listenOnUnused(tub2)
self.assertNotEqual(portnum, portnum2)
+ @retry(stop=stop_after_attempt(7))
def test_specific_port(self):
"""The given port is used."""
tub = self.create_tub("utils/ListenOnUsed/test_givenport")
From cc494e3d3978dd79eb67b3397bc8abcb29b7d1c4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 28 Jul 2020 11:11:05 -0400
Subject: [PATCH 19/21] Document that API returns native strings.
---
src/allmydata/util/iputil.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 36f4bb79b..890ff98e7 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -102,13 +102,18 @@ except ImportError:
increase_rlimits = _increase_rlimits
def get_local_addresses_sync():
+ """
+ Return a list of IPv4 addresses (as dotted-quad native strings) that are
+ currently configured on this host, sorted in descending order of how likely
+ we think they are to work.
+ """
return [native_str(a) for a in _synchronously_find_addresses_via_config()]
def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
"""
Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad
- strings) that are currently configured on this host, sorted in descending
- order of how likely we think they are to work.
+ native strings) that are currently configured on this host, sorted in
+ descending order of how likely we think they are to work.
@param target: we want to learn an IP address they could try using to
connect to us; The default value is fine, but it might help if you
@@ -137,7 +142,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
def get_local_ip_for(target):
"""Find out what our IP address is for use by a given target.
- @return: the IP address as a dotted-quad string which could be used by
+ @return: the IP address as a dotted-quad native string which could be used
to connect to us. It might work for them, it might not. If
there is no suitable address (perhaps we don't currently have an
externally-visible interface), this will return None.
@@ -172,7 +177,7 @@ def get_local_ip_for(target):
return localip
finally:
d = port.stopListening()
- d.addErrback(log.err)
+ 3 d.addErrback(log.err)
except (socket.error, CannotListenError):
# no route to that host
localip = None
From 8c9c691c02c420253a79393669856d0bea07f4e3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 28 Jul 2020 11:11:27 -0400
Subject: [PATCH 20/21] Fix typo.
---
src/allmydata/util/iputil.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py
index 890ff98e7..8754fca53 100644
--- a/src/allmydata/util/iputil.py
+++ b/src/allmydata/util/iputil.py
@@ -177,7 +177,7 @@ def get_local_ip_for(target):
return localip
finally:
d = port.stopListening()
- 3 d.addErrback(log.err)
+ d.addErrback(log.err)
except (socket.error, CannotListenError):
# no route to that host
localip = None
From 199c4f6acd350885e28e30b91216c77d1b739b0e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 28 Jul 2020 13:00:44 -0400
Subject: [PATCH 21/21] Try to fix nix.
---
nix/tahoe-lafs.nix | 1 +
1 file changed, 1 insertion(+)
diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index 0e941bce1..f2e61d6c2 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -59,6 +59,7 @@ python.pkgs.buildPythonPackage rec {
fixtures
beautifulsoup4
html5lib
+ tenacity
];
checkPhase = ''