From c6d4ec0295446071c6e1dd8036e876d1ed317137 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:33:04 -0400 Subject: [PATCH 1/4] Port the test module to Python 3. --- src/allmydata/test/test_happiness.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index a8ed06363..ffba402ef 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -1,5 +1,15 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 87fe265d6..be0a382e8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -52,6 +52,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", + "allmydata.test.test_happiness", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", From d8f74770ebd1a6f27f2a69d24ba22c5a0f83f010 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:38:19 -0400 Subject: [PATCH 2/4] Port to Python 3. --- misc/python3/ratchet-passing | 15 +++++++++ newsfragments/3370.minor | 0 src/allmydata/immutable/happiness_upload.py | 35 +++++++++++++-------- src/allmydata/util/_python3.py | 1 + 4 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3370.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 951603419..48db826f2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -57,6 +57,21 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls allmydata.test.test_dictutil.DictUtil.test_auxdict allmydata.test.test_dictutil.DictUtil.test_dict_of_sets +allmydata.test.test_happiness.Happiness.test_100 +allmydata.test.test_happiness.Happiness.test_calc_happy +allmydata.test.test_happiness.Happiness.test_everything_broken +allmydata.test.test_happiness.Happiness.test_hypothesis0 +allmydata.test.test_happiness.Happiness.test_hypothesis_0 +allmydata.test.test_happiness.Happiness.test_hypothesis_1 +allmydata.test.test_happiness.Happiness.test_placement_1 +allmydata.test.test_happiness.Happiness.test_placement_simple +allmydata.test.test_happiness.Happiness.test_redistribute +allmydata.test.test_happiness.Happiness.test_unhappy +allmydata.test.test_happiness.HappinessUtils.test_residual_0 +allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph +allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph +allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy +allmydata.test.test_happiness.PlacementTests.test_more_hypothesis allmydata.test.test_hashtree.Complete.test_create allmydata.test.test_hashtree.Complete.test_dump allmydata.test.test_hashtree.Complete.test_needed_hashes diff --git a/newsfragments/3370.minor b/newsfragments/3370.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index 75edb74d9..a177660cc 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,5 +1,14 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from Queue import PriorityQueue +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things for external Python 2 code. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + +from queue import PriorityQueue def augmenting_path_for(graph): @@ -35,9 +44,9 @@ def bfs(graph, s): GRAY = 1 # BLACK vertices are those we have seen and explored BLACK = 2 - color = [WHITE for i in xrange(len(graph))] - predecessor = [None for i in xrange(len(graph))] - distance = [-1 for i in xrange(len(graph))] + color = [WHITE for i in range(len(graph))] + predecessor = [None for i in range(len(graph))] + distance = [-1 for i in range(len(graph))] queue = [s] # vertices that we haven't explored yet. color[s] = GRAY distance[s] = 0 @@ -58,9 +67,9 @@ def residual_network(graph, f): flow network represented by my graph and f arguments. graph is a flow network in adjacency-list form, and f is a flow in graph. """ - new_graph = [[] for i in xrange(len(graph))] - cf = [[0 for s in xrange(len(graph))] for sh in xrange(len(graph))] - for i in xrange(len(graph)): + new_graph = [[] for i in range(len(graph))] + cf = [[0 for s in range(len(graph))] for sh in range(len(graph))] + for i in range(len(graph)): for v in graph[i]: if f[i][v] == 1: # We add an edge (v, i) with cf[v,i] = 1. This means @@ -135,7 +144,7 @@ def _compute_maximum_graph(graph, shareIndices): return {} dim = len(graph) - flow_function = [[0 for sh in xrange(dim)] for s in xrange(dim)] + flow_function = [[0 for sh in range(dim)] for s in range(dim)] residual_graph, residual_function = residual_network(graph, flow_function) while augmenting_path_for(residual_graph): @@ -260,9 +269,9 @@ def _servermap_flow_graph(peers, shares, servermap): #print "share_to_index %s" % share_to_index #print "servermap %s" % servermap for peer in peers: - if servermap.has_key(peer): + if peer in servermap: for s in servermap[peer]: - if share_to_index.has_key(s): + if s in share_to_index: indexedShares.append(share_to_index[s]) graph.insert(peer_to_index[peer], indexedShares) for share in shares: @@ -373,7 +382,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): new_mappings = _calculate_mappings(new_peers, new_shares) #print "new_peers %s" % new_peers #print "new_mappings %s" % new_mappings - mappings = dict(readonly_mappings.items() + existing_mappings.items() + new_mappings.items()) + mappings = dict(list(readonly_mappings.items()) + list(existing_mappings.items()) + list(new_mappings.items())) homeless_shares = set() for share in mappings: if mappings[share] is None: @@ -384,7 +393,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): mappings, homeless_shares, { k: v - for k, v in peers_to_shares.items() + for k, v in list(peers_to_shares.items()) if k not in readonly_peers } ) @@ -401,5 +410,5 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): return { k: v.pop() if v else next(peer_iter) - for k, v in mappings.items() + for k, v in list(mappings.items()) } diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index be0a382e8..d0b2d4a21 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.immutable._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", From 4d2193fe13bd12ff9d73025bb54dcaf25c7e0bad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:12:12 -0400 Subject: [PATCH 3/4] Fix the module name. --- src/allmydata/util/_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index aa1ab2fb9..608d6ebea 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,7 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", - "allmydata.immutable._python3", + "allmydata.immutable.happiness_upload", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", From 504258622c8e3cd87a2a4fdcc3b1ac1902618c78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 14:54:12 -0400 Subject: [PATCH 4/4] Docstring. --- src/allmydata/immutable/happiness_upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index a177660cc..9716aaef2 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,3 +1,9 @@ +""" +Algorithms for figuring out happiness, the number of unique nodes the data is +on. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function