From da57a7fc21f4bd988f129eadc299bd688b85efe9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 22 Aug 2020 08:14:08 -0400 Subject: [PATCH 1/6] Add codecov settings --- .codecov.yml | 33 +++++++++++++++++++++++++++++++++ newsfragments/3391.minor | 0 2 files changed, 33 insertions(+) create mode 100644 .codecov.yml create mode 100644 newsfragments/3391.minor diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..4916bf166 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,33 @@ +# Override defaults for codecov.io checks. +# +# Codecov's defaults seem to leave red marks in GitHub CI checks in a +# rather arbitrary manner, probably because computers are bad with +# floating point numbers. +# +# Documentation is at https://docs.codecov.io/docs/codecov-yaml; +# reference is at https://docs.codecov.io/docs/codecovyml-reference. +# +# To validate this file, use: +# +# curl --data-binary @.codecov.yml https://codecov.io/validate + +# Allow coverage percentage a precision of zero decimals, and round to +# the nearest number (for example, 89.957 to to 90; 89.497 to 89%). +# Coverage above 90% is good, below 80% is bad. +coverage: + round: nearest + range: 80..90 + precision: 0 + + # Aim for a target test coverage of 90% in codecov/project check (do + # not allow project coverage to drop below that), and allow + # codecov/patch a threshold of 1% (allow coverage in changes to drop + # by that much, and no less). That should be good enough for us. + status: + project: + default: + target: 90% + threshold: 1% + patch: + default: + threshold: 1% diff --git a/newsfragments/3391.minor b/newsfragments/3391.minor new file mode 100644 index 000000000..e69de29bb From 621abcd61179b42c8738a054019031c49e197ad2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 8 Sep 2020 16:07:46 -0400 Subject: [PATCH 2/6] Mention known issue with non-determinism in coverage data --- .codecov.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index 4916bf166..57abf7c0a 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,15 +1,16 @@ # Override defaults for codecov.io checks. # -# Codecov's defaults seem to leave red marks in GitHub CI checks in a -# rather arbitrary manner, probably because computers are bad with -# floating point numbers. -# # Documentation is at https://docs.codecov.io/docs/codecov-yaml; # reference is at https://docs.codecov.io/docs/codecovyml-reference. # # To validate this file, use: # # curl --data-binary @.codecov.yml https://codecov.io/validate +# +# Codecov's defaults seem to leave red marks in GitHub CI checks in a +# rather arbitrary manner, probably because of non-determinism in +# coverage (see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2891) +# and maybe because computers are bad with floating point numbers. # Allow coverage percentage a precision of zero decimals, and round to # the nearest number (for example, 89.957 to to 90; 89.497 to 89%). From e86e0d761f8d84d453c3863d92e60b38b6b30016 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 11 Sep 2020 14:49:30 -0400 Subject: [PATCH 3/6] Better test coverage for allmydata.storage.immutable. --- src/allmydata/test/test_storage.py | 37 +++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 4bbbcda30..bba4ae964 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,7 +30,7 @@ from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32 from allmydata.storage.server import StorageServer from allmydata.storage.mutable import MutableShareFile -from allmydata.storage.immutable import BucketWriter, BucketReader +from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError from allmydata.storage.lease import LeaseInfo @@ -2968,3 +2968,38 @@ class Stats(unittest.TestCase): self.failUnless(output["get"]["95_0_percentile"] is None, output) self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) + + +class ShareFileTests(unittest.TestCase): + """Tests for allmydata.storage.immutable.ShareFile.""" + + def get_sharefile(self): + sf = ShareFile(self.mktemp(), max_size=1000, create=True) + sf.write_share_data(0, b"abc") + sf.write_share_data(2, b"DEF") + # Should be b'abDEF' now. + return sf + + def test_read_write(self): + """Basic writes can be read.""" + sf = self.get_sharefile() + self.assertEqual(sf.read_share_data(0, 3), b"abD") + self.assertEqual(sf.read_share_data(1, 4), b"bDEF") + + def test_reads_beyond_file_end(self): + """Reads beyond the file size are truncated.""" + sf = self.get_sharefile() + self.assertEqual(sf.read_share_data(0, 10), b"abDEF") + self.assertEqual(sf.read_share_data(5, 10), b"") + + def test_too_large_write(self): + """Can't do write larger than file size.""" + sf = self.get_sharefile() + with self.assertRaises(DataTooLargeError): + sf.write_share_data(0, b"x" * 3000) + + def test_no_leases_cancelled(self): + """If no leases were cancelled, IndexError is raised.""" + sf = self.get_sharefile() + with self.assertRaises(IndexError): + sf.cancel_lease(b"garbage") From 6f5c32c4616b2d0e8becdacf920caa8307acec93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 11 Sep 2020 14:51:20 -0400 Subject: [PATCH 4/6] News fragment. --- newsfragments/3409.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3409.minor diff --git a/newsfragments/3409.minor b/newsfragments/3409.minor new file mode 100644 index 000000000..e69de29bb From 7de84e32a3536024ceb244739629b09913f46c34 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 11 Sep 2020 15:02:42 -0400 Subject: [PATCH 5/6] Port to Python 3. --- src/allmydata/storage/immutable.py | 13 ++++++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 8d83ec1b3..778c0ddf8 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -1,4 +1,15 @@ -from future.utils import bytes_to_native_str +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, bytes_to_native_str +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, stat, struct, time diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index afdbea1f0..5a26d2cbc 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -37,6 +37,7 @@ PORTED_MODULES = [ "allmydata.monitor", "allmydata.storage.crawler", "allmydata.storage.expirer", + "allmydata.storage.immutable", "allmydata.test.common_py3", "allmydata.uri", "allmydata.util._python3", From 26a8a0bfc5d0d7447b081934c87e3075ffda1ef7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Sep 2020 13:29:28 -0400 Subject: [PATCH 6/6] Make stats code deal with keys that are unicode. --- src/allmydata/stats.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 34907b1fa..28a451ead 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import json import os import pprint @@ -155,6 +157,8 @@ class StatsProvider(Referenceable, service.MultiService): service.MultiService.startService(self) def count(self, name, delta=1): + if isinstance(name, unicode): + name = name.encode("utf-8") val = self.counters.setdefault(name, 0) self.counters[name] = val + delta @@ -170,7 +174,18 @@ class StatsProvider(Referenceable, service.MultiService): return ret def remote_get_stats(self): - return self.get_stats() + # The remote API expects keys to be bytes: + def to_bytes(d): + result = {} + for (k, v) in d.items(): + if isinstance(k, unicode): + k = k.encode("utf-8") + result[k] = v + return result + + stats = self.get_stats() + return {b"counters": to_bytes(stats["counters"]), + b"stats": to_bytes(stats["stats"])} def _connected(self, gatherer, nickname): gatherer.callRemoteOnly('provide', self, nickname or '')