Merge remote-tracking branch 'origin/master' into 3415.storage-server-python-3

.circleci/config.yml

@@ -1,8 +1,17 @@
 # https://circleci.com/docs/2.0/
 
-version: 2
+# We use version 2.1 of CircleCI's configuration format (the docs are still at
+# the 2.0 link) in order to have access to Windows executors. This means we
+# can't use dots in job names anymore. They have a new "parameters" feature
+# that is supposed to remove the need to have version numbers in job names (the
+# source of our dots), but switching to that is going to be a bigger refactor:
+#
+#   https://discuss.circleci.com/t/v2-1-job-name-validation/31123
+#   https://circleci.com/docs/2.0/reusing-config/
+#
+version: 2.1
+
 workflows:
-  version: 2
   ci:
     jobs:
       # Platforms
@@ -11,13 +20,13 @@ workflows:
           requires:
             - "debian-9"
 
-      - "ubuntu-20.04"
+      - "ubuntu-20-04"
-      - "ubuntu-18.04":
+      - "ubuntu-18-04":
           requires:
-            - "ubuntu-20.04"
+            - "ubuntu-20-04"
-      - "ubuntu-16.04":
+      - "ubuntu-16-04":
           requires:
-            - "ubuntu-20.04"
+            - "ubuntu-20-04"
 
       - "fedora-29"
       - "fedora-28":
@@ -26,13 +35,13 @@ workflows:
 
       - "centos-8"
 
-      - "nixos-19.09"
+      - "nixos-19-09"
 
       # Test against PyPy 2.7
-      - "pypy2.7-buster"
+      - "pypy27-buster"
 
       # Just one Python 3.6 configuration while the port is in-progress.
-      - "python3.6"
+      - "python36"
 
       # Other assorted tasks and configurations
       - "lint"
@@ -67,13 +76,13 @@ workflows:
     jobs:
       - "build-image-debian-8"
       - "build-image-debian-9"
-      - "build-image-ubuntu-16.04"
+      - "build-image-ubuntu-16-04"
-      - "build-image-ubuntu-18.04"
+      - "build-image-ubuntu-18-04"
-      - "build-image-ubuntu-20.04"
+      - "build-image-ubuntu-20-04"
       - "build-image-fedora-28"
       - "build-image-fedora-29"
       - "build-image-centos-8"
-      - "build-image-pypy-2.7-buster"
+      - "build-image-pypy27-buster"
       - "build-image-python36-ubuntu"
 
 
@@ -203,7 +212,7 @@ jobs:
         user: "nobody"
 
 
-  pypy2.7-buster:
+  pypy27-buster:
     <<: *DEBIAN
     docker:
       - image: "tahoelafsci/pypy:buster-py2"
@@ -260,21 +269,21 @@ jobs:
       - run: *RUN_TESTS
 
 
-  ubuntu-16.04:
+  ubuntu-16-04:
     <<: *DEBIAN
     docker:
       - image: "tahoelafsci/ubuntu:16.04-py2.7"
         user: "nobody"
 
 
-  ubuntu-18.04: &UBUNTU_18_04
+  ubuntu-18-04: &UBUNTU_18_04
     <<: *DEBIAN
     docker:
       - image: "tahoelafsci/ubuntu:18.04-py2.7"
         user: "nobody"
 
 
-  python3.6:
+  python36:
     <<: *UBUNTU_18_04
     docker:
       - image: "tahoelafsci/ubuntu:18.04-py3"
@@ -289,7 +298,7 @@ jobs:
       TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
 
 
-  ubuntu-20.04:
+  ubuntu-20-04:
     <<: *DEBIAN
     docker:
       - image: "tahoelafsci/ubuntu:20.04"
@@ -331,7 +340,7 @@ jobs:
         user: "nobody"
 
 
-  nixos-19.09:
+  nixos-19-09:
     docker:
       # Run in a highly Nix-capable environment.
       - image: "nixorg/nix:circleci"
@@ -476,7 +485,7 @@ jobs:
       PYTHON_VERSION: "2.7"
 
 
-  build-image-ubuntu-16.04:
+  build-image-ubuntu-16-04:
     <<: *BUILD_IMAGE
 
     environment:
@@ -485,7 +494,7 @@ jobs:
       PYTHON_VERSION: "2.7"
 
 
-  build-image-ubuntu-18.04:
+  build-image-ubuntu-18-04:
     <<: *BUILD_IMAGE
 
     environment:
@@ -503,7 +512,7 @@ jobs:
       PYTHON_VERSION: "3"
 
 
-  build-image-ubuntu-20.04:
+  build-image-ubuntu-20-04:
     <<: *BUILD_IMAGE
 
     environment:
@@ -539,7 +548,7 @@ jobs:
       TAG: "29"
 
 
-  build-image-pypy-2.7-buster:
+  build-image-pypy27-buster:
     <<: *BUILD_IMAGE
 
     environment:

.codecov.yml

@@ -0,0 +1,34 @@
+# Override defaults for codecov.io checks.
+#
+# Documentation is at https://docs.codecov.io/docs/codecov-yaml;
+# reference is at https://docs.codecov.io/docs/codecovyml-reference.
+#
+# To validate this file, use:
+#
+#   curl --data-binary @.codecov.yml https://codecov.io/validate
+#
+# Codecov's defaults seem to leave red marks in GitHub CI checks in a
+# rather arbitrary manner, probably because of non-determinism in
+# coverage (see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2891)
+# and maybe because computers are bad with floating point numbers.
+
+# Allow coverage percentage a precision of zero decimals, and round to
+# the nearest number (for example, 89.957 to to 90; 89.497 to 89%).
+# Coverage above 90% is good, below 80% is bad.
+coverage:
+  round: nearest
+  range: 80..90
+  precision: 0
+
+  # Aim for a target test coverage of 90% in codecov/project check (do
+  # not allow project coverage to drop below that), and allow
+  # codecov/patch a threshold of 1% (allow coverage in changes to drop
+  # by that much, and no less).  That should be good enough for us.
+  status:
+    project:
+      default:
+        target: 90%
+        threshold: 1%
+    patch:
+      default:
+        threshold: 1%

src/allmydata/storage/common.py

@@ -1,4 +1,14 @@
-from future.utils import PY3
+"""
+Ported to Python 3.
+"""
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2, PY3
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os.path
 from allmydata.util import base32

src/allmydata/storage/lease.py

@@ -1,3 +1,16 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import struct, time
 
 class LeaseInfo(object):

src/allmydata/storage/mutable.py

@@ -1,3 +1,16 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os, stat, struct
 
 from allmydata.interfaces import BadWriteEnablerError

src/allmydata/storage/shares.py

@@ -1,4 +1,15 @@
-#! /usr/bin/python
+"""
+Ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from allmydata.storage.mutable import MutableShareFile
 from allmydata.storage.immutable import ShareFile

src/allmydata/test/test_storage.py

@@ -29,10 +29,12 @@ import itertools
 from allmydata import interfaces
 from allmydata.util import fileutil, hashutil, base32
 from allmydata.storage.server import StorageServer
+from allmydata.storage.shares import get_share_file
 from allmydata.storage.mutable import MutableShareFile
 from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile
 from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \
-     UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError
+     UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \
+     si_b2a, si_a2b
 from allmydata.storage.lease import LeaseInfo
 from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \
      ReadBucketProxy
@@ -52,6 +54,42 @@ from allmydata.storage_client import (
 from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin
 
 
+class UtilTests(unittest.TestCase):
+    """Tests for allmydata.storage.common and .shares."""
+
+    def test_encoding(self):
+        """b2a/a2b are the same as base32."""
+        s = b"\xFF HELLO \xF3"
+        result = si_b2a(s)
+        self.assertEqual(base32.b2a(s), result)
+        self.assertEqual(si_a2b(result), s)
+
+    def test_storage_index_to_dir(self):
+        """storage_index_to_dir creates a native string path."""
+        s = b"\xFF HELLO \xF3"
+        path = storage_index_to_dir(s)
+        parts = os.path.split(path)
+        self.assertEqual(parts[0], parts[1][:2])
+        self.assertIsInstance(path, native_str)
+
+    def test_get_share_file_mutable(self):
+        """A mutable share is identified by get_share_file()."""
+        path = self.mktemp()
+        msf = MutableShareFile(path)
+        msf.create(b"12", b"abc")  # arbitrary values
+        loaded = get_share_file(path)
+        self.assertIsInstance(loaded, MutableShareFile)
+        self.assertEqual(loaded.home, path)
+
+    def test_get_share_file_immutable(self):
+        """An immutable share is identified by get_share_file()."""
+        path = self.mktemp()
+        _ = ShareFile(path, max_size=1000, create=True)
+        loaded = get_share_file(path)
+        self.assertIsInstance(loaded, ShareFile)
+        self.assertEqual(loaded.home, path)
+
+
 class FakeStatsProvider(object):
     def count(self, name, delta=1):
         pass

src/allmydata/util/_python3.py

@@ -35,10 +35,14 @@ PORTED_MODULES = [
     "allmydata.immutable.happiness_upload",
     "allmydata.interfaces",
     "allmydata.monitor",
+    "allmydata.storage.common",
     "allmydata.storage.crawler",
     "allmydata.storage.expirer",
     "allmydata.storage.immutable",
+    "allmydata.storage.lease",
+    "allmydata.storage.mutable",
     "allmydata.storage.server",
+    "allmydata.storage.shares",
     "allmydata.test.common_py3",
     "allmydata.uri",
     "allmydata.util._python3",