diff --git a/.circleci/config.yml b/.circleci/config.yml index 56bc3e07e..f662a3702 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,17 @@ # https://circleci.com/docs/2.0/ -version: 2 +# We use version 2.1 of CircleCI's configuration format (the docs are still at +# the 2.0 link) in order to have access to Windows executors. This means we +# can't use dots in job names anymore. They have a new "parameters" feature +# that is supposed to remove the need to have version numbers in job names (the +# source of our dots), but switching to that is going to be a bigger refactor: +# +# https://discuss.circleci.com/t/v2-1-job-name-validation/31123 +# https://circleci.com/docs/2.0/reusing-config/ +# +version: 2.1 + workflows: - version: 2 ci: jobs: # Platforms @@ -11,13 +20,13 @@ workflows: requires: - "debian-9" - - "ubuntu-20.04" - - "ubuntu-18.04": + - "ubuntu-20-04" + - "ubuntu-18-04": requires: - - "ubuntu-20.04" - - "ubuntu-16.04": + - "ubuntu-20-04" + - "ubuntu-16-04": requires: - - "ubuntu-20.04" + - "ubuntu-20-04" - "fedora-29" - "fedora-28": @@ -26,13 +35,13 @@ workflows: - "centos-8" - - "nixos-19.09" + - "nixos-19-09" # Test against PyPy 2.7 - - "pypy2.7-buster" + - "pypy27-buster" # Just one Python 3.6 configuration while the port is in-progress. - - "python3.6" + - "python36" # Other assorted tasks and configurations - "lint" @@ -67,13 +76,13 @@ workflows: jobs: - "build-image-debian-8" - "build-image-debian-9" - - "build-image-ubuntu-16.04" - - "build-image-ubuntu-18.04" - - "build-image-ubuntu-20.04" + - "build-image-ubuntu-16-04" + - "build-image-ubuntu-18-04" + - "build-image-ubuntu-20-04" - "build-image-fedora-28" - "build-image-fedora-29" - "build-image-centos-8" - - "build-image-pypy-2.7-buster" + - "build-image-pypy27-buster" - "build-image-python36-ubuntu" @@ -203,7 +212,7 @@ jobs: user: "nobody" - pypy2.7-buster: + pypy27-buster: <<: *DEBIAN docker: - image: "tahoelafsci/pypy:buster-py2" @@ -260,21 +269,21 @@ jobs: - run: *RUN_TESTS - ubuntu-16.04: + ubuntu-16-04: <<: *DEBIAN docker: - image: "tahoelafsci/ubuntu:16.04-py2.7" user: "nobody" - ubuntu-18.04: &UBUNTU_18_04 + ubuntu-18-04: &UBUNTU_18_04 <<: *DEBIAN docker: - image: "tahoelafsci/ubuntu:18.04-py2.7" user: "nobody" - python3.6: + python36: <<: *UBUNTU_18_04 docker: - image: "tahoelafsci/ubuntu:18.04-py3" @@ -289,7 +298,7 @@ jobs: TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage" - ubuntu-20.04: + ubuntu-20-04: <<: *DEBIAN docker: - image: "tahoelafsci/ubuntu:20.04" @@ -331,7 +340,7 @@ jobs: user: "nobody" - nixos-19.09: + nixos-19-09: docker: # Run in a highly Nix-capable environment. - image: "nixorg/nix:circleci" @@ -476,7 +485,7 @@ jobs: PYTHON_VERSION: "2.7" - build-image-ubuntu-16.04: + build-image-ubuntu-16-04: <<: *BUILD_IMAGE environment: @@ -485,7 +494,7 @@ jobs: PYTHON_VERSION: "2.7" - build-image-ubuntu-18.04: + build-image-ubuntu-18-04: <<: *BUILD_IMAGE environment: @@ -503,7 +512,7 @@ jobs: PYTHON_VERSION: "3" - build-image-ubuntu-20.04: + build-image-ubuntu-20-04: <<: *BUILD_IMAGE environment: @@ -539,7 +548,7 @@ jobs: TAG: "29" - build-image-pypy-2.7-buster: + build-image-pypy27-buster: <<: *BUILD_IMAGE environment: diff --git a/newsfragments/3406.minor b/newsfragments/3406.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3411.minor b/newsfragments/3411.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 55036eea7..cb6116e5b 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -1,4 +1,14 @@ -from future.utils import PY3 +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path from allmydata.util import base32 diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 4d2dce8c4..187f32406 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import struct, time class LeaseInfo(object): diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index a1eddf6cc..a44a2e18d 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os, stat, struct from allmydata.interfaces import BadWriteEnablerError diff --git a/src/allmydata/storage/shares.py b/src/allmydata/storage/shares.py index bd94c0f3f..ec6c0a501 100644 --- a/src/allmydata/storage/shares.py +++ b/src/allmydata/storage/shares.py @@ -1,4 +1,15 @@ -#! /usr/bin/python +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import ShareFile diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bba4ae964..ee2412f78 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -29,10 +29,12 @@ import itertools from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32 from allmydata.storage.server import StorageServer +from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \ - UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError + UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ + si_b2a, si_a2b from allmydata.storage.lease import LeaseInfo from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \ ReadBucketProxy @@ -52,6 +54,42 @@ from allmydata.storage_client import ( from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin +class UtilTests(unittest.TestCase): + """Tests for allmydata.storage.common and .shares.""" + + def test_encoding(self): + """b2a/a2b are the same as base32.""" + s = b"\xFF HELLO \xF3" + result = si_b2a(s) + self.assertEqual(base32.b2a(s), result) + self.assertEqual(si_a2b(result), s) + + def test_storage_index_to_dir(self): + """storage_index_to_dir creates a native string path.""" + s = b"\xFF HELLO \xF3" + path = storage_index_to_dir(s) + parts = os.path.split(path) + self.assertEqual(parts[0], parts[1][:2]) + self.assertIsInstance(path, native_str) + + def test_get_share_file_mutable(self): + """A mutable share is identified by get_share_file().""" + path = self.mktemp() + msf = MutableShareFile(path) + msf.create(b"12", b"abc") # arbitrary values + loaded = get_share_file(path) + self.assertIsInstance(loaded, MutableShareFile) + self.assertEqual(loaded.home, path) + + def test_get_share_file_immutable(self): + """An immutable share is identified by get_share_file().""" + path = self.mktemp() + _ = ShareFile(path, max_size=1000, create=True) + loaded = get_share_file(path) + self.assertIsInstance(loaded, ShareFile) + self.assertEqual(loaded.home, path) + + class FakeStatsProvider(object): def count(self, name, delta=1): pass diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5a26d2cbc..9cb7d0e2c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -35,9 +35,13 @@ PORTED_MODULES = [ "allmydata.immutable.happiness_upload", "allmydata.interfaces", "allmydata.monitor", + "allmydata.storage.common", "allmydata.storage.crawler", "allmydata.storage.expirer", "allmydata.storage.immutable", + "allmydata.storage.lease", + "allmydata.storage.mutable", + "allmydata.storage.shares", "allmydata.test.common_py3", "allmydata.uri", "allmydata.util._python3",