Merge branch '3370.happiness-upload-python-3' into 3373.happinessutil-python-3

.circleci/Dockerfile.slackware

@@ -1,49 +0,0 @@
-ARG TAG
-FROM vbatts/slackware:${TAG}
-
-ENV WHEELHOUSE_PATH /tmp/wheelhouse
-ENV VIRTUALENV_PATH /tmp/venv
-# This will get updated by the CircleCI checkout step.
-ENV BUILD_SRC_ROOT /tmp/project
-
-# Be careful with slackpkg.  If the package name given doesn't match anything,
-# slackpkg still claims to succeed but you're totally screwed.  Slackware
-# updates versions of packaged software so including too much version prefix
-# is a good way to have your install commands suddenly begin not installing
-# anything.
-RUN slackpkg update && \
-    slackpkg install \
-        openssh-7 git-2 \
-        ca-certificates \
-        sudo-1 \
-        make-4 \
-        automake-1 \
-        kernel-headers \
-        glibc-2 \
-        binutils-2 \
-        gcc-5 \
-        gcc-g++-5 \
-        python-2 \
-        libffi-3 \
-        libyaml-0 \
-        sqlite-3 \
-        icu4c-56 \
-        libmpc-1 </dev/null && \
-    slackpkg upgrade \
-        openssl-1 </dev/null
-
-# neither virtualenv nor pip is packaged.
-# do it the hard way.
-# and it is extra hard since it is slackware.
-RUN slackpkg install \
-        cyrus-sasl-2 \
-        curl-7 </dev/null && \
-    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
-    python get-pip.py && \
-    pip install virtualenv
-
-# Get the project source.  This is better than it seems.  CircleCI will
-# *update* this checkout on each job run, saving us more time per-job.
-COPY . ${BUILD_SRC_ROOT}
-
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/config.yml

@@ -11,10 +11,13 @@ workflows:
           requires:
             - "debian-9"
 
-      - "ubuntu-18.04"
+      - "ubuntu-20.04"
+      - "ubuntu-18.04":
+          requires:
+            - "ubuntu-20.04"
       - "ubuntu-16.04":
           requires:
-            - "ubuntu-18.04"
+            - "ubuntu-20.04"
 
       - "fedora-29"
       - "fedora-28":
@@ -23,8 +26,6 @@ workflows:
 
       - "centos-8"
 
-      - "slackware-14.2"
-
       - "nixos-19.09"
 
       # Test against PyPy 2.7
@@ -65,10 +66,10 @@ workflows:
       - "build-image-debian-9"
       - "build-image-ubuntu-16.04"
       - "build-image-ubuntu-18.04"
+      - "build-image-ubuntu-20.04"
       - "build-image-fedora-28"
       - "build-image-fedora-29"
       - "build-image-centos-8"
-      - "build-image-slackware-14.2"
       - "build-image-pypy-2.7-buster"
 
 
@@ -268,6 +269,13 @@ jobs:
         user: "nobody"
 
 
+  ubuntu-20.04:
+    <<: *DEBIAN
+    docker:
+      - image: "tahoelafsci/ubuntu:20.04"
+        user: "nobody"
+
+
   centos-8: &RHEL_DERIV
     docker:
       - image: "tahoelafsci/centos:8"
@@ -303,26 +311,6 @@ jobs:
         user: "nobody"
 
 
-  slackware-14.2:
-    docker:
-      - image: "tahoelafsci/slackware:14.2"
-        user: "nobody"
-
-    environment: *UTF_8_ENVIRONMENT
-
-    # pip cannot install packages if the working directory is not readable.
-    # We want to run a lot of steps as nobody instead of as root.
-    working_directory: "/tmp/project"
-
-    steps:
-      - "checkout"
-      - run: *SETUP_VIRTUALENV
-      - run: *RUN_TESTS
-      - store_test_results: *STORE_TEST_RESULTS
-      - store_artifacts: *STORE_TEST_LOG
-      - store_artifacts: *STORE_OTHER_ARTIFACTS
-      - run: *SUBMIT_COVERAGE
-
   nixos-19.09:
     docker:
       # Run in a highly Nix-capable environment.
@@ -480,6 +468,14 @@ jobs:
       TAG: "18.04"
 
 
+  build-image-ubuntu-20.04:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "20.04"
+
+
   build-image-centos-8:
     <<: *BUILD_IMAGE
 
@@ -504,14 +500,6 @@ jobs:
       TAG: "29"
 
 
-  build-image-slackware-14.2:
-    <<: *BUILD_IMAGE
-
-    environment:
-      DISTRO: "slackware"
-      TAG: "14.2"
-
-
   build-image-pypy-2.7-buster:
     <<: *BUILD_IMAGE
 

misc/python3/ratchet-passing

@@ -101,6 +101,14 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr
 allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route
 allmydata.test.test_iputil.ListenOnUsed.test_random_port
 allmydata.test.test_iputil.ListenOnUsed.test_specific_port
+allmydata.test.test_log.Log.test_default_facility
+allmydata.test.test_log.Log.test_err
+allmydata.test.test_log.Log.test_grandparent_id
+allmydata.test.test_log.Log.test_no_prefix
+allmydata.test.test_log.Log.test_numming
+allmydata.test.test_log.Log.test_parent_id
+allmydata.test.test_log.Log.test_with_bytes_prefix
+allmydata.test.test_log.Log.test_with_prefix
 allmydata.test.test_netstring.Netstring.test_encode
 allmydata.test.test_netstring.Netstring.test_extra
 allmydata.test.test_netstring.Netstring.test_nested

newsfragments/3323.removed

@@ -0,0 +1 @@
+Slackware 14.2 is no longer a Tahoe-LAFS supported platform.

newsfragments/3328.installation

@@ -0,0 +1 @@
+Tahoe-LAFS now supports Ubuntu 20.04.

src/allmydata/hashtree.py

@@ -45,6 +45,8 @@ Written by Connelly Barnes in 2005 and released into the
 public domain  with no warranty of any kind, either expressed
 or implied.  It probably won't make your computer catch on fire,
 or eat  your children, but it might.  Use at your own risk.
+
+Ported to Python 3.
 """
 
 from __future__ import absolute_import

src/allmydata/immutable/happiness_upload.py

@@ -1,3 +1,9 @@
+"""
+Algorithms for figuring out happiness, the number of unique nodes the data is
+on.
+
+Ported to Python 3.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

src/allmydata/test/test_log.py

@@ -0,0 +1,156 @@
+"""
+Tests for allmydata.util.log.
+
+Ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from twisted.trial import unittest
+from twisted.python.failure import Failure
+
+from foolscap.logging import log
+
+from allmydata.util import log as tahoe_log
+
+
+class SampleError(Exception):
+    pass
+
+
+class Log(unittest.TestCase):
+    def setUp(self):
+        self.messages = []
+
+        def msg(msg, facility, parent, *args, **kwargs):
+            self.messages.append((msg, facility, parent, args, kwargs))
+            return "msg{}".format(len(self.messages))
+
+        self.patch(log, "msg", msg)
+
+    def test_err(self):
+        """Logging with log.err() causes tests to fail."""
+        try:
+            raise SampleError("simple sample")
+        except:
+            f = Failure()
+        tahoe_log.err(format="intentional sample error",
+                      failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ")
+        result = self.flushLoggedErrors(SampleError)
+        self.assertEqual(len(result), 1)
+
+    def test_default_facility(self):
+        """
+        If facility is passed to PrefixingLogMixin.__init__, it is used as
+        default facility.
+        """
+        class LoggingObject1(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject1(facility="defaultfac")
+        obj.log("hello")
+        obj.log("world", facility="override")
+        self.assertEqual(self.messages[-2][1], "defaultfac")
+        self.assertEqual(self.messages[-1][1], "override")
+
+    def test_with_prefix(self):
+        """
+        If prefix is passed to PrefixingLogMixin.__init__, it is used in
+        message rendering.
+        """
+        class LoggingObject4(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject4("fac", prefix="pre1")
+        obj.log("hello")
+        obj.log("world")
+        self.assertEqual(self.messages[-2][0], '<LoggingObject4 #1>(pre1): hello')
+        self.assertEqual(self.messages[-1][0], '<LoggingObject4 #1>(pre1): world')
+
+    def test_with_bytes_prefix(self):
+        """
+        If bytes prefix is passed to PrefixingLogMixin.__init__, it is used in
+        message rendering.
+        """
+        class LoggingObject5(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject5("fac", prefix=b"pre1")
+        obj.log("hello")
+        obj.log("world")
+        self.assertEqual(self.messages[-2][0], '<LoggingObject5 #1>(pre1): hello')
+        self.assertEqual(self.messages[-1][0], '<LoggingObject5 #1>(pre1): world')
+
+    def test_no_prefix(self):
+        """
+        If no prefix is passed to PrefixingLogMixin.__init__, it is not used in
+        message rendering.
+        """
+        class LoggingObject2(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject2()
+        obj.log("hello")
+        obj.log("world")
+        self.assertEqual(self.messages[-2][0], '<LoggingObject2 #1>: hello')
+        self.assertEqual(self.messages[-1][0], '<LoggingObject2 #1>: world')
+
+    def test_numming(self):
+        """
+        Objects inheriting from PrefixingLogMixin get a unique number from a
+        class-specific counter.
+        """
+        class LoggingObject3(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject3()
+        obj2 = LoggingObject3()
+        obj.log("hello")
+        obj2.log("world")
+        self.assertEqual(self.messages[-2][0], '<LoggingObject3 #1>: hello')
+        self.assertEqual(self.messages[-1][0], '<LoggingObject3 #2>: world')
+
+    def test_parent_id(self):
+        """
+        The parent message id can be passed in, otherwise the first message's
+        id is used as the parent.
+
+        This logic is pretty bogus, but that's what the code does.
+        """
+        class LoggingObject1(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject1()
+        result = obj.log("zero")
+        self.assertEqual(result, "msg1")
+        obj.log("one", parent="par1")
+        obj.log("two", parent="par2")
+        obj.log("three")
+        obj.log("four")
+        self.assertEqual([m[2] for m in self.messages],
+                         [None, "par1", "par2", "msg1", "msg1"])
+
+    def test_grandparent_id(self):
+        """
+        If grandparent message id is given, it's used as parent id of the first
+        message.
+        """
+        class LoggingObject1(tahoe_log.PrefixingLogMixin):
+            pass
+
+        obj = LoggingObject1(grandparentmsgid="grand")
+        result = obj.log("zero")
+        self.assertEqual(result, "msg1")
+        obj.log("one", parent="par1")
+        obj.log("two", parent="par2")
+        obj.log("three")
+        obj.log("four")
+        self.assertEqual([m[2] for m in self.messages],
+                         ["grand", "par1", "par2", "msg1", "msg1"])

src/allmydata/test/test_util.py

@@ -5,13 +5,11 @@ import os, time, sys
 import yaml
 
 from twisted.trial import unittest
-from twisted.python.failure import Failure
 
 from allmydata.util import idlib, mathutil
 from allmydata.util import fileutil
 from allmydata.util import pollmixin
 from allmydata.util import yamlutil
-from allmydata.util import log as tahoe_log
 from allmydata.util.fileutil import EncryptedTemporaryFile
 from allmydata.test.common_util import ReallyEqualMixin
 
@@ -452,20 +450,6 @@ class EqButNotIs(object):
         return self.x == other
 
 
-class SampleError(Exception):
-    pass
-
-class Log(unittest.TestCase):
-    def test_err(self):
-        try:
-            raise SampleError("simple sample")
-        except:
-            f = Failure()
-        tahoe_log.err(format="intentional sample error",
-                      failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ")
-        self.flushLoggedErrors(SampleError)
-
-
 class YAML(unittest.TestCase):
     def test_convert(self):
         data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])

src/allmydata/util/_python3.py

@@ -23,6 +23,8 @@ PORTED_MODULES = [
     "allmydata.crypto.util",
     "allmydata.hashtree",
     "allmydata.immutable.happiness_upload",
+    "allmydata.test.common_py3",
+    "allmydata.util._python3",
     "allmydata.util.abbreviate",
     "allmydata.util.assertutil",
     "allmydata.util.base32",
@@ -34,17 +36,16 @@ PORTED_MODULES = [
     "allmydata.util.hashutil",
     "allmydata.util.humanreadable",
     "allmydata.util.iputil",
+    "allmydata.util.log",
     "allmydata.util.mathutil",
     "allmydata.util.namespace",
     "allmydata.util.netstring",
     "allmydata.util.observer",
     "allmydata.util.pipeline",
     "allmydata.util.pollmixin",
-    "allmydata.util._python3",
     "allmydata.util.spans",
     "allmydata.util.statistics",
     "allmydata.util.time_format",
-    "allmydata.test.common_py3",
 ]
 
 PORTED_TEST_MODULES = [
@@ -59,6 +60,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_hashutil",
     "allmydata.test.test_humanreadable",
     "allmydata.test.test_iputil",
+    "allmydata.test.test_log",
     "allmydata.test.test_netstring",
     "allmydata.test.test_observer",
     "allmydata.test.test_pipeline",

src/allmydata/util/log.py

@@ -1,4 +1,18 @@
-from allmydata.util import nummedobj
+"""
+Logging utilities.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from pyutil import nummedobj
 
 from foolscap.logging import log
 from twisted.python import log as tw_log
@@ -36,8 +50,8 @@ class LogMixin(object):
     def log(self, msg, facility=None, parent=None, *args, **kwargs):
         if facility is None:
             facility = self._facility
-        pmsgid = None
+        pmsgid = parent
-        if parent is None:
+        if pmsgid is None:
             pmsgid = self._parentmsgid
             if pmsgid is None:
                 pmsgid = self._grandparentmsgid
@@ -54,6 +68,8 @@ class PrefixingLogMixin(nummedobj.NummedObj, LogMixin):
         LogMixin.__init__(self, facility, grandparentmsgid)
 
         if prefix:
+            if isinstance(prefix, bytes):
+                prefix = prefix.decode("utf-8", errors="replace")
             self._prefix = "%s(%s): " % (self.__repr__(), prefix)
         else:
             self._prefix = "%s: " % (self.__repr__(),)

src/allmydata/util/nummedobj.py

@@ -1,42 +0,0 @@
-import collections, itertools, functools
-
-objnums = collections.defaultdict(itertools.count)
-
-
-@functools.total_ordering
-class NummedObj(object):
-    """
-    This is useful for nicer debug printouts.  Instead of objects of the same class being
-    distinguished from one another by their memory address, they each get a unique number, which
-    can be read as "the first object of this class", "the second object of this class", etc.  This
-    is especially useful because separate runs of a program will yield identical debug output,
-    (assuming that the objects get created in the same order in each run).  This makes it possible
-    to diff outputs from separate runs to see what changed, without having to ignore a difference
-    on every line due to different memory addresses of objects.
-    """
-
-    def __init__(self, klass=None):
-        """
-        @param klass: in which class are you counted?  If default value of `None', then self.__class__ will be used.
-        """
-        if klass is None:
-            klass = self.__class__
-        self._classname = klass.__name__
-
-        self._objid = objnums[self._classname].next()
-
-    def __repr__(self):
-        return "<%s #%d>" % (self._classname, self._objid,)
-
-    def __lt__(self, other):
-        if isinstance(other, NummedObj):
-            return (self._objid, self._classname,) < (other._objid, other._classname,)
-        return NotImplemented
-
-    def __eq__(self, other):
-        if isinstance(other, NummedObj):
-            return (self._objid, self._classname,) == (other._objid, other._classname,)
-        return NotImplemented
-
-    def __hash__(self):
-        return id(self)

tox.ini

@@ -210,7 +210,9 @@ extras =
 deps =
     {[testenv]deps}
     packaging
-    pyinstaller
+    # PyInstaller 4.0 drops Python 2 support.  When we finish porting to
+    # Python 3 we can reconsider this constraint.
+    pyinstaller < 4.0
 # Setting PYTHONHASHSEED to a known value assists with reproducible builds.
 # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build
 setenv=PYTHONHASHSEED=1