Merge remote-tracking branch 'origin/master' into integration/storage-economics

.appveyor.yml

@@ -51,6 +51,11 @@ test_script:
   # to put the Python version you want to use on PATH.
   - |
     %PYTHON%\Scripts\tox.exe -e coverage
+    %PYTHON%\Scripts\tox.exe -e pyinstaller
+  # To verify that the resultant PyInstaller-generated binary executes
+  # cleanly (i.e., that it terminates with an exit code of 0 and isn't
+  # failing due to import/packaging-related errors, etc.).
+  - dist\Tahoe-LAFS\tahoe.exe --version
 
 after_test:
   # This builds the main tahoe wheel, and wheels for all dependencies.

.circleci/config.yml

@@ -30,6 +30,7 @@ workflows:
 
       # Other assorted tasks and configurations
       - "lint"
+      - "pyinstaller"
       - "deprecations"
       - "c-locale"
       # Any locale other than C or UTF-8.
@@ -87,6 +88,31 @@ jobs:
           command: |
             ~/.local/bin/tox -e codechecks
 
+  pyinstaller:
+    docker:
+      - image: "circleci/python:2"
+
+    steps:
+      - "checkout"
+
+      - run:
+          name: "Install tox"
+          command: |
+            pip install --user tox
+
+      - run:
+          name: "Make PyInstaller executable"
+          command: |
+            ~/.local/bin/tox -e pyinstaller
+
+      - run:
+          # To verify that the resultant PyInstaller-generated binary executes
+          # cleanly (i.e., that it terminates with an exit code of 0 and isn't
+          # failing due to import/packaging-related errors, etc.).
+          name: "Test PyInstaller executable"
+          command: |
+            dist/Tahoe-LAFS/tahoe --version
+
   debian-9: &DEBIAN
     docker:
       - image: "tahoelafsci/debian:9"

.travis.yml

@@ -29,6 +29,10 @@ script:
     else
       tox -e ${T}
     fi
+    # To verify that the resultant PyInstaller-generated binary executes
+    # cleanly (i.e., that it terminates with an exit code of 0 and isn't
+    # failing due to import/packaging-related errors, etc.).
+    if [ "${T}" = "pyinstaller" ]; then dist/Tahoe-LAFS/tahoe --version; fi
 
 after_success:
   - if [ "${T}" = "coverage" ]; then codecov; fi

misc/python3/tahoe-depgraph.py

@@ -111,7 +111,7 @@ def main(target):
         json_dump(mf.as_json(), outfile)
         outfile.write('\n')
 
-    ported_modules_path = os.path.join(target, "misc", "python3", "ported-modules.txt")
+    ported_modules_path = os.path.join(target, "src", "allmydata", "ported-modules.txt")
     with open(ported_modules_path) as ported_modules:
         port_status = dict.fromkeys((line.strip() for line in ported_modules), "ported")
     with open('tahoe-ported.json', 'wb') as outfile:

setup.py

@@ -55,6 +55,12 @@ install_requires = [
     # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs
     "foolscap >= 0.12.6",
 
+    # * cryptography 2.6 introduced some ed25519 APIs we rely on.  Note that
+    #   Twisted[conch] also depends on cryptography and Twisted[tls]
+    #   transitively depends on cryptography.  So it's anyone's guess what
+    #   version of cryptography will *really* be installed.
+    "cryptography >= 2.6",
+
     # * On Linux we need at least Twisted 10.1.0 for inotify support
     #   used by the drop-upload frontend.
     # * We also need Twisted 10.1.0 for the FTP frontend in order for
@@ -368,7 +374,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
                                       "static/*.js", "static/*.png", "static/*.css",
                                       "static/img/*.png",
                                       "static/css/*.css",
-                                      ]
+                                      ],
+                    "allmydata": ["ported-modules.txt"],
                     },
       include_package_data=True,
       setup_requires=setup_requires,

src/allmydata/test/test_python3.py

@@ -0,0 +1,107 @@
+"""
+Tests related to the Python 3 porting effort itself.
+"""
+
+from pkg_resources import (
+    resource_stream,
+)
+
+from twisted.python.modules import (
+    getModule,
+)
+from twisted.trial.unittest import (
+    SynchronousTestCase,
+)
+
+
+class Python3PortingEffortTests(SynchronousTestCase):
+    def test_finished_porting(self):
+        """
+        Tahoe-LAFS has been ported to Python 3.
+        """
+        tahoe_lafs_module_names = set(all_module_names("allmydata"))
+        ported_names = set(ported_module_names())
+        self.assertEqual(
+            tahoe_lafs_module_names - ported_names,
+            set(),
+            "Some unported modules remain: {}".format(
+                unported_report(
+                    tahoe_lafs_module_names,
+                    ported_names,
+                ),
+            ),
+        )
+    test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed"
+
+    def test_ported_modules_exist(self):
+        """
+        All modules listed as ported exist and belong to Tahoe-LAFS.
+        """
+        tahoe_lafs_module_names = set(all_module_names("allmydata"))
+        ported_names = set(ported_module_names())
+        unknown = ported_names - tahoe_lafs_module_names
+        self.assertEqual(
+            unknown,
+            set(),
+            "Some supposedly-ported modules weren't found: {}.".format(sorted(unknown)),
+        )
+
+    def test_ported_modules_distinct(self):
+        """
+        The ported modules list doesn't contain duplicates.
+        """
+        ported_names_list = ported_module_names()
+        ported_names_list.sort()
+        ported_names_set = set(ported_names_list)
+        ported_names_unique_list = list(ported_names_set)
+        ported_names_unique_list.sort()
+        self.assertEqual(
+            ported_names_list,
+            ported_names_unique_list,
+        )
+
+
+def all_module_names(toplevel):
+    """
+    :param unicode toplevel: The name of a top-level Python package.
+
+    :return iterator[unicode]: An iterator of ``unicode`` giving the names of
+        all modules within the given top-level Python package.
+    """
+    allmydata = getModule(toplevel)
+    for module in allmydata.walkModules():
+        yield module.name.decode("utf-8")
+
+
+def ported_module_names():
+    """
+    :return list[unicode]: A ``set`` of ``unicode`` giving the names of
+        Tahoe-LAFS modules which have been ported to Python 3.
+    """
+    return resource_stream(
+        "allmydata",
+        u"ported-modules.txt",
+    ).read().splitlines()
+
+
+def unported_report(tahoe_lafs_module_names, ported_names):
+    return """
+Ported files: {} / {}
+Ported lines: {} / {}
+""".format(
+    len(ported_names),
+    len(tahoe_lafs_module_names),
+    sum(map(count_lines, ported_names)),
+    sum(map(count_lines, tahoe_lafs_module_names)),
+)
+
+def count_lines(module_name):
+    module = getModule(module_name)
+    try:
+        source = module.filePath.getContent()
+    except Exception as e:
+        print(module_name, e)
+        return 0
+    lines = source.splitlines()
+    nonblank = filter(None, lines)
+    return len(nonblank)

src/allmydata/test/test_runner.py

@@ -170,21 +170,6 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin):
         d.addCallback(_cb)
         return d
 
-    def test_version_no_noise(self):
-        d = self.run_bintahoe(["--version"])
-        def _cb(res):
-            out, err, rc_or_sig = res
-            self.failUnlessEqual(rc_or_sig, 0, str(res))
-            self.failUnless(out.startswith(allmydata.__appname__+':'), str(res))
-            self.failIfIn("DeprecationWarning", out, str(res))
-            errlines = err.split("\n")
-            self.failIf([True for line in errlines if (line != "" and "UserWarning: Unbuilt egg for setuptools" not in line
-                                                                  and "from pkg_resources import load_entry_point" not in line)], str(res))
-            if err != "":
-                raise unittest.SkipTest("This test is known not to pass on Ubuntu Lucid; see #1235.")
-        d.addCallback(_cb)
-        return d
-
     @inlineCallbacks
     def test_help_eliot_destinations(self):
         out, err, rc_or_sig = yield self.run_bintahoe(["--help-eliot-destinations"])

src/allmydata/test/test_system.py

@@ -2428,7 +2428,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         def _run_in_subprocess(ignored, verb, *args, **kwargs):
             stdin = kwargs.get("stdin")
-            env = kwargs.get("env")
+            env = kwargs.get("env", os.environ)
+            # Python warnings from the child process don't matter.
+            env["PYTHONWARNINGS"] = "ignore"
             newargs = ["--node-directory", self.getdir("client0"), verb] + list(args)
             return self.run_bintahoe(newargs, stdin=stdin, env=env)
 

src/allmydata/test/web/test_root.py

@@ -1,8 +1,17 @@
+from mock import Mock
+
 from twisted.trial import unittest
+from twisted.web.test.requesthelper import DummyRequest
 
 from ...storage_client import NativeStorageServer
 from ...web.root import Root
 from ...util.connection_status import ConnectionStatus
+from allmydata.web.root import URIHandler
+from allmydata.web.common import WebError
+
+from hypothesis import given
+from hypothesis.strategies import text
+
 
 from ..common import (
     EMPTY_CLIENT_CONFIG,
@@ -14,6 +23,7 @@ class FakeRoot(Root):
     def now_fn(self):
         return 0
 
+
 class FakeContext(object):
     def __init__(self):
         self.slots = {}
@@ -21,12 +31,62 @@ class FakeContext(object):
     def fillSlots(self, slotname, contents):
         self.slots[slotname] = contents
 
+
+class RenderSlashUri(unittest.TestCase):
+    """
+    Ensure that URIs starting with /uri?uri= only accept valid
+    capabilities
+    """
+
+    def setUp(self):
+        self.request = DummyRequest(b"/uri")
+        self.request.fields = {}
+
+        def prepathURL():
+            return b"http://127.0.0.1.99999/" + b"/".join(self.request.prepath)
+
+        self.request.prePathURL = prepathURL
+        self.client = Mock()
+        self.res = URIHandler(self.client)
+
+    def test_valid(self):
+        """
+        A valid capbility does not result in error
+        """
+        self.request.args[b"uri"] = [(
+            b"URI:CHK:nt2xxmrccp7sursd6yh2thhcky:"
+            b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882"
+        )]
+        self.res.render_GET(self.request)
+
+    def test_invalid(self):
+        """
+        A (trivially) invalid capbility is an error
+        """
+        self.request.args[b"uri"] = [b"not a capability"]
+        with self.assertRaises(WebError):
+            self.res.render_GET(self.request)
+
+    @given(
+        text()
+    )
+    def test_hypothesis_error_caps(self, cap):
+        """
+        Let hypothesis try a bunch of invalid capabilities
+        """
+        self.request.args[b"uri"] = [cap.encode('utf8')]
+        with self.assertRaises(WebError):
+            self.res.render_GET(self.request)
+
+
 class RenderServiceRow(unittest.TestCase):
     def test_missing(self):
-        # minimally-defined static servers just need anonymous-storage-FURL
-        # and permutation-seed-base32. The WUI used to have problems
-        # rendering servers that lacked nickname and version. This tests that
-        # we can render such minimal servers.
+        """
+        minimally-defined static servers just need anonymous-storage-FURL
+        and permutation-seed-base32. The WUI used to have problems
+        rendering servers that lacked nickname and version. This tests that
+        we can render such minimal servers.
+        """
         ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
                "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
                }

src/allmydata/version_checks.py

@@ -23,11 +23,21 @@ from .util import (
     verlib,
 )
 
-_INSTALL_REQUIRES = list(
-    str(req)
-    for req
-    in pkg_resources.get_distribution(__appname__).requires()
-)
+if getattr(sys, 'frozen', None):
+    # "Frozen" python interpreters (i.e., standalone executables
+    # generated by PyInstaller and other, similar utilities) run
+    # independently of a traditional setuptools-based packaging
+    # environment, and so pkg_resources.get_distribution() cannot be
+    # used in such cases to gather a list of requirements at runtime
+    # (and because a frozen application is one that has already been
+    # "installed", an empty list suffices here).
+    _INSTALL_REQUIRES = []
+else:
+    _INSTALL_REQUIRES = list(
+        str(req)
+        for req
+        in pkg_resources.get_distribution(__appname__).requires()
+    )
 
 class PackagingError(EnvironmentError):
     """

src/allmydata/web/root.py

@@ -1,7 +1,17 @@
-import time, os, json
+import os
+import time
+import json
+import urllib
 
-from twisted.web import http
-from nevow import rend, url, tags as T
+from twisted.web import (
+    http,
+    resource,
+)
+from twisted.web.util import redirectTo
+
+from hyperlink import URL
+
+from nevow import rend, tags as T
 from nevow.inevow import IRequest
 from nevow.static import File as nevow_File # TODO: merge with static.File?
 from nevow.util import resource_filename
@@ -28,31 +38,53 @@ from allmydata.web.common import (
 from allmydata.web.private import (
     create_private_tree,
 )
+from allmydata import uri
 
-class URIHandler(RenderMixin, rend.Page):
-    # I live at /uri . There are several operations defined on /uri itself,
-    # mostly involved with creation of unlinked files and directories.
+class URIHandler(resource.Resource, object):
+    """
+    I live at /uri . There are several operations defined on /uri itself,
+    mostly involved with creation of unlinked files and directories.
+    """
 
     def __init__(self, client):
-        rend.Page.__init__(self, client)
+        super(URIHandler, self).__init__()
         self.client = client
 
-    def render_GET(self, ctx):
-        req = IRequest(ctx)
-        uri = get_arg(req, "uri", None)
-        if uri is None:
+    def render_GET(self, req):
+        """
+        Historically, accessing this via "GET /uri?uri=<capabilitiy>"
+        was/is a feature -- which simply redirects to the more-common
+        "GET /uri/<capability>" with any other query args
+        preserved. New code should use "/uri/<cap>"
+        """
+        uri_arg = req.args.get(b"uri", [None])[0]
+        if uri_arg is None:
             raise WebError("GET /uri requires uri=")
-        there = url.URL.fromContext(ctx)
-        there = there.clear("uri")
-        # I thought about escaping the childcap that we attach to the URL
-        # here, but it seems that nevow does that for us.
-        there = there.child(uri)
-        return there
 
-    def render_PUT(self, ctx):
-        req = IRequest(ctx)
-        # either "PUT /uri" to create an unlinked file, or
-        # "PUT /uri?t=mkdir" to create an unlinked directory
+        # shennanigans like putting "%2F" or just "/" itself, or ../
+        # etc in the <cap> might be a vector for weirdness so we
+        # validate that this is a valid capability before proceeding.
+        cap = uri.from_string(uri_arg)
+        if isinstance(cap, uri.UnknownURI):
+            raise WebError("Invalid capability")
+
+        # so, using URL.from_text(req.uri) isn't going to work because
+        # it seems Nevow was creating absolute URLs including
+        # host/port whereas req.uri is absolute (but lacks host/port)
+        redir_uri = URL.from_text(req.prePathURL().decode('utf8'))
+        redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8'))
+        # add back all the query args that AREN'T "?uri="
+        for k, values in req.args.items():
+            if k != b"uri":
+                for v in values:
+                    redir_uri = redir_uri.add(k.decode('utf8'), v.decode('utf8'))
+        return redirectTo(redir_uri.to_text().encode('utf8'), req)
+
+    def render_PUT(self, req):
+        """
+        either "PUT /uri" to create an unlinked file, or
+        "PUT /uri?t=mkdir" to create an unlinked directory
+        """
         t = get_arg(req, "t", "").strip()
         if t == "":
             file_format = get_format(req, "CHK")
@@ -63,15 +95,18 @@ class URIHandler(RenderMixin, rend.Page):
                 return unlinked.PUTUnlinkedCHK(req, self.client)
         if t == "mkdir":
             return unlinked.PUTUnlinkedCreateDirectory(req, self.client)
-        errmsg = ("/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, "
-                  "and POST?t=mkdir")
+        errmsg = (
+            "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, "
+            "and POST?t=mkdir"
+        )
         raise WebError(errmsg, http.BAD_REQUEST)
 
-    def render_POST(self, ctx):
-        # "POST /uri?t=upload&file=newfile" to upload an
-        # unlinked file or "POST /uri?t=mkdir" to create a
-        # new directory
-        req = IRequest(ctx)
+    def render_POST(self, req):
+        """
+        "POST /uri?t=upload&file=newfile" to upload an
+        unlinked file or "POST /uri?t=mkdir" to create a
+        new directory
+        """
         t = get_arg(req, "t", "").strip()
         if t in ("", "upload"):
             file_format = get_format(req)
@@ -92,14 +127,20 @@ class URIHandler(RenderMixin, rend.Page):
                   "and POST?t=mkdir")
         raise WebError(errmsg, http.BAD_REQUEST)
 
-    def childFactory(self, ctx, name):
-        # 'name' is expected to be a URI
+    def getChild(self, name, req):
+        """
+        Most requests look like /uri/<cap> so this fetches the capability
+        and creates and appropriate handler (depending on the kind of
+        capability it was passed).
+        """
         try:
             node = self.client.create_node_from_uri(name)
             return directory.make_handler_for(node, self.client)
         except (TypeError, AssertionError):
-            raise WebError("'%s' is not a valid file- or directory- cap"
-                           % name)
+            raise WebError(
+                "'{}' is not a valid file- or directory- cap".format(name)
+            )
+
 
 class FileHandler(rend.Page):
     # I handle /file/$FILECAP[/IGNORED] , which provides a URL from which a

tox.ini

@@ -185,7 +185,9 @@ deps =
 # Setting PYTHONHASHSEED to a known value assists with reproducible builds.
 # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build
 setenv=PYTHONHASHSEED=1
-commands=pyinstaller -y --clean pyinstaller.spec
+commands=
+    pip freeze
+    pyinstaller -y --clean pyinstaller.spec
 
 [testenv:tarballs]
 deps =