Merge remote-tracking branch 'origin/master' into 3672.non-utf-8-bytes-in-logs

.github/workflows/ci.yml

@@ -27,15 +27,6 @@ jobs:
 
     steps:
 
-      # Get vcpython27 on Windows + Python 2.7, to build netifaces
-      # extension.  See https://chocolatey.org/packages/vcpython27 and
-      # https://github.com/crazy-max/ghaction-chocolatey
-      - name: Install MSVC 9.0 for Python 2.7 [Windows]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install vcpython27
-
       # See https://github.com/actions/checkout. A fetch-depth of 0
       # fetches all tags and branches.
       - name: Check out Tahoe-LAFS sources
@@ -44,10 +35,35 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
+        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
+      # See note below about need for using 32-bit Python 2.7 on
+      # Windows.  The extra handling here for Python 3.6 on Windows is
+      # because I could not figure out the right GitHub Actions
+      # expression to do this in a better way.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x64]
+        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '3.6' ) }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: 'x64'
+
+      # We use netifaces, which does not ship a 64-bit wheel for the
+      # Python 2.7 + Windows combination, but it ships a 32-bit wheel.
+      # Since MS has removed vcpython27 compiler downloads from their
+      # usual download site, building a netifaces wheel locally is not
+      # an option anymore.  So let us just test with 32-bit Python on
+      # Windows.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
+        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '2.7' ) }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: 'x86'
+
       # To use pip caching with GitHub Actions in an OS-independent
       # manner, we need `pip cache dir` command, which became
       # available since pip v20.1+.  At the time of writing this,
@@ -164,15 +180,6 @@ jobs:
 
     steps:
 
-      # Get vcpython27 for Windows + Python 2.7, to build netifaces
-      # extension.  See https://chocolatey.org/packages/vcpython27 and
-      # https://github.com/crazy-max/ghaction-chocolatey
-      - name: Install MSVC 9.0 for Python 2.7 [Windows]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install vcpython27
-
       - name: Install Tor [Ubuntu]
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt install tor
@@ -193,10 +200,19 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
+        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
+      # See this step under coverage job.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: 'x86'
+
       - name: Get pip cache directory
         id: pip-cache
         run: |
@@ -242,25 +258,25 @@ jobs:
 
     steps:
 
-      # Get vcpython27 for Windows + Python 2.7, to build netifaces
-      # extension.  See https://chocolatey.org/packages/vcpython27 and
-      # https://github.com/crazy-max/ghaction-chocolatey
-      - name: Install MSVC 9.0 for Python 2.7 [Windows]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install vcpython27
-
       - name: Check out Tahoe-LAFS sources
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
+        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
+      # See this step under coverage job.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: 'x86'
+
       - name: Get pip cache directory
         id: pip-cache
         run: |

docs/frontends/FTP-and-SFTP.rst

@@ -7,11 +7,10 @@ Tahoe-LAFS SFTP Frontend
 1.  `SFTP Background`_
 2.  `Tahoe-LAFS Support`_
 3.  `Creating an Account File`_
-4.  `Running An Account Server (accounts.url)`_
-5.  `Configuring SFTP Access`_
-6.  `Dependencies`_
-7.  `Immutable and Mutable Files`_
-8.  `Known Issues`_
+4.  `Configuring SFTP Access`_
+5.  `Dependencies`_
+6.  `Immutable and Mutable Files`_
+7.  `Known Issues`_
 
 
 SFTP Background
@@ -78,33 +77,6 @@ start with "ssh-".
 Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in
 the next sections.
 
-Running An Account Server (accounts.url)
-========================================
-
-The accounts.url directive allows access requests to be controlled by an
-HTTP-based login service, useful for centralized deployments. This was used
-by AllMyData to provide web-based file access, where the service used a
-simple PHP script and database lookups to map an account email address and
-password to a Tahoe-LAFS directory cap. The service will receive a
-multipart/form-data POST, just like one created with a <form> and <input>
-fields, with three parameters:
-
-• action: "authenticate" (this is a static string)
-• email: USERNAME (Tahoe-LAFS has no notion of email addresses, but the
-  authentication service uses them as account names, so the interface
-  presents this argument as "email" rather than "username").
-• passwd: PASSWORD
-
-It should return a single string that either contains a Tahoe-LAFS directory
-cap (URI:DIR2:...), or "0" to indicate a login failure.
-
-Tahoe-LAFS recommends the service be secure, preferably localhost-only.  This
-makes it harder for attackers to brute force the password or use DNS
-poisoning to cause the Tahoe-LAFS gateway to talk with the wrong server,
-thereby revealing the usernames and passwords.
-
-Public key authentication is not supported when an account server is used.
-
 Configuring SFTP Access
 =======================
 

integration/test_servers_of_happiness.py

@@ -30,7 +30,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
         proto,
         sys.executable,
         [
-            sys.executable, '-m', 'allmydata.scripts.runner',
+            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', node_dir,
             'put', __file__,
         ]

integration/test_tor.py

@@ -46,7 +46,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
         proto,
         sys.executable,
         (
-            sys.executable, '-m', 'allmydata.scripts.runner',
+            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'carol'),
             'put', gold_path,
         )
@@ -60,7 +60,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
         proto,
         sys.executable,
         (
-            sys.executable, '-m', 'allmydata.scripts.runner',
+            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'dave'),
             'get', cap,
         )
@@ -84,7 +84,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
             proto,
             sys.executable,
             (
-                sys.executable, '-m', 'allmydata.scripts.runner',
+                sys.executable, '-b', '-m', 'allmydata.scripts.runner',
                 'create-node',
                 '--nickname', name,
                 '--introducer', introducer_furl,

integration/util.py

@@ -152,9 +152,9 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
     `--coverage` option if the `request` indicates we should.
     """
     if request.config.getoption('coverage'):
-        args = [sys.executable, '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage']
+        args = [sys.executable, '-b', '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage']
     else:
-        args = [sys.executable, '-m', 'allmydata.scripts.runner']
+        args = [sys.executable, '-b', '-m', 'allmydata.scripts.runner']
     args += other_args
     return reactor.spawnProcess(
         proto,

newsfragments/3652.removed

@@ -0,0 +1 @@
+Removed support for the Account Server frontend authentication type.

newsfragments/3681.installation

@@ -0,0 +1,3 @@
+Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has
+removed vcpython27 compiler downloads from their site, and Tahoe-LAFS
+needs vcpython27 to build and install netifaces on 64-bit Windows.

src/allmydata/__init__.py

@@ -8,7 +8,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2
+from future.utils import PY2, PY3
 if PY2:
     # Don't import future str() so we don't break Foolscap serialization on Python 2.
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
@@ -62,3 +62,18 @@ standard_library.install_aliases()
 from ._monkeypatch import patch
 patch()
 del patch
+
+
+# On Python 3, turn BytesWarnings into exceptions. This can have potential
+# production impact... if BytesWarnings are actually present in the codebase.
+# Given that this has been enabled before Python 3 Tahoe-LAFS was publicly
+# released, no such code should exist, and this will ensure it doesn't get
+# added either.
+#
+# Also note that BytesWarnings only happen if Python is run with -b option, so
+# in practice this should only affect tests.
+if PY3:
+    import warnings
+    # Error on BytesWarnings, to catch things like str(b""), but only for
+    # allmydata code.
+    warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*")

src/allmydata/client.py

@@ -116,7 +116,6 @@ _client_config = configutil.ValidConfiguration(
         ),
         "sftpd": (
             "accounts.file",
-            "accounts.url",
             "enabled",
             "host_privkey_file",
             "host_pubkey_file",
@@ -1042,13 +1041,12 @@ class _Client(node.Node, pollmixin.PollMixin):
             accountfile = self.config.get_config("sftpd", "accounts.file", None)
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
-            accounturl = self.config.get_config("sftpd", "accounts.url", None)
             sftp_portstr = self.config.get_config("sftpd", "port", "tcp:8022")
             pubkey_file = self.config.get_config("sftpd", "host_pubkey_file")
             privkey_file = self.config.get_config("sftpd", "host_privkey_file")
 
             from allmydata.frontends import sftpd
-            s = sftpd.SFTPServer(self, accountfile, accounturl,
+            s = sftpd.SFTPServer(self, accountfile,
                                  sftp_portstr, pubkey_file, privkey_file)
             s.setServiceParent(self)
 

src/allmydata/frontends/auth.py

@@ -1,14 +1,10 @@
-import os
-
 from zope.interface import implementer
-from twisted.web.client import getPage
 from twisted.internet import defer
 from twisted.cred import error, checkers, credentials
 from twisted.conch.ssh import keys
 from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB
 
 from allmydata.util.dictutil import BytesKeyDict
-from allmydata.util import base32
 from allmydata.util.fileutil import abspath_expanduser_unicode
 
 
@@ -86,54 +82,3 @@ class AccountFileChecker(object):
         d = defer.maybeDeferred(creds.checkPassword, correct)
         d.addCallback(self._cbPasswordMatch, str(creds.username))
         return d
-
-
-@implementer(checkers.ICredentialsChecker)
-class AccountURLChecker(object):
-    credentialInterfaces = (credentials.IUsernamePassword,)
-
-    def __init__(self, client, auth_url):
-        self.client = client
-        self.auth_url = auth_url
-
-    def _cbPasswordMatch(self, rootcap, username):
-        return FTPAvatarID(username, rootcap)
-
-    def post_form(self, username, password):
-        sepbase = base32.b2a(os.urandom(4))
-        sep = "--" + sepbase
-        form = []
-        form.append(sep)
-        fields = {"action": "authenticate",
-                  "email": username,
-                  "passwd": password,
-                  }
-        for name, value in fields.iteritems():
-            form.append('Content-Disposition: form-data; name="%s"' % name)
-            form.append('')
-            assert isinstance(value, str)
-            form.append(value)
-            form.append(sep)
-        form[-1] += "--"
-        body = "\r\n".join(form) + "\r\n"
-        headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
-                   }
-        return getPage(self.auth_url, method="POST",
-                       postdata=body, headers=headers,
-                       followRedirect=True, timeout=30)
-
-    def _parse_response(self, res):
-        rootcap = res.strip()
-        if rootcap == "0":
-            raise error.UnauthorizedLogin
-        return rootcap
-
-    def requestAvatarId(self, credentials):
-        # construct a POST to the login form. While this could theoretically
-        # be done with something like the stdlib 'email' package, I can't
-        # figure out how, so we just slam together a form manually.
-        d = self.post_form(credentials.username, credentials.password)
-        d.addCallback(self._parse_response)
-        d.addCallback(self._cbPasswordMatch, str(credentials.username))
-        return d
-

src/allmydata/frontends/sftpd.py

@@ -1983,7 +1983,7 @@ class ShellSession(PrefixingLogMixin):
 components.registerAdapter(ShellSession, SFTPUserHandler, ISession)
 
 
-from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme
+from allmydata.frontends.auth import AccountFileChecker, NeedRootcapLookupScheme
 
 @implementer(portal.IRealm)
 class Dispatcher(object):
@@ -2000,7 +2000,7 @@ class Dispatcher(object):
 class SFTPServer(service.MultiService):
     name = "frontend:sftp"
 
-    def __init__(self, client, accountfile, accounturl,
+    def __init__(self, client, accountfile,
                  sftp_portstr, pubkey_file, privkey_file):
         precondition(isinstance(accountfile, (str, type(None))), accountfile)
         precondition(isinstance(pubkey_file, str), pubkey_file)
@@ -2013,12 +2013,9 @@ class SFTPServer(service.MultiService):
         if accountfile:
             c = AccountFileChecker(self, accountfile)
             p.registerChecker(c)
-        if accounturl:
-            c = AccountURLChecker(self, accounturl)
-            p.registerChecker(c)
-        if not accountfile and not accounturl:
+        if not accountfile:
             # we could leave this anonymous, with just the /uri/CAP form
-            raise NeedRootcapLookupScheme("must provide an account file or URL")
+            raise NeedRootcapLookupScheme("must provide an account file")
 
         pubkey = keys.Key.fromFile(pubkey_file.encode(get_filesystem_encoding()))
         privkey = keys.Key.fromFile(privkey_file.encode(get_filesystem_encoding()))

src/allmydata/test/__init__.py

@@ -20,11 +20,10 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-import warnings
 from traceback import extract_stack, format_list
 
 from foolscap.pb import Listener
@@ -33,11 +32,6 @@ from twisted.application import service
 
 from foolscap.logging.incident import IncidentQualifier
 
-if PY3:
-    # Error on BytesWarnings, to catch things like str(b""), but only for
-    # allmydata code.
-    warnings.filterwarnings("error", category=BytesWarning, module="allmydata.*")
-
 
 class NonQualifier(IncidentQualifier, object):
     def check_event(self, ev):

src/allmydata/test/cli_node_api.py

@@ -154,6 +154,7 @@ class CLINodeAPI(object):
         exe = sys.executable
         argv = [
             exe,
+            "-b",
             u"-m",
             u"allmydata.scripts.runner",
         ] + argv

src/allmydata/test/test_download.py

@@ -1304,7 +1304,7 @@ class MyShare(object):
         self._dyhb_rtt = rtt
 
     def __repr__(self):
-        return "sh%d-on-%s" % (self._shnum, self._server.get_name())
+        return "sh%d-on-%s" % (self._shnum, str(self._server.get_name(), "ascii"))
 
 class MySegmentFetcher(SegmentFetcher):
     def __init__(self, *args, **kwargs):
@@ -1383,7 +1383,7 @@ class Selection(unittest.TestCase):
             self.failUnless(node.failed)
             self.failUnless(node.failed.check(NotEnoughSharesError))
             sname = serverA.get_name()
-            self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused="  % sname,
+            self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused="  % str(sname, "ascii"),
                               str(node.failed))
         d.addCallback(_check2)
         return d
@@ -1605,7 +1605,7 @@ class Selection(unittest.TestCase):
             self.failUnless(node.failed)
             self.failUnless(node.failed.check(NotEnoughSharesError))
             sname = servers[b"peer-2"].get_name()
-            self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % sname,
+            self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % str(sname, "ascii"),
                               str(node.failed))
         d.addCallback(_check4)
         return d

src/allmydata/test/test_runner.py

@@ -88,7 +88,7 @@ def run_bintahoe(extra_argv, python_options=None):
     argv = [executable]
     if python_options is not None:
         argv.extend(python_options)
-    argv.extend([u"-m", u"allmydata.scripts.runner"])
+    argv.extend([u"-b", u"-m", u"allmydata.scripts.runner"])
     argv.extend(extra_argv)
     argv = list(unicode_to_argv(arg) for arg in argv)
     p = Popen(argv, stdout=PIPE, stderr=PIPE)
@@ -515,7 +515,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
             0,
             "Expected error message from '{}', got something else: {}".format(
                 description,
-                p.get_buffered_output(),
+                str(p.get_buffered_output(), "utf-8"),
             ),
         )
 

src/allmydata/test/test_system.py

@@ -76,7 +76,7 @@ class RunBinTahoeMixin(object):
         # support env yet and is also synchronous.  If we could get rid of
         # this in favor of that, though, it would probably be an improvement.
         command = sys.executable
-        argv = python_options + ["-m", "allmydata.scripts.runner"] + args
+        argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args
 
         if env is None:
             env = os.environ