Merge branch '3716.allmydata-scripts-python-3-part-1' into 3718.allmydata-scripts-python-3-part-2

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
+          - windows-latest
           - macos-latest
           - ubuntu-latest
         python-version:
@@ -26,11 +27,6 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
-        include:
-          # For now we're only doing Windows on 2.7, will be fixed in
-          # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701
-          - os: windows-latest
-            python-version: 2.7
 
     steps:
       # See https://github.com/actions/checkout. A fetch-depth of 0

README.rst

@@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site <http:
 
 Get involved with the Tahoe-LAFS community:
 
--  Chat with Tahoe-LAFS developers at #tahoe-lafs chat on irc.freenode.net or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
+-  Chat with Tahoe-LAFS developers at ``#tahoe-lafs`` channel on `libera.chat <https://libera.chat/>`__ IRC network or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
 
 -  Join our `weekly conference calls <https://www.tahoe-lafs.org/trac/tahoe-lafs/wiki/WeeklyMeeting>`__ with core developers and interested community members.
 

docs/frontends/CLI.rst

@@ -514,10 +514,10 @@ Command Examples
  the pattern will be matched against any level of the directory tree;
  it's still impossible to specify absolute path exclusions.
 
-``tahoe backup --exclude-from=/path/to/filename ~ work:backups``
+``tahoe backup --exclude-from-utf-8=/path/to/filename ~ work:backups``
 
- ``--exclude-from`` is similar to ``--exclude``, but reads exclusion
+ ``--exclude-from-utf-8`` is similar to ``--exclude``, but reads exclusion
- patterns from ``/path/to/filename``, one per line.
+ patterns from a UTF-8-encoded ``/path/to/filename``, one per line.
 
 ``tahoe backup --exclude-vcs ~ work:backups``
 

docs/running.rst

@@ -235,7 +235,7 @@ Socialize
 =========
 
 You can chat with other users of and hackers of this software on the
-#tahoe-lafs IRC channel at ``irc.freenode.net``, or on the `tahoe-dev mailing
+#tahoe-lafs IRC channel at ``irc.libera.chat``, or on the `tahoe-dev mailing
 list`_.
 
 .. _tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev

integration/test_get_put.py

@@ -0,0 +1,64 @@
+"""
+Integration tests for getting and putting files, including reading from stdin
+and stdout.
+"""
+
+from subprocess import Popen, PIPE
+
+import pytest
+
+from .util import run_in_thread, cli
+
+DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11"
+try:
+    DATA.decode("utf-8")
+except UnicodeDecodeError:
+    pass  # great, what we want
+else:
+    raise ValueError("BUG, the DATA string was decoded from UTF-8")
+
+
+@pytest.fixture(scope="session")
+def get_put_alias(alice):
+    cli(alice, "create-alias", "getput")
+
+
+def read_bytes(path):
+    with open(path, "rb") as f:
+        return f.read()
+
+
+@run_in_thread
+def test_put_from_stdin(alice, get_put_alias, tmpdir):
+    """
+    It's possible to upload a file via `tahoe put`'s STDIN, and then download
+    it to a file.
+    """
+    tempfile = str(tmpdir.join("file"))
+    p = Popen(
+        ["tahoe", "--node-directory", alice.node_dir, "put", "-", "getput:fromstdin"],
+        stdin=PIPE
+    )
+    p.stdin.write(DATA)
+    p.stdin.close()
+    assert p.wait() == 0
+
+    cli(alice, "get", "getput:fromstdin", tempfile)
+    assert read_bytes(tempfile) == DATA
+
+
+def test_get_to_stdout(alice, get_put_alias, tmpdir):
+    """
+    It's possible to upload a file, and then download it to stdout.
+    """
+    tempfile = tmpdir.join("file")
+    with tempfile.open("wb") as f:
+        f.write(DATA)
+    cli(alice, "put", str(tempfile), "getput:tostdout")
+
+    p = Popen(
+        ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"],
+        stdout=PIPE
+    )
+    assert p.stdout.read() == DATA
+    assert p.wait() == 0

newsfragments/3716.incompat

@@ -0,0 +1 @@
+tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded.

newsfragments/3721.documentation

@@ -0,0 +1 @@
+Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat.

src/allmydata/scripts/cli.py

@@ -357,12 +357,12 @@ class BackupOptions(FileStoreOptions):
             exclude = self['exclude']
             exclude.add(g)
 
-    def opt_exclude_from(self, filepath):
+    def opt_exclude_from_utf_8(self, filepath):
         """Ignore file matching glob patterns listed in file, one per
         line. The file is assumed to be in the argv encoding."""
         abs_filepath = argv_to_abspath(filepath)
         try:
-            exclude_file = open(abs_filepath)
+            exclude_file = open(abs_filepath, "r", encoding="utf-8")
         except Exception as e:
             raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % (
                 quote_local_unicode_path(abs_filepath), e))

src/allmydata/scripts/tahoe_cp.py

@@ -701,6 +701,8 @@ class Copier(object):
 
 
     def need_to_copy_bytes(self, source, target):
+        # This should likley be a method call! but enabling that triggers
+        # additional bugs. https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3719
         if source.need_to_copy_bytes:
             # mutable tahoe files, and local files
             return True

src/allmydata/test/cli/test_backup.py

@@ -10,7 +10,6 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-
 import os.path
 from six.moves import cStringIO as StringIO
 from datetime import timedelta
@@ -354,14 +353,14 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         exclusion_string = "_darcs\n*py\n.svn"
         excl_filepath = os.path.join(basedir, 'exclusion')
         fileutil.write(excl_filepath, exclusion_string)
-        backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to'])
+        backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to'])
         filtered = list(backup_options.filter_listdir(subdir_listdir))
         self._check_filtering(filtered, subdir_listdir, (u'another_doc.lyx', u'CVS'),
                               (u'.svn', u'_darcs', u'run_snake_run.py'))
         # test BackupConfigurationError
         self.failUnlessRaises(cli.BackupConfigurationError,
                               parse,
-                              ['--exclude-from', excl_filepath + '.no', 'from', 'to'])
+                              ['--exclude-from-utf-8', excl_filepath + '.no', 'from', 'to'])
 
         # test that an iterator works too
         backup_options = parse(['--exclude', '*lyx', 'from', 'to'])
@@ -372,7 +371,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
     def test_exclude_options_unicode(self):
         nice_doc = u"nice_d\u00F8c.lyx"
         try:
-            doc_pattern_arg = u"*d\u00F8c*".encode(get_io_encoding())
+            doc_pattern_arg_unicode = doc_pattern_arg = u"*d\u00F8c*"
+            if PY2:
+                doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding())
         except UnicodeEncodeError:
             raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
 
@@ -394,10 +395,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'),
                              (nice_doc, u'lib.a'))
         # read exclude patterns from file
-        exclusion_string = doc_pattern_arg + b"\nlib.?"
+        exclusion_string = (doc_pattern_arg_unicode + "\nlib.?").encode("utf-8")
         excl_filepath = os.path.join(basedir, 'exclusion')
         fileutil.write(excl_filepath, exclusion_string)
-        backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to'])
+        backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to'])
         filtered = list(backup_options.filter_listdir(root_listdir))
         self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'),
                              (nice_doc, u'lib.a'))
@@ -420,20 +421,20 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         ns = Namespace()
         ns.called = False
         original_open = open
-        def call_file(name, *args):
+        def call_file(name, *args, **kwargs):
             if name.endswith("excludes.dummy"):
                 ns.called = True
                 self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file))
                 return StringIO()
             else:
-                return original_open(name, *args)
+                return original_open(name, *args, **kwargs)
 
         if PY2:
             from allmydata.scripts import cli as module_to_patch
         else:
             import builtins as module_to_patch
         patcher = MonkeyPatcher((module_to_patch, 'open', call_file))
-        patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to'])
+        patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from-utf-8', unicode_to_argv(exclude_file), 'from', 'to'])
         self.failUnless(ns.called)
 
     def test_ignore_symlinks(self):

src/allmydata/test/cli/test_check.py

@@ -15,7 +15,7 @@ from six.moves import cStringIO as StringIO
 
 from allmydata import uri
 from allmydata.util import base32
-from allmydata.util.encodingutil import to_bytes
+from allmydata.util.encodingutil import to_bytes, quote_output_u
 from allmydata.mutable.publish import MutableData
 from allmydata.immutable import upload
 from allmydata.scripts import debug
@@ -168,7 +168,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.uris = {}
         self.fileurls = {}
         DATA = b"data" * 100
-        quoted_good = u"'g\u00F6\u00F6d'"
+        quoted_good = quote_output_u("g\u00F6\u00F6d")
 
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):

src/allmydata/test/cli/test_cp.py

@@ -238,6 +238,66 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         return d
 
+    @defer.inlineCallbacks
+    def test_cp_duplicate_directories(self):
+        self.basedir = "cli/Cp/cp_duplicate_directories"
+        self.set_up_grid(oneshare=True)
+
+        filename = os.path.join(self.basedir, "file")
+        data = b"abc\xff\x00\xee"
+        with open(filename, "wb") as f:
+            f.write(data)
+
+        yield self.do_cli("create-alias", "tahoe")
+        (rc, out, err) = yield self.do_cli("mkdir", "tahoe:test1")
+        self.assertEqual(rc, 0, (rc, err))
+        dircap = out.strip()
+
+        (rc, out, err) = yield self.do_cli("cp", filename, "tahoe:test1/file")
+        self.assertEqual(rc, 0, (rc, err))
+
+        # Now duplicate dirnode, testing duplicates on destination side:
+        (rc, out, err) = yield self.do_cli(
+            "cp", "--recursive", dircap, "tahoe:test2/")
+        self.assertEqual(rc, 0, (rc, err))
+        (rc, out, err) = yield self.do_cli(
+            "cp", "--recursive", dircap, "tahoe:test3/")
+        self.assertEqual(rc, 0, (rc, err))
+
+        # Now copy to local directory, testing duplicates on origin side:
+        yield self.do_cli("cp", "--recursive", "tahoe:", self.basedir)
+
+        for i in range(1, 4):
+            with open(os.path.join(self.basedir, "test%d" % (i,), "file"), "rb") as f:
+                self.assertEquals(f.read(), data)
+
+    @defer.inlineCallbacks
+    def test_cp_immutable_file(self):
+        self.basedir = "cli/Cp/cp_immutable_file"
+        self.set_up_grid(oneshare=True)
+
+        filename = os.path.join(self.basedir, "source_file")
+        data = b"abc\xff\x00\xee"
+        with open(filename, "wb") as f:
+            f.write(data)
+
+        # Create immutable file:
+        yield self.do_cli("create-alias", "tahoe")
+        (rc, out, _) = yield self.do_cli("put", filename, "tahoe:file1")
+        filecap = out.strip()
+        self.assertEqual(rc, 0)
+
+        # Copy it:
+        (rc, _, _) = yield self.do_cli("cp", "tahoe:file1", "tahoe:file2")
+        self.assertEqual(rc, 0)
+
+        # Make sure resulting file is the same:
+        (rc, _, _) = yield self.do_cli("cp", "--recursive", "--caps-only",
+                                       "tahoe:", self.basedir)
+        self.assertEqual(rc, 0)
+        with open(os.path.join(self.basedir, "file2")) as f:
+            self.assertEqual(f.read().strip(), filecap)
+
     def test_cp_replaces_mutable_file_contents(self):
         self.basedir = "cli/Cp/cp_replaces_mutable_file_contents"
         self.set_up_grid(oneshare=True)

src/allmydata/test/cli/test_create_alias.py

@@ -10,7 +10,6 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from six import ensure_text
 from six.moves import StringIO
 import os.path
 from twisted.trial import unittest
@@ -20,7 +19,7 @@ from allmydata.util import fileutil
 from allmydata.scripts.common import get_aliases
 from allmydata.scripts import cli, runner
 from ..no_network import GridTestMixin
-from allmydata.util.encodingutil import quote_output
+from allmydata.util.encodingutil import quote_output_u
 from .common import CLITestMixin
 
 class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
@@ -182,7 +181,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
             self.assertEqual(len(err), 0, err)
-            self.failUnlessIn(u"Alias %s created" % ensure_text(quote_output(etudes_arg)), out)
+            self.failUnlessIn(u"Alias %s created" % (quote_output_u(etudes_arg),), out)
 
             aliases = get_aliases(self.get_clientdir())
             self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:"))

src/allmydata/test/cli/test_put.py

@@ -486,3 +486,20 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.failUnlessReallyEqual(rc_out_err[1], DATA))
 
         return d
+
+    def test_no_leading_slash(self):
+        self.basedir = "cli/Put/leading_slash"
+        self.set_up_grid(oneshare=True)
+
+        fn1 = os.path.join(self.basedir, "DATA1")
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res:
+                      self.do_cli("put", fn1, "tahoe:/uploaded.txt"))
+        def _check(args):
+            (rc, out, err) = args
+            self.assertEqual(rc, 1)
+            self.failUnlessIn("must not start with a slash", err)
+            self.assertEqual(len(out), 0, out)
+        d.addCallback(_check)
+        return d

src/allmydata/test/cli/test_status.py

@@ -130,9 +130,10 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(_check)
         return d
 
-    @mock.patch('sys.stdout')
+    @defer.inlineCallbacks
-    def test_help(self, fake):
+    def test_help(self):
-        return self.do_cli('status', '--help')
+        rc, _, _ = yield self.do_cli('status', '--help')
+        self.assertEqual(rc, 0)
 
 
 class CommandStatus(unittest.TestCase):

src/allmydata/test/common_util.py

@@ -12,6 +12,7 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
+import sys
 import time
 import signal
 from random import randrange
@@ -85,7 +86,7 @@ def run_cli_native(verb, *args, **kwargs):
         bytes.
     """
     nodeargs = kwargs.pop("nodeargs", [])
-    encoding = kwargs.pop("encoding", None) or "utf-8"
+    encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8"
     return_bytes = kwargs.pop("return_bytes", False)
     verb = maybe_unicode_to_argv(verb)
     args = [maybe_unicode_to_argv(a) for a in args]

src/allmydata/test/test_encodingutil.py

@@ -379,7 +379,10 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase):
         check(u"\n",       u"\"\\x0a\"", quote_newlines=True)
 
     def test_quote_output_default(self):
-        self.test_quote_output_utf8(None)
+        """Default is the encoding of sys.stdout if known, otherwise utf-8."""
+        encoding = getattr(sys.stdout, "encoding") or "utf-8"
+        self.assertEqual(quote_output(u"\u2621"),
+                         quote_output(u"\u2621", encoding=encoding))
 
 
 def win32_other(win32, other):

src/allmydata/test/test_runner.py

@@ -17,6 +17,7 @@ from six import ensure_text
 
 import os.path, re, sys
 from os import linesep
+import locale
 
 from eliot import (
     log_call,
@@ -92,8 +93,12 @@ def run_bintahoe(extra_argv, python_options=None):
     argv.extend(extra_argv)
     argv = list(unicode_to_argv(arg) for arg in argv)
     p = Popen(argv, stdout=PIPE, stderr=PIPE)
-    out = p.stdout.read().decode("utf-8")
+    if PY2:
-    err = p.stderr.read().decode("utf-8")
+        encoding = "utf-8"
+    else:
+        encoding = locale.getpreferredencoding(False)
+    out = p.stdout.read().decode(encoding)
+    err = p.stderr.read().decode(encoding)
     returncode = p.wait()
     return (out, err, returncode)
 
@@ -103,7 +108,7 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase):
         """
         The runner script receives unmangled non-ASCII values in argv.
         """
-        tricky = u"\u2621"
+        tricky = u"\u00F6"
         out, err, returncode = run_bintahoe([tricky])
         self.assertEqual(returncode, 1)
         self.assertIn(u"Unknown command: " + tricky, out)

src/allmydata/test/test_windows.py

@@ -79,6 +79,7 @@ slow_settings = settings(
 )
 
 @skipUnless(platform.isWindows(), "get_argv is Windows-only")
+@skipUnless(PY2, "Not used on Python 3.")
 class GetArgvTests(SyncTestCase):
     """
     Tests for ``get_argv``.
@@ -172,6 +173,7 @@ class GetArgvTests(SyncTestCase):
 
 
 @skipUnless(platform.isWindows(), "intended for Windows-only codepaths")
+@skipUnless(PY2, "Not used on Python 3.")
 class UnicodeOutputTests(SyncTestCase):
     """
     Tests for writing unicode to stdout and stderr.

src/allmydata/util/encodingutil.py

@@ -256,7 +256,11 @@ def quote_output_u(*args, **kwargs):
     result = quote_output(*args, **kwargs)
     if isinstance(result, unicode):
         return result
-    return result.decode(kwargs.get("encoding", None) or io_encoding)
+    # Since we're quoting, the assumption is this will be read by a human, and
+    # therefore printed, so stdout's encoding is the plausible one. io_encoding
+    # is now always utf-8.
+    return result.decode(kwargs.get("encoding", None) or
+                         getattr(sys.stdout, "encoding") or io_encoding)
 
 
 def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None):
@@ -276,7 +280,10 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None):
     On Python 3, returns Unicode strings.
     """
     precondition(isinstance(s, (bytes, unicode)), s)
-    encoding = encoding or io_encoding
+    # Since we're quoting, the assumption is this will be read by a human, and
+    # therefore printed, so stdout's encoding is the plausible one. io_encoding
+    # is now always utf-8.
+    encoding = encoding or getattr(sys.stdout, "encoding") or io_encoding
 
     if quote_newlines is None:
         quote_newlines = quotemarks
@@ -284,7 +291,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None):
     def _encode(s):
         if isinstance(s, bytes):
             try:
-                s = s.decode('utf-8')
+                s = s.decode("utf-8")
             except UnicodeDecodeError:
                 return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),)
 

src/allmydata/windows/fixups.py

@@ -1,4 +1,6 @@
 from __future__ import print_function
+
+from future.utils import PY3
 from past.builtins import unicode
 
 # This code isn't loadable or sensible except on Windows.  Importers all know
@@ -122,6 +124,10 @@ def initialize():
 
     SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX)
 
+    if PY3:
+        # The rest of this appears to be Python 2-specific
+        return
+
     original_stderr = sys.stderr
 
     # If any exception occurs in this code, we'll probably try to print it on stderr,

tox.ini

@@ -9,9 +9,9 @@
 python =
     2.7: py27-coverage,codechecks
     3.6: py36-coverage
-    3.7: py37-coverage
+    3.7: py37-coverage,typechecks,codechecks3
     3.8: py38-coverage
-    3.9: py39-coverage,typechecks,codechecks3
+    3.9: py39-coverage
     pypy-3.7: pypy3
 
 [pytest]