Merge remote-tracking branch 'origin/master' into 3716.allmydata-scripts-python-3-part-1

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
+          - windows-latest
           - macos-latest
           - ubuntu-latest
         python-version:
@@ -26,11 +27,6 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
-        include:
-          # For now we're only doing Windows on 2.7, will be fixed in
-          # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701
-          - os: windows-latest
-            python-version: 2.7
 
     steps:
       # See https://github.com/actions/checkout. A fetch-depth of 0

README.rst

@@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site <http:
 
 Get involved with the Tahoe-LAFS community:
 
--  Chat with Tahoe-LAFS developers at #tahoe-lafs chat on irc.freenode.net or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
+-  Chat with Tahoe-LAFS developers at ``#tahoe-lafs`` channel on `libera.chat <https://libera.chat/>`__ IRC network or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
 
 -  Join our `weekly conference calls <https://www.tahoe-lafs.org/trac/tahoe-lafs/wiki/WeeklyMeeting>`__ with core developers and interested community members.
 

docs/running.rst

@@ -235,7 +235,7 @@ Socialize
 =========
 
 You can chat with other users of and hackers of this software on the
-#tahoe-lafs IRC channel at ``irc.freenode.net``, or on the `tahoe-dev mailing
+#tahoe-lafs IRC channel at ``irc.libera.chat``, or on the `tahoe-dev mailing
 list`_.
 
 .. _tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev

integration/test_get_put.py

@@ -0,0 +1,64 @@
+"""
+Integration tests for getting and putting files, including reading from stdin
+and stdout.
+"""
+
+from subprocess import Popen, PIPE
+
+import pytest
+
+from .util import run_in_thread, cli
+
+DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11"
+try:
+    DATA.decode("utf-8")
+except UnicodeDecodeError:
+    pass  # great, what we want
+else:
+    raise ValueError("BUG, the DATA string was decoded from UTF-8")
+
+
+@pytest.fixture(scope="session")
+def get_put_alias(alice):
+    cli(alice, "create-alias", "getput")
+
+
+def read_bytes(path):
+    with open(path, "rb") as f:
+        return f.read()
+
+
+@run_in_thread
+def test_put_from_stdin(alice, get_put_alias, tmpdir):
+    """
+    It's possible to upload a file via `tahoe put`'s STDIN, and then download
+    it to a file.
+    """
+    tempfile = str(tmpdir.join("file"))
+    p = Popen(
+        ["tahoe", "--node-directory", alice.node_dir, "put", "-", "getput:fromstdin"],
+        stdin=PIPE
+    )
+    p.stdin.write(DATA)
+    p.stdin.close()
+    assert p.wait() == 0
+
+    cli(alice, "get", "getput:fromstdin", tempfile)
+    assert read_bytes(tempfile) == DATA
+
+
+def test_get_to_stdout(alice, get_put_alias, tmpdir):
+    """
+    It's possible to upload a file, and then download it to stdout.
+    """
+    tempfile = tmpdir.join("file")
+    with tempfile.open("wb") as f:
+        f.write(DATA)
+    cli(alice, "put", str(tempfile), "getput:tostdout")
+
+    p = Popen(
+        ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"],
+        stdout=PIPE
+    )
+    assert p.stdout.read() == DATA
+    assert p.wait() == 0

newsfragments/3721.documentation

@@ -0,0 +1 @@
+Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat.

src/allmydata/scripts/tahoe_cp.py

@@ -701,6 +701,8 @@ class Copier(object):
 
 
     def need_to_copy_bytes(self, source, target):
+        # This should likley be a method call! but enabling that triggers
+        # additional bugs. https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3719
         if source.need_to_copy_bytes:
             # mutable tahoe files, and local files
             return True

src/allmydata/test/cli/test_backup.py

@@ -6,16 +6,19 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2
+from future.utils import PY2, PY3
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
+    import __builtin__ as builtins
+else:
+    import builtins
+from six import ensure_str
 
 import os.path
 from six.moves import cStringIO as StringIO
 from datetime import timedelta
 import re
-import sys
+import locale
 
 from twisted.trial import unittest
 from twisted.python.monkey import MonkeyPatcher
@@ -373,7 +376,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
     def test_exclude_options_unicode(self):
         nice_doc = u"nice_d\u00F8c.lyx"
         try:
-            doc_pattern_arg = quote_output(u"*d\u00F8c*", sys.stdout.encoding)
+            doc_pattern_arg = u"*d\u00F8c*"
+            if PY2:
+                doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding())
         except UnicodeEncodeError:
             raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
 
@@ -395,7 +400,11 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'),
                              (nice_doc, u'lib.a'))
         # read exclude patterns from file
-        exclusion_string = doc_pattern_arg + b"\nlib.?"
+        exclusion_string = doc_pattern_arg + ensure_str("\nlib.?")
+        if PY3:
+            # On Python 2 this gives some garbage encoding. Also on Python 2 we
+            # expect exclusion string to be bytes.
+            exclusion_string = exclusion_string.encode(locale.getpreferredencoding(False))
         excl_filepath = os.path.join(basedir, 'exclusion')
         fileutil.write(excl_filepath, exclusion_string)
         backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to'])

src/allmydata/test/cli/test_cp.py

@@ -238,6 +238,66 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         return d
 
+    @defer.inlineCallbacks
+    def test_cp_duplicate_directories(self):
+        self.basedir = "cli/Cp/cp_duplicate_directories"
+        self.set_up_grid(oneshare=True)
+
+        filename = os.path.join(self.basedir, "file")
+        data = b"abc\xff\x00\xee"
+        with open(filename, "wb") as f:
+            f.write(data)
+
+        yield self.do_cli("create-alias", "tahoe")
+        (rc, out, err) = yield self.do_cli("mkdir", "tahoe:test1")
+        self.assertEqual(rc, 0, (rc, err))
+        dircap = out.strip()
+
+        (rc, out, err) = yield self.do_cli("cp", filename, "tahoe:test1/file")
+        self.assertEqual(rc, 0, (rc, err))
+
+        # Now duplicate dirnode, testing duplicates on destination side:
+        (rc, out, err) = yield self.do_cli(
+            "cp", "--recursive", dircap, "tahoe:test2/")
+        self.assertEqual(rc, 0, (rc, err))
+        (rc, out, err) = yield self.do_cli(
+            "cp", "--recursive", dircap, "tahoe:test3/")
+        self.assertEqual(rc, 0, (rc, err))
+
+        # Now copy to local directory, testing duplicates on origin side:
+        yield self.do_cli("cp", "--recursive", "tahoe:", self.basedir)
+
+        for i in range(1, 4):
+            with open(os.path.join(self.basedir, "test%d" % (i,), "file"), "rb") as f:
+                self.assertEquals(f.read(), data)
+
+    @defer.inlineCallbacks
+    def test_cp_immutable_file(self):
+        self.basedir = "cli/Cp/cp_immutable_file"
+        self.set_up_grid(oneshare=True)
+
+        filename = os.path.join(self.basedir, "source_file")
+        data = b"abc\xff\x00\xee"
+        with open(filename, "wb") as f:
+            f.write(data)
+
+        # Create immutable file:
+        yield self.do_cli("create-alias", "tahoe")
+        (rc, out, _) = yield self.do_cli("put", filename, "tahoe:file1")
+        filecap = out.strip()
+        self.assertEqual(rc, 0)
+
+        # Copy it:
+        (rc, _, _) = yield self.do_cli("cp", "tahoe:file1", "tahoe:file2")
+        self.assertEqual(rc, 0)
+
+        # Make sure resulting file is the same:
+        (rc, _, _) = yield self.do_cli("cp", "--recursive", "--caps-only",
+                                       "tahoe:", self.basedir)
+        self.assertEqual(rc, 0)
+        with open(os.path.join(self.basedir, "file2")) as f:
+            self.assertEqual(f.read().strip(), filecap)
+
     def test_cp_replaces_mutable_file_contents(self):
         self.basedir = "cli/Cp/cp_replaces_mutable_file_contents"
         self.set_up_grid(oneshare=True)

src/allmydata/test/cli/test_put.py

@@ -486,3 +486,20 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.failUnlessReallyEqual(rc_out_err[1], DATA))
 
         return d
+
+    def test_no_leading_slash(self):
+        self.basedir = "cli/Put/leading_slash"
+        self.set_up_grid(oneshare=True)
+
+        fn1 = os.path.join(self.basedir, "DATA1")
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res:
+                      self.do_cli("put", fn1, "tahoe:/uploaded.txt"))
+        def _check(args):
+            (rc, out, err) = args
+            self.assertEqual(rc, 1)
+            self.failUnlessIn("must not start with a slash", err)
+            self.assertEqual(len(out), 0, out)
+        d.addCallback(_check)
+        return d

src/allmydata/test/test_runner.py

@@ -17,6 +17,7 @@ from six import ensure_text
 
 import os.path, re, sys
 from os import linesep
+import locale
 
 from eliot import (
     log_call,
@@ -92,8 +93,12 @@ def run_bintahoe(extra_argv, python_options=None):
     argv.extend(extra_argv)
     argv = list(unicode_to_argv(arg) for arg in argv)
     p = Popen(argv, stdout=PIPE, stderr=PIPE)
-    out = p.stdout.read().decode("utf-8")
-    err = p.stderr.read().decode("utf-8")
+    if PY2:
+        encoding = "utf-8"
+    else:
+        encoding = locale.getpreferredencoding(False)
+    out = p.stdout.read().decode(encoding)
+    err = p.stderr.read().decode(encoding)
     returncode = p.wait()
     return (out, err, returncode)
 
@@ -103,7 +108,7 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase):
         """
         The runner script receives unmangled non-ASCII values in argv.
         """
-        tricky = u"\u2621"
+        tricky = u"\u00F6"
         out, err, returncode = run_bintahoe([tricky])
         self.assertEqual(returncode, 1)
         self.assertIn(u"Unknown command: " + tricky, out)

src/allmydata/test/test_windows.py

@@ -79,6 +79,7 @@ slow_settings = settings(
 )
 
 @skipUnless(platform.isWindows(), "get_argv is Windows-only")
+@skipUnless(PY2, "Not used on Python 3.")
 class GetArgvTests(SyncTestCase):
     """
     Tests for ``get_argv``.
@@ -172,6 +173,7 @@ class GetArgvTests(SyncTestCase):
 
 
 @skipUnless(platform.isWindows(), "intended for Windows-only codepaths")
+@skipUnless(PY2, "Not used on Python 3.")
 class UnicodeOutputTests(SyncTestCase):
     """
     Tests for writing unicode to stdout and stderr.

src/allmydata/windows/fixups.py

@@ -1,4 +1,6 @@
 from __future__ import print_function
+
+from future.utils import PY3
 from past.builtins import unicode
 
 # This code isn't loadable or sensible except on Windows.  Importers all know
@@ -122,6 +124,10 @@ def initialize():
 
     SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX)
 
+    if PY3:
+        # The rest of this appears to be Python 2-specific
+        return
+
     original_stderr = sys.stderr
 
     # If any exception occurs in this code, we'll probably try to print it on stderr,

tox.ini

@@ -9,9 +9,9 @@
 python =
     2.7: py27-coverage,codechecks
     3.6: py36-coverage
-    3.7: py37-coverage
+    3.7: py37-coverage,typechecks,codechecks3
     3.8: py38-coverage
-    3.9: py39-coverage,typechecks,codechecks3
+    3.9: py39-coverage
     pypy-3.7: pypy3
 
 [pytest]