Merge branch '2329-2': fix "tahoe cp -r" exception

src/allmydata/scripts/tahoe_cp.py

@@ -2,6 +2,7 @@
 import os.path
 import urllib
 import simplejson
+from collections import defaultdict
 from cStringIO import StringIO
 from twisted.python.failure import Failure
 from allmydata.scripts.common import get_alias, escape_path, \
@@ -12,7 +13,7 @@ from allmydata.util import fileutil
 from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath
 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
     quote_local_unicode_path, to_str
-from allmydata.util.assertutil import precondition
+from allmydata.util.assertutil import precondition, _assert
 
 
 class MissingSourceError(TahoeError):
@@ -115,7 +116,7 @@ class LocalDirectorySource:
                 child = LocalDirectorySource(self.progressfunc, pn, n)
                 self.children[n] = child
                 if recurse:
-                    child.populate(True)
+                    child.populate(recurse=True)
             elif os.path.isfile(pn):
                 self.children[n] = LocalFileSource(pn, n)
             else:
@@ -143,7 +144,7 @@ class LocalDirectoryTarget:
                 child = LocalDirectoryTarget(self.progressfunc, pn)
                 self.children[n] = child
                 if recurse:
-                    child.populate(True)
+                    child.populate(recurse=True)
             else:
                 assert os.path.isfile(pn)
                 self.children[n] = LocalFileTarget(pn)
@@ -151,12 +152,14 @@ class LocalDirectoryTarget:
     def get_child_target(self, name):
         precondition(isinstance(name, unicode), name)
         if self.children is None:
-            self.populate(False)
+            self.populate(recurse=False)
         if name in self.children:
             return self.children[name]
         pathname = os.path.join(self.pathname, name)
         os.makedirs(pathname)
-        return LocalDirectoryTarget(self.progressfunc, pathname)
+        child = LocalDirectoryTarget(self.progressfunc, pathname)
+        self.children[name] = child
+        return child
 
     def put_file(self, name, inf):
         precondition(isinstance(name, unicode), name)
@@ -277,7 +280,7 @@ class TahoeDirectorySource:
                     if readcap:
                         self.cache[readcap] = child
                     if recurse:
-                        child.populate(True)
+                        child.populate(recurse=True)
                 self.children[name] = child
             else:
                 # TODO: there should be an option to skip unknown nodes.
@@ -337,6 +340,7 @@ class TahoeDirectoryTarget:
         self.children = None
 
     def just_created(self, writecap):
+        # TODO: maybe integrate this with the constructor
         self.writecap = writecap
         self.readcap = uri.from_string(writecap).get_readonly().to_string()
         self.mutable = True
@@ -376,7 +380,7 @@ class TahoeDirectoryTarget:
                     if readcap:
                         self.cache[readcap] = child
                     if recurse:
-                        child.populate(True)
+                        child.populate(recurse=True)
                 self.children[name] = child
             else:
                 # TODO: there should be an option to skip unknown nodes.
@@ -388,7 +392,7 @@ class TahoeDirectoryTarget:
         # return a new target for a named subdirectory of this dir
         precondition(isinstance(name, unicode), name)
         if self.children is None:
-            self.populate(False)
+            self.populate(recurse=False)
         if name in self.children:
             return self.children[name]
         writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
@@ -405,7 +409,7 @@ class TahoeDirectoryTarget:
             inf = inf.read()
 
         if self.children is None:
-            self.populate(False)
+            self.populate(recurse=False)
 
         # Check to see if we already have a mutable file by this name.
         # If so, overwrite that file in place.
@@ -439,6 +443,12 @@ class TahoeDirectoryTarget:
         body = simplejson.dumps(set_data)
         POST(url, body)
 
+FileSources = (LocalFileSource, TahoeFileSource)
+DirectorySources = (LocalDirectorySource, TahoeDirectorySource)
+FileTargets = (LocalFileTarget, TahoeFileTarget)
+DirectoryTargets = (LocalDirectoryTarget, TahoeDirectoryTarget)
+MissingTargets = (LocalMissingTarget, TahoeMissingTarget)
+
 class Copier:
 
     def do_copy(self, options, progressfunc=None):
@@ -476,58 +486,83 @@ class Copier:
             return 1
 
     def try_copy(self):
+        """
+        All usage errors are caught here, not in a subroutine. This bottoms
+        out in copy_file_to_file() or copy_things_to_directory().
+        """
         source_specs = self.options.sources
         destination_spec = self.options.destination
         recursive = self.options["recursive"]
 
         target = self.get_target_info(destination_spec)
+        precondition(isinstance(target, FileTargets + DirectoryTargets + MissingTargets), target)
+        target_has_trailing_slash = destination_spec.endswith("/")
 
         sources = [] # list of source objects
         for ss in source_specs:
-            sources.append(self.get_source_info(ss))
+            si = self.get_source_info(ss)
+            precondition(isinstance(si, FileSources + DirectorySources), si)
+            sources.append(si)
 
-        have_source_dirs = any([isinstance(s, (LocalDirectorySource,
+        # if any source is a directory, must use -r
-                                               TahoeDirectorySource))
+        # if target is missing:
+        #    if source is a single file, target will be a file
+        #    else target will be a directory, so mkdir it
+        # if there are multiple sources, target must be a dir
+        # if target is a file, source must be a single file
+        # if target is directory, sources must be named or a dir
+
+        have_source_dirs = any([isinstance(s, DirectorySources)
                                 for s in sources])
-
         if have_source_dirs and not recursive:
+            # 'cp dir target' without -r: error
             self.to_stderr("cannot copy directories without --recursive")
             return 1
+        del recursive # -r is only used for signalling errors
 
-        if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
+        if isinstance(target, FileTargets):
-            # cp STUFF foo.txt, where foo.txt already exists. This limits the
+            target_is_file = True
-            # possibilities considerably.
+        elif isinstance(target, DirectoryTargets):
-            if len(sources) > 1:
+            target_is_file = False
-                self.to_stderr("target %s is not a directory" % quote_output(destination_spec))
+        else: # isinstance(target, MissingTargets)
-                return 1
+            if len(sources) == 1 and isinstance(sources[0], FileSources):
-            if have_source_dirs:
+                target_is_file = True
+            else:
+                target_is_file = False
+
+        if target_is_file and target_has_trailing_slash:
+            self.to_stderr("target is not a directory, but ends with a slash")
+            return 1
+
+        if len(sources) > 1 and target_is_file:
+            self.to_stderr("copying multiple things requires target be a directory")
+            return 1
+
+        if target_is_file:
+            _assert(len(sources) == 1, sources)
+            if not isinstance(sources[0], FileSources):
+                # 'cp -r dir existingfile': error
                 self.to_stderr("cannot copy directory into a file")
                 return 1
-            return self.copy_file(sources[0], target)
+            return self.copy_file_to_file(sources[0], target)
 
-        if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
+        # else target is a directory, so each source must be one of:
-            if recursive:
+        # * a named file (copied to a new file under the target)
-                return self.copy_to_directory(sources, target)
+        # * a named directory (causes a new directory of the same name to be
-            if len(sources) > 1:
+        #   created under the target, then the contents of the source are
-                # if we have -r, we'll auto-create the target directory. Without
+        #   copied into that directory)
-                # it, we'll only create a file.
+        # * an unnamed directory (the contents of the source are copied into
-                self.to_stderr("cannot copy multiple files into a file without -r")
+        #   the target, without a new directory being made)
+        #
+        # If any source is an unnamed file, throw an error, since we have no
+        # way to name the output file.
+        _assert(isinstance(target, DirectoryTargets + MissingTargets), target)
+
+        for source in sources:
+            if isinstance(source, FileSources) and source.basename() is None:
+                self.to_stderr("when copying into a directory, all source files must have names, but %s is unnamed" % quote_output(source_specs[0]))
                 return 1
-            # cp file1 newfile
+        return self.copy_things_to_directory(sources, target)
-            return self.copy_file(sources[0], target)
-
-        if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
-            # We're copying to an existing directory -- make sure that we
-            # have target names for everything
-            for source in sources:
-                if source.basename() is None and isinstance(source, TahoeFileSource):
-                    self.to_stderr(
-                        "error: you must specify a destination filename")
-                    return 1
-            return self.copy_to_directory(sources, target)
-
-        self.to_stderr("unknown target")
-        return 1
 
     def to_stderr(self, text):
         print >>self.stderr, text
@@ -580,6 +615,9 @@ class Copier:
         return t
 
     def get_source_info(self, source_spec):
+        """
+        This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
+        """
         precondition(isinstance(source_spec, unicode), source_spec)
         rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
         path = path_utf8.decode("utf-8")
@@ -632,109 +670,6 @@ class Copier:
         return t
 
 
-    def dump_graph(self, s, indent=" "):
-        for name, child in s.children.items():
-            print "%s%s: %r" % (indent, quote_output(name), child)
-            if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
-                self.dump_graph(child, indent+"  ")
-
-    def copy_to_directory(self, sources, target):
-        # step one: build a recursive graph of the source tree. This returns
-        # a dictionary, with child names as keys, and values that are either
-        # Directory or File instances (local or tahoe).
-        source_dirs = self.build_graphs(sources)
-        source_files = [s for s in sources
-                        if isinstance(s, (LocalFileSource, TahoeFileSource))]
-
-        #print "graphs"
-        #for s in source_dirs:
-        #    self.dump_graph(s)
-
-        # step two: create the top-level target directory object
-        if isinstance(target, LocalMissingTarget):
-            os.makedirs(target.pathname)
-            target = LocalDirectoryTarget(self.progress, target.pathname)
-        elif isinstance(target, TahoeMissingTarget):
-            writecap = mkdir(target.url)
-            target = TahoeDirectoryTarget(self.nodeurl, self.cache,
-                                          self.progress)
-            target.just_created(writecap)
-        assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
-        target.populate(False)
-
-        # step three: find a target for each source node, creating
-        # directories as necessary. 'targetmap' is a dictionary that uses
-        # target Directory instances as keys, and has values of
-        # (name->sourceobject) dicts for all the files that need to wind up
-        # there.
-
-        # sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
-        # target is LocalDirectory/TahoeDirectory
-
-        self.progress("attaching sources to targets, "
-                      "%d files / %d dirs in root" %
-                      (len(source_files), len(source_dirs)))
-
-        self.targetmap = {}
-        self.files_to_copy = 0
-
-        for s in source_files:
-            self.attach_to_target(s, s.basename(), target)
-
-        for (name, source) in source_dirs:
-            new_target = target.get_child_target(name)
-            self.assign_targets(source, new_target)
-
-        self.progress("targets assigned, %s dirs, %s files" %
-                      (len(self.targetmap), self.files_to_copy))
-
-        self.progress("starting copy, %d files, %d directories" %
-                      (self.files_to_copy, len(self.targetmap)))
-        self.files_copied = 0
-        self.targets_finished = 0
-
-        # step four: walk through the list of targets. For each one, copy all
-        # the files. If the target is a TahoeDirectory, upload and create
-        # read-caps, then do a set_children to the target directory.
-
-        for target in self.targetmap:
-            self.copy_files_to_target(self.targetmap[target], target)
-            self.targets_finished += 1
-            self.progress("%d/%d directories" %
-                          (self.targets_finished, len(self.targetmap)))
-
-        return self.announce_success("files copied")
-
-    def attach_to_target(self, source, name, target):
-        precondition(isinstance(name, unicode), name)
-        if target not in self.targetmap:
-            self.targetmap[target] = {}
-        self.targetmap[target][name] = source
-        self.files_to_copy += 1
-
-    def assign_targets(self, source, target):
-        # copy everything in the source into the target
-        precondition(isinstance(source, (LocalDirectorySource, TahoeDirectorySource)), source)
-
-        for name, child in source.children.items():
-            if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
-                # we will need a target directory for this one
-                subtarget = target.get_child_target(name)
-                self.assign_targets(child, subtarget)
-            else:
-                precondition(isinstance(child, (LocalFileSource, TahoeFileSource)), child)
-                self.attach_to_target(child, name, target)
-
-    def copy_files_to_target(self, targetmap, target):
-        for name, source in targetmap.items():
-            precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
-            self.copy_file_into(source, name, target)
-            self.files_copied += 1
-            self.progress("%d/%d files, %d/%d directories" %
-                          (self.files_copied, self.files_to_copy,
-                           self.targets_finished, len(self.targetmap)))
-        target.set_children()
-
     def need_to_copy_bytes(self, source, target):
         if source.need_to_copy_bytes:
             # mutable tahoe files, and local files
@@ -748,10 +683,9 @@ class Copier:
             print >>self.stdout, "Success: %s" % msg
         return 0
 
-    def copy_file(self, source, target):
+    def copy_file_to_file(self, source, target):
-        precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
+        precondition(isinstance(source, FileSources), source)
-        precondition(isinstance(target, (LocalFileTarget, TahoeFileTarget,
+        precondition(isinstance(target, FileTargets + MissingTargets), target)
-                                         LocalMissingTarget, TahoeMissingTarget)), target)
         if self.need_to_copy_bytes(source, target):
             # if the target is a local directory, this will just write the
             # bytes to disk. If it is a tahoe directory, it will upload the
@@ -765,9 +699,115 @@ class Copier:
         target.put_uri(source.bestcap())
         return self.announce_success("file linked")
 
-    def copy_file_into(self, source, name, target):
+    def copy_things_to_directory(self, sources, target):
-        precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
+        # step one: if the target is missing, we should mkdir it
-        precondition(isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)), target)
+        target = self.maybe_create_target(target)
+        target.populate(recurse=False)
+
+        # step two: scan any source dirs, recursively, to find children
+        for s in sources:
+            if isinstance(s, DirectorySources):
+                s.populate(recurse=True)
+            if isinstance(s, FileSources):
+                # each source must have a name, or be a directory
+                _assert(s.basename() is not None, s)
+
+        # step three: find a target for each source node, creating
+        # directories as necessary. 'targetmap' is a dictionary that uses
+        # target Directory instances as keys, and has values of (name:
+        # sourceobject) dicts for all the files that need to wind up there.
+        targetmap = self.build_targetmap(sources, target)
+
+        # step four: walk through the list of targets. For each one, copy all
+        # the files. If the target is a TahoeDirectory, upload and create
+        # read-caps, then do a set_children to the target directory.
+        self.copy_to_targetmap(targetmap)
+
+        return self.announce_success("files copied")
+
+    def maybe_create_target(self, target):
+        if isinstance(target, LocalMissingTarget):
+            os.makedirs(target.pathname)
+            target = LocalDirectoryTarget(self.progress, target.pathname)
+        elif isinstance(target, TahoeMissingTarget):
+            writecap = mkdir(target.url)
+            target = TahoeDirectoryTarget(self.nodeurl, self.cache,
+                                          self.progress)
+            target.just_created(writecap)
+        # afterwards, or otherwise, it will be a directory
+        precondition(isinstance(target, DirectoryTargets), target)
+        return target
+
+    def build_targetmap(self, sources, target):
+        num_source_files = len([s for s in sources
+                                if isinstance(s, FileSources)])
+        num_source_dirs = len([s for s in sources
+                               if isinstance(s, DirectorySources)])
+        self.progress("attaching sources to targets, "
+                      "%d files / %d dirs in root" %
+                      (num_source_files, num_source_dirs))
+
+        # this maps each target directory to a list of source files that need
+        # to be copied into it. All source files have names.
+        targetmap = defaultdict(list)
+
+        for s in sources:
+            if isinstance(s, FileSources):
+                targetmap[target].append(s)
+            else:
+                _assert(isinstance(s, DirectorySources), s)
+                name = s.basename()
+                if name is not None:
+                    # named sources get a new directory. see #2329
+                    new_target = target.get_child_target(name)
+                else:
+                    # unnamed sources have their contents copied directly
+                    new_target = target
+                self.assign_targets(targetmap, s, new_target)
+
+        self.progress("targets assigned, %s dirs, %s files" %
+                      (len(targetmap), self.count_files_to_copy(targetmap)))
+        return targetmap
+
+    def assign_targets(self, targetmap, source, target):
+        # copy everything in the source into the target
+        precondition(isinstance(source, DirectorySources), source)
+        for name, child in source.children.items():
+            if isinstance(child, DirectorySources):
+                # we will need a target directory for this one
+                subtarget = target.get_child_target(name)
+                self.assign_targets(targetmap, child, subtarget)
+            else:
+                _assert(isinstance(child, FileSources), child)
+                targetmap[target].append(child)
+
+    def copy_to_targetmap(self, targetmap):
+        files_to_copy = self.count_files_to_copy(targetmap)
+        self.progress("starting copy, %d files, %d directories" %
+                      (files_to_copy, len(targetmap)))
+        files_copied = 0
+        targets_finished = 0
+
+        for target, sources in targetmap.items():
+            _assert(isinstance(target, DirectoryTargets), target)
+            for source in sources:
+                _assert(isinstance(source, FileSources), source)
+                self.copy_file_into_dir(source, source.basename(), target)
+                files_copied += 1
+                self.progress("%d/%d files, %d/%d directories" %
+                              (files_copied, files_to_copy,
+                               targets_finished, len(targetmap)))
+            target.set_children()
+            targets_finished += 1
+            self.progress("%d/%d directories" %
+                          (targets_finished, len(targetmap)))
+
+    def count_files_to_copy(self, targetmap):
+        return sum([len(sources) for sources in targetmap.values()])
+
+    def copy_file_into_dir(self, source, name, target):
+        precondition(isinstance(source, FileSources), source)
+        precondition(isinstance(target, DirectoryTargets), target)
         precondition(isinstance(name, unicode), name)
         if self.need_to_copy_bytes(source, target):
             # if the target is a local directory, this will just write the
@@ -786,14 +826,6 @@ class Copier:
         if self.progressfunc:
             self.progressfunc(message)
 
-    def build_graphs(self, sources):
-        graphs = []
-        for source in sources:
-            if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
-                source.populate(True)
-                graphs.append((source.basename(), source))
-        return graphs
-
 
 def copy(options):
     return Copier().do_copy(options)

src/allmydata/test/test_cli_cp.py

@@ -1,11 +1,13 @@
 import os.path, simplejson
 from twisted.trial import unittest
 from twisted.python import usage
+from twisted.internet import defer
 
 from allmydata.scripts import cli
 from allmydata.util import fileutil
 from allmydata.util.encodingutil import (quote_output, get_io_encoding,
                                          unicode_to_output, to_str)
+from allmydata.util.assertutil import _assert
 from .no_network import GridTestMixin
 from .test_cli import CLITestMixin
 
@@ -119,7 +121,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir))
         def _resp((rc, out, err)):
             self.failUnlessReallyEqual(rc, 1)
-            self.failUnlessIn("error: you must specify a destination filename",
+            self.failUnlessIn("when copying into a directory, all source files must have names, but",
                               err)
             self.failUnlessReallyEqual(out, "")
         d.addCallback(_resp)
@@ -653,3 +655,348 @@ starting copy, 2 files, 1 directories
             (rc, out, err) = res
             self.failUnlessIn("Success: file copied", out, str(res))
         return d
+
+# these test cases come from ticket #2329 comment 40
+# trailing slash on target *directory* should not matter, test both
+# trailing slash on files should cause error
+
+COPYOUT_TESTCASES = """
+cp    $FILECAP          to/existing-file : to/existing-file
+cp -r $FILECAP          to/existing-file : to/existing-file
+cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
+cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
+cp    $DIRCAP           to/existing-file : E4-NEED-R
+cp -r $DIRCAP           to/existing-file : E5-DIRTOFILE
+cp    $FILECAP $DIRCAP  to/existing-file : E4-NEED-R
+cp -r $FILECAP $DIRCAP  to/existing-file : E6-MANYONE
+
+cp    $FILECAP          to/existing-file/ : E7-BADSLASH
+cp -r $FILECAP          to/existing-file/ : E7-BADSLASH
+cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
+cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
+cp    $DIRCAP           to/existing-file/ : E4-NEED-R
+cp -r $DIRCAP           to/existing-file/ : E7-BADSLASH
+cp    $FILECAP $DIRCAP  to/existing-file/ : E4-NEED-R
+cp -r $FILECAP $DIRCAP  to/existing-file/ : E7-BADSLASH
+
+# single source to a (present) target directory
+cp    $FILECAP       to : E2-DESTNAME
+cp -r $FILECAP       to : E2-DESTNAME
+cp    $DIRCAP/file   to : to/file
+cp -r $DIRCAP/file   to : to/file
+cp    $PARENTCAP/dir to : E4-NEED-R
+cp -r $PARENTCAP/dir to : to/dir/file
+cp    $DIRCAP        to : E4-NEED-R
+cp -r $DIRCAP        to : to/file
+cp    $DIRALIAS      to : E4-NEED-R
+cp -r $DIRALIAS      to : to/file
+
+cp    $FILECAP       to/ : E2-DESTNAME
+cp -r $FILECAP       to/ : E2-DESTNAME
+cp    $DIRCAP/file   to/ : to/file
+cp -r $DIRCAP/file   to/ : to/file
+cp    $PARENTCAP/dir to/ : E4-NEED-R
+cp -r $PARENTCAP/dir to/ : to/dir/file
+cp    $DIRCAP        to/ : E4-NEED-R
+cp -r $DIRCAP        to/ : to/file
+cp    $DIRALIAS      to/ : E4-NEED-R
+cp -r $DIRALIAS      to/ : to/file
+
+# multiple sources to a (present) target directory
+cp    $DIRCAP/file $PARENTCAP/dir2/file2 to : to/file,to/file2
+cp    $DIRCAP/file $FILECAP              to : E2-DESTNAME
+cp    $DIRCAP $FILECAP                   to : E4-NEED-R
+cp -r $DIRCAP $FILECAP                   to : E2-DESTNAME
+      # namedfile, unnameddir, nameddir
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to : to/file3,to/file,to/dir2/file2
+      # namedfile, unnameddir, nameddir, unnamedfile
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E2-DESTNAME
+
+cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/ : to/file,to/file2
+cp    $DIRCAP/file $FILECAP           to/    : E2-DESTNAME
+cp    $DIRCAP $FILECAP                to/    : E4-NEED-R
+cp -r $DIRCAP $FILECAP                to/    : E2-DESTNAME
+      # namedfile, unnameddir, nameddir
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/ : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/ : to/file3,to/file,to/dir2/file2
+      # namedfile, unnameddir, nameddir, unnamedfile
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E2-DESTNAME
+
+# single sources to a missing target: should mkdir or create a file
+cp    $FILECAP       to/missing : to/missing
+cp -r $FILECAP       to/missing : to/missing
+cp    $DIRCAP/file   to/missing : to/missing
+cp -r $DIRCAP/file   to/missing : to/missing
+cp    $PARENTCAP/dir to/missing : E4-NEED-R
+cp -r $PARENTCAP/dir to/missing : to/missing/dir/file
+cp    $DIRCAP        to/missing : E4-NEED-R
+cp -r $DIRCAP        to/missing : to/missing/file
+cp    $DIRALIAS      to/missing : E4-NEED-R
+cp -r $DIRALIAS      to/missing : to/missing/file
+
+cp    $FILECAP       to/missing/ : E7-BADSLASH
+cp -r $FILECAP       to/missing/ : E7-BADSLASH
+cp    $DIRCAP/file   to/missing/ : E7-BADSLASH
+cp -r $DIRCAP/file   to/missing/ : E7-BADSLASH
+cp    $PARENTCAP/dir to/missing/ : E4-NEED-R
+cp -r $PARENTCAP/dir to/missing/ : to/missing/dir/file
+cp    $DIRCAP        to/missing/ : E4-NEED-R
+cp -r $DIRCAP        to/missing/ : to/missing/file
+cp    $DIRALIAS      to/missing/ : E4-NEED-R
+cp -r $DIRALIAS      to/missing/ : to/missing/file
+
+# multiple things to a missing target: should mkdir
+cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
+cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
+cp    $DIRCAP/file $FILECAP              to/missing : E2-DESTNAME
+cp -r $DIRCAP/file $FILECAP              to/missing : E2-DESTNAME
+cp    $DIRCAP $FILECAP                   to/missing : E4-NEED-R
+cp -r $DIRCAP $FILECAP                   to/missing : E2-DESTNAME
+      # namedfile, unnameddir, nameddir
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/missing : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/missing : to/missing/file3,to/missing/file,to/missing/dir2/file2
+      # namedfile, unnameddir, nameddir, unnamedfile
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E2-DESTNAME
+
+cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
+cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
+cp    $DIRCAP/file $FILECAP           to/missing/    : E2-DESTNAME
+cp -r $DIRCAP/file $FILECAP           to/missing/    : E2-DESTNAME
+cp    $DIRCAP $FILECAP                to/missing/    : E4-NEED-R
+cp -r $DIRCAP $FILECAP                to/missing/    : E2-DESTNAME
+      # namedfile, unnameddir, nameddir
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/missing/ : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/missing/ : to/missing/file3,to/missing/file,to/missing/dir2/file2
+      # namedfile, unnameddir, nameddir, unnamedfile
+cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E4-NEED-R
+cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E2-DESTNAME
+
+# make sure empty directories are copied too
+cp -r $PARENTCAP/dir4 to  : to/dir4/emptydir/
+cp -r $PARENTCAP/dir4 to/ : to/dir4/emptydir/
+
+# name collisions: ensure files are copied in order
+cp -r $PARENTCAP/dir6/dir $PARENTCAP/dir5/dir to : to/dir/collide=5
+cp -r $PARENTCAP/dir5/dir $PARENTCAP/dir6/dir to : to/dir/collide=6
+cp -r $DIRCAP6 $DIRCAP5 to : to/dir/collide=5
+cp -r $DIRCAP5 $DIRCAP6 to : to/dir/collide=6
+
+"""
+
+class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
+    FILE_CONTENTS = "file text"
+    FILE_CONTENTS_5 = "5"
+    FILE_CONTENTS_6 = "6"
+
+    def do_setup(self):
+        # first we build a tahoe filesystem that contains:
+        #  $PARENTCAP
+        #  $PARENTCAP/dir  == $DIRCAP == alias:
+        #  $PARENTCAP/dir/file == $FILECAP
+        #  $PARENTCAP/dir2        (named directory)
+        #  $PARENTCAP/dir2/file2
+        #  $PARENTCAP/dir3/file3  (a second named file)
+        #  $PARENTCAP/dir4
+        #  $PARENTCAP/dir4/emptydir/ (an empty directory)
+        #  $PARENTCAP/dir5 == $DIRCAP5
+        #  $PARENTCAP/dir5/dir/collide (contents are "5")
+        #  $PARENTCAP/dir6 == $DIRCAP6
+        #  $PARENTCAP/dir6/dir/collide (contents are "6")
+
+        source_file = os.path.join(self.basedir, "file")
+        fileutil.write(source_file, self.FILE_CONTENTS)
+        source_file_5 = os.path.join(self.basedir, "file5")
+        fileutil.write(source_file_5, self.FILE_CONTENTS_5)
+        source_file_6 = os.path.join(self.basedir, "file6")
+        fileutil.write(source_file_6, self.FILE_CONTENTS_6)
+
+        d = self.do_cli("mkdir")
+        def _stash_parentdircap(res):
+            (rc, out, err) = res
+            self.failUnlessEqual(rc, 0, str(res))
+            self.failUnlessEqual(err, "", str(res))
+            self.PARENTCAP = out.strip()
+            return self.do_cli("mkdir", "%s/dir" % self.PARENTCAP)
+        d.addCallback(_stash_parentdircap)
+        def _stash_dircap(res):
+            (rc, out, err) = res
+            self.failUnlessEqual(rc, 0, str(res))
+            self.failUnlessEqual(err, "", str(res))
+            self.DIRCAP = out.strip()
+            return self.do_cli("add-alias", "ALIAS", self.DIRCAP)
+        d.addCallback(_stash_dircap)
+        d.addCallback(lambda ign:
+            self.do_cli("put", source_file, "%s/dir/file" % self.PARENTCAP))
+        def _stash_filecap(res):
+            (rc, out, err) = res
+            self.failUnlessEqual(rc, 0, str(res))
+            self.failUnlessEqual(err.strip(), "201 Created", str(res))
+            self.FILECAP = out.strip()
+            assert self.FILECAP.startswith("URI:LIT:")
+        d.addCallback(_stash_filecap)
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir2" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("put", source_file, "%s/dir2/file2" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir3" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("put", source_file, "%s/dir3/file3" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir4" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir4/emptydir" % self.PARENTCAP))
+
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir5" % self.PARENTCAP))
+        def _stash_dircap_5(res):
+            (rc, out, err) = res
+            self.failUnlessEqual(rc, 0, str(res))
+            self.failUnlessEqual(err, "", str(res))
+            self.DIRCAP5 = out.strip()
+        d.addCallback(_stash_dircap_5)
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir5/dir" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("put", source_file_5, "%s/dir5/dir/collide" % self.PARENTCAP))
+
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir6" % self.PARENTCAP))
+        def _stash_dircap_6(res):
+            (rc, out, err) = res
+            self.failUnlessEqual(rc, 0, str(res))
+            self.failUnlessEqual(err, "", str(res))
+            self.DIRCAP6 = out.strip()
+        d.addCallback(_stash_dircap_6)
+        d.addCallback(lambda ign:
+            self.do_cli("mkdir", "%s/dir6/dir" % self.PARENTCAP))
+        d.addCallback(lambda ign:
+            self.do_cli("put", source_file_6, "%s/dir6/dir/collide" % self.PARENTCAP))
+
+        return d
+
+    def check_output(self):
+        # locate the files and directories created (if any) under to/
+        top = os.path.join(self.basedir, "to")
+        results = set()
+        for (dirpath, dirnames, filenames) in os.walk(top):
+            assert dirpath.startswith(top)
+            here = "/".join(dirpath.split(os.sep)[len(top.split(os.sep))-1:])
+            results.add(here+"/")
+            for fn in filenames:
+                contents = fileutil.read(os.path.join(dirpath, fn))
+                if contents == self.FILE_CONTENTS:
+                    results.add("%s/%s" % (here, fn))
+                elif contents == self.FILE_CONTENTS_5:
+                    results.add("%s/%s=5" % (here, fn))
+                elif contents == self.FILE_CONTENTS_6:
+                    results.add("%s/%s=6" % (here, fn))
+        return results
+
+    def run_one_case(self, case):
+        cmd = (case
+               .replace("$PARENTCAP", self.PARENTCAP)
+               .replace("$DIRCAP5", self.DIRCAP5)
+               .replace("$DIRCAP6", self.DIRCAP6)
+               .replace("$DIRCAP", self.DIRCAP)
+               .replace("$DIRALIAS", "ALIAS:")
+               .replace("$FILECAP", self.FILECAP)
+               .split())
+        target = cmd[-1]
+        _assert(target == "to" or target.startswith("to/"), target)
+        cmd[-1] = os.path.abspath(os.path.join(self.basedir, cmd[-1]))
+
+        # reset
+        targetdir = os.path.abspath(os.path.join(self.basedir, "to"))
+        fileutil.rm_dir(targetdir)
+        os.mkdir(targetdir)
+
+        if target.rstrip("/") == "to/existing-file":
+            fileutil.write(cmd[-1], "existing file contents\n")
+
+        # The abspath() for cmd[-1] strips a trailing slash, and we want to
+        # test what happens when it is present. So put it back.
+        if target.endswith("/"):
+            cmd[-1] += "/"
+
+        d = self.do_cli(*cmd)
+        def _check(res):
+            (rc, out, err) = res
+            err = err.strip()
+            if rc == 0:
+                return self.check_output()
+            if rc == 1:
+                self.failUnlessEqual(out, "", str(res))
+                if "when copying into a directory, all source files must have names, but" in err:
+                    return set(["E2-DESTNAME"])
+                if err == "cannot copy directories without --recursive":
+                    return set(["E4-NEED-R"])
+                if err == "cannot copy directory into a file":
+                    return set(["E5-DIRTOFILE"])
+                if err == "copying multiple things requires target be a directory":
+                    return set(["E6-MANYONE"])
+                if err == "target is not a directory, but ends with a slash":
+                    return set(["E7-BADSLASH"])
+            self.fail("unrecognized error ('%s') %s" % (case, res))
+        d.addCallback(_check)
+        return d
+
+    def do_one_test(self, case, orig_expected):
+        expected = set(orig_expected)
+        printable_expected = ",".join(sorted(expected))
+        #print "---", case, ":", printable_expected
+
+        for f in orig_expected:
+            # f is "dir/file" or "dir/sub/file" or "dir/" or "dir/sub/"
+            # we want all parent directories in the set, with trailing /
+            pieces = f.rstrip("/").split("/")
+            for i in range(1,len(pieces)):
+                parent = "/".join(pieces[:i])
+                expected.add(parent+"/")
+
+        d = self.run_one_case(case)
+        def _dump(got):
+            ok = "ok" if got == expected else "FAIL"
+            printable_got = ",".join(sorted(got))
+            print "%-31s: got %-19s, want %-19s %s" % (case, printable_got,
+                                                       printable_expected, ok)
+            return got
+        #d.addCallback(_dump)
+        def _check(got):
+            self.failUnlessEqual(got, expected, case)
+        d.addCallback(_check)
+        return d
+
+    def do_tests(self):
+        # then we run various forms of "cp [-r] TAHOETHING to[/missing]"
+        # and see what happens.
+        d = defer.succeed(None)
+        #print
+
+        for line in COPYOUT_TESTCASES.splitlines():
+            if "#" in line:
+                line = line[:line.find("#")]
+            line = line.strip()
+            if not line:
+                continue
+            case, expected = line.split(":")
+            case = case.strip()
+            expected = frozenset(expected.strip().split(","))
+
+            d.addCallback(lambda ign, case=case, expected=expected:
+                          self.do_one_test(case, expected))
+
+        return d
+
+    def test_cp_out(self):
+        # test copying all sorts of things out of a tahoe filesystem
+        self.basedir = "cli_cp/CopyOut/cp_out"
+        self.set_up_grid(num_servers=1)
+
+        d = self.do_setup()
+        d.addCallback(lambda ign: self.do_tests())
+        return d