mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2024-12-21 13:57:51 +00:00
Merge branch '2329-2': fix "tahoe cp -r" exception
This commit is contained in:
Brian Warner
commit
e60392a479
@ -2,6 +2,7 @@
|
||||
import os.path
|
||||
import urllib
|
||||
import simplejson
|
||||
from collections import defaultdict
|
||||
from cStringIO import StringIO
|
||||
from twisted.python.failure import Failure
|
||||
from allmydata.scripts.common import get_alias, escape_path, \
|
||||
@ -12,7 +13,7 @@ from allmydata.util import fileutil
|
||||
from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath
|
||||
from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
|
||||
quote_local_unicode_path, to_str
|
||||
from allmydata.util.assertutil import precondition
|
||||
from allmydata.util.assertutil import precondition, _assert
|
||||
|
||||
|
||||
class MissingSourceError(TahoeError):
|
||||
@ -115,7 +116,7 @@ class LocalDirectorySource:
|
||||
child = LocalDirectorySource(self.progressfunc, pn, n)
|
||||
self.children[n] = child
|
||||
if recurse:
|
||||
child.populate(True)
|
||||
child.populate(recurse=True)
|
||||
elif os.path.isfile(pn):
|
||||
self.children[n] = LocalFileSource(pn, n)
|
||||
else:
|
||||
@ -143,7 +144,7 @@ class LocalDirectoryTarget:
|
||||
child = LocalDirectoryTarget(self.progressfunc, pn)
|
||||
self.children[n] = child
|
||||
if recurse:
|
||||
child.populate(True)
|
||||
child.populate(recurse=True)
|
||||
else:
|
||||
assert os.path.isfile(pn)
|
||||
self.children[n] = LocalFileTarget(pn)
|
||||
@ -151,12 +152,14 @@ class LocalDirectoryTarget:
|
||||
def get_child_target(self, name):
|
||||
precondition(isinstance(name, unicode), name)
|
||||
if self.children is None:
|
||||
self.populate(False)
|
||||
self.populate(recurse=False)
|
||||
if name in self.children:
|
||||
return self.children[name]
|
||||
pathname = os.path.join(self.pathname, name)
|
||||
os.makedirs(pathname)
|
||||
return LocalDirectoryTarget(self.progressfunc, pathname)
|
||||
child = LocalDirectoryTarget(self.progressfunc, pathname)
|
||||
self.children[name] = child
|
||||
return child
|
||||
|
||||
def put_file(self, name, inf):
|
||||
precondition(isinstance(name, unicode), name)
|
||||
@ -277,7 +280,7 @@ class TahoeDirectorySource:
|
||||
if readcap:
|
||||
self.cache[readcap] = child
|
||||
if recurse:
|
||||
child.populate(True)
|
||||
child.populate(recurse=True)
|
||||
self.children[name] = child
|
||||
else:
|
||||
# TODO: there should be an option to skip unknown nodes.
|
||||
@ -337,6 +340,7 @@ class TahoeDirectoryTarget:
|
||||
self.children = None
|
||||
|
||||
def just_created(self, writecap):
|
||||
# TODO: maybe integrate this with the constructor
|
||||
self.writecap = writecap
|
||||
self.readcap = uri.from_string(writecap).get_readonly().to_string()
|
||||
self.mutable = True
|
||||
@ -376,7 +380,7 @@ class TahoeDirectoryTarget:
|
||||
if readcap:
|
||||
self.cache[readcap] = child
|
||||
if recurse:
|
||||
child.populate(True)
|
||||
child.populate(recurse=True)
|
||||
self.children[name] = child
|
||||
else:
|
||||
# TODO: there should be an option to skip unknown nodes.
|
||||
@ -388,7 +392,7 @@ class TahoeDirectoryTarget:
|
||||
# return a new target for a named subdirectory of this dir
|
||||
precondition(isinstance(name, unicode), name)
|
||||
if self.children is None:
|
||||
self.populate(False)
|
||||
self.populate(recurse=False)
|
||||
if name in self.children:
|
||||
return self.children[name]
|
||||
writecap = make_tahoe_subdirectory(self.nodeurl, self.writecap, name)
|
||||
@ -405,7 +409,7 @@ class TahoeDirectoryTarget:
|
||||
inf = inf.read()
|
||||
|
||||
if self.children is None:
|
||||
self.populate(False)
|
||||
self.populate(recurse=False)
|
||||
|
||||
# Check to see if we already have a mutable file by this name.
|
||||
# If so, overwrite that file in place.
|
||||
@ -439,6 +443,12 @@ class TahoeDirectoryTarget:
|
||||
body = simplejson.dumps(set_data)
|
||||
POST(url, body)
|
||||
|
||||
FileSources = (LocalFileSource, TahoeFileSource)
|
||||
DirectorySources = (LocalDirectorySource, TahoeDirectorySource)
|
||||
FileTargets = (LocalFileTarget, TahoeFileTarget)
|
||||
DirectoryTargets = (LocalDirectoryTarget, TahoeDirectoryTarget)
|
||||
MissingTargets = (LocalMissingTarget, TahoeMissingTarget)
|
||||
|
||||
class Copier:
|
||||
|
||||
def do_copy(self, options, progressfunc=None):
|
||||
@ -476,58 +486,83 @@ class Copier:
|
||||
return 1
|
||||
|
||||
def try_copy(self):
|
||||
"""
|
||||
All usage errors are caught here, not in a subroutine. This bottoms
|
||||
out in copy_file_to_file() or copy_things_to_directory().
|
||||
"""
|
||||
source_specs = self.options.sources
|
||||
destination_spec = self.options.destination
|
||||
recursive = self.options["recursive"]
|
||||
|
||||
target = self.get_target_info(destination_spec)
|
||||
precondition(isinstance(target, FileTargets + DirectoryTargets + MissingTargets), target)
|
||||
target_has_trailing_slash = destination_spec.endswith("/")
|
||||
|
||||
sources = [] # list of source objects
|
||||
for ss in source_specs:
|
||||
sources.append(self.get_source_info(ss))
|
||||
si = self.get_source_info(ss)
|
||||
precondition(isinstance(si, FileSources + DirectorySources), si)
|
||||
sources.append(si)
|
||||
|
||||
have_source_dirs = any([isinstance(s, (LocalDirectorySource,
|
||||
TahoeDirectorySource))
|
||||
# if any source is a directory, must use -r
|
||||
# if target is missing:
|
||||
# if source is a single file, target will be a file
|
||||
# else target will be a directory, so mkdir it
|
||||
# if there are multiple sources, target must be a dir
|
||||
# if target is a file, source must be a single file
|
||||
# if target is directory, sources must be named or a dir
|
||||
|
||||
have_source_dirs = any([isinstance(s, DirectorySources)
|
||||
for s in sources])
|
||||
|
||||
if have_source_dirs and not recursive:
|
||||
# 'cp dir target' without -r: error
|
||||
self.to_stderr("cannot copy directories without --recursive")
|
||||
return 1
|
||||
del recursive # -r is only used for signalling errors
|
||||
|
||||
if isinstance(target, (LocalFileTarget, TahoeFileTarget)):
|
||||
# cp STUFF foo.txt, where foo.txt already exists. This limits the
|
||||
# possibilities considerably.
|
||||
if len(sources) > 1:
|
||||
self.to_stderr("target %s is not a directory" % quote_output(destination_spec))
|
||||
return 1
|
||||
if have_source_dirs:
|
||||
if isinstance(target, FileTargets):
|
||||
target_is_file = True
|
||||
elif isinstance(target, DirectoryTargets):
|
||||
target_is_file = False
|
||||
else: # isinstance(target, MissingTargets)
|
||||
if len(sources) == 1 and isinstance(sources[0], FileSources):
|
||||
target_is_file = True
|
||||
else:
|
||||
target_is_file = False
|
||||
|
||||
if target_is_file and target_has_trailing_slash:
|
||||
self.to_stderr("target is not a directory, but ends with a slash")
|
||||
return 1
|
||||
|
||||
if len(sources) > 1 and target_is_file:
|
||||
self.to_stderr("copying multiple things requires target be a directory")
|
||||
return 1
|
||||
|
||||
if target_is_file:
|
||||
_assert(len(sources) == 1, sources)
|
||||
if not isinstance(sources[0], FileSources):
|
||||
# 'cp -r dir existingfile': error
|
||||
self.to_stderr("cannot copy directory into a file")
|
||||
return 1
|
||||
return self.copy_file(sources[0], target)
|
||||
return self.copy_file_to_file(sources[0], target)
|
||||
|
||||
if isinstance(target, (LocalMissingTarget, TahoeMissingTarget)):
|
||||
if recursive:
|
||||
return self.copy_to_directory(sources, target)
|
||||
if len(sources) > 1:
|
||||
# if we have -r, we'll auto-create the target directory. Without
|
||||
# it, we'll only create a file.
|
||||
self.to_stderr("cannot copy multiple files into a file without -r")
|
||||
# else target is a directory, so each source must be one of:
|
||||
# * a named file (copied to a new file under the target)
|
||||
# * a named directory (causes a new directory of the same name to be
|
||||
# created under the target, then the contents of the source are
|
||||
# copied into that directory)
|
||||
# * an unnamed directory (the contents of the source are copied into
|
||||
# the target, without a new directory being made)
|
||||
#
|
||||
# If any source is an unnamed file, throw an error, since we have no
|
||||
# way to name the output file.
|
||||
_assert(isinstance(target, DirectoryTargets + MissingTargets), target)
|
||||
|
||||
for source in sources:
|
||||
if isinstance(source, FileSources) and source.basename() is None:
|
||||
self.to_stderr("when copying into a directory, all source files must have names, but %s is unnamed" % quote_output(source_specs[0]))
|
||||
return 1
|
||||
# cp file1 newfile
|
||||
return self.copy_file(sources[0], target)
|
||||
|
||||
if isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)):
|
||||
# We're copying to an existing directory -- make sure that we
|
||||
# have target names for everything
|
||||
for source in sources:
|
||||
if source.basename() is None and isinstance(source, TahoeFileSource):
|
||||
self.to_stderr(
|
||||
"error: you must specify a destination filename")
|
||||
return 1
|
||||
return self.copy_to_directory(sources, target)
|
||||
|
||||
self.to_stderr("unknown target")
|
||||
return 1
|
||||
return self.copy_things_to_directory(sources, target)
|
||||
|
||||
def to_stderr(self, text):
|
||||
print >>self.stderr, text
|
||||
@ -580,6 +615,9 @@ class Copier:
|
||||
return t
|
||||
|
||||
def get_source_info(self, source_spec):
|
||||
"""
|
||||
This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
|
||||
"""
|
||||
precondition(isinstance(source_spec, unicode), source_spec)
|
||||
rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
|
||||
path = path_utf8.decode("utf-8")
|
||||
@ -632,109 +670,6 @@ class Copier:
|
||||
return t
|
||||
|
||||
|
||||
def dump_graph(self, s, indent=" "):
|
||||
for name, child in s.children.items():
|
||||
print "%s%s: %r" % (indent, quote_output(name), child)
|
||||
if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
|
||||
self.dump_graph(child, indent+" ")
|
||||
|
||||
def copy_to_directory(self, sources, target):
|
||||
# step one: build a recursive graph of the source tree. This returns
|
||||
# a dictionary, with child names as keys, and values that are either
|
||||
# Directory or File instances (local or tahoe).
|
||||
source_dirs = self.build_graphs(sources)
|
||||
source_files = [s for s in sources
|
||||
if isinstance(s, (LocalFileSource, TahoeFileSource))]
|
||||
|
||||
#print "graphs"
|
||||
#for s in source_dirs:
|
||||
# self.dump_graph(s)
|
||||
|
||||
# step two: create the top-level target directory object
|
||||
if isinstance(target, LocalMissingTarget):
|
||||
os.makedirs(target.pathname)
|
||||
target = LocalDirectoryTarget(self.progress, target.pathname)
|
||||
elif isinstance(target, TahoeMissingTarget):
|
||||
writecap = mkdir(target.url)
|
||||
target = TahoeDirectoryTarget(self.nodeurl, self.cache,
|
||||
self.progress)
|
||||
target.just_created(writecap)
|
||||
assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget))
|
||||
target.populate(False)
|
||||
|
||||
# step three: find a target for each source node, creating
|
||||
# directories as necessary. 'targetmap' is a dictionary that uses
|
||||
# target Directory instances as keys, and has values of
|
||||
# (name->sourceobject) dicts for all the files that need to wind up
|
||||
# there.
|
||||
|
||||
# sources are all LocalFile/LocalDirectory/TahoeFile/TahoeDirectory
|
||||
# target is LocalDirectory/TahoeDirectory
|
||||
|
||||
self.progress("attaching sources to targets, "
|
||||
"%d files / %d dirs in root" %
|
||||
(len(source_files), len(source_dirs)))
|
||||
|
||||
self.targetmap = {}
|
||||
self.files_to_copy = 0
|
||||
|
||||
for s in source_files:
|
||||
self.attach_to_target(s, s.basename(), target)
|
||||
|
||||
for (name, source) in source_dirs:
|
||||
new_target = target.get_child_target(name)
|
||||
self.assign_targets(source, new_target)
|
||||
|
||||
self.progress("targets assigned, %s dirs, %s files" %
|
||||
(len(self.targetmap), self.files_to_copy))
|
||||
|
||||
self.progress("starting copy, %d files, %d directories" %
|
||||
(self.files_to_copy, len(self.targetmap)))
|
||||
self.files_copied = 0
|
||||
self.targets_finished = 0
|
||||
|
||||
# step four: walk through the list of targets. For each one, copy all
|
||||
# the files. If the target is a TahoeDirectory, upload and create
|
||||
# read-caps, then do a set_children to the target directory.
|
||||
|
||||
for target in self.targetmap:
|
||||
self.copy_files_to_target(self.targetmap[target], target)
|
||||
self.targets_finished += 1
|
||||
self.progress("%d/%d directories" %
|
||||
(self.targets_finished, len(self.targetmap)))
|
||||
|
||||
return self.announce_success("files copied")
|
||||
|
||||
def attach_to_target(self, source, name, target):
|
||||
precondition(isinstance(name, unicode), name)
|
||||
if target not in self.targetmap:
|
||||
self.targetmap[target] = {}
|
||||
self.targetmap[target][name] = source
|
||||
self.files_to_copy += 1
|
||||
|
||||
def assign_targets(self, source, target):
|
||||
# copy everything in the source into the target
|
||||
precondition(isinstance(source, (LocalDirectorySource, TahoeDirectorySource)), source)
|
||||
|
||||
for name, child in source.children.items():
|
||||
if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)):
|
||||
# we will need a target directory for this one
|
||||
subtarget = target.get_child_target(name)
|
||||
self.assign_targets(child, subtarget)
|
||||
else:
|
||||
precondition(isinstance(child, (LocalFileSource, TahoeFileSource)), child)
|
||||
self.attach_to_target(child, name, target)
|
||||
|
||||
def copy_files_to_target(self, targetmap, target):
|
||||
for name, source in targetmap.items():
|
||||
precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
|
||||
self.copy_file_into(source, name, target)
|
||||
self.files_copied += 1
|
||||
self.progress("%d/%d files, %d/%d directories" %
|
||||
(self.files_copied, self.files_to_copy,
|
||||
self.targets_finished, len(self.targetmap)))
|
||||
target.set_children()
|
||||
|
||||
def need_to_copy_bytes(self, source, target):
|
||||
if source.need_to_copy_bytes:
|
||||
# mutable tahoe files, and local files
|
||||
@ -748,10 +683,9 @@ class Copier:
|
||||
print >>self.stdout, "Success: %s" % msg
|
||||
return 0
|
||||
|
||||
def copy_file(self, source, target):
|
||||
precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
|
||||
precondition(isinstance(target, (LocalFileTarget, TahoeFileTarget,
|
||||
LocalMissingTarget, TahoeMissingTarget)), target)
|
||||
def copy_file_to_file(self, source, target):
|
||||
precondition(isinstance(source, FileSources), source)
|
||||
precondition(isinstance(target, FileTargets + MissingTargets), target)
|
||||
if self.need_to_copy_bytes(source, target):
|
||||
# if the target is a local directory, this will just write the
|
||||
# bytes to disk. If it is a tahoe directory, it will upload the
|
||||
@ -765,9 +699,115 @@ class Copier:
|
||||
target.put_uri(source.bestcap())
|
||||
return self.announce_success("file linked")
|
||||
|
||||
def copy_file_into(self, source, name, target):
|
||||
precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source)
|
||||
precondition(isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)), target)
|
||||
def copy_things_to_directory(self, sources, target):
|
||||
# step one: if the target is missing, we should mkdir it
|
||||
target = self.maybe_create_target(target)
|
||||
target.populate(recurse=False)
|
||||
|
||||
# step two: scan any source dirs, recursively, to find children
|
||||
for s in sources:
|
||||
if isinstance(s, DirectorySources):
|
||||
s.populate(recurse=True)
|
||||
if isinstance(s, FileSources):
|
||||
# each source must have a name, or be a directory
|
||||
_assert(s.basename() is not None, s)
|
||||
|
||||
# step three: find a target for each source node, creating
|
||||
# directories as necessary. 'targetmap' is a dictionary that uses
|
||||
# target Directory instances as keys, and has values of (name:
|
||||
# sourceobject) dicts for all the files that need to wind up there.
|
||||
targetmap = self.build_targetmap(sources, target)
|
||||
|
||||
# step four: walk through the list of targets. For each one, copy all
|
||||
# the files. If the target is a TahoeDirectory, upload and create
|
||||
# read-caps, then do a set_children to the target directory.
|
||||
self.copy_to_targetmap(targetmap)
|
||||
|
||||
return self.announce_success("files copied")
|
||||
|
||||
def maybe_create_target(self, target):
|
||||
if isinstance(target, LocalMissingTarget):
|
||||
os.makedirs(target.pathname)
|
||||
target = LocalDirectoryTarget(self.progress, target.pathname)
|
||||
elif isinstance(target, TahoeMissingTarget):
|
||||
writecap = mkdir(target.url)
|
||||
target = TahoeDirectoryTarget(self.nodeurl, self.cache,
|
||||
self.progress)
|
||||
target.just_created(writecap)
|
||||
# afterwards, or otherwise, it will be a directory
|
||||
precondition(isinstance(target, DirectoryTargets), target)
|
||||
return target
|
||||
|
||||
def build_targetmap(self, sources, target):
|
||||
num_source_files = len([s for s in sources
|
||||
if isinstance(s, FileSources)])
|
||||
num_source_dirs = len([s for s in sources
|
||||
if isinstance(s, DirectorySources)])
|
||||
self.progress("attaching sources to targets, "
|
||||
"%d files / %d dirs in root" %
|
||||
(num_source_files, num_source_dirs))
|
||||
|
||||
# this maps each target directory to a list of source files that need
|
||||
# to be copied into it. All source files have names.
|
||||
targetmap = defaultdict(list)
|
||||
|
||||
for s in sources:
|
||||
if isinstance(s, FileSources):
|
||||
targetmap[target].append(s)
|
||||
else:
|
||||
_assert(isinstance(s, DirectorySources), s)
|
||||
name = s.basename()
|
||||
if name is not None:
|
||||
# named sources get a new directory. see #2329
|
||||
new_target = target.get_child_target(name)
|
||||
else:
|
||||
# unnamed sources have their contents copied directly
|
||||
new_target = target
|
||||
self.assign_targets(targetmap, s, new_target)
|
||||
|
||||
self.progress("targets assigned, %s dirs, %s files" %
|
||||
(len(targetmap), self.count_files_to_copy(targetmap)))
|
||||
return targetmap
|
||||
|
||||
def assign_targets(self, targetmap, source, target):
|
||||
# copy everything in the source into the target
|
||||
precondition(isinstance(source, DirectorySources), source)
|
||||
for name, child in source.children.items():
|
||||
if isinstance(child, DirectorySources):
|
||||
# we will need a target directory for this one
|
||||
subtarget = target.get_child_target(name)
|
||||
self.assign_targets(targetmap, child, subtarget)
|
||||
else:
|
||||
_assert(isinstance(child, FileSources), child)
|
||||
targetmap[target].append(child)
|
||||
|
||||
def copy_to_targetmap(self, targetmap):
|
||||
files_to_copy = self.count_files_to_copy(targetmap)
|
||||
self.progress("starting copy, %d files, %d directories" %
|
||||
(files_to_copy, len(targetmap)))
|
||||
files_copied = 0
|
||||
targets_finished = 0
|
||||
|
||||
for target, sources in targetmap.items():
|
||||
_assert(isinstance(target, DirectoryTargets), target)
|
||||
for source in sources:
|
||||
_assert(isinstance(source, FileSources), source)
|
||||
self.copy_file_into_dir(source, source.basename(), target)
|
||||
files_copied += 1
|
||||
self.progress("%d/%d files, %d/%d directories" %
|
||||
(files_copied, files_to_copy,
|
||||
targets_finished, len(targetmap)))
|
||||
target.set_children()
|
||||
targets_finished += 1
|
||||
self.progress("%d/%d directories" %
|
||||
(targets_finished, len(targetmap)))
|
||||
|
||||
def count_files_to_copy(self, targetmap):
|
||||
return sum([len(sources) for sources in targetmap.values()])
|
||||
|
||||
def copy_file_into_dir(self, source, name, target):
|
||||
precondition(isinstance(source, FileSources), source)
|
||||
precondition(isinstance(target, DirectoryTargets), target)
|
||||
precondition(isinstance(name, unicode), name)
|
||||
if self.need_to_copy_bytes(source, target):
|
||||
# if the target is a local directory, this will just write the
|
||||
@ -786,14 +826,6 @@ class Copier:
|
||||
if self.progressfunc:
|
||||
self.progressfunc(message)
|
||||
|
||||
def build_graphs(self, sources):
|
||||
graphs = []
|
||||
for source in sources:
|
||||
if isinstance(source, (LocalDirectorySource, TahoeDirectorySource)):
|
||||
source.populate(True)
|
||||
graphs.append((source.basename(), source))
|
||||
return graphs
|
||||
|
||||
|
||||
def copy(options):
|
||||
return Copier().do_copy(options)
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
import os.path, simplejson
|
||||
from twisted.trial import unittest
|
||||
from twisted.python import usage
|
||||
from twisted.internet import defer
|
||||
|
||||
from allmydata.scripts import cli
|
||||
from allmydata.util import fileutil
|
||||
from allmydata.util.encodingutil import (quote_output, get_io_encoding,
|
||||
unicode_to_output, to_str)
|
||||
from allmydata.util.assertutil import _assert
|
||||
from .no_network import GridTestMixin
|
||||
from .test_cli import CLITestMixin
|
||||
|
||||
@ -119,7 +121,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||
d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir))
|
||||
def _resp((rc, out, err)):
|
||||
self.failUnlessReallyEqual(rc, 1)
|
||||
self.failUnlessIn("error: you must specify a destination filename",
|
||||
self.failUnlessIn("when copying into a directory, all source files must have names, but",
|
||||
err)
|
||||
self.failUnlessReallyEqual(out, "")
|
||||
d.addCallback(_resp)
|
||||
@ -653,3 +655,348 @@ starting copy, 2 files, 1 directories
|
||||
(rc, out, err) = res
|
||||
self.failUnlessIn("Success: file copied", out, str(res))
|
||||
return d
|
||||
|
||||
# these test cases come from ticket #2329 comment 40
|
||||
# trailing slash on target *directory* should not matter, test both
|
||||
# trailing slash on files should cause error
|
||||
|
||||
COPYOUT_TESTCASES = """
|
||||
cp $FILECAP to/existing-file : to/existing-file
|
||||
cp -r $FILECAP to/existing-file : to/existing-file
|
||||
cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
|
||||
cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
|
||||
cp $DIRCAP to/existing-file : E4-NEED-R
|
||||
cp -r $DIRCAP to/existing-file : E5-DIRTOFILE
|
||||
cp $FILECAP $DIRCAP to/existing-file : E4-NEED-R
|
||||
cp -r $FILECAP $DIRCAP to/existing-file : E6-MANYONE
|
||||
|
||||
cp $FILECAP to/existing-file/ : E7-BADSLASH
|
||||
cp -r $FILECAP to/existing-file/ : E7-BADSLASH
|
||||
cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
|
||||
cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
|
||||
cp $DIRCAP to/existing-file/ : E4-NEED-R
|
||||
cp -r $DIRCAP to/existing-file/ : E7-BADSLASH
|
||||
cp $FILECAP $DIRCAP to/existing-file/ : E4-NEED-R
|
||||
cp -r $FILECAP $DIRCAP to/existing-file/ : E7-BADSLASH
|
||||
|
||||
# single source to a (present) target directory
|
||||
cp $FILECAP to : E2-DESTNAME
|
||||
cp -r $FILECAP to : E2-DESTNAME
|
||||
cp $DIRCAP/file to : to/file
|
||||
cp -r $DIRCAP/file to : to/file
|
||||
cp $PARENTCAP/dir to : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir to : to/dir/file
|
||||
cp $DIRCAP to : E4-NEED-R
|
||||
cp -r $DIRCAP to : to/file
|
||||
cp $DIRALIAS to : E4-NEED-R
|
||||
cp -r $DIRALIAS to : to/file
|
||||
|
||||
cp $FILECAP to/ : E2-DESTNAME
|
||||
cp -r $FILECAP to/ : E2-DESTNAME
|
||||
cp $DIRCAP/file to/ : to/file
|
||||
cp -r $DIRCAP/file to/ : to/file
|
||||
cp $PARENTCAP/dir to/ : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir to/ : to/dir/file
|
||||
cp $DIRCAP to/ : E4-NEED-R
|
||||
cp -r $DIRCAP to/ : to/file
|
||||
cp $DIRALIAS to/ : E4-NEED-R
|
||||
cp -r $DIRALIAS to/ : to/file
|
||||
|
||||
# multiple sources to a (present) target directory
|
||||
cp $DIRCAP/file $PARENTCAP/dir2/file2 to : to/file,to/file2
|
||||
cp $DIRCAP/file $FILECAP to : E2-DESTNAME
|
||||
cp $DIRCAP $FILECAP to : E4-NEED-R
|
||||
cp -r $DIRCAP $FILECAP to : E2-DESTNAME
|
||||
# namedfile, unnameddir, nameddir
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : to/file3,to/file,to/dir2/file2
|
||||
# namedfile, unnameddir, nameddir, unnamedfile
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E2-DESTNAME
|
||||
|
||||
cp $DIRCAP/file $PARENTCAP/dir2/file2 to/ : to/file,to/file2
|
||||
cp $DIRCAP/file $FILECAP to/ : E2-DESTNAME
|
||||
cp $DIRCAP $FILECAP to/ : E4-NEED-R
|
||||
cp -r $DIRCAP $FILECAP to/ : E2-DESTNAME
|
||||
# namedfile, unnameddir, nameddir
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : to/file3,to/file,to/dir2/file2
|
||||
# namedfile, unnameddir, nameddir, unnamedfile
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E2-DESTNAME
|
||||
|
||||
# single sources to a missing target: should mkdir or create a file
|
||||
cp $FILECAP to/missing : to/missing
|
||||
cp -r $FILECAP to/missing : to/missing
|
||||
cp $DIRCAP/file to/missing : to/missing
|
||||
cp -r $DIRCAP/file to/missing : to/missing
|
||||
cp $PARENTCAP/dir to/missing : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir to/missing : to/missing/dir/file
|
||||
cp $DIRCAP to/missing : E4-NEED-R
|
||||
cp -r $DIRCAP to/missing : to/missing/file
|
||||
cp $DIRALIAS to/missing : E4-NEED-R
|
||||
cp -r $DIRALIAS to/missing : to/missing/file
|
||||
|
||||
cp $FILECAP to/missing/ : E7-BADSLASH
|
||||
cp -r $FILECAP to/missing/ : E7-BADSLASH
|
||||
cp $DIRCAP/file to/missing/ : E7-BADSLASH
|
||||
cp -r $DIRCAP/file to/missing/ : E7-BADSLASH
|
||||
cp $PARENTCAP/dir to/missing/ : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir to/missing/ : to/missing/dir/file
|
||||
cp $DIRCAP to/missing/ : E4-NEED-R
|
||||
cp -r $DIRCAP to/missing/ : to/missing/file
|
||||
cp $DIRALIAS to/missing/ : E4-NEED-R
|
||||
cp -r $DIRALIAS to/missing/ : to/missing/file
|
||||
|
||||
# multiple things to a missing target: should mkdir
|
||||
cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
|
||||
cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
|
||||
cp $DIRCAP/file $FILECAP to/missing : E2-DESTNAME
|
||||
cp -r $DIRCAP/file $FILECAP to/missing : E2-DESTNAME
|
||||
cp $DIRCAP $FILECAP to/missing : E4-NEED-R
|
||||
cp -r $DIRCAP $FILECAP to/missing : E2-DESTNAME
|
||||
# namedfile, unnameddir, nameddir
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing : to/missing/file3,to/missing/file,to/missing/dir2/file2
|
||||
# namedfile, unnameddir, nameddir, unnamedfile
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E2-DESTNAME
|
||||
|
||||
cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
|
||||
cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
|
||||
cp $DIRCAP/file $FILECAP to/missing/ : E2-DESTNAME
|
||||
cp -r $DIRCAP/file $FILECAP to/missing/ : E2-DESTNAME
|
||||
cp $DIRCAP $FILECAP to/missing/ : E4-NEED-R
|
||||
cp -r $DIRCAP $FILECAP to/missing/ : E2-DESTNAME
|
||||
# namedfile, unnameddir, nameddir
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing/ : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing/ : to/missing/file3,to/missing/file,to/missing/dir2/file2
|
||||
# namedfile, unnameddir, nameddir, unnamedfile
|
||||
cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E4-NEED-R
|
||||
cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E2-DESTNAME
|
||||
|
||||
# make sure empty directories are copied too
|
||||
cp -r $PARENTCAP/dir4 to : to/dir4/emptydir/
|
||||
cp -r $PARENTCAP/dir4 to/ : to/dir4/emptydir/
|
||||
|
||||
# name collisions: ensure files are copied in order
|
||||
cp -r $PARENTCAP/dir6/dir $PARENTCAP/dir5/dir to : to/dir/collide=5
|
||||
cp -r $PARENTCAP/dir5/dir $PARENTCAP/dir6/dir to : to/dir/collide=6
|
||||
cp -r $DIRCAP6 $DIRCAP5 to : to/dir/collide=5
|
||||
cp -r $DIRCAP5 $DIRCAP6 to : to/dir/collide=6
|
||||
|
||||
"""
|
||||
|
||||
class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||
FILE_CONTENTS = "file text"
|
||||
FILE_CONTENTS_5 = "5"
|
||||
FILE_CONTENTS_6 = "6"
|
||||
|
||||
def do_setup(self):
|
||||
# first we build a tahoe filesystem that contains:
|
||||
# $PARENTCAP
|
||||
# $PARENTCAP/dir == $DIRCAP == alias:
|
||||
# $PARENTCAP/dir/file == $FILECAP
|
||||
# $PARENTCAP/dir2 (named directory)
|
||||
# $PARENTCAP/dir2/file2
|
||||
# $PARENTCAP/dir3/file3 (a second named file)
|
||||
# $PARENTCAP/dir4
|
||||
# $PARENTCAP/dir4/emptydir/ (an empty directory)
|
||||
# $PARENTCAP/dir5 == $DIRCAP5
|
||||
# $PARENTCAP/dir5/dir/collide (contents are "5")
|
||||
# $PARENTCAP/dir6 == $DIRCAP6
|
||||
# $PARENTCAP/dir6/dir/collide (contents are "6")
|
||||
|
||||
source_file = os.path.join(self.basedir, "file")
|
||||
fileutil.write(source_file, self.FILE_CONTENTS)
|
||||
source_file_5 = os.path.join(self.basedir, "file5")
|
||||
fileutil.write(source_file_5, self.FILE_CONTENTS_5)
|
||||
source_file_6 = os.path.join(self.basedir, "file6")
|
||||
fileutil.write(source_file_6, self.FILE_CONTENTS_6)
|
||||
|
||||
d = self.do_cli("mkdir")
|
||||
def _stash_parentdircap(res):
|
||||
(rc, out, err) = res
|
||||
self.failUnlessEqual(rc, 0, str(res))
|
||||
self.failUnlessEqual(err, "", str(res))
|
||||
self.PARENTCAP = out.strip()
|
||||
return self.do_cli("mkdir", "%s/dir" % self.PARENTCAP)
|
||||
d.addCallback(_stash_parentdircap)
|
||||
def _stash_dircap(res):
|
||||
(rc, out, err) = res
|
||||
self.failUnlessEqual(rc, 0, str(res))
|
||||
self.failUnlessEqual(err, "", str(res))
|
||||
self.DIRCAP = out.strip()
|
||||
return self.do_cli("add-alias", "ALIAS", self.DIRCAP)
|
||||
d.addCallback(_stash_dircap)
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("put", source_file, "%s/dir/file" % self.PARENTCAP))
|
||||
def _stash_filecap(res):
|
||||
(rc, out, err) = res
|
||||
self.failUnlessEqual(rc, 0, str(res))
|
||||
self.failUnlessEqual(err.strip(), "201 Created", str(res))
|
||||
self.FILECAP = out.strip()
|
||||
assert self.FILECAP.startswith("URI:LIT:")
|
||||
d.addCallback(_stash_filecap)
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir2" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("put", source_file, "%s/dir2/file2" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir3" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("put", source_file, "%s/dir3/file3" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir4" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir4/emptydir" % self.PARENTCAP))
|
||||
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir5" % self.PARENTCAP))
|
||||
def _stash_dircap_5(res):
|
||||
(rc, out, err) = res
|
||||
self.failUnlessEqual(rc, 0, str(res))
|
||||
self.failUnlessEqual(err, "", str(res))
|
||||
self.DIRCAP5 = out.strip()
|
||||
d.addCallback(_stash_dircap_5)
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir5/dir" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("put", source_file_5, "%s/dir5/dir/collide" % self.PARENTCAP))
|
||||
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir6" % self.PARENTCAP))
|
||||
def _stash_dircap_6(res):
|
||||
(rc, out, err) = res
|
||||
self.failUnlessEqual(rc, 0, str(res))
|
||||
self.failUnlessEqual(err, "", str(res))
|
||||
self.DIRCAP6 = out.strip()
|
||||
d.addCallback(_stash_dircap_6)
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("mkdir", "%s/dir6/dir" % self.PARENTCAP))
|
||||
d.addCallback(lambda ign:
|
||||
self.do_cli("put", source_file_6, "%s/dir6/dir/collide" % self.PARENTCAP))
|
||||
|
||||
return d
|
||||
|
||||
def check_output(self):
|
||||
# locate the files and directories created (if any) under to/
|
||||
top = os.path.join(self.basedir, "to")
|
||||
results = set()
|
||||
for (dirpath, dirnames, filenames) in os.walk(top):
|
||||
assert dirpath.startswith(top)
|
||||
here = "/".join(dirpath.split(os.sep)[len(top.split(os.sep))-1:])
|
||||
results.add(here+"/")
|
||||
for fn in filenames:
|
||||
contents = fileutil.read(os.path.join(dirpath, fn))
|
||||
if contents == self.FILE_CONTENTS:
|
||||
results.add("%s/%s" % (here, fn))
|
||||
elif contents == self.FILE_CONTENTS_5:
|
||||
results.add("%s/%s=5" % (here, fn))
|
||||
elif contents == self.FILE_CONTENTS_6:
|
||||
results.add("%s/%s=6" % (here, fn))
|
||||
return results
|
||||
|
||||
def run_one_case(self, case):
|
||||
cmd = (case
|
||||
.replace("$PARENTCAP", self.PARENTCAP)
|
||||
.replace("$DIRCAP5", self.DIRCAP5)
|
||||
.replace("$DIRCAP6", self.DIRCAP6)
|
||||
.replace("$DIRCAP", self.DIRCAP)
|
||||
.replace("$DIRALIAS", "ALIAS:")
|
||||
.replace("$FILECAP", self.FILECAP)
|
||||
.split())
|
||||
target = cmd[-1]
|
||||
_assert(target == "to" or target.startswith("to/"), target)
|
||||
cmd[-1] = os.path.abspath(os.path.join(self.basedir, cmd[-1]))
|
||||
|
||||
# reset
|
||||
targetdir = os.path.abspath(os.path.join(self.basedir, "to"))
|
||||
fileutil.rm_dir(targetdir)
|
||||
os.mkdir(targetdir)
|
||||
|
||||
if target.rstrip("/") == "to/existing-file":
|
||||
fileutil.write(cmd[-1], "existing file contents\n")
|
||||
|
||||
# The abspath() for cmd[-1] strips a trailing slash, and we want to
|
||||
# test what happens when it is present. So put it back.
|
||||
if target.endswith("/"):
|
||||
cmd[-1] += "/"
|
||||
|
||||
d = self.do_cli(*cmd)
|
||||
def _check(res):
|
||||
(rc, out, err) = res
|
||||
err = err.strip()
|
||||
if rc == 0:
|
||||
return self.check_output()
|
||||
if rc == 1:
|
||||
self.failUnlessEqual(out, "", str(res))
|
||||
if "when copying into a directory, all source files must have names, but" in err:
|
||||
return set(["E2-DESTNAME"])
|
||||
if err == "cannot copy directories without --recursive":
|
||||
return set(["E4-NEED-R"])
|
||||
if err == "cannot copy directory into a file":
|
||||
return set(["E5-DIRTOFILE"])
|
||||
if err == "copying multiple things requires target be a directory":
|
||||
return set(["E6-MANYONE"])
|
||||
if err == "target is not a directory, but ends with a slash":
|
||||
return set(["E7-BADSLASH"])
|
||||
self.fail("unrecognized error ('%s') %s" % (case, res))
|
||||
d.addCallback(_check)
|
||||
return d
|
||||
|
||||
def do_one_test(self, case, orig_expected):
|
||||
expected = set(orig_expected)
|
||||
printable_expected = ",".join(sorted(expected))
|
||||
#print "---", case, ":", printable_expected
|
||||
|
||||
for f in orig_expected:
|
||||
# f is "dir/file" or "dir/sub/file" or "dir/" or "dir/sub/"
|
||||
# we want all parent directories in the set, with trailing /
|
||||
pieces = f.rstrip("/").split("/")
|
||||
for i in range(1,len(pieces)):
|
||||
parent = "/".join(pieces[:i])
|
||||
expected.add(parent+"/")
|
||||
|
||||
d = self.run_one_case(case)
|
||||
def _dump(got):
|
||||
ok = "ok" if got == expected else "FAIL"
|
||||
printable_got = ",".join(sorted(got))
|
||||
print "%-31s: got %-19s, want %-19s %s" % (case, printable_got,
|
||||
printable_expected, ok)
|
||||
return got
|
||||
#d.addCallback(_dump)
|
||||
def _check(got):
|
||||
self.failUnlessEqual(got, expected, case)
|
||||
d.addCallback(_check)
|
||||
return d
|
||||
|
||||
def do_tests(self):
|
||||
# then we run various forms of "cp [-r] TAHOETHING to[/missing]"
|
||||
# and see what happens.
|
||||
d = defer.succeed(None)
|
||||
#print
|
||||
|
||||
for line in COPYOUT_TESTCASES.splitlines():
|
||||
if "#" in line:
|
||||
line = line[:line.find("#")]
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
case, expected = line.split(":")
|
||||
case = case.strip()
|
||||
expected = frozenset(expected.strip().split(","))
|
||||
|
||||
d.addCallback(lambda ign, case=case, expected=expected:
|
||||
self.do_one_test(case, expected))
|
||||
|
||||
return d
|
||||
|
||||
def test_cp_out(self):
|
||||
# test copying all sorts of things out of a tahoe filesystem
|
||||
self.basedir = "cli_cp/CopyOut/cp_out"
|
||||
self.set_up_grid(num_servers=1)
|
||||
|
||||
d = self.do_setup()
|
||||
d.addCallback(lambda ign: self.do_tests())
|
||||
return d
|
||||
|
||||
Loading…
Reference in New Issue
Block a user