ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-18 18:56:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

tahoe_backup.py: display warnings on errors instead of stopping the whole backup. Fix #729.

Browse Source 
This patch displays a warning to the user in two cases:
  
  1. When special files like symlinks, fifos, devices, etc. are found in the
     local source.
  
  2. If files or directories are not readables by the user running the 'tahoe
     backup' command.

In verbose mode, the number of skipped files and directories is printed at the
end of the backup.

Exit status returned by 'tahoe backup':

  - 0 everything went fine
  - 1 the backup failed
  - 2 files were skipped during the backup
This commit is contained in:
francois 2010-01-20 01:42:49 -08:00
parent 66646d9dd6
commit b03406af9d
2 changed files with 153 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/allmydata/scripts/tahoe_backup.py
View File

@ -65,9 +65,11 @@ class BackerUpper:
self.files_uploaded = 0
self.files_reused = 0
self.files_checked = 0
self.files_skipped = 0
self.directories_created = 0
self.directories_reused = 0
self.directories_checked = 0
self.directories_skipped = 0
def run(self):
options = self.options
@ -123,16 +125,25 @@ class BackerUpper:
if self.verbosity >= 1:
print >>stdout, (" %d files uploaded (%d reused), "
"%d directories created (%d reused)"
"%d files skipped, "
"%d directories created (%d reused), "
"%d directories skipped"
% (self.files_uploaded,
self.files_reused,
self.files_skipped,
self.directories_created,
self.directories_reused))
self.directories_reused,
self.directories_skipped))
if self.verbosity >= 2:
print >>stdout, (" %d files checked, %d directories checked"
% (self.files_checked,
self.directories_checked))
print >>stdout, " backup done, elapsed time: %s" % elapsed_time
# The command exits with code 2 if files or directories were skipped
if self.files_skipped or self.directories_skipped:
return 2
# done!
return 0
@ -140,13 +151,24 @@ class BackerUpper:
if self.verbosity >= 2:
print >>self.options.stdout, msg
def warn(self, msg):
print >>self.options.stderr, msg
def process(self, localpath):
# returns newdircap
self.verboseprint("processing %s" % localpath)
create_contents = {} # childname -> (type, rocap, metadata)
compare_contents = {} # childname -> rocap
for child in self.options.filter_listdir(os.listdir(localpath)):
try:
children = os.listdir(localpath)
except EnvironmentError:
self.directories_skipped += 1
self.warn("WARNING: permission denied on directory %s" % localpath)
children = []
for child in self.options.filter_listdir(children):
childpath = os.path.join(localpath, child)
child = unicode(child)
if os.path.isdir(childpath):
@ -157,12 +179,17 @@ class BackerUpper:
create_contents[child] = ("dirnode", childcap, metadata)
compare_contents[child] = childcap
elif os.path.isfile(childpath):
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
try:
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
except EnvironmentError:
self.files_skipped += 1
self.warn("WARNING: permission denied on file %s" % childpath)
else:
raise BackupProcessingError("Cannot backup child %r" % childpath)
self.files_skipped += 1
self.warn("WARNING: cannot backup special file %s" % childpath)
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
@ -245,6 +272,7 @@ class BackerUpper:
r.did_check_healthy(cr)
return False, r
# This function will raise an IOError exception when called on an unreadable file
def upload(self, childpath):
#self.verboseprint("uploading %s.." % childpath)
metadata = get_local_metadata(childpath)

122
src/allmydata/test/test_cli.py
View File

@ -1072,7 +1072,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
f.close()
def count_output(self, out):
mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), (\d+) directories created \((\d+) reused\)", out)
mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), "
"(\d)+ files skipped, "
"(\d+) directories created \((\d+) reused\), "
"(\d+) directories skipped", out)
return [int(s) for s in mo.groups()]
def count_output2(self, out):
@ -1117,13 +1120,15 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
def _check0((rc, out, err)):
self.failUnlessEqual(err, "")
self.failUnlessEqual(rc, 0)
fu, fr, dc, dr = self.count_output(out)
fu, fr, fs, dc, dr, ds = self.count_output(out)
# foo.txt, bar.txt, blah.txt
self.failUnlessEqual(fu, 3)
self.failUnlessEqual(fr, 0)
self.failUnlessEqual(fs, 0)
# empty, home, home/parent, home/parent/subdir
self.failUnlessEqual(dc, 4)
self.failUnlessEqual(dr, 0)
self.failUnlessEqual(ds, 0)
d.addCallback(_check0)
d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups"))
@ -1172,13 +1177,15 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
self.failUnlessEqual(err, "")
self.failUnlessEqual(rc, 0)
if have_bdb:
fu, fr, dc, dr = self.count_output(out)
fu, fr, fs, dc, dr, ds = self.count_output(out)
# foo.txt, bar.txt, blah.txt
self.failUnlessEqual(fu, 0)
self.failUnlessEqual(fr, 3)
self.failUnlessEqual(fs, 0)
# empty, home, home/parent, home/parent/subdir
self.failUnlessEqual(dc, 0)
self.failUnlessEqual(dr, 4)
self.failUnlessEqual(ds, 0)
d.addCallback(_check4a)
if have_bdb:
@ -1203,14 +1210,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
# re-use all of them too.
self.failUnlessEqual(err, "")
self.failUnlessEqual(rc, 0)
fu, fr, dc, dr = self.count_output(out)
fu, fr, fs, dc, dr, ds = self.count_output(out)
fchecked, dchecked = self.count_output2(out)
self.failUnlessEqual(fchecked, 3)
self.failUnlessEqual(fu, 0)
self.failUnlessEqual(fr, 3)
self.failUnlessEqual(fs, 0)
self.failUnlessEqual(dchecked, 4)
self.failUnlessEqual(dc, 0)
self.failUnlessEqual(dr, 4)
self.failUnlessEqual(ds, 0)
d.addCallback(_check4b)
d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
@ -1247,14 +1256,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
self.failUnlessEqual(err, "")
self.failUnlessEqual(rc, 0)
if have_bdb:
fu, fr, dc, dr = self.count_output(out)
fu, fr, fs, dc, dr, ds = self.count_output(out)
# new foo.txt, surprise file, subfile, empty
self.failUnlessEqual(fu, 4)
# old bar.txt
self.failUnlessEqual(fr, 1)
self.failUnlessEqual(fs, 0)
# home, parent, subdir, blah.txt, surprisedir
self.failUnlessEqual(dc, 5)
self.failUnlessEqual(dr, 0)
self.failUnlessEqual(ds, 0)
d.addCallback(_check5a)
d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
def _check6((rc, out, err)):
@ -1356,6 +1367,107 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
_check_filtering(filtered, root_listdir, ('lib.a', '_darcs', 'subdir'),
('nice_doc.lyx',))
def test_ignore_symlinks(self):
if not hasattr(os, 'symlink'):
raise unittest.SkipTest("There is no symlink on this platform.")
self.basedir = os.path.dirname(self.mktemp())
self.set_up_grid()
source = os.path.join(self.basedir, "home")
self.writeto("foo.txt", "foo")
os.symlink(os.path.join(source, "foo.txt"), os.path.join(source, "foo2.txt"))
d = self.do_cli("create-alias", "tahoe")
d.addCallback(lambda res: self.do_cli("backup", "--verbose", source, "tahoe:test"))
def _check((rc, out, err)):
self.failUnlessEqual(rc, 2)
self.failUnlessEqual(err, "WARNING: cannot backup special file %s\n" % os.path.join(source, "foo2.txt"))
fu, fr, fs, dc, dr, ds = self.count_output(out)
# foo.txt
self.failUnlessEqual(fu, 1)
self.failUnlessEqual(fr, 0)
# foo2.txt
self.failUnlessEqual(fs, 1)
# home
self.failUnlessEqual(dc, 1)
self.failUnlessEqual(dr, 0)
self.failUnlessEqual(ds, 0)
d.addCallback(_check)
return d
def test_ignore_unreadable_file(self):
self.basedir = os.path.dirname(self.mktemp())
self.set_up_grid()
source = os.path.join(self.basedir, "home")
self.writeto("foo.txt", "foo")
os.chmod(os.path.join(source, "foo.txt"), 0000)
d = self.do_cli("create-alias", "tahoe")
d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test"))
def _check((rc, out, err)):
self.failUnlessEqual(rc, 2)
self.failUnlessEqual(err, "WARNING: permission denied on file %s\n" % os.path.join(source, "foo.txt"))
fu, fr, fs, dc, dr, ds = self.count_output(out)
self.failUnlessEqual(fu, 0)
self.failUnlessEqual(fr, 0)
# foo.txt
self.failUnlessEqual(fs, 1)
# home
self.failUnlessEqual(dc, 1)
self.failUnlessEqual(dr, 0)
self.failUnlessEqual(ds, 0)
d.addCallback(_check)
# This is necessary for the temp files to be correctly removed
def _cleanup(self):
os.chmod(os.path.join(source, "foo.txt"), 0644)
d.addCallback(_cleanup)
d.addErrback(_cleanup)
return d
def test_ignore_unreadable_directory(self):
self.basedir = os.path.dirname(self.mktemp())
self.set_up_grid()
source = os.path.join(self.basedir, "home")
os.mkdir(source)
os.mkdir(os.path.join(source, "test"))
os.chmod(os.path.join(source, "test"), 0000)
d = self.do_cli("create-alias", "tahoe")
d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test"))
def _check((rc, out, err)):
self.failUnlessEqual(rc, 2)
self.failUnlessEqual(err, "WARNING: permission denied on directory %s\n" % os.path.join(source, "test"))
fu, fr, fs, dc, dr, ds = self.count_output(out)
self.failUnlessEqual(fu, 0)
self.failUnlessEqual(fr, 0)
self.failUnlessEqual(fs, 0)
# home, test
self.failUnlessEqual(dc, 2)
self.failUnlessEqual(dr, 0)
# test
self.failUnlessEqual(ds, 1)
d.addCallback(_check)
# This is necessary for the temp files to be correctly removed
def _cleanup(self):
os.chmod(os.path.join(source, "test"), 0655)
d.addCallback(_cleanup)
d.addErrback(_cleanup)
return d
class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
def test_check(self):