import time from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer from allmydata import uri, dirnode from allmydata.client import Client from allmydata.immutable import upload from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ ExistingChildError, NoSuchChildError, NotDeepImmutableError, \ IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError from allmydata.mutable.filenode import MutableFileNode from allmydata.mutable.common import UncoordinatedWriteError from allmydata.util import hashutil, base32 from allmydata.monitor import Monitor from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \ ErrorMixin from allmydata.test.no_network import GridTestMixin from allmydata.unknown import UnknownNode from allmydata.nodemaker import NodeMaker from base64 import b32decode import common_util as testutil class Dirnode(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin): timeout = 240 # It takes longer than 120 seconds on Francois's arm box. def test_basic(self): self.basedir = "dirnode/Dirnode/test_basic" self.set_up_grid() c = self.g.clients[0] d = c.create_dirnode() def _done(res): self.failUnless(isinstance(res, dirnode.DirectoryNode)) rep = str(res) self.failUnless("RW-MUT" in rep) d.addCallback(_done) return d def test_initial_children(self): self.basedir = "dirnode/Dirnode/test_initial_children" self.set_up_grid() c = self.g.clients[0] nm = c.nodemaker setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" one_uri = "URI:LIT:n5xgk" # LIT for "one" kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), } d = c.create_dirnode(kids) def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) rep = str(dn) self.failUnless("RW-MUT" in rep) return dn.list() d.addCallback(_created) def _check_kids(children): self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] self.failUnlessEqual(one_node.get_size(), 3) self.failUnlessEqual(two_node.get_size(), 14861) self.failUnless(isinstance(one_metadata, dict), one_metadata) self.failUnlessEqual(two_metadata["metakey"], "metavalue") d.addCallback(_check_kids) d.addCallback(lambda ign: nm.create_new_mutable_directory(kids)) d.addCallback(lambda dn: dn.list()) d.addCallback(_check_kids) future_writecap = "x-tahoe-crazy://I_am_from_the_future." future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." future_node = UnknownNode(future_writecap, future_readcap) bad_kids1 = {u"one": (future_node, {})} d.addCallback(lambda ign: self.shouldFail(AssertionError, "bad_kids1", "does not accept UnknownNode", nm.create_new_mutable_directory, bad_kids1)) bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} d.addCallback(lambda ign: self.shouldFail(AssertionError, "bad_kids2", "requires metadata to be a dict", nm.create_new_mutable_directory, bad_kids2)) return d def test_immutable(self): self.basedir = "dirnode/Dirnode/test_immutable" self.set_up_grid() c = self.g.clients[0] nm = c.nodemaker setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" one_uri = "URI:LIT:n5xgk" # LIT for "one" mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), } d = c.create_immutable_dirnode(kids) def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() self.failUnlessIn("CHK", cap.to_string()) self.cap = cap return dn.list() d.addCallback(_created) def _check_kids(children): self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] self.failUnlessEqual(one_node.get_size(), 3) self.failUnlessEqual(two_node.get_size(), 14861) self.failUnless(isinstance(one_metadata, dict), one_metadata) self.failUnlessEqual(two_metadata["metakey"], "metavalue") d.addCallback(_check_kids) d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) d.addCallback(lambda dn: dn.list()) d.addCallback(_check_kids) future_writecap = "x-tahoe-crazy://I_am_from_the_future." future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." future_node = UnknownNode(future_writecap, future_readcap) bad_kids1 = {u"one": (future_node, {})} d.addCallback(lambda ign: self.shouldFail(AssertionError, "bad_kids1", "does not accept UnknownNode", c.create_immutable_dirnode, bad_kids1)) bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} d.addCallback(lambda ign: self.shouldFail(AssertionError, "bad_kids2", "requires metadata to be a dict", c.create_immutable_dirnode, bad_kids2)) bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})} d.addCallback(lambda ign: self.shouldFail(NotDeepImmutableError, "bad_kids3", "is not immutable", c.create_immutable_dirnode, bad_kids3)) bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})} d.addCallback(lambda ign: self.shouldFail(NotDeepImmutableError, "bad_kids4", "is not immutable", c.create_immutable_dirnode, bad_kids4)) d.addCallback(lambda ign: c.create_immutable_dirnode({})) def _created_empty(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() self.failUnlessIn("LIT", cap.to_string()) self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:") self.cap = cap return dn.list() d.addCallback(_created_empty) d.addCallback(lambda kids: self.failUnlessEqual(kids, {})) smallkids = {u"o": (nm.create_from_cap(one_uri), {})} d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids)) def _created_small(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() self.failUnlessIn("LIT", cap.to_string()) self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy") self.cap = cap return dn.list() d.addCallback(_created_small) d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"])) # now test n.create_subdirectory(mutable=False) d.addCallback(lambda ign: c.create_dirnode()) def _made_parent(n): d = n.create_subdirectory(u"subdir", kids, mutable=False) d.addCallback(lambda sd: sd.list()) d.addCallback(_check_kids) d.addCallback(lambda ign: n.list()) d.addCallback(lambda children: self.failUnlessEqual(children.keys(), [u"subdir"])) d.addCallback(lambda ign: n.get(u"subdir")) d.addCallback(lambda sd: sd.list()) d.addCallback(_check_kids) d.addCallback(lambda ign: n.get(u"subdir")) d.addCallback(lambda sd: self.failIf(sd.is_mutable())) bad_kids = {u"one": (nm.create_from_cap(mut_writecap), {})} d.addCallback(lambda ign: self.shouldFail(NotDeepImmutableError, "YZ", "is not immutable", n.create_subdirectory, u"sub2", bad_kids, mutable=False)) return d d.addCallback(_made_parent) return d def test_check(self): self.basedir = "dirnode/Dirnode/test_check" self.set_up_grid() c = self.g.clients[0] d = c.create_dirnode() d.addCallback(lambda dn: dn.check(Monitor())) def _done(res): self.failUnless(res.is_healthy()) d.addCallback(_done) return d def _test_deepcheck_create(self): # create a small tree with a loop, and some non-directories # root/ # root/subdir/ # root/subdir/file1 # root/subdir/link -> root # root/rodir c = self.g.clients[0] d = c.create_dirnode() def _created_root(rootnode): self._rootnode = rootnode return rootnode.create_subdirectory(u"subdir") d.addCallback(_created_root) def _created_subdir(subdir): self._subdir = subdir d = subdir.add_file(u"file1", upload.Data("data"*100, None)) d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode)) d.addCallback(lambda res: c.create_dirnode()) d.addCallback(lambda dn: self._rootnode.set_uri(u"rodir", dn.get_uri(), dn.get_readonly_uri())) return d d.addCallback(_created_subdir) def _done(res): return self._rootnode d.addCallback(_done) return d def test_deepcheck(self): self.basedir = "dirnode/Dirnode/test_deepcheck" self.set_up_grid() d = self._test_deepcheck_create() d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done()) def _check_results(r): self.failUnless(IDeepCheckResults.providedBy(r)) c = r.get_counters() self.failUnlessEqual(c, {"count-objects-checked": 4, "count-objects-healthy": 4, "count-objects-unhealthy": 0, "count-objects-unrecoverable": 0, "count-corrupt-shares": 0, }) self.failIf(r.get_corrupt_shares()) self.failUnlessEqual(len(r.get_all_results()), 4) d.addCallback(_check_results) return d def test_deepcheck_and_repair(self): self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair" self.set_up_grid() d = self._test_deepcheck_create() d.addCallback(lambda rootnode: rootnode.start_deep_check_and_repair().when_done()) def _check_results(r): self.failUnless(IDeepCheckAndRepairResults.providedBy(r)) c = r.get_counters() self.failUnlessEqual(c, {"count-objects-checked": 4, "count-objects-healthy-pre-repair": 4, "count-objects-unhealthy-pre-repair": 0, "count-objects-unrecoverable-pre-repair": 0, "count-corrupt-shares-pre-repair": 0, "count-objects-healthy-post-repair": 4, "count-objects-unhealthy-post-repair": 0, "count-objects-unrecoverable-post-repair": 0, "count-corrupt-shares-post-repair": 0, "count-repairs-attempted": 0, "count-repairs-successful": 0, "count-repairs-unsuccessful": 0, }) self.failIf(r.get_corrupt_shares()) self.failIf(r.get_remaining_corrupt_shares()) self.failUnlessEqual(len(r.get_all_results()), 4) d.addCallback(_check_results) return d def _mark_file_bad(self, rootnode): self.delete_shares_numbered(rootnode.get_uri(), [0]) return rootnode def test_deepcheck_problems(self): self.basedir = "dirnode/Dirnode/test_deepcheck_problems" self.set_up_grid() d = self._test_deepcheck_create() d.addCallback(lambda rootnode: self._mark_file_bad(rootnode)) d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done()) def _check_results(r): c = r.get_counters() self.failUnlessEqual(c, {"count-objects-checked": 4, "count-objects-healthy": 3, "count-objects-unhealthy": 1, "count-objects-unrecoverable": 0, "count-corrupt-shares": 0, }) #self.failUnlessEqual(len(r.get_problems()), 1) # TODO d.addCallback(_check_results) return d def test_readonly(self): self.basedir = "dirnode/Dirnode/test_readonly" self.set_up_grid() c = self.g.clients[0] nm = c.nodemaker filecap = make_chk_file_uri(1234) filenode = nm.create_from_cap(filecap) uploadable = upload.Data("some data", convergence="some convergence string") d = c.create_dirnode() def _created(rw_dn): d2 = rw_dn.set_uri(u"child", filecap, filecap) d2.addCallback(lambda res: rw_dn) return d2 d.addCallback(_created) def _ready(rw_dn): ro_uri = rw_dn.get_readonly_uri() ro_dn = c.create_node_from_uri(ro_uri) self.failUnless(ro_dn.is_readonly()) self.failUnless(ro_dn.is_mutable()) self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, ro_dn.set_uri, u"newchild", filecap, filecap) self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, ro_dn.set_node, u"newchild", filenode) self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None, ro_dn.set_nodes, { u"newchild": (filenode, None) }) self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, ro_dn.add_file, u"newchild", uploadable) self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, ro_dn.delete, u"child") self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, ro_dn.create_subdirectory, u"newchild") self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None, ro_dn.set_metadata_for, u"child", {}) self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, ro_dn.move_child_to, u"child", rw_dn) self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, rw_dn.move_child_to, u"child", ro_dn) return ro_dn.list() d.addCallback(_ready) def _listed(children): self.failUnless(u"child" in children) d.addCallback(_listed) return d def failUnlessGreaterThan(self, a, b): self.failUnless(a > b, "%r should be > %r" % (a, b)) def failUnlessGreaterOrEqualThan(self, a, b): self.failUnless(a >= b, "%r should be >= %r" % (a, b)) def test_create(self): self.basedir = "dirnode/Dirnode/test_create" self.set_up_grid() c = self.g.clients[0] self.expected_manifest = [] self.expected_verifycaps = set() self.expected_storage_indexes = set() d = c.create_dirnode() def _then(n): # / self.rootnode = n self.failUnless(n.is_mutable()) u = n.get_uri() self.failUnless(u) self.failUnless(u.startswith("URI:DIR2:"), u) u_ro = n.get_readonly_uri() self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro) u_v = n.get_verify_cap().to_string() self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v) u_r = n.get_repair_cap().to_string() self.failUnlessEqual(u_r, u) self.expected_manifest.append( ((), u) ) self.expected_verifycaps.add(u_v) si = n.get_storage_index() self.expected_storage_indexes.add(base32.b2a(si)) expected_si = n._uri._filenode_uri.storage_index self.failUnlessEqual(si, expected_si) d = n.list() d.addCallback(lambda res: self.failUnlessEqual(res, {})) d.addCallback(lambda res: n.has_child(u"missing")) d.addCallback(lambda res: self.failIf(res)) fake_file_uri = make_mutable_file_uri() other_file_uri = make_mutable_file_uri() m = c.nodemaker.create_from_cap(fake_file_uri) ffu_v = m.get_verify_cap().to_string() self.expected_manifest.append( ((u"child",) , m.get_uri()) ) self.expected_verifycaps.add(ffu_v) self.expected_storage_indexes.add(base32.b2a(m.get_storage_index())) d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri, fake_file_uri)) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_uri-no", "child 'child' already exists", n.set_uri, u"child", other_file_uri, other_file_uri, overwrite=False)) # / # /child = mutable d.addCallback(lambda res: n.create_subdirectory(u"subdir")) # / # /child = mutable # /subdir = directory def _created(subdir): self.failUnless(isinstance(subdir, dirnode.DirectoryNode)) self.subdir = subdir new_v = subdir.get_verify_cap().to_string() assert isinstance(new_v, str) self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) ) self.expected_verifycaps.add(new_v) si = subdir.get_storage_index() self.expected_storage_indexes.add(base32.b2a(si)) d.addCallback(_created) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "mkdir-no", "child 'subdir' already exists", n.create_subdirectory, u"subdir", overwrite=False)) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), sorted([u"child", u"subdir"]))) d.addCallback(lambda res: n.start_deep_stats().when_done()) def _check_deepstats(stats): self.failUnless(isinstance(stats, dict)) expected = {"count-immutable-files": 0, "count-mutable-files": 1, "count-literal-files": 0, "count-files": 1, "count-directories": 2, "size-immutable-files": 0, "size-literal-files": 0, #"size-directories": 616, # varies #"largest-directory": 616, "largest-directory-children": 2, "largest-immutable-file": 0, } for k,v in expected.iteritems(): self.failUnlessEqual(stats[k], v, "stats[%s] was %s, not %s" % (k, stats[k], v)) self.failUnless(stats["size-directories"] > 500, stats["size-directories"]) self.failUnless(stats["largest-directory"] > 500, stats["largest-directory"]) self.failUnlessEqual(stats["size-files-histogram"], []) d.addCallback(_check_deepstats) d.addCallback(lambda res: n.build_manifest().when_done()) def _check_manifest(res): manifest = res["manifest"] self.failUnlessEqual(sorted(manifest), sorted(self.expected_manifest)) stats = res["stats"] _check_deepstats(stats) self.failUnlessEqual(self.expected_verifycaps, res["verifycaps"]) self.failUnlessEqual(self.expected_storage_indexes, res["storage-index"]) d.addCallback(_check_manifest) def _add_subsubdir(res): return self.subdir.create_subdirectory(u"subsubdir") d.addCallback(_add_subsubdir) # / # /child = mutable # /subdir = directory # /subdir/subsubdir = directory d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir")) d.addCallback(lambda subsubdir: self.failUnless(isinstance(subsubdir, dirnode.DirectoryNode))) d.addCallback(lambda res: n.get_child_at_path(u"")) d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(), n.get_uri())) d.addCallback(lambda res: n.get_metadata_for(u"child")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) d.addCallback(lambda res: self.shouldFail(NoSuchChildError, "gcamap-no", "nope", n.get_child_and_metadata_at_path, u"subdir/nope")) d.addCallback(lambda res: n.get_child_and_metadata_at_path(u"")) def _check_child_and_metadata1(res): child, metadata = res self.failUnless(isinstance(child, dirnode.DirectoryNode)) # edge-metadata needs at least one path segment self.failUnlessEqual(sorted(metadata.keys()), []) d.addCallback(_check_child_and_metadata1) d.addCallback(lambda res: n.get_child_and_metadata_at_path(u"child")) def _check_child_and_metadata2(res): child, metadata = res self.failUnlessEqual(child.get_uri(), fake_file_uri) self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])) d.addCallback(_check_child_and_metadata2) d.addCallback(lambda res: n.get_child_and_metadata_at_path(u"subdir/subsubdir")) def _check_child_and_metadata3(res): child, metadata = res self.failUnless(isinstance(child, dirnode.DirectoryNode)) self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])) d.addCallback(_check_child_and_metadata3) # set_uri + metadata # it should be possible to add a child without any metadata d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri, fake_file_uri, {})) d.addCallback(lambda res: n.get_metadata_for(u"c2")) d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe'])) # You can't override the link timestamps. d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri, fake_file_uri, { 'tahoe': {'linkcrtime': "bogus"}})) d.addCallback(lambda res: n.get_metadata_for(u"c2")) def _has_good_linkcrtime(metadata): self.failUnless(metadata.has_key('tahoe')) self.failUnless(metadata['tahoe'].has_key('linkcrtime')) self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus') d.addCallback(_has_good_linkcrtime) # if we don't set any defaults, the child should get timestamps d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri, fake_file_uri)) d.addCallback(lambda res: n.get_metadata_for(u"c3")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) # or we can add specific metadata at set_uri() time, which # overrides the timestamps d.addCallback(lambda res: n.set_uri(u"c4", fake_file_uri, fake_file_uri, {"key": "value"})) d.addCallback(lambda res: n.get_metadata_for(u"c4")) d.addCallback(lambda metadata: self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"c2")) d.addCallback(lambda res: n.delete(u"c3")) d.addCallback(lambda res: n.delete(u"c4")) # set_node + metadata # it should be possible to add a child without any metadata d.addCallback(lambda res: n.set_node(u"d2", n, {})) d.addCallback(lambda res: c.create_dirnode()) d.addCallback(lambda n2: self.shouldFail(ExistingChildError, "set_node-no", "child 'd2' already exists", n.set_node, u"d2", n2, overwrite=False)) d.addCallback(lambda res: n.get_metadata_for(u"d2")) d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe'])) # if we don't set any defaults, the child should get timestamps d.addCallback(lambda res: n.set_node(u"d3", n)) d.addCallback(lambda res: n.get_metadata_for(u"d3")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) # or we can add specific metadata at set_node() time, which # overrides the timestamps d.addCallback(lambda res: n.set_node(u"d4", n, {"key": "value"})) d.addCallback(lambda res: n.get_metadata_for(u"d4")) d.addCallback(lambda metadata: self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"d2")) d.addCallback(lambda res: n.delete(u"d3")) d.addCallback(lambda res: n.delete(u"d4")) # metadata through set_children() d.addCallback(lambda res: n.set_children({ u"e1": (fake_file_uri, fake_file_uri), u"e2": (fake_file_uri, fake_file_uri, {}), u"e3": (fake_file_uri, fake_file_uri, {"key": "value"}), })) d.addCallback(lambda n2: self.failUnlessIdentical(n2, n)) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_children-no", "child 'e1' already exists", n.set_children, { u"e1": (other_file_uri, other_file_uri), u"new": (other_file_uri, other_file_uri), }, overwrite=False)) # and 'new' should not have been created d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failIf(u"new" in children)) d.addCallback(lambda res: n.get_metadata_for(u"e1")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) d.addCallback(lambda res: n.get_metadata_for(u"e2")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(['tahoe']))) d.addCallback(lambda res: n.get_metadata_for(u"e3")) d.addCallback(lambda metadata: self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"e1")) d.addCallback(lambda res: n.delete(u"e2")) d.addCallback(lambda res: n.delete(u"e3")) # metadata through set_nodes() d.addCallback(lambda res: n.set_nodes({ u"f1": (n, None), u"f2": (n, {}), u"f3": (n, {"key": "value"}), })) d.addCallback(lambda n2: self.failUnlessIdentical(n2, n)) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_nodes-no", "child 'f1' already exists", n.set_nodes, { u"f1": (n, None), u"new": (n, None), }, overwrite=False)) # and 'new' should not have been created d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failIf(u"new" in children)) d.addCallback(lambda res: n.get_metadata_for(u"f1")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) d.addCallback(lambda res: n.get_metadata_for(u"f2")) d.addCallback( lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(['tahoe']))) d.addCallback(lambda res: n.get_metadata_for(u"f3")) d.addCallback(lambda metadata: self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"f1")) d.addCallback(lambda res: n.delete(u"f2")) d.addCallback(lambda res: n.delete(u"f3")) d.addCallback(lambda res: n.set_metadata_for(u"child", {"tags": ["web2.0-compatible"]})) d.addCallback(lambda n1: n1.get_metadata_for(u"child")) d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {"tags": ["web2.0-compatible"]})) def _start(res): self._start_timestamp = time.time() d.addCallback(_start) # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all # floats to hundredeths (it uses str(num) instead of repr(num)). # simplejson-1.7.3 does not have this bug. To prevent this bug # from causing the test to fail, stall for more than a few # hundrededths of a second. d.addCallback(self.stall, 0.1) d.addCallback(lambda res: n.add_file(u"timestamps", upload.Data("stamp me", convergence="some convergence string"))) d.addCallback(self.stall, 0.1) def _stop(res): self._stop_timestamp = time.time() d.addCallback(_stop) d.addCallback(lambda res: n.get_metadata_for(u"timestamps")) def _check_timestamp1(metadata): self.failUnless("ctime" in metadata) self.failUnless("mtime" in metadata) self.failUnlessGreaterOrEqualThan(metadata["ctime"], self._start_timestamp) self.failUnlessGreaterOrEqualThan(self._stop_timestamp, metadata["ctime"]) self.failUnlessGreaterOrEqualThan(metadata["mtime"], self._start_timestamp) self.failUnlessGreaterOrEqualThan(self._stop_timestamp, metadata["mtime"]) # Our current timestamp rules say that replacing an existing # child should preserve the 'ctime' but update the mtime self._old_ctime = metadata["ctime"] self._old_mtime = metadata["mtime"] d.addCallback(_check_timestamp1) d.addCallback(self.stall, 2.0) # accomodate low-res timestamps d.addCallback(lambda res: n.set_node(u"timestamps", n)) d.addCallback(lambda res: n.get_metadata_for(u"timestamps")) def _check_timestamp2(metadata): self.failUnlessEqual(metadata["ctime"], self._old_ctime, "%s != %s" % (metadata["ctime"], self._old_ctime)) self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime) return n.delete(u"timestamps") d.addCallback(_check_timestamp2) # also make sure we can add/update timestamps on a # previously-existing child that didn't have any, since there are # a lot of 0.7.0-generated edges around out there d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {})) d.addCallback(lambda res: n.set_node(u"no_timestamps", n)) d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) d.addCallback(lambda res: n.delete(u"no_timestamps")) d.addCallback(lambda res: n.delete(u"subdir")) d.addCallback(lambda old_child: self.failUnlessEqual(old_child.get_uri(), self.subdir.get_uri())) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), sorted([u"child"]))) uploadable1 = upload.Data("some data", convergence="converge") d.addCallback(lambda res: n.add_file(u"newfile", uploadable1)) d.addCallback(lambda newnode: self.failUnless(IImmutableFileNode.providedBy(newnode))) uploadable2 = upload.Data("some data", convergence="stuff") d.addCallback(lambda res: self.shouldFail(ExistingChildError, "add_file-no", "child 'newfile' already exists", n.add_file, u"newfile", uploadable2, overwrite=False)) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), sorted([u"child", u"newfile"]))) d.addCallback(lambda res: n.get_metadata_for(u"newfile")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"]))) uploadable3 = upload.Data("some data", convergence="converge") d.addCallback(lambda res: n.add_file(u"newfile-metadata", uploadable3, {"key": "value"})) d.addCallback(lambda newnode: self.failUnless(IImmutableFileNode.providedBy(newnode))) d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata")) d.addCallback(lambda metadata: self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"newfile-metadata")) d.addCallback(lambda res: n.create_subdirectory(u"subdir2")) def _created2(subdir2): self.subdir2 = subdir2 # put something in the way, to make sure it gets overwritten return subdir2.add_file(u"child", upload.Data("overwrite me", "converge")) d.addCallback(_created2) d.addCallback(lambda res: n.move_child_to(u"child", self.subdir2)) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), sorted([u"newfile", u"subdir2"]))) d.addCallback(lambda res: self.subdir2.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), sorted([u"child"]))) d.addCallback(lambda res: self.subdir2.get(u"child")) d.addCallback(lambda child: self.failUnlessEqual(child.get_uri(), fake_file_uri)) # move it back, using new_child_name= d.addCallback(lambda res: self.subdir2.move_child_to(u"child", n, u"newchild")) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), sorted([u"newchild", u"newfile", u"subdir2"]))) d.addCallback(lambda res: self.subdir2.list()) d.addCallback(lambda children: self.failUnlessEqual(sorted(children.keys()), [])) # now make sure that we honor overwrite=False d.addCallback(lambda res: self.subdir2.set_uri(u"newchild", other_file_uri, other_file_uri)) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "move_child_to-no", "child 'newchild' already exists", n.move_child_to, u"newchild", self.subdir2, overwrite=False)) d.addCallback(lambda res: self.subdir2.get(u"newchild")) d.addCallback(lambda child: self.failUnlessEqual(child.get_uri(), other_file_uri)) return d d.addCallback(_then) d.addErrback(self.explain_error) return d def test_create_subdirectory(self): self.basedir = "dirnode/Dirnode/test_create_subdirectory" self.set_up_grid() c = self.g.clients[0] nm = c.nodemaker d = c.create_dirnode() def _then(n): # / self.rootnode = n fake_file_uri = make_mutable_file_uri() other_file_uri = make_mutable_file_uri() md = {"metakey": "metavalue"} kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}), u"kid2": (nm.create_from_cap(other_file_uri), md), } d = n.create_subdirectory(u"subdir", kids) def _check(sub): d = n.get_child_at_path(u"subdir") d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(), sub.get_uri())) d.addCallback(lambda ign: sub.list()) return d d.addCallback(_check) def _check_kids(kids2): self.failUnlessEqual(sorted(kids.keys()), sorted(kids2.keys())) self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue") d.addCallback(_check_kids) return d d.addCallback(_then) return d class MinimalFakeMutableFile: def get_writekey(self): return "writekey" class Packing(unittest.TestCase): # This is a base32-encoded representation of the directory tree # root/file1 # root/file2 # root/file3 # as represented after being fed to _pack_contents. # We have it here so we can decode it, feed it to # _unpack_contents, and verify that _unpack_contents # works correctly. known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY===" def test_unpack_and_pack_behavior(self): known_tree = b32decode(self.known_tree) nodemaker = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10}, None) writecap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" filenode = nodemaker.create_from_cap(writecap) node = dirnode.DirectoryNode(filenode, nodemaker, None) children = node._unpack_contents(known_tree) self._check_children(children) packed_children = node._pack_contents(children) children = node._unpack_contents(packed_children) self._check_children(children) def _check_children(self, children): # Are all the expected child nodes there? self.failUnless(children.has_key(u'file1')) self.failUnless(children.has_key(u'file2')) self.failUnless(children.has_key(u'file3')) # Are the metadata for child 3 right? file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5" file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5" file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269} self.failUnlessEqual(file3_metadata, children[u'file3'][1]) self.failUnlessEqual(file3_rocap, children[u'file3'][0].get_readonly_uri()) self.failUnlessEqual(file3_rwcap, children[u'file3'][0].get_uri()) # Are the metadata for child 2 right? file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4" file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4" file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218} self.failUnlessEqual(file2_metadata, children[u'file2'][1]) self.failUnlessEqual(file2_rocap, children[u'file2'][0].get_readonly_uri()) self.failUnlessEqual(file2_rwcap, children[u'file2'][0].get_uri()) # Are the metadata for child 1 right? file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10" file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10" file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661} self.failUnlessEqual(file1_metadata, children[u'file1'][1]) self.failUnlessEqual(file1_rocap, children[u'file1'][0].get_readonly_uri()) self.failUnlessEqual(file1_rwcap, children[u'file1'][0].get_uri()) def _make_kids(self, nm, which): caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861", "lit": "URI:LIT:n5xgk", # LIT for "one" "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq", "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q", "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq", "dirread": "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq", } kids = {} for name in which: kids[unicode(name)] = (nm.create_from_cap(caps[name]), {}) return kids def test_deep_immutable(self): nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10}, None) fn = MinimalFakeMutableFile() kids = self._make_kids(nm, ["imm", "lit", "write", "read", "dirwrite", "dirread"]) packed = dirnode.pack_children(fn, kids, deep_immutable=False) self.failUnlessIn("lit", packed) kids = self._make_kids(nm, ["imm", "lit"]) packed = dirnode.pack_children(fn, kids, deep_immutable=True) self.failUnlessIn("lit", packed) kids = self._make_kids(nm, ["imm", "lit", "write"]) self.failUnlessRaises(dirnode.MustBeDeepImmutable, dirnode.pack_children, fn, kids, deep_immutable=True) # read-only is not enough: all children must be immutable kids = self._make_kids(nm, ["imm", "lit", "read"]) self.failUnlessRaises(dirnode.MustBeDeepImmutable, dirnode.pack_children, fn, kids, deep_immutable=True) kids = self._make_kids(nm, ["imm", "lit", "dirwrite"]) self.failUnlessRaises(dirnode.MustBeDeepImmutable, dirnode.pack_children, fn, kids, deep_immutable=True) kids = self._make_kids(nm, ["imm", "lit", "dirread"]) self.failUnlessRaises(dirnode.MustBeDeepImmutable, dirnode.pack_children, fn, kids, deep_immutable=True) class FakeMutableFile: implements(IMutableFileNode) counter = 0 def __init__(self, initial_contents=""): self.data = self._get_initial_contents(initial_contents) counter = FakeMutableFile.counter FakeMutableFile.counter += 1 writekey = hashutil.ssk_writekey_hash(str(counter)) fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter)) self.uri = uri.WriteableSSKFileURI(writekey, fingerprint) def _get_initial_contents(self, contents): if isinstance(contents, str): return contents if contents is None: return "" assert callable(contents), "%s should be callable, not %s" % \ (contents, type(contents)) return contents(self) def get_cap(self): return self.uri def get_uri(self): return self.uri.to_string() def download_best_version(self): return defer.succeed(self.data) def get_writekey(self): return "writekey" def is_readonly(self): return False def is_mutable(self): return True def modify(self, modifier): self.data = modifier(self.data, None, True) return defer.succeed(None) class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None): return defer.succeed(FakeMutableFile(contents)) class FakeClient2(Client): def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, None, {"k":3,"n":10}, None) def create_node_from_uri(self, rwcap, rocap): return self.nodemaker.create_from_cap(rwcap, rocap) class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin): def setUp(self): client = FakeClient2() self.nodemaker = client.nodemaker def test_from_future(self): # create a dirnode that contains unknown URI types, and make sure we # tolerate them properly. Since dirnodes aren't allowed to add # unknown node types, we have to be tricky. d = self.nodemaker.create_new_mutable_directory() future_writecap = "x-tahoe-crazy://I_am_from_the_future." future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." future_node = UnknownNode(future_writecap, future_readcap) def _then(n): self._node = n return n.set_node(u"future", future_node) d.addCallback(_then) # we should be prohibited from adding an unknown URI to a directory, # since we don't know how to diminish the cap to a readcap (for the # dirnode's rocap slot), and we don't want to accidentally grant # write access to a holder of the dirnode's readcap. d.addCallback(lambda ign: self.shouldFail(CannotPackUnknownNodeError, "copy unknown", "cannot pack unknown node as child add", self._node.set_uri, u"add", future_writecap, future_readcap)) d.addCallback(lambda ign: self._node.list()) def _check(children): self.failUnlessEqual(len(children), 1) (fn, metadata) = children[u"future"] self.failUnless(isinstance(fn, UnknownNode), fn) self.failUnlessEqual(fn.get_uri(), future_writecap) self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) # but we *should* be allowed to copy this node, because the # UnknownNode contains all the information that was in the # original directory (readcap and writecap), so we're preserving # everything. return self._node.set_node(u"copy", fn) d.addCallback(_check) d.addCallback(lambda ign: self._node.list()) def _check2(children): self.failUnlessEqual(len(children), 2) (fn, metadata) = children[u"copy"] self.failUnless(isinstance(fn, UnknownNode), fn) self.failUnlessEqual(fn.get_uri(), future_writecap) self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) return d class DeepStats(unittest.TestCase): timeout = 240 # It takes longer than 120 seconds on Francois's arm box. def test_stats(self): ds = dirnode.DeepStats(None) ds.add("count-files") ds.add("size-immutable-files", 123) ds.histogram("size-files-histogram", 123) ds.max("largest-directory", 444) s = ds.get_results() self.failUnlessEqual(s["count-files"], 1) self.failUnlessEqual(s["size-immutable-files"], 123) self.failUnlessEqual(s["largest-directory"], 444) self.failUnlessEqual(s["count-literal-files"], 0) ds.add("count-files") ds.add("size-immutable-files", 321) ds.histogram("size-files-histogram", 321) ds.max("largest-directory", 2) s = ds.get_results() self.failUnlessEqual(s["count-files"], 2) self.failUnlessEqual(s["size-immutable-files"], 444) self.failUnlessEqual(s["largest-directory"], 444) self.failUnlessEqual(s["count-literal-files"], 0) self.failUnlessEqual(s["size-files-histogram"], [ (101, 316, 1), (317, 1000, 1) ]) ds = dirnode.DeepStats(None) for i in range(1, 1100): ds.histogram("size-files-histogram", i) ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB s = ds.get_results() self.failUnlessEqual(s["size-files-histogram"], [ (1, 3, 3), (4, 10, 7), (11, 31, 21), (32, 100, 69), (101, 316, 216), (317, 1000, 684), (1001, 3162, 99), (3162277660169L, 10000000000000L, 1), ]) class UCWEingMutableFileNode(MutableFileNode): please_ucwe_after_next_upload = False def _upload(self, new_contents, servermap): d = MutableFileNode._upload(self, new_contents, servermap) def _ucwe(res): if self.please_ucwe_after_next_upload: self.please_ucwe_after_next_upload = False raise UncoordinatedWriteError() return res d.addCallback(_ucwe) return d class UCWEingNodeMaker(NodeMaker): def _create_mutable(self, cap): n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder, self.default_encoding_parameters, self.history) return n.init_from_cap(cap) class Deleter(GridTestMixin, unittest.TestCase): timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box. def test_retry(self): # ticket #550, a dirnode.delete which experiences an # UncoordinatedWriteError will fail with an incorrect "you're # deleting something which isn't there" NoSuchChildError exception. # to trigger this, we start by creating a directory with a single # file in it. Then we create a special dirnode that uses a modified # MutableFileNode which will raise UncoordinatedWriteError once on # demand. We then call dirnode.delete, which ought to retry and # succeed. self.basedir = self.mktemp() self.set_up_grid() c0 = self.g.clients[0] d = c0.create_dirnode() small = upload.Data("Small enough for a LIT", None) def _created_dir(dn): self.root = dn self.root_uri = dn.get_uri() return dn.add_file(u"file", small) d.addCallback(_created_dir) def _do_delete(ignored): nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder, c0.get_history(), c0.getServiceNamed("uploader"), c0.downloader, c0.download_cache_dirman, c0.get_encoding_parameters(), c0._key_generator) n = nm.create_from_cap(self.root_uri) assert n._node.please_ucwe_after_next_upload == False n._node.please_ucwe_after_next_upload = True # This should succeed, not raise an exception return n.delete(u"file") d.addCallback(_do_delete) return d class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): def test_overwrite(self): # note: This functionality could be tested without actually creating # several RSA keys. It would be faster without the GridTestMixin: use # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead # of dn.add_file, and use a special NodeMaker that creates fake # mutable files. self.basedir = "dirnode/Adder/test_overwrite" self.set_up_grid() c = self.g.clients[0] fileuri = make_chk_file_uri(1234) filenode = c.nodemaker.create_from_cap(fileuri) d = c.create_dirnode() def _create_directory_tree(root_node): # Build # root/file1 # root/file2 # root/dir1 d = root_node.add_file(u'file1', upload.Data("Important Things", None)) d.addCallback(lambda res: root_node.add_file(u'file2', upload.Data("Sekrit Codes", None))) d.addCallback(lambda res: root_node.create_subdirectory(u"dir1")) d.addCallback(lambda res: root_node) return d d.addCallback(_create_directory_tree) def _test_adder(root_node): d = root_node.set_node(u'file1', filenode) # We've overwritten file1. Let's try it with a directory d.addCallback(lambda res: root_node.create_subdirectory(u'dir2')) d.addCallback(lambda res: root_node.set_node(u'dir2', filenode)) # We try overwriting a file with a child while also specifying # overwrite=False. We should receive an ExistingChildError # when we do this. d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_node", "child 'file1' already exists", root_node.set_node, u"file1", filenode, overwrite=False)) # If we try with a directory, we should see the same thing d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_node", "child 'dir1' already exists", root_node.set_node, u'dir1', filenode, overwrite=False)) d.addCallback(lambda res: root_node.set_node(u'file1', filenode, overwrite="only-files")) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_node", "child 'dir1' already exists", root_node.set_node, u'dir1', filenode, overwrite="only-files")) return d d.addCallback(_test_adder) return d