ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-03-23 04:05:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

dirnode manifest: add verifycaps, both to internal API and to webapi. This will give the manual-GC tools more to work with, so they can estimate how much space will be freed.

Browse Source
This commit is contained in:
Brian Warner 2008-11-24 14:40:46 -07:00
parent 2308440fb6
commit bc53c24003
8 changed files with 62 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/CLI.txt
View File

@ -329,6 +329,7 @@ tahoe mv tahoe:uploaded.txt fun:uploaded.txt
tahoe manifest tahoe:
tahoe manifest --storage-index tahoe:
tahoe manifest --raw tahoe:
This performs a recursive walk of the given directory, visiting every file
and directory that can be reached from that point. It then emits one line to
@ -339,7 +340,9 @@ tahoe manifest --storage-index tahoe:
If --storage-index is added, each line will instead contain the object's
storage index. This (string) value is useful to determine which share files
(on the server) are associated with this directory tree.
(on the server) are associated with this directory tree. If --raw is
provided instead, the output will be a JSON-encoded dictionary that includes
keys for storage index strings, verifycaps, and deep-stats.
tahoe stats tahoe:

3
docs/frontends/webapi.txt
View File

@ -964,11 +964,12 @@ POST $DIRURL?t=start-manifest (must add &ophandle=XYZ)
by a space.
If output=JSON is added to the queryargs, then the results will be a
JSON-formatted dictionary with five keys:
JSON-formatted dictionary with six keys:
finished (bool): if False then you must reload the page until True
origin_si (base32 str): the storage index of the starting point
manifest: list of (path, cap) tuples, where path is a list of strings.
verifycaps: list of (printable) verify cap strings
storage-index: list of (base32) storage index strings
stats: a dictionary with the same keys as the t=deep-stats command
(described below)

5
src/allmydata/dirnode.py
View File

@ -641,17 +641,22 @@ class ManifestWalker(DeepStats):
DeepStats.__init__(self, origin)
self.manifest = []
self.storage_index_strings = set()
self.verifycaps = set()
def add_node(self, node, path):
self.manifest.append( (tuple(path), node.get_uri()) )
si = node.get_storage_index()
if si:
self.storage_index_strings.add(base32.b2a(si))
v = node.get_verifier()
if v:
self.verifycaps.add(v.to_string())
return DeepStats.add_node(self, node, path)
def get_results(self):
stats = DeepStats.get_results(self)
return {"manifest": self.manifest,
"verifycaps": self.verifycaps,
"storage-index": self.storage_index_strings,
"stats": stats,
}

9
src/allmydata/interfaces.py
View File

@ -863,16 +863,19 @@ class IDirectoryNode(IMutableFilesystemNode):
I also compute deep-stats as described below.
I return a Monitor. The Monitor's results will be a dictionary with
three elements:
four elements:
res['manifest']: a list of (path, cap) tuples for all nodes
(directories and files) reachable from this one.
'path' will be a tuple of unicode strings. The
origin dirnode will be represented by an empty path
tuple.
res['verifycaps']: a list of (printable) verifycap strings, one for
each reachable non-LIT node. This is a set:
it will contain no duplicates.
res['storage-index']: a list of (base32) storage index strings,
one for each reachable node. This is a set:
duplicates have been removed.
one for each reachable non-LIT node. This is
a set: it will contain no duplicates.
res['stats']: a dictionary, the same that is generated by
start_deep_stats() below.

18
src/allmydata/test/test_dirnode.py
View File

@ -9,7 +9,7 @@ from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \
ExistingChildError, NoSuchChildError, \
IDeepCheckResults, IDeepCheckAndRepairResults
from allmydata.util import hashutil
from allmydata.util import hashutil, base32
from allmydata.monitor import Monitor
from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
FakeDirectoryNode, create_chk_filenode, ErrorMixin
@ -281,6 +281,8 @@ class Dirnode(unittest.TestCase,
def test_create(self):
self.expected_manifest = []
self.expected_verifycaps = set()
self.expected_storage_indexes = set()
d = self.client.create_empty_dirnode()
def _then(n):
@ -294,8 +296,11 @@ class Dirnode(unittest.TestCase,
u_v = n.get_verifier().to_string()
self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
self.expected_manifest.append( ((), u) )
self.expected_verifycaps.add(u_v)
si = n.get_storage_index()
self.expected_storage_indexes.add(base32.b2a(si))
expected_si = n._uri._filenode_uri.storage_index
self.failUnlessEqual(n.get_storage_index(), expected_si)
self.failUnlessEqual(si, expected_si)
d = n.list()
d.addCallback(lambda res: self.failUnlessEqual(res, {}))
@ -306,6 +311,8 @@ class Dirnode(unittest.TestCase,
m = Marker(fake_file_uri)
ffu_v = m.get_verifier().to_string()
self.expected_manifest.append( ((u"child",) , m.get_uri()) )
self.expected_verifycaps.add(ffu_v)
self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri))
d.addCallback(lambda res:
self.shouldFail(ExistingChildError, "set_uri-no",
@ -326,6 +333,9 @@ class Dirnode(unittest.TestCase,
new_v = subdir.get_verifier().to_string()
assert isinstance(new_v, str)
self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
self.expected_verifycaps.add(new_v)
si = subdir.get_storage_index()
self.expected_storage_indexes.add(base32.b2a(si))
d.addCallback(_created)
d.addCallback(lambda res:
@ -372,6 +382,10 @@ class Dirnode(unittest.TestCase,
sorted(self.expected_manifest))
stats = res["stats"]
_check_deepstats(stats)
self.failUnlessEqual(self.expected_verifycaps,
res["verifycaps"])
self.failUnlessEqual(self.expected_storage_indexes,
res["storage-index"])
d.addCallback(_check_manifest)
def _add_subsubdir(res):

23
src/allmydata/test/test_system.py
View File

@ -2423,6 +2423,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
"--node-directory", basedir,
"--storage-index", self.root_uri]))
def _check2((out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
@ -2430,6 +2431,28 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
d.addCallback(_check2)
d.addCallback(lambda res:
self._run_cli(["manifest",
"--node-directory", basedir,
"--raw", self.root_uri]))
def _check2r((out,err)):
self.failUnlessEqual(err, "")
data = simplejson.loads(out)
sis = data["storage-index"]
self.failUnlessEqual(len(sis), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in sis)
self.failUnless(base32.b2a(self.mutable.get_storage_index()) in sis)
self.failUnless(base32.b2a(self.large.get_storage_index()) in sis)
self.failUnlessEqual(data["stats"]["count-files"], 4)
self.failUnlessEqual(data["origin"],
base32.b2a(self.root.get_storage_index()))
verifycaps = data["verifycaps"]
self.failUnlessEqual(len(verifycaps), 3)
self.failUnless(self.root.get_verifier().to_string() in verifycaps)
self.failUnless(self.mutable.get_verifier().to_string() in verifycaps)
self.failUnless(self.large.get_verifier().to_string() in verifycaps)
d.addCallback(_check2r)
d.addCallback(lambda res:
self._run_cli(["stats",
"--node-directory", basedir,

1
src/allmydata/test/test_web.py
View File

@ -950,6 +950,7 @@ class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
self.failUnless("finished" in res)
self.failUnless("origin" in res)
self.failUnless("storage-index" in res)
self.failUnless("verifycaps" in res)
self.failUnless("stats" in res)
d.addCallback(_got_json)
return d

8
src/allmydata/web/directory.py
View File

@ -731,9 +731,11 @@ class ManifestResults(rend.Page, ReloadMixin):
def json(self, ctx):
inevow.IRequest(ctx).setHeader("content-type", "text/plain")
m = self.monitor
status = {"manifest": m.get_status()["manifest"],
"storage-index": list(m.get_status()["storage-index"]),
"stats": m.get_status()["stats"],
s = m.get_status()
status = {"manifest": s["manifest"],
"verifycaps": list(s["verifycaps"]),
"storage-index": list(s["storage-index"]),
"stats": s["stats"],
"finished": m.is_finished(),
"origin": base32.b2a(m.origin_si),
}