diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index 6abbd42a3..1353f233a 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -1232,6 +1232,7 @@ POST $DIRURL?t=start-deep-stats (must add &ophandle=XYZ) count-literal-files: same, for LIT files (data contained inside the URI) count-files: sum of the above three count-directories: count of directories + count-unknown: count of unrecognized objects (perhaps from the future) size-immutable-files: total bytes for all CHK files in the set, =deep-size size-mutable-files (TODO): same, for current version of all mutable files size-literal-files: same, for LIT files diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 08d559e55..8c1dc21fb 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -20,9 +20,10 @@ from allmydata.introducer.client import IntroducerClient from allmydata.util import hashutil, base32, pollmixin, cachedir, log from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date -from allmydata.uri import LiteralFileURI +from allmydata.uri import LiteralFileURI, UnknownURI from allmydata.dirnode import NewDirectoryNode from allmydata.mutable.filenode import MutableFileNode +from allmydata.unknown import UnknownNode from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import IURI, INewDirectoryURI, IStatsProducer, \ @@ -404,11 +405,17 @@ class Client(node.Node, pollmixin.PollMixin): # dirnodes. The first takes a URI and produces a filenode or (new-style) # dirnode. The other three create brand-new filenodes/dirnodes. - def create_node_from_uri(self, u, readcap=None): + def create_node_from_uri(self, writecap, readcap=None): # this returns synchronously. + u = writecap or readcap if not u: - u = readcap + # maybe the writecap was hidden because we're in a readonly + # directory, and the future cap format doesn't have a readcap, or + # something. + return UnknownNode(writecap, readcap) u = IURI(u) + if isinstance(u, UnknownURI): + return UnknownNode(writecap, readcap) u_s = u.to_string() if u_s not in self._node_cache: if IReadonlyNewDirectoryURI.providedBy(u): @@ -427,13 +434,12 @@ class Client(node.Node, pollmixin.PollMixin): else: assert IMutableFileURI.providedBy(u), u node = MutableFileNode(self).init_from_uri(u) - self._node_cache[u_s] = node + self._node_cache[u_s] = node # note: WeakValueDictionary return self._node_cache[u_s] def create_empty_dirnode(self): - n = NewDirectoryNode(self) - d = n.create(self._generate_pubprivkeys, self.DEFAULT_MUTABLE_KEYSIZE) - d.addCallback(lambda res: n) + d = self.create_mutable_file() + d.addCallback(NewDirectoryNode.create_with_mutablefile, self) return d def create_mutable_file(self, contents="", keysize=None): diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index b8113b3aa..731431992 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -7,9 +7,11 @@ from foolscap.api import fireEventually import simplejson from allmydata.mutable.common import NotMutableError from allmydata.mutable.filenode import MutableFileNode +from allmydata.unknown import UnknownNode from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\ IURI, IFileNode, IMutableFileURI, IFilesystemNode, \ - ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable + ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable, \ + CannotPackUnknownNodeError from allmydata.check_results import DeepCheckResults, \ DeepCheckAndRepairResults from allmydata.monitor import Monitor @@ -137,6 +139,12 @@ class NewDirectoryNode: self._node.init_from_uri(self._uri.get_filenode_uri()) return self + @classmethod + def create_with_mutablefile(cls, filenode, client): + self = cls(client) + self._node = filenode + return self._filenode_created(filenode) + def create(self, keypair_generator=None, keysize=None): """ Returns a deferred that eventually fires with self once the directory @@ -226,9 +234,7 @@ class NewDirectoryNode: for name in sorted(children.keys()): child, metadata = children[name] assert isinstance(name, unicode) - assert (IFileNode.providedBy(child) - or IMutableFileNode.providedBy(child) - or IDirectoryNode.providedBy(child)), (name,child) + assert IFilesystemNode.providedBy(child), (name,child) assert isinstance(metadata, dict) rwcap = child.get_uri() # might be RO if the child is not writeable if rwcap is None: @@ -381,6 +387,13 @@ class NewDirectoryNode: precondition(isinstance(name, unicode), name) precondition(isinstance(child_uri, str), child_uri) child_node = self._create_node(child_uri, None) + if isinstance(child_node, UnknownNode): + # don't be willing to pack unknown nodes: we might accidentally + # put some write-authority into the rocap slot because we don't + # know how to diminish the URI they gave us. We don't even know + # if they gave us a readcap or a writecap. + msg = "cannot pack unknown node as child %s" % str(name) + raise CannotPackUnknownNodeError(msg) d = self.set_node(name, child_node, metadata, overwrite) d.addCallback(lambda res: child_node) return d @@ -397,7 +410,11 @@ class NewDirectoryNode: assert len(e) == 3 name, child_uri, metadata = e assert isinstance(name, unicode) - a.set_node(name, self._create_node(child_uri, None), metadata) + child_node = self._create_node(child_uri, None) + if isinstance(child_node, UnknownNode): + msg = "cannot pack unknown node as child %s" % str(name) + raise CannotPackUnknownNodeError(msg) + a.set_node(name, child_node, metadata) return self._node.modify(a.modify) def set_node(self, name, child, metadata=None, overwrite=True): @@ -560,12 +577,15 @@ class NewDirectoryNode: dirkids = [] filekids = [] for name, (child, metadata) in sorted(children.iteritems()): + childpath = path + [name] + if isinstance(child, UnknownNode): + walker.add_node(child, childpath) + continue verifier = child.get_verify_cap() # allow LIT files (for which verifier==None) to be processed if (verifier is not None) and (verifier in found): continue found.add(verifier) - childpath = path + [name] if IDirectoryNode.providedBy(child): dirkids.append( (child, childpath) ) else: @@ -618,6 +638,7 @@ class DeepStats: "count-literal-files", "count-files", "count-directories", + "count-unknown", "size-immutable-files", #"size-mutable-files", "size-literal-files", @@ -640,7 +661,9 @@ class DeepStats: monitor.set_status(self.get_results()) def add_node(self, node, childpath): - if IDirectoryNode.providedBy(node): + if isinstance(node, UnknownNode): + self.add("count-unknown") + elif IDirectoryNode.providedBy(node): self.add("count-directories") elif IMutableFileNode.providedBy(node): self.add("count-files") diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 86e831243..42188e7be 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -478,6 +478,9 @@ class INewDirectoryURI(Interface): class IReadonlyNewDirectoryURI(Interface): pass +class CannotPackUnknownNodeError(Exception): + """UnknownNodes (using filecaps from the future that we don't understand) + cannot yet be copied safely, so I refuse to copy them.""" class IFilesystemNode(Interface): def get_uri(): diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index b652fbab5..38cbdc226 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -8,7 +8,8 @@ import allmydata from allmydata import client from allmydata.storage_client import StorageFarmBroker from allmydata.introducer.client import IntroducerClient -from allmydata.util import base32 +from allmydata.util import base32, fileutil +from allmydata.interfaces import IFilesystemNode, IFileNode, IDirectoryNode from foolscap.api import flushEventualQueue import common_util as testutil @@ -234,3 +235,39 @@ class Run(unittest.TestCase, testutil.StallMixin): d.addCallback(_restart) return d +class NodeMaker(unittest.TestCase): + def test_maker(self): + basedir = "client/NodeMaker/maker" + fileutil.make_dirs(basedir) + f = open(os.path.join(basedir, "tahoe.cfg"), "w") + f.write(BASECONFIG) + f.close() + c = client.Client(basedir) + + n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") + self.failUnless(IFilesystemNode.providedBy(n)) + self.failUnless(IFileNode.providedBy(n)) + self.failIf(IDirectoryNode.providedBy(n)) + self.failUnless(n.is_readonly()) + self.failIf(n.is_mutable()) + + n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + self.failUnless(IFilesystemNode.providedBy(n)) + self.failIf(IFileNode.providedBy(n)) + self.failUnless(IDirectoryNode.providedBy(n)) + self.failIf(n.is_readonly()) + self.failUnless(n.is_mutable()) + + n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + self.failUnless(IFilesystemNode.providedBy(n)) + self.failIf(IFileNode.providedBy(n)) + self.failUnless(IDirectoryNode.providedBy(n)) + self.failUnless(n.is_readonly()) + self.failUnless(n.is_mutable()) + + future = "x-tahoe-crazy://future_cap_format." + n = c.create_node_from_uri(future) + self.failUnless(IFilesystemNode.providedBy(n)) + self.failIf(IFileNode.providedBy(n)) + self.failIf(IDirectoryNode.providedBy(n)) + self.failUnlessEqual(n.get_uri(), future) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 0321726a8..555d647e7 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -4,11 +4,12 @@ from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer from allmydata import uri, dirnode +from allmydata.client import Client from allmydata.immutable import upload from allmydata.interfaces import IURI, IClient, IMutableFileNode, \ INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \ ExistingChildError, NoSuchChildError, \ - IDeepCheckResults, IDeepCheckAndRepairResults + IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError from allmydata.mutable.filenode import MutableFileNode from allmydata.mutable.common import UncoordinatedWriteError from allmydata.util import hashutil, base32 @@ -17,6 +18,7 @@ from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \ FakeDirectoryNode, create_chk_filenode, ErrorMixin from allmydata.test.no_network import GridTestMixin from allmydata.check_results import CheckResults, CheckAndRepairResults +from allmydata.unknown import UnknownNode import common_util as testutil # to test dirnode.py, we want to construct a tree of real DirectoryNodes that @@ -715,6 +717,83 @@ class Dirnode(unittest.TestCase, d.addErrback(self.explain_error) return d +class FakeMutableFile: + counter = 0 + def __init__(self, initial_contents=""): + self.data = initial_contents + counter = FakeMutableFile.counter + FakeMutableFile.counter += 1 + writekey = hashutil.ssk_writekey_hash(str(counter)) + fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter)) + self.uri = uri.WriteableSSKFileURI(writekey, fingerprint) + def get_uri(self): + return self.uri.to_string() + def download_best_version(self): + return defer.succeed(self.data) + def get_writekey(self): + return "writekey" + def is_readonly(self): + return False + def is_mutable(self): + return True + def modify(self, modifier): + self.data = modifier(self.data, None, True) + return defer.succeed(None) + +class FakeClient2(Client): + def __init__(self): + pass + def create_mutable_file(self, initial_contents=""): + return defer.succeed(FakeMutableFile(initial_contents)) + +class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin): + def setUp(self): + self.client = FakeClient2() + + def test_from_future(self): + # create a dirnode that contains unknown URI types, and make sure we + # tolerate them properly. Since dirnodes aren't allowed to add + # unknown node types, we have to be tricky. + d = self.client.create_empty_dirnode() + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_writecap, future_readcap) + def _then(n): + self._node = n + return n.set_node(u"future", future_node) + d.addCallback(_then) + + # we should be prohibited from adding an unknown URI to a directory, + # since we don't know how to diminish the cap to a readcap (for the + # dirnode's rocap slot), and we don't want to accidentally grant + # write access to a holder of the dirnode's readcap. + d.addCallback(lambda ign: + self.shouldFail(CannotPackUnknownNodeError, + "copy unknown", + "cannot pack unknown node as child add", + self._node.set_uri, u"add", future_writecap)) + d.addCallback(lambda ign: self._node.list()) + def _check(children): + self.failUnlessEqual(len(children), 1) + (fn, metadata) = children[u"future"] + self.failUnless(isinstance(fn, UnknownNode), fn) + self.failUnlessEqual(fn.get_uri(), future_writecap) + self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) + # but we *should* be allowed to copy this node, because the + # UnknownNode contains all the information that was in the + # original directory (readcap and writecap), so we're preserving + # everything. + return self._node.set_node(u"copy", fn) + d.addCallback(_check) + d.addCallback(lambda ign: self._node.list()) + def _check2(children): + self.failUnlessEqual(len(children), 2) + (fn, metadata) = children[u"copy"] + self.failUnless(isinstance(fn, UnknownNode), fn) + self.failUnlessEqual(fn.get_uri(), future_writecap) + self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) + return d + class DeepStats(unittest.TestCase): timeout = 240 # It takes longer than 120 seconds on Francois's arm box. def test_stats(self): diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index e81d466d3..7ac63469d 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -61,7 +61,7 @@ class Compare(unittest.TestCase): def test_is_uri(self): lit1 = uri.LiteralFileURI("some data").to_string() self.failUnless(uri.is_uri(lit1)) - self.failIf(uri.is_uri("this is not a uri")) + self.failIf(uri.is_uri(None)) class CHKFile(unittest.TestCase): def test_pack(self): @@ -175,10 +175,12 @@ class Extension(unittest.TestCase): readable = uri.unpack_extension_readable(ext) class Invalid(unittest.TestCase): - def test_create_invalid(self): - not_uri = "I am not a URI" - self.failUnlessRaises(TypeError, uri.from_string, not_uri) - + def test_from_future(self): + # any URI type that we don't recognize should be treated as unknown + future_uri = "I am a URI from the future. Whatever you do, don't " + u = uri.from_string(future_uri) + self.failUnless(isinstance(u, uri.UnknownURI)) + self.failUnlessEqual(u.to_string(), future_uri) class Constraint(unittest.TestCase): def test_constraint(self): diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index c6638ce70..6a1a8638e 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -11,6 +11,7 @@ from allmydata import interfaces, uri, webish from allmydata.storage.shares import get_share_file from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import upload, download +from allmydata.unknown import UnknownNode from allmydata.web import status, common from allmydata.scripts.debug import CorruptShareOptions, corrupt_share from allmydata.util import fileutil, base32 @@ -2781,6 +2782,73 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): d.addErrback(self.explain_web_error) return d + def test_unknown(self): + self.basedir = "web/Grid/unknown" + self.set_up_grid() + c0 = self.g.clients[0] + self.uris = {} + self.fileurls = {} + + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + # the future cap format may contain slashes, which must be tolerated + expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap, + safe="") + future_node = UnknownNode(future_writecap, future_readcap) + + d = c0.create_empty_dirnode() + def _stash_root_and_create_file(n): + self.rootnode = n + self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/" + self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/" + return self.rootnode.set_node(u"future", future_node) + d.addCallback(_stash_root_and_create_file) + # make sure directory listing tolerates unknown nodes + d.addCallback(lambda ign: self.GET(self.rooturl)) + def _check_html(res): + self.failUnlessIn("future", res) + # find the More Info link for "future", should be relative + mo = re.search(r'More Info', res) + info_url = mo.group(1) + self.failUnlessEqual(info_url, "future?t=info") + + d.addCallback(_check_html) + d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) + def _check_json(res, expect_writecap): + data = simplejson.loads(res) + self.failUnlessEqual(data[0], "dirnode") + f = data[1]["children"]["future"] + self.failUnlessEqual(f[0], "unknown") + if expect_writecap: + self.failUnlessEqual(f[1]["rw_uri"], future_writecap) + else: + self.failIfIn("rw_uri", f[1]) + self.failUnlessEqual(f[1]["ro_uri"], future_readcap) + self.failUnless("metadata" in f[1]) + d.addCallback(_check_json, expect_writecap=True) + d.addCallback(lambda ign: self.GET(expected_info_url)) + def _check_info(res, expect_readcap): + self.failUnlessIn("Object Type: unknown", res) + self.failUnlessIn(future_writecap, res) + if expect_readcap: + self.failUnlessIn(future_readcap, res) + self.failIfIn("Raw data as", res) + self.failIfIn("Directory writecap", res) + self.failIfIn("Checker Operations", res) + self.failIfIn("Mutable File Operations", res) + self.failIfIn("Directory Operations", res) + d.addCallback(_check_info, expect_readcap=False) + d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info")) + d.addCallback(_check_info, expect_readcap=True) + + # and make sure that a read-only version of the directory can be + # rendered too. This version will not have future_writecap + d.addCallback(lambda ign: self.GET(self.rourl)) + d.addCallback(_check_html) + d.addCallback(lambda ign: self.GET(self.rourl+"?t=json")) + d.addCallback(_check_json, expect_writecap=False) + return d + def test_deep_check(self): self.basedir = "web/Grid/deep_check" self.set_up_grid() @@ -2809,6 +2877,13 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): convergence=""))) d.addCallback(_stash_uri, "sick") + # this tests that deep-check and stream-manifest will ignore + # UnknownNode instances. Hopefully this will also cover deep-stats. + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_writecap, future_readcap) + d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node)) + def _clobber_shares(ignored): self.delete_shares_numbered(self.uris["sick"], [0,1]) d.addCallback(_clobber_shares) @@ -2817,13 +2892,19 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): # root/good # root/small # root/sick + # root/future d.addCallback(self.CHECK, "root", "t=stream-deep-check") def _done(res): - units = [simplejson.loads(line) - for line in res.splitlines() - if line] - self.failUnlessEqual(len(units), 4+1) + try: + units = [simplejson.loads(line) + for line in res.splitlines() + if line] + except ValueError: + print "response is:", res + print "undecodeable line was '%s'" % line + raise + self.failUnlessEqual(len(units), 5+1) # should be parent-first u0 = units[0] self.failUnlessEqual(u0["path"], []) @@ -2844,8 +2925,27 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): self.failUnlessEqual(s["count-immutable-files"], 2) self.failUnlessEqual(s["count-literal-files"], 1) self.failUnlessEqual(s["count-directories"], 1) + self.failUnlessEqual(s["count-unknown"], 1) d.addCallback(_done) + d.addCallback(self.CHECK, "root", "t=stream-manifest") + def _check_manifest(res): + self.failUnless(res.endswith("\n")) + units = [simplejson.loads(t) for t in res[:-1].split("\n")] + self.failUnlessEqual(len(units), 5+1) + self.failUnlessEqual(units[-1]["type"], "stats") + first = units[0] + self.failUnlessEqual(first["path"], []) + self.failUnlessEqual(first["cap"], self.rootnode.get_uri()) + self.failUnlessEqual(first["type"], "directory") + stats = units[-1]["stats"] + self.failUnlessEqual(stats["count-immutable-files"], 2) + self.failUnlessEqual(stats["count-literal-files"], 1) + self.failUnlessEqual(stats["count-mutable-files"], 0) + self.failUnlessEqual(stats["count-immutable-files"], 2) + self.failUnlessEqual(stats["count-unknown"], 1) + d.addCallback(_check_manifest) + # now add root/subdir and root/subdir/grandchild, then make subdir # unrecoverable, then see what happens @@ -2866,6 +2966,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): # root/good # root/small # root/sick + # root/future # root/subdir [unrecoverable] # root/subdir/grandchild @@ -2888,7 +2989,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): error_line) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [simplejson.loads(line) for line in lines[:first_error]] - self.failUnlessEqual(len(units), 5) # includes subdir + self.failUnlessEqual(len(units), 6) # includes subdir last_unit = units[-1] self.failUnlessEqual(last_unit["path"], ["subdir"]) d.addCallback(_check_broken_manifest) @@ -2909,7 +3010,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): error_line) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [simplejson.loads(line) for line in lines[:first_error]] - self.failUnlessEqual(len(units), 5) # includes subdir + self.failUnlessEqual(len(units), 6) # includes subdir last_unit = units[-1] self.failUnlessEqual(last_unit["path"], ["subdir"]) r = last_unit["check-results"]["results"] diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py new file mode 100644 index 000000000..c64e63685 --- /dev/null +++ b/src/allmydata/unknown.py @@ -0,0 +1,25 @@ +from zope.interface import implements +from twisted.internet import defer +from allmydata.interfaces import IFilesystemNode + +class UnknownNode: + implements(IFilesystemNode) + def __init__(self, writecap, readcap): + assert writecap is None or isinstance(writecap, str) + self.writecap = writecap + assert readcap is None or isinstance(readcap, str) + self.readcap = readcap + def get_uri(self): + return self.writecap + def get_readonly_uri(self): + return self.readcap + def get_storage_index(self): + return None + def get_verify_cap(self): + return None + def get_repair_cap(self): + return None + def check(self, monitor, verify, add_lease): + return defer.succeed(None) + def check_and_repair(self, monitor, verify, add_lease): + return defer.succeed(None) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 297fe93c8..9be92a418 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -428,10 +428,16 @@ class NewDirectoryURIVerifier(_NewDirectoryBaseURI): def get_filenode_uri(self): return self._filenode_uri - +class UnknownURI: + def __init__(self, uri): + self._uri = uri + def to_string(self): + return self._uri def from_string(s): - if s.startswith('URI:CHK:'): + if not isinstance(s, str): + raise TypeError("unknown URI type: %s.." % str(s)[:100]) + elif s.startswith('URI:CHK:'): return CHKFileURI.init_from_string(s) elif s.startswith('URI:CHK-Verifier:'): return CHKFileVerifierURI.init_from_string(s) @@ -449,8 +455,7 @@ def from_string(s): return ReadonlyNewDirectoryURI.init_from_string(s) elif s.startswith('URI:DIR2-Verifier:'): return NewDirectoryURIVerifier.init_from_string(s) - else: - raise TypeError("unknown URI type: %s.." % s[:12]) + return UnknownURI(s) registerAdapter(from_string, str, IURI) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 5c83b51b9..1bfd38262 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -15,7 +15,7 @@ from foolscap.api import fireEventually from allmydata.util import base32, time_format from allmydata.uri import from_string_dirnode from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \ - ExistingChildError, NoSuchChildError + IFilesystemNode, ExistingChildError, NoSuchChildError from allmydata.monitor import Monitor, OperationCancelledError from allmydata import dirnode from allmydata.web.common import text_plain, WebError, \ @@ -46,7 +46,7 @@ def make_handler_for(node, client, parentnode=None, name=None): return FileNodeHandler(client, node, parentnode, name) if IDirectoryNode.providedBy(node): return DirectoryNodeHandler(client, node, parentnode, name) - raise WebError("Cannot provide handler for '%s'" % node) + return UnknownNodeHandler(client, node, parentnode, name) class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): addSlash = True @@ -617,11 +617,9 @@ class DirectoryAsHTML(rend.Page): times.append("m: " + mtime) ctx.fillSlots("times", times) - assert (IFileNode.providedBy(target) - or IDirectoryNode.providedBy(target) - or IMutableFileNode.providedBy(target)), target - - quoted_uri = urllib.quote(target.get_uri()) + assert IFilesystemNode.providedBy(target), target + writecap = target.get_uri() or "" + quoted_uri = urllib.quote(writecap, safe="") # escape slashes too if IMutableFileNode.providedBy(target): # to prevent javascript in displayed .html files from stealing a @@ -650,7 +648,7 @@ class DirectoryAsHTML(rend.Page): elif IDirectoryNode.providedBy(target): # directory - uri_link = "%s/uri/%s/" % (root, urllib.quote(target.get_uri())) + uri_link = "%s/uri/%s/" % (root, urllib.quote(writecap)) ctx.fillSlots("filename", T.a(href=uri_link)[html.escape(name)]) if target.is_readonly(): @@ -661,6 +659,15 @@ class DirectoryAsHTML(rend.Page): ctx.fillSlots("size", "-") info_link = "%s/uri/%s/?t=info" % (root, quoted_uri) + else: + # unknown + ctx.fillSlots("filename", html.escape(name)) + ctx.fillSlots("type", "?") + ctx.fillSlots("size", "-") + # use a directory-relative info link, so we can extract both the + # writecap and the readcap + info_link = "%s?t=info" % urllib.quote(name) + ctx.fillSlots("info", T.a(href=info_link)["More Info"]) return ctx.tag @@ -727,20 +734,23 @@ def DirectoryJSONMetadata(ctx, dirnode): def _got(children): kids = {} for name, (childnode, metadata) in children.iteritems(): - if childnode.is_readonly(): - rw_uri = None - ro_uri = childnode.get_uri() - else: - rw_uri = childnode.get_uri() - ro_uri = childnode.get_readonly_uri() + assert IFilesystemNode.providedBy(childnode), childnode + rw_uri = childnode.get_uri() + ro_uri = childnode.get_readonly_uri() + if (IDirectoryNode.providedBy(childnode) + or IFileNode.providedBy(childnode)): + if childnode.is_readonly(): + rw_uri = None if IFileNode.providedBy(childnode): kiddata = ("filenode", {'size': childnode.get_size(), - 'metadata': metadata, + 'mutable': childnode.is_mutable(), }) + elif IDirectoryNode.providedBy(childnode): + kiddata = ("dirnode", {'mutable': childnode.is_mutable(), + }) else: - assert IDirectoryNode.providedBy(childnode), (childnode, - children,) - kiddata = ("dirnode", {'metadata': metadata}) + kiddata = ("unknown", {}) + kiddata[1]["metadata"] = metadata if ro_uri: kiddata[1]["ro_uri"] = ro_uri if rw_uri: @@ -748,7 +758,6 @@ def DirectoryJSONMetadata(ctx, dirnode): verifycap = childnode.get_verify_cap() if verifycap: kiddata[1]['verify_uri'] = verifycap.to_string() - kiddata[1]['mutable'] = childnode.is_mutable() kids[name] = kiddata if dirnode.is_readonly(): drw_uri = None @@ -879,13 +888,16 @@ class ManifestResults(rend.Page, ReloadMixin): ctx.fillSlots("path", self.slashify_path(path)) root = get_root(ctx) # TODO: we need a clean consistent way to get the type of a cap string - if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"): - nameurl = urllib.quote(path[-1].encode("utf-8")) - uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap), - nameurl) + if cap: + if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"): + nameurl = urllib.quote(path[-1].encode("utf-8")) + uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap), + nameurl) + else: + uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe="")) + ctx.fillSlots("cap", T.a(href=uri_link)[cap]) else: - uri_link = "%s/uri/%s" % (root, urllib.quote(cap)) - ctx.fillSlots("cap", T.a(href=uri_link)[cap]) + ctx.fillSlots("cap", "") return ctx.tag class DeepSizeResults(rend.Page): @@ -951,8 +963,10 @@ class ManifestStreamer(dirnode.DeepStats): if IDirectoryNode.providedBy(node): d["type"] = "directory" - else: + elif IFileNode.providedBy(node): d["type"] = "file" + else: + d["type"] = "unknown" v = node.get_verify_cap() if v: @@ -1058,3 +1072,19 @@ class DeepCheckStreamer(dirnode.DeepStats): assert "\n" not in j self.req.write(j+"\n") return "" + +class UnknownNodeHandler(RenderMixin, rend.Page): + + def __init__(self, client, node, parentnode=None, name=None): + rend.Page.__init__(self) + assert node + self.node = node + + def render_GET(self, ctx): + req = IRequest(ctx) + t = get_arg(req, "t", "").strip() + if t == "info": + return MoreInfo(self.node) + raise WebError("GET unknown: can only do t=info, not t=%s" % t) + + diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 727d9b68b..05c122791 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -6,10 +6,11 @@ from twisted.internet import defer from nevow import url, rend from nevow.inevow import IRequest -from allmydata.interfaces import ExistingChildError +from allmydata.interfaces import ExistingChildError, CannotPackUnknownNodeError from allmydata.monitor import Monitor from allmydata.immutable.upload import FileHandle from allmydata.immutable.filenode import LiteralFileNode +from allmydata.unknown import UnknownNode from allmydata.util import log, base32 from allmydata.web.common import text_plain, WebError, RenderMixin, \ @@ -55,7 +56,14 @@ class ReplaceMeMixin: def replace_me_with_a_childcap(self, req, client, replace): req.content.seek(0) childcap = req.content.read() - childnode = client.create_node_from_uri(childcap) + childnode = client.create_node_from_uri(childcap, childcap+"readonly") + if isinstance(childnode, UnknownNode): + # don't be willing to pack unknown nodes: we might accidentally + # put some write-authority into the rocap slot because we don't + # know how to diminish the URI they gave us. We don't even know + # if they gave us a readcap or a writecap. + msg = "cannot attach unknown node as child %s" % str(self.name) + raise CannotPackUnknownNodeError(msg) d = self.parentnode.set_node(self.name, childnode, overwrite=replace) d.addCallback(lambda res: childnode.get_uri()) return d diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index e7c42e60b..edaab44bb 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -6,7 +6,7 @@ from nevow import rend, tags as T from nevow.inevow import IRequest from allmydata.util import base32 -from allmydata.interfaces import IDirectoryNode +from allmydata.interfaces import IDirectoryNode, IFileNode from allmydata.web.common import getxmlfile from allmydata.mutable.common import UnrecoverableFileError # TODO: move @@ -21,14 +21,16 @@ class MoreInfo(rend.Page): def get_type(self): node = self.original - si = node.get_storage_index() if IDirectoryNode.providedBy(node): return "directory" - if si: - if node.is_mutable(): - return "mutable file" - return "immutable file" - return "LIT file" + if IFileNode.providedBy(node): + si = node.get_storage_index() + if si: + if node.is_mutable(): + return "mutable file" + return "immutable file" + return "LIT file" + return "unknown" def render_title(self, ctx, data): node = self.original @@ -55,11 +57,15 @@ class MoreInfo(rend.Page): si = node.get_storage_index() if IDirectoryNode.providedBy(node): d = node._node.get_size_of_best_version() - elif node.is_mutable(): - d = node.get_size_of_best_version() + elif IFileNode.providedBy(node): + if node.is_mutable(): + d = node.get_size_of_best_version() + else: + # for immutable files and LIT files, we get the size from the + # URI + d = defer.succeed(node.get_size()) else: - # for immutable files and LIT files, we get the size from the URI - d = defer.succeed(node.get_size()) + d = defer.succeed("?") def _handle_unrecoverable(f): f.trap(UnrecoverableFileError) return "?" @@ -92,15 +98,22 @@ class MoreInfo(rend.Page): node = self.original if IDirectoryNode.providedBy(node): node = node._node - if node.is_readonly(): + if ((IDirectoryNode.providedBy(node) or IFileNode.providedBy(node)) + and node.is_readonly()): return "" - return ctx.tag[node.get_uri()] + writecap = node.get_uri() + if not writecap: + return "" + return ctx.tag[writecap] def render_file_readcap(self, ctx, data): node = self.original if IDirectoryNode.providedBy(node): node = node._node - return ctx.tag[node.get_readonly_uri()] + readcap = node.get_readonly_uri() + if not readcap: + return "" + return ctx.tag[readcap] def render_file_verifycap(self, ctx, data): node = self.original @@ -122,10 +135,14 @@ class MoreInfo(rend.Page): node = self.original if IDirectoryNode.providedBy(node): node = node._node + elif IFileNode.providedBy(node): + pass + else: + return "" root = self.get_root(ctx) quoted_uri = urllib.quote(node.get_uri()) text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri) - return ctx.tag[text_plain_url] + return T.li["Raw data as ", T.a(href=text_plain_url)["text/plain"]] def render_is_checkable(self, ctx, data): node = self.original @@ -167,7 +184,8 @@ class MoreInfo(rend.Page): node = self.original if IDirectoryNode.providedBy(node): return "" - if node.is_mutable() and not node.is_readonly(): + if (IFileNode.providedBy(node) + and node.is_mutable() and not node.is_readonly()): return ctx.tag return "" diff --git a/src/allmydata/web/info.xhtml b/src/allmydata/web/info.xhtml index 5e0a09c16..9237f38fe 100644 --- a/src/allmydata/web/info.xhtml +++ b/src/allmydata/web/info.xhtml @@ -45,7 +45,7 @@
  • JSON
  • -
  • Raw data as text/plain
  • +