Tolerate unknown URI types in directory structures. Part of #683.

The idea is that future versions of Tahoe will add new URI types that this
version won't recognize, but might store them in directories that we *can*
read. We should handle these "objects from the future" as best we can.
Previous releases of Tahoe would just explode. With this change, we'll
continue to be able to work with everything else in the directory.

The code change is to wrap anything we don't recognize as an UnknownNode
instance (as opposed to a FileNode or DirectoryNode). Then webapi knows how
to render these (mostly by leaving fields blank), deep-check knows to skip
over them, deep-stats counts them in "count-unknown". You can rename and
delete these things, but you can't add new ones (because we wouldn't know how
to generate a readcap to put into the dirnode's rocap slot, and because this
lets us catch typos better).

docs/frontends/webapi.txt

@@ -1232,6 +1232,7 @@ POST $DIRURL?t=start-deep-stats    (must add &ophandle=XYZ)
    count-literal-files: same, for LIT files (data contained inside the URI)
    count-files: sum of the above three
    count-directories: count of directories
+   count-unknown: count of unrecognized objects (perhaps from the future)
    size-immutable-files: total bytes for all CHK files in the set, =deep-size
    size-mutable-files (TODO): same, for current version of all mutable files
    size-literal-files: same, for LIT files

src/allmydata/client.py

@@ -20,9 +20,10 @@ from allmydata.introducer.client import IntroducerClient
 from allmydata.util import hashutil, base32, pollmixin, cachedir, log
 from allmydata.util.abbreviate import parse_abbreviated_size
 from allmydata.util.time_format import parse_duration, parse_date
-from allmydata.uri import LiteralFileURI
+from allmydata.uri import LiteralFileURI, UnknownURI
 from allmydata.dirnode import NewDirectoryNode
 from allmydata.mutable.filenode import MutableFileNode
+from allmydata.unknown import UnknownNode
 from allmydata.stats import StatsProvider
 from allmydata.history import History
 from allmydata.interfaces import IURI, INewDirectoryURI, IStatsProducer, \
@@ -404,11 +405,17 @@ class Client(node.Node, pollmixin.PollMixin):
     # dirnodes. The first takes a URI and produces a filenode or (new-style)
     # dirnode. The other three create brand-new filenodes/dirnodes.
 
-    def create_node_from_uri(self, u, readcap=None):
+    def create_node_from_uri(self, writecap, readcap=None):
         # this returns synchronously.
+        u = writecap or readcap
         if not u:
-            u = readcap
+            # maybe the writecap was hidden because we're in a readonly
+            # directory, and the future cap format doesn't have a readcap, or
+            # something.
+            return UnknownNode(writecap, readcap)
         u = IURI(u)
+        if isinstance(u, UnknownURI):
+            return UnknownNode(writecap, readcap)
         u_s = u.to_string()
         if u_s not in self._node_cache:
             if IReadonlyNewDirectoryURI.providedBy(u):
@@ -427,13 +434,12 @@ class Client(node.Node, pollmixin.PollMixin):
             else:
                 assert IMutableFileURI.providedBy(u), u
                 node = MutableFileNode(self).init_from_uri(u)
-            self._node_cache[u_s] = node
+            self._node_cache[u_s] = node  # note: WeakValueDictionary
         return self._node_cache[u_s]
 
     def create_empty_dirnode(self):
-        n = NewDirectoryNode(self)
-        d = n.create(self._generate_pubprivkeys, self.DEFAULT_MUTABLE_KEYSIZE)
-        d.addCallback(lambda res: n)
+        d = self.create_mutable_file()
+        d.addCallback(NewDirectoryNode.create_with_mutablefile, self)
         return d
 
     def create_mutable_file(self, contents="", keysize=None):

src/allmydata/dirnode.py

@@ -7,9 +7,11 @@ from foolscap.api import fireEventually
 import simplejson
 from allmydata.mutable.common import NotMutableError
 from allmydata.mutable.filenode import MutableFileNode
+from allmydata.unknown import UnknownNode
 from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\
      IURI, IFileNode, IMutableFileURI, IFilesystemNode, \
-     ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable
+     ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable, \
+     CannotPackUnknownNodeError
 from allmydata.check_results import DeepCheckResults, \
      DeepCheckAndRepairResults
 from allmydata.monitor import Monitor
@@ -137,6 +139,12 @@ class NewDirectoryNode:
         self._node.init_from_uri(self._uri.get_filenode_uri())
         return self
 
+    @classmethod
+    def create_with_mutablefile(cls, filenode, client):
+        self = cls(client)
+        self._node = filenode
+        return self._filenode_created(filenode)
+
     def create(self, keypair_generator=None, keysize=None):
         """
         Returns a deferred that eventually fires with self once the directory
@@ -226,9 +234,7 @@ class NewDirectoryNode:
         for name in sorted(children.keys()):
             child, metadata = children[name]
             assert isinstance(name, unicode)
-            assert (IFileNode.providedBy(child)
-                    or IMutableFileNode.providedBy(child)
-                    or IDirectoryNode.providedBy(child)), (name,child)
+            assert IFilesystemNode.providedBy(child), (name,child)
             assert isinstance(metadata, dict)
             rwcap = child.get_uri() # might be RO if the child is not writeable
             if rwcap is None:
@@ -381,6 +387,13 @@ class NewDirectoryNode:
         precondition(isinstance(name, unicode), name)
         precondition(isinstance(child_uri, str), child_uri)
         child_node = self._create_node(child_uri, None)
+        if isinstance(child_node, UnknownNode):
+            # don't be willing to pack unknown nodes: we might accidentally
+            # put some write-authority into the rocap slot because we don't
+            # know how to diminish the URI they gave us. We don't even know
+            # if they gave us a readcap or a writecap.
+            msg = "cannot pack unknown node as child %s" % str(name)
+            raise CannotPackUnknownNodeError(msg)
         d = self.set_node(name, child_node, metadata, overwrite)
         d.addCallback(lambda res: child_node)
         return d
@@ -397,7 +410,11 @@ class NewDirectoryNode:
                 assert len(e) == 3
                 name, child_uri, metadata = e
             assert isinstance(name, unicode)
-            a.set_node(name, self._create_node(child_uri, None), metadata)
+            child_node = self._create_node(child_uri, None)
+            if isinstance(child_node, UnknownNode):
+                msg = "cannot pack unknown node as child %s" % str(name)
+                raise CannotPackUnknownNodeError(msg)
+            a.set_node(name, child_node, metadata)
         return self._node.modify(a.modify)
 
     def set_node(self, name, child, metadata=None, overwrite=True):
@@ -560,12 +577,15 @@ class NewDirectoryNode:
         dirkids = []
         filekids = []
         for name, (child, metadata) in sorted(children.iteritems()):
+            childpath = path + [name]
+            if isinstance(child, UnknownNode):
+                walker.add_node(child, childpath)
+                continue
             verifier = child.get_verify_cap()
             # allow LIT files (for which verifier==None) to be processed
             if (verifier is not None) and (verifier in found):
                 continue
             found.add(verifier)
-            childpath = path + [name]
             if IDirectoryNode.providedBy(child):
                 dirkids.append( (child, childpath) )
             else:
@@ -618,6 +638,7 @@ class DeepStats:
                   "count-literal-files",
                   "count-files",
                   "count-directories",
+                  "count-unknown",
                   "size-immutable-files",
                   #"size-mutable-files",
                   "size-literal-files",
@@ -640,7 +661,9 @@ class DeepStats:
         monitor.set_status(self.get_results())
 
     def add_node(self, node, childpath):
-        if IDirectoryNode.providedBy(node):
+        if isinstance(node, UnknownNode):
+            self.add("count-unknown")
+        elif IDirectoryNode.providedBy(node):
             self.add("count-directories")
         elif IMutableFileNode.providedBy(node):
             self.add("count-files")

src/allmydata/interfaces.py

@@ -478,6 +478,9 @@ class INewDirectoryURI(Interface):
 class IReadonlyNewDirectoryURI(Interface):
     pass
 
+class CannotPackUnknownNodeError(Exception):
+    """UnknownNodes (using filecaps from the future that we don't understand)
+    cannot yet be copied safely, so I refuse to copy them."""
 
 class IFilesystemNode(Interface):
     def get_uri():

src/allmydata/test/test_client.py

@@ -8,7 +8,8 @@ import allmydata
 from allmydata import client
 from allmydata.storage_client import StorageFarmBroker
 from allmydata.introducer.client import IntroducerClient
-from allmydata.util import base32
+from allmydata.util import base32, fileutil
+from allmydata.interfaces import IFilesystemNode, IFileNode, IDirectoryNode
 from foolscap.api import flushEventualQueue
 import common_util as testutil
 
@@ -234,3 +235,39 @@ class Run(unittest.TestCase, testutil.StallMixin):
         d.addCallback(_restart)
         return d
 
+class NodeMaker(unittest.TestCase):
+    def test_maker(self):
+        basedir = "client/NodeMaker/maker"
+        fileutil.make_dirs(basedir)
+        f = open(os.path.join(basedir, "tahoe.cfg"), "w")
+        f.write(BASECONFIG)
+        f.close()
+        c = client.Client(basedir)
+
+        n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
+        self.failUnless(IFilesystemNode.providedBy(n))
+        self.failUnless(IFileNode.providedBy(n))
+        self.failIf(IDirectoryNode.providedBy(n))
+        self.failUnless(n.is_readonly())
+        self.failIf(n.is_mutable())
+
+        n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        self.failUnless(IFilesystemNode.providedBy(n))
+        self.failIf(IFileNode.providedBy(n))
+        self.failUnless(IDirectoryNode.providedBy(n))
+        self.failIf(n.is_readonly())
+        self.failUnless(n.is_mutable())
+
+        n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        self.failUnless(IFilesystemNode.providedBy(n))
+        self.failIf(IFileNode.providedBy(n))
+        self.failUnless(IDirectoryNode.providedBy(n))
+        self.failUnless(n.is_readonly())
+        self.failUnless(n.is_mutable())
+
+        future = "x-tahoe-crazy://future_cap_format."
+        n = c.create_node_from_uri(future)
+        self.failUnless(IFilesystemNode.providedBy(n))
+        self.failIf(IFileNode.providedBy(n))
+        self.failIf(IDirectoryNode.providedBy(n))
+        self.failUnlessEqual(n.get_uri(), future)

src/allmydata/test/test_dirnode.py

@@ -4,11 +4,12 @@ from zope.interface import implements
 from twisted.trial import unittest
 from twisted.internet import defer
 from allmydata import uri, dirnode
+from allmydata.client import Client
 from allmydata.immutable import upload
 from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
      INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \
      ExistingChildError, NoSuchChildError, \
-     IDeepCheckResults, IDeepCheckAndRepairResults
+     IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError
 from allmydata.mutable.filenode import MutableFileNode
 from allmydata.mutable.common import UncoordinatedWriteError
 from allmydata.util import hashutil, base32
@@ -17,6 +18,7 @@ from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
      FakeDirectoryNode, create_chk_filenode, ErrorMixin
 from allmydata.test.no_network import GridTestMixin
 from allmydata.check_results import CheckResults, CheckAndRepairResults
+from allmydata.unknown import UnknownNode
 import common_util as testutil
 
 # to test dirnode.py, we want to construct a tree of real DirectoryNodes that
@@ -715,6 +717,83 @@ class Dirnode(unittest.TestCase,
         d.addErrback(self.explain_error)
         return d
 
+class FakeMutableFile:
+    counter = 0
+    def __init__(self, initial_contents=""):
+        self.data = initial_contents
+        counter = FakeMutableFile.counter
+        FakeMutableFile.counter += 1
+        writekey = hashutil.ssk_writekey_hash(str(counter))
+        fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
+        self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
+    def get_uri(self):
+        return self.uri.to_string()
+    def download_best_version(self):
+        return defer.succeed(self.data)
+    def get_writekey(self):
+        return "writekey"
+    def is_readonly(self):
+        return False
+    def is_mutable(self):
+        return True
+    def modify(self, modifier):
+        self.data = modifier(self.data, None, True)
+        return defer.succeed(None)
+
+class FakeClient2(Client):
+    def __init__(self):
+        pass
+    def create_mutable_file(self, initial_contents=""):
+        return defer.succeed(FakeMutableFile(initial_contents))
+
+class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
+    def setUp(self):
+        self.client = FakeClient2()
+
+    def test_from_future(self):
+        # create a dirnode that contains unknown URI types, and make sure we
+        # tolerate them properly. Since dirnodes aren't allowed to add
+        # unknown node types, we have to be tricky.
+        d = self.client.create_empty_dirnode()
+        future_writecap = "x-tahoe-crazy://I_am_from_the_future."
+        future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
+        future_node = UnknownNode(future_writecap, future_readcap)
+        def _then(n):
+            self._node = n
+            return n.set_node(u"future", future_node)
+        d.addCallback(_then)
+
+        # we should be prohibited from adding an unknown URI to a directory,
+        # since we don't know how to diminish the cap to a readcap (for the
+        # dirnode's rocap slot), and we don't want to accidentally grant
+        # write access to a holder of the dirnode's readcap.
+        d.addCallback(lambda ign:
+             self.shouldFail(CannotPackUnknownNodeError,
+                             "copy unknown",
+                             "cannot pack unknown node as child add",
+                             self._node.set_uri, u"add", future_writecap))
+        d.addCallback(lambda ign: self._node.list())
+        def _check(children):
+            self.failUnlessEqual(len(children), 1)
+            (fn, metadata) = children[u"future"]
+            self.failUnless(isinstance(fn, UnknownNode), fn)
+            self.failUnlessEqual(fn.get_uri(), future_writecap)
+            self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
+            # but we *should* be allowed to copy this node, because the
+            # UnknownNode contains all the information that was in the
+            # original directory (readcap and writecap), so we're preserving
+            # everything.
+            return self._node.set_node(u"copy", fn)
+        d.addCallback(_check)
+        d.addCallback(lambda ign: self._node.list())
+        def _check2(children):
+            self.failUnlessEqual(len(children), 2)
+            (fn, metadata) = children[u"copy"]
+            self.failUnless(isinstance(fn, UnknownNode), fn)
+            self.failUnlessEqual(fn.get_uri(), future_writecap)
+            self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
+        return d
+
 class DeepStats(unittest.TestCase):
     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
     def test_stats(self):

src/allmydata/test/test_uri.py

@@ -61,7 +61,7 @@ class Compare(unittest.TestCase):
     def test_is_uri(self):
         lit1 = uri.LiteralFileURI("some data").to_string()
         self.failUnless(uri.is_uri(lit1))
-        self.failIf(uri.is_uri("this is not a uri"))
+        self.failIf(uri.is_uri(None))
 
 class CHKFile(unittest.TestCase):
     def test_pack(self):
@@ -175,10 +175,12 @@ class Extension(unittest.TestCase):
         readable = uri.unpack_extension_readable(ext)
 
 class Invalid(unittest.TestCase):
-    def test_create_invalid(self):
-        not_uri = "I am not a URI"
-        self.failUnlessRaises(TypeError, uri.from_string, not_uri)
-
+    def test_from_future(self):
+        # any URI type that we don't recognize should be treated as unknown
+        future_uri = "I am a URI from the future. Whatever you do, don't "
+        u = uri.from_string(future_uri)
+        self.failUnless(isinstance(u, uri.UnknownURI))
+        self.failUnlessEqual(u.to_string(), future_uri)
 
 class Constraint(unittest.TestCase):
     def test_constraint(self):

src/allmydata/test/test_web.py

@@ -11,6 +11,7 @@ from allmydata import interfaces, uri, webish
 from allmydata.storage.shares import get_share_file
 from allmydata.storage_client import StorageFarmBroker
 from allmydata.immutable import upload, download
+from allmydata.unknown import UnknownNode
 from allmydata.web import status, common
 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
 from allmydata.util import fileutil, base32
@@ -2781,6 +2782,73 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
         d.addErrback(self.explain_web_error)
         return d
 
+    def test_unknown(self):
+        self.basedir = "web/Grid/unknown"
+        self.set_up_grid()
+        c0 = self.g.clients[0]
+        self.uris = {}
+        self.fileurls = {}
+
+        future_writecap = "x-tahoe-crazy://I_am_from_the_future."
+        future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
+        # the future cap format may contain slashes, which must be tolerated
+        expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap,
+                                                           safe="")
+        future_node = UnknownNode(future_writecap, future_readcap)
+
+        d = c0.create_empty_dirnode()
+        def _stash_root_and_create_file(n):
+            self.rootnode = n
+            self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
+            self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
+            return self.rootnode.set_node(u"future", future_node)
+        d.addCallback(_stash_root_and_create_file)
+        # make sure directory listing tolerates unknown nodes
+        d.addCallback(lambda ign: self.GET(self.rooturl))
+        def _check_html(res):
+            self.failUnlessIn("<td>future</td>", res)
+            # find the More Info link for "future", should be relative
+            mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
+            info_url = mo.group(1)
+            self.failUnlessEqual(info_url, "future?t=info")
+
+        d.addCallback(_check_html)
+        d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
+        def _check_json(res, expect_writecap):
+            data = simplejson.loads(res)
+            self.failUnlessEqual(data[0], "dirnode")
+            f = data[1]["children"]["future"]
+            self.failUnlessEqual(f[0], "unknown")
+            if expect_writecap:
+                self.failUnlessEqual(f[1]["rw_uri"], future_writecap)
+            else:
+                self.failIfIn("rw_uri", f[1])
+            self.failUnlessEqual(f[1]["ro_uri"], future_readcap)
+            self.failUnless("metadata" in f[1])
+        d.addCallback(_check_json, expect_writecap=True)
+        d.addCallback(lambda ign: self.GET(expected_info_url))
+        def _check_info(res, expect_readcap):
+            self.failUnlessIn("Object Type: <span>unknown</span>", res)
+            self.failUnlessIn(future_writecap, res)
+            if expect_readcap:
+                self.failUnlessIn(future_readcap, res)
+            self.failIfIn("Raw data as", res)
+            self.failIfIn("Directory writecap", res)
+            self.failIfIn("Checker Operations", res)
+            self.failIfIn("Mutable File Operations", res)
+            self.failIfIn("Directory Operations", res)
+        d.addCallback(_check_info, expect_readcap=False)
+        d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info"))
+        d.addCallback(_check_info, expect_readcap=True)
+
+        # and make sure that a read-only version of the directory can be
+        # rendered too. This version will not have future_writecap
+        d.addCallback(lambda ign: self.GET(self.rourl))
+        d.addCallback(_check_html)
+        d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
+        d.addCallback(_check_json, expect_writecap=False)
+        return d
+
     def test_deep_check(self):
         self.basedir = "web/Grid/deep_check"
         self.set_up_grid()
@@ -2809,6 +2877,13 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
                                                         convergence="")))
         d.addCallback(_stash_uri, "sick")
 
+        # this tests that deep-check and stream-manifest will ignore
+        # UnknownNode instances. Hopefully this will also cover deep-stats.
+        future_writecap = "x-tahoe-crazy://I_am_from_the_future."
+        future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
+        future_node = UnknownNode(future_writecap, future_readcap)
+        d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node))
+
         def _clobber_shares(ignored):
             self.delete_shares_numbered(self.uris["sick"], [0,1])
         d.addCallback(_clobber_shares)
@@ -2817,13 +2892,19 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
         # root/good
         # root/small
         # root/sick
+        # root/future
 
         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
         def _done(res):
-            units = [simplejson.loads(line)
-                     for line in res.splitlines()
-                     if line]
-            self.failUnlessEqual(len(units), 4+1)
+            try:
+                units = [simplejson.loads(line)
+                         for line in res.splitlines()
+                         if line]
+            except ValueError:
+                print "response is:", res
+                print "undecodeable line was '%s'" % line
+                raise
+            self.failUnlessEqual(len(units), 5+1)
             # should be parent-first
             u0 = units[0]
             self.failUnlessEqual(u0["path"], [])
@@ -2844,8 +2925,27 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
             self.failUnlessEqual(s["count-immutable-files"], 2)
             self.failUnlessEqual(s["count-literal-files"], 1)
             self.failUnlessEqual(s["count-directories"], 1)
+            self.failUnlessEqual(s["count-unknown"], 1)
         d.addCallback(_done)
 
+        d.addCallback(self.CHECK, "root", "t=stream-manifest")
+        def _check_manifest(res):
+            self.failUnless(res.endswith("\n"))
+            units = [simplejson.loads(t) for t in res[:-1].split("\n")]
+            self.failUnlessEqual(len(units), 5+1)
+            self.failUnlessEqual(units[-1]["type"], "stats")
+            first = units[0]
+            self.failUnlessEqual(first["path"], [])
+            self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
+            self.failUnlessEqual(first["type"], "directory")
+            stats = units[-1]["stats"]
+            self.failUnlessEqual(stats["count-immutable-files"], 2)
+            self.failUnlessEqual(stats["count-literal-files"], 1)
+            self.failUnlessEqual(stats["count-mutable-files"], 0)
+            self.failUnlessEqual(stats["count-immutable-files"], 2)
+            self.failUnlessEqual(stats["count-unknown"], 1)
+        d.addCallback(_check_manifest)
+
         # now add root/subdir and root/subdir/grandchild, then make subdir
         # unrecoverable, then see what happens
 
@@ -2866,6 +2966,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
         # root/good
         # root/small
         # root/sick
+        # root/future
         # root/subdir [unrecoverable]
         # root/subdir/grandchild
 
@@ -2888,7 +2989,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
                               error_line)
             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
             units = [simplejson.loads(line) for line in lines[:first_error]]
-            self.failUnlessEqual(len(units), 5) # includes subdir
+            self.failUnlessEqual(len(units), 6) # includes subdir
             last_unit = units[-1]
             self.failUnlessEqual(last_unit["path"], ["subdir"])
         d.addCallback(_check_broken_manifest)
@@ -2909,7 +3010,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
                               error_line)
             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
             units = [simplejson.loads(line) for line in lines[:first_error]]
-            self.failUnlessEqual(len(units), 5) # includes subdir
+            self.failUnlessEqual(len(units), 6) # includes subdir
             last_unit = units[-1]
             self.failUnlessEqual(last_unit["path"], ["subdir"])
             r = last_unit["check-results"]["results"]

src/allmydata/unknown.py

@@ -0,0 +1,25 @@
+from zope.interface import implements
+from twisted.internet import defer
+from allmydata.interfaces import IFilesystemNode
+
+class UnknownNode:
+    implements(IFilesystemNode)
+    def __init__(self, writecap, readcap):
+        assert writecap is None or isinstance(writecap, str)
+        self.writecap = writecap
+        assert readcap is None or isinstance(readcap, str)
+        self.readcap = readcap
+    def get_uri(self):
+        return self.writecap
+    def get_readonly_uri(self):
+        return self.readcap
+    def get_storage_index(self):
+        return None
+    def get_verify_cap(self):
+        return None
+    def get_repair_cap(self):
+        return None
+    def check(self, monitor, verify, add_lease):
+        return defer.succeed(None)
+    def check_and_repair(self, monitor, verify, add_lease):
+        return defer.succeed(None)

src/allmydata/uri.py

@@ -428,10 +428,16 @@ class NewDirectoryURIVerifier(_NewDirectoryBaseURI):
     def get_filenode_uri(self):
         return self._filenode_uri
 
-
+class UnknownURI:
+    def __init__(self, uri):
+        self._uri = uri
+    def to_string(self):
+        return self._uri
 
 def from_string(s):
-    if s.startswith('URI:CHK:'):
+    if not isinstance(s, str):
+        raise TypeError("unknown URI type: %s.." % str(s)[:100])
+    elif s.startswith('URI:CHK:'):
         return CHKFileURI.init_from_string(s)
     elif s.startswith('URI:CHK-Verifier:'):
         return CHKFileVerifierURI.init_from_string(s)
@@ -449,8 +455,7 @@ def from_string(s):
         return ReadonlyNewDirectoryURI.init_from_string(s)
     elif s.startswith('URI:DIR2-Verifier:'):
         return NewDirectoryURIVerifier.init_from_string(s)
-    else:
-        raise TypeError("unknown URI type: %s.." % s[:12])
+    return UnknownURI(s)
 
 registerAdapter(from_string, str, IURI)
 

src/allmydata/web/directory.py

@@ -15,7 +15,7 @@ from foolscap.api import fireEventually
 from allmydata.util import base32, time_format
 from allmydata.uri import from_string_dirnode
 from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \
-     ExistingChildError, NoSuchChildError
+     IFilesystemNode, ExistingChildError, NoSuchChildError
 from allmydata.monitor import Monitor, OperationCancelledError
 from allmydata import dirnode
 from allmydata.web.common import text_plain, WebError, \
@@ -46,7 +46,7 @@ def make_handler_for(node, client, parentnode=None, name=None):
         return FileNodeHandler(client, node, parentnode, name)
     if IDirectoryNode.providedBy(node):
         return DirectoryNodeHandler(client, node, parentnode, name)
-    raise WebError("Cannot provide handler for '%s'" % node)
+    return UnknownNodeHandler(client, node, parentnode, name)
 
 class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
     addSlash = True
@@ -617,11 +617,9 @@ class DirectoryAsHTML(rend.Page):
                 times.append("m: " + mtime)
         ctx.fillSlots("times", times)
 
-        assert (IFileNode.providedBy(target)
-                or IDirectoryNode.providedBy(target)
-                or IMutableFileNode.providedBy(target)), target
-
-        quoted_uri = urllib.quote(target.get_uri())
+        assert IFilesystemNode.providedBy(target), target
+        writecap = target.get_uri() or ""
+        quoted_uri = urllib.quote(writecap, safe="") # escape slashes too
 
         if IMutableFileNode.providedBy(target):
             # to prevent javascript in displayed .html files from stealing a
@@ -650,7 +648,7 @@ class DirectoryAsHTML(rend.Page):
 
         elif IDirectoryNode.providedBy(target):
             # directory
-            uri_link = "%s/uri/%s/" % (root, urllib.quote(target.get_uri()))
+            uri_link = "%s/uri/%s/" % (root, urllib.quote(writecap))
             ctx.fillSlots("filename",
                           T.a(href=uri_link)[html.escape(name)])
             if target.is_readonly():
@@ -661,6 +659,15 @@ class DirectoryAsHTML(rend.Page):
             ctx.fillSlots("size", "-")
             info_link = "%s/uri/%s/?t=info" % (root, quoted_uri)
 
+        else:
+            # unknown
+            ctx.fillSlots("filename", html.escape(name))
+            ctx.fillSlots("type", "?")
+            ctx.fillSlots("size", "-")
+            # use a directory-relative info link, so we can extract both the
+            # writecap and the readcap
+            info_link = "%s?t=info" % urllib.quote(name)
+
         ctx.fillSlots("info", T.a(href=info_link)["More Info"])
 
         return ctx.tag
@@ -727,20 +734,23 @@ def DirectoryJSONMetadata(ctx, dirnode):
     def _got(children):
         kids = {}
         for name, (childnode, metadata) in children.iteritems():
-            if childnode.is_readonly():
-                rw_uri = None
-                ro_uri = childnode.get_uri()
-            else:
-                rw_uri = childnode.get_uri()
-                ro_uri = childnode.get_readonly_uri()
+            assert IFilesystemNode.providedBy(childnode), childnode
+            rw_uri = childnode.get_uri()
+            ro_uri = childnode.get_readonly_uri()
+            if (IDirectoryNode.providedBy(childnode)
+                or IFileNode.providedBy(childnode)):
+                if childnode.is_readonly():
+                    rw_uri = None
             if IFileNode.providedBy(childnode):
                 kiddata = ("filenode", {'size': childnode.get_size(),
-                                        'metadata': metadata,
+                                        'mutable': childnode.is_mutable(),
                                         })
+            elif IDirectoryNode.providedBy(childnode):
+                kiddata = ("dirnode", {'mutable': childnode.is_mutable(),
+                                       })
             else:
-                assert IDirectoryNode.providedBy(childnode), (childnode,
-                                                              children,)
-                kiddata = ("dirnode", {'metadata': metadata})
+                kiddata = ("unknown", {})
+            kiddata[1]["metadata"] = metadata
             if ro_uri:
                 kiddata[1]["ro_uri"] = ro_uri
             if rw_uri:
@@ -748,7 +758,6 @@ def DirectoryJSONMetadata(ctx, dirnode):
             verifycap = childnode.get_verify_cap()
             if verifycap:
                 kiddata[1]['verify_uri'] = verifycap.to_string()
-            kiddata[1]['mutable'] = childnode.is_mutable()
             kids[name] = kiddata
         if dirnode.is_readonly():
             drw_uri = None
@@ -879,13 +888,16 @@ class ManifestResults(rend.Page, ReloadMixin):
         ctx.fillSlots("path", self.slashify_path(path))
         root = get_root(ctx)
         # TODO: we need a clean consistent way to get the type of a cap string
-        if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"):
-            nameurl = urllib.quote(path[-1].encode("utf-8"))
-            uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap),
-                                                   nameurl)
+        if cap:
+            if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"):
+                nameurl = urllib.quote(path[-1].encode("utf-8"))
+                uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap),
+                                                       nameurl)
+            else:
+                uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe=""))
+            ctx.fillSlots("cap", T.a(href=uri_link)[cap])
         else:
-            uri_link = "%s/uri/%s" % (root, urllib.quote(cap))
-        ctx.fillSlots("cap", T.a(href=uri_link)[cap])
+            ctx.fillSlots("cap", "")
         return ctx.tag
 
 class DeepSizeResults(rend.Page):
@@ -951,8 +963,10 @@ class ManifestStreamer(dirnode.DeepStats):
 
         if IDirectoryNode.providedBy(node):
             d["type"] = "directory"
-        else:
+        elif IFileNode.providedBy(node):
             d["type"] = "file"
+        else:
+            d["type"] = "unknown"
 
         v = node.get_verify_cap()
         if v:
@@ -1058,3 +1072,19 @@ class DeepCheckStreamer(dirnode.DeepStats):
         assert "\n" not in j
         self.req.write(j+"\n")
         return ""
+
+class UnknownNodeHandler(RenderMixin, rend.Page):
+
+    def __init__(self, client, node, parentnode=None, name=None):
+        rend.Page.__init__(self)
+        assert node
+        self.node = node
+
+    def render_GET(self, ctx):
+        req = IRequest(ctx)
+        t = get_arg(req, "t", "").strip()
+        if t == "info":
+            return MoreInfo(self.node)
+        raise WebError("GET unknown: can only do t=info, not t=%s" % t)
+
+

src/allmydata/web/filenode.py

@@ -6,10 +6,11 @@ from twisted.internet import defer
 from nevow import url, rend
 from nevow.inevow import IRequest
 
-from allmydata.interfaces import ExistingChildError
+from allmydata.interfaces import ExistingChildError, CannotPackUnknownNodeError
 from allmydata.monitor import Monitor
 from allmydata.immutable.upload import FileHandle
 from allmydata.immutable.filenode import LiteralFileNode
+from allmydata.unknown import UnknownNode
 from allmydata.util import log, base32
 
 from allmydata.web.common import text_plain, WebError, RenderMixin, \
@@ -55,7 +56,14 @@ class ReplaceMeMixin:
     def replace_me_with_a_childcap(self, req, client, replace):
         req.content.seek(0)
         childcap = req.content.read()
-        childnode = client.create_node_from_uri(childcap)
+        childnode = client.create_node_from_uri(childcap, childcap+"readonly")
+        if isinstance(childnode, UnknownNode):
+            # don't be willing to pack unknown nodes: we might accidentally
+            # put some write-authority into the rocap slot because we don't
+            # know how to diminish the URI they gave us. We don't even know
+            # if they gave us a readcap or a writecap.
+            msg = "cannot attach unknown node as child %s" % str(self.name)
+            raise CannotPackUnknownNodeError(msg)
         d = self.parentnode.set_node(self.name, childnode, overwrite=replace)
         d.addCallback(lambda res: childnode.get_uri())
         return d

src/allmydata/web/info.py

@@ -6,7 +6,7 @@ from nevow import rend, tags as T
 from nevow.inevow import IRequest
 
 from allmydata.util import base32
-from allmydata.interfaces import IDirectoryNode
+from allmydata.interfaces import IDirectoryNode, IFileNode
 from allmydata.web.common import getxmlfile
 from allmydata.mutable.common import UnrecoverableFileError # TODO: move
 
@@ -21,14 +21,16 @@ class MoreInfo(rend.Page):
 
     def get_type(self):
         node = self.original
-        si = node.get_storage_index()
         if IDirectoryNode.providedBy(node):
             return "directory"
-        if si:
-            if node.is_mutable():
-                return "mutable file"
-            return "immutable file"
-        return "LIT file"
+        if IFileNode.providedBy(node):
+            si = node.get_storage_index()
+            if si:
+                if node.is_mutable():
+                    return "mutable file"
+                return "immutable file"
+            return "LIT file"
+        return "unknown"
 
     def render_title(self, ctx, data):
         node = self.original
@@ -55,11 +57,15 @@ class MoreInfo(rend.Page):
         si = node.get_storage_index()
         if IDirectoryNode.providedBy(node):
             d = node._node.get_size_of_best_version()
-        elif node.is_mutable():
-            d = node.get_size_of_best_version()
+        elif IFileNode.providedBy(node):
+            if node.is_mutable():
+                d = node.get_size_of_best_version()
+            else:
+                # for immutable files and LIT files, we get the size from the
+                # URI
+                d = defer.succeed(node.get_size())
         else:
-            # for immutable files and LIT files, we get the size from the URI
-            d = defer.succeed(node.get_size())
+            d = defer.succeed("?")
         def _handle_unrecoverable(f):
             f.trap(UnrecoverableFileError)
             return "?"
@@ -92,15 +98,22 @@ class MoreInfo(rend.Page):
         node = self.original
         if IDirectoryNode.providedBy(node):
             node = node._node
-        if node.is_readonly():
+        if ((IDirectoryNode.providedBy(node) or IFileNode.providedBy(node))
+            and node.is_readonly()):
             return ""
-        return ctx.tag[node.get_uri()]
+        writecap = node.get_uri()
+        if not writecap:
+            return ""
+        return ctx.tag[writecap]
 
     def render_file_readcap(self, ctx, data):
         node = self.original
         if IDirectoryNode.providedBy(node):
             node = node._node
-        return ctx.tag[node.get_readonly_uri()]
+        readcap = node.get_readonly_uri()
+        if not readcap:
+            return ""
+        return ctx.tag[readcap]
 
     def render_file_verifycap(self, ctx, data):
         node = self.original
@@ -122,10 +135,14 @@ class MoreInfo(rend.Page):
         node = self.original
         if IDirectoryNode.providedBy(node):
             node = node._node
+        elif IFileNode.providedBy(node):
+            pass
+        else:
+            return ""
         root = self.get_root(ctx)
         quoted_uri = urllib.quote(node.get_uri())
         text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
-        return ctx.tag[text_plain_url]
+        return T.li["Raw data as ", T.a(href=text_plain_url)["text/plain"]]
 
     def render_is_checkable(self, ctx, data):
         node = self.original
@@ -167,7 +184,8 @@ class MoreInfo(rend.Page):
         node = self.original
         if IDirectoryNode.providedBy(node):
             return ""
-        if node.is_mutable() and not node.is_readonly():
+        if (IFileNode.providedBy(node)
+            and node.is_mutable() and not node.is_readonly()):
             return ctx.tag
         return ""
 

src/allmydata/web/info.xhtml

@@ -45,7 +45,7 @@
     </tr>
   </table></li>
   <li><a href="?t=json">JSON</a></li>
-  <li>Raw data as <a><n:attr name="href" n:render="raw_link" />text/plain</a></li>
+  <li n:render="raw_link" />
 </ul>
 
 <div n:render="is_checkable">