diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt
index 6abbd42a3..1353f233a 100644
--- a/docs/frontends/webapi.txt
+++ b/docs/frontends/webapi.txt
@@ -1232,6 +1232,7 @@ POST $DIRURL?t=start-deep-stats (must add &ophandle=XYZ)
count-literal-files: same, for LIT files (data contained inside the URI)
count-files: sum of the above three
count-directories: count of directories
+ count-unknown: count of unrecognized objects (perhaps from the future)
size-immutable-files: total bytes for all CHK files in the set, =deep-size
size-mutable-files (TODO): same, for current version of all mutable files
size-literal-files: same, for LIT files
diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 08d559e55..8c1dc21fb 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -20,9 +20,10 @@ from allmydata.introducer.client import IntroducerClient
from allmydata.util import hashutil, base32, pollmixin, cachedir, log
from allmydata.util.abbreviate import parse_abbreviated_size
from allmydata.util.time_format import parse_duration, parse_date
-from allmydata.uri import LiteralFileURI
+from allmydata.uri import LiteralFileURI, UnknownURI
from allmydata.dirnode import NewDirectoryNode
from allmydata.mutable.filenode import MutableFileNode
+from allmydata.unknown import UnknownNode
from allmydata.stats import StatsProvider
from allmydata.history import History
from allmydata.interfaces import IURI, INewDirectoryURI, IStatsProducer, \
@@ -404,11 +405,17 @@ class Client(node.Node, pollmixin.PollMixin):
# dirnodes. The first takes a URI and produces a filenode or (new-style)
# dirnode. The other three create brand-new filenodes/dirnodes.
- def create_node_from_uri(self, u, readcap=None):
+ def create_node_from_uri(self, writecap, readcap=None):
# this returns synchronously.
+ u = writecap or readcap
if not u:
- u = readcap
+ # maybe the writecap was hidden because we're in a readonly
+ # directory, and the future cap format doesn't have a readcap, or
+ # something.
+ return UnknownNode(writecap, readcap)
u = IURI(u)
+ if isinstance(u, UnknownURI):
+ return UnknownNode(writecap, readcap)
u_s = u.to_string()
if u_s not in self._node_cache:
if IReadonlyNewDirectoryURI.providedBy(u):
@@ -427,13 +434,12 @@ class Client(node.Node, pollmixin.PollMixin):
else:
assert IMutableFileURI.providedBy(u), u
node = MutableFileNode(self).init_from_uri(u)
- self._node_cache[u_s] = node
+ self._node_cache[u_s] = node # note: WeakValueDictionary
return self._node_cache[u_s]
def create_empty_dirnode(self):
- n = NewDirectoryNode(self)
- d = n.create(self._generate_pubprivkeys, self.DEFAULT_MUTABLE_KEYSIZE)
- d.addCallback(lambda res: n)
+ d = self.create_mutable_file()
+ d.addCallback(NewDirectoryNode.create_with_mutablefile, self)
return d
def create_mutable_file(self, contents="", keysize=None):
diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py
index b8113b3aa..731431992 100644
--- a/src/allmydata/dirnode.py
+++ b/src/allmydata/dirnode.py
@@ -7,9 +7,11 @@ from foolscap.api import fireEventually
import simplejson
from allmydata.mutable.common import NotMutableError
from allmydata.mutable.filenode import MutableFileNode
+from allmydata.unknown import UnknownNode
from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\
IURI, IFileNode, IMutableFileURI, IFilesystemNode, \
- ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable
+ ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable, \
+ CannotPackUnknownNodeError
from allmydata.check_results import DeepCheckResults, \
DeepCheckAndRepairResults
from allmydata.monitor import Monitor
@@ -137,6 +139,12 @@ class NewDirectoryNode:
self._node.init_from_uri(self._uri.get_filenode_uri())
return self
+ @classmethod
+ def create_with_mutablefile(cls, filenode, client):
+ self = cls(client)
+ self._node = filenode
+ return self._filenode_created(filenode)
+
def create(self, keypair_generator=None, keysize=None):
"""
Returns a deferred that eventually fires with self once the directory
@@ -226,9 +234,7 @@ class NewDirectoryNode:
for name in sorted(children.keys()):
child, metadata = children[name]
assert isinstance(name, unicode)
- assert (IFileNode.providedBy(child)
- or IMutableFileNode.providedBy(child)
- or IDirectoryNode.providedBy(child)), (name,child)
+ assert IFilesystemNode.providedBy(child), (name,child)
assert isinstance(metadata, dict)
rwcap = child.get_uri() # might be RO if the child is not writeable
if rwcap is None:
@@ -381,6 +387,13 @@ class NewDirectoryNode:
precondition(isinstance(name, unicode), name)
precondition(isinstance(child_uri, str), child_uri)
child_node = self._create_node(child_uri, None)
+ if isinstance(child_node, UnknownNode):
+ # don't be willing to pack unknown nodes: we might accidentally
+ # put some write-authority into the rocap slot because we don't
+ # know how to diminish the URI they gave us. We don't even know
+ # if they gave us a readcap or a writecap.
+ msg = "cannot pack unknown node as child %s" % str(name)
+ raise CannotPackUnknownNodeError(msg)
d = self.set_node(name, child_node, metadata, overwrite)
d.addCallback(lambda res: child_node)
return d
@@ -397,7 +410,11 @@ class NewDirectoryNode:
assert len(e) == 3
name, child_uri, metadata = e
assert isinstance(name, unicode)
- a.set_node(name, self._create_node(child_uri, None), metadata)
+ child_node = self._create_node(child_uri, None)
+ if isinstance(child_node, UnknownNode):
+ msg = "cannot pack unknown node as child %s" % str(name)
+ raise CannotPackUnknownNodeError(msg)
+ a.set_node(name, child_node, metadata)
return self._node.modify(a.modify)
def set_node(self, name, child, metadata=None, overwrite=True):
@@ -560,12 +577,15 @@ class NewDirectoryNode:
dirkids = []
filekids = []
for name, (child, metadata) in sorted(children.iteritems()):
+ childpath = path + [name]
+ if isinstance(child, UnknownNode):
+ walker.add_node(child, childpath)
+ continue
verifier = child.get_verify_cap()
# allow LIT files (for which verifier==None) to be processed
if (verifier is not None) and (verifier in found):
continue
found.add(verifier)
- childpath = path + [name]
if IDirectoryNode.providedBy(child):
dirkids.append( (child, childpath) )
else:
@@ -618,6 +638,7 @@ class DeepStats:
"count-literal-files",
"count-files",
"count-directories",
+ "count-unknown",
"size-immutable-files",
#"size-mutable-files",
"size-literal-files",
@@ -640,7 +661,9 @@ class DeepStats:
monitor.set_status(self.get_results())
def add_node(self, node, childpath):
- if IDirectoryNode.providedBy(node):
+ if isinstance(node, UnknownNode):
+ self.add("count-unknown")
+ elif IDirectoryNode.providedBy(node):
self.add("count-directories")
elif IMutableFileNode.providedBy(node):
self.add("count-files")
diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 86e831243..42188e7be 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -478,6 +478,9 @@ class INewDirectoryURI(Interface):
class IReadonlyNewDirectoryURI(Interface):
pass
+class CannotPackUnknownNodeError(Exception):
+ """UnknownNodes (using filecaps from the future that we don't understand)
+ cannot yet be copied safely, so I refuse to copy them."""
class IFilesystemNode(Interface):
def get_uri():
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index b652fbab5..38cbdc226 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -8,7 +8,8 @@ import allmydata
from allmydata import client
from allmydata.storage_client import StorageFarmBroker
from allmydata.introducer.client import IntroducerClient
-from allmydata.util import base32
+from allmydata.util import base32, fileutil
+from allmydata.interfaces import IFilesystemNode, IFileNode, IDirectoryNode
from foolscap.api import flushEventualQueue
import common_util as testutil
@@ -234,3 +235,39 @@ class Run(unittest.TestCase, testutil.StallMixin):
d.addCallback(_restart)
return d
+class NodeMaker(unittest.TestCase):
+ def test_maker(self):
+ basedir = "client/NodeMaker/maker"
+ fileutil.make_dirs(basedir)
+ f = open(os.path.join(basedir, "tahoe.cfg"), "w")
+ f.write(BASECONFIG)
+ f.close()
+ c = client.Client(basedir)
+
+ n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
+ self.failUnless(IFilesystemNode.providedBy(n))
+ self.failUnless(IFileNode.providedBy(n))
+ self.failIf(IDirectoryNode.providedBy(n))
+ self.failUnless(n.is_readonly())
+ self.failIf(n.is_mutable())
+
+ n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+ self.failUnless(IFilesystemNode.providedBy(n))
+ self.failIf(IFileNode.providedBy(n))
+ self.failUnless(IDirectoryNode.providedBy(n))
+ self.failIf(n.is_readonly())
+ self.failUnless(n.is_mutable())
+
+ n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+ self.failUnless(IFilesystemNode.providedBy(n))
+ self.failIf(IFileNode.providedBy(n))
+ self.failUnless(IDirectoryNode.providedBy(n))
+ self.failUnless(n.is_readonly())
+ self.failUnless(n.is_mutable())
+
+ future = "x-tahoe-crazy://future_cap_format."
+ n = c.create_node_from_uri(future)
+ self.failUnless(IFilesystemNode.providedBy(n))
+ self.failIf(IFileNode.providedBy(n))
+ self.failIf(IDirectoryNode.providedBy(n))
+ self.failUnlessEqual(n.get_uri(), future)
diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py
index 0321726a8..555d647e7 100644
--- a/src/allmydata/test/test_dirnode.py
+++ b/src/allmydata/test/test_dirnode.py
@@ -4,11 +4,12 @@ from zope.interface import implements
from twisted.trial import unittest
from twisted.internet import defer
from allmydata import uri, dirnode
+from allmydata.client import Client
from allmydata.immutable import upload
from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \
ExistingChildError, NoSuchChildError, \
- IDeepCheckResults, IDeepCheckAndRepairResults
+ IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError
from allmydata.mutable.filenode import MutableFileNode
from allmydata.mutable.common import UncoordinatedWriteError
from allmydata.util import hashutil, base32
@@ -17,6 +18,7 @@ from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
FakeDirectoryNode, create_chk_filenode, ErrorMixin
from allmydata.test.no_network import GridTestMixin
from allmydata.check_results import CheckResults, CheckAndRepairResults
+from allmydata.unknown import UnknownNode
import common_util as testutil
# to test dirnode.py, we want to construct a tree of real DirectoryNodes that
@@ -715,6 +717,83 @@ class Dirnode(unittest.TestCase,
d.addErrback(self.explain_error)
return d
+class FakeMutableFile:
+ counter = 0
+ def __init__(self, initial_contents=""):
+ self.data = initial_contents
+ counter = FakeMutableFile.counter
+ FakeMutableFile.counter += 1
+ writekey = hashutil.ssk_writekey_hash(str(counter))
+ fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
+ self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
+ def get_uri(self):
+ return self.uri.to_string()
+ def download_best_version(self):
+ return defer.succeed(self.data)
+ def get_writekey(self):
+ return "writekey"
+ def is_readonly(self):
+ return False
+ def is_mutable(self):
+ return True
+ def modify(self, modifier):
+ self.data = modifier(self.data, None, True)
+ return defer.succeed(None)
+
+class FakeClient2(Client):
+ def __init__(self):
+ pass
+ def create_mutable_file(self, initial_contents=""):
+ return defer.succeed(FakeMutableFile(initial_contents))
+
+class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
+ def setUp(self):
+ self.client = FakeClient2()
+
+ def test_from_future(self):
+ # create a dirnode that contains unknown URI types, and make sure we
+ # tolerate them properly. Since dirnodes aren't allowed to add
+ # unknown node types, we have to be tricky.
+ d = self.client.create_empty_dirnode()
+ future_writecap = "x-tahoe-crazy://I_am_from_the_future."
+ future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
+ future_node = UnknownNode(future_writecap, future_readcap)
+ def _then(n):
+ self._node = n
+ return n.set_node(u"future", future_node)
+ d.addCallback(_then)
+
+ # we should be prohibited from adding an unknown URI to a directory,
+ # since we don't know how to diminish the cap to a readcap (for the
+ # dirnode's rocap slot), and we don't want to accidentally grant
+ # write access to a holder of the dirnode's readcap.
+ d.addCallback(lambda ign:
+ self.shouldFail(CannotPackUnknownNodeError,
+ "copy unknown",
+ "cannot pack unknown node as child add",
+ self._node.set_uri, u"add", future_writecap))
+ d.addCallback(lambda ign: self._node.list())
+ def _check(children):
+ self.failUnlessEqual(len(children), 1)
+ (fn, metadata) = children[u"future"]
+ self.failUnless(isinstance(fn, UnknownNode), fn)
+ self.failUnlessEqual(fn.get_uri(), future_writecap)
+ self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
+ # but we *should* be allowed to copy this node, because the
+ # UnknownNode contains all the information that was in the
+ # original directory (readcap and writecap), so we're preserving
+ # everything.
+ return self._node.set_node(u"copy", fn)
+ d.addCallback(_check)
+ d.addCallback(lambda ign: self._node.list())
+ def _check2(children):
+ self.failUnlessEqual(len(children), 2)
+ (fn, metadata) = children[u"copy"]
+ self.failUnless(isinstance(fn, UnknownNode), fn)
+ self.failUnlessEqual(fn.get_uri(), future_writecap)
+ self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
+ return d
+
class DeepStats(unittest.TestCase):
timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
def test_stats(self):
diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py
index e81d466d3..7ac63469d 100644
--- a/src/allmydata/test/test_uri.py
+++ b/src/allmydata/test/test_uri.py
@@ -61,7 +61,7 @@ class Compare(unittest.TestCase):
def test_is_uri(self):
lit1 = uri.LiteralFileURI("some data").to_string()
self.failUnless(uri.is_uri(lit1))
- self.failIf(uri.is_uri("this is not a uri"))
+ self.failIf(uri.is_uri(None))
class CHKFile(unittest.TestCase):
def test_pack(self):
@@ -175,10 +175,12 @@ class Extension(unittest.TestCase):
readable = uri.unpack_extension_readable(ext)
class Invalid(unittest.TestCase):
- def test_create_invalid(self):
- not_uri = "I am not a URI"
- self.failUnlessRaises(TypeError, uri.from_string, not_uri)
-
+ def test_from_future(self):
+ # any URI type that we don't recognize should be treated as unknown
+ future_uri = "I am a URI from the future. Whatever you do, don't "
+ u = uri.from_string(future_uri)
+ self.failUnless(isinstance(u, uri.UnknownURI))
+ self.failUnlessEqual(u.to_string(), future_uri)
class Constraint(unittest.TestCase):
def test_constraint(self):
diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py
index c6638ce70..6a1a8638e 100644
--- a/src/allmydata/test/test_web.py
+++ b/src/allmydata/test/test_web.py
@@ -11,6 +11,7 @@ from allmydata import interfaces, uri, webish
from allmydata.storage.shares import get_share_file
from allmydata.storage_client import StorageFarmBroker
from allmydata.immutable import upload, download
+from allmydata.unknown import UnknownNode
from allmydata.web import status, common
from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
from allmydata.util import fileutil, base32
@@ -2781,6 +2782,73 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
d.addErrback(self.explain_web_error)
return d
+ def test_unknown(self):
+ self.basedir = "web/Grid/unknown"
+ self.set_up_grid()
+ c0 = self.g.clients[0]
+ self.uris = {}
+ self.fileurls = {}
+
+ future_writecap = "x-tahoe-crazy://I_am_from_the_future."
+ future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
+ # the future cap format may contain slashes, which must be tolerated
+ expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap,
+ safe="")
+ future_node = UnknownNode(future_writecap, future_readcap)
+
+ d = c0.create_empty_dirnode()
+ def _stash_root_and_create_file(n):
+ self.rootnode = n
+ self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
+ self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
+ return self.rootnode.set_node(u"future", future_node)
+ d.addCallback(_stash_root_and_create_file)
+ # make sure directory listing tolerates unknown nodes
+ d.addCallback(lambda ign: self.GET(self.rooturl))
+ def _check_html(res):
+ self.failUnlessIn("
future | ", res)
+ # find the More Info link for "future", should be relative
+ mo = re.search(r'More Info', res)
+ info_url = mo.group(1)
+ self.failUnlessEqual(info_url, "future?t=info")
+
+ d.addCallback(_check_html)
+ d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
+ def _check_json(res, expect_writecap):
+ data = simplejson.loads(res)
+ self.failUnlessEqual(data[0], "dirnode")
+ f = data[1]["children"]["future"]
+ self.failUnlessEqual(f[0], "unknown")
+ if expect_writecap:
+ self.failUnlessEqual(f[1]["rw_uri"], future_writecap)
+ else:
+ self.failIfIn("rw_uri", f[1])
+ self.failUnlessEqual(f[1]["ro_uri"], future_readcap)
+ self.failUnless("metadata" in f[1])
+ d.addCallback(_check_json, expect_writecap=True)
+ d.addCallback(lambda ign: self.GET(expected_info_url))
+ def _check_info(res, expect_readcap):
+ self.failUnlessIn("Object Type: unknown", res)
+ self.failUnlessIn(future_writecap, res)
+ if expect_readcap:
+ self.failUnlessIn(future_readcap, res)
+ self.failIfIn("Raw data as", res)
+ self.failIfIn("Directory writecap", res)
+ self.failIfIn("Checker Operations", res)
+ self.failIfIn("Mutable File Operations", res)
+ self.failIfIn("Directory Operations", res)
+ d.addCallback(_check_info, expect_readcap=False)
+ d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info"))
+ d.addCallback(_check_info, expect_readcap=True)
+
+ # and make sure that a read-only version of the directory can be
+ # rendered too. This version will not have future_writecap
+ d.addCallback(lambda ign: self.GET(self.rourl))
+ d.addCallback(_check_html)
+ d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
+ d.addCallback(_check_json, expect_writecap=False)
+ return d
+
def test_deep_check(self):
self.basedir = "web/Grid/deep_check"
self.set_up_grid()
@@ -2809,6 +2877,13 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
convergence="")))
d.addCallback(_stash_uri, "sick")
+ # this tests that deep-check and stream-manifest will ignore
+ # UnknownNode instances. Hopefully this will also cover deep-stats.
+ future_writecap = "x-tahoe-crazy://I_am_from_the_future."
+ future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
+ future_node = UnknownNode(future_writecap, future_readcap)
+ d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node))
+
def _clobber_shares(ignored):
self.delete_shares_numbered(self.uris["sick"], [0,1])
d.addCallback(_clobber_shares)
@@ -2817,13 +2892,19 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
# root/good
# root/small
# root/sick
+ # root/future
d.addCallback(self.CHECK, "root", "t=stream-deep-check")
def _done(res):
- units = [simplejson.loads(line)
- for line in res.splitlines()
- if line]
- self.failUnlessEqual(len(units), 4+1)
+ try:
+ units = [simplejson.loads(line)
+ for line in res.splitlines()
+ if line]
+ except ValueError:
+ print "response is:", res
+ print "undecodeable line was '%s'" % line
+ raise
+ self.failUnlessEqual(len(units), 5+1)
# should be parent-first
u0 = units[0]
self.failUnlessEqual(u0["path"], [])
@@ -2844,8 +2925,27 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
self.failUnlessEqual(s["count-immutable-files"], 2)
self.failUnlessEqual(s["count-literal-files"], 1)
self.failUnlessEqual(s["count-directories"], 1)
+ self.failUnlessEqual(s["count-unknown"], 1)
d.addCallback(_done)
+ d.addCallback(self.CHECK, "root", "t=stream-manifest")
+ def _check_manifest(res):
+ self.failUnless(res.endswith("\n"))
+ units = [simplejson.loads(t) for t in res[:-1].split("\n")]
+ self.failUnlessEqual(len(units), 5+1)
+ self.failUnlessEqual(units[-1]["type"], "stats")
+ first = units[0]
+ self.failUnlessEqual(first["path"], [])
+ self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
+ self.failUnlessEqual(first["type"], "directory")
+ stats = units[-1]["stats"]
+ self.failUnlessEqual(stats["count-immutable-files"], 2)
+ self.failUnlessEqual(stats["count-literal-files"], 1)
+ self.failUnlessEqual(stats["count-mutable-files"], 0)
+ self.failUnlessEqual(stats["count-immutable-files"], 2)
+ self.failUnlessEqual(stats["count-unknown"], 1)
+ d.addCallback(_check_manifest)
+
# now add root/subdir and root/subdir/grandchild, then make subdir
# unrecoverable, then see what happens
@@ -2866,6 +2966,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
# root/good
# root/small
# root/sick
+ # root/future
# root/subdir [unrecoverable]
# root/subdir/grandchild
@@ -2888,7 +2989,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
error_line)
self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
units = [simplejson.loads(line) for line in lines[:first_error]]
- self.failUnlessEqual(len(units), 5) # includes subdir
+ self.failUnlessEqual(len(units), 6) # includes subdir
last_unit = units[-1]
self.failUnlessEqual(last_unit["path"], ["subdir"])
d.addCallback(_check_broken_manifest)
@@ -2909,7 +3010,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
error_line)
self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
units = [simplejson.loads(line) for line in lines[:first_error]]
- self.failUnlessEqual(len(units), 5) # includes subdir
+ self.failUnlessEqual(len(units), 6) # includes subdir
last_unit = units[-1]
self.failUnlessEqual(last_unit["path"], ["subdir"])
r = last_unit["check-results"]["results"]
diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py
new file mode 100644
index 000000000..c64e63685
--- /dev/null
+++ b/src/allmydata/unknown.py
@@ -0,0 +1,25 @@
+from zope.interface import implements
+from twisted.internet import defer
+from allmydata.interfaces import IFilesystemNode
+
+class UnknownNode:
+ implements(IFilesystemNode)
+ def __init__(self, writecap, readcap):
+ assert writecap is None or isinstance(writecap, str)
+ self.writecap = writecap
+ assert readcap is None or isinstance(readcap, str)
+ self.readcap = readcap
+ def get_uri(self):
+ return self.writecap
+ def get_readonly_uri(self):
+ return self.readcap
+ def get_storage_index(self):
+ return None
+ def get_verify_cap(self):
+ return None
+ def get_repair_cap(self):
+ return None
+ def check(self, monitor, verify, add_lease):
+ return defer.succeed(None)
+ def check_and_repair(self, monitor, verify, add_lease):
+ return defer.succeed(None)
diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py
index 297fe93c8..9be92a418 100644
--- a/src/allmydata/uri.py
+++ b/src/allmydata/uri.py
@@ -428,10 +428,16 @@ class NewDirectoryURIVerifier(_NewDirectoryBaseURI):
def get_filenode_uri(self):
return self._filenode_uri
-
+class UnknownURI:
+ def __init__(self, uri):
+ self._uri = uri
+ def to_string(self):
+ return self._uri
def from_string(s):
- if s.startswith('URI:CHK:'):
+ if not isinstance(s, str):
+ raise TypeError("unknown URI type: %s.." % str(s)[:100])
+ elif s.startswith('URI:CHK:'):
return CHKFileURI.init_from_string(s)
elif s.startswith('URI:CHK-Verifier:'):
return CHKFileVerifierURI.init_from_string(s)
@@ -449,8 +455,7 @@ def from_string(s):
return ReadonlyNewDirectoryURI.init_from_string(s)
elif s.startswith('URI:DIR2-Verifier:'):
return NewDirectoryURIVerifier.init_from_string(s)
- else:
- raise TypeError("unknown URI type: %s.." % s[:12])
+ return UnknownURI(s)
registerAdapter(from_string, str, IURI)
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 5c83b51b9..1bfd38262 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -15,7 +15,7 @@ from foolscap.api import fireEventually
from allmydata.util import base32, time_format
from allmydata.uri import from_string_dirnode
from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \
- ExistingChildError, NoSuchChildError
+ IFilesystemNode, ExistingChildError, NoSuchChildError
from allmydata.monitor import Monitor, OperationCancelledError
from allmydata import dirnode
from allmydata.web.common import text_plain, WebError, \
@@ -46,7 +46,7 @@ def make_handler_for(node, client, parentnode=None, name=None):
return FileNodeHandler(client, node, parentnode, name)
if IDirectoryNode.providedBy(node):
return DirectoryNodeHandler(client, node, parentnode, name)
- raise WebError("Cannot provide handler for '%s'" % node)
+ return UnknownNodeHandler(client, node, parentnode, name)
class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
addSlash = True
@@ -617,11 +617,9 @@ class DirectoryAsHTML(rend.Page):
times.append("m: " + mtime)
ctx.fillSlots("times", times)
- assert (IFileNode.providedBy(target)
- or IDirectoryNode.providedBy(target)
- or IMutableFileNode.providedBy(target)), target
-
- quoted_uri = urllib.quote(target.get_uri())
+ assert IFilesystemNode.providedBy(target), target
+ writecap = target.get_uri() or ""
+ quoted_uri = urllib.quote(writecap, safe="") # escape slashes too
if IMutableFileNode.providedBy(target):
# to prevent javascript in displayed .html files from stealing a
@@ -650,7 +648,7 @@ class DirectoryAsHTML(rend.Page):
elif IDirectoryNode.providedBy(target):
# directory
- uri_link = "%s/uri/%s/" % (root, urllib.quote(target.get_uri()))
+ uri_link = "%s/uri/%s/" % (root, urllib.quote(writecap))
ctx.fillSlots("filename",
T.a(href=uri_link)[html.escape(name)])
if target.is_readonly():
@@ -661,6 +659,15 @@ class DirectoryAsHTML(rend.Page):
ctx.fillSlots("size", "-")
info_link = "%s/uri/%s/?t=info" % (root, quoted_uri)
+ else:
+ # unknown
+ ctx.fillSlots("filename", html.escape(name))
+ ctx.fillSlots("type", "?")
+ ctx.fillSlots("size", "-")
+ # use a directory-relative info link, so we can extract both the
+ # writecap and the readcap
+ info_link = "%s?t=info" % urllib.quote(name)
+
ctx.fillSlots("info", T.a(href=info_link)["More Info"])
return ctx.tag
@@ -727,20 +734,23 @@ def DirectoryJSONMetadata(ctx, dirnode):
def _got(children):
kids = {}
for name, (childnode, metadata) in children.iteritems():
- if childnode.is_readonly():
- rw_uri = None
- ro_uri = childnode.get_uri()
- else:
- rw_uri = childnode.get_uri()
- ro_uri = childnode.get_readonly_uri()
+ assert IFilesystemNode.providedBy(childnode), childnode
+ rw_uri = childnode.get_uri()
+ ro_uri = childnode.get_readonly_uri()
+ if (IDirectoryNode.providedBy(childnode)
+ or IFileNode.providedBy(childnode)):
+ if childnode.is_readonly():
+ rw_uri = None
if IFileNode.providedBy(childnode):
kiddata = ("filenode", {'size': childnode.get_size(),
- 'metadata': metadata,
+ 'mutable': childnode.is_mutable(),
})
+ elif IDirectoryNode.providedBy(childnode):
+ kiddata = ("dirnode", {'mutable': childnode.is_mutable(),
+ })
else:
- assert IDirectoryNode.providedBy(childnode), (childnode,
- children,)
- kiddata = ("dirnode", {'metadata': metadata})
+ kiddata = ("unknown", {})
+ kiddata[1]["metadata"] = metadata
if ro_uri:
kiddata[1]["ro_uri"] = ro_uri
if rw_uri:
@@ -748,7 +758,6 @@ def DirectoryJSONMetadata(ctx, dirnode):
verifycap = childnode.get_verify_cap()
if verifycap:
kiddata[1]['verify_uri'] = verifycap.to_string()
- kiddata[1]['mutable'] = childnode.is_mutable()
kids[name] = kiddata
if dirnode.is_readonly():
drw_uri = None
@@ -879,13 +888,16 @@ class ManifestResults(rend.Page, ReloadMixin):
ctx.fillSlots("path", self.slashify_path(path))
root = get_root(ctx)
# TODO: we need a clean consistent way to get the type of a cap string
- if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"):
- nameurl = urllib.quote(path[-1].encode("utf-8"))
- uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap),
- nameurl)
+ if cap:
+ if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"):
+ nameurl = urllib.quote(path[-1].encode("utf-8"))
+ uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap),
+ nameurl)
+ else:
+ uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe=""))
+ ctx.fillSlots("cap", T.a(href=uri_link)[cap])
else:
- uri_link = "%s/uri/%s" % (root, urllib.quote(cap))
- ctx.fillSlots("cap", T.a(href=uri_link)[cap])
+ ctx.fillSlots("cap", "")
return ctx.tag
class DeepSizeResults(rend.Page):
@@ -951,8 +963,10 @@ class ManifestStreamer(dirnode.DeepStats):
if IDirectoryNode.providedBy(node):
d["type"] = "directory"
- else:
+ elif IFileNode.providedBy(node):
d["type"] = "file"
+ else:
+ d["type"] = "unknown"
v = node.get_verify_cap()
if v:
@@ -1058,3 +1072,19 @@ class DeepCheckStreamer(dirnode.DeepStats):
assert "\n" not in j
self.req.write(j+"\n")
return ""
+
+class UnknownNodeHandler(RenderMixin, rend.Page):
+
+ def __init__(self, client, node, parentnode=None, name=None):
+ rend.Page.__init__(self)
+ assert node
+ self.node = node
+
+ def render_GET(self, ctx):
+ req = IRequest(ctx)
+ t = get_arg(req, "t", "").strip()
+ if t == "info":
+ return MoreInfo(self.node)
+ raise WebError("GET unknown: can only do t=info, not t=%s" % t)
+
+
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index 727d9b68b..05c122791 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -6,10 +6,11 @@ from twisted.internet import defer
from nevow import url, rend
from nevow.inevow import IRequest
-from allmydata.interfaces import ExistingChildError
+from allmydata.interfaces import ExistingChildError, CannotPackUnknownNodeError
from allmydata.monitor import Monitor
from allmydata.immutable.upload import FileHandle
from allmydata.immutable.filenode import LiteralFileNode
+from allmydata.unknown import UnknownNode
from allmydata.util import log, base32
from allmydata.web.common import text_plain, WebError, RenderMixin, \
@@ -55,7 +56,14 @@ class ReplaceMeMixin:
def replace_me_with_a_childcap(self, req, client, replace):
req.content.seek(0)
childcap = req.content.read()
- childnode = client.create_node_from_uri(childcap)
+ childnode = client.create_node_from_uri(childcap, childcap+"readonly")
+ if isinstance(childnode, UnknownNode):
+ # don't be willing to pack unknown nodes: we might accidentally
+ # put some write-authority into the rocap slot because we don't
+ # know how to diminish the URI they gave us. We don't even know
+ # if they gave us a readcap or a writecap.
+ msg = "cannot attach unknown node as child %s" % str(self.name)
+ raise CannotPackUnknownNodeError(msg)
d = self.parentnode.set_node(self.name, childnode, overwrite=replace)
d.addCallback(lambda res: childnode.get_uri())
return d
diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py
index e7c42e60b..edaab44bb 100644
--- a/src/allmydata/web/info.py
+++ b/src/allmydata/web/info.py
@@ -6,7 +6,7 @@ from nevow import rend, tags as T
from nevow.inevow import IRequest
from allmydata.util import base32
-from allmydata.interfaces import IDirectoryNode
+from allmydata.interfaces import IDirectoryNode, IFileNode
from allmydata.web.common import getxmlfile
from allmydata.mutable.common import UnrecoverableFileError # TODO: move
@@ -21,14 +21,16 @@ class MoreInfo(rend.Page):
def get_type(self):
node = self.original
- si = node.get_storage_index()
if IDirectoryNode.providedBy(node):
return "directory"
- if si:
- if node.is_mutable():
- return "mutable file"
- return "immutable file"
- return "LIT file"
+ if IFileNode.providedBy(node):
+ si = node.get_storage_index()
+ if si:
+ if node.is_mutable():
+ return "mutable file"
+ return "immutable file"
+ return "LIT file"
+ return "unknown"
def render_title(self, ctx, data):
node = self.original
@@ -55,11 +57,15 @@ class MoreInfo(rend.Page):
si = node.get_storage_index()
if IDirectoryNode.providedBy(node):
d = node._node.get_size_of_best_version()
- elif node.is_mutable():
- d = node.get_size_of_best_version()
+ elif IFileNode.providedBy(node):
+ if node.is_mutable():
+ d = node.get_size_of_best_version()
+ else:
+ # for immutable files and LIT files, we get the size from the
+ # URI
+ d = defer.succeed(node.get_size())
else:
- # for immutable files and LIT files, we get the size from the URI
- d = defer.succeed(node.get_size())
+ d = defer.succeed("?")
def _handle_unrecoverable(f):
f.trap(UnrecoverableFileError)
return "?"
@@ -92,15 +98,22 @@ class MoreInfo(rend.Page):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
- if node.is_readonly():
+ if ((IDirectoryNode.providedBy(node) or IFileNode.providedBy(node))
+ and node.is_readonly()):
return ""
- return ctx.tag[node.get_uri()]
+ writecap = node.get_uri()
+ if not writecap:
+ return ""
+ return ctx.tag[writecap]
def render_file_readcap(self, ctx, data):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
- return ctx.tag[node.get_readonly_uri()]
+ readcap = node.get_readonly_uri()
+ if not readcap:
+ return ""
+ return ctx.tag[readcap]
def render_file_verifycap(self, ctx, data):
node = self.original
@@ -122,10 +135,14 @@ class MoreInfo(rend.Page):
node = self.original
if IDirectoryNode.providedBy(node):
node = node._node
+ elif IFileNode.providedBy(node):
+ pass
+ else:
+ return ""
root = self.get_root(ctx)
quoted_uri = urllib.quote(node.get_uri())
text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
- return ctx.tag[text_plain_url]
+ return T.li["Raw data as ", T.a(href=text_plain_url)["text/plain"]]
def render_is_checkable(self, ctx, data):
node = self.original
@@ -167,7 +184,8 @@ class MoreInfo(rend.Page):
node = self.original
if IDirectoryNode.providedBy(node):
return ""
- if node.is_mutable() and not node.is_readonly():
+ if (IFileNode.providedBy(node)
+ and node.is_mutable() and not node.is_readonly()):
return ctx.tag
return ""
diff --git a/src/allmydata/web/info.xhtml b/src/allmydata/web/info.xhtml
index 5e0a09c16..9237f38fe 100644
--- a/src/allmydata/web/info.xhtml
+++ b/src/allmydata/web/info.xhtml
@@ -45,7 +45,7 @@
JSON
- Raw data as text/plain
+