mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-01-01 10:46:42 +00:00
fc3bd0c987
Now upload or encode methods take a required argument named "convergence" which can be either None, indicating no convergent encryption at all, or a string, which is the "added secret" to be mixed in to the content hash key. If you want traditional convergent encryption behavior, set the added secret to be the empty string. This patch also renames "content hash key" to "convergent encryption" in a argument names and variable names. (A different and larger renaming is needed in order to clarify that Tahoe supports immutable files which are not encrypted content-hash-key a.k.a. convergent encryption.) This patch also changes a few unit tests to use non-convergent encryption, because it doesn't matter for what they are testing and non-convergent encryption is slightly faster.
478 lines
22 KiB
Python
478 lines
22 KiB
Python
|
|
import time
|
|
from zope.interface import implements
|
|
from twisted.trial import unittest
|
|
from allmydata import uri, dirnode, upload
|
|
from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
|
|
INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode
|
|
from allmydata.util import hashutil, testutil
|
|
from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
|
|
FakeDirectoryNode, create_chk_filenode
|
|
from twisted.internet import defer, reactor
|
|
|
|
# to test dirnode.py, we want to construct a tree of real DirectoryNodes that
|
|
# contain pointers to fake files. We start with a fake MutableFileNode that
|
|
# stores all of its data in a static table.
|
|
|
|
class Marker:
|
|
implements(IFileNode, IMutableFileNode) # sure, why not
|
|
def __init__(self, nodeuri):
|
|
if not isinstance(nodeuri, str):
|
|
nodeuri = nodeuri.to_string()
|
|
self.nodeuri = nodeuri
|
|
si = hashutil.tagged_hash("tag1", nodeuri)[:16]
|
|
fp = hashutil.tagged_hash("tag2", nodeuri)
|
|
self.verifieruri = uri.SSKVerifierURI(storage_index=si,
|
|
fingerprint=fp).to_string()
|
|
def get_uri(self):
|
|
return self.nodeuri
|
|
def get_readonly_uri(self):
|
|
return self.nodeuri
|
|
def get_verifier(self):
|
|
return self.verifieruri
|
|
|
|
# dirnode requires three methods from the client: upload(),
|
|
# create_node_from_uri(), and create_empty_dirnode(). Of these, upload() is
|
|
# only used by the convenience composite method add_file().
|
|
|
|
class FakeClient:
|
|
implements(IClient)
|
|
|
|
def upload(self, uploadable):
|
|
d = uploadable.get_size()
|
|
d.addCallback(lambda size: uploadable.read(size))
|
|
def _got_data(datav):
|
|
data = "".join(datav)
|
|
n = create_chk_filenode(self, data)
|
|
results = upload.UploadResults()
|
|
results.uri = n.get_uri()
|
|
return results
|
|
d.addCallback(_got_data)
|
|
return d
|
|
|
|
def create_node_from_uri(self, u):
|
|
u = IURI(u)
|
|
if (INewDirectoryURI.providedBy(u)
|
|
or IReadonlyNewDirectoryURI.providedBy(u)):
|
|
return FakeDirectoryNode(self).init_from_uri(u)
|
|
return Marker(u.to_string())
|
|
|
|
def create_empty_dirnode(self):
|
|
n = FakeDirectoryNode(self)
|
|
d = n.create()
|
|
d.addCallback(lambda res: n)
|
|
return d
|
|
|
|
|
|
class Dirnode(unittest.TestCase, testutil.ShouldFailMixin):
|
|
def setUp(self):
|
|
self.client = FakeClient()
|
|
|
|
def test_basic(self):
|
|
d = self.client.create_empty_dirnode()
|
|
def _done(res):
|
|
self.failUnless(isinstance(res, FakeDirectoryNode))
|
|
rep = str(res)
|
|
self.failUnless("RW" in rep)
|
|
d.addCallback(_done)
|
|
return d
|
|
|
|
def test_corrupt(self):
|
|
d = self.client.create_empty_dirnode()
|
|
def _created(dn):
|
|
u = make_mutable_file_uri()
|
|
d = dn.set_uri(u"child", u, {})
|
|
d.addCallback(lambda res: dn.list())
|
|
def _check1(children):
|
|
self.failUnless(u"child" in children)
|
|
d.addCallback(_check1)
|
|
d.addCallback(lambda res:
|
|
self.shouldFail(KeyError, "get bogus", None,
|
|
dn.get, u"bogus"))
|
|
def _corrupt(res):
|
|
filenode = dn._node
|
|
si = IURI(filenode.get_uri()).storage_index
|
|
old_contents = filenode.all_contents[si]
|
|
# we happen to know that the writecap is encrypted near the
|
|
# end of the string. Flip one of its bits and make sure we
|
|
# detect the corruption.
|
|
new_contents = testutil.flip_bit(old_contents, -10)
|
|
# TODO: also test flipping bits in the other portions
|
|
filenode.all_contents[si] = new_contents
|
|
d.addCallback(_corrupt)
|
|
def _check2(res):
|
|
self.shouldFail(hashutil.IntegrityCheckError, "corrupt",
|
|
"HMAC does not match, crypttext is corrupted",
|
|
dn.list)
|
|
d.addCallback(_check2)
|
|
return d
|
|
d.addCallback(_created)
|
|
return d
|
|
|
|
def test_check(self):
|
|
d = self.client.create_empty_dirnode()
|
|
d.addCallback(lambda dn: dn.check())
|
|
def _done(res):
|
|
pass
|
|
d.addCallback(_done)
|
|
return d
|
|
|
|
def test_readonly(self):
|
|
fileuri = make_chk_file_uri(1234)
|
|
filenode = self.client.create_node_from_uri(fileuri)
|
|
uploadable = upload.Data("some data", convergence="some convergence string")
|
|
|
|
d = self.client.create_empty_dirnode()
|
|
def _created(rw_dn):
|
|
d2 = rw_dn.set_uri(u"child", fileuri)
|
|
d2.addCallback(lambda res: rw_dn)
|
|
return d2
|
|
d.addCallback(_created)
|
|
|
|
def _ready(rw_dn):
|
|
ro_uri = rw_dn.get_readonly_uri()
|
|
ro_dn = self.client.create_node_from_uri(ro_uri)
|
|
self.failUnless(ro_dn.is_readonly())
|
|
self.failUnless(ro_dn.is_mutable())
|
|
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
ro_dn.set_uri, u"newchild", fileuri)
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
ro_dn.set_node, u"newchild", filenode)
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
ro_dn.add_file, u"newchild", uploadable)
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
ro_dn.delete, u"child")
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
ro_dn.create_empty_directory, u"newchild")
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
ro_dn.move_child_to, u"child", rw_dn)
|
|
self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
|
|
rw_dn.move_child_to, u"child", ro_dn)
|
|
return ro_dn.list()
|
|
d.addCallback(_ready)
|
|
def _listed(children):
|
|
self.failUnless(u"child" in children)
|
|
d.addCallback(_listed)
|
|
return d
|
|
|
|
def failUnlessGreaterThan(self, a, b):
|
|
self.failUnless(a > b, "%r should be > %r" % (a, b))
|
|
|
|
def failUnlessGreaterOrEqualThan(self, a, b):
|
|
self.failUnless(a >= b, "%r should be >= %r" % (a, b))
|
|
|
|
def stall(self, res, delay=1.0):
|
|
d = defer.Deferred()
|
|
reactor.callLater(delay, d.callback, res)
|
|
return d
|
|
|
|
def test_create(self):
|
|
self.expected_manifest = []
|
|
|
|
d = self.client.create_empty_dirnode()
|
|
def _then(n):
|
|
self.failUnless(n.is_mutable())
|
|
u = n.get_uri()
|
|
self.failUnless(u)
|
|
self.failUnless(u.startswith("URI:DIR2:"), u)
|
|
u_ro = n.get_readonly_uri()
|
|
self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
|
|
u_v = n.get_verifier()
|
|
self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
|
|
self.expected_manifest.append(u_v)
|
|
|
|
d = n.list()
|
|
d.addCallback(lambda res: self.failUnlessEqual(res, {}))
|
|
d.addCallback(lambda res: n.has_child(u"missing"))
|
|
d.addCallback(lambda res: self.failIf(res))
|
|
fake_file_uri = make_mutable_file_uri()
|
|
m = Marker(fake_file_uri)
|
|
ffu_v = m.get_verifier()
|
|
assert isinstance(ffu_v, str)
|
|
self.expected_manifest.append(ffu_v)
|
|
d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri))
|
|
|
|
d.addCallback(lambda res: n.create_empty_directory(u"subdir"))
|
|
def _created(subdir):
|
|
self.failUnless(isinstance(subdir, FakeDirectoryNode))
|
|
self.subdir = subdir
|
|
new_v = subdir.get_verifier()
|
|
assert isinstance(new_v, str)
|
|
self.expected_manifest.append(new_v)
|
|
d.addCallback(_created)
|
|
|
|
d.addCallback(lambda res: n.list())
|
|
d.addCallback(lambda children:
|
|
self.failUnlessEqual(sorted(children.keys()),
|
|
sorted([u"child", u"subdir"])))
|
|
|
|
d.addCallback(lambda res: n.build_manifest())
|
|
def _check_manifest(manifest):
|
|
self.failUnlessEqual(sorted(manifest),
|
|
sorted(self.expected_manifest))
|
|
d.addCallback(_check_manifest)
|
|
|
|
def _add_subsubdir(res):
|
|
return self.subdir.create_empty_directory(u"subsubdir")
|
|
d.addCallback(_add_subsubdir)
|
|
d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
|
|
d.addCallback(lambda subsubdir:
|
|
self.failUnless(isinstance(subsubdir,
|
|
FakeDirectoryNode)))
|
|
d.addCallback(lambda res: n.get_child_at_path(u""))
|
|
d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
|
|
n.get_uri()))
|
|
|
|
d.addCallback(lambda res: n.get_metadata_for(u"child"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
|
|
# set_uri + metadata
|
|
# it should be possible to add a child without any metadata
|
|
d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri, {}))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"c2"))
|
|
d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
|
|
|
|
# if we don't set any defaults, the child should get timestamps
|
|
d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"c3"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
|
|
# or we can add specific metadata at set_uri() time, which
|
|
# overrides the timestamps
|
|
d.addCallback(lambda res: n.set_uri(u"c4", fake_file_uri,
|
|
{"key": "value"}))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"c4"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(metadata, {"key": "value"}))
|
|
|
|
d.addCallback(lambda res: n.delete(u"c2"))
|
|
d.addCallback(lambda res: n.delete(u"c3"))
|
|
d.addCallback(lambda res: n.delete(u"c4"))
|
|
|
|
# set_node + metadata
|
|
# it should be possible to add a child without any metadata
|
|
d.addCallback(lambda res: n.set_node(u"d2", n, {}))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"d2"))
|
|
d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
|
|
|
|
# if we don't set any defaults, the child should get timestamps
|
|
d.addCallback(lambda res: n.set_node(u"d3", n))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"d3"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
|
|
# or we can add specific metadata at set_node() time, which
|
|
# overrides the timestamps
|
|
d.addCallback(lambda res: n.set_node(u"d4", n,
|
|
{"key": "value"}))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"d4"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(metadata, {"key": "value"}))
|
|
|
|
d.addCallback(lambda res: n.delete(u"d2"))
|
|
d.addCallback(lambda res: n.delete(u"d3"))
|
|
d.addCallback(lambda res: n.delete(u"d4"))
|
|
|
|
# metadata through set_children()
|
|
d.addCallback(lambda res: n.set_children([ (u"e1", fake_file_uri),
|
|
(u"e2", fake_file_uri, {}),
|
|
(u"e3", fake_file_uri,
|
|
{"key": "value"}),
|
|
]))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"e1"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"e2"))
|
|
d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"e3"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(metadata, {"key": "value"}))
|
|
|
|
d.addCallback(lambda res: n.delete(u"e1"))
|
|
d.addCallback(lambda res: n.delete(u"e2"))
|
|
d.addCallback(lambda res: n.delete(u"e3"))
|
|
|
|
# metadata through set_nodes()
|
|
d.addCallback(lambda res: n.set_nodes([ (u"f1", n),
|
|
(u"f2", n, {}),
|
|
(u"f3", n,
|
|
{"key": "value"}),
|
|
]))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"f1"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"f2"))
|
|
d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"f3"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(metadata, {"key": "value"}))
|
|
|
|
d.addCallback(lambda res: n.delete(u"f1"))
|
|
d.addCallback(lambda res: n.delete(u"f2"))
|
|
d.addCallback(lambda res: n.delete(u"f3"))
|
|
|
|
|
|
d.addCallback(lambda res:
|
|
n.set_metadata_for(u"child",
|
|
{"tags": ["web2.0-compatible"]}))
|
|
d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(metadata,
|
|
{"tags": ["web2.0-compatible"]}))
|
|
|
|
def _start(res):
|
|
self._start_timestamp = time.time()
|
|
d.addCallback(_start)
|
|
# simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
|
|
# floats to hundredeths (it uses str(num) instead of repr(num)).
|
|
# simplejson-1.7.3 does not have this bug. To prevent this bug
|
|
# from causing the test to fail, stall for more than a few
|
|
# hundrededths of a second.
|
|
d.addCallback(self.stall, 0.1)
|
|
d.addCallback(lambda res: n.add_file(u"timestamps",
|
|
upload.Data("stamp me", convergence="some convergence string")))
|
|
d.addCallback(self.stall, 0.1)
|
|
def _stop(res):
|
|
self._stop_timestamp = time.time()
|
|
d.addCallback(_stop)
|
|
|
|
d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
|
|
def _check_timestamp1(metadata):
|
|
self.failUnless("ctime" in metadata)
|
|
self.failUnless("mtime" in metadata)
|
|
self.failUnlessGreaterOrEqualThan(metadata["ctime"],
|
|
self._start_timestamp)
|
|
self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
|
|
metadata["ctime"])
|
|
self.failUnlessGreaterOrEqualThan(metadata["mtime"],
|
|
self._start_timestamp)
|
|
self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
|
|
metadata["mtime"])
|
|
# Our current timestamp rules say that replacing an existing
|
|
# child should preserve the 'ctime' but update the mtime
|
|
self._old_ctime = metadata["ctime"]
|
|
self._old_mtime = metadata["mtime"]
|
|
d.addCallback(_check_timestamp1)
|
|
d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
|
|
d.addCallback(lambda res: n.set_node(u"timestamps", n))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
|
|
def _check_timestamp2(metadata):
|
|
self.failUnlessEqual(metadata["ctime"], self._old_ctime,
|
|
"%s != %s" % (metadata["ctime"],
|
|
self._old_ctime))
|
|
self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
|
|
return n.delete(u"timestamps")
|
|
d.addCallback(_check_timestamp2)
|
|
|
|
# also make sure we can add/update timestamps on a
|
|
# previously-existing child that didn't have any, since there are
|
|
# a lot of 0.7.0-generated edges around out there
|
|
d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
|
|
d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
d.addCallback(lambda res: n.delete(u"no_timestamps"))
|
|
|
|
d.addCallback(lambda res: n.delete(u"subdir"))
|
|
d.addCallback(lambda old_child:
|
|
self.failUnlessEqual(old_child.get_uri(),
|
|
self.subdir.get_uri()))
|
|
|
|
d.addCallback(lambda res: n.list())
|
|
d.addCallback(lambda children:
|
|
self.failUnlessEqual(sorted(children.keys()),
|
|
sorted([u"child"])))
|
|
|
|
uploadable = upload.Data("some data", convergence="some convergence string")
|
|
d.addCallback(lambda res: n.add_file(u"newfile", uploadable))
|
|
d.addCallback(lambda newnode:
|
|
self.failUnless(IFileNode.providedBy(newnode)))
|
|
d.addCallback(lambda res: n.list())
|
|
d.addCallback(lambda children:
|
|
self.failUnlessEqual(sorted(children.keys()),
|
|
sorted([u"child", u"newfile"])))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(sorted(metadata.keys()),
|
|
["ctime", "mtime"]))
|
|
|
|
uploadable = upload.Data("some data", convergence="some convergence string")
|
|
d.addCallback(lambda res: n.add_file(u"newfile-metadata",
|
|
uploadable,
|
|
{"key": "value"}))
|
|
d.addCallback(lambda newnode:
|
|
self.failUnless(IFileNode.providedBy(newnode)))
|
|
d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
|
|
d.addCallback(lambda metadata:
|
|
self.failUnlessEqual(metadata, {"key": "value"}))
|
|
d.addCallback(lambda res: n.delete(u"newfile-metadata"))
|
|
|
|
d.addCallback(lambda res: n.create_empty_directory(u"subdir2"))
|
|
def _created2(subdir2):
|
|
self.subdir2 = subdir2
|
|
d.addCallback(_created2)
|
|
|
|
d.addCallback(lambda res:
|
|
n.move_child_to(u"child", self.subdir2))
|
|
d.addCallback(lambda res: n.list())
|
|
d.addCallback(lambda children:
|
|
self.failUnlessEqual(sorted(children.keys()),
|
|
sorted([u"newfile", u"subdir2"])))
|
|
d.addCallback(lambda res: self.subdir2.list())
|
|
d.addCallback(lambda children:
|
|
self.failUnlessEqual(sorted(children.keys()),
|
|
sorted([u"child"])))
|
|
|
|
return d
|
|
|
|
d.addCallback(_then)
|
|
|
|
return d
|
|
|
|
|
|
netstring = hashutil.netstring
|
|
split_netstring = dirnode.split_netstring
|
|
|
|
class Netstring(unittest.TestCase):
|
|
def test_split(self):
|
|
a = netstring("hello") + netstring("world")
|
|
self.failUnlessEqual(split_netstring(a, 2), ("hello", "world"))
|
|
self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world"))
|
|
self.failUnlessEqual(split_netstring(a, 2, True),
|
|
("hello", "world", ""))
|
|
self.failUnlessRaises(ValueError, split_netstring, a, 3)
|
|
self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2)
|
|
self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False)
|
|
|
|
def test_extra(self):
|
|
a = netstring("hello")
|
|
self.failUnlessEqual(split_netstring(a, 1, True), ("hello", ""))
|
|
b = netstring("hello") + "extra stuff"
|
|
self.failUnlessEqual(split_netstring(b, 1, True),
|
|
("hello", "extra stuff"))
|
|
|
|
def test_nested(self):
|
|
a = netstring("hello") + netstring("world") + "extra stuff"
|
|
b = netstring("a") + netstring("is") + netstring(a) + netstring(".")
|
|
top = split_netstring(b, 4)
|
|
self.failUnlessEqual(len(top), 4)
|
|
self.failUnlessEqual(top[0], "a")
|
|
self.failUnlessEqual(top[1], "is")
|
|
self.failUnlessEqual(top[2], a)
|
|
self.failUnlessEqual(top[3], ".")
|
|
self.failUnlessRaises(ValueError, split_netstring, a, 2)
|
|
self.failUnlessRaises(ValueError, split_netstring, a, 2, False)
|
|
bottom = split_netstring(a, 2, True)
|
|
self.failUnlessEqual(bottom, ("hello", "world", "extra stuff"))
|
|
|