ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-19 21:17:54 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

more filetree work, more tests now pass

Browse Source
This commit is contained in:
Brian Warner 2007-01-20 15:50:21 -07:00
parent b61a4ff371
commit ce4610c3e6
6 changed files with 89 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/allmydata/filetree/directory.py
View File

@ -3,7 +3,7 @@ from zope.interface import implements
from twisted.internet import defer
from cStringIO import StringIO
from allmydata.filetree.interfaces import (
INode, IDirectoryNode, ISubTree,
INode, INodeMaker, IDirectoryNode, ISubTree,
ICHKDirectoryNode, ISSKDirectoryNode,
NoSuchChildError,
)
@ -19,17 +19,14 @@ from allmydata.util import bencode
# each time the vdrive changes, update the local drive to match, and
# vice versa.
# from the itertools 'recipes' page
from itertools import izip, tee
def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..."
a, b = tee(iterable)
try:
b.next()
except StopIteration:
pass
from itertools import islice, izip
def in_pairs(iterable):
"s -> (s0,s1), (s2,s3), (s4,s5), ..."
a = islice(iterable, 0, None, 2)
b = islice(iterable, 1, None, 2)
return izip(a, b)
class SubTreeNode:
implements(INode, IDirectoryNode)
@ -95,15 +92,16 @@ class SubTreeNode:
data.append(self.children[name].serialize_node())
return data
def populate_node(self, data, node_maker):
def populate_dirnode(self, data, node_maker):
assert INodeMaker(node_maker)
assert len(data) % 2 == 0
for (name, child_data) in pairwise(data):
for (name, child_data) in in_pairs(data):
if isinstance(child_data, (list, tuple)):
child = SubTreeNode(self.enclosing_tree)
child.populate_node(child_data)
child.populate_dirnode(child_data, node_maker)
else:
assert isinstance(child_data, str)
child = node_maker(child_data)
child = node_maker.make_node_from_serialized(child_data)
self.children[name] = child
@ -141,7 +139,7 @@ class _DirectorySubTree(object):
def _populate_from_data(self, data, node_maker):
self.root = SubTreeNode(self)
self.root.populate_node(bencode.bdecode(data), node_maker)
self.root.populate_dirnode(bencode.bdecode(data), node_maker)
return self
def serialize_subtree_to_file(self, f):
@ -205,14 +203,16 @@ class LocalFileSubTree(_DirectorySubTree):
f = open(self.filename, "rb")
data = f.read()
f.close()
return defer.succeed(self._populate_from_data(node_maker))
d = defer.succeed(data)
d.addCallback(self._populate_from_data, node_maker)
return d
def create_node_now(self):
return LocalFileSubTreeNode().new(self.filename)
def _update(self):
f = open(self.filename, "wb")
self.serialize_to_file(f)
self.serialize_subtree_to_file(f)
f.close()
def update_now(self, uploader):
@ -258,7 +258,7 @@ class CHKDirectorySubTree(_DirectorySubTree):
def update_now(self, uploader):
f = StringIO()
self.serialize_to_file(f)
self.serialize_subtree_to_file(f)
data = f.getvalue()
d = uploader.upload_data(data)
def _uploaded(uri):
@ -271,7 +271,7 @@ class CHKDirectorySubTree(_DirectorySubTree):
# this is the CHK form
old_uri = self.uri
f, filename = workqueue.create_tempfile(".chkdir")
self.serialize_to_file(f)
self.serialize_subtree_to_file(f)
f.close()
boxname = workqueue.create_boxname()
workqueue.add_upload_chk(filename, boxname)
@ -339,7 +339,7 @@ class SSKDirectorySubTree(_DirectorySubTree):
raise RuntimeError("This SSKDirectorySubTree is not mutable")
f = StringIO()
self.serialize_to_file(f)
self.serialize_subtree_to_file(f)
data = f.getvalue()
self.version += 1
@ -350,7 +350,7 @@ class SSKDirectorySubTree(_DirectorySubTree):
def update(self, workqueue):
# this is the SSK form
f, filename = workqueue.create_tempfile(".sskdir")
self.serialize_to_file(f)
self.serialize_subtree_to_file(f)
f.close()
oldversion = self.version

22
src/allmydata/filetree/interfaces.py
View File

@ -17,8 +17,11 @@ class INode(Interface):
.prefix ."""
def populate_node(body, node_maker):
"""vdrive.make_node_from_serialized() will first use the prefix from
the .prefix attribute to decide what kind of Node to create. It will
then call this function with the body to populate the new Node."""
the .prefix attribute to decide what kind of Node to create. They
will then call this populate_node() method with the body to populate
the new Node. 'node_maker' provides INodeMaker, which provides that
same make_node_from_serialized function to create any internal child
nodes that might be necessary."""
class IFileNode(Interface):
"""This is a file which can be retrieved."""
@ -190,6 +193,21 @@ class ISubTree(Interface):
(SSKDirectorySubTrees and redirections).
"""
class INodeMaker(Interface):
def make_node_from_serialized(serialized):
"""Turn a string into an INode, which contains information about
the file or directory (like a URI), but does not contain the actual
contents. An IOpener can be used later to retrieve the contents
(which means downloading the file if this is an IFileNode, or
perhaps creating a new subtree from the contents)."""
class ISubTreeMaker(Interface):
def make_subtree_from_node(node, parent_is_mutable):
"""Turn an INode into an ISubTree (using an internal opener to
download the data, if necessary).
This returns a Deferred that fires with the ISubTree instance.
"""
#class IMutableSubTree(Interface):
# def mutation_affects_parent():
# """This returns True for CHK nodes where you must inform the parent

4
src/allmydata/filetree/opener.py
View File

@ -3,7 +3,7 @@ from zope.interface import implements
from twisted.internet import defer
from allmydata.filetree import interfaces, directory, redirect
#from allmydata.filetree.file import CHKFile, MutableSSKFile, ImmutableSSKFile
from allmydata.filetree.interfaces import INode, IDirectoryNode
from allmydata.filetree.interfaces import INode, IDirectoryNode, INodeMaker
all_openable_subtree_types = [
directory.LocalFileSubTree,
@ -27,6 +27,7 @@ class Opener(object):
def _create(self, node, parent_is_mutable, node_maker):
assert INode(node)
assert INodeMaker(node_maker)
for subtree_class in all_openable_subtree_types:
if isinstance(node, subtree_class.node_class):
subtree = subtree_class()
@ -41,6 +42,7 @@ class Opener(object):
def open(self, node, parent_is_mutable, node_maker):
assert INode(node)
assert not isinstance(node, IDirectoryNode)
assert INodeMaker(node_maker)
# is it in cache? To check this we need to use the node's serialized
# form, since nodes are instances and don't compare by value

16
src/allmydata/filetree/redirect.py
View File

@ -3,7 +3,7 @@ from cStringIO import StringIO
from zope.interface import implements
from twisted.internet import defer
from allmydata.filetree.interfaces import ISubTree
from allmydata.filetree.interfaces import ISubTree, INodeMaker
from allmydata.filetree.basenode import BaseDataNode
from allmydata.util import bencode
@ -33,7 +33,8 @@ class _BaseRedirection(object):
return self.child_node.serialize_node()
def _populate_from_data(self, data, node_maker):
self.child_node = node_maker(data)
assert INodeMaker(node_maker)
self.child_node = node_maker.make_node_from_serialized(data)
return self
@ -64,7 +65,9 @@ class LocalFileRedirection(_BaseRedirection):
# note: we don't cache the contents of the file. TODO: consider
# doing this based upon mtime. It is important that we be able to
# notice if the file has been changed.
return defer.succeed(self._populate_from_data(data, node_maker))
d = defer.succeed(data)
d.addCallback(self._populate_from_data, node_maker)
return d
def is_mutable(self):
return True
@ -161,10 +164,11 @@ class QueenOrLocalFileRedirection(_BaseRedirection):
# TODO: queen?
# TODO: pubsub so we can cache the queen's results
d = self._queen.callRemote("lookup_handle", self.handle)
d.addCallback(self._choose_winner, local_version_and_data, node_maker)
d.addCallback(self._choose_winner, local_version_and_data)
d.addCallback(self._populate_from_data, node_maker)
return d
def _choose_winner(self, queen_version_and_data, local_version_and_data, node_maker):
def _choose_winner(self, queen_version_and_data, local_version_and_data):
queen_version, queen_data = bencode.bdecode(queen_version_and_data)
local_version, local_data = bencode.bdecode(local_version_and_data)
if queen_version > local_version:
@ -174,7 +178,7 @@ class QueenOrLocalFileRedirection(_BaseRedirection):
data = local_data
self.version = local_version
# NOTE: two layers of bencoding here, TODO
return self._populate_from_data(data, node_maker)
return data
def is_mutable(self):
return True

19
src/allmydata/filetree/vdrive.py
View File

@ -1,8 +1,8 @@
from zope.interface import implements
from allmydata.filetree import opener, directory, redirect
from allmydata.filetree import opener, directory, file, redirect
from allmydata.filetree.interfaces import (
IVirtualDrive, INode, ISubTree, IFileNode, IDirectoryNode,
IVirtualDrive, INodeMaker, INode, ISubTree, IFileNode, IDirectoryNode,
NoSuchDirectoryError, NoSuchChildError, PathAlreadyExistsError,
PathDoesNotExistError,
)
@ -12,8 +12,11 @@ from allmydata.upload import IUploadable
# node specification strings (found inside the serialized form of subtrees)
# into Nodes (which live in the in-RAM form of subtrees).
all_node_types = [
directory.LocalFileSubTreeNode,
directory.CHKDirectorySubTreeNode,
directory.SSKDirectorySubTreeNode,
file.CHKFileNode,
file.SSKFileNode,
redirect.LocalFileRedirectionNode,
redirect.QueenRedirectionNode,
redirect.HTTPRedirectionNode,
@ -21,7 +24,7 @@ all_node_types = [
]
class VirtualDrive(object):
implements(IVirtualDrive)
implements(IVirtualDrive, INodeMaker)
def __init__(self, workqueue, downloader, root_node):
assert INode(root_node)
@ -33,6 +36,8 @@ class VirtualDrive(object):
self.root_node = root_node
# these are called when loading and creating nodes
# INodeMaker
def make_node_from_serialized(self, serialized):
# this turns a string into an INode, which contains information about
# the file or directory (like a URI), but does not contain the actual
@ -47,14 +52,14 @@ class VirtualDrive(object):
for node_class in all_node_types:
if prefix == node_class.prefix:
node = node_class()
node.populate_node(body, self.make_node_from_serialized)
node.populate_node(body, self)
return node
raise RuntimeError("unable to handle subtree type '%s'" % prefix)
raise RuntimeError("unable to handle node type '%s'" % prefix)
# ISubTreeMaker
def make_subtree_from_node(self, node, parent_is_mutable):
assert INode(node)
return self.opener.open(node, parent_is_mutable,
self.make_subtree_from_node)
return self.opener.open(node, parent_is_mutable, self)
# these methods are used to walk through our subtrees

30
src/allmydata/test/test_filetree_new.py
View File

@ -1,7 +1,7 @@
#from zope.interface import implements
from twisted.trial import unittest
#from twisted.internet import defer
from twisted.internet import defer
#from allmydata.filetree.interfaces import IOpener, IDirectoryNode
#from allmydata.filetree.directory import (ImmutableDirectorySubTree,
# SubTreeNode,
@ -314,8 +314,15 @@ class Redirect(unittest.TestCase):
import os.path
from allmydata.filetree import directory, redirect, vdrive
from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode)
from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode, IFileNode)
from allmydata.filetree.file import CHKFileNode
from allmydata.util import bencode
class InPairs(unittest.TestCase):
def test_in_pairs(self):
l = range(8)
pairs = list(directory.in_pairs(l))
self.failUnlessEqual(pairs, [(0,1), (2,3), (4,5), (6,7)])
class Stuff(unittest.TestCase):
@ -372,11 +379,14 @@ class Stuff(unittest.TestCase):
del root, subdir1, subdir2, subdir3, subdir4
# leaving file1 for later use
# now serialize it and reconstruct it
# now serialize it and examine the results
f = StringIO()
subtree.serialize_subtree_to_file(f)
data = f.getvalue()
#print data
unpacked = bencode.bdecode(data)
#print unpacked
del f, data, unpacked
node = subtree.create_node_now()
self.failUnless(isinstance(node, directory.LocalFileSubTreeNode))
@ -384,9 +394,14 @@ class Stuff(unittest.TestCase):
self.failUnless(isinstance(node_s, str))
self.failUnless(node_s.startswith("LocalFileDirectory:"))
self.failUnless("dirtree.save" in node_s)
del node, node_s
d = defer.maybeDeferred(subtree.update_now, None)
def _updated(node):
# now reconstruct it
return v.make_subtree_from_node(node, False)
d.addCallback(_updated)
# now reconstruct it
d = v.make_subtree_from_node(node, False)
def _opened(new_subtree):
res = new_subtree.get_node_for_path([])
(found_path, root, remaining_path) = res
@ -396,7 +411,9 @@ class Stuff(unittest.TestCase):
self.failUnless(IDirectoryNode.providedBy(root))
self.failUnlessEqual(root.list(), ["foo.txt", "subdir1"])
file1a = root.get("foo.txt")
self.failUnless(isinstance(CHKFileNode, file1a))
self.failUnless(INode(file1a))
self.failUnless(isinstance(file1a, CHKFileNode))
self.failUnless(IFileNode(file1a))
self.failUnlessEqual(file1a.get_uri(), "uri1")
subdir1 = root.get("subdir1")
subdir2 = subdir1.get("subdir2")
@ -405,7 +422,6 @@ class Stuff(unittest.TestCase):
self.failUnlessEqual(subdir2.list(), [])
d.addCallback(_opened)
return d
testDirectory.todo = "not working yet"
def testVdrive(self):
# create some stuff, see if we can import everything