webapi: checkpointing more test progress

This commit is contained in:
Brian Warner 2007-07-07 00:16:36 -07:00
parent 6570253d6b
commit d501984eba
2 changed files with 333 additions and 71 deletions

View File

@ -8,6 +8,7 @@ from twisted.web import client, error
from twisted.python import failure
from allmydata import webish, interfaces, dirnode, uri
from allmydata.encode import NotEnoughPeersError
from allmydata.util import fileutil
import itertools
# create a fake uploader/downloader, and a couple of fake dirnodes, then
@ -64,12 +65,14 @@ class MyUploader(service.Service):
class MyDirectoryNode(dirnode.MutableDirectoryNode):
def __init__(self, nodes, uri=None):
self._nodes = nodes
def __init__(self, nodes, files, client, uri=None):
self._my_nodes = nodes
self._my_files = files
self._my_client = client
if uri is None:
uri = str(uri_counter.next())
self._uri = str(uri)
self._nodes[self._uri] = self
self._my_nodes[self._uri] = self
self.children = {}
self._mutable = True
@ -79,22 +82,38 @@ class MyDirectoryNode(dirnode.MutableDirectoryNode):
def get(self, name):
def _try():
uri = self.children[name]
if uri not in self._nodes:
if uri not in self._my_nodes:
raise IndexError("this isn't supposed to happen")
return self._nodes[uri]
return self._my_nodes[uri]
return defer.maybeDeferred(_try)
def set_uri(self, name, child_uri):
self.children[name] = child_uri
return defer.succeed(None)
def add_file(self, name, uploadable):
f = uploadable.get_filehandle()
data = f.read()
uri = str(uri_counter.next())
self._my_files[uri] = data
self._my_nodes[uri] = MyFileNode(uri, self._my_client)
uploadable.close_filehandle(f)
self.children[name] = uri
return defer.succeed(self._my_nodes[uri])
def delete(self, name):
def _try():
del self.children[name]
return defer.maybeDeferred(_try)
def create_empty_directory(self, name):
node = MyDirectoryNode(self._nodes)
node = MyDirectoryNode(self._my_nodes, self._my_files, self._my_client)
self.children[name] = node.get_uri()
return defer.succeed(node)
def list(self):
kids = dict([(name, self._nodes[uri])
kids = dict([(name, self._my_nodes[uri])
for name,uri in self.children.iteritems()])
return defer.succeed(kids)
@ -134,44 +153,57 @@ class Web(unittest.TestCase):
ul = MyUploader(self.files)
ul.setServiceParent(self.s)
v.public_root = MyDirectoryNode(self.nodes)
v.private_root = MyDirectoryNode(self.nodes)
foo = MyDirectoryNode(self.nodes)
v.public_root = self.makedir()
self.public_root = v.public_root
v.private_root = self.makedir()
foo = self.makedir()
self._foo_node = foo
self._foo_uri = foo.get_uri()
self._foo_readonly_uri = foo.get_immutable_uri()
v.public_root.children["foo"] = foo.get_uri()
self.BAR_CONTENTS = "bar.txt contents"
bar_uri = uri.pack_uri("SI"+"0"*30,
"K"+"0"*15,
"EH"+"0"*30,
25, 100, 123)
bar_txt = MyFileNode(bar_uri, self.s)
self._bar_txt_uri = bar_txt.get_uri()
self.nodes[bar_uri] = bar_txt
self.files[bar_txt.get_uri()] = self.BAR_CONTENTS
foo.children["bar.txt"] = bar_txt.get_uri()
self._bar_txt_uri = self.makefile(0)
self.BAR_CONTENTS = self.files[self._bar_txt_uri]
foo.children["bar.txt"] = self._bar_txt_uri
foo.children["empty"] = self.makedir().get_uri()
sub_uri = foo.children["sub"] = self.makedir().get_uri()
sub = self.nodes[sub_uri]
foo.children["sub"] = MyDirectoryNode(self.nodes).get_uri()
blocking_uri = self.makefile(1)
foo.children["blockingfile"] = blocking_uri
blocking_uri = uri.pack_uri("SI"+"1"*30,
"K"+"1"*15,
"EH"+"1"*30,
25, 100, 124)
blocking_file = MyFileNode(blocking_uri, self.s)
self.nodes[blocking_uri] = blocking_file
self.files[blocking_uri] = "blocking contents"
foo.children["blockingfile"] = blocking_file.get_uri()
baz_file = self.makefile(2)
sub.children["baz.txt"] = baz_file
# public/
# public/foo/
# public/foo/bar.txt
# public/foo/sub/
# public/foo/blockingfile
# public/foo/empty/
# public/foo/sub/
# public/foo/sub/baz.txt
self.NEWFILE_CONTENTS = "newfile contents\n"
def makefile(self, number):
n = str(number)
assert len(n) == 1
newuri = uri.pack_uri("SI" + n*30,
"K" + n*15,
"EH" + n*30,
25, 100, 123+number)
assert newuri not in self.nodes
assert newuri not in self.files
node = MyFileNode(newuri, self.s)
self.nodes[newuri] = node
contents = "contents of file %s\n" % n
self.files[newuri] = contents
return newuri
def makedir(self):
node = MyDirectoryNode(self.nodes, self.files, self.s)
return node
def tearDown(self):
return self.s.stopService()
@ -191,6 +223,7 @@ class Web(unittest.TestCase):
return client.getPage(url, method="DELETE")
def POST(self, urlpath, data):
raise unittest.SkipTest("not yet")
url = self.webish_url + urlpath
return client.getPage(url, method="POST", postdata=data)
@ -211,8 +244,8 @@ class Web(unittest.TestCase):
res.trap(error.Error)
self.failUnlessEqual(res.value.status, "404")
else:
self.fail("%s was supposed to raise %s, not get '%s'" %
(which, expected_failure, res))
self.fail("%s was supposed to Error(404), not get '%s'" %
(which, res))
def test_create(self): # YES
pass
@ -274,12 +307,21 @@ class Web(unittest.TestCase):
"403 Forbidden")
return d
def test_DELETE_FILEURL(self):
def test_DELETE_FILEURL(self): # YES
d = self.DELETE("/vdrive/global/foo/bar.txt")
def _check(res):
self.failIf("bar.txt" in self._foo_node.children)
d.addCallback(_check)
return d
def test_DELETE_FILEURL_missing(self):
def test_DELETE_FILEURL_missing(self): # YES
d = self.DELETE("/vdrive/global/foo/missing")
d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
return d
def test_DELETE_FILEURL_missing2(self): # YES
d = self.DELETE("/vdrive/global/missing/missing")
d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
return d
def test_GET_FILEURL_json(self): # YES
@ -301,7 +343,7 @@ class Web(unittest.TestCase):
def test_GET_FILEURL_localfile(self): # YES
localfile = os.path.abspath("web/GET_FILEURL_localfile")
os.makedirs("web")
fileutil.make_dirs("web")
d = self.GET("/vdrive/global/foo/bar.txt?localfile=%s" % localfile)
def _done(res):
self.failUnless(os.path.exists(localfile))
@ -317,7 +359,7 @@ class Web(unittest.TestCase):
old_LOCALHOST = webish.LOCALHOST
webish.LOCALHOST = "127.0.0.2"
localfile = os.path.abspath("web/GET_FILEURL_localfile_nonlocal")
os.makedirs("web")
fileutil.make_dirs("web")
d = self.GET("/vdrive/global/foo/bar.txt?localfile=%s" % localfile)
d.addBoth(self.shouldFail, error.Error, "localfile non-local",
"403 Forbidden")
@ -331,9 +373,20 @@ class Web(unittest.TestCase):
d.addBoth(_reset)
return d
def test_GET_FILEURL_localfile_nonabsolute(self):
localfile = "web/nonabsolute/path"
fileutil.make_dirs("web/nonabsolute")
d = self.GET("/vdrive/global/foo/bar.txt?localfile=%s" % localfile)
d.addBoth(self.shouldFail, error.Error, "localfile non-absolute",
"403 Forbidden")
def _check(res):
self.failIf(os.path.exists(localfile))
d.addCallback(_check)
return d
def test_PUT_NEWFILEURL_localfile(self): # YES
localfile = os.path.abspath("web/PUT_NEWFILEURL_localfile")
os.makedirs("web")
fileutil.make_dirs("web")
f = open(localfile, "wb")
f.write(self.NEWFILE_CONTENTS)
f.close()
@ -349,7 +402,7 @@ class Web(unittest.TestCase):
def test_PUT_NEWFILEURL_localfile_mkdirs(self): # YES
localfile = os.path.abspath("web/PUT_NEWFILEURL_localfile_mkdirs")
os.makedirs("web")
fileutil.make_dirs("web")
f = open(localfile, "wb")
f.write(self.NEWFILE_CONTENTS)
f.close()
@ -436,32 +489,127 @@ class Web(unittest.TestCase):
d.addCallback(_check)
return d
def test_DELETE_DIRURL(self):
def test_DELETE_DIRURL(self): # YES
d = self.DELETE("/vdrive/global/foo")
def _check(res):
self.failIf("foo" in self.public_root.children)
d.addCallback(_check)
return d
def test_DELETE_DIRURL_missing(self):
def test_DELETE_DIRURL_missing(self): # YES
d = self.DELETE("/vdrive/global/foo/missing")
d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
def _check(res):
self.failUnless("foo" in self.public_root.children)
d.addCallback(_check)
return d
def test_DELETE_DIRURL_missing2(self): # YES
d = self.DELETE("/vdrive/global/missing")
d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
return d
def test_GET_DIRURL_localdir(self):
def test_walker(self): # YES
out = []
def _visitor(path, node):
out.append((path, node))
return defer.succeed(None)
w = webish.DirnodeWalkerMixin()
d = w.walk(self.public_root, _visitor)
def _check(res):
names = [path for (path,node) in out]
self.failUnlessEqual(sorted(names),
[('foo',),
('foo','bar.txt'),
('foo','blockingfile'),
('foo', 'empty'),
('foo', 'sub'),
('foo','sub','baz.txt'),
])
subindex = names.index( ('foo', 'sub') )
bazindex = names.index( ('foo', 'sub', 'baz.txt') )
self.failUnless(subindex < bazindex)
for path,node in out:
if path[-1] in ('bar.txt', 'blockingfile', 'baz.txt'):
self.failUnless(interfaces.IFileNode.providedBy(node))
else:
self.failUnless(interfaces.IDirectoryNode.providedBy(node))
d.addCallback(_check)
return d
def test_GET_DIRURL_localdir(self): # YES
localdir = os.path.abspath("web/GET_DIRURL_localdir")
os.makedirs("web")
fileutil.make_dirs("web")
d = self.GET("/vdrive/global/foo?localdir=%s" % localdir)
def _check(res):
barfile = os.path.join(localdir, "bar.txt")
self.failUnless(os.path.exists(barfile))
data = open(barfile, "rb").read()
self.failUnlessEqual(data, self.BAR_CONTENTS)
blockingfile = os.path.join(localdir, "blockingfile")
self.failUnless(os.path.exists(blockingfile))
subdir = os.path.join(localdir, "sub")
self.failUnless(os.path.isdir(subdir))
d.addCallback(_check)
return d
def test_PUT_NEWDIRURL_localdir(self):
def touch(self, localdir, filename):
path = os.path.join(localdir, filename)
f = open(path, "w")
f.write("contents of %s\n" % filename)
f.close()
def test_PUT_NEWDIRURL_localdir(self): # NO
localdir = os.path.abspath("web/PUT_NEWDIRURL_localdir")
os.makedirs("web")
# create some files there
d = self.GET("/vdrive/global/foo/newdir?localdir=%s" % localdir)
fileutil.make_dirs(os.path.join(localdir, "web"))
fileutil.make_dirs(os.path.join(localdir, "web/one"))
fileutil.make_dirs(os.path.join(localdir, "web/two"))
fileutil.make_dirs(os.path.join(localdir, "web/three"))
self.touch(localdir, "web/three/foo.txt")
self.touch(localdir, "web/three/bar.txt")
self.touch(localdir, "web/zap.zip")
d = self.PUT("/vdrive/global/foo/newdir?localdir=%s" % localdir, "")
def _check(res):
self.failUnless("newdir" in self._foo_node.children)
webnode = self.nodes[self._foo_node.children["newdir"]]
self.failUnlessEqual(sorted(webnode.children.keys()),
sorted(["one", "two", "three", "zap.zip"]))
threenode = self.nodes[webnode.children["three"]]
self.failUnlessEqual(sorted(threenode.children.keys()),
sorted(["foo.txt", "bar.txt"]))
barnode = self.nodes[threenode.children["foo.txt"]]
contents = self.files[barnode.get_uri()]
self.failUnlessEqual(contents, "contents of web/three/bar.txt")
d.addCallback(_check)
return d
def test_PUT_NEWDIRURL_localdir_mkdirs(self):
def test_PUT_NEWDIRURL_localdir_mkdirs(self): # NO
localdir = os.path.abspath("web/PUT_NEWDIRURL_localdir_mkdirs")
os.makedirs("web")
# create some files there
d = self.GET("/vdrive/global/foo/subdir/newdir?localdir=%s" % localdir)
fileutil.make_dirs(os.path.join(localdir, "web"))
fileutil.make_dirs(os.path.join(localdir, "web/one"))
fileutil.make_dirs(os.path.join(localdir, "web/two"))
fileutil.make_dirs(os.path.join(localdir, "web/three"))
self.touch(localdir, "web/three/foo.txt")
self.touch(localdir, "web/three/bar.txt")
self.touch(localdir, "web/zap.zip")
d = self.PUT("/vdrive/global/foo/subdir/newdir?localdir=%s" % localdir,
"")
def _check(res):
self.failUnless("subdir" in self._foo_node.children)
subnode = self.nodes[self._foo_node.children["subdir"]]
self.failUnless("newdir" in subnode.children)
webnode = self.nodes[subnode.children["newdir"]]
self.failUnlessEqual(sorted(webnode.children.keys()),
sorted(["one", "two", "three", "zap.zip"]))
threenode = self.nodes[webnode.children["three"]]
self.failUnlessEqual(sorted(threenode.children.keys()),
sorted(["foo.txt", "bar.txt"]))
barnode = self.nodes[threenode.children["foo.txt"]]
contents = self.files[barnode.get_uri()]
self.failUnlessEqual(contents, "contents of web/three/bar.txt")
d.addCallback(_check)
return d
def test_POST_upload(self):
@ -485,14 +633,17 @@ class Web(unittest.TestCase):
return d
def test_URI_GET(self):
raise unittest.SkipTest("not yet")
d = self.GET("/uri/%s/bar.txt" % foo_uri)
return d
def test_PUT_NEWFILEURL_uri(self):
raise unittest.SkipTest("not yet")
d = self.PUT("/vdrive/global/foo/new.txt?uri", new_uri)
return d
def test_XMLRPC(self):
raise unittest.SkipTest("not yet")
pass

View File

@ -1,11 +1,12 @@
import os.path
from twisted.application import service, strports
from twisted.web import static, resource, server, html, http
from twisted.python import util, log
from twisted.internet import defer
from nevow import inevow, rend, loaders, appserver, url, tags as T
from nevow.static import File as nevow_File # TODO: merge with static.File?
from allmydata.util import idlib
from allmydata.util import idlib, fileutil
from allmydata.uri import unpack_uri
from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode
from allmydata.dirnode import FileNode
@ -327,6 +328,18 @@ class NeedLocalhostError:
req.setHeader("content-type", "text/plain")
return "localfile= or localdir= requires a local connection"
class NeedAbsolutePathError:
implements(inevow.IResource)
def locateChild(self, ctx, segments):
return rend.NotFound
def renderHTTP(self, ctx):
req = inevow.IRequest(ctx)
req.setResponseCode(http.FORBIDDEN)
req.setHeader("content-type", "text/plain")
return "localfile= or localdir= requires an absolute path"
class LocalFileDownloader(resource.Resource):
@ -372,13 +385,68 @@ class FileURI(FileJSONMetadata):
file_uri = self._filenode.get_uri()
return file_uri
class LocalDirectoryDownloader(resource.Resource):
def __init__(self, dirnode):
self._dirnode = dirnode
class DirnodeWalkerMixin:
"""Visit all nodes underneath (and including) the rootnode, one at a
time. For each one, call the visitor. The visitor will see the
IDirectoryNode before it sees any of the IFileNodes inside. If the
visitor returns a Deferred, I do not call the visitor again until it has
fired.
"""
def renderHTTP(self, ctx):
dl = get_downloader_service(ctx)
pass # TODO
def _walk_if_we_could_use_generators(self, rootnode, rootpath=()):
# this is what we'd be doing if we didn't have the Deferreds and thus
# could use generators
yield rootpath, rootnode
for childname, childnode in rootnode.list().items():
childpath = rootpath + (childname,)
if IFileNode.providedBy(childnode):
yield childpath, childnode
elif IDirectoryNode.providedBy(childnode):
for res in self._walk_if_we_could_use_generators(childnode,
childpath):
yield res
def walk(self, rootnode, visitor, rootpath=()):
d = rootnode.list()
def _listed(listing):
return listing.items()
d.addCallback(_listed)
d.addCallback(self._handle_items, visitor, rootpath)
return d
def _handle_items(self, items, visitor, rootpath):
if not items:
return
childname, childnode = items[0]
childpath = rootpath + (childname,)
d = defer.maybeDeferred(visitor, childpath, childnode)
if IDirectoryNode.providedBy(childnode):
d.addCallback(lambda res: self.walk(childnode, visitor, childpath))
d.addCallback(lambda res:
self._handle_items(items[1:], visitor, rootpath))
return d
class LocalDirectoryDownloader(resource.Resource, DirnodeWalkerMixin):
def __init__(self, dirnode, localdir):
self._dirnode = dirnode
self._localdir = localdir
def _handle(self, path, node):
print "DONWLOADING", path, node
localfile = os.path.join(self._localdir, os.sep.join(path))
if IDirectoryNode.providedBy(node):
fileutil.make_dirs(localfile)
elif IFileNode.providedBy(node):
target = download.FileName(localfile)
return node.download(target)
def render(self, req):
d = self.walk(self._dirnode, self._handle)
def _done(res):
req.setHeader("content-type", "text/plain")
return "operation complete"
d.addCallback(_done)
return d
class DirectoryJSONMetadata(rend.Page):
def __init__(self, dirnode):
@ -441,11 +509,20 @@ class DELETEHandler(rend.Page):
self._name = name
def renderHTTP(self, ctx):
print "DELETEHandler.renderHTTP", self._name
req = inevow.IRequest(ctx)
d = self._node.delete(self._name)
def _done(res):
# what should this return??
return "%s deleted" % self._name
d.addCallback(_done)
def _trap_missing(f):
print "TRAPPED MISSING"
f.trap(KeyError)
req.setResponseCode(http.NOT_FOUND)
req.setHeader("content-type", "text/plain")
return "no such child %s" % self._name
d.addErrback(_trap_missing)
return d
class PUTHandler(rend.Page):
@ -521,17 +598,8 @@ class PUTHandler(rend.Page):
def _upload_localfile(self, node, localfile, name):
uploadable = upload.FileName(localfile)
d = self._uploader.upload(uploadable)
def _uploaded(uri):
print "SETTING URI", name, uri
d1 = node.set_uri(name, uri)
d1.addCallback(lambda res: uri)
return d1
d.addCallback(_uploaded)
def _done(uri):
log.msg("webish upload complete")
return uri
d.addCallback(_done)
d = node.add_file(name, uploadable)
d.addCallback(lambda filenode: filenode.get_uri())
return d
def _attach_uri(self, parentnode, contents, name):
@ -543,7 +611,40 @@ class PUTHandler(rend.Page):
return d
def _upload_localdir(self, node, localdir):
pass # TODO
# build up a list of files to upload
all_files = []
all_dirs = []
for root, dirs, files in os.walk(localdir):
path = tuple(root.split(os.sep))
for d in dirs:
all_dirs.append(path + (d,))
for f in files:
all_files.append(path + (f,))
d = defer.succeed(None)
for dir in all_dirs:
if dir:
d.addCallback(self._makedir, node, dir)
for f in all_files:
d.addCallback(self._upload_one_file, node, localdir, f)
return d
def _makedir(self, res, node, dir):
d = defer.succeed(None)
# get the parent. As long as os.walk gives us parents before
# children, this ought to work
d.addCallback(lambda res: node.get_child_at_path(dir[:-1]))
# then create the child directory
d.addCallback(lambda parent: parent.create_empty_directory(dir[-1]))
return d
def _upload_one_file(self, res, node, localdir, f):
# get the parent. We can be sure this exists because we already
# went through and created all the directories we require.
localfile = os.path.join(localdir, f)
d = node.get_child_at_path(f[:-1])
d.addCallback(self._upload_localfile, localfile, f[-1])
return d
class Manifest(rend.Page):
docFactory = getxmlfile("manifest.xhtml")
@ -595,10 +696,15 @@ class VDrive(rend.Page):
localfile = None
if "localfile" in req.args:
localfile = req.args["localfile"][0]
if localfile != os.path.abspath(localfile):
return NeedAbsolutePathError(), ()
localdir = None
if "localdir" in req.args:
localdir = req.args["localdir"][0]
if (localfile or localdir) and req.getHost().host != LOCALHOST:
if localdir != os.path.abspath(localdir):
return NeedAbsolutePathError(), ()
if localfile or localdir:
if req.getHost().host != LOCALHOST:
return NeedLocalhostError(), ()
# TODO: think about clobbering/revealing config files and node secrets
@ -651,13 +757,18 @@ class VDrive(rend.Page):
# the node must exist, and our operation will be performed on the
# node itself.
d = self.get_child_at_path(path)
d.addCallback(lambda node: POSTHandler(node), ())
def _got(node):
return POSTHandler(node), ()
d.addCallback(_got)
elif method == "DELETE":
# the node must exist, and our operation will be performed on its
# parent node.
assert path # you can't delete the root
print "AT DELETE"
d = self.get_child_at_path(path[:-1])
d.addCallback(lambda node: DELETEHandler(node, path[-1]), )
def _got(node):
return DELETEHandler(node, path[-1]), ()
d.addCallback(_got)
elif method in ("PUT",):
# the node may or may not exist, and our operation may involve
# all the ancestors of the node.