ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-18 18:56:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

webapi: handle format=, remove mutable-type=

Browse Source 
* fix CLI commands (put, mkdir) to send format=, not mutable-type=
* fix tests
* test_cli: fix tests that observe t=json output, don't ignore failures in
  'tahoe put'
* fix handling of version= to make it easier to use the default
* interpret ?mutable=true&format=MDMF as MDMF, not SDMF
This commit is contained in:
Brian Warner 2011-10-13 09:29:51 -07:00
parent bd642739cb
commit dad354b275
11 changed files with 113 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/allmydata/client.py
View File

@ -499,7 +499,7 @@ class Client(node.Node, pollmixin.PollMixin):
# may get an opaque node if there were any problems.
return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)
def create_dirnode(self, initial_children={}, version=SDMF_VERSION):
def create_dirnode(self, initial_children={}, version=None):
d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
return d

8
src/allmydata/nodemaker.py
View File

@ -109,8 +109,9 @@ class NodeMaker:
return self._create_dirnode(filenode)
return None
def create_mutable_file(self, contents=None, keysize=None,
version=SDMF_VERSION):
def create_mutable_file(self, contents=None, keysize=None, version=None):
if version is None:
version = SDMF_VERSION
n = MutableFileNode(self.storage_broker, self.secret_holder,
self.default_encoding_parameters, self.history)
d = self.key_generator.generate(keysize)
@ -118,8 +119,7 @@ class NodeMaker:
d.addCallback(lambda res: n)
return d
def create_new_mutable_directory(self, initial_children={},
version=SDMF_VERSION):
def create_new_mutable_directory(self, initial_children={}, version=None):
# initial_children must have metadata (i.e. {} instead of None)
for (name, (node, metadata)) in initial_children.iteritems():
precondition(isinstance(metadata, dict),

4
src/allmydata/scripts/tahoe_mkdir.py
View File

@ -23,7 +23,7 @@ def mkdir(options):
# create a new unlinked directory
url = nodeurl + "uri?t=mkdir"
if options["mutable-type"]:
url += "&mutable-type=%s" % urllib.quote(options['mutable-type'])
url += "&format=%s" % urllib.quote(options['mutable-type'])
resp = do_http("POST", url)
rc = check_http_error(resp, stderr)
if rc:
@ -40,7 +40,7 @@ def mkdir(options):
url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap),
urllib.quote(path))
if options['mutable-type']:
url += "&mutable-type=%s" % urllib.quote(options['mutable-type'])
url += "&format=%s" % urllib.quote(options['mutable-type'])
resp = do_http("POST", url)
check_http_error(resp, stderr)

8
src/allmydata/scripts/tahoe_put.py
View File

@ -63,11 +63,15 @@ def put(options):
else:
# unlinked upload
url = nodeurl + "uri"
file_format = None
if mutable:
url += "?mutable=true"
file_format = "SDMF"
if mutable_type:
assert mutable
url += "&mutable-type=%s" % mutable_type
file_format = mutable_type.upper()
if file_format:
url += "?format=%s" % file_format
if from_file:
infileobj = open(os.path.expanduser(from_file), "rb")

17
src/allmydata/test/test_cli.py
View File

@ -1145,7 +1145,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
def _check_mdmf_json(self, (rc, json, err)):
self.failUnlessEqual(rc, 0)
self.failUnlessEqual(err, "")
self.failUnlessIn('"mutable-type": "mdmf"', json)
self.failUnlessIn('"format": "mdmf"', json)
# We also want a valid MDMF cap to be in the json.
self.failUnlessIn("URI:MDMF", json)
self.failUnlessIn("URI:MDMF-RO", json)
@ -1154,7 +1154,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
def _check_sdmf_json(self, (rc, json, err)):
self.failUnlessEqual(rc, 0)
self.failUnlessEqual(err, "")
self.failUnlessIn('"mutable-type": "sdmf"', json)
self.failUnlessIn('"format": "sdmf"', json)
# We also want to see the appropriate SDMF caps.
self.failUnlessIn("URI:SSK", json)
self.failUnlessIn("URI:SSK-RO", json)
@ -1171,6 +1171,9 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
def _put_and_ls(ign, mutable_type, filename):
d2 = self.do_cli("put", "--mutable", "--mutable-type="+mutable_type,
fn1, filename)
def _dont_fail((rc, out, err)):
self.failUnlessEqual(rc, 0)
d2.addCallback(_dont_fail)
d2.addCallback(lambda ign: self.do_cli("ls", "--json", filename))
return d2
@ -1615,8 +1618,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failUnlessIn(self._sdmf_uri, out)
self.failUnlessIn(self._sdmf_readonly_uri, out)
self.failUnlessIn(self._imm_uri, out)
self.failUnlessIn('"mutable-type": "sdmf"', out)
self.failUnlessIn('"mutable-type": "mdmf"', out)
self.failUnlessIn('"format": "sdmf"', out)
self.failUnlessIn('"format": "mdmf"', out)
d.addCallback(_got_json)
return d
@ -3315,7 +3318,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
d2.addCallback(lambda ign: self.do_cli("ls", "--json", dirname))
d2.addCallback(_check, uri_prefix)
d2.addCallback(lambda ign: self.do_cli("ls", "--json", self._filecap))
d2.addCallback(_check, '"mutable-type": "%s"' % (mutable_type.lower(),))
d2.addCallback(_check, '"format": "%s"' % (mutable_type.lower(),))
return d2
d.addCallback(_mkdir, "sdmf", "URI:DIR2", "tahoe:foo")
@ -3345,13 +3348,13 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
d.addCallback(_stash_dircap)
d.addCallback(lambda res: self.do_cli("ls", "--json",
self._filecap))
d.addCallback(_check, '"mutable-type": "sdmf"')
d.addCallback(_check, '"format": "sdmf"')
d.addCallback(lambda res: self.do_cli("mkdir", "--mutable-type=mdmf"))
d.addCallback(_check, "URI:DIR2-MDMF")
d.addCallback(_stash_dircap)
d.addCallback(lambda res: self.do_cli("ls", "--json",
self._filecap))
d.addCallback(_check, '"mutable-type": "mdmf"')
d.addCallback(_check, '"format": "mdmf"')
return d
def test_mkdir_bad_mutable_type(self):

23
src/allmydata/test/test_web.py
View File

@ -2123,17 +2123,27 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
filename = format + ".txt"
d = self.POST("/uri?t=upload&format=" + format,
file=(filename, self.NEWFILE_CONTENTS * 300000))
def _got_filecap(filecap):
self.failUnless(filecap.startswith(uri_prefix))
def _got_results(results):
if format.upper() in ("SDMF", "MDMF"):
# webapi.rst says this returns a filecap
filecap = results
else:
# for immutable, it returns an "upload results page", and
# the filecap is buried inside
line = [l for l in results.split("\n") if "URI: " in l][0]
mo = re.search(r'<span>([^<]+)</span>', line)
filecap = mo.group(1)
self.failUnless(filecap.startswith(uri_prefix),
(uri_prefix, filecap))
return self.GET("/uri/%s?t=json" % filecap)
d.addCallback(_got_filecap)
d.addCallback(_got_results)
def _got_json(json):
data = simplejson.loads(json)
data = data[1]
self.failUnlessIn("format", data)
self.failUnlessEqual(data["format"], format)
self.failUnlessEqual(data["format"], format.lower())
d.addCallback(_got_json)
return d
d = defer.succeed(None)
d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
@ -2165,8 +2175,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
data = simplejson.loads(json)
data = data[1]
self.failUnlessIn("format", data)
self.failUnlessEqual(data["format"], format)
self.failUnlessEqual(data["format"], format.lower())
d.addCallback(_got_json)
return d
d = defer.succeed(None)
d.addCallback(_check_upload, "chk", "URI:CHK")

28
src/allmydata/web/common.py
View File

@ -34,17 +34,29 @@ def parse_replace_arg(replace):
return boolean_of_arg(replace)
def parse_mutable_type_arg(arg):
def get_format(req, default="CHK"):
arg = get_arg(req, "format", None)
if not arg:
return None # interpreted by the caller as "let the nodemaker decide"
if boolean_of_arg(get_arg(req, "mutable", "false")):
return "SDMF"
return default
if arg.upper() == "CHK":
return "CHK"
elif arg.upper() == "SDMF":
return "SDMF"
elif arg.upper() == "MDMF":
return "MDMF"
else:
raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg,
http.BAD_REQUEST)
arg = arg.lower()
if arg == "mdmf":
return MDMF_VERSION
elif arg == "sdmf":
def get_mutable_type(file_format): # accepts result of get_format()
if file_format == "SDMF":
return SDMF_VERSION
return "invalid"
elif file_format == "MDMF":
return MDMF_VERSION
else:
return None
def parse_offset_arg(offset):

45
src/allmydata/web/directory.py
View File

@ -25,7 +25,7 @@ from allmydata.web.common import text_plain, WebError, \
boolean_of_arg, get_arg, get_root, parse_replace_arg, \
should_create_intermediate_directories, \
getxmlfile, RenderMixin, humanize_failure, convert_children_json, \
parse_mutable_type_arg
get_format, get_mutable_type
from allmydata.web.filenode import ReplaceMeMixin, \
FileNodeHandler, PlaceHolderNodeHandler
from allmydata.web.check_results import CheckResults, \
@ -107,17 +107,12 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
kids_json = req.content.read()
kids = convert_children_json(self.client.nodemaker,
kids_json)
file_format = get_format(req, None)
mutable = True
mt = get_mutable_type(file_format)
if t == "mkdir-immutable":
mutable = False
mt = None
if mutable:
arg = get_arg(req, "mutable-type", None)
mt = parse_mutable_type_arg(arg)
if mt is "invalid":
raise WebError("Unknown type: %s" % arg,
http.BAD_REQUEST)
d = self.node.create_subdirectory(name, kids,
mutable=mutable,
mutable_version=mt)
@ -251,15 +246,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
name = name.decode("utf-8")
replace = boolean_of_arg(get_arg(req, "replace", "true"))
kids = {}
arg = get_arg(req, "mutable-type", None)
mt = parse_mutable_type_arg(arg)
if mt is not None and mt is not "invalid":
d = self.node.create_subdirectory(name, kids, overwrite=replace,
mt = get_mutable_type(get_format(req, None))
d = self.node.create_subdirectory(name, kids, overwrite=replace,
mutable_version=mt)
elif mt is "invalid":
raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
else:
d = self.node.create_subdirectory(name, kids, overwrite=replace)
d.addCallback(lambda child: child.get_uri()) # TODO: urlencode
return d
@ -275,15 +264,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
req.content.seek(0)
kids_json = req.content.read()
kids = convert_children_json(self.client.nodemaker, kids_json)
arg = get_arg(req, "mutable-type", None)
mt = parse_mutable_type_arg(arg)
if mt is not None and mt is not "invalid":
d = self.node.create_subdirectory(name, kids, overwrite=False,
mutable_version=mt)
elif mt is "invalid":
raise WebError("Unknown type: %s" % arg)
else:
d = self.node.create_subdirectory(name, kids, overwrite=False)
mt = get_mutable_type(get_format(req, None))
d = self.node.create_subdirectory(name, kids, overwrite=False,
mutable_version=mt)
d.addCallback(lambda child: child.get_uri()) # TODO: urlencode
return d
@ -882,16 +865,16 @@ def DirectoryJSONMetadata(ctx, dirnode):
kiddata = ("filenode", {'size': childnode.get_size(),
'mutable': childnode.is_mutable(),
})
if childnode.is_mutable() and \
childnode.get_version() is not None:
if childnode.is_mutable():
mutable_type = childnode.get_version()
assert mutable_type in (SDMF_VERSION, MDMF_VERSION)
if mutable_type == MDMF_VERSION:
mutable_type = "mdmf"
file_format = "mdmf"
else:
mutable_type = "sdmf"
kiddata[1]['mutable-type'] = mutable_type
file_format = "sdmf"
else:
file_format = "chk"
kiddata[1]['format'] = file_format
elif IDirectoryNode.providedBy(childnode):
kiddata = ("dirnode", {'mutable': childnode.is_mutable()})

36
src/allmydata/web/filenode.py
View File

@ -18,7 +18,7 @@ from allmydata.blacklist import FileProhibited, ProhibitedNode
from allmydata.web.common import text_plain, WebError, RenderMixin, \
boolean_of_arg, get_arg, should_create_intermediate_directories, \
MyExceptionHandler, parse_replace_arg, parse_offset_arg, \
parse_mutable_type_arg
get_format, get_mutable_type
from allmydata.web.check_results import CheckResults, \
CheckAndRepairResults, LiteralCheckResults
from allmydata.web.info import MoreInfo
@ -26,13 +26,9 @@ from allmydata.web.info import MoreInfo
class ReplaceMeMixin:
def replace_me_with_a_child(self, req, client, replace):
# a new file is being uploaded in our place.
mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
if mutable:
arg = get_arg(req, "mutable-type", None)
mutable_type = parse_mutable_type_arg(arg)
if mutable_type is "invalid":
raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
file_format = get_format(req, "CHK")
if file_format in ("SDMF", "MDMF"):
mutable_type = get_mutable_type(file_format)
data = MutableFileHandle(req.content)
d = client.create_mutable_file(data, version=mutable_type)
def _uploaded(newnode):
@ -42,6 +38,7 @@ class ReplaceMeMixin:
return d2
d.addCallback(_uploaded)
else:
assert file_format == "CHK"
uploadable = FileHandle(req.content, convergence=client.convergence)
d = self.parentnode.add_file(self.name, uploadable,
overwrite=replace)
@ -70,15 +67,10 @@ class ReplaceMeMixin:
def replace_me_with_a_formpost(self, req, client, replace):
# create a new file, maybe mutable, maybe immutable
mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
# create an immutable file
file_format = get_format(req, "CHK")
contents = req.fields["file"]
if mutable:
arg = get_arg(req, "mutable-type", None)
mutable_type = parse_mutable_type_arg(arg)
if mutable_type is "invalid":
raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
if file_format in ("SDMF", "MDMF"):
mutable_type = get_mutable_type(file_format)
uploadable = MutableFileHandle(contents.file)
d = client.create_mutable_file(uploadable, version=mutable_type)
def _uploaded(newnode):
@ -518,14 +510,16 @@ def FileJSONMetadata(ctx, filenode, edge_metadata):
if edge_metadata is not None:
data[1]['metadata'] = edge_metadata
if filenode.is_mutable() and filenode.get_version() is not None:
if filenode.is_mutable():
mutable_type = filenode.get_version()
assert mutable_type in (MDMF_VERSION, SDMF_VERSION)
assert mutable_type in (SDMF_VERSION, MDMF_VERSION)
if mutable_type == MDMF_VERSION:
mutable_type = "mdmf"
file_format = "mdmf"
else:
mutable_type = "sdmf"
data[1]['mutable-type'] = mutable_type
file_format = "sdmf"
else:
file_format = "chk"
data[1]['format'] = file_format
return text_plain(simplejson.dumps(data, indent=1) + "\n", ctx)

22
src/allmydata/web/root.py
View File

@ -16,7 +16,7 @@ from allmydata.interfaces import IFileNode
from allmydata.web import filenode, directory, unlinked, status, operations
from allmydata.web import reliability, storage
from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \
get_arg, RenderMixin, boolean_of_arg, parse_mutable_type_arg
get_arg, RenderMixin, get_format, get_mutable_type
class URIHandler(RenderMixin, rend.Page):
@ -45,14 +45,9 @@ class URIHandler(RenderMixin, rend.Page):
# "PUT /uri?t=mkdir" to create an unlinked directory
t = get_arg(req, "t", "").strip()
if t == "":
mutable = boolean_of_arg(get_arg(req, "mutable", "false").strip())
if mutable:
arg = get_arg(req, "mutable-type", None)
version = parse_mutable_type_arg(arg)
if version == "invalid":
errmsg = "Unknown type: %s" % arg
raise WebError(errmsg, http.BAD_REQUEST)
file_format = get_format(req, "CHK")
if file_format in ("SDMF", "MDMF"):
version = get_mutable_type(file_format)
return unlinked.PUTUnlinkedSSK(req, self.client, version)
else:
return unlinked.PUTUnlinkedCHK(req, self.client)
@ -69,12 +64,9 @@ class URIHandler(RenderMixin, rend.Page):
req = IRequest(ctx)
t = get_arg(req, "t", "").strip()
if t in ("", "upload"):
mutable = bool(get_arg(req, "mutable", "").strip())
if mutable:
arg = get_arg(req, "mutable-type", None)
version = parse_mutable_type_arg(arg)
if version is "invalid":
raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
file_format = get_format(req)
if file_format in ("SDMF", "MDMF"):
version = get_mutable_type(file_format)
return unlinked.POSTUnlinkedSSK(req, self.client, version)
else:
return unlinked.POSTUnlinkedCHK(req, self.client)

36
src/allmydata/web/unlinked.py
View File

@ -6,7 +6,7 @@ from nevow import rend, url, tags as T
from allmydata.immutable.upload import FileHandle
from allmydata.mutable.publish import MutableFileHandle
from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \
convert_children_json, WebError, parse_mutable_type_arg
convert_children_json, WebError, get_format, get_mutable_type
from allmydata.web import status
def PUTUnlinkedCHK(req, client):
@ -27,15 +27,14 @@ def PUTUnlinkedSSK(req, client, version):
def PUTUnlinkedCreateDirectory(req, client):
# "PUT /uri?t=mkdir", to create an unlinked directory.
arg = get_arg(req, "mutable-type", None)
mt = parse_mutable_type_arg(arg)
if mt is not None and mt is not "invalid":
d = client.create_dirnode(version=mt)
elif mt is "invalid":
msg = "Unknown type: %s" % arg
raise WebError(msg, http.BAD_REQUEST)
else:
d = client.create_dirnode()
file_format = get_format(req, None)
if file_format == "CHK":
raise WebError("format=CHK not currently accepted for PUT /uri?t=mkdir",
http.BAD_REQUEST)
mt = None
if file_format:
mt = get_mutable_type(file_format)
d = client.create_dirnode(version=mt)
d.addCallback(lambda dirnode: dirnode.get_uri())
# XXX add redirect_to_result
return d
@ -112,15 +111,14 @@ def POSTUnlinkedCreateDirectory(req, client):
raise WebError("t=mkdir does not accept children=, "
"try t=mkdir-with-children instead",
http.BAD_REQUEST)
arg = get_arg(req, "mutable-type", None)
mt = parse_mutable_type_arg(arg)
if mt is not None and mt is not "invalid":
d = client.create_dirnode(version=mt)
elif mt is "invalid":
msg = "Unknown type: %s" % arg
raise WebError(msg, http.BAD_REQUEST)
else:
d = client.create_dirnode()
file_format = get_format(req, None)
if file_format == "CHK":
raise WebError("format=CHK not currently accepted for POST /uri?t=mkdir",
http.BAD_REQUEST)
mt = None
if file_format:
mt = get_mutable_type(file_format)
d = client.create_dirnode(version=mt)
redirect = get_arg(req, "redirect_to_result", "false")
if boolean_of_arg(redirect):
def _then_redir(res):