webapi: handle format=, remove mutable-type=

* fix CLI commands (put, mkdir) to send format=, not mutable-type=
* fix tests
* test_cli: fix tests that observe t=json output, don't ignore failures in
  'tahoe put'
* fix handling of version= to make it easier to use the default
* interpret ?mutable=true&format=MDMF as MDMF, not SDMF

src/allmydata/client.py

@@ -499,7 +499,7 @@ class Client(node.Node, pollmixin.PollMixin):
         # may get an opaque node if there were any problems.
         return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)
 
-    def create_dirnode(self, initial_children={}, version=SDMF_VERSION):
+    def create_dirnode(self, initial_children={}, version=None):
         d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
         return d
 

src/allmydata/nodemaker.py

@@ -109,8 +109,9 @@ class NodeMaker:
             return self._create_dirnode(filenode)
         return None
 
-    def create_mutable_file(self, contents=None, keysize=None,
+    def create_mutable_file(self, contents=None, keysize=None, version=None):
-                            version=SDMF_VERSION):
+        if version is None:
+            version = SDMF_VERSION
         n = MutableFileNode(self.storage_broker, self.secret_holder,
                             self.default_encoding_parameters, self.history)
         d = self.key_generator.generate(keysize)
@@ -118,8 +119,7 @@ class NodeMaker:
         d.addCallback(lambda res: n)
         return d
 
-    def create_new_mutable_directory(self, initial_children={},
+    def create_new_mutable_directory(self, initial_children={}, version=None):
-                                     version=SDMF_VERSION):
         # initial_children must have metadata (i.e. {} instead of None)
         for (name, (node, metadata)) in initial_children.iteritems():
             precondition(isinstance(metadata, dict),

src/allmydata/scripts/tahoe_mkdir.py

@@ -23,7 +23,7 @@ def mkdir(options):
         # create a new unlinked directory
         url = nodeurl + "uri?t=mkdir"
         if options["mutable-type"]:
-            url += "&mutable-type=%s" % urllib.quote(options['mutable-type'])
+            url += "&format=%s" % urllib.quote(options['mutable-type'])
         resp = do_http("POST", url)
         rc = check_http_error(resp, stderr)
         if rc:
@@ -40,7 +40,7 @@ def mkdir(options):
     url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap),
                                            urllib.quote(path))
     if options['mutable-type']:
-        url += "&mutable-type=%s" % urllib.quote(options['mutable-type'])
+        url += "&format=%s" % urllib.quote(options['mutable-type'])
 
     resp = do_http("POST", url)
     check_http_error(resp, stderr)

src/allmydata/scripts/tahoe_put.py

@@ -63,11 +63,15 @@ def put(options):
     else:
         # unlinked upload
         url = nodeurl + "uri"
+
+    file_format = None
     if mutable:
-        url += "?mutable=true"
+        file_format = "SDMF"
     if mutable_type:
         assert mutable
-        url += "&mutable-type=%s" % mutable_type
+        file_format = mutable_type.upper()
+    if file_format:
+        url += "?format=%s" % file_format
 
     if from_file:
         infileobj = open(os.path.expanduser(from_file), "rb")

src/allmydata/test/test_cli.py

@@ -1145,7 +1145,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
     def _check_mdmf_json(self, (rc, json, err)):
          self.failUnlessEqual(rc, 0)
          self.failUnlessEqual(err, "")
-         self.failUnlessIn('"mutable-type": "mdmf"', json)
+         self.failUnlessIn('"format": "mdmf"', json)
          # We also want a valid MDMF cap to be in the json.
          self.failUnlessIn("URI:MDMF", json)
          self.failUnlessIn("URI:MDMF-RO", json)
@@ -1154,7 +1154,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
     def _check_sdmf_json(self, (rc, json, err)):
         self.failUnlessEqual(rc, 0)
         self.failUnlessEqual(err, "")
-        self.failUnlessIn('"mutable-type": "sdmf"', json)
+        self.failUnlessIn('"format": "sdmf"', json)
         # We also want to see the appropriate SDMF caps.
         self.failUnlessIn("URI:SSK", json)
         self.failUnlessIn("URI:SSK-RO", json)
@@ -1171,6 +1171,9 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _put_and_ls(ign, mutable_type, filename):
             d2 = self.do_cli("put", "--mutable", "--mutable-type="+mutable_type,
                              fn1, filename)
+            def _dont_fail((rc, out, err)):
+                self.failUnlessEqual(rc, 0)
+            d2.addCallback(_dont_fail)
             d2.addCallback(lambda ign: self.do_cli("ls", "--json", filename))
             return d2
 
@@ -1615,8 +1618,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessIn(self._sdmf_uri, out)
             self.failUnlessIn(self._sdmf_readonly_uri, out)
             self.failUnlessIn(self._imm_uri, out)
-            self.failUnlessIn('"mutable-type": "sdmf"', out)
+            self.failUnlessIn('"format": "sdmf"', out)
-            self.failUnlessIn('"mutable-type": "mdmf"', out)
+            self.failUnlessIn('"format": "mdmf"', out)
         d.addCallback(_got_json)
         return d
 
@@ -3315,7 +3318,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
             d2.addCallback(lambda ign: self.do_cli("ls", "--json", dirname))
             d2.addCallback(_check, uri_prefix)
             d2.addCallback(lambda ign: self.do_cli("ls", "--json", self._filecap))
-            d2.addCallback(_check, '"mutable-type": "%s"' % (mutable_type.lower(),))
+            d2.addCallback(_check, '"format": "%s"' % (mutable_type.lower(),))
             return d2
 
         d.addCallback(_mkdir, "sdmf", "URI:DIR2", "tahoe:foo")
@@ -3345,13 +3348,13 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(_stash_dircap)
         d.addCallback(lambda res: self.do_cli("ls", "--json",
                                               self._filecap))
-        d.addCallback(_check, '"mutable-type": "sdmf"')
+        d.addCallback(_check, '"format": "sdmf"')
         d.addCallback(lambda res: self.do_cli("mkdir", "--mutable-type=mdmf"))
         d.addCallback(_check, "URI:DIR2-MDMF")
         d.addCallback(_stash_dircap)
         d.addCallback(lambda res: self.do_cli("ls", "--json",
                                               self._filecap))
-        d.addCallback(_check, '"mutable-type": "mdmf"')
+        d.addCallback(_check, '"format": "mdmf"')
         return d
 
     def test_mkdir_bad_mutable_type(self):

src/allmydata/test/test_web.py

@@ -2123,17 +2123,27 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             filename = format + ".txt"
             d = self.POST("/uri?t=upload&format=" + format,
                           file=(filename, self.NEWFILE_CONTENTS * 300000))
-            def _got_filecap(filecap):
+            def _got_results(results):
-                self.failUnless(filecap.startswith(uri_prefix))
+                if format.upper() in ("SDMF", "MDMF"):
+                    # webapi.rst says this returns a filecap
+                    filecap = results
+                else:
+                    # for immutable, it returns an "upload results page", and
+                    # the filecap is buried inside
+                    line = [l for l in results.split("\n") if "URI: " in l][0]
+                    mo = re.search(r'<span>([^<]+)</span>', line)
+                    filecap = mo.group(1)
+                self.failUnless(filecap.startswith(uri_prefix),
+                                (uri_prefix, filecap))
                 return self.GET("/uri/%s?t=json" % filecap)
-            d.addCallback(_got_filecap)
+            d.addCallback(_got_results)
             def _got_json(json):
                 data = simplejson.loads(json)
                 data = data[1]
                 self.failUnlessIn("format", data)
-                self.failUnlessEqual(data["format"], format)
+                self.failUnlessEqual(data["format"], format.lower())
             d.addCallback(_got_json)
-
+            return d
         d = defer.succeed(None)
         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
@@ -2165,8 +2175,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 data = simplejson.loads(json)
                 data = data[1]
                 self.failUnlessIn("format", data)
-                self.failUnlessEqual(data["format"], format)
+                self.failUnlessEqual(data["format"], format.lower())
             d.addCallback(_got_json)
+            return d
 
         d = defer.succeed(None)
         d.addCallback(_check_upload, "chk", "URI:CHK")

src/allmydata/web/common.py

@@ -34,17 +34,29 @@ def parse_replace_arg(replace):
         return boolean_of_arg(replace)
 
 
-def parse_mutable_type_arg(arg):
+def get_format(req, default="CHK"):
+    arg = get_arg(req, "format", None)
     if not arg:
-        return None # interpreted by the caller as "let the nodemaker decide"
+        if boolean_of_arg(get_arg(req, "mutable", "false")):
+            return "SDMF"
+        return default
+    if arg.upper() == "CHK":
+        return "CHK"
+    elif arg.upper() == "SDMF":
+        return "SDMF"
+    elif arg.upper() == "MDMF":
+        return "MDMF"
+    else:
+        raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg,
+                       http.BAD_REQUEST)
 
-    arg = arg.lower()
+def get_mutable_type(file_format): # accepts result of get_format()
-    if arg == "mdmf":
+    if file_format == "SDMF":
-        return MDMF_VERSION
-    elif arg == "sdmf":
         return SDMF_VERSION
-
+    elif file_format == "MDMF":
-    return "invalid"
+        return MDMF_VERSION
+    else:
+        return None
 
 
 def parse_offset_arg(offset):

src/allmydata/web/directory.py

@@ -25,7 +25,7 @@ from allmydata.web.common import text_plain, WebError, \
      boolean_of_arg, get_arg, get_root, parse_replace_arg, \
      should_create_intermediate_directories, \
      getxmlfile, RenderMixin, humanize_failure, convert_children_json, \
-     parse_mutable_type_arg
+     get_format, get_mutable_type
 from allmydata.web.filenode import ReplaceMeMixin, \
      FileNodeHandler, PlaceHolderNodeHandler
 from allmydata.web.check_results import CheckResults, \
@@ -107,17 +107,12 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
                         kids_json = req.content.read()
                         kids = convert_children_json(self.client.nodemaker,
                                                      kids_json)
+                    file_format = get_format(req, None)
                     mutable = True
+                    mt = get_mutable_type(file_format)
                     if t == "mkdir-immutable":
                         mutable = False
 
-                    mt = None
-                    if mutable:
-                        arg = get_arg(req, "mutable-type", None)
-                        mt = parse_mutable_type_arg(arg)
-                        if mt is "invalid":
-                            raise WebError("Unknown type: %s" % arg,
-                                           http.BAD_REQUEST)
                     d = self.node.create_subdirectory(name, kids,
                                                       mutable=mutable,
                                                       mutable_version=mt)
@@ -251,15 +246,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         name = name.decode("utf-8")
         replace = boolean_of_arg(get_arg(req, "replace", "true"))
         kids = {}
-        arg = get_arg(req, "mutable-type", None)
+        mt = get_mutable_type(get_format(req, None))
-        mt = parse_mutable_type_arg(arg)
+        d = self.node.create_subdirectory(name, kids, overwrite=replace,
-        if mt is not None and mt is not "invalid":
-            d = self.node.create_subdirectory(name, kids, overwrite=replace,
                                           mutable_version=mt)
-        elif mt is "invalid":
-            raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
-        else:
-            d = self.node.create_subdirectory(name, kids, overwrite=replace)
         d.addCallback(lambda child: child.get_uri()) # TODO: urlencode
         return d
 
@@ -275,15 +264,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         req.content.seek(0)
         kids_json = req.content.read()
         kids = convert_children_json(self.client.nodemaker, kids_json)
-        arg = get_arg(req, "mutable-type", None)
+        mt = get_mutable_type(get_format(req, None))
-        mt = parse_mutable_type_arg(arg)
+        d = self.node.create_subdirectory(name, kids, overwrite=False,
-        if mt is not None and mt is not "invalid":
+                                          mutable_version=mt)
-            d = self.node.create_subdirectory(name, kids, overwrite=False,
-                                              mutable_version=mt)
-        elif mt is "invalid":
-            raise WebError("Unknown type: %s" % arg)
-        else:
-            d = self.node.create_subdirectory(name, kids, overwrite=False)
         d.addCallback(lambda child: child.get_uri()) # TODO: urlencode
         return d
 
@@ -882,16 +865,16 @@ def DirectoryJSONMetadata(ctx, dirnode):
                 kiddata = ("filenode", {'size': childnode.get_size(),
                                         'mutable': childnode.is_mutable(),
                                         })
-                if childnode.is_mutable() and \
+                if childnode.is_mutable():
-                    childnode.get_version() is not None:
                     mutable_type = childnode.get_version()
                     assert mutable_type in (SDMF_VERSION, MDMF_VERSION)
-
                     if mutable_type == MDMF_VERSION:
-                        mutable_type = "mdmf"
+                        file_format = "mdmf"
                     else:
-                        mutable_type = "sdmf"
+                        file_format = "sdmf"
-                    kiddata[1]['mutable-type'] = mutable_type
+                else:
+                    file_format = "chk"
+                kiddata[1]['format'] = file_format
 
             elif IDirectoryNode.providedBy(childnode):
                 kiddata = ("dirnode", {'mutable': childnode.is_mutable()})

src/allmydata/web/filenode.py

@@ -18,7 +18,7 @@ from allmydata.blacklist import FileProhibited, ProhibitedNode
 from allmydata.web.common import text_plain, WebError, RenderMixin, \
      boolean_of_arg, get_arg, should_create_intermediate_directories, \
      MyExceptionHandler, parse_replace_arg, parse_offset_arg, \
-     parse_mutable_type_arg
+     get_format, get_mutable_type
 from allmydata.web.check_results import CheckResults, \
      CheckAndRepairResults, LiteralCheckResults
 from allmydata.web.info import MoreInfo
@@ -26,13 +26,9 @@ from allmydata.web.info import MoreInfo
 class ReplaceMeMixin:
     def replace_me_with_a_child(self, req, client, replace):
         # a new file is being uploaded in our place.
-        mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
+        file_format = get_format(req, "CHK")
-        if mutable:
+        if file_format in ("SDMF", "MDMF"):
-            arg = get_arg(req, "mutable-type", None)
+            mutable_type = get_mutable_type(file_format)
-            mutable_type = parse_mutable_type_arg(arg)
-            if mutable_type is "invalid":
-                raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
-
             data = MutableFileHandle(req.content)
             d = client.create_mutable_file(data, version=mutable_type)
             def _uploaded(newnode):
@@ -42,6 +38,7 @@ class ReplaceMeMixin:
                 return d2
             d.addCallback(_uploaded)
         else:
+            assert file_format == "CHK"
             uploadable = FileHandle(req.content, convergence=client.convergence)
             d = self.parentnode.add_file(self.name, uploadable,
                                          overwrite=replace)
@@ -70,15 +67,10 @@ class ReplaceMeMixin:
 
     def replace_me_with_a_formpost(self, req, client, replace):
         # create a new file, maybe mutable, maybe immutable
-        mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
+        file_format = get_format(req, "CHK")
-
-        # create an immutable file
         contents = req.fields["file"]
-        if mutable:
+        if file_format in ("SDMF", "MDMF"):
-            arg = get_arg(req, "mutable-type", None)
+            mutable_type = get_mutable_type(file_format)
-            mutable_type = parse_mutable_type_arg(arg)
-            if mutable_type is "invalid":
-                raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
             uploadable = MutableFileHandle(contents.file)
             d = client.create_mutable_file(uploadable, version=mutable_type)
             def _uploaded(newnode):
@@ -518,14 +510,16 @@ def FileJSONMetadata(ctx, filenode, edge_metadata):
     if edge_metadata is not None:
         data[1]['metadata'] = edge_metadata
 
-    if filenode.is_mutable() and filenode.get_version() is not None:
+    if filenode.is_mutable():
         mutable_type = filenode.get_version()
-        assert mutable_type in (MDMF_VERSION, SDMF_VERSION)
+        assert mutable_type in (SDMF_VERSION, MDMF_VERSION)
         if mutable_type == MDMF_VERSION:
-            mutable_type = "mdmf"
+            file_format = "mdmf"
         else:
-            mutable_type = "sdmf"
+            file_format = "sdmf"
-        data[1]['mutable-type'] = mutable_type
+    else:
+        file_format = "chk"
+    data[1]['format'] = file_format
 
     return text_plain(simplejson.dumps(data, indent=1) + "\n", ctx)
 

src/allmydata/web/root.py

@@ -16,7 +16,7 @@ from allmydata.interfaces import IFileNode
 from allmydata.web import filenode, directory, unlinked, status, operations
 from allmydata.web import reliability, storage
 from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \
-     get_arg, RenderMixin, boolean_of_arg, parse_mutable_type_arg
+     get_arg, RenderMixin, get_format, get_mutable_type
 
 
 class URIHandler(RenderMixin, rend.Page):
@@ -45,14 +45,9 @@ class URIHandler(RenderMixin, rend.Page):
         # "PUT /uri?t=mkdir" to create an unlinked directory
         t = get_arg(req, "t", "").strip()
         if t == "":
-            mutable = boolean_of_arg(get_arg(req, "mutable", "false").strip())
+            file_format = get_format(req, "CHK")
-            if mutable:
+            if file_format in ("SDMF", "MDMF"):
-                arg = get_arg(req, "mutable-type", None)
+                version = get_mutable_type(file_format)
-                version = parse_mutable_type_arg(arg)
-                if version == "invalid":
-                    errmsg = "Unknown type: %s" % arg
-                    raise WebError(errmsg, http.BAD_REQUEST)
-
                 return unlinked.PUTUnlinkedSSK(req, self.client, version)
             else:
                 return unlinked.PUTUnlinkedCHK(req, self.client)
@@ -69,12 +64,9 @@ class URIHandler(RenderMixin, rend.Page):
         req = IRequest(ctx)
         t = get_arg(req, "t", "").strip()
         if t in ("", "upload"):
-            mutable = bool(get_arg(req, "mutable", "").strip())
+            file_format = get_format(req)
-            if mutable:
+            if file_format in ("SDMF", "MDMF"):
-                arg = get_arg(req, "mutable-type", None)
+                version = get_mutable_type(file_format)
-                version = parse_mutable_type_arg(arg)
-                if version is "invalid":
-                    raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
                 return unlinked.POSTUnlinkedSSK(req, self.client, version)
             else:
                 return unlinked.POSTUnlinkedCHK(req, self.client)

src/allmydata/web/unlinked.py

@@ -6,7 +6,7 @@ from nevow import rend, url, tags as T
 from allmydata.immutable.upload import FileHandle
 from allmydata.mutable.publish import MutableFileHandle
 from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \
-     convert_children_json, WebError, parse_mutable_type_arg
+     convert_children_json, WebError, get_format, get_mutable_type
 from allmydata.web import status
 
 def PUTUnlinkedCHK(req, client):
@@ -27,15 +27,14 @@ def PUTUnlinkedSSK(req, client, version):
 
 def PUTUnlinkedCreateDirectory(req, client):
     # "PUT /uri?t=mkdir", to create an unlinked directory.
-    arg = get_arg(req, "mutable-type", None)
+    file_format = get_format(req, None)
-    mt = parse_mutable_type_arg(arg)
+    if file_format == "CHK":
-    if mt is not None and mt is not "invalid":
+        raise WebError("format=CHK not currently accepted for PUT /uri?t=mkdir",
-        d = client.create_dirnode(version=mt)
+                       http.BAD_REQUEST)
-    elif mt is "invalid":
+    mt = None
-        msg = "Unknown type: %s" % arg
+    if file_format:
-        raise WebError(msg, http.BAD_REQUEST)
+        mt = get_mutable_type(file_format)
-    else:
+    d = client.create_dirnode(version=mt)
-        d = client.create_dirnode()
     d.addCallback(lambda dirnode: dirnode.get_uri())
     # XXX add redirect_to_result
     return d
@@ -112,15 +111,14 @@ def POSTUnlinkedCreateDirectory(req, client):
             raise WebError("t=mkdir does not accept children=, "
                            "try t=mkdir-with-children instead",
                            http.BAD_REQUEST)
-    arg = get_arg(req, "mutable-type", None)
+    file_format = get_format(req, None)
-    mt = parse_mutable_type_arg(arg)
+    if file_format == "CHK":
-    if mt is not None and mt is not "invalid":
+        raise WebError("format=CHK not currently accepted for POST /uri?t=mkdir",
-        d = client.create_dirnode(version=mt)
+                       http.BAD_REQUEST)
-    elif mt is "invalid":
+    mt = None
-        msg = "Unknown type: %s" % arg
+    if file_format:
-        raise WebError(msg, http.BAD_REQUEST)
+        mt = get_mutable_type(file_format)
-    else:
+    d = client.create_dirnode(version=mt)
-        d = client.create_dirnode()
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):