webish: add POST t=mutable, make it replace files in-place, add t=overwrite

docs/webapi.txt

@@ -324,6 +324,7 @@ c. POST forms
   t=upload
   name=childname  (optional)
   file=newfile
+
   This instructs the node to upload a file into the given directory. We need
   this because forms are the only way for a web browser to upload a file
   (browsers do not know how to do PUT or DELETE). The file's contents and the
@@ -331,13 +332,29 @@ c. POST forms
   used to upload a single file at a time. To avoid confusion, name= is not
   allowed to contain a slash (a 400 Bad Request error will result).
 
+
   POST $URL
   t=upload
   name=childname  (optional)
-  mutable="on"
+  mutable="true"
   file=newfile
+
   This instructs the node to upload a file into the given directory, using a
-  mutable file (SSK) rather than the usual immutable file (CHK).
+  mutable file (SSK) rather than the usual immutable file (CHK). As a result,
+  further operations to the same $URL will not cause the identity of the file
+  to change.
+
+
+  POST $URL
+  t=overwrite
+  file=newfile
+
+  This is used to replace the existing (mutable) file's contents with new
+  ones. It may only be used when $URL refers to a mutable file, as created by
+  POST $URL?t=upload&mutable=true, or PUT /uri?t=mutable . The name
+  associated with the uploaded file is ignored. TODO: rethink this, it's kind
+  of weird.
+
 
   POST $URL
   t=mkdir
@@ -346,6 +363,7 @@ c. POST forms
   This instructs the node to create a new empty directory. The name of the
   new child directory will be included in the form's arguments.
 
+
   POST $URL
   t=uri
   name=childname
@@ -355,6 +373,7 @@ c. POST forms
   like the PUT $URL?t=uri method). The name and URI of the new child
   will be included in the form's arguments.
 
+
   POST $URL
   t=delete
   name=childname
@@ -362,6 +381,7 @@ c. POST forms
   This instructs the node to delete a file from the given directory. The name
   of the child to be deleted will be included in the form's arguments.
 
+
   POST $URL
   t=rename
   from_name=oldchildname

src/allmydata/test/common.py

@@ -81,7 +81,9 @@ class FakeMutableFileNode:
         self.storage_index = self.my_uri.storage_index
         return self
     def get_uri(self):
-        return self.my_uri
+        return self.my_uri.to_string()
+    def get_readonly_uri(self):
+        return self.my_uri.get_readonly().to_string()
     def is_readonly(self):
         return self.my_uri.is_readonly()
     def is_mutable(self):

src/allmydata/test/test_web.py

@@ -1,5 +1,6 @@
 
 import re, os.path, urllib
+import simplejson
 from twisted.application import service
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -8,7 +9,7 @@ from twisted.python import failure, log
 from allmydata import webish, interfaces, provisioning
 from allmydata.util import fileutil
 from allmydata.test.common import NonGridDirectoryNode, FakeCHKFileNode, FakeMutableFileNode, create_chk_filenode
-from allmydata.interfaces import IURI, INewDirectoryURI, IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI
+from allmydata.interfaces import IURI, INewDirectoryURI, IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
 
 # create a fake uploader/downloader, and a couple of fake dirnodes, then
 # create a webserver that works against them
@@ -130,15 +131,8 @@ class WebMixin(object):
     def failUnlessIsBarDotTxt(self, res):
         self.failUnlessEqual(res, self.BAR_CONTENTS)
 
-    def worlds_cheapest_json_decoder(self, json):
-        # don't write tests that use 'true' or 'false' as filenames
-        json = re.sub('false', 'False', json)
-        json = re.sub('true', 'True', json)
-        json = re.sub(r'\\/', '/', json)
-        return eval(json)
-
     def failUnlessIsBarJSON(self, res):
-        data = self.worlds_cheapest_json_decoder(res)
+        data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
         self.failUnlessEqual(data[0], "filenode")
         self.failUnless(isinstance(data[1], dict))
@@ -147,7 +141,7 @@ class WebMixin(object):
         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 
     def failUnlessIsFooJSON(self, res):
-        data = self.worlds_cheapest_json_decoder(res)
+        data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
         self.failUnlessEqual(data[0], "dirnode", res)
         self.failUnless(isinstance(data[1], dict))
@@ -894,6 +888,61 @@ class Web(WebMixin, unittest.TestCase):
                                                       self.NEWFILE_CONTENTS))
         return d
 
+    def test_POST_upload_mutable(self):
+        # this creates a mutable file
+        d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
+                      file=("new.txt", self.NEWFILE_CONTENTS))
+        fn = self._foo_node
+        d.addCallback(self.failUnlessURIMatchesChild, fn, "new.txt")
+        d.addCallback(lambda res:
+                      self.failUnlessChildContentsAre(fn, "new.txt",
+                                                      self.NEWFILE_CONTENTS))
+        d.addCallback(lambda res: self._foo_node.get("new.txt"))
+        def _got(newnode):
+            self.failUnless(IMutableFileNode.providedBy(newnode))
+            self.failUnless(newnode.is_mutable())
+            self.failIf(newnode.is_readonly())
+            self._mutable_uri = newnode.get_uri()
+        d.addCallback(_got)
+
+        # now upload it again and make sure that the URI doesn't change
+        NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
+        d.addCallback(lambda res:
+                      self.POST(self.public_url + "/foo", t="upload",
+                                mutable="true",
+                                file=("new.txt", NEWER_CONTENTS)))
+        d.addCallback(self.failUnlessURIMatchesChild, fn, "new.txt")
+        d.addCallback(lambda res:
+                      self.failUnlessChildContentsAre(fn, "new.txt",
+                                                      NEWER_CONTENTS))
+        d.addCallback(lambda res: self._foo_node.get("new.txt"))
+        def _got2(newnode):
+            self.failUnless(IMutableFileNode.providedBy(newnode))
+            self.failUnless(newnode.is_mutable())
+            self.failIf(newnode.is_readonly())
+            self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
+        d.addCallback(_got2)
+
+        # also test t=overwrite while we're here
+        EVEN_NEWER_CONTENTS = NEWER_CONTENTS + "even newer\n"
+        d.addCallback(lambda res:
+                      self.POST(self.public_url + "/foo/new.txt",
+                                t="overwrite",
+                                file=("new.txt", EVEN_NEWER_CONTENTS)))
+        d.addCallback(self.failUnlessURIMatchesChild, fn, "new.txt")
+        d.addCallback(lambda res:
+                      self.failUnlessChildContentsAre(fn, "new.txt",
+                                                      EVEN_NEWER_CONTENTS))
+        d.addCallback(lambda res: self._foo_node.get("new.txt"))
+        def _got3(newnode):
+            self.failUnless(IMutableFileNode.providedBy(newnode))
+            self.failUnless(newnode.is_mutable())
+            self.failIf(newnode.is_readonly())
+            self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
+        d.addCallback(_got3)
+
+        return d
+
     def test_POST_upload_replace(self):
         d = self.POST(self.public_url + "/foo", t="upload",
                       file=("bar.txt", self.NEWFILE_CONTENTS))
@@ -1118,7 +1167,7 @@ class Web(WebMixin, unittest.TestCase):
         return d
 
     def failUnlessIsEmptyJSON(self, res):
-        data = self.worlds_cheapest_json_decoder(res)
+        data = simplejson.loads(res)
         self.failUnlessEqual(data[0], "dirnode", data)
         self.failUnlessEqual(len(data[1]["children"]), 0)
 

src/allmydata/webish.py

@@ -788,13 +788,26 @@ class POSTHandler(rend.Page):
                 data = contents.file.read()
                 uploadable = upload.FileHandle(contents.file)
                 d = self._check_replacement(name)
-                d.addCallback(lambda res:
+                d.addCallback(lambda res: self._node.has_child(name))
-                              IClient(ctx).create_mutable_file(data))
+                def _checked(present):
-                def _uploaded(newnode):
+                    if present:
-                    d1 = self._node.set_node(name, newnode)
+                        # modify the existing one instead of creating a new
-                    d1.addCallback(lambda res: newnode.get_uri())
+                        # one
-                    return d1
+                        d2 = self._node.get(name)
-                d.addCallback(_uploaded)
+                        def _got_newnode(newnode):
+                            d3 = newnode.replace(data)
+                            d3.addCallback(lambda res: newnode.get_uri())
+                            return d3
+                        d2.addCallback(_got_newnode)
+                    else:
+                        d2 = IClient(ctx).create_mutable_file(data)
+                        def _uploaded(newnode):
+                            d1 = self._node.set_node(name, newnode)
+                            d1.addCallback(lambda res: newnode.get_uri())
+                            return d1
+                        d2.addCallback(_uploaded)
+                    return d2
+                d.addCallback(_checked)
             else:
                 contents = req.fields["file"]
                 name = name or contents.filename
@@ -814,7 +827,8 @@ class POSTHandler(rend.Page):
             # SDMF: files are small, and we can only upload data.
             contents.file.seek(0)
             data = contents.file.read()
-            d = self._node.get(name)
+            # TODO: 'name' handling needs review
+            d = defer.succeed(self._node)
             def _got_child(child_node):
                 child_node.replace(data)
                 return child_node.get_uri()