a first crack at the "put" command-line

There are actually two versions in this patch, one of which requires twisted.web2 and the other of which uses the Python standard library's socket module.  The socketish one doesn't know when the web server is done so it hangs after doing its thing.  (Oh -- maybe I should add an HTTP header asking the web server to close the connection when finished.)  The web2ish one works better in that respect.  Neither of them handle error responses from the server very well yet.

After lunch I intend to finish the socketish one.

To try one, mv src/allmydata/scripts/tahoe_put-{socketish,web2ish}.py src/allmydata/scripts/tahoe_put.py .

If you want to try the web2ish one, and you can't find a web2 package to install, you can get one from:

http://allmydata.org/~zooko/repos/twistedweb2snarf/

src/allmydata/scripts/cli.py

@@ -2,7 +2,13 @@
 import os.path, sys
 from twisted.python import usage
 
-class VDriveOptions(usage.Options):
+class OptionsMixin(usage.Options):
+    optFlags = [
+        ["quiet", "q", "Operate silently."],
+        ["version", "V", "Display version numbers and exit."],
+        ]
+
+class VDriveOptions(OptionsMixin):
     optParameters = [
         ["vdrive", "d", "global",
          "which virtual drive to use: 'global' or 'private'"],
@@ -24,13 +30,27 @@ class GetOptions(VDriveOptions):
         return "%s get VDRIVE_FILE LOCAL_FILE" % (os.path.basename(sys.argv[0]),)
 
     longdesc = """Retrieve a file from the virtual drive and write it to the
-    local disk. If LOCAL_FILE is omitted or '-', the contents of the file
+    local filesystem. If LOCAL_FILE is omitted or '-', the contents of the file
     will be written to stdout."""
 
+class PutOptions(VDriveOptions):
+    def parseArgs(self, local_filename, vdrive_filename):
+        self['vdrive_filename'] = vdrive_filename
+        self['local_filename'] = local_filename
+
+    def getSynopsis(self):
+        return "%s put LOCAL_FILEVDRI VE_FILE" % (os.path.basename(sys.argv[0]),)
+
+    longdesc = """Put a file into the virtual drive (copying the file's
+    contents from the local filesystem). If LOCAL_FILE is omitted or '-', the
+    contents of the file will be read from stdin."""
+
+
 
 subCommands = [
     ["ls", None, ListOptions, "List a directory"],
     ["get", None, GetOptions, "Retrieve a file from the virtual drive."],
+    ["put", None, PutOptions, "Upload a file into the virtual drive."],
     ]
 
 def list(config, stdout, stderr):
@@ -59,8 +79,24 @@ def get(config, stdout, stderr):
                   (vdrive_filename, local_filename)
     return rc
 
+def put(config, stdout, stderr):
+    from allmydata.scripts import tahoe_put
+    vdrive_filename = config['vdrive_filename']
+    local_filename = config['local_filename']
+    if config['quiet']:
+        verbosity = 0
+    else:
+        verbosity = 2
+    rc = tahoe_put.put(config['server'],
+                       config['vdrive'],
+                       vdrive_filename,
+                       local_filename,
+                       verbosity)
+    return rc
+
 dispatch = {
     "ls": list,
     "get": get,
+    "put": put,
     }
 

src/allmydata/scripts/runner.py

@@ -3,16 +3,11 @@ import sys
 from cStringIO import StringIO
 from twisted.python import usage
 
-from allmydata.scripts import debug, create_node, startstop_node, cli
+import debug, create_node, startstop_node, cli
 
-class Options(usage.Options):
+class Options(cli.OptionsMixin):
     synopsis = "Usage:  allmydata <command> [command options]"
 
-    optFlags = [
-        ["quiet", "q", "Operate silently."],
-        ["version", "V", "Display version numbers and exit."],
-        ]
-
     subCommands = []
     subCommands += create_node.subCommands
     subCommands += startstop_node.subCommands

src/allmydata/scripts/tahoe_put-socketish.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import re, socket, sys
+
+SERVERURL_RE=re.compile("http://([^:]*)(:([1-9][0-9]*))?")
+
+def put(serverurl, vdrive, vdrive_fname, local_fname, verbosity):
+    """
+    @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
+
+    @return: a Deferred which eventually fires with the exit code
+    """
+    mo = SERVERURL_RE.match(serverurl)
+    if not mo:
+        raise ValueError("serverurl is required to look like \"http://HOSTNAMEORADDR:PORT\"")
+    host = mo.group(1)
+    port = int(mo.group(3))
+
+    url = "/vdrive/" + vdrive + "/"
+    if vdrive_fname:
+        url += vdrive_fname
+
+    if local_fname is None or local_fname == "-":
+        infileobj = sys.stdin
+    else:
+        infileobj = open(local_fname, "rb")
+
+    so = socket.socket()
+    so.connect((host, port,))
+
+    CHUNKSIZE=2**16
+    data = "PUT %s HTTP/1.1\r\nHostname: %s\r\n\r\n" % (url, host,)
+    while data:
+        try:
+            sent = so.send(data)
+            print "XXXXXXXX I just sent %s" % (data[:sent],)
+        except Exception, le:
+            print "BOOOOO le: %r" % (le,)
+            return -1
+
+        if sent == len(data):
+            data = infileobj.read(CHUNKSIZE)
+        else:
+            data = data[sent:]
+
+    respbuf = []
+    data = so.recv(CHUNKSIZE)
+    while data:
+        print "WHEEEEE okay now we've got some more data: %r" % (data,)
+        respbuf.append(data)
+        data = so.recv(CHUNKSIZE)
+
+    so.shutdown(socket.SHUT_WR)
+    data = so.recv(CHUNKSIZE)
+    while data:
+        print "WHEEEEE 22222 okay now we've got some more data: %r" % (data,)
+        respbuf.append(data)
+        data = so.recv(CHUNKSIZE)
+
+def main():
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option("-d", "--vdrive", dest="vdrive", default="global")
+    parser.add_option("-s", "--server", dest="server", default="http://tahoebs1.allmydata.com:8011")
+
+    (options, args) = parser.parse_args()
+
+    local_file = args[0]
+    vdrive_file = None
+    if len(args) > 1:
+        vdrive_file = args[1]
+
+    put(options.server, options.vdrive, vdrive_file, local_file)
+
+if __name__ == '__main__':
+    main()

src/allmydata/scripts/tahoe_put-web2ish.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+import re, sys
+
+from twisted import web2 
+from twisted.web2 import client, http, stream
+import twisted.web2.client.http
+
+from twisted.internet import defer, reactor, protocol
+
+SERVERURL_RE=re.compile("http://([^:]*)(:([1-9][0-9]*))?")
+
+def _put(serverurl, vdrive, vdrive_fname, local_fname, verbosity):
+    """
+    @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
+
+    @return: a Deferred which eventually fires with the exit code
+    """
+    mo = SERVERURL_RE.match(serverurl)
+    if not mo:
+        raise ValueError("serverurl is required to look like \"http://HOSTNAMEORADDR:PORT\"")
+    host = mo.group(1)
+    port = int(mo.group(3))
+
+    d = defer.Deferred()
+    
+    url = "/vdrive/" + vdrive + "/"
+    if vdrive_fname:
+        url += vdrive_fname
+
+    if local_fname is None or local_fname == "-":
+        infileobj = sys.stdin
+    else:
+        infileobj = open(local_fname, "rb")
+    instream = web2.stream.FileStream(infileobj)
+    
+    d2 = protocol.ClientCreator(reactor, web2.client.http.HTTPClientProtocol).connectTCP(host, port)
+
+    def got_resp(resp):
+        # If this isn't a 200 or 201, then write out the response data and
+        # exit with resp.code as our exit value.
+        if resp.code not in (200, 201,):
+            def writeit(data):
+                sys.stdout.write(data)
+
+            def exit(dummy):
+                d.errback(resp.code)
+
+            return web2.http.stream.readStream(resp.stream, writeit).addCallback(exit)
+
+        # If we are in quiet mode, then just exit with the resp.code.
+        if verbosity == 0:
+            d.callback(resp.code)
+            return
+
+        # Else, this is a successful request and we are not in quiet mode:
+        uribuffer = []
+        def gather_uri(data):
+            uribuffer.append(data)
+
+        def output_result(thingie):
+            uri = ''.join(uribuffer)
+            outbuf = []
+            if resp.code == 200:
+                outbuf.append("200 (OK); ")
+            elif resp.code == 201:
+                outbuf.append("201 (Created); ")
+             
+            if verbosity == 2:
+                if resp.code == 200:
+                    outbuf.append("modified existing mapping in vdrive %s of name %s to point to " % (vdrive, vdrive_fname,))
+                elif resp.code == 201:
+                    outbuf.append("created new mapping in vdrive %s of name %s to point to " % (vdrive, vdrive_fname,))
+
+            outbuf.append("URI: %s" % (uri,))
+        
+            sys.stdout.write(''.join(outbuf))
+
+            d.callback(resp.code)
+
+        web2.http.stream.readStream(resp.stream, gather_uri).addCallback(output_result)
+                
+    def send_req(proto):
+        proto.submitRequest(web2.client.http.ClientRequest('PUT', url, {}, instream)).addCallback(got_resp)
+
+    d2.addCallback(send_req)
+
+    return d
+
+def put(server, vdrive, vdrive_fname, local_fname, verbosity):
+    """
+    This starts the reactor, does the PUT command, waits for the result, stops
+    the reactor, and returns the exit code. 
+
+    @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
+
+    @return: the exit code
+    """
+    d = _put(server, vdrive, vdrive_fname, local_fname, verbosity)
+    exitcode = [ None ]
+    def exit(result):
+        exitcode[0] = result
+        reactor.stop()
+        return result
+        
+    d.addCallbacks(exit, exit)
+    reactor.run()
+    return exitcode[0]
+