offloaded: add a system test, make it pass. files are now being uploaded through the helper.

src/allmydata/client.py

@@ -151,13 +151,13 @@ class Client(node.Node, Referenceable, testutil.PollMixin):
             return
         h = Helper(os.path.join(self.basedir, "helper"))
         h.setServiceParent(self)
-        helper_furl = self.tub.registerReference(h)
         # TODO: this is confusing. BASEDIR/private/helper.furl is created by
         # the helper. BASEDIR/helper.furl is consumed by the client who wants
         # to use the helper. I like having the filename be the same, since
         # that makes 'cp' work smoothly, but the difference between config
         # inputs and generated outputs is hard to see.
-        self.write_private_config("helper.furl", helper_furl + "\n")
+        helper_furlfile = os.path.join(self.basedir, "private", "helper.furl")
+        self.tub.registerReference(h, furlFile=helper_furlfile)
 
     def remote_get_versions(self):
         return str(allmydata.__version__), str(self.OLDEST_SUPPORTED_VERSION)

src/allmydata/interfaces.py

@@ -1269,9 +1269,9 @@ class RIEncryptedUploadable(RemoteInterface):
         return None
 
     def read_encrypted(offset=long, length=long):
-        return str
+        return ListOf(str)
 
-    def get_plaintext_hashtree_leaves(first=int, last=int):
+    def get_plaintext_hashtree_leaves(first=int, last=int, num_segments=int):
         return ListOf(Hash)
 
     def get_plaintext_hash():

src/allmydata/offloaded.py

@@ -55,6 +55,7 @@ class CiphertextReader:
     def __init__(self, remote_reader, storage_index):
         self.rr = remote_reader
         self.storage_index = storage_index
+        self._offset = 0
 
     def get_size(self):
         return self.rr.callRemote("get_size")
@@ -65,7 +66,12 @@ class CiphertextReader:
     def set_serialized_encoding_parameters(self, params):
         pass # ??
     def read_encrypted(self, length):
-        return self.rr.callRemote("read_encrypted", length)
+        d = self.rr.callRemote("read_encrypted", self._offset, length)
+        def _done(strings):
+            self._offset += sum([len(data) for data in strings])
+            return strings
+        d.addCallback(_done)
+        return d
     def get_plaintext_hashtree_leaves(self, first, last, num_segments):
         return self.rr.callRemote("get_plaintext_hashtree_leaves",
                                   first, last, num_segments)

src/allmydata/test/test_helper.py

@@ -105,4 +105,3 @@ class AssistedUpload(unittest.TestCase):
         d.addCallback(_uploaded)
 
         return d
-

src/allmydata/test/test_system.py

@@ -64,23 +64,40 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase):
         q = self.introducer
         self.introducer_furl = q.introducer_url
         self.clients = []
+        basedirs = []
         for i in range(self.numclients):
             basedir = self.getdir("client%d" % i)
+            basedirs.append(basedir)
             fileutil.make_dirs(basedir)
             if i == 0:
+                # client[0] runs a webserver and a helper
                 open(os.path.join(basedir, "webport"), "w").write("tcp:0:interface=127.0.0.1")
+                open(os.path.join(basedir, "run_helper"), "w").write("yes\n")
             if self.createprivdir:
                 fileutil.make_dirs(os.path.join(basedir, "private"))
                 open(os.path.join(basedir, "private", "root_dir.cap"), "w")
             open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl)
 
-        # this starts all the clients
-        for i in range(self.numclients):
-            basedir = self.getdir("client%d" % i)
-            c = self.add_service(client.Client(basedir=basedir))
-            self.clients.append(c)
-        log.msg("STARTING")
-        d = self.wait_for_connections()
+        # start client[0], wait for it's tub to be ready (at which point it
+        # will have registered the helper furl).
+        c = self.add_service(client.Client(basedir=basedirs[0]))
+        self.clients.append(c)
+        d = c.when_tub_ready()
+        def _ready(res):
+            f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
+            helper_furl = f.read()
+            f.close()
+            f = open(os.path.join(basedirs[3],"helper.furl"), "w")
+            f.write(helper_furl)
+            f.close()
+
+            # this starts the rest of the clients
+            for i in range(1, self.numclients):
+                c = self.add_service(client.Client(basedir=basedirs[i]))
+                self.clients.append(c)
+            log.msg("STARTING")
+            return self.wait_for_connections()
+        d.addCallback(_ready)
         def _connected(res):
             log.msg("CONNECTED")
             # now find out where the web port was
@@ -239,6 +256,20 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase):
             d1.addBoth(_baduri_should_fail)
             return d1
         d.addCallback(_download_nonexistent_uri)
+
+        def _upload_with_helper(res):
+            DATA = "Data that needs help to upload" * 1000
+            u = upload.Data(DATA)
+            d = self.clients[3].upload(u)
+            def _uploaded(uri):
+                return self.downloader.download_to_data(uri)
+            d.addCallback(_uploaded)
+            def _check(newdata):
+                self.failUnlessEqual(newdata, DATA)
+            d.addCallback(_check)
+            return d
+        d.addCallback(_upload_with_helper)
+
         return d
     test_upload_and_download.timeout = 4800
 

src/allmydata/upload.py

@@ -555,15 +555,18 @@ class RemoteEncryptedUploabable(Referenceable):
     def remote_set_segment_size(self, segment_size):
         self._eu.set_segment_size(segment_size)
     def remote_read_encrypted(self, offset, length):
-        assert offset == self._offset # we don't yet implement seek
+        # we don't yet implement seek
+        assert offset == self._offset, "%d != %d" % (offset, self._offset)
         d = self._eu.read_encrypted(length)
-        def _read(data):
-            self._offset += len(data)
-            return data
+        def _read(strings):
+            self._offset += sum([len(data) for data in strings])
+            return strings
         d.addCallback(_read)
         return d
-    def remote_get_plaintext_hashtree_leaves(self, first, last):
-        return self._eu.get_plaintext_hashtree_leaves(first, last)
+    def remote_get_plaintext_hashtree_leaves(self, first, last, num_segments):
+        d = self._eu.get_plaintext_hashtree_leaves(first, last, num_segments)
+        d.addCallback(list)
+        return d
     def remote_get_plaintext_hash(self):
         return self._eu.get_plaintext_hash()