upload: finish refactoring, all unit tests pass now

src/allmydata/test/test_encode.py

@@ -3,7 +3,7 @@ from zope.interface import implements
 from twisted.trial import unittest
 from twisted.internet import defer
 from twisted.python.failure import Failure
-from allmydata import encode, download, hashtree
+from allmydata import encode, upload, download, hashtree
 from allmydata.util import hashutil
 from allmydata.uri import pack_uri
 from allmydata.Crypto.Cipher import AES
@@ -154,10 +154,12 @@ class Encode(unittest.TestCase):
         # force use of multiple segments
         options = {"max_segment_size": max_segment_size}
         e = encode.Encoder(options)
+        e.set_size(datalen)
+        e.set_uploadable(upload.Data(data))
         nonkey = "\x00" * 16
-        e.setup(StringIO(data), nonkey)
+        e.set_encryption_key(nonkey)
+        e.setup()
         assert e.num_shares == NUM_SHARES # else we'll be completely confused
-        e.setup_codec() # need to rebuild the codec for that change
         assert (NUM_SEGMENTS-1)*e.segment_size < len(data) <= NUM_SEGMENTS*e.segment_size
         shareholders = {}
         all_shareholders = []
@@ -279,11 +281,12 @@ class Roundtrip(unittest.TestCase):
         options = {"max_segment_size": max_segment_size,
                    "needed_and_happy_and_total_shares": k_and_happy_and_n}
         e = encode.Encoder(options)
+        e.set_size(len(data))
+        e.set_uploadable(upload.Data(data))
         nonkey = "\x00" * 16
-        e.setup(StringIO(data), nonkey)
-
+        e.set_encryption_key(nonkey)
+        e.setup()
         assert e.num_shares == NUM_SHARES # else we'll be completely confused
-        e.setup_codec() # need to rebuild the codec for that change
 
         shareholders = {}
         all_peers = []
@@ -292,16 +295,7 @@ class Roundtrip(unittest.TestCase):
             peer = FakeBucketWriterProxy(mode)
             shareholders[shnum] = peer
         e.set_shareholders(shareholders)
-        plaintext_hasher = hashutil.plaintext_hasher()
-        plaintext_hasher.update(data)
-        cryptor = AES.new(key=nonkey, mode=AES.MODE_CTR,
-                          counterstart="\x00"*16)
-        crypttext_hasher = hashutil.crypttext_hasher()
-        crypttext_hasher.update(cryptor.encrypt(data))
 
-        e.set_uri_extension_data({'crypttext_hash': crypttext_hasher.digest(),
-                                  'plaintext_hash': plaintext_hasher.digest(),
-                                  })
         d = e.start()
         def _sent(uri_extension_hash):
             return (uri_extension_hash, e, shareholders)

src/allmydata/test/test_upload.py

@@ -1,4 +1,5 @@
 
+import os
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import defer
@@ -9,6 +10,56 @@ from allmydata.uri import unpack_uri, unpack_lit
 from allmydata.util.assertutil import precondition
 from foolscap import eventual
 
+class Uploadable(unittest.TestCase):
+    def shouldEqual(self, data, expected):
+        self.failUnless(isinstance(data, list))
+        for e in data:
+            self.failUnless(isinstance(e, str))
+        s = "".join(data)
+        self.failUnlessEqual(s, expected)
+
+    def test_filehandle(self):
+        s = StringIO("a"*41)
+        u = upload.FileHandle(s)
+        d = u.get_size()
+        d.addCallback(self.failUnlessEqual, 41)
+        d.addCallback(lambda res: u.read(1))
+        d.addCallback(self.shouldEqual, "a")
+        d.addCallback(lambda res: u.read(80))
+        d.addCallback(self.shouldEqual, "a"*40)
+        d.addCallback(lambda res: u.close()) # this doesn't close the filehandle
+        d.addCallback(lambda res: s.close()) # that privilege is reserved for us
+        return d
+
+    def test_filename(self):
+        basedir = "upload/Uploadable/test_filename"
+        os.makedirs(basedir)
+        fn = os.path.join(basedir, "file")
+        f = open(fn, "w")
+        f.write("a"*41)
+        f.close()
+        u = upload.FileName(fn)
+        d = u.get_size()
+        d.addCallback(self.failUnlessEqual, 41)
+        d.addCallback(lambda res: u.read(1))
+        d.addCallback(self.shouldEqual, "a")
+        d.addCallback(lambda res: u.read(80))
+        d.addCallback(self.shouldEqual, "a"*40)
+        d.addCallback(lambda res: u.close())
+        return d
+
+    def test_data(self):
+        s = "a"*41
+        u = upload.Data(s)
+        d = u.get_size()
+        d.addCallback(self.failUnlessEqual, 41)
+        d.addCallback(lambda res: u.read(1))
+        d.addCallback(self.shouldEqual, "a")
+        d.addCallback(lambda res: u.read(80))
+        d.addCallback(self.shouldEqual, "a"*40)
+        d.addCallback(lambda res: u.close())
+        return d
+
 class FakePeer:
     def __init__(self, mode="good"):
         self.ss = FakeStorageServer(mode)

src/allmydata/upload.py

@@ -330,9 +330,11 @@ class CHKUploader:
                         )
 
 def read_this_many_bytes(uploadable, size, prepend_data=[]):
+    if size == 0:
+        return defer.succeed([])
     d = uploadable.read(size)
     def _got(data):
-        assert isinstance(list)
+        assert isinstance(data, list)
         bytes = sum([len(piece) for piece in data])
         assert bytes > 0
         assert bytes <= size
@@ -443,7 +445,7 @@ class Uploader(service.MultiService):
             uploader_class = self.uploader_class
             if size <= self.URI_LIT_SIZE_THRESHOLD:
                 uploader_class = LiteralUploader
-            uploader = self.uploader_class(self.parent, uploadable, options)
+            uploader = uploader_class(self.parent, uploadable, options)
             uploader.set_params(self.parent.get_encoding_parameters()
                                 or self.DEFAULT_ENCODING_PARAMETERS)
             return uploader.start()