test_encode: make sure encode_new can produce the data it is supposed to

src/allmydata/encode_new.py

@@ -86,18 +86,22 @@ class Encoder(object):
         self.required_shares = 25
 
         self.segment_size = min(2*MiB, self.file_size)
-        self.num_segments = mathutil.div_ceil(self.file_size, self.segment_size)
 
     def get_reservation_size(self):
-        self.num_shares = 100
-        self.share_size = mathutil.div_ceil(self.file_size, self.required_shares)
+        share_size = mathutil.div_ceil(self.file_size, self.required_shares)
         overhead = self.compute_overhead()
-        return self.share_size + overhead
+        return share_size + overhead
+    def compute_overhead(self):
+        return 0
 
     def set_shareholders(self, landlords):
         self.landlords = landlords.copy()
 
     def start(self):
+        self.num_segments = mathutil.div_ceil(self.file_size,
+                                              self.segment_size)
+        self.share_size = mathutil.div_ceil(self.file_size,
+                                            self.required_shares)
         self.setup_encryption()
         self.setup_encoder()
         d = defer.succeed(None)
@@ -167,10 +171,7 @@ class Encoder(object):
         return defer.DeferredList(dl)
 
     def send_subshare(self, shareid, segment_num, subshare):
-        #if False:
-        #    offset = hash_size + segment_num * segment_size
-        #    return self.send(shareid, "write", subshare, offset)
-        return self.send(shareid, "put_subshare", segment_num, subshare)
+        return self.send(shareid, "put_block", segment_num, subshare)
 
     def send(self, shareid, methname, *args, **kwargs):
         ll = self.landlords[shareid]
@@ -191,9 +192,6 @@ class Encoder(object):
         # all_hashes[1] is the left child, == hash(ah[3]+ah[4])
         # all_hashes[n] == hash(all_hashes[2*n+1] + all_hashes[2*n+2])
         self.share_root_hashes[shareid] = t[0]
-        if False:
-            block = "".join(all_hashes)
-            return self.send(shareid, "write", block, offset=0)
         return self.send(shareid, "put_block_hashes", all_hashes)
 
     def send_all_share_hash_trees(self):

src/allmydata/test/test_encode.py

@@ -21,3 +21,73 @@ class Encode(unittest.TestCase):
         d = e.start()
         return d
 
+class FakePeer:
+    def __init__(self):
+        self.blocks = {}
+        self.blockhashes = None
+        self.sharehashes = None
+        self.closed = False
+
+    def callRemote(self, methname, *args, **kwargs):
+        def _call():
+            meth = getattr(self, methname)
+            return meth(*args, **kwargs)
+        return defer.maybeDeferred(_call)
+
+    def put_block(self, segmentnum, data):
+        assert not self.closed
+        assert segmentnum not in self.blocks
+        self.blocks[segmentnum] = data
+    
+    def put_block_hashes(self, blockhashes):
+        assert not self.closed
+        assert self.blockhashes is None
+        self.blockhashes = blockhashes
+        
+    def put_share_hashes(self, sharehashes):
+        assert not self.closed
+        assert self.sharehashes is None
+        self.sharehashes = sharehashes
+
+    def close(self):
+        assert not self.closed
+        self.closed = True
+
+
+class UpDown(unittest.TestCase):
+    def test_send(self):
+        e = encode_new.Encoder()
+        data = "happy happy joy joy" * 4
+        e.setup(StringIO(data))
+        NUM_SHARES = 100
+        assert e.num_shares == NUM_SHARES # else we'll be completely confused
+        e.segment_size = 25 # force use of multiple segments
+        NUM_SEGMENTS = 4
+        assert (NUM_SEGMENTS-1)*e.segment_size < len(data) <= NUM_SEGMENTS*e.segment_size
+        shareholders = {}
+        all_shareholders = []
+        for shnum in range(NUM_SHARES):
+            peer = FakePeer()
+            shareholders[shnum] = peer
+            all_shareholders.append(peer)
+        e.set_shareholders(shareholders)
+        d = e.start()
+        def _check(roothash):
+            self.failUnless(isinstance(roothash, str))
+            self.failUnlessEqual(len(roothash), 32)
+            for i,peer in enumerate(all_shareholders):
+                self.failUnless(peer.closed)
+                self.failUnlessEqual(len(peer.blocks), NUM_SEGMENTS)
+                #self.failUnlessEqual(len(peer.blockhashes), NUM_SEGMENTS)
+                # that isn't true: each peer gets a full tree, so it's more
+                # like 2n-1 but with rounding to a power of two
+                for h in peer.blockhashes:
+                    self.failUnlessEqual(len(h), 32)
+                #self.failUnlessEqual(len(peer.sharehashes), NUM_SHARES)
+                # that isn't true: each peer only gets the chain they need
+                for (hashnum, h) in peer.sharehashes:
+                    self.failUnless(isinstance(hashnum, int))
+                    self.failUnlessEqual(len(h), 32)
+        d.addCallback(_check)
+
+        return d