mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2024-12-28 16:58:53 +00:00
b315619d6b
Refactor into a class the logic of asking each server in turn until one of them gives an answer that validates. It is called ValidatedThingObtainer. Refactor the downloading and verification of the URI Extension Block into a class named ValidatedExtendedURIProxy. The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any unncessary information, but of course it still accepts such information for backwards compatibility (so that this new download code is able to download files uploaded with old, and for that matter with current, upload code). The new logic of validating UEBs follows the practice of doing all validation up front. This practice advises one to isolate the validation of incoming data into one place, so that all of the rest of the code can assume only valid data. If any redundant information is present in the UEB+URI, the new code cross-checks and asserts that it is all fully consistent. This closes some issues where the uploader could have uploaded inconsistent redundant data, which would probably have caused the old downloader to simply reject that download after getting a Python exception, but perhaps could have caused greater harm to the old downloader. I removed the notion of selecting an erasure codec from codec.py based on the string that was passed in the UEB. Currently "crs" is the only such string that works, so "_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the "validate up front" strategy -- now if someone sets a different string than "crs" in their UEB, the downloader will reject the download in the "validate this UEB" function instead of in a separate "select the codec instance" function. I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not produce this information any more (since it potentially exposes confidential information about the file), and the unit tests for it were disabled. The downloader before this patch would check that plaintext hash or plaintext merkle tree if they were present, but not complain if they were absent. The new downloader in this patch complains if they are present and doesn't check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt the hashes which are stored in the UEB to preserve confidentiality. This would be a double- check on the correctness of our own source code -- the current Merkle Tree over the ciphertext is already sufficient to guarantee the integrity of the download unless there is a bug in our Merkle Tree or AES implementation.) This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
95 lines
4.0 KiB
Python
95 lines
4.0 KiB
Python
|
|
import os
|
|
from twisted.trial import unittest
|
|
from twisted.python import log
|
|
from allmydata.codec import CRSEncoder, CRSDecoder
|
|
import random
|
|
from allmydata.util import mathutil
|
|
|
|
class T(unittest.TestCase):
|
|
def do_test(self, size, required_shares, max_shares, fewer_shares=None):
|
|
data0s = [os.urandom(mathutil.div_ceil(size, required_shares)) for i in range(required_shares)]
|
|
enc = CRSEncoder()
|
|
enc.set_params(size, required_shares, max_shares)
|
|
params = enc.get_params()
|
|
assert params == (size, required_shares, max_shares)
|
|
log.msg("params: %s" % (params,))
|
|
d = enc.encode(data0s)
|
|
def _done_encoding_all((shares, shareids)):
|
|
self.failUnlessEqual(len(shares), max_shares)
|
|
self.shares = shares
|
|
self.shareids = shareids
|
|
d.addCallback(_done_encoding_all)
|
|
if fewer_shares is not None:
|
|
# also validate that the desired_shareids= parameter works
|
|
desired_shareids = random.sample(range(max_shares), fewer_shares)
|
|
d.addCallback(lambda res: enc.encode(data0s, desired_shareids))
|
|
def _check_fewer_shares((some_shares, their_shareids)):
|
|
self.failUnlessEqual(tuple(their_shareids), tuple(desired_shareids))
|
|
d.addCallback(_check_fewer_shares)
|
|
|
|
def _decode((shares, shareids)):
|
|
dec = CRSDecoder()
|
|
dec.set_params(*params)
|
|
d1 = dec.decode(shares, shareids)
|
|
return d1
|
|
|
|
def _check_data(decoded_shares):
|
|
self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s)))
|
|
self.failUnlessEqual(len(decoded_shares), len(data0s))
|
|
for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)):
|
|
self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],))
|
|
self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",))
|
|
# 0data0sclipped = tuple(data0s)
|
|
# data0sclipped[-1] =
|
|
# self.failUnless(tuple(decoded_shares) == tuple(data0s))
|
|
|
|
def _decode_some(res):
|
|
log.msg("_decode_some")
|
|
# decode with a minimal subset of the shares
|
|
some_shares = self.shares[:required_shares]
|
|
some_shareids = self.shareids[:required_shares]
|
|
return _decode((some_shares, some_shareids))
|
|
d.addCallback(_decode_some)
|
|
d.addCallback(_check_data)
|
|
|
|
def _decode_some_random(res):
|
|
log.msg("_decode_some_random")
|
|
# use a randomly-selected minimal subset
|
|
l = random.sample(zip(self.shares, self.shareids), required_shares)
|
|
some_shares = [ x[0] for x in l ]
|
|
some_shareids = [ x[1] for x in l ]
|
|
return _decode((some_shares, some_shareids))
|
|
d.addCallback(_decode_some_random)
|
|
d.addCallback(_check_data)
|
|
|
|
def _decode_multiple(res):
|
|
log.msg("_decode_multiple")
|
|
# make sure we can re-use the decoder object
|
|
shares1 = random.sample(self.shares, required_shares)
|
|
sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares)
|
|
shares1 = [ x[0] for x in sharesl1 ]
|
|
shareids1 = [ x[1] for x in sharesl1 ]
|
|
sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares)
|
|
shares2 = [ x[0] for x in sharesl2 ]
|
|
shareids2 = [ x[1] for x in sharesl2 ]
|
|
dec = CRSDecoder()
|
|
dec.set_params(*params)
|
|
d1 = dec.decode(shares1, shareids1)
|
|
d1.addCallback(_check_data)
|
|
d1.addCallback(lambda res: dec.decode(shares2, shareids2))
|
|
d1.addCallback(_check_data)
|
|
return d1
|
|
d.addCallback(_decode_multiple)
|
|
|
|
return d
|
|
|
|
def test_encode(self):
|
|
return self.do_test(1000, 25, 100)
|
|
|
|
def test_encode1(self):
|
|
return self.do_test(8, 8, 16)
|
|
|
|
def test_encode2(self):
|
|
return self.do_test(125, 25, 100, 90)
|