2007-01-05 04:52:51 +00:00
|
|
|
|
2007-02-02 00:13:01 +00:00
|
|
|
import os
|
2007-01-05 04:52:51 +00:00
|
|
|
from twisted.trial import unittest
|
2007-01-05 07:06:42 +00:00
|
|
|
from twisted.python import log
|
download: refactor handling of URI Extension Block and crypttext hash tree, simplify things
Refactor into a class the logic of asking each server in turn until one of them gives an answer
that validates. It is called ValidatedThingObtainer.
Refactor the downloading and verification of the URI Extension Block into a class named
ValidatedExtendedURIProxy.
The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any
unncessary information, but of course it still accepts such information for backwards
compatibility (so that this new download code is able to download files uploaded with old, and
for that matter with current, upload code).
The new logic of validating UEBs follows the practice of doing all validation up front. This
practice advises one to isolate the validation of incoming data into one place, so that all of
the rest of the code can assume only valid data.
If any redundant information is present in the UEB+URI, the new code cross-checks and asserts
that it is all fully consistent. This closes some issues where the uploader could have
uploaded inconsistent redundant data, which would probably have caused the old downloader to
simply reject that download after getting a Python exception, but perhaps could have caused
greater harm to the old downloader.
I removed the notion of selecting an erasure codec from codec.py based on the string that was
passed in the UEB. Currently "crs" is the only such string that works, so
"_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the
"validate up front" strategy -- now if someone sets a different string than "crs" in their UEB,
the downloader will reject the download in the "validate this UEB" function instead of in a
separate "select the codec instance" function.
I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not
produce this information any more (since it potentially exposes confidential information about
the file), and the unit tests for it were disabled. The downloader before this patch would
check that plaintext hash or plaintext merkle tree if they were present, but not complain if
they were absent. The new downloader in this patch complains if they are present and doesn't
check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt
the hashes which are stored in the UEB to preserve confidentiality. This would be a double-
check on the correctness of our own source code -- the current Merkle Tree over the ciphertext
is already sufficient to guarantee the integrity of the download unless there is a bug in our
Merkle Tree or AES implementation.)
This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the
uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more
meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
2008-12-05 15:17:54 +00:00
|
|
|
from allmydata.codec import CRSEncoder, CRSDecoder
|
2007-01-05 04:52:51 +00:00
|
|
|
import random
|
2007-02-01 23:07:00 +00:00
|
|
|
from allmydata.util import mathutil
|
2007-01-05 04:52:51 +00:00
|
|
|
|
download: refactor handling of URI Extension Block and crypttext hash tree, simplify things
Refactor into a class the logic of asking each server in turn until one of them gives an answer
that validates. It is called ValidatedThingObtainer.
Refactor the downloading and verification of the URI Extension Block into a class named
ValidatedExtendedURIProxy.
The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any
unncessary information, but of course it still accepts such information for backwards
compatibility (so that this new download code is able to download files uploaded with old, and
for that matter with current, upload code).
The new logic of validating UEBs follows the practice of doing all validation up front. This
practice advises one to isolate the validation of incoming data into one place, so that all of
the rest of the code can assume only valid data.
If any redundant information is present in the UEB+URI, the new code cross-checks and asserts
that it is all fully consistent. This closes some issues where the uploader could have
uploaded inconsistent redundant data, which would probably have caused the old downloader to
simply reject that download after getting a Python exception, but perhaps could have caused
greater harm to the old downloader.
I removed the notion of selecting an erasure codec from codec.py based on the string that was
passed in the UEB. Currently "crs" is the only such string that works, so
"_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the
"validate up front" strategy -- now if someone sets a different string than "crs" in their UEB,
the downloader will reject the download in the "validate this UEB" function instead of in a
separate "select the codec instance" function.
I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not
produce this information any more (since it potentially exposes confidential information about
the file), and the unit tests for it were disabled. The downloader before this patch would
check that plaintext hash or plaintext merkle tree if they were present, but not complain if
they were absent. The new downloader in this patch complains if they are present and doesn't
check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt
the hashes which are stored in the UEB to preserve confidentiality. This would be a double-
check on the correctness of our own source code -- the current Merkle Tree over the ciphertext
is already sufficient to guarantee the integrity of the download unless there is a bug in our
Merkle Tree or AES implementation.)
This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the
uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more
meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
2008-12-05 15:17:54 +00:00
|
|
|
class T(unittest.TestCase):
|
2007-01-16 04:22:22 +00:00
|
|
|
def do_test(self, size, required_shares, max_shares, fewer_shares=None):
|
2007-02-01 23:07:00 +00:00
|
|
|
data0s = [os.urandom(mathutil.div_ceil(size, required_shares)) for i in range(required_shares)]
|
download: refactor handling of URI Extension Block and crypttext hash tree, simplify things
Refactor into a class the logic of asking each server in turn until one of them gives an answer
that validates. It is called ValidatedThingObtainer.
Refactor the downloading and verification of the URI Extension Block into a class named
ValidatedExtendedURIProxy.
The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any
unncessary information, but of course it still accepts such information for backwards
compatibility (so that this new download code is able to download files uploaded with old, and
for that matter with current, upload code).
The new logic of validating UEBs follows the practice of doing all validation up front. This
practice advises one to isolate the validation of incoming data into one place, so that all of
the rest of the code can assume only valid data.
If any redundant information is present in the UEB+URI, the new code cross-checks and asserts
that it is all fully consistent. This closes some issues where the uploader could have
uploaded inconsistent redundant data, which would probably have caused the old downloader to
simply reject that download after getting a Python exception, but perhaps could have caused
greater harm to the old downloader.
I removed the notion of selecting an erasure codec from codec.py based on the string that was
passed in the UEB. Currently "crs" is the only such string that works, so
"_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the
"validate up front" strategy -- now if someone sets a different string than "crs" in their UEB,
the downloader will reject the download in the "validate this UEB" function instead of in a
separate "select the codec instance" function.
I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not
produce this information any more (since it potentially exposes confidential information about
the file), and the unit tests for it were disabled. The downloader before this patch would
check that plaintext hash or plaintext merkle tree if they were present, but not complain if
they were absent. The new downloader in this patch complains if they are present and doesn't
check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt
the hashes which are stored in the UEB to preserve confidentiality. This would be a double-
check on the correctness of our own source code -- the current Merkle Tree over the ciphertext
is already sufficient to guarantee the integrity of the download unless there is a bug in our
Merkle Tree or AES implementation.)
This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the
uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more
meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
2008-12-05 15:17:54 +00:00
|
|
|
enc = CRSEncoder()
|
2007-01-16 04:22:22 +00:00
|
|
|
enc.set_params(size, required_shares, max_shares)
|
download: refactor handling of URI Extension Block and crypttext hash tree, simplify things
Refactor into a class the logic of asking each server in turn until one of them gives an answer
that validates. It is called ValidatedThingObtainer.
Refactor the downloading and verification of the URI Extension Block into a class named
ValidatedExtendedURIProxy.
The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any
unncessary information, but of course it still accepts such information for backwards
compatibility (so that this new download code is able to download files uploaded with old, and
for that matter with current, upload code).
The new logic of validating UEBs follows the practice of doing all validation up front. This
practice advises one to isolate the validation of incoming data into one place, so that all of
the rest of the code can assume only valid data.
If any redundant information is present in the UEB+URI, the new code cross-checks and asserts
that it is all fully consistent. This closes some issues where the uploader could have
uploaded inconsistent redundant data, which would probably have caused the old downloader to
simply reject that download after getting a Python exception, but perhaps could have caused
greater harm to the old downloader.
I removed the notion of selecting an erasure codec from codec.py based on the string that was
passed in the UEB. Currently "crs" is the only such string that works, so
"_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the
"validate up front" strategy -- now if someone sets a different string than "crs" in their UEB,
the downloader will reject the download in the "validate this UEB" function instead of in a
separate "select the codec instance" function.
I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not
produce this information any more (since it potentially exposes confidential information about
the file), and the unit tests for it were disabled. The downloader before this patch would
check that plaintext hash or plaintext merkle tree if they were present, but not complain if
they were absent. The new downloader in this patch complains if they are present and doesn't
check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt
the hashes which are stored in the UEB to preserve confidentiality. This would be a double-
check on the correctness of our own source code -- the current Merkle Tree over the ciphertext
is already sufficient to guarantee the integrity of the download unless there is a bug in our
Merkle Tree or AES implementation.)
This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the
uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more
meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
2008-12-05 15:17:54 +00:00
|
|
|
params = enc.get_params()
|
|
|
|
|
assert params == (size, required_shares, max_shares)
|
|
|
|
|
log.msg("params: %s" % (params,))
|
2007-02-01 23:07:00 +00:00
|
|
|
d = enc.encode(data0s)
|
|
|
|
|
def _done_encoding_all((shares, shareids)):
|
2007-01-16 04:22:22 +00:00
|
|
|
self.failUnlessEqual(len(shares), max_shares)
|
2007-01-05 04:52:51 +00:00
|
|
|
self.shares = shares
|
2007-02-01 23:07:00 +00:00
|
|
|
self.shareids = shareids
|
2007-01-16 04:22:22 +00:00
|
|
|
d.addCallback(_done_encoding_all)
|
|
|
|
|
if fewer_shares is not None:
|
2007-02-01 23:07:00 +00:00
|
|
|
# also validate that the desired_shareids= parameter works
|
|
|
|
|
desired_shareids = random.sample(range(max_shares), fewer_shares)
|
|
|
|
|
d.addCallback(lambda res: enc.encode(data0s, desired_shareids))
|
|
|
|
|
def _check_fewer_shares((some_shares, their_shareids)):
|
|
|
|
|
self.failUnlessEqual(tuple(their_shareids), tuple(desired_shareids))
|
2007-01-16 04:22:22 +00:00
|
|
|
d.addCallback(_check_fewer_shares)
|
2007-01-05 04:52:51 +00:00
|
|
|
|
2007-02-01 23:07:00 +00:00
|
|
|
def _decode((shares, shareids)):
|
download: refactor handling of URI Extension Block and crypttext hash tree, simplify things
Refactor into a class the logic of asking each server in turn until one of them gives an answer
that validates. It is called ValidatedThingObtainer.
Refactor the downloading and verification of the URI Extension Block into a class named
ValidatedExtendedURIProxy.
The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any
unncessary information, but of course it still accepts such information for backwards
compatibility (so that this new download code is able to download files uploaded with old, and
for that matter with current, upload code).
The new logic of validating UEBs follows the practice of doing all validation up front. This
practice advises one to isolate the validation of incoming data into one place, so that all of
the rest of the code can assume only valid data.
If any redundant information is present in the UEB+URI, the new code cross-checks and asserts
that it is all fully consistent. This closes some issues where the uploader could have
uploaded inconsistent redundant data, which would probably have caused the old downloader to
simply reject that download after getting a Python exception, but perhaps could have caused
greater harm to the old downloader.
I removed the notion of selecting an erasure codec from codec.py based on the string that was
passed in the UEB. Currently "crs" is the only such string that works, so
"_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the
"validate up front" strategy -- now if someone sets a different string than "crs" in their UEB,
the downloader will reject the download in the "validate this UEB" function instead of in a
separate "select the codec instance" function.
I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not
produce this information any more (since it potentially exposes confidential information about
the file), and the unit tests for it were disabled. The downloader before this patch would
check that plaintext hash or plaintext merkle tree if they were present, but not complain if
they were absent. The new downloader in this patch complains if they are present and doesn't
check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt
the hashes which are stored in the UEB to preserve confidentiality. This would be a double-
check on the correctness of our own source code -- the current Merkle Tree over the ciphertext
is already sufficient to guarantee the integrity of the download unless there is a bug in our
Merkle Tree or AES implementation.)
This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the
uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more
meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
2008-12-05 15:17:54 +00:00
|
|
|
dec = CRSDecoder()
|
|
|
|
|
dec.set_params(*params)
|
2007-02-01 23:07:00 +00:00
|
|
|
d1 = dec.decode(shares, shareids)
|
2007-01-05 04:52:51 +00:00
|
|
|
return d1
|
|
|
|
|
|
2007-01-25 00:23:22 +00:00
|
|
|
def _check_data(decoded_shares):
|
2007-03-28 05:57:15 +00:00
|
|
|
self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s)))
|
2007-02-01 23:07:00 +00:00
|
|
|
self.failUnlessEqual(len(decoded_shares), len(data0s))
|
2007-03-28 05:57:15 +00:00
|
|
|
for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)):
|
|
|
|
|
self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],))
|
|
|
|
|
self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",))
|
|
|
|
|
# 0data0sclipped = tuple(data0s)
|
2007-11-01 22:34:12 +00:00
|
|
|
# data0sclipped[-1] =
|
2007-03-28 05:57:15 +00:00
|
|
|
# self.failUnless(tuple(decoded_shares) == tuple(data0s))
|
2007-01-05 07:06:42 +00:00
|
|
|
|
2007-01-05 04:52:51 +00:00
|
|
|
def _decode_some(res):
|
2007-01-05 07:06:42 +00:00
|
|
|
log.msg("_decode_some")
|
2007-01-05 04:52:51 +00:00
|
|
|
# decode with a minimal subset of the shares
|
|
|
|
|
some_shares = self.shares[:required_shares]
|
2007-02-01 23:07:00 +00:00
|
|
|
some_shareids = self.shareids[:required_shares]
|
|
|
|
|
return _decode((some_shares, some_shareids))
|
2007-01-05 04:52:51 +00:00
|
|
|
d.addCallback(_decode_some)
|
|
|
|
|
d.addCallback(_check_data)
|
|
|
|
|
|
|
|
|
|
def _decode_some_random(res):
|
2007-01-05 07:06:42 +00:00
|
|
|
log.msg("_decode_some_random")
|
2007-01-05 04:52:51 +00:00
|
|
|
# use a randomly-selected minimal subset
|
2007-02-01 23:07:00 +00:00
|
|
|
l = random.sample(zip(self.shares, self.shareids), required_shares)
|
|
|
|
|
some_shares = [ x[0] for x in l ]
|
|
|
|
|
some_shareids = [ x[1] for x in l ]
|
|
|
|
|
return _decode((some_shares, some_shareids))
|
2007-01-05 04:52:51 +00:00
|
|
|
d.addCallback(_decode_some_random)
|
|
|
|
|
d.addCallback(_check_data)
|
|
|
|
|
|
|
|
|
|
def _decode_multiple(res):
|
2007-01-05 07:06:42 +00:00
|
|
|
log.msg("_decode_multiple")
|
2007-01-05 04:52:51 +00:00
|
|
|
# make sure we can re-use the decoder object
|
|
|
|
|
shares1 = random.sample(self.shares, required_shares)
|
2007-02-01 23:07:00 +00:00
|
|
|
sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares)
|
|
|
|
|
shares1 = [ x[0] for x in sharesl1 ]
|
|
|
|
|
shareids1 = [ x[1] for x in sharesl1 ]
|
|
|
|
|
sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares)
|
|
|
|
|
shares2 = [ x[0] for x in sharesl2 ]
|
|
|
|
|
shareids2 = [ x[1] for x in sharesl2 ]
|
download: refactor handling of URI Extension Block and crypttext hash tree, simplify things
Refactor into a class the logic of asking each server in turn until one of them gives an answer
that validates. It is called ValidatedThingObtainer.
Refactor the downloading and verification of the URI Extension Block into a class named
ValidatedExtendedURIProxy.
The new logic of validating UEBs is minimalist: it doesn't require the UEB to contain any
unncessary information, but of course it still accepts such information for backwards
compatibility (so that this new download code is able to download files uploaded with old, and
for that matter with current, upload code).
The new logic of validating UEBs follows the practice of doing all validation up front. This
practice advises one to isolate the validation of incoming data into one place, so that all of
the rest of the code can assume only valid data.
If any redundant information is present in the UEB+URI, the new code cross-checks and asserts
that it is all fully consistent. This closes some issues where the uploader could have
uploaded inconsistent redundant data, which would probably have caused the old downloader to
simply reject that download after getting a Python exception, but perhaps could have caused
greater harm to the old downloader.
I removed the notion of selecting an erasure codec from codec.py based on the string that was
passed in the UEB. Currently "crs" is the only such string that works, so
"_assert(codec_name == 'crs')" is simpler and more explicit. This is also in keeping with the
"validate up front" strategy -- now if someone sets a different string than "crs" in their UEB,
the downloader will reject the download in the "validate this UEB" function instead of in a
separate "select the codec instance" function.
I removed the code to check plaintext hashes and plaintext Merkle Trees. Uploaders do not
produce this information any more (since it potentially exposes confidential information about
the file), and the unit tests for it were disabled. The downloader before this patch would
check that plaintext hash or plaintext merkle tree if they were present, but not complain if
they were absent. The new downloader in this patch complains if they are present and doesn't
check them. (We might in the future re-introduce such hashes over the plaintext, but encrypt
the hashes which are stored in the UEB to preserve confidentiality. This would be a double-
check on the correctness of our own source code -- the current Merkle Tree over the ciphertext
is already sufficient to guarantee the integrity of the download unless there is a bug in our
Merkle Tree or AES implementation.)
This patch increases the lines-of-code count by 8 (from 17,770 to 17,778), and reduces the
uncovered-by-tests lines-of-code count by 24 (from 1408 to 1384). Those numbers would be more
meaningful if we omitted src/allmydata/util/ from the test-coverage statistics.
2008-12-05 15:17:54 +00:00
|
|
|
dec = CRSDecoder()
|
|
|
|
|
dec.set_params(*params)
|
2007-02-01 23:07:00 +00:00
|
|
|
d1 = dec.decode(shares1, shareids1)
|
2007-01-05 04:52:51 +00:00
|
|
|
d1.addCallback(_check_data)
|
2007-02-01 23:07:00 +00:00
|
|
|
d1.addCallback(lambda res: dec.decode(shares2, shareids2))
|
2007-01-05 04:52:51 +00:00
|
|
|
d1.addCallback(_check_data)
|
|
|
|
|
return d1
|
|
|
|
|
d.addCallback(_decode_multiple)
|
|
|
|
|
|
|
|
|
|
return d
|
|
|
|
|
|
|
|
|
|
def test_encode(self):
|
|
|
|
|
return self.do_test(1000, 25, 100)
|
|
|
|
|
|
2007-01-05 06:50:21 +00:00
|
|
|
def test_encode1(self):
|
|
|
|
|
return self.do_test(8, 8, 16)
|
|
|
|
|
|
2007-01-05 07:06:42 +00:00
|
|
|
def test_encode2(self):
|
2007-03-28 05:57:15 +00:00
|
|
|
return self.do_test(125, 25, 100, 90)
|
