ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-06-05 17:11:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

change HashTree to use new hashutil convenience methods, thus fixing a security flaw

Browse Source
This commit is contained in:
Zooko O'Whielacronx 2007-03-29 18:12:35 -07:00
parent 99a046ab51
commit c70b4773d3
1 changed files with 5 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

549
src/allmydata/chunk.py
View File

@ -51,11 +51,9 @@ import sha
import os import os
#import os.path #import os.path
from allmydata.util import bencode from allmydata.util.hashutil import tagged_hash, tagged_pair_hash
__all__ = ['CompleteChunkFile', 'PartialChunkFile'] __version__ = '1.0.0-allmydata'
__version__ = '1.0.0'
BLOCK_SIZE = 65536 BLOCK_SIZE = 65536
MAX_CHUNK_SIZE = BLOCK_SIZE + 4096 MAX_CHUNK_SIZE = BLOCK_SIZE + 4096
@ -171,557 +169,20 @@ class HashTree(CompleteBinaryTreeMixin, list):
then this is used as the bottom row of the hash tree. then this is used as the bottom row of the hash tree.
The augmenting is done so that if the augmented element is at The augmenting is done so that if the augmented element is at
index C{i}, then its value is C{hash(bencode.bencode((i, '')))}. index C{i}, then its value is C{hash(tagged_hash('Merkle tree empty leaf', '%d'%i))}.
""" """
# Augment the list. # Augment the list.
start = len(L) start = len(L)
end = roundup_pow2(len(L)) end = roundup_pow2(len(L))
L = L + [None] * (end - start) L = L + [None] * (end - start)
for i in range(start, end): for i in range(start, end):
L[i] = hash(bencode.bencode((i, ''))) L[i] = tagged_hash('Merkle tree empty leaf', "%d"%i)
# Form each row of the tree. # Form each row of the tree.
rows = [L] rows = [L]
while len(rows[-1]) != 1: while len(rows[-1]) != 1:
last = rows[-1] last = rows[-1]
rows += [[hash(last[2*i] + last[2*i+1]) for i in xrange(len(last)//2)]] rows += [[tagged_pair_hash('Merkle tree internal node', last[2*i], last[2*i+1]) for i in xrange(len(last)//2)]]
# Flatten the list of rows into a single list. # Flatten the list of rows into a single list.
rows.reverse() rows.reverse()
self[:] = sum(rows, []) self[:] = sum(rows, [])
class BlockFile:
"""
Reads and writes blocks of data to a binary file.
It is assumed that the binary file does not change in size.
@ivar file_name: Full path to file.
@ivar file_size: Size of file in bytes.
@ivar block_size: Size of each block.
"""
def __init__(self, file_name, mode, block_size, file_size=None):
"""
Initialize block reader or writer on given file name.
If mode is 'r', the file must already exist and it is opened for
reading only. If mode is 'w', the file will be created with size
C{file_size} if it does not exist, and it is opened for reading
and writing.
Note that C{file_size} is ignored if the file already exists.
"""
self.mode = mode
self.file_name = os.path.abspath(file_name)
assert self.mode in ['r', 'w']
if mode == 'r':
f = open(self.file_name, 'rb')
f.close()
# Create file if it doesn't exist.
created = False
if mode == 'w' and not os.path.exists(self.file_name):
created = True
buf = ' ' * 1024
f = open(self.file_name, 'wb')
for i in xrange(file_size // len(buf)):
f.write(buf)
f.write(' ' * (file_size % len(buf)))
f.close()
self.file_size = os.stat(self.file_name).st_size
if created:
assert self.file_size == file_size
self.block_size = block_size
self.__block_count = self.file_size // self.block_size
if self.file_size % self.block_size == 0:
self.last_block_size = self.block_size
else:
self.last_block_size = self.file_size % self.block_size
self.__block_count += 1
def __getitem__(self, i):
"""
Get block i.
"""
if i < 0 or i >= len(self):
raise IndexError('block index out of range: ' + repr(i))
f = open(self.file_name, 'rb')
try:
f.seek(i * self.block_size)
ans = f.read(self.block_size)
finally:
f.close()
return ans
def __setitem__(self, i, s):
"""
Set block i.
"""
if self.mode != 'w':
raise ValueError('file opened for reading only')
if i < 0 or i >= len(self):
raise IndexError('block index out of range: ' + repr(i))
if i < len(self) - 1:
if len(s) != self.block_size:
raise ValueError('length of value must equal block_size')
else:
if len(s) != self.last_block_size:
raise ValueError('length of value must equal last_block_size')
f = open(self.file_name, 'rb+')
try:
f.seek(i * self.block_size)
f.write(s)
finally:
f.close()
def __len__(self):
"""
Get number of blocks.
"""
return int(self.__block_count)
class MetaFile(CompleteBinaryTreeMixin):
"""
A L{HashTree} stored on disk, with a timestamp.
The list of hashes can be accessed using subscripting and
C{__len__}, in the same manner as for L{HashTree}.
Note that the constructor takes the entire list associated with
the L{HashTree}, not just the bottom row of the tree.
@ivar meta_name: Full path to metafile.
"""
def __init__(self, meta_name, mode, L=None):
"""
Open an existing meta-file for reading or writing.
If C{mode} is 'r', the meta-file must already exist and it is
opened for reading only, and the list C{L} is ignored. If C{mode}
is 'w', the file will be created if it does not exist (from the
list of hashes given in C{L}), and it is opened for reading and
writing.
"""
self.meta_name = os.path.abspath(meta_name)
self.mode = mode
assert self.mode in ['r', 'w']
# A timestamp is stored at index 0. The MetaFile instance
# offsets all indices passed to __getitem__, __setitem__ by
# this offset, and pretends it has length equal to
# self.sublength.
self.offset = 1
if self.mode == 'w':
suggested_length = len(hash('')) * (len(L)+self.offset)
else:
suggested_length = None
created = False
if self.mode == 'w' and not os.path.exists(self.meta_name):
created = True
self.block_file = BlockFile(self.meta_name, self.mode,
len(hash('')),
suggested_length)
self.sublength = len(self.block_file) - self.offset
if created:
for i in xrange(len(L)):
self.block_file[i + self.offset] = L[i]
def __getitem__(self, i):
if i < 0 or i >= self.sublength:
raise IndexError('bad meta-file block index')
return self.block_file[i + self.offset]
def __setitem__(self, i, value):
if i < 0 or i >= self.sublength:
raise IndexError('bad meta-file block index')
self.block_file[i + self.offset] = value
def __len__(self):
return self.sublength
def set_timestamp(self, file_name):
"""
Set meta file's timestamp equal to the timestamp for C{file_name}.
"""
st = os.stat(file_name)
timestamp = bencode.bencode((st.st_size, st.st_mtime))
self.block_file[0] = sha.new(timestamp).digest()
def check_timestamp(self, file_name):
"""
True if meta file's timestamp equals timestamp for C{file_name}.
"""
st = os.stat(file_name)
timestamp = bencode.bencode((st.st_size, st.st_mtime))
return self.block_file[0] == sha.new(timestamp).digest()
class CompleteChunkFile(BlockFile):
"""
Reads chunks from a fully-downloaded file.
A chunk C{i} is created from block C{i}. Block C{i} is unencoded
data read from the file by the L{BlockFile}. Chunk C{i} is
an encoded string created from block C{i}.
Chunks can be read using list subscripting. The total number of
chunks (equals the total number of blocks) is given by L{__len__}.
@ivar file_name: Full path to file.
@ivar file_size: Size of file in bytes.
@ivar file_hash: Hash of file.
@ivar meta_name: Full path to metafile, or C{None}.
@ivar tree: L{HashTree} or L{MetaFile} instance for the file.
One can extract a hash from any node in the hash
tree.
"""
def __init__(self, file_name, meta_name=None, callback=None):
"""
Initialize reader on the given file name.
The entire file will be read and the hash will be computed from
the file. This may take a long time, so C{callback()} is called
frequently during this process. This allows you to reduce CPU
usage if you wish.
The C{meta_name} argument is optional. If it is specified, then the
hashes for C{file_name} will be stored under the file
C{meta_name}. If a C{CompleteChunkFile} is created on the same
file and metafile in the future, then the hashes will not need to
be recomputed and the constructor will return instantly. The
metafile contains a file and date stamp, so that if the file stored
in C{file_name} is modified, then the hashes will be recomputed.
"""
BlockFile.__init__(self, file_name, 'r', block_size=65536)
# Whether we need to compute the hash tree
compute_tree = False
self.meta_name = meta_name
if self.meta_name != None:
self.meta_name = os.path.abspath(self.meta_name)
self.meta = None
if self.meta_name == None:
compute_tree = True
else:
try:
meta = MetaFile(self.meta_name, 'r')
assert meta.check_timestamp(self.file_name)
except (IOError, AssertionError):
compute_tree = True
# Compute the hash tree if needed.
if compute_tree:
chunk_hashes = [None] * len(self)
for i in xrange(len(self)):
triple = (self.file_size, i, BlockFile.__getitem__(self, i))
chunk_hashes[i] = hash(bencode.bencode(triple))
if callback:
callback()
self.tree = HashTree(chunk_hashes)
del chunk_hashes
# If a meta-file was given, make self.tree be a MetaFile instance.
if self.meta_name != None:
if compute_tree:
# Did we compute the hash tree? Then store it to disk.
self.tree = MetaFile(self.meta_name, 'w', self.tree)
# Update its timestamp to be consistent with the file we
# just hashed.
self.tree.set_timestamp(self.file_name)
else:
# Read existing file from disk.
self.tree = MetaFile(self.meta_name, 'r')
self.file_hash = self.tree[0]
def __getitem__(self, i):
"""
Get chunk C{i}.
Raises C{ValueError} if the file's contents changed since the
CompleteFileChunkReader was instantiated.
"""
return encode_chunk(BlockFile.__getitem__(self, i), i,
self.file_size, self.tree)
def encode_chunk(block, index, file_size, tree):
"""
Encode a chunk.
Given a block at index C{index} in a file with size C{file_size},
and a L{HashTree} or L{MetaFile} instance C{tree}, computes and
returns a chunk string for the given block.
The C{tree} argument needs to have correct hashes only at certain
indices. Check out the code for details. In any case, if a hash
is wrong an exception will be raised.
"""
block_count = (len(tree) + 1) // 2
if index < 0 or index >= block_count:
raise IndexError('block index out of range: ' + repr(index))
suffix = bencode.bencode((file_size, index, block))
current = len(tree) - block_count + index
prefix = []
while current > 0:
sibling = tree.sibling(current)
prefix += [tree[current], tree[sibling]]
current = tree.parent(current)
prefix = ''.join(prefix)
# Encode the chunk
chunk = bencode.bencode((prefix, suffix))
# Check to make sure it decodes properly.
decode_chunk(chunk, file_size, tree)
return chunk
def decode_chunk(chunk, file_size, tree):
"""
Decode a chunk.
Given file with size C{file_size} and a L{HashTree} or L{MetaFile}
instance C{tree}, return C{(index, block, tree_items)}. Here
C{index} is the block index where string C{block} should be placed
in the file. Also C{tree_items} is a dict mapping indices within
the L{HashTree} or L{MetaFile} tree object associated with the
given file to the corresponding hashes at those indices. These
have been verified against the file's hash, so it is known that
they are correct.
Raises C{ValueError} if chunk verification fails.
"""
file_hash = tree[0]
block_count = (len(tree) + 1) // 2
try:
# Decode the chunk
try:
(prefix, suffix) = bencode.bdecode(chunk)
except:
raise AssertionError()
assert isinstance(prefix, str)
assert isinstance(suffix, str)
# Verify the suffix against the hashes in the prefix.
hash_len = len(hash(''))
L = [prefix[hash_len*i:hash_len*(i+1)] for i in range(len(prefix)//hash_len)]
L += [file_hash]
assert L[0] == hash(suffix)
branches = []
for i in range(0, len(L)-1, 2):
if hash(L[i] + L[i+1]) == L[i+2]:
branches += [0]
elif hash(L[i+1] + L[i]) == L[i+2]:
branches += [1]
else:
raise AssertionError()
# Decode the suffix
try:
(claim_file_size, claim_index, block) = bencode.bdecode(suffix)
except:
raise AssertionError()
assert isinstance(claim_file_size, int) or isinstance(claim_file_size, long)
assert isinstance(claim_index, int) or isinstance(claim_index, long)
assert isinstance(block, str)
assert file_size == claim_file_size
# Compute the index of the block, and check it.
found_index = sum([branches[i]*2**i for i in range(len(branches))])
assert found_index == claim_index
# Now fill in the tree_items dict.
tree_items = {}
current = (len(tree) - block_count) + found_index
i = 0
while current > 0 and i + 1 < len(L):
tree_items[current] = L[i]
# Next item is our sibling.
tree_items[tree.sibling(current)] = L[i+1]
i += 2
current = tree.parent(current)
return (found_index, block, tree_items)
except AssertionError:
raise ValueError('corrupt chunk')
class PartialChunkFile(BlockFile):
"""
Reads and writes chunks to a partially downloaded file.
@ivar file_name: Full path to file.
@ivar file_size: Size of file in bytes.
@ivar file_hash: Hash of file.
@ivar meta_name: Full path to metafile.
@ivar tree: L{MetaFile} instance for the file.
The hashes in this hash tree are valid only for
nodes that we have been sent hashes for.
"""
def __init__(self, file_name, meta_name, file_hash=None, file_size=None):
"""
Initialize reader/writer for the given file name and metafile name.
If neither C{file_name} nor C{meta_file} exist, then both are
created. The C{file_hash} and C{file_size} arguments are used to
initialize the two files.
If both C{file_name} and C{meta_file} exist, then the hash and
file size arguments are ignored, and those values are instead read
from the files.
If one file exists and the other does not, an C{IOError} is raised.
"""
self.meta_name = os.path.abspath(meta_name)
meta_exists = os.path.exists(self.meta_name)
file_exists = os.path.exists(os.path.abspath(file_name))
BlockFile.__init__(self, os.path.abspath(file_name), 'w',
BLOCK_SIZE, file_size)
if file_exists and not meta_exists:
raise IOError('metafile ' + repr(self.meta_name) +
' missing for file ' + repr(self.file_name))
if meta_exists and not file_exists:
raise IOError('file ' + repr(self.file_name) +
' missing for metafile ' + repr(self.meta_name))
tree_count = 2 * roundup_pow2(len(self)) - 1
self.tree = MetaFile(self.meta_name, 'w', [hash('')] * tree_count)
if not meta_exists and not file_exists:
self.tree[0] = file_hash
self.file_hash = self.tree[0]
def __getitem__(self, i):
"""
Get chunk C{i}.
Raises C{ValueError} if chunk has not yet been downloaded or is
corrupted.
"""
return encode_chunk(BlockFile.__getitem__(self, i), i,
self.file_size, self.tree)
def __setitem__(self, i, chunk):
"""
Set chunk C{i}.
Raises C{ValueError} if the chunk is invalid.
"""
(index, block, tree_items) = decode_chunk(chunk,
self.file_size, self.tree)
if index != i:
raise ValueError('incorrect index for chunk')
BlockFile.__setitem__(self, index, block)
for (tree_index, tree_value) in tree_items.items():
self.tree[tree_index] = tree_value
def test(filename1='temp-out', metaname1='temp-out.meta',
filename2='temp-out2', metaname2='temp-out2.meta'):
"""
Unit tests.
"""
print 'Testing:'
import random
ntests = 100
max_file_size = 200000
# Test CompleteChunkFile.
if os.path.exists(metaname1):
os.remove(metaname1)
for i in range(ntests):
fsize = random.randrange(max_file_size)
# Make some random string of size 'fsize' to go in the file.
s = ''.join([sha.new(str(j)).digest() for j in range(fsize//20+1)])
assert len(s) >= fsize
s = s[:fsize]
f = open(filename1, 'wb')
f.write(s)
f.close()
C = CompleteChunkFile(filename1)
for j in range(len(C)):
C[j]
C = CompleteChunkFile(filename1, metaname1)
for j in range(len(C)):
C[j]
C = CompleteChunkFile(filename1, metaname1)
for j in range(len(C)):
C[j]
os.remove(metaname1)
os.remove(filename1)
print ' CompleteChunkFile: OK'
# Test PartialChunkFile
for i in range(ntests):
fsize = random.randrange(max_file_size)
# Make some random string of size 'fsize' to go in the file.
s = ''.join([sha.new(str(j)).digest() for j in range(fsize//20+1)])
assert len(s) >= fsize
s = s[:fsize]
f = open(filename1, 'wb')
f.write(s)
f.close()
C1 = CompleteChunkFile(filename1)
if os.path.exists(filename2):
os.remove(filename2)
if os.path.exists(metaname2):
os.remove(metaname2)
C2 = PartialChunkFile(filename2, metaname2, C1.file_hash, C1.file_size)
assert len(C1) == len(C2)
assert C2.tree[0] == C1.tree[0]
for j in range(len(C2)):
try:
C2[j]
ok = False
except ValueError:
ok = True
if not ok:
raise AssertionError()
for j in range(len(C2)//2):
k = random.randrange(len(C2))
if len(C1) > 1:
assert C1[k] != C1[(k+1)%len(C1)]
try:
C2[k] = C1[(k+1)%len(C1)]
ok = False
except ValueError:
ok = True
if not ok:
raise AssertionError()
C2[k] = C1[k]
assert C2[k] == C1[k]
for j in range(len(C2)):
C2[j] = C1[j]
assert C2[j] == C1[j]
os.remove(filename1)
os.remove(filename2)
os.remove(metaname2)
print ' PartialChunkFile: OK'
if __name__ == '__main__':
test()