tahoe-lafs/misc/simulators/storage-overhead.py

89 lines
3.0 KiB
Python
Raw Normal View History

#!/usr/bin/env python
from __future__ import print_function
import sys, math
from allmydata import uri, storage
from allmydata.immutable import upload
from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE
from allmydata.util import mathutil
def roundup(size, blocksize=4096):
return blocksize * mathutil.div_ceil(size, blocksize)
class BigFakeString:
def __init__(self, length):
self.length = length
self.fp = 0
def seek(self, offset, whence=0):
if whence == 0:
self.fp = offset
elif whence == 1:
self.fp += offset
elif whence == 2:
self.fp = self.length - offset
def tell(self):
return self.fp
def calc(filesize, params=(3,7,10), segsize=DEFAULT_MAX_SEGMENT_SIZE):
num_shares = params[2]
if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD:
urisize = len(uri.LiteralFileURI("A"*filesize).to_string())
sharesize = 0
sharespace = 0
else:
u = upload.FileUploader(None) # XXX changed
u.set_params(params)
# unfortunately, Encoder doesn't currently lend itself to answering
# this question without measuring a filesize, so we have to give it a
# fake one
data = BigFakeString(filesize)
u.set_filehandle(data)
u.set_encryption_key("a"*16)
sharesize, blocksize = u.setup_encoder()
# how much overhead?
# 0x20 bytes of offsets
# 0x04 bytes of extension length
# 0x1ad bytes of extension (=429)
# total is 465 bytes
num_segments = mathutil.div_ceil(filesize, segsize)
num_share_hashes = int(math.log(mathutil.next_power_of_k(num_shares, 2),
2)) + 1
sharesize = storage.allocated_size(sharesize, num_segments,
num_share_hashes,
429)
sharespace = num_shares * roundup(sharesize)
urisize = len(uri.pack_uri(storage_index="a"*32,
key="a"*16,
uri_extension_hash="a"*32,
needed_shares=params[0],
total_shares=params[2],
size=filesize))
return urisize, sharesize, sharespace
def main():
filesize = int(sys.argv[1])
urisize, sharesize, sharespace = calc(filesize)
print("urisize:", urisize)
print("sharesize: %10d" % sharesize)
print("sharespace: %10d" % sharespace)
print("desired expansion: %1.1f" % (1.0 * 10 / 3))
print("effective expansion: %1.1f" % (1.0 * sharespace / filesize))
def chart():
filesize = 2
while filesize < 2**20:
urisize, sharesize, sharespace = calc(int(filesize))
expansion = 1.0 * sharespace / int(filesize)
print("%d,%d,%d,%1.2f" % (int(filesize), urisize, sharespace, expansion))
filesize = filesize * 2**0.5
if __name__ == '__main__':
if sys.argv[1] == "chart":
chart()
else:
main()