ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-20 13:33:09 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Explicit bytes.

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2020-08-20 14:32:41 -04:00
parent 8136b21f46
commit 5d2bdf5883
2 changed files with 33 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/allmydata/storage/mutable.py
View File

@ -87,7 +87,7 @@ class MutableShareFile(object):
self.MAGIC, my_nodeid, write_enabler, self.MAGIC, my_nodeid, write_enabler,
data_length, extra_lease_offset, data_length, extra_lease_offset,
) )
leases = ("\x00" * self.LEASE_SIZE) * 4 leases = (b"\x00" * self.LEASE_SIZE) * 4
f.write(header + leases) f.write(header + leases)
# data goes here, empty after creation # data goes here, empty after creation
f.write(struct.pack(">L", num_extra_leases)) f.write(struct.pack(">L", num_extra_leases))
@ -155,7 +155,7 @@ class MutableShareFile(object):
# Zero out the old lease info (in order to minimize the chance that # Zero out the old lease info (in order to minimize the chance that
# it could accidentally be exposed to a reader later, re #1528). # it could accidentally be exposed to a reader later, re #1528).
f.seek(old_extra_lease_offset) f.seek(old_extra_lease_offset)
f.write('\x00' * leases_size) f.write(b'\x00' * leases_size)
f.flush() f.flush()
# An interrupt here will corrupt the leases. # An interrupt here will corrupt the leases.
@ -194,7 +194,7 @@ class MutableShareFile(object):
# Fill any newly exposed empty space with 0's. # Fill any newly exposed empty space with 0's.
if offset > data_length: if offset > data_length:
f.seek(self.DATA_OFFSET+data_length) f.seek(self.DATA_OFFSET+data_length)
f.write('\x00'*(offset - data_length)) f.write(b'\x00'*(offset - data_length))
f.flush() f.flush()
new_data_length = offset+length new_data_length = offset+length
@ -326,10 +326,10 @@ class MutableShareFile(object):
modified = 0 modified = 0
remaining = 0 remaining = 0
blank_lease = LeaseInfo(owner_num=0, blank_lease = LeaseInfo(owner_num=0,
renew_secret="\x00"*32, renew_secret=b"\x00"*32,
cancel_secret="\x00"*32, cancel_secret=b"\x00"*32,
expiration_time=0, expiration_time=0,
nodeid="\x00"*20) nodeid=b"\x00"*20)
with open(self.home, 'rb+') as f: with open(self.home, 'rb+') as f:
for (leasenum,lease) in self._enumerate_leases(f): for (leasenum,lease) in self._enumerate_leases(f):
accepting_nodeids.add(lease.nodeid) accepting_nodeids.add(lease.nodeid)

52
src/allmydata/test/test_storage_web.py
View File

@ -1,5 +1,7 @@
""" """
Tests for twisted.storage that uses Web APIs. Tests for twisted.storage that uses Web APIs.
Partially ported to Python 3.
""" """
from __future__ import absolute_import from __future__ import absolute_import
@ -283,27 +285,27 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
def make_shares(self, ss): def make_shares(self, ss):
def make(si): def make(si):
return (si, hashutil.tagged_hash("renew", si), return (si, hashutil.tagged_hash(b"renew", si),
hashutil.tagged_hash("cancel", si)) hashutil.tagged_hash(b"cancel", si))
def make_mutable(si): def make_mutable(si):
return (si, hashutil.tagged_hash("renew", si), return (si, hashutil.tagged_hash(b"renew", si),
hashutil.tagged_hash("cancel", si), hashutil.tagged_hash(b"cancel", si),
hashutil.tagged_hash("write-enabler", si)) hashutil.tagged_hash(b"write-enabler", si))
def make_extra_lease(si, num): def make_extra_lease(si, num):
return (hashutil.tagged_hash("renew-%d" % num, si), return (hashutil.tagged_hash(b"renew-%d" % num, si),
hashutil.tagged_hash("cancel-%d" % num, si)) hashutil.tagged_hash(b"cancel-%d" % num, si))
immutable_si_0, rs0, cs0 = make("\x00" * 16) immutable_si_0, rs0, cs0 = make(b"\x00" * 16)
immutable_si_1, rs1, cs1 = make("\x01" * 16) immutable_si_1, rs1, cs1 = make(b"\x01" * 16)
rs1a, cs1a = make_extra_lease(immutable_si_1, 1) rs1a, cs1a = make_extra_lease(immutable_si_1, 1)
mutable_si_2, rs2, cs2, we2 = make_mutable("\x02" * 16) mutable_si_2, rs2, cs2, we2 = make_mutable(b"\x02" * 16)
mutable_si_3, rs3, cs3, we3 = make_mutable("\x03" * 16) mutable_si_3, rs3, cs3, we3 = make_mutable(b"\x03" * 16)
rs3a, cs3a = make_extra_lease(mutable_si_3, 1) rs3a, cs3a = make_extra_lease(mutable_si_3, 1)
sharenums = [0] sharenums = [0]
canary = FakeCanary() canary = FakeCanary()
# note: 'tahoe debug dump-share' will not handle this file, since the # note: 'tahoe debug dump-share' will not handle this file, since the
# inner contents are not a valid CHK share # inner contents are not a valid CHK share
data = "\xff" * 1000 data = b"\xff" * 1000
a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums, a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums,
1000, canary) 1000, canary)
@ -330,7 +332,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
def test_basic(self): def test_basic(self):
basedir = "storage/LeaseCrawler/basic" basedir = "storage/LeaseCrawler/basic"
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
ss = InstrumentedStorageServer(basedir, "\x00" * 20) ss = InstrumentedStorageServer(basedir, b"\x00" * 20)
# make it start sooner than usual. # make it start sooner than usual.
lc = ss.lease_checker lc = ss.lease_checker
lc.slow_start = 0 lc.slow_start = 0
@ -347,7 +349,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
storage_index_to_dir(immutable_si_0), storage_index_to_dir(immutable_si_0),
"not-a-share") "not-a-share")
f = open(fn, "wb") f = open(fn, "wb")
f.write("I am not a share.\n") f.write(b"I am not a share.\n")
f.close() f.close()
# this is before the crawl has started, so we're not in a cycle yet # this is before the crawl has started, so we're not in a cycle yet
@ -513,7 +515,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
# setting expiration_time to 2000 means that any lease which is more # setting expiration_time to 2000 means that any lease which is more
# than 2000s old will be expired. # than 2000s old will be expired.
ss = InstrumentedStorageServer(basedir, "\x00" * 20, ss = InstrumentedStorageServer(basedir, b"\x00" * 20,
expiration_enabled=True, expiration_enabled=True,
expiration_mode="age", expiration_mode="age",
expiration_override_lease_duration=2000) expiration_override_lease_duration=2000)
@ -653,7 +655,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
# is more than 2000s old will be expired. # is more than 2000s old will be expired.
now = time.time() now = time.time()
then = int(now - 2000) then = int(now - 2000)
ss = InstrumentedStorageServer(basedir, "\x00" * 20, ss = InstrumentedStorageServer(basedir, b"\x00" * 20,
expiration_enabled=True, expiration_enabled=True,
expiration_mode="cutoff-date", expiration_mode="cutoff-date",
expiration_cutoff_date=then) expiration_cutoff_date=then)
@ -800,7 +802,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
now = time.time() now = time.time()
then = int(now - 2000) then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20, ss = StorageServer(basedir, b"\x00" * 20,
expiration_enabled=True, expiration_enabled=True,
expiration_mode="cutoff-date", expiration_mode="cutoff-date",
expiration_cutoff_date=then, expiration_cutoff_date=then,
@ -857,7 +859,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
now = time.time() now = time.time()
then = int(now - 2000) then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20, ss = StorageServer(basedir, b"\x00" * 20,
expiration_enabled=True, expiration_enabled=True,
expiration_mode="cutoff-date", expiration_mode="cutoff-date",
expiration_cutoff_date=then, expiration_cutoff_date=then,
@ -913,14 +915,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
basedir = "storage/LeaseCrawler/bad_mode" basedir = "storage/LeaseCrawler/bad_mode"
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
e = self.failUnlessRaises(ValueError, e = self.failUnlessRaises(ValueError,
StorageServer, basedir, "\x00" * 20, StorageServer, basedir, b"\x00" * 20,
expiration_mode="bogus") expiration_mode="bogus")
self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e)) self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e))
def test_limited_history(self): def test_limited_history(self):
basedir = "storage/LeaseCrawler/limited_history" basedir = "storage/LeaseCrawler/limited_history"
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
ss = StorageServer(basedir, "\x00" * 20) ss = StorageServer(basedir, b"\x00" * 20)
# make it start sooner than usual. # make it start sooner than usual.
lc = ss.lease_checker lc = ss.lease_checker
lc.slow_start = 0 lc.slow_start = 0
@ -952,7 +954,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
def test_unpredictable_future(self): def test_unpredictable_future(self):
basedir = "storage/LeaseCrawler/unpredictable_future" basedir = "storage/LeaseCrawler/unpredictable_future"
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
ss = StorageServer(basedir, "\x00" * 20) ss = StorageServer(basedir, b"\x00" * 20)
# make it start sooner than usual. # make it start sooner than usual.
lc = ss.lease_checker lc = ss.lease_checker
lc.slow_start = 0 lc.slow_start = 0
@ -1015,7 +1017,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
def test_no_st_blocks(self): def test_no_st_blocks(self):
basedir = "storage/LeaseCrawler/no_st_blocks" basedir = "storage/LeaseCrawler/no_st_blocks"
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20, ss = No_ST_BLOCKS_StorageServer(basedir, b"\x00" * 20,
expiration_mode="age", expiration_mode="age",
expiration_override_lease_duration=-1000) expiration_override_lease_duration=-1000)
# a negative expiration_time= means the "configured-" # a negative expiration_time= means the "configured-"
@ -1054,7 +1056,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
] ]
basedir = "storage/LeaseCrawler/share_corruption" basedir = "storage/LeaseCrawler/share_corruption"
fileutil.make_dirs(basedir) fileutil.make_dirs(basedir)
ss = InstrumentedStorageServer(basedir, "\x00" * 20) ss = InstrumentedStorageServer(basedir, b"\x00" * 20)
w = StorageStatus(ss) w = StorageStatus(ss)
# make it start sooner than usual. # make it start sooner than usual.
lc = ss.lease_checker lc = ss.lease_checker
@ -1072,7 +1074,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
fn = os.path.join(ss.sharedir, storage_index_to_dir(first), "0") fn = os.path.join(ss.sharedir, storage_index_to_dir(first), "0")
f = open(fn, "rb+") f = open(fn, "rb+")
f.seek(0) f.seek(0)
f.write("BAD MAGIC") f.write(b"BAD MAGIC")
f.close() f.close()
# if get_share_file() doesn't see the correct mutable magic, it # if get_share_file() doesn't see the correct mutable magic, it
# assumes the file is an immutable share, and then # assumes the file is an immutable share, and then
@ -1081,7 +1083,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
# UnknownImmutableContainerVersionError. # UnknownImmutableContainerVersionError.
# also create an empty bucket # also create an empty bucket
empty_si = base32.b2a("\x04"*16) empty_si = base32.b2a(b"\x04"*16)
empty_bucket_dir = os.path.join(ss.sharedir, empty_bucket_dir = os.path.join(ss.sharedir,
storage_index_to_dir(empty_si)) storage_index_to_dir(empty_si))
fileutil.make_dirs(empty_bucket_dir) fileutil.make_dirs(empty_bucket_dir)