ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-01 16:58:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1200 from tahoe-lafs/3896-mutable-http-part-4

Browse Source 
Mutable http storage API, part 4

Fixes ticket:3896
This commit is contained in:
Itamar Turner-Trauring 2022-06-10 09:47:15 -04:00 committed by GitHub
commit a78f50470f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 500 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3896.minor Normal file
View File

11
src/allmydata/storage/http_client.py
View File

@ -7,7 +7,7 @@ from __future__ import annotations
from typing import Union, Optional, Sequence, Mapping
from base64 import b64encode
from attrs import define, asdict, frozen
from attrs import define, asdict, frozen, field
# TODO Make sure to import Python version?
from cbor2 import loads, dumps
@ -586,7 +586,7 @@ class StorageClientImmutables(object):
)
@inlineCallbacks
def list_shares(self, storage_index): # type: (bytes,) -> Deferred[set[int]]
def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
"""
Return the set of shares for a given storage index.
"""
@ -646,8 +646,8 @@ class ReadVector:
class TestWriteVectors:
"""Test and write vectors for a specific share."""
test_vectors: Sequence[TestVector]
write_vectors: Sequence[WriteVector]
test_vectors: Sequence[TestVector] = field(factory=list)
write_vectors: Sequence[WriteVector] = field(factory=list)
new_length: Optional[int] = None
def asdict(self) -> dict:
@ -696,7 +696,6 @@ class StorageClientMutables:
Given a mapping between share numbers and test/write vectors, the tests
are done and if they are valid the writes are done.
"""
# TODO unit test all the things
url = self._client.relative_url(
"/v1/mutable/{}/read-test-write".format(_encode_si(storage_index))
)
@ -731,7 +730,6 @@ class StorageClientMutables:
"""
Download a chunk of data from a share.
"""
# TODO unit test all the things
return read_share_chunk(
self._client, "mutable", storage_index, share_number, offset, length
)
@ -741,7 +739,6 @@ class StorageClientMutables:
"""
List the share numbers for a given storage index.
"""
# TODO unit test all the things
url = self._client.relative_url(
"/v1/mutable/{}/shares".format(_encode_si(storage_index))
)

142
src/allmydata/storage/http_server.py
View File

@ -2,7 +2,8 @@
HTTP server for storage.
"""
from typing import Dict, List, Set, Tuple, Any
from __future__ import annotations
from typing import Dict, List, Set, Tuple, Any, Callable
from functools import wraps
from base64 import b64decode
@ -262,7 +263,7 @@ _SCHEMAS = {
* share_number: {
"test": [* {"offset": uint, "size": uint, "specimen": bstr}]
"write": [* {"offset": uint, "data": bstr}]
"new-length": uint // null
"new-length": uint / null
}
}
"read-vector": [* {"offset": uint, "size": uint}]
@ -273,6 +274,61 @@ _SCHEMAS = {
}
def read_range(request, read_data: Callable[[int, int], bytes]) -> None:
"""
Read an optional ``Range`` header, reads data appropriately via the given
callable, writes the data to the request.
Only parses a subset of ``Range`` headers that we support: must be set,
bytes only, only a single range, the end must be explicitly specified.
Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is
not possible or the header isn't set.
Takes a function that will do the actual reading given the start offset and
a length to read.
The resulting data is written to the request.
"""
if request.getHeader("range") is None:
# Return the whole thing.
start = 0
while True:
# TODO should probably yield to event loop occasionally...
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872
data = read_data(start, start + 65536)
if not data:
request.finish()
return
request.write(data)
start += len(data)
range_header = parse_range_header(request.getHeader("range"))
if (
range_header is None # failed to parse
or range_header.units != "bytes"
or len(range_header.ranges) > 1 # more than one range
or range_header.ranges[0][1] is None # range without end
):
raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)
offset, end = range_header.ranges[0]
# TODO limit memory usage
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872
data = read_data(offset, end - offset)
request.setResponseCode(http.PARTIAL_CONTENT)
if len(data):
# For empty bodies the content-range header makes no sense since
# the end of the range is inclusive.
request.setHeader(
"content-range",
ContentRange("bytes", offset, offset + len(data)).to_header(),
)
request.write(data)
request.finish()
class HTTPServer(object):
"""
A HTTP interface to the storage server.
@ -492,44 +548,7 @@ class HTTPServer(object):
request.setResponseCode(http.NOT_FOUND)
return b""
if request.getHeader("range") is None:
# Return the whole thing.
start = 0
while True:
# TODO should probably yield to event loop occasionally...
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872
data = bucket.read(start, start + 65536)
if not data:
request.finish()
return
request.write(data)
start += len(data)
range_header = parse_range_header(request.getHeader("range"))
if (
range_header is None
or range_header.units != "bytes"
or len(range_header.ranges) > 1 # more than one range
or range_header.ranges[0][1] is None # range without end
):
request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
return b""
offset, end = range_header.ranges[0]
# TODO limit memory usage
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872
data = bucket.read(offset, end - offset)
request.setResponseCode(http.PARTIAL_CONTENT)
if len(data):
# For empty bodies the content-range header makes no sense since
# the end of the range is inclusive.
request.setHeader(
"content-range",
ContentRange("bytes", offset, offset + len(data)).to_header(),
)
return data
return read_range(request, bucket.read)
@_authorized_route(
_app,
@ -581,7 +600,6 @@ class HTTPServer(object):
)
def mutable_read_test_write(self, request, authorization, storage_index):
"""Read/test/write combined operation for mutables."""
# TODO unit tests
rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"])
secrets = (
authorization[Secrets.WRITE_ENABLER],
@ -617,40 +635,15 @@ class HTTPServer(object):
)
def read_mutable_chunk(self, request, authorization, storage_index, share_number):
"""Read a chunk from a mutable."""
if request.getHeader("range") is None:
# TODO in follow-up ticket
raise NotImplementedError()
def read_data(offset, length):
try:
return self._storage_server.slot_readv(
storage_index, [share_number], [(offset, length)]
)[share_number][0]
except KeyError:
raise _HTTPError(http.NOT_FOUND)
# TODO reduce duplication with immutable reads?
# TODO unit tests, perhaps shared if possible
range_header = parse_range_header(request.getHeader("range"))
if (
range_header is None
or range_header.units != "bytes"
or len(range_header.ranges) > 1 # more than one range
or range_header.ranges[0][1] is None # range without end
):
request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
return b""
offset, end = range_header.ranges[0]
# TODO limit memory usage
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872
data = self._storage_server.slot_readv(
storage_index, [share_number], [(offset, end - offset)]
)[share_number][0]
# TODO reduce duplication?
request.setResponseCode(http.PARTIAL_CONTENT)
if len(data):
# For empty bodies the content-range header makes no sense since
# the end of the range is inclusive.
request.setHeader(
"content-range",
ContentRange("bytes", offset, offset + len(data)).to_header(),
)
return data
return read_range(request, read_data)
@_authorized_route(
_app,
@ -673,7 +666,6 @@ class HTTPServer(object):
self, request, authorization, storage_index, share_number
):
"""Indicate that given share is corrupt, with a text reason."""
# TODO unit test all the paths
if share_number not in {
shnum for (shnum, _) in self._storage_server.get_shares(storage_index)
}:

614
src/allmydata/test/test_storage_http.py
View File

@ -5,7 +5,7 @@ Tests for HTTP storage client + server.
from base64 import b64encode
from contextlib import contextmanager
from os import urandom
from typing import Union, Callable, Tuple, Iterable
from cbor2 import dumps
from pycddl import ValidationError as CDDLValidationError
from hypothesis import assume, given, strategies as st
@ -23,6 +23,7 @@ from werkzeug.exceptions import NotFound as WNotFound
from .common import SyncTestCase
from ..storage.http_common import get_content_type, CBOR_MIME_TYPE
from ..storage.common import si_b2a
from ..storage.lease import LeaseInfo
from ..storage.server import StorageServer
from ..storage.http_server import (
HTTPServer,
@ -40,6 +41,12 @@ from ..storage.http_client import (
UploadProgress,
StorageClientGeneral,
_encode_si,
StorageClientMutables,
TestWriteVectors,
WriteVector,
ReadVector,
ReadTestWriteResult,
TestVector,
)
@ -781,141 +788,6 @@ class ImmutableHTTPAPITests(SyncTestCase):
)
)
def upload(self, share_number, data_length=26):
"""
Create a share, return (storage_index, uploaded_data).
"""
uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[
:data_length
]
(upload_secret, _, storage_index, _) = self.create_upload(
{share_number}, data_length
)
result_of(
self.imm_client.write_share_chunk(
storage_index,
share_number,
upload_secret,
0,
uploaded_data,
)
)
return storage_index, uploaded_data
def test_read_of_wrong_storage_index_fails(self):
"""
Reading from unknown storage index results in 404.
"""
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(
self.imm_client.read_share_chunk(
b"1" * 16,
1,
0,
10,
)
)
def test_read_of_wrong_share_number_fails(self):
"""
Reading from unknown storage index results in 404.
"""
storage_index, _ = self.upload(1)
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(
self.imm_client.read_share_chunk(
storage_index,
7, # different share number
0,
10,
)
)
def test_read_with_negative_offset_fails(self):
"""
Malformed or unsupported Range headers result in 416 (requested range
not satisfiable) error.
"""
storage_index, _ = self.upload(1)
def check_bad_range(bad_range_value):
client = StorageClientImmutables(
StorageClientWithHeadersOverride(
self.http.client, {"range": bad_range_value}
)
)
with assert_fails_with_http_code(
self, http.REQUESTED_RANGE_NOT_SATISFIABLE
):
result_of(
client.read_share_chunk(
storage_index,
1,
0,
10,
)
)
# Bad unit
check_bad_range("molluscs=0-9")
# Negative offsets
check_bad_range("bytes=-2-9")
check_bad_range("bytes=0--10")
# Negative offset no endpoint
check_bad_range("bytes=-300-")
check_bad_range("bytes=")
# Multiple ranges are currently unsupported, even if they're
# semantically valid under HTTP:
check_bad_range("bytes=0-5, 6-7")
# Ranges without an end are currently unsupported, even if they're
# semantically valid under HTTP.
check_bad_range("bytes=0-")
@given(data_length=st.integers(min_value=1, max_value=300000))
def test_read_with_no_range(self, data_length):
"""
A read with no range returns the whole immutable.
"""
storage_index, uploaded_data = self.upload(1, data_length)
response = result_of(
self.http.client.request(
"GET",
self.http.client.relative_url(
"/v1/immutable/{}/1".format(_encode_si(storage_index))
),
)
)
self.assertEqual(response.code, http.OK)
self.assertEqual(result_of(response.content()), uploaded_data)
def test_validate_content_range_response_to_read(self):
"""
The server responds to ranged reads with an appropriate Content-Range
header.
"""
storage_index, _ = self.upload(1, 26)
def check_range(requested_range, expected_response):
headers = Headers()
headers.setRawHeaders("range", [requested_range])
response = result_of(
self.http.client.request(
"GET",
self.http.client.relative_url(
"/v1/immutable/{}/1".format(_encode_si(storage_index))
),
headers=headers,
)
)
self.assertEqual(
response.headers.getRawHeaders("content-range"), [expected_response]
)
check_range("bytes=0-10", "bytes 0-10/*")
# Can't go beyond the end of the immutable!
check_range("bytes=10-100", "bytes 10-25/*")
def test_timed_out_upload_allows_reupload(self):
"""
If an in-progress upload times out, it is cancelled altogether,
@ -1056,25 +928,237 @@ class ImmutableHTTPAPITests(SyncTestCase):
),
)
def test_lease_renew_and_add(self):
def test_lease_on_unknown_storage_index(self):
"""
It's possible the renew the lease on an uploaded immutable, by using
the same renewal secret, or add a new lease by choosing a different
renewal secret.
An attempt to renew an unknown storage index will result in a HTTP 404.
"""
# Create immutable:
(upload_secret, lease_secret, storage_index, _) = self.create_upload({0}, 100)
storage_index = urandom(16)
secret = b"A" * 32
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(
self.general_client.add_or_renew_lease(storage_index, secret, secret)
)
class MutableHTTPAPIsTests(SyncTestCase):
"""Tests for mutable APIs."""
def setUp(self):
super(MutableHTTPAPIsTests, self).setUp()
self.http = self.useFixture(HttpTestFixture())
self.mut_client = StorageClientMutables(self.http.client)
def create_upload(self, data=b"abcdef"):
"""
Utility that creates shares 0 and 1 with bodies
``{data}-{share_number}``.
"""
write_secret = urandom(32)
lease_secret = urandom(32)
storage_index = urandom(16)
result_of(
self.imm_client.write_share_chunk(
self.mut_client.read_test_write_chunks(
storage_index,
0,
upload_secret,
0,
b"A" * 100,
write_secret,
lease_secret,
lease_secret,
{
0: TestWriteVectors(
write_vectors=[WriteVector(offset=0, data=data + b"-0")]
),
1: TestWriteVectors(
write_vectors=[
WriteVector(offset=0, data=data),
WriteVector(offset=len(data), data=b"-1"),
]
),
},
[],
)
)
return storage_index, write_secret, lease_secret
[lease] = self.http.storage_server.get_leases(storage_index)
def test_write_can_be_read(self):
"""
Written data can be read using ``read_share_chunk``.
"""
storage_index, _, _ = self.create_upload()
data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7))
data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8))
self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1"))
def test_read_before_write(self):
"""In combo read/test/write operation, reads happen before writes."""
storage_index, write_secret, lease_secret = self.create_upload()
result = result_of(
self.mut_client.read_test_write_chunks(
storage_index,
write_secret,
lease_secret,
lease_secret,
{
0: TestWriteVectors(
write_vectors=[WriteVector(offset=1, data=b"XYZ")]
),
},
[ReadVector(0, 8)],
)
)
# Reads are from before the write:
self.assertEqual(
result,
ReadTestWriteResult(
success=True, reads={0: [b"abcdef-0"], 1: [b"abcdef-1"]}
),
)
# But the write did happen:
data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8))
data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8))
self.assertEqual((data0, data1), (b"aXYZef-0", b"abcdef-1"))
def test_conditional_write(self):
"""Uploads only happen if the test passes."""
storage_index, write_secret, lease_secret = self.create_upload()
result_failed = result_of(
self.mut_client.read_test_write_chunks(
storage_index,
write_secret,
lease_secret,
lease_secret,
{
0: TestWriteVectors(
test_vectors=[TestVector(1, 4, b"FAIL")],
write_vectors=[WriteVector(offset=1, data=b"XYZ")],
),
},
[],
)
)
self.assertFalse(result_failed.success)
# This time the test matches:
result = result_of(
self.mut_client.read_test_write_chunks(
storage_index,
write_secret,
lease_secret,
lease_secret,
{
0: TestWriteVectors(
test_vectors=[TestVector(1, 4, b"bcde")],
write_vectors=[WriteVector(offset=1, data=b"XYZ")],
),
},
[ReadVector(0, 8)],
)
)
self.assertTrue(result.success)
self.assertEqual(
result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)),
b"aXYZef-0",
)
def test_list_shares(self):
"""``list_shares()`` returns the shares for a given storage index."""
storage_index, _, _ = self.create_upload()
self.assertEqual(result_of(self.mut_client.list_shares(storage_index)), {0, 1})
def test_non_existent_list_shares(self):
"""A non-existent storage index errors when shares are listed."""
with self.assertRaises(ClientException) as exc:
result_of(self.mut_client.list_shares(urandom(32)))
self.assertEqual(exc.exception.code, http.NOT_FOUND)
def test_wrong_write_enabler(self):
"""Writes with the wrong write enabler fail, and are not processed."""
storage_index, write_secret, lease_secret = self.create_upload()
with self.assertRaises(ClientException) as exc:
result_of(
self.mut_client.read_test_write_chunks(
storage_index,
urandom(32),
lease_secret,
lease_secret,
{
0: TestWriteVectors(
write_vectors=[WriteVector(offset=1, data=b"XYZ")]
),
},
[ReadVector(0, 8)],
)
)
self.assertEqual(exc.exception.code, http.UNAUTHORIZED)
# The write did not happen:
self.assertEqual(
result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)),
b"abcdef-0",
)
class SharedImmutableMutableTestsMixin:
"""
Shared tests for mutables and immutables where the API is the same.
"""
KIND: str # either "mutable" or "immutable"
general_client: StorageClientGeneral
client: Union[StorageClientImmutables, StorageClientMutables]
clientFactory: Callable[
[StorageClient], Union[StorageClientImmutables, StorageClientMutables]
]
def upload(self, share_number: int, data_length=26) -> Tuple[bytes, bytes, bytes]:
"""
Create a share, return (storage_index, uploaded_data, lease secret).
"""
raise NotImplementedError
def get_leases(self, storage_index: bytes) -> Iterable[LeaseInfo]:
"""Get leases for the storage index."""
raise NotImplementedError()
def test_advise_corrupt_share(self):
"""
Advising share was corrupted succeeds from HTTP client's perspective,
and calls appropriate method on server.
"""
corrupted = []
self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append(
args
)
storage_index, _, _ = self.upload(13)
reason = "OHNO \u1235"
result_of(self.client.advise_corrupt_share(storage_index, 13, reason))
self.assertEqual(
corrupted,
[(self.KIND.encode("ascii"), storage_index, 13, reason.encode("utf-8"))],
)
def test_advise_corrupt_share_unknown(self):
"""
Advising an unknown share was corrupted results in 404.
"""
storage_index, _, _ = self.upload(13)
reason = "OHNO \u1235"
result_of(self.client.advise_corrupt_share(storage_index, 13, reason))
for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]:
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(self.client.advise_corrupt_share(si, share_number, reason))
def test_lease_renew_and_add(self):
"""
It's possible the renew the lease on an uploaded mutable/immutable, by
using the same renewal secret, or add a new lease by choosing a
different renewal secret.
"""
# Create a storage index:
storage_index, _, lease_secret = self.upload(0)
[lease] = self.get_leases(storage_index)
initial_expiration_time = lease.get_expiration_time()
# Time passes:
@ -1098,47 +1182,207 @@ class ImmutableHTTPAPITests(SyncTestCase):
)
)
[lease1, lease2] = self.http.storage_server.get_leases(storage_index)
[lease1, lease2] = self.get_leases(storage_index)
self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167)
self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177)
def test_lease_on_unknown_storage_index(self):
def test_read_of_wrong_storage_index_fails(self):
"""
An attempt to renew an unknown storage index will result in a HTTP 404.
Reading from unknown storage index results in 404.
"""
storage_index = urandom(16)
secret = b"A" * 32
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(self.general_client.add_or_renew_lease(storage_index, secret, secret))
def test_advise_corrupt_share(self):
"""
Advising share was corrupted succeeds from HTTP client's perspective,
and calls appropriate method on server.
"""
corrupted = []
self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append(
args
)
storage_index, _ = self.upload(13)
reason = "OHNO \u1235"
result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason))
self.assertEqual(
corrupted, [(b"immutable", storage_index, 13, reason.encode("utf-8"))]
)
def test_advise_corrupt_share_unknown(self):
"""
Advising an unknown share was corrupted results in 404.
"""
storage_index, _ = self.upload(13)
reason = "OHNO \u1235"
result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason))
for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]:
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(
self.imm_client.advise_corrupt_share(si, share_number, reason)
result_of(
self.client.read_share_chunk(
b"1" * 16,
1,
0,
10,
)
)
def test_read_of_wrong_share_number_fails(self):
"""
Reading from unknown storage index results in 404.
"""
storage_index, _, _ = self.upload(1)
with assert_fails_with_http_code(self, http.NOT_FOUND):
result_of(
self.client.read_share_chunk(
storage_index,
7, # different share number
0,
10,
)
)
def test_read_with_negative_offset_fails(self):
"""
Malformed or unsupported Range headers result in 416 (requested range
not satisfiable) error.
"""
storage_index, _, _ = self.upload(1)
def check_bad_range(bad_range_value):
client = self.clientFactory(
StorageClientWithHeadersOverride(
self.http.client, {"range": bad_range_value}
)
)
with assert_fails_with_http_code(
self, http.REQUESTED_RANGE_NOT_SATISFIABLE
):
result_of(
client.read_share_chunk(
storage_index,
1,
0,
10,
)
)
# Bad unit
check_bad_range("molluscs=0-9")
# Negative offsets
check_bad_range("bytes=-2-9")
check_bad_range("bytes=0--10")
# Negative offset no endpoint
check_bad_range("bytes=-300-")
check_bad_range("bytes=")
# Multiple ranges are currently unsupported, even if they're
# semantically valid under HTTP:
check_bad_range("bytes=0-5, 6-7")
# Ranges without an end are currently unsupported, even if they're
# semantically valid under HTTP.
check_bad_range("bytes=0-")
@given(data_length=st.integers(min_value=1, max_value=300000))
def test_read_with_no_range(self, data_length):
"""
A read with no range returns the whole mutable/immutable.
"""
storage_index, uploaded_data, _ = self.upload(1, data_length)
response = result_of(
self.http.client.request(
"GET",
self.http.client.relative_url(
"/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
),
)
)
self.assertEqual(response.code, http.OK)
self.assertEqual(result_of(response.content()), uploaded_data)
def test_validate_content_range_response_to_read(self):
"""
The server responds to ranged reads with an appropriate Content-Range
header.
"""
storage_index, _, _ = self.upload(1, 26)
def check_range(requested_range, expected_response):
headers = Headers()
headers.setRawHeaders("range", [requested_range])
response = result_of(
self.http.client.request(
"GET",
self.http.client.relative_url(
"/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
),
headers=headers,
)
)
self.assertEqual(
response.headers.getRawHeaders("content-range"), [expected_response]
)
check_range("bytes=0-10", "bytes 0-10/*")
# Can't go beyond the end of the mutable/immutable!
check_range("bytes=10-100", "bytes 10-25/*")
class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase):
"""Shared tests, running on immutables."""
KIND = "immutable"
clientFactory = StorageClientImmutables
def setUp(self):
super(ImmutableSharedTests, self).setUp()
self.http = self.useFixture(HttpTestFixture())
self.client = self.clientFactory(self.http.client)
self.general_client = StorageClientGeneral(self.http.client)
def upload(self, share_number, data_length=26):
"""
Create a share, return (storage_index, uploaded_data, lease_secret).
"""
uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[
:data_length
]
upload_secret = urandom(32)
lease_secret = urandom(32)
storage_index = urandom(16)
result_of(
self.client.create(
storage_index,
{share_number},
data_length,
upload_secret,
lease_secret,
lease_secret,
)
)
result_of(
self.client.write_share_chunk(
storage_index,
share_number,
upload_secret,
0,
uploaded_data,
)
)
return storage_index, uploaded_data, lease_secret
def get_leases(self, storage_index):
return self.http.storage_server.get_leases(storage_index)
class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase):
"""Shared tests, running on mutables."""
KIND = "mutable"
clientFactory = StorageClientMutables
def setUp(self):
super(MutableSharedTests, self).setUp()
self.http = self.useFixture(HttpTestFixture())
self.client = self.clientFactory(self.http.client)
self.general_client = StorageClientGeneral(self.http.client)
def upload(self, share_number, data_length=26):
"""
Create a share, return (storage_index, uploaded_data, lease_secret).
"""
data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[:data_length]
write_secret = urandom(32)
lease_secret = urandom(32)
storage_index = urandom(16)
result_of(
self.client.read_test_write_chunks(
storage_index,
write_secret,
lease_secret,
lease_secret,
{
share_number: TestWriteVectors(
write_vectors=[WriteVector(offset=0, data=data)]
),
},
[],
)
)
return storage_index, data, lease_secret
def get_leases(self, storage_index):
return self.http.storage_server.get_slot_leases(storage_index)