Merge pull request #1143 from tahoe-lafs/3800-lease-istorageserver-tests

IStorageServer tests for leases

Fixes ticket:3800

src/allmydata/test/test_istorageserver.py

@@ -24,10 +24,10 @@ from twisted.internet.defer import inlineCallbacks, returnValue
 
 from foolscap.api import Referenceable, RemoteException
 
-from allmydata.interfaces import IStorageServer
+from allmydata.interfaces import IStorageServer  # really, IStorageClient
 from .common_system import SystemTestMixin
 from .common import AsyncTestCase
-
+from allmydata.storage.server import StorageServer  # not a IStorageServer!!
 
 # Use random generator with known seed, so results are reproducible if tests
 # are run in the same order.
@@ -56,7 +56,7 @@ class IStorageServerSharedAPIsTestsMixin(object):
     """
     Tests for ``IStorageServer``'s shared APIs.
 
-    ``self.storage_server`` is expected to provide ``IStorageServer``.
+    ``self.storage_client`` is expected to provide ``IStorageServer``.
     """
 
     @inlineCallbacks
@@ -65,7 +65,7 @@ class IStorageServerSharedAPIsTestsMixin(object):
         ``IStorageServer`` returns a dictionary where the key is an expected
         protocol version.
         """
-        result = yield self.storage_server.get_version()
+        result = yield self.storage_client.get_version()
         self.assertIsInstance(result, dict)
         self.assertIn(b"http://allmydata.org/tahoe/protocols/storage/v1", result)
 
@@ -74,11 +74,16 @@ class IStorageServerImmutableAPIsTestsMixin(object):
     """
     Tests for ``IStorageServer``'s immutable APIs.
 
-    ``self.storage_server`` is expected to provide ``IStorageServer``.
+    ``self.storage_client`` is expected to provide ``IStorageServer``.
 
     ``self.disconnect()`` should disconnect and then reconnect, creating a new
-    ``self.storage_server``.  Some implementations may wish to skip tests using
+    ``self.storage_client``.  Some implementations may wish to skip tests using
     this; HTTP has no notion of disconnection.
+
+    ``self.server`` is expected to be the corresponding
+    ``allmydata.storage.server.StorageServer`` instance.  Time should be
+    instrumented, such that ``self.fake_time()`` and ``self.fake_sleep()``
+    return and advance the server time, respectively.
     """
 
     @inlineCallbacks
@@ -87,7 +92,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         allocate_buckets() with a new storage index returns the matching
         shares.
         """
-        (already_got, allocated) = yield self.storage_server.allocate_buckets(
+        (already_got, allocated) = yield self.storage_client.allocate_buckets(
             new_storage_index(),
             renew_secret=new_secret(),
             cancel_secret=new_secret(),
@@ -110,7 +115,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             new_secret(),
             new_secret(),
         )
-        (already_got, allocated) = yield self.storage_server.allocate_buckets(
+        (already_got, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -118,7 +123,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             allocated_size=1024,
             canary=Referenceable(),
         )
-        (already_got2, allocated2) = yield self.storage_server.allocate_buckets(
+        (already_got2, allocated2) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -146,7 +151,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             new_secret(),
             new_secret(),
         )
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -162,7 +167,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         yield abort_or_disconnect(allocated[0])
 
         # Write different data with no complaint:
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -198,7 +203,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             new_secret(),
             new_secret(),
         )
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -219,7 +224,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         # Bucket 0 has partial write.
         yield allocated[0].callRemote("write", 0, b"1" * 512)
 
-        (already_got, _) = yield self.storage_server.allocate_buckets(
+        (already_got, _) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -242,7 +247,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             new_secret(),
             new_secret(),
         )
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -261,7 +266,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         yield allocated[2].callRemote("write", 0, b"3" * 512)
         yield allocated[2].callRemote("close")
 
-        buckets = yield self.storage_server.get_buckets(storage_index)
+        buckets = yield self.storage_client.get_buckets(storage_index)
         self.assertEqual(set(buckets.keys()), {1, 2})
 
         self.assertEqual(
@@ -282,7 +287,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             new_secret(),
             new_secret(),
         )
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -307,7 +312,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
             new_secret(),
             new_secret(),
         )
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret,
             cancel_secret,
@@ -321,7 +326,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         yield allocated[0].callRemote("write", 5, b"1" * 20)
         yield allocated[0].callRemote("close")
 
-        buckets = yield self.storage_server.get_buckets(storage_index)
+        buckets = yield self.storage_client.get_buckets(storage_index)
         self.assertEqual(set(buckets.keys()), {0})
 
         self.assertEqual((yield buckets[0].callRemote("read", 0, 25)), b"1" * 25)
@@ -346,7 +351,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         ``IStorageServer.get_buckets()`` implementations.
         """
         storage_index = new_storage_index()
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret=new_secret(),
             cancel_secret=new_secret(),
@@ -362,7 +367,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         # Bucket 2 is partially written
         yield allocated[2].callRemote("write", 0, b"1" * 5)
 
-        buckets = yield self.storage_server.get_buckets(storage_index)
+        buckets = yield self.storage_client.get_buckets(storage_index)
         self.assertEqual(set(buckets.keys()), {1})
 
     @inlineCallbacks
@@ -375,7 +380,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         length = 256 * 17
 
         storage_index = new_storage_index()
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
             renew_secret=new_secret(),
             cancel_secret=new_secret(),
@@ -388,7 +393,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         yield allocated[0].callRemote("write", 0, total_data)
         yield allocated[0].callRemote("close")
 
-        buckets = yield self.storage_server.get_buckets(storage_index)
+        buckets = yield self.storage_client.get_buckets(storage_index)
         bucket = buckets[0]
         for start, to_read in [
             (0, 250),  # fraction
@@ -408,10 +413,12 @@ class IStorageServerImmutableAPIsTestsMixin(object):
     def create_share(self):
         """Create a share, return the storage index."""
         storage_index = new_storage_index()
-        (_, allocated) = yield self.storage_server.allocate_buckets(
+        renew_secret = new_secret()
+        cancel_secret = new_secret()
+        (_, allocated) = yield self.storage_client.allocate_buckets(
             storage_index,
-            renew_secret=new_secret(),
+            renew_secret=renew_secret,
-            cancel_secret=new_secret(),
+            cancel_secret=cancel_secret,
             sharenums=set(range(1)),
             allocated_size=10,
             canary=Referenceable(),
@@ -419,7 +426,7 @@ class IStorageServerImmutableAPIsTestsMixin(object):
 
         yield allocated[0].callRemote("write", 0, b"0123456789")
         yield allocated[0].callRemote("close")
-        returnValue(storage_index)
+        returnValue((storage_index, renew_secret, cancel_secret))
 
     @inlineCallbacks
     def test_bucket_advise_corrupt_share(self):
@@ -428,8 +435,8 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         ``IStorageServer.get_buckets()`` does not result in error (other
         behavior is opaque at this level of abstraction).
         """
-        storage_index = yield self.create_share()
+        storage_index, _, _ = yield self.create_share()
-        buckets = yield self.storage_server.get_buckets(storage_index)
+        buckets = yield self.storage_client.get_buckets(storage_index)
         yield buckets[0].callRemote("advise_corrupt_share", b"OH NO")
 
     @inlineCallbacks
@@ -439,17 +446,72 @@ class IStorageServerImmutableAPIsTestsMixin(object):
         result in error (other behavior is opaque at this level of
         abstraction).
         """
-        storage_index = yield self.create_share()
+        storage_index, _, _ = yield self.create_share()
-        yield self.storage_server.advise_corrupt_share(
+        yield self.storage_client.advise_corrupt_share(
             b"immutable", storage_index, 0, b"ono"
         )
 
+    @inlineCallbacks
+    def test_allocate_buckets_creates_lease(self):
+        """
+        When buckets are created using ``allocate_buckets()``, a lease is
+        created once writing is done.
+        """
+        storage_index, _, _ = yield self.create_share()
+        [lease] = self.server.get_leases(storage_index)
+        # Lease expires in 31 days.
+        self.assertTrue(
+            lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10)
+        )
+
+    @inlineCallbacks
+    def test_add_lease_renewal(self):
+        """
+        If the lease secret is reused, ``add_lease()`` extends the existing
+        lease.
+        """
+        storage_index, renew_secret, cancel_secret = yield self.create_share()
+        [lease] = self.server.get_leases(storage_index)
+        initial_expiration_time = lease.get_expiration_time()
+
+        # Time passes:
+        self.fake_sleep(178)
+
+        # We renew the lease:
+        yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret)
+        [lease] = self.server.get_leases(storage_index)
+        new_expiration_time = lease.get_expiration_time()
+        self.assertEqual(new_expiration_time - initial_expiration_time, 178)
+
+    @inlineCallbacks
+    def test_add_new_lease(self):
+        """
+        If a new lease secret is used, ``add_lease()`` creates a new lease.
+        """
+        storage_index, _, _ = yield self.create_share()
+        [lease] = self.server.get_leases(storage_index)
+        initial_expiration_time = lease.get_expiration_time()
+
+        # Time passes:
+        self.fake_sleep(167)
+
+        # We create a new lease:
+        renew_secret = new_secret()
+        cancel_secret = new_secret()
+        yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret)
+        [lease1, lease2] = self.server.get_leases(storage_index)
+        self.assertEqual(lease1.get_expiration_time(), initial_expiration_time)
+        self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167)
+
 
 class IStorageServerMutableAPIsTestsMixin(object):
     """
     Tests for ``IStorageServer``'s mutable APIs.
 
-    ``self.storage_server`` is expected to provide ``IStorageServer``.
+    ``self.storage_client`` is expected to provide ``IStorageServer``.
+
+    ``self.server`` is expected to be the corresponding
+    ``allmydata.storage.server.StorageServer`` instance.
 
     ``STARAW`` is short for ``slot_testv_and_readv_and_writev``.
     """
@@ -460,7 +522,7 @@ class IStorageServerMutableAPIsTestsMixin(object):
 
     def staraw(self, *args, **kwargs):
         """Like ``slot_testv_and_readv_and_writev``, but less typing."""
-        return self.storage_server.slot_testv_and_readv_and_writev(*args, **kwargs)
+        return self.storage_client.slot_testv_and_readv_and_writev(*args, **kwargs)
 
     @inlineCallbacks
     def test_STARAW_reads_after_write(self):
@@ -756,7 +818,7 @@ class IStorageServerMutableAPIsTestsMixin(object):
         )
         self.assertEqual(written, True)
 
-        reads = yield self.storage_server.slot_readv(
+        reads = yield self.storage_client.slot_readv(
             storage_index,
             shares=[0, 1],
             # Whole thing, partial, going beyond the edge, completely outside
@@ -787,7 +849,7 @@ class IStorageServerMutableAPIsTestsMixin(object):
         )
         self.assertEqual(written, True)
 
-        reads = yield self.storage_server.slot_readv(
+        reads = yield self.storage_client.slot_readv(
             storage_index,
             shares=[],
             readv=[(0, 7)],
@@ -798,12 +860,8 @@ class IStorageServerMutableAPIsTestsMixin(object):
         )
 
     @inlineCallbacks
-    def test_advise_corrupt_share(self):
+    def create_slot(self):
-        """
+        """Create a slot with sharenum 0."""
-        Calling ``advise_corrupt_share()`` on a mutable share does not
-        result in error (other behavior is opaque at this level of
-        abstraction).
-        """
         secrets = self.new_secrets()
         storage_index = new_storage_index()
         (written, _) = yield self.staraw(
@@ -815,11 +873,129 @@ class IStorageServerMutableAPIsTestsMixin(object):
             r_vector=[],
         )
         self.assertEqual(written, True)
+        returnValue((secrets, storage_index))
 
-        yield self.storage_server.advise_corrupt_share(
+    @inlineCallbacks
+    def test_advise_corrupt_share(self):
+        """
+        Calling ``advise_corrupt_share()`` on a mutable share does not
+        result in error (other behavior is opaque at this level of
+        abstraction).
+        """
+        secrets, storage_index = yield self.create_slot()
+
+        yield self.storage_client.advise_corrupt_share(
             b"mutable", storage_index, 0, b"ono"
         )
 
+    @inlineCallbacks
+    def test_STARAW_create_lease(self):
+        """
+        When STARAW creates a new slot, it also creates a lease.
+        """
+        _, storage_index = yield self.create_slot()
+        [lease] = self.server.get_slot_leases(storage_index)
+        # Lease expires in 31 days.
+        self.assertTrue(
+            lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10)
+        )
+
+    @inlineCallbacks
+    def test_STARAW_renews_lease(self):
+        """
+        When STARAW is run on an existing slot with same renewal secret, it
+        renews the lease.
+        """
+        secrets, storage_index = yield self.create_slot()
+        [lease] = self.server.get_slot_leases(storage_index)
+        initial_expire = lease.get_expiration_time()
+
+        # Time passes...
+        self.fake_sleep(17)
+
+        # We do another write:
+        (written, _) = yield self.staraw(
+            storage_index,
+            secrets,
+            tw_vectors={
+                0: ([], [(0, b"1234567")], 7),
+            },
+            r_vector=[],
+        )
+        self.assertEqual(written, True)
+
+        # The lease has been renewed:
+        [lease] = self.server.get_slot_leases(storage_index)
+        self.assertEqual(lease.get_expiration_time() - initial_expire, 17)
+
+    @inlineCallbacks
+    def test_STARAW_new_lease(self):
+        """
+        When STARAW is run with a new renewal secret on an existing slot, it
+        adds a new lease.
+        """
+        secrets, storage_index = yield self.create_slot()
+        [lease] = self.server.get_slot_leases(storage_index)
+        initial_expire = lease.get_expiration_time()
+
+        # Time passes...
+        self.fake_sleep(19)
+
+        # We do another write:
+        (written, _) = yield self.staraw(
+            storage_index,
+            (secrets[0], new_secret(), new_secret()),
+            tw_vectors={
+                0: ([], [(0, b"1234567")], 7),
+            },
+            r_vector=[],
+        )
+        self.assertEqual(written, True)
+
+        # A new lease was added:
+        [lease1, lease2] = self.server.get_slot_leases(storage_index)
+        self.assertEqual(lease1.get_expiration_time(), initial_expire)
+        self.assertEqual(lease2.get_expiration_time() - initial_expire, 19)
+
+    @inlineCallbacks
+    def test_add_lease_renewal(self):
+        """
+        If the lease secret is reused, ``add_lease()`` extends the existing
+        lease.
+        """
+        secrets, storage_index = yield self.create_slot()
+        [lease] = self.server.get_slot_leases(storage_index)
+        initial_expiration_time = lease.get_expiration_time()
+
+        # Time passes:
+        self.fake_sleep(178)
+
+        # We renew the lease:
+        yield self.storage_client.add_lease(storage_index, secrets[1], secrets[2])
+        [lease] = self.server.get_slot_leases(storage_index)
+        new_expiration_time = lease.get_expiration_time()
+        self.assertEqual(new_expiration_time - initial_expiration_time, 178)
+
+    @inlineCallbacks
+    def test_add_new_lease(self):
+        """
+        If a new lease secret is used, ``add_lease()`` creates a new lease.
+        """
+        secrets, storage_index = yield self.create_slot()
+        [lease] = self.server.get_slot_leases(storage_index)
+        initial_expiration_time = lease.get_expiration_time()
+
+        # Time passes:
+        self.fake_sleep(167)
+
+        # We create a new lease:
+        renew_secret = new_secret()
+        cancel_secret = new_secret()
+        yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret)
+        [lease1, lease2] = self.server.get_slot_leases(storage_index)
+        self.assertEqual(lease1.get_expiration_time(), initial_expiration_time)
+        self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167)
+
 
 class _FoolscapMixin(SystemTestMixin):
     """Run tests on Foolscap version of ``IStorageServer."""
@@ -833,8 +1009,24 @@ class _FoolscapMixin(SystemTestMixin):
         self.basedir = "test_istorageserver/" + self.id()
         yield SystemTestMixin.setUp(self)
         yield self.set_up_nodes(1)
-        self.storage_server = self._get_native_server().get_storage_server()
+        self.storage_client = self._get_native_server().get_storage_server()
-        self.assertTrue(IStorageServer.providedBy(self.storage_server))
+        self.assertTrue(IStorageServer.providedBy(self.storage_client))
+        self.server = None
+        for s in self.clients[0].services:
+            if isinstance(s, StorageServer):
+                self.server = s
+                break
+        assert self.server is not None, "Couldn't find StorageServer"
+        self._current_time = 123456
+        self.server._get_current_time = self.fake_time
+
+    def fake_time(self):
+        """Return the current fake, test-controlled, time."""
+        return self._current_time
+
+    def fake_sleep(self, seconds):
+        """Advance the fake time by the given number of seconds."""
+        self._current_time += seconds
 
     @inlineCallbacks
     def tearDown(self):
@@ -846,10 +1038,10 @@ class _FoolscapMixin(SystemTestMixin):
         """
         Disconnect and then reconnect with a new ``IStorageServer``.
         """
-        current = self.storage_server
+        current = self.storage_client
         yield self.bounce_client(0)
-        self.storage_server = self._get_native_server().get_storage_server()
+        self.storage_client = self._get_native_server().get_storage_server()
-        assert self.storage_server is not current
+        assert self.storage_client is not current
 
 
 class FoolscapSharedAPIsTests(