Merge remote-tracking branch 'origin/master' into 3940-http-timeouts

docs/proposed/http-storage-node-protocol.rst

@@ -30,12 +30,12 @@ Glossary
    introducer
      a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers
 
-   fURL
+   :ref:`fURLs <fURLs>`
      a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol
      (the storage service is an example of such an object)
 
-   NURL
-     a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap
+   :ref:`NURLs <NURLs>`
+     a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap
 
    swissnum
      a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service
@@ -579,24 +579,6 @@ Responses:
   the response is ``CONFLICT``.
   At this point the only thing to do is abort the upload and start from scratch (see below).
 
-``PUT /storage/v1/immutable/:storage_index/:share_number/abort``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-This cancels an *in-progress* upload.
-
-The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
-
-    X-Tahoe-Authorization: upload-secret <base64-upload-secret>
-
-The response code:
-
-* When the upload is still in progress and therefore the abort has succeeded,
-  the response is ``OK``.
-  Future uploads can start from scratch with no pre-existing upload state stored on the server.
-* If the uploaded has already finished, the response is 405 (Method Not Allowed)
-  and no change is made.
-
-
 Discussion
 ``````````
 
@@ -615,6 +597,25 @@ From RFC 7231::
    PATCH method defined in [RFC5789]).
 
 
+
+``PUT /storage/v1/immutable/:storage_index/:share_number/abort``
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+This cancels an *in-progress* upload.
+
+The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
+
+    X-Tahoe-Authorization: upload-secret <base64-upload-secret>
+
+The response code:
+
+* When the upload is still in progress and therefore the abort has succeeded,
+  the response is ``OK``.
+  Future uploads can start from scratch with no pre-existing upload state stored on the server.
+* If the uploaded has already finished, the response is 405 (Method Not Allowed)
+  and no change is made.
+
+
 ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
@@ -624,7 +625,7 @@ corruption. It also includes potentially important details about the share.
 
 For example::
 
-  {"reason": u"expected hash abcd, got hash efgh"}
+  {"reason": "expected hash abcd, got hash efgh"}
 
 .. share-type, storage-index, and share-number are inferred from the URL
 
@@ -798,6 +799,7 @@ Immutable Data
      <first 16 bytes of share data>
 
      200 OK
+     { "required": [ {"begin": 16, "end": 48 } ] }
 
      PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
      Authorization: Tahoe-LAFS nurl-swissnum
@@ -806,6 +808,7 @@ Immutable Data
      <second 16 bytes of share data>
 
      200 OK
+     { "required": [ {"begin": 32, "end": 48 } ] }
 
      PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
      Authorization: Tahoe-LAFS nurl-swissnum
@@ -822,6 +825,7 @@ Immutable Data
      Range: bytes=0-47
 
      200 OK
+     Content-Range: bytes 0-47/48
      <complete 48 bytes of previously uploaded data>
 
 #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``::
@@ -905,9 +909,12 @@ otherwise it will read a byte which won't match `b""`::
 
 #. Download the contents of share number ``3``::
 
-     GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10
+     GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3
      Authorization: Tahoe-LAFS nurl-swissnum
+     Range: bytes=0-16
 
+     200 OK
+     Content-Range: bytes 0-15/16
      <complete 16 bytes of previously uploaded data>
 
 #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``::

docs/specifications/url.rst

@@ -7,6 +7,8 @@ These are not to be confused with the URI-like capabilities Tahoe-LAFS uses to r
 An attempt is also made to outline the rationale for certain choices about these URLs.
 The intended audience for this document is Tahoe-LAFS maintainers and other developers interested in interoperating with Tahoe-LAFS or these URLs.
 
+.. _furls:
+
 Background
 ----------
 
@@ -31,6 +33,8 @@ The client's use of the swissnum is what allows the server to authorize the clie
 
 .. _`swiss number`: http://wiki.erights.org/wiki/Swiss_number
 
+.. _NURLs:
+
 NURLs
 -----
 

newsfragments/3922.documentation

@@ -0,0 +1 @@
+Several minor errors in the Great Black Swamp proposed specification document have been fixed.

src/allmydata/history.py

@@ -20,7 +20,7 @@ class History(object):
     MAX_UPLOAD_STATUSES = 10
     MAX_MAPUPDATE_STATUSES = 20
     MAX_PUBLISH_STATUSES = 20
-    MAX_RETRIEVE_STATUSES = 20
+    MAX_RETRIEVE_STATUSES = 40
 
     def __init__(self, stats_provider=None):
         self.stats_provider = stats_provider

src/allmydata/storage_client.py

@@ -970,6 +970,7 @@ class HTTPNativeStorageServer(service.MultiService):
         self._connection_status = connection_status.ConnectionStatus.unstarted()
         self._version = None
         self._last_connect_time = None
+        self._connecting_deferred = None
 
     def get_permutation_seed(self):
         return self._permutation_seed
@@ -1060,20 +1061,30 @@ class HTTPNativeStorageServer(service.MultiService):
 
     def stop_connecting(self):
         self._lc.stop()
+        if self._connecting_deferred is not None:
+            self._connecting_deferred.cancel()
 
     def try_to_connect(self):
         self._connect()
 
     def _connect(self):
         result = self._istorage_server.get_version()
+
+        def remove_connecting_deferred(result):
+            self._connecting_deferred = None
+            return result
+
         # Set a short timeout since we're relying on this for server liveness.
-        result.addTimeout(5, self._reactor)
-        result.addCallbacks(
+        self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth(
+                remove_connecting_deferred).addCallbacks(
             self._got_version,
             self._failed_to_connect
         )
 
     def stopService(self):
+        if self._connecting_deferred is not None:
+            self._connecting_deferred.cancel()
+
         result = service.MultiService.stopService(self)
         if self._lc.running:
             self._lc.stop()

src/allmydata/test/common_system.py

@@ -648,7 +648,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
     def setUp(self):
         self._http_client_pools = []
-        http_client.StorageClient.start_test_mode(self._http_client_pools.append)
+        http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool)
         self.addCleanup(http_client.StorageClient.stop_test_mode)
         self.port_assigner = SameProcessStreamEndpointAssigner()
         self.port_assigner.setUp()
@@ -657,6 +657,23 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         self.sparent = service.MultiService()
         self.sparent.startService()
 
+    def _got_new_http_connection_pool(self, pool):
+        # Register the pool for shutdown later:
+        self._http_client_pools.append(pool)
+        # Disable retries:
+        pool.retryAutomatically = False
+        # Make a much more aggressive timeout for connections, we're connecting
+        # locally after all... and also make sure it's lower than the delay we
+        # add in tearDown, to prevent dirty reactor issues.
+        getConnection = pool.getConnection
+
+        def getConnectionWithTimeout(*args, **kwargs):
+            d = getConnection(*args, **kwargs)
+            d.addTimeout(1, reactor)
+            return d
+
+        pool.getConnection = getConnectionWithTimeout
+
     def close_idle_http_connections(self):
         """Close all HTTP client connections that are just hanging around."""
         return defer.gatherResults(
@@ -668,7 +685,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         d = self.sparent.stopService()
         d.addBoth(flush_but_dont_ignore)
         d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x))
-        d.addBoth(lambda x: deferLater(reactor, 0.02, lambda: x))
+        d.addBoth(lambda x: deferLater(reactor, 2, lambda: x))
         return d
 
     def getdir(self, subdir):

src/allmydata/test/test_storage_https.py

@@ -144,6 +144,10 @@ class PinningHTTPSValidation(AsyncTestCase):
             response = await self.request(url, certificate)
             self.assertEqual(await response.content(), b"YOYODYNE")
 
+        # We keep getting TLSMemoryBIOProtocol being left around, so try harder
+        # to wait for it to finish.
+        await deferLater(reactor, 0.01)
+
     @async_to_deferred
     async def test_server_certificate_has_wrong_hash(self):
         """