Merge remote-tracking branch 'origin/master' into 3922.minor-gbs-text-fixes

Makefile

@@ -224,3 +224,62 @@ src/allmydata/_version.py:
 
 .tox/create-venvs.log: tox.ini setup.py
 	tox --notest -p all | tee -a "$(@)"
+
+
+# to make a new release:
+# - create a ticket for the release in Trac
+# - ensure local copy is up-to-date
+# - create a branch like "XXXX.release" from up-to-date master
+# - in the branch, run "make release"
+# - run "make release-test"
+# - perform any other sanity-checks on the release
+# - run "make release-upload"
+# Note that several commands below hard-code "meejah"; if you are
+# someone else please adjust them.
+release:
+	@echo "Is checkout clean?"
+	git diff-files --quiet
+	git diff-index --quiet --cached HEAD --
+
+	@echo "Clean docs build area"
+	rm -rf docs/_build/
+
+	@echo "Install required build software"
+	python3 -m pip install --editable .[build]
+
+	@echo "Test README"
+	python3 setup.py check -r -s
+
+	@echo "Update NEWS"
+	python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag`
+	git add -u
+	git commit -m "update NEWS for release"
+
+# note that this always bumps the "middle" number, e.g. from 1.17.1 -> 1.18.0
+# and produces a tag into the Git repository
+	@echo "Bump version and create tag"
+	python3 misc/build_helpers/update-version.py
+
+	@echo "Build and sign wheel"
+	python3 setup.py bdist_wheel
+	gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl
+	ls dist/*`git describe | cut -b 12-`*
+
+	@echo "Build and sign source-dist"
+	python3 setup.py sdist
+	gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz
+	ls dist/*`git describe | cut -b 12-`*
+
+# basically just a bare-minimum smoke-test that it installs and runs
+release-test:
+	gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc
+	gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc
+	virtualenv testmf_venv
+	testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl
+	testmf_venv/bin/tahoe --version
+	rm -rf testmf_venv
+
+release-upload:
+	scp dist/*`git describe | cut -b 12-`* meejah@tahoe-lafs.org:/home/source/downloads
+	git push origin_push tahoe-lafs-`git describe | cut -b 12-`
+	twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc

NEWS.rst

@@ -5,6 +5,47 @@ User-Visible Changes in Tahoe-LAFS
 ==================================
 
 .. towncrier start line
+Release 1.18.0 (2022-10-02)
+'''''''''''''''''''''''''''
+
+Backwards Incompatible Changes
+------------------------------
+
+- Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. (`#3865 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3865>`_)
+- Python 3.7 or later is now required; Python 2 is no longer supported. (`#3873 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3873>`_)
+- Share corruption reports stored on disk are now always encoded in UTF-8. (`#3879 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3879>`_)
+- Record both the PID and the process creation-time:
+
+  a new kind of pidfile in `running.process` records both
+  the PID and the creation-time of the process. This facilitates
+  automatic discovery of a "stale" pidfile that points to a
+  currently-running process. If the recorded creation-time matches
+  the creation-time of the running process, then it is a still-running
+  `tahoe run` process. Otherwise, the file is stale.
+
+  The `twistd.pid` file is no longer present. (`#3926 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3926>`_)
+
+
+Features
+--------
+
+- The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification.
+
+  Some code existed to allow tests to shorten this and it's
+  conceptually possible a modified client produced mutables
+  with different key-sizes. However, the spec says that they
+  must be 2048 bits. If you happen to have a capability with
+  a key-size different from 2048 you may use 1.17.1 or earlier
+  to read the content. (`#3828 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3828>`_)
+- "make" based release automation (`#3846 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3846>`_)
+
+
+Misc/Other
+----------
+
+- `#3327 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3327>`_, `#3526 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3526>`_, `#3697 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3697>`_, `#3709 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3709>`_, `#3786 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3786>`_, `#3788 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3788>`_, `#3802 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3802>`_, `#3816 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3816>`_, `#3855 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3855>`_, `#3858 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3858>`_, `#3859 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3859>`_, `#3860 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860>`_, `#3867 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3867>`_, `#3868 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3868>`_, `#3871 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3871>`_, `#3872 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872>`_, `#3875 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3875>`_, `#3876 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3876>`_, `#3877 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3877>`_, `#3881 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3881>`_, `#3882 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3882>`_, `#3883 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3883>`_, `#3889 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3889>`_, `#3890 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3890>`_, `#3891 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3891>`_, `#3893 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3893>`_, `#3895 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3895>`_, `#3896 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3896>`_, `#3898 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3898>`_, `#3900 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3900>`_, `#3909 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3909>`_, `#3913 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3913>`_, `#3915 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3915>`_, `#3916 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3916>`_
+
+
 Release 1.17.1 (2022-01-07)
 '''''''''''''''''''''''''''
 

docs/check_running.py

@@ -0,0 +1,47 @@
+
+import psutil
+import filelock
+
+
+def can_spawn_tahoe(pidfile):
+    """
+    Determine if we can spawn a Tahoe-LAFS for the given pidfile. That
+    pidfile may be deleted if it is stale.
+
+    :param pathlib.Path pidfile: the file to check, that is the Path
+        to "running.process" in a Tahoe-LAFS configuration directory
+
+    :returns bool: True if we can spawn `tahoe run` here
+    """
+    lockpath = pidfile.parent / (pidfile.name + ".lock")
+    with filelock.FileLock(lockpath):
+        try:
+            with pidfile.open("r") as f:
+                pid, create_time = f.read().strip().split(" ", 1)
+        except FileNotFoundError:
+            return True
+
+        # somewhat interesting: we have a pidfile
+        pid = int(pid)
+        create_time = float(create_time)
+
+        try:
+            proc = psutil.Process(pid)
+            # most interesting case: there _is_ a process running at the
+            # recorded PID -- but did it just happen to get that PID, or
+            # is it the very same one that wrote the file?
+            if create_time == proc.create_time():
+                # _not_ stale! another intance is still running against
+                # this configuration
+                return False
+
+        except psutil.NoSuchProcess:
+            pass
+
+        # the file is stale
+        pidfile.unlink()
+        return True
+
+
+from pathlib import Path
+print("can spawn?", can_spawn_tahoe(Path("running.process")))

docs/proposed/http-storage-node-protocol.rst

@@ -395,8 +395,8 @@ Encoding
 General
 ~~~~~~~
 
-``GET /v1/version``
+``GET /storage/v1/version``
-!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Retrieve information about the version of the storage server.
 Information is returned as an encoded mapping.
@@ -409,14 +409,13 @@ For example::
       "tolerates-immutable-read-overrun": true,
       "delete-mutable-shares-with-zero-length-writev": true,
       "fills-holes-with-zero-bytes": true,
-      "prevents-read-past-end-of-share-data": true,
+      "prevents-read-past-end-of-share-data": true
-      "gbs-anonymous-storage-url": "pb://...#v=1"
       },
     "application-version": "1.13.0"
     }
 
-``PUT /v1/lease/:storage_index``
+``PUT /storage/v1/lease/:storage_index``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Either renew or create a new lease on the bucket addressed by ``storage_index``.
 
@@ -468,8 +467,8 @@ Immutable
 Writing
 ~~~~~~~
 
-``POST /v1/immutable/:storage_index``
+``POST /storage/v1/immutable/:storage_index``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Initialize an immutable storage index with some buckets.
 The buckets may have share data written to them once.
@@ -504,7 +503,7 @@ Handling repeat calls:
 Discussion
 ``````````
 
-We considered making this ``POST /v1/immutable`` instead.
+We considered making this ``POST /storage/v1/immutable`` instead.
 The motivation was to keep *storage index* out of the request URL.
 Request URLs have an elevated chance of being logged by something.
 We were concerned that having the *storage index* logged may increase some risks.
@@ -539,8 +538,8 @@ Rejected designs for upload secrets:
   it must contain randomness.
   Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better.
 
-``PATCH /v1/immutable/:storage_index/:share_number``
+``PATCH /storage/v1/immutable/:storage_index/:share_number``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Write data for the indicated share.
 The share number must belong to the storage index.
@@ -598,8 +597,9 @@ From RFC 7231::
    PATCH method defined in [RFC5789]).
 
 
-``PUT /v1/immutable/:storage_index/:share_number/abort``
+
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+``PUT /storage/v1/immutable/:storage_index/:share_number/abort``
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 This cancels an *in-progress* upload.
 
@@ -616,8 +616,8 @@ The response code:
   and no change is made.
 
 
-``POST /v1/immutable/:storage_index/:share_number/corrupt``
+``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Advise the server the data read from the indicated share was corrupt. The
 request body includes an human-meaningful text string with details about the
@@ -635,8 +635,8 @@ couldn't be found.
 Reading
 ~~~~~~~
 
-``GET /v1/immutable/:storage_index/shares``
+``GET /storage/v1/immutable/:storage_index/shares``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Retrieve a list (semantically, a set) indicating all shares available for the
 indicated storage index. For example::
@@ -645,8 +645,8 @@ indicated storage index. For example::
 
 An unknown storage index results in an empty list.
 
-``GET /v1/immutable/:storage_index/:share_number``
+``GET /storage/v1/immutable/:storage_index/:share_number``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Read a contiguous sequence of bytes from one share in one bucket.
 The response body is the raw share data (i.e., ``application/octet-stream``).
@@ -686,8 +686,8 @@ Mutable
 Writing
 ~~~~~~~
 
-``POST /v1/mutable/:storage_index/read-test-write``
+``POST /storage/v1/mutable/:storage_index/read-test-write``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 General purpose read-test-and-write operation for mutable storage indexes.
 A mutable storage index is also called a "slot"
@@ -742,18 +742,18 @@ As a result, if there is no data at all, an empty bytestring is returned no matt
 Reading
 ~~~~~~~
 
-``GET /v1/mutable/:storage_index/shares``
+``GET /storage/v1/mutable/:storage_index/shares``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Retrieve a set indicating all shares available for the indicated storage index.
 For example (this is shown as list, since it will be list for JSON, but will be set for CBOR)::
 
   [1, 5]
 
-``GET /v1/mutable/:storage_index/:share_number``
+``GET /storage/v1/mutable/:storage_index/:share_number``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Read data from the indicated mutable shares, just like ``GET /v1/immutable/:storage_index``
+Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``
 
 The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content).
 Interpretation and response behavior is as specified in RFC 7233 § 4.1.
@@ -765,8 +765,8 @@ The resulting ``Content-Range`` header will be consistent with the returned data
 If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used.
 
 
-``POST /v1/mutable/:storage_index/:share_number/corrupt``
+``POST /storage/v1/mutable/:storage_index/:share_number/corrupt``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Advise the server the data read from the indicated share was corrupt.
 Just like the immutable version.
@@ -779,7 +779,7 @@ Immutable Data
 
 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded::
 
-     POST /v1/immutable/AAAAAAAAAAAAAAAA
+     POST /storage/v1/immutable/AAAAAAAAAAAAAAAA
      Authorization: Tahoe-LAFS nurl-swissnum
      X-Tahoe-Authorization: lease-renew-secret efgh
      X-Tahoe-Authorization: lease-cancel-secret jjkl
@@ -792,7 +792,7 @@ Immutable Data
 
 #. Upload the content for immutable share ``7``::
 
-     PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7
+     PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
      Authorization: Tahoe-LAFS nurl-swissnum
      Content-Range: bytes 0-15/48
      X-Tahoe-Authorization: upload-secret xyzf
@@ -801,7 +801,7 @@ Immutable Data
      200 OK
      { "required": [ {"begin": 16, "end": 48 } ] }
 
-     PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7
+     PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
      Authorization: Tahoe-LAFS nurl-swissnum
      Content-Range: bytes 16-31/48
      X-Tahoe-Authorization: upload-secret xyzf
@@ -810,7 +810,7 @@ Immutable Data
      200 OK
      { "required": [ {"begin": 32, "end": 48 } ] }
 
-     PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7
+     PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
      Authorization: Tahoe-LAFS nurl-swissnum
      Content-Range: bytes 32-47/48
      X-Tahoe-Authorization: upload-secret xyzf
@@ -820,7 +820,7 @@ Immutable Data
 
 #. Download the content of the previously uploaded immutable share ``7``::
 
-     GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7
+     GET /storage/v1/immutable/AAAAAAAAAAAAAAAA?share=7
      Authorization: Tahoe-LAFS nurl-swissnum
      Range: bytes=0-47
 
@@ -830,7 +830,7 @@ Immutable Data
 
 #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``::
 
-     PUT /v1/lease/AAAAAAAAAAAAAAAA
+     PUT /storage/v1/lease/AAAAAAAAAAAAAAAA
      Authorization: Tahoe-LAFS nurl-swissnum
      X-Tahoe-Authorization: lease-cancel-secret jjkl
      X-Tahoe-Authorization: lease-renew-secret efgh
@@ -845,7 +845,7 @@ The special test vector of size 1 but empty bytes will only pass
 if there is no existing share,
 otherwise it will read a byte which won't match `b""`::
 
-     POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
+     POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
      Authorization: Tahoe-LAFS nurl-swissnum
      X-Tahoe-Authorization: write-enabler abcd
      X-Tahoe-Authorization: lease-cancel-secret efgh
@@ -877,7 +877,7 @@ otherwise it will read a byte which won't match `b""`::
 
 #. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail)::
 
-     POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
+     POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
      Authorization: Tahoe-LAFS nurl-swissnum
      X-Tahoe-Authorization: write-enabler abcd
      X-Tahoe-Authorization: lease-cancel-secret efgh
@@ -909,7 +909,7 @@ otherwise it will read a byte which won't match `b""`::
 
 #. Download the contents of share number ``3``::
 
-     GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3
+     GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3
      Authorization: Tahoe-LAFS nurl-swissnum
      Range: bytes=0-16
 
@@ -919,7 +919,7 @@ otherwise it will read a byte which won't match `b""`::
 
 #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``::
 
-     PUT /v1/lease/BBBBBBBBBBBBBBBB
+     PUT /storage/v1/lease/BBBBBBBBBBBBBBBB
      Authorization: Tahoe-LAFS nurl-swissnum
      X-Tahoe-Authorization: lease-cancel-secret efgh
      X-Tahoe-Authorization: lease-renew-secret ijkl

docs/running.rst

@@ -124,6 +124,35 @@ Tahoe-LAFS.
 .. _magic wormhole: https://magic-wormhole.io/
 
 
+Multiple Instances
+------------------
+
+Running multiple instances against the same configuration directory isn't supported.
+This will lead to undefined behavior and could corrupt the configuration or state.
+
+We attempt to avoid this situation with a "pidfile"-style file in the config directory called ``running.process``.
+There may be a parallel file called ``running.process.lock`` in existence.
+
+The ``.lock`` file exists to make sure only one process modifies ``running.process`` at once.
+The lock file is managed by the `lockfile <https://pypi.org/project/lockfile/>`_ library.
+If you wish to make use of ``running.process`` for any reason you should also lock it and follow the semantics of lockfile.
+
+If ``running.process`` exists then it contains the PID and the creation-time of the process.
+When no such file exists, there is no other process running on this configuration.
+If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config.
+To tell the difference, determine if the PID in the file exists currently.
+If it does, check the creation-time of the process versus the one in the file.
+If these match, there is another process currently running and using this config.
+Otherwise, the file is stale -- it should be removed before starting Tahoe-LAFS.
+
+Some example Python code to check the above situations:
+
+.. literalinclude:: check_running.py
+
+
+
+
+
 A note about small grids
 ------------------------
 

docs/specifications/url.rst

@@ -51,27 +51,27 @@ This can be considered to expand to "**N**\ ew URLs" or "Authe\ **N**\ ticating
 The anticipated use for a **NURL** will still be to establish a TLS connection to a peer.
 The protocol run over that TLS connection could be Foolscap though it is more likely to be an HTTP-based protocol (such as GBS).
 
+Unlike fURLs, only a single net-loc is included, for consistency with other forms of URLs.
+As a result, multiple NURLs may be available for a single server.
+
 Syntax
 ------
 
 The EBNF for a NURL is as follows::
 
-  nurl         = scheme, hash, "@", net-loc-list, "/", swiss-number, [ version1 ]
+  nurl         = tcp-nurl | tor-nurl | i2p-nurl
-
+  tcp-nurl     = "pb://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ]
-  scheme       = "pb://"
+  tor-nurl     = "pb+tor://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ]
+  i2p-nurl     = "pb+i2p://", hash, "@", i2p-loc, "/", swiss-number, [ version1 ]
 
   hash         = unreserved
 
-  net-loc-list = net-loc, [ { ",", net-loc } ]
+  tcp-loc      = hostname, [ ":" port ]
-  net-loc      = tcp-loc | tor-loc | i2p-loc
-
-  tcp-loc      = [ "tcp:" ], hostname, [ ":" port ]
-  tor-loc      = "tor:", hostname, [ ":" port ]
-  i2p-loc      = "i2p:", i2p-addr, [ ":" port ]
-
-  i2p-addr     = { unreserved }, ".i2p"
   hostname     = domain | IPv4address | IPv6address
 
+  i2p-loc      = i2p-addr, [ ":" port ]
+  i2p-addr     = { unreserved }, ".i2p"
+
   swiss-number = segment
 
   version1     = "#v=1"
@@ -91,11 +91,13 @@ These differences are separated into distinct versions.
 Version 0
 ---------
 
-A Foolscap fURL is considered the canonical definition of a version 0 NURL.
+In theory, a Foolscap fURL with a single netloc is considered the canonical definition of a version 0 NURL.
 Notably,
 the hash component is defined as the base32-encoded SHA1 hash of the DER form of an x509v3 certificate.
 A version 0 NURL is identified by the absence of the ``v=1`` fragment.
 
+In practice, real world fURLs may have more than one netloc, so lack of version fragment will likely just involve dispatching the fURL to a different parser.
+
 Examples
 ~~~~~~~~
 
@@ -107,11 +109,8 @@ Version 1
 
 The hash component of a version 1 NURL differs in three ways from the prior version.
 
-1. The hash function used is SHA3-224 instead of SHA1.
+1. The hash function used is SHA-256, to match RFC 7469.
-   The security of SHA1 `continues to be eroded`_.
+   The security of SHA1 `continues to be eroded`_; Latacora `SHA-2`_.
-   Contrariwise SHA3 is currently the most recent addition to the SHA family by NIST.
-   The 224 bit instance is chosen to keep the output short and because it offers greater collision resistance than SHA1 was thought to offer even at its inception
-   (prior to security research showing actual collision resistance is lower).
 2. The hash is computed over the certificate's SPKI instead of the whole certificate.
    This allows certificate re-generation so long as the public key remains the same.
    This is useful to allow contact information to be updated or extension of validity period.
@@ -126,7 +125,7 @@ The hash component of a version 1 NURL differs in three ways from the prior vers
    *all* certificate fields should be considered within the context of the relationship identified by the SPKI hash.
 
 3. The hash is encoded using urlsafe-base64 (without padding) instead of base32.
-   This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 224 bit hash.
+   This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 256 bit hash.
 
 A version 1 NURL is identified by the presence of the ``v=1`` fragment.
 Though the length of the hash string (38 bytes) could also be used to differentiate it from a version 0 NURL,
@@ -144,7 +143,8 @@ Examples
 * ``pb://azEu8vlRpnEeYm0DySQDeNY3Z2iJXHC_bsbaAw@localhost:47877/64i4aokv4ej#v=1``
 
 .. _`continues to be eroded`: https://en.wikipedia.org/wiki/SHA-1#Cryptanalysis_and_validation
-.. _`explored by the web community`: https://www.imperialviolet.org/2011/05/04/pinning.html
+.. _`SHA-2`: https://latacora.micro.blog/2018/04/03/cryptographic-right-answers.html
+.. _`explored by the web community`: https://www.rfc-editor.org/rfc/rfc7469
 .. _Foolscap: https://github.com/warner/foolscap
 
 .. [1] ``foolscap.furl.decode_furl`` is taken as the canonical definition of the syntax of a fURL.

docs/stats.rst

@@ -264,3 +264,18 @@ the "tahoe-conf" file for notes about configuration and installing these
 plugins into a Munin environment.
 
 .. _Munin: http://munin-monitoring.org/
+
+
+Scraping Stats Values in OpenMetrics Format
+===========================================
+
+Time Series DataBase (TSDB) software like Prometheus_ and VictoriaMetrics_ can
+parse statistics from the e.g. http://localhost:3456/statistics?t=openmetrics
+URL in OpenMetrics_ format. Software like Grafana_ can then be used to graph
+and alert on these numbers. You can find a pre-configured dashboard for
+Grafana at https://grafana.com/grafana/dashboards/16894-tahoe-lafs/.
+
+.. _OpenMetrics: https://openmetrics.io/
+.. _Prometheus: https://prometheus.io/
+.. _VictoriaMetrics: https://victoriametrics.com/
+.. _Grafana: https://grafana.com/

integration/test_i2p.py

@@ -55,7 +55,7 @@ def i2p_network(reactor, temp_dir, request):
         proto,
         which("docker"),
         (
-            "docker", "run", "-p", "7656:7656", "purplei2p/i2pd",
+            "docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.43.0",
             # Bad URL for reseeds, so it can't talk to other routers.
             "--reseed.urls", "http://localhost:1/",
         ),
@@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request):
 
     def cleanup():
         try:
-            proto.transport.signalProcess("KILL")
+            proto.transport.signalProcess("INT")
             util.block_with_timeout(proto.exited, reactor)
         except ProcessExitedAlready:
             pass

misc/build_helpers/update-version.py

@@ -0,0 +1,95 @@
+#
+# this updates the (tagged) version of the software
+#
+# Any "options" are hard-coded in here (e.g. the GnuPG key to use)
+#
+
+author = "meejah <meejah@meejah.ca>"
+
+
+import sys
+import time
+from datetime import datetime
+from packaging.version import Version
+
+from dulwich.repo import Repo
+from dulwich.porcelain import (
+    tag_list,
+    tag_create,
+    status,
+)
+
+from twisted.internet.task import (
+    react,
+)
+from twisted.internet.defer import (
+    ensureDeferred,
+)
+
+
+def existing_tags(git):
+    versions = sorted(
+        Version(v.decode("utf8").lstrip("tahoe-lafs-"))
+        for v in tag_list(git)
+        if v.startswith(b"tahoe-lafs-")
+    )
+    return versions
+
+
+def create_new_version(git):
+    versions = existing_tags(git)
+    biggest = versions[-1]
+
+    return Version(
+        "{}.{}.{}".format(
+            biggest.major,
+            biggest.minor + 1,
+            0,
+        )
+    )
+
+
+async def main(reactor):
+    git = Repo(".")
+
+    st = status(git)
+    if any(st.staged.values()) or st.unstaged:
+        print("unclean checkout; aborting")
+        raise SystemExit(1)
+
+    v = create_new_version(git)
+    if "--no-tag" in sys.argv:
+        print(v)
+        return
+
+    print("Existing tags: {}".format("\n".join(str(x) for x in existing_tags(git))))
+    print("New tag will be {}".format(v))
+
+    # the "tag time" is seconds from the epoch .. we quantize these to
+    # the start of the day in question, in UTC.
+    now = datetime.now()
+    s = now.utctimetuple()
+    ts = int(
+        time.mktime(
+            time.struct_time((s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0))
+        )
+    )
+    tag_create(
+        repo=git,
+        tag="tahoe-lafs-{}".format(str(v)).encode("utf8"),
+        author=author.encode("utf8"),
+        message="Release {}".format(v).encode("utf8"),
+        annotated=True,
+        objectish=b"HEAD",
+        sign=author.encode("utf8"),
+        tag_time=ts,
+        tag_timezone=0,
+    )
+
+    print("Tag created locally, it is not pushed")
+    print("To push it run something like:")
+    print("   git push origin {}".format(v))
+
+
+if __name__ == "__main__":
+    react(lambda r: ensureDeferred(main(r)))

newsfragments/3526.minor

@@ -1 +0,0 @@
-

newsfragments/3697.minor

@@ -1 +0,0 @@
-Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only).

newsfragments/3828.feature

@@ -1,8 +0,0 @@
-The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification.
-
-Some code existed to allow tests to shorten this and it's
-conceptually possible a modified client produced mutables
-with different key-sizes. However, the spec says that they
-must be 2048 bits. If you happen to have a capability with
-a key-size different from 2048 you may use 1.17.1 or earlier
-to read the content.

newsfragments/3865.incompat

@@ -1 +0,0 @@
-Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates.

newsfragments/3873.incompat

@@ -1 +0,0 @@
-Python 3.7 or later is now required; Python 2 is no longer supported.

newsfragments/3879.incompat

@@ -1 +0,0 @@
-Share corruption reports stored on disk are now always encoded in UTF-8.

newsfragments/3902.feature

@@ -0,0 +1 @@
+The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well.

newsfragments/3938.bugfix

@@ -0,0 +1 @@
+Work with (and require) newer versions of pycddl.

nix/sources.json

@@ -53,10 +53,10 @@
         "homepage": "",
         "owner": "DavHau",
         "repo": "pypi-deps-db",
-        "rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a",
+        "rev": "5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6",
-        "sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x",
+        "sha256": "0pc6mj7rzvmhh303rvj5wf4hrksm4h2rf4fsvqs0ljjdmgxrqm3f",
         "type": "tarball",
-        "url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz",
+        "url": "https://github.com/DavHau/pypi-deps-db/archive/5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

relnotes.txt

@@ -1,6 +1,6 @@
-ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1
+ANNOUNCING Tahoe, the Least-Authority File Store, v1.18.0
 
-The Tahoe-LAFS team is pleased to announce version 1.17.1 of
+The Tahoe-LAFS team is pleased to announce version 1.18.0 of
 Tahoe-LAFS, an extremely reliable decentralized storage
 system. Get it with "pip install tahoe-lafs", or download a
 tarball here:
@@ -15,10 +15,12 @@ unique security and fault-tolerance properties:
 
   https://tahoe-lafs.readthedocs.org/en/latest/about.html
 
-The previous stable release of Tahoe-LAFS was v1.17.0, released on
+The previous stable release of Tahoe-LAFS was v1.17.1, released on
-December 6, 2021.
+January 7, 2022.
 
-This release fixes two Python3-releated regressions and 4 minor bugs.
+This release drops support for Python 2 and for Python 3.6 and earlier.
+twistd.pid is no longer used (in favour of one with pid + process creation time).
+A collection of minor bugs and issues were also fixed.
 
 Please see ``NEWS.rst`` [1] for a complete list of changes.
 
@@ -132,24 +134,23 @@ Of Fame" [13].
 
 ACKNOWLEDGEMENTS
 
-This is the nineteenth release of Tahoe-LAFS to be created
+This is the twentieth release of Tahoe-LAFS to be created solely as a
-solely as a labor of love by volunteers. Thank you very much
+labor of love by volunteers. Thank you very much to the team of
-to the team of "hackers in the public interest" who make
+"hackers in the public interest" who make Tahoe-LAFS possible.
-Tahoe-LAFS possible.
 
 meejah
 on behalf of the Tahoe-LAFS team
 
-January 7, 2022
+October 1, 2022
 Planet Earth
 
 
-[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst
+[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/NEWS.rst
 [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst
 [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects
-[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL
+[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.GPL
-[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst
+[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.TGPPL.rst
-[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html
+[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.18.0/INSTALL.html
 [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev
 [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap
 [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS

setup.py

@@ -137,7 +137,11 @@ install_requires = [
     "werkzeug != 2.2.0",
     "treq",
     "cbor2",
-    "pycddl",
+    "pycddl >= 0.2",
+
+    # for pid-file support
+    "psutil",
+    "filelock",
 ]
 
 setup_requires = [
@@ -376,8 +380,15 @@ setup(name="tahoe-lafs", # also set in __init__.py
           # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some
           # discussion.
           ':sys_platform=="win32"': ["pywin32 != 226"],
+          "build": [
+              "dulwich",
+              "gpg",
+          ],
           "test": [
               "flake8",
+              # On Python 3.7, importlib_metadata v5 breaks flake8.
+              # https://github.com/python/importlib_metadata/issues/407
+              "importlib_metadata<5; python_version < '3.8'",
               # Pin a specific pyflakes so we don't have different folks
               # disagreeing on what is or is not a lint issue.  We can bump
               # this version from time to time, but we will do it

src/allmydata/client.py

@@ -1,17 +1,9 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import
+from __future__ import annotations
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
-    # Don't use future str to prevent leaking future's newbytes into foolscap, which they break.
-    from past.builtins import unicode as str
 
+from typing import Optional
 import os, stat, time, weakref
 from base64 import urlsafe_b64encode
 from functools import partial
@@ -591,6 +583,10 @@ def anonymous_storage_enabled(config):
 
 @implementer(IStatsProducer)
 class _Client(node.Node, pollmixin.PollMixin):
+    """
+    This class should be refactored; see
+    https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931
+    """
 
     STOREDIR = 'storage'
     NODETYPE = "client"
@@ -658,6 +654,14 @@ class _Client(node.Node, pollmixin.PollMixin):
         if webport:
             self.init_web(webport) # strports string
 
+        # TODO this may be the wrong location for now? but as temporary measure
+        # it allows us to get NURLs for testing in test_istorageserver.py. This
+        # will eventually get fixed one way or another in
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901. See also
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 for the bigger
+        # picture issue.
+        self.storage_nurls : Optional[set] = None
+
     def init_stats_provider(self):
         self.stats_provider = StatsProvider(self)
         self.stats_provider.setServiceParent(self)
@@ -818,6 +822,10 @@ class _Client(node.Node, pollmixin.PollMixin):
         if anonymous_storage_enabled(self.config):
             furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding())
             furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file)
+            (_, _, swissnum) = decode_furl(furl)
+            self.storage_nurls = self.tub.negotiationClass.add_storage_server(
+                ss, swissnum.encode("ascii")
+            )
             announcement["anonymous-storage-FURL"] = furl
 
         enabled_storage_servers = self._enable_storage_servers(

src/allmydata/immutable/encode.py

@@ -694,3 +694,24 @@ class Encoder(object):
         return self.uri_extension_data
     def get_uri_extension_hash(self):
         return self.uri_extension_hash
+
+    def get_uri_extension_size(self):
+        """
+        Calculate the size of the URI extension that gets written at the end of
+        immutables.
+
+        This may be done earlier than actual encoding, so e.g. we might not
+        know the crypttext hashes, but that's fine for our purposes since we
+        only care about the length.
+        """
+        params = self.uri_extension_data.copy()
+        params["crypttext_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE
+        params["crypttext_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE
+        params["share_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE
+        assert params.keys() == {
+            "codec_name", "codec_params", "size", "segment_size", "num_segments",
+            "needed_shares", "total_shares", "tail_codec_params",
+            "crypttext_hash", "crypttext_root_hash", "share_root_hash"
+        }, params.keys()
+        uri_extension = uri.pack_extension(params)
+        return len(uri_extension)

src/allmydata/immutable/layout.py

@@ -19,6 +19,7 @@ from allmydata.util import mathutil, observer, pipeline, log
 from allmydata.util.assertutil import precondition
 from allmydata.storage.server import si_b2a
 
+
 class LayoutInvalid(Exception):
     """ There is something wrong with these bytes so they can't be
     interpreted as the kind of immutable file that I know how to download."""
@@ -90,7 +91,7 @@ FORCE_V2 = False # set briefly by unit tests to make small-sized V2 shares
 
 def make_write_bucket_proxy(rref, server,
                             data_size, block_size, num_segments,
-                            num_share_hashes, uri_extension_size_max):
+                            num_share_hashes, uri_extension_size):
     # Use layout v1 for small files, so they'll be readable by older versions
     # (<tahoe-1.3.0). Use layout v2 for large files; they'll only be readable
     # by tahoe-1.3.0 or later.
@@ -99,11 +100,11 @@ def make_write_bucket_proxy(rref, server,
             raise FileTooLargeError
         wbp = WriteBucketProxy(rref, server,
                                data_size, block_size, num_segments,
-                               num_share_hashes, uri_extension_size_max)
+                               num_share_hashes, uri_extension_size)
     except FileTooLargeError:
         wbp = WriteBucketProxy_v2(rref, server,
                                   data_size, block_size, num_segments,
-                                  num_share_hashes, uri_extension_size_max)
+                                  num_share_hashes, uri_extension_size)
     return wbp
 
 @implementer(IStorageBucketWriter)
@@ -112,20 +113,20 @@ class WriteBucketProxy(object):
     fieldstruct = ">L"
 
     def __init__(self, rref, server, data_size, block_size, num_segments,
-                 num_share_hashes, uri_extension_size_max, pipeline_size=50000):
+                 num_share_hashes, uri_extension_size, pipeline_size=50000):
         self._rref = rref
         self._server = server
         self._data_size = data_size
         self._block_size = block_size
         self._num_segments = num_segments
+        self._written_bytes = 0
 
         effective_segments = mathutil.next_power_of_k(num_segments,2)
         self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE
         # how many share hashes are included in each share? This will be
         # about ln2(num_shares).
         self._share_hashtree_size = num_share_hashes * (2+HASH_SIZE)
-        # we commit to not sending a uri extension larger than this
+        self._uri_extension_size = uri_extension_size
-        self._uri_extension_size_max = uri_extension_size_max
 
         self._create_offsets(block_size, data_size)
 
@@ -137,7 +138,7 @@ class WriteBucketProxy(object):
 
     def get_allocated_size(self):
         return (self._offsets['uri_extension'] + self.fieldsize +
-                self._uri_extension_size_max)
+                self._uri_extension_size)
 
     def _create_offsets(self, block_size, data_size):
         if block_size >= 2**32 or data_size >= 2**32:
@@ -195,6 +196,14 @@ class WriteBucketProxy(object):
         return self._write(offset, data)
 
     def put_crypttext_hashes(self, hashes):
+        # plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and
+        # so is not explicitly written, but we need to write everything, so
+        # fill it in with nulls.
+        d = self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size)
+        d.addCallback(lambda _: self._really_put_crypttext_hashes(hashes))
+        return d
+
+    def _really_put_crypttext_hashes(self, hashes):
         offset = self._offsets['crypttext_hash_tree']
         assert isinstance(hashes, list)
         data = b"".join(hashes)
@@ -233,8 +242,7 @@ class WriteBucketProxy(object):
     def put_uri_extension(self, data):
         offset = self._offsets['uri_extension']
         assert isinstance(data, bytes)
-        precondition(len(data) <= self._uri_extension_size_max,
+        precondition(len(data) == self._uri_extension_size)
-                     len(data), self._uri_extension_size_max)
         length = struct.pack(self.fieldstruct, len(data))
         return self._write(offset, length+data)
 
@@ -244,11 +252,12 @@ class WriteBucketProxy(object):
         # would reduce the foolscap CPU overhead per share, but wouldn't
         # reduce the number of round trips, so it might not be worth the
         # effort.
-
+        self._written_bytes += len(data)
         return self._pipeline.add(len(data),
                                   self._rref.callRemote, "write", offset, data)
 
     def close(self):
+        assert self._written_bytes == self.get_allocated_size(), f"{self._written_bytes} != {self.get_allocated_size()}"
         d = self._pipeline.add(0, self._rref.callRemote, "close")
         d.addCallback(lambda ign: self._pipeline.flush())
         return d
@@ -303,8 +312,6 @@ class WriteBucketProxy_v2(WriteBucketProxy):
 @implementer(IStorageBucketReader)
 class ReadBucketProxy(object):
 
-    MAX_UEB_SIZE = 2000 # actual size is closer to 419, but varies by a few bytes
-
     def __init__(self, rref, server, storage_index):
         self._rref = rref
         self._server = server
@@ -332,11 +339,6 @@ class ReadBucketProxy(object):
         # TODO: for small shares, read the whole bucket in _start()
         d = self._fetch_header()
         d.addCallback(self._parse_offsets)
-        # XXX The following two callbacks implement a slightly faster/nicer
-        # way to get the ueb and sharehashtree, but it requires that the
-        # storage server be >= v1.3.0.
-        # d.addCallback(self._fetch_sharehashtree_and_ueb)
-        # d.addCallback(self._parse_sharehashtree_and_ueb)
         def _fail_waiters(f):
             self._ready.fire(f)
         def _notify_waiters(result):
@@ -381,29 +383,6 @@ class ReadBucketProxy(object):
             self._offsets[field] = offset
         return self._offsets
 
-    def _fetch_sharehashtree_and_ueb(self, offsets):
-        sharehashtree_size = offsets['uri_extension'] - offsets['share_hashes']
-        return self._read(offsets['share_hashes'],
-                          self.MAX_UEB_SIZE+sharehashtree_size)
-
-    def _parse_sharehashtree_and_ueb(self, data):
-        sharehashtree_size = self._offsets['uri_extension'] - self._offsets['share_hashes']
-        if len(data) < sharehashtree_size:
-            raise LayoutInvalid("share hash tree truncated -- should have at least %d bytes -- not %d" % (sharehashtree_size, len(data)))
-        if sharehashtree_size % (2+HASH_SIZE) != 0:
-            raise LayoutInvalid("share hash tree malformed -- should have an even multiple of %d bytes -- not %d" % (2+HASH_SIZE, sharehashtree_size))
-        self._share_hashes = []
-        for i in range(0, sharehashtree_size, 2+HASH_SIZE):
-            hashnum = struct.unpack(">H", data[i:i+2])[0]
-            hashvalue = data[i+2:i+2+HASH_SIZE]
-            self._share_hashes.append( (hashnum, hashvalue) )
-
-        i = self._offsets['uri_extension']-self._offsets['share_hashes']
-        if len(data) < i+self._fieldsize:
-            raise LayoutInvalid("not enough bytes to encode URI length -- should be at least %d bytes long, not %d " % (i+self._fieldsize, len(data),))
-        length = struct.unpack(self._fieldstruct, data[i:i+self._fieldsize])[0]
-        self._ueb_data = data[i+self._fieldsize:i+self._fieldsize+length]
-
     def _get_block_data(self, unused, blocknum, blocksize, thisblocksize):
         offset = self._offsets['data'] + blocknum * blocksize
         return self._read(offset, thisblocksize)
@@ -446,20 +425,18 @@ class ReadBucketProxy(object):
         else:
             return defer.succeed([])
 
-    def _get_share_hashes(self, unused=None):
-        if hasattr(self, '_share_hashes'):
-            return self._share_hashes
-        return self._get_share_hashes_the_old_way()
-
     def get_share_hashes(self):
         d = self._start_if_needed()
         d.addCallback(self._get_share_hashes)
         return d
 
-    def _get_share_hashes_the_old_way(self):
+    def _get_share_hashes(self, _ignore):
         """ Tahoe storage servers < v1.3.0 would return an error if you tried
         to read past the end of the share, so we need to use the offset and
-        read just that much."""
+        read just that much.
+
+        HTTP-based storage protocol also doesn't like reading past the end.
+        """
         offset = self._offsets['share_hashes']
         size = self._offsets['uri_extension'] - offset
         if size % (2+HASH_SIZE) != 0:
@@ -477,32 +454,29 @@ class ReadBucketProxy(object):
         d.addCallback(_unpack_share_hashes)
         return d
 
-    def _get_uri_extension_the_old_way(self, unused=None):
+    def _get_uri_extension(self, unused=None):
         """ Tahoe storage servers < v1.3.0 would return an error if you tried
         to read past the end of the share, so we need to fetch the UEB size
-        and then read just that much."""
+        and then read just that much.
+
+        HTTP-based storage protocol also doesn't like reading past the end.
+        """
         offset = self._offsets['uri_extension']
         d = self._read(offset, self._fieldsize)
         def _got_length(data):
             if len(data) != self._fieldsize:
                 raise LayoutInvalid("not enough bytes to encode URI length -- should be %d bytes long, not %d " % (self._fieldsize, len(data),))
             length = struct.unpack(self._fieldstruct, data)[0]
-            if length >= 2**31:
+            if length >= 2000:
-                # URI extension blocks are around 419 bytes long, so this
+                # URI extension blocks are around 419 bytes long; in previous
-                # must be corrupted. Anyway, the foolscap interface schema
+                # versions of the code 1000 was used as a default catchall. So
-                # for "read" will not allow >= 2**31 bytes length.
+                # 2000 or more must be corrupted.
                 raise RidiculouslyLargeURIExtensionBlock(length)
 
             return self._read(offset+self._fieldsize, length)
         d.addCallback(_got_length)
         return d
 
-    def _get_uri_extension(self, unused=None):
-        if hasattr(self, '_ueb_data'):
-            return self._ueb_data
-        else:
-            return self._get_uri_extension_the_old_way()
-
     def get_uri_extension(self):
         d = self._start_if_needed()
         d.addCallback(self._get_uri_extension)

src/allmydata/immutable/upload.py

@@ -242,31 +242,26 @@ class UploadResults(object):
     def get_verifycapstr(self):
         return self._verifycapstr
 
-# our current uri_extension is 846 bytes for small files, a few bytes
-# more for larger ones (since the filesize is encoded in decimal in a
-# few places). Ask for a little bit more just in case we need it. If
-# the extension changes size, we can change EXTENSION_SIZE to
-# allocate a more accurate amount of space.
-EXTENSION_SIZE = 1000
-# TODO: actual extensions are closer to 419 bytes, so we can probably lower
-# this.
 
 def pretty_print_shnum_to_servers(s):
     return ', '.join([ "sh%s: %s" % (k, '+'.join([idlib.shortnodeid_b2a(x) for x in v])) for k, v in s.items() ])
 
+
 class ServerTracker(object):
     def __init__(self, server,
                  sharesize, blocksize, num_segments, num_share_hashes,
                  storage_index,
-                 bucket_renewal_secret, bucket_cancel_secret):
+                 bucket_renewal_secret, bucket_cancel_secret,
+                 uri_extension_size):
         self._server = server
         self.buckets = {} # k: shareid, v: IRemoteBucketWriter
         self.sharesize = sharesize
+        self.uri_extension_size = uri_extension_size
 
         wbp = layout.make_write_bucket_proxy(None, None, sharesize,
                                              blocksize, num_segments,
                                              num_share_hashes,
-                                             EXTENSION_SIZE)
+                                             uri_extension_size)
         self.wbp_class = wbp.__class__ # to create more of them
         self.allocated_size = wbp.get_allocated_size()
         self.blocksize = blocksize
@@ -314,7 +309,7 @@ class ServerTracker(object):
                                 self.blocksize,
                                 self.num_segments,
                                 self.num_share_hashes,
-                                EXTENSION_SIZE)
+                                self.uri_extension_size)
             b[sharenum] = bp
         self.buckets.update(b)
         return (alreadygot, set(b.keys()))
@@ -487,7 +482,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
     def get_shareholders(self, storage_broker, secret_holder,
                          storage_index, share_size, block_size,
                          num_segments, total_shares, needed_shares,
-                         min_happiness):
+                         min_happiness, uri_extension_size):
         """
         @return: (upload_trackers, already_serverids), where upload_trackers
                  is a set of ServerTracker instances that have agreed to hold
@@ -529,7 +524,8 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
         # figure out how much space to ask for
         wbp = layout.make_write_bucket_proxy(None, None,
                                              share_size, 0, num_segments,
-                                             num_share_hashes, EXTENSION_SIZE)
+                                             num_share_hashes,
+                                             uri_extension_size)
         allocated_size = wbp.get_allocated_size()
 
         # decide upon the renewal/cancel secrets, to include them in the
@@ -554,7 +550,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
         def _create_server_tracker(server, renew, cancel):
             return ServerTracker(
                 server, share_size, block_size, num_segments, num_share_hashes,
-                storage_index, renew, cancel,
+                storage_index, renew, cancel, uri_extension_size
             )
 
         readonly_trackers, write_trackers = self._create_trackers(
@@ -1326,7 +1322,8 @@ class CHKUploader(object):
         d = server_selector.get_shareholders(storage_broker, secret_holder,
                                              storage_index,
                                              share_size, block_size,
-                                             num_segments, n, k, desired)
+                                             num_segments, n, k, desired,
+                                             encoder.get_uri_extension_size())
         def _done(res):
             self._server_selection_elapsed = time.time() - server_selection_started
             return res

src/allmydata/node.py

@@ -55,6 +55,8 @@ from allmydata.util.yamlutil import (
 from . import (
     __full_version__,
 )
+from .protocol_switch import create_tub_with_https_support
+
 
 def _common_valid_config():
     return configutil.ValidConfiguration({
@@ -706,7 +708,10 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han
     :param dict tub_options: every key-value pair in here will be set in
         the new Tub via `Tub.setOption`
     """
-    tub = Tub(**kwargs)
+    # We listen simulataneously for both Foolscap and HTTPS on the same port,
+    # so we have to create a special Foolscap Tub for that to work:
+    tub = create_tub_with_https_support(**kwargs)
+
     for (name, value) in list(tub_options.items()):
         tub.setOption(name, value)
     handlers = default_connection_handlers.copy()

src/allmydata/protocol_switch.py

@@ -0,0 +1,210 @@
+"""
+Support for listening with both HTTPS and Foolscap on the same port.
+
+The goal is to make the transition from Foolscap to HTTPS-based protocols as
+simple as possible, with no extra configuration needed.  Listening on the same
+port means a user upgrading Tahoe-LAFS will automatically get HTTPS working
+with no additional changes.
+
+Use ``create_tub_with_https_support()`` creates a new ``Tub`` that has its
+``negotiationClass`` modified to be a new subclass tied to that specific
+``Tub`` instance.  Calling ``tub.negotiationClass.add_storage_server(...)``
+then adds relevant information for a storage server once it becomes available
+later in the configuration process.
+"""
+
+from __future__ import annotations
+
+from itertools import chain
+
+from twisted.internet.protocol import Protocol
+from twisted.internet.interfaces import IDelayedCall
+from twisted.internet.ssl import CertificateOptions
+from twisted.web.server import Site
+from twisted.protocols.tls import TLSMemoryBIOFactory
+from twisted.internet import reactor
+
+from hyperlink import DecodedURL
+from foolscap.negotiate import Negotiation
+from foolscap.api import Tub
+
+from .storage.http_server import HTTPServer, build_nurl
+from .storage.server import StorageServer
+
+
+class _PretendToBeNegotiation(type):
+    """
+    Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a
+    ``Negotiation`` instance, since Foolscap does some checks like
+    ``assert isinstance(protocol, tub.negotiationClass)`` in its internals,
+    and sometimes that ``protocol`` is a ``_FoolscapOrHttps`` instance, but
+    sometimes it's a ``Negotiation`` instance.
+    """
+
+    def __instancecheck__(self, instance):
+        return issubclass(instance.__class__, self) or isinstance(instance, Negotiation)
+
+
+class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
+    """
+    Based on initial query, decide whether we're talking Foolscap or HTTP.
+
+    Additionally, pretends to be a ``foolscap.negotiate.Negotiation`` instance,
+    since these are created by Foolscap's ``Tub``, by setting this to be the
+    tub's ``negotiationClass``.
+
+    Do not instantiate directly, use ``create_tub_with_https_support(...)``
+    instead.  The way this class works is that a new subclass is created for a
+    specific ``Tub`` instance.
+    """
+
+    # These are class attributes; they will be set by
+    # create_tub_with_https_support() and add_storage_server().
+
+    # The Twisted HTTPS protocol factory wrapping the storage server HTTP API:
+    https_factory: TLSMemoryBIOFactory
+    # The tub that created us:
+    tub: Tub
+
+    @classmethod
+    def add_storage_server(
+        cls, storage_server: StorageServer, swissnum: bytes
+    ) -> set[DecodedURL]:
+        """
+        Update a ``_FoolscapOrHttps`` subclass for a specific ``Tub`` instance
+        with the class attributes it requires for a specific storage server.
+
+        Returns the resulting NURLs.
+        """
+        # We need to be a subclass:
+        assert cls != _FoolscapOrHttps
+        # The tub instance must already be set:
+        assert hasattr(cls, "tub")
+        assert isinstance(cls.tub, Tub)
+
+        # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate
+        # instance.
+        certificate_options = CertificateOptions(
+            privateKey=cls.tub.myCertificate.privateKey.original,
+            certificate=cls.tub.myCertificate.original,
+        )
+
+        http_storage_server = HTTPServer(storage_server, swissnum)
+        cls.https_factory = TLSMemoryBIOFactory(
+            certificate_options,
+            False,
+            Site(http_storage_server.get_resource()),
+        )
+
+        storage_nurls = set()
+        # Individual hints can be in the form
+        # "tcp:host:port,tcp:host:port,tcp:host:port".
+        for location_hint in chain.from_iterable(
+            hints.split(",") for hints in cls.tub.locationHints
+        ):
+            if location_hint.startswith("tcp:"):
+                _, hostname, port = location_hint.split(":")
+                port = int(port)
+                storage_nurls.add(
+                    build_nurl(
+                        hostname,
+                        port,
+                        str(swissnum, "ascii"),
+                        cls.tub.myCertificate.original.to_cryptography(),
+                    )
+                )
+            # TODO this is probably where we'll have to support Tor and I2P?
+            # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9
+            # for discussion (there will be separate tickets added for those at
+            # some point.)
+        return storage_nurls
+
+    def __init__(self, *args, **kwargs):
+        self._foolscap: Negotiation = Negotiation(*args, **kwargs)
+
+    def __setattr__(self, name, value):
+        if name in {"_foolscap", "_buffer", "transport", "__class__", "_timeout"}:
+            object.__setattr__(self, name, value)
+        else:
+            setattr(self._foolscap, name, value)
+
+    def __getattr__(self, name):
+        return getattr(self._foolscap, name)
+
+    def _convert_to_negotiation(self):
+        """
+        Convert self to a ``Negotiation`` instance.
+        """
+        self.__class__ = Negotiation  # type: ignore
+        self.__dict__ = self._foolscap.__dict__
+
+    def initClient(self, *args, **kwargs):
+        # After creation, a Negotiation instance either has initClient() or
+        # initServer() called. Since this is a client, we're never going to do
+        # HTTP, so we can immediately become a Negotiation instance.
+        assert not hasattr(self, "_buffer")
+        self._convert_to_negotiation()
+        return self.initClient(*args, **kwargs)
+
+    def connectionMade(self):
+        self._buffer: bytes = b""
+        self._timeout: IDelayedCall = reactor.callLater(
+            30, self.transport.abortConnection
+        )
+
+    def connectionLost(self, reason):
+        if self._timeout.active():
+            self._timeout.cancel()
+
+    def dataReceived(self, data: bytes) -> None:
+        """Handle incoming data.
+
+        Once we've decided which protocol we are, update self.__class__, at
+        which point all methods will be called on the new class.
+        """
+        self._buffer += data
+        if len(self._buffer) < 8:
+            return
+
+        # Check if it looks like a Foolscap request. If so, it can handle this
+        # and later data, otherwise assume HTTPS.
+        self._timeout.cancel()
+        if self._buffer.startswith(b"GET /id/"):
+            # We're a Foolscap Negotiation server protocol instance:
+            transport = self.transport
+            buf = self._buffer
+            self._convert_to_negotiation()
+            self.makeConnection(transport)
+            self.dataReceived(buf)
+            return
+        else:
+            # We're a HTTPS protocol instance, serving the storage protocol:
+            assert self.transport is not None
+            protocol = self.https_factory.buildProtocol(self.transport.getPeer())
+            protocol.makeConnection(self.transport)
+            protocol.dataReceived(self._buffer)
+
+            # Update the factory so it knows we're transforming to a new
+            # protocol object (we'll do that next)
+            value = self.https_factory.protocols.pop(protocol)
+            self.https_factory.protocols[self] = value
+
+            # Transform self into the TLS protocol 🪄
+            self.__class__ = protocol.__class__
+            self.__dict__ = protocol.__dict__
+
+
+def create_tub_with_https_support(**kwargs) -> Tub:
+    """
+    Create a new Tub that also supports HTTPS.
+
+    This involves creating a new protocol switch class for the specific ``Tub``
+    instance.
+    """
+    the_tub = Tub(**kwargs)
+
+    class FoolscapOrHttpForTub(_FoolscapOrHttps):
+        tub = the_tub
+
+    the_tub.negotiationClass = FoolscapOrHttpForTub  # type: ignore
+    return the_tub

src/allmydata/scripts/runner.py

@@ -47,11 +47,6 @@ if _default_nodedir:
     NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]"
 
 
-# XXX all this 'dispatch' stuff needs to be unified + fixed up
-_control_node_dispatch = {
-    "run": tahoe_run.run,
-}
-
 process_control_commands = [
     ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"),
 ]  # type: SubCommands
@@ -195,6 +190,7 @@ def parse_or_exit(config, argv, stdout, stderr):
     return config
 
 def dispatch(config,
+             reactor,
              stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr):
     command = config.subCommand
     so = config.subOptions
@@ -206,8 +202,8 @@ def dispatch(config,
 
     if command in create_dispatch:
         f = create_dispatch[command]
-    elif command in _control_node_dispatch:
+    elif command == "run":
-        f = _control_node_dispatch[command]
+        f = lambda config: tahoe_run.run(reactor, config)
     elif command in debug.dispatch:
         f = debug.dispatch[command]
     elif command in admin.dispatch:
@@ -361,7 +357,7 @@ def _run_with_reactor(reactor, config, argv, stdout, stderr):
         stderr,
     )
     d.addCallback(_maybe_enable_eliot_logging, reactor)
-    d.addCallback(dispatch, stdout=stdout, stderr=stderr)
+    d.addCallback(dispatch, reactor, stdout=stdout, stderr=stderr)
     def _show_exception(f):
         # when task.react() notices a non-SystemExit exception, it does
         # log.err() with the failure and then exits with rc=1. We want this

src/allmydata/scripts/tahoe_run.py

@@ -19,6 +19,7 @@ import os, sys
 from allmydata.scripts.common import BasedirOptions
 from twisted.scripts import twistd
 from twisted.python import usage
+from twisted.python.filepath import FilePath
 from twisted.python.reflect import namedAny
 from twisted.internet.defer import maybeDeferred
 from twisted.application.service import Service
@@ -27,6 +28,13 @@ from allmydata.scripts.default_nodedir import _default_nodedir
 from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
 from allmydata.util.configutil import UnknownConfigError
 from allmydata.util.deferredutil import HookMixin
+from allmydata.util.pid import (
+    parse_pidfile,
+    check_pid_process,
+    cleanup_pidfile,
+    ProcessInTheWay,
+    InvalidPidFile,
+)
 from allmydata.storage.crawler import (
     MigratePickleFileError,
 )
@@ -35,35 +43,34 @@ from allmydata.node import (
     PrivacyError,
 )
 
+
 def get_pidfile(basedir):
     """
     Returns the path to the PID file.
     :param basedir: the node's base directory
     :returns: the path to the PID file
     """
-    return os.path.join(basedir, u"twistd.pid")
+    return os.path.join(basedir, u"running.process")
+
 
 def get_pid_from_pidfile(pidfile):
     """
     Tries to read and return the PID stored in the node's PID file
-    (twistd.pid).
+
     :param pidfile: try to read this PID file
     :returns: A numeric PID on success, ``None`` if PID file absent or
               inaccessible, ``-1`` if PID file invalid.
     """
     try:
-        with open(pidfile, "r") as f:
+        pid, _ = parse_pidfile(pidfile)
-            pid = f.read()
     except EnvironmentError:
         return None
-
+    except InvalidPidFile:
-    try:
-        pid = int(pid)
-    except ValueError:
         return -1
 
     return pid
 
+
 def identify_node_type(basedir):
     """
     :return unicode: None or one of: 'client' or 'introducer'.
@@ -206,7 +213,7 @@ class DaemonizeTahoeNodePlugin(object):
         return DaemonizeTheRealService(self.nodetype, self.basedir, so)
 
 
-def run(config, runApp=twistd.runApp):
+def run(reactor, config, runApp=twistd.runApp):
     """
     Runs a Tahoe-LAFS node in the foreground.
 
@@ -227,10 +234,15 @@ def run(config, runApp=twistd.runApp):
         print("%s is not a recognizable node directory" % quoted_basedir, file=err)
         return 1
 
-    twistd_args = ["--nodaemon", "--rundir", basedir]
+    twistd_args = [
+        # ensure twistd machinery does not daemonize.
+        "--nodaemon",
+        "--rundir", basedir,
+    ]
     if sys.platform != "win32":
-        pidfile = get_pidfile(basedir)
+        # turn off Twisted's pid-file to use our own -- but not on
-        twistd_args.extend(["--pidfile", pidfile])
+        # windows, because twistd doesn't know about pidfiles there
+        twistd_args.extend(["--pidfile", None])
     twistd_args.extend(config.twistd_args)
     twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin
 
@@ -246,10 +258,18 @@ def run(config, runApp=twistd.runApp):
         return 1
     twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)}
 
-    # handle invalid PID file (twistd might not start otherwise)
+    # our own pid-style file contains PID and process creation time
-    if sys.platform != "win32" and get_pid_from_pidfile(pidfile) == -1:
+    pidfile = FilePath(get_pidfile(config['basedir']))
-        print("found invalid PID file in %s - deleting it" % basedir, file=err)
+    try:
-        os.remove(pidfile)
+        check_pid_process(pidfile)
+    except (ProcessInTheWay, InvalidPidFile) as e:
+        print("ERROR: {}".format(e), file=err)
+        return 1
+    else:
+        reactor.addSystemEventTrigger(
+            "after", "shutdown",
+            lambda: cleanup_pidfile(pidfile)
+        )
 
     # We always pass --nodaemon so twistd.runApp does not daemonize.
     print("running node in %s" % (quoted_basedir,), file=out)

src/allmydata/storage/http_client.py

@@ -83,35 +83,35 @@ _SCHEMAS = {
     "allocate_buckets": Schema(
         """
     response = {
-      already-have: #6.258([* uint])
+      already-have: #6.258([0*256 uint])
-      allocated: #6.258([* uint])
+      allocated: #6.258([0*256 uint])
     }
     """
     ),
     "immutable_write_share_chunk": Schema(
         """
     response = {
-      required: [* {begin: uint, end: uint}]
+      required: [0* {begin: uint, end: uint}]
     }
     """
     ),
     "list_shares": Schema(
         """
-    response = #6.258([* uint])
+    response = #6.258([0*256 uint])
     """
     ),
     "mutable_read_test_write": Schema(
         """
         response = {
           "success": bool,
-          "data": {* share_number: [* bstr]}
+          "data": {0*256 share_number: [0* bstr]}
         }
         share_number = uint
         """
     ),
     "mutable_list_shares": Schema(
         """
-        response = #6.258([* uint])
+        response = #6.258([0*256 uint])
         """
     ),
 }
@@ -392,7 +392,7 @@ class StorageClientGeneral(object):
         """
         Return the version metadata for the server.
         """
-        url = self._client.relative_url("/v1/version")
+        url = self._client.relative_url("/storage/v1/version")
         response = yield self._client.request("GET", url)
         decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"])
         returnValue(decoded_response)
@@ -408,7 +408,7 @@ class StorageClientGeneral(object):
         Otherwise a new lease is added.
         """
         url = self._client.relative_url(
-            "/v1/lease/{}".format(_encode_si(storage_index))
+            "/storage/v1/lease/{}".format(_encode_si(storage_index))
         )
         response = yield self._client.request(
             "PUT",
@@ -457,7 +457,9 @@ def read_share_chunk(
     always provided by the current callers.
     """
     url = client.relative_url(
-        "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number)
+        "/storage/v1/{}/{}/{}".format(
+            share_type, _encode_si(storage_index), share_number
+        )
     )
     response = yield client.request(
         "GET",
@@ -518,7 +520,7 @@ async def advise_corrupt_share(
 ):
     assert isinstance(reason, str)
     url = client.relative_url(
-        "/v1/{}/{}/{}/corrupt".format(
+        "/storage/v1/{}/{}/{}/corrupt".format(
             share_type, _encode_si(storage_index), share_number
         )
     )
@@ -563,7 +565,9 @@ class StorageClientImmutables(object):
         Result fires when creating the storage index succeeded, if creating the
         storage index failed the result will fire with an exception.
         """
-        url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index))
+        url = self._client.relative_url(
+            "/storage/v1/immutable/" + _encode_si(storage_index)
+        )
         message = {"share-numbers": share_numbers, "allocated-size": allocated_size}
 
         response = yield self._client.request(
@@ -588,7 +592,9 @@ class StorageClientImmutables(object):
     ) -> Deferred[None]:
         """Abort the upload."""
         url = self._client.relative_url(
-            "/v1/immutable/{}/{}/abort".format(_encode_si(storage_index), share_number)
+            "/storage/v1/immutable/{}/{}/abort".format(
+                _encode_si(storage_index), share_number
+            )
         )
         response = yield self._client.request(
             "PUT",
@@ -620,7 +626,9 @@ class StorageClientImmutables(object):
         been uploaded.
         """
         url = self._client.relative_url(
-            "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number)
+            "/storage/v1/immutable/{}/{}".format(
+                _encode_si(storage_index), share_number
+            )
         )
         response = yield self._client.request(
             "PATCH",
@@ -668,7 +676,7 @@ class StorageClientImmutables(object):
         Return the set of shares for a given storage index.
         """
         url = self._client.relative_url(
-            "/v1/immutable/{}/shares".format(_encode_si(storage_index))
+            "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
         )
         response = yield self._client.request(
             "GET",
@@ -774,7 +782,7 @@ class StorageClientMutables:
         are done and if they are valid the writes are done.
         """
         url = self._client.relative_url(
-            "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index))
+            "/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index))
         )
         message = {
             "test-write-vectors": {
@@ -817,7 +825,7 @@ class StorageClientMutables:
         List the share numbers for a given storage index.
         """
         url = self._client.relative_url(
-            "/v1/mutable/{}/shares".format(_encode_si(storage_index))
+            "/storage/v1/mutable/{}/shares".format(_encode_si(storage_index))
         )
         response = await self._client.request("GET", url)
         if response.code == http.OK:

src/allmydata/storage/http_server.py

@@ -4,12 +4,13 @@ HTTP server for storage.
 
 from __future__ import annotations
 
-from typing import Dict, List, Set, Tuple, Any, Callable, Union
+from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast
 from functools import wraps
 from base64 import b64decode
 import binascii
 from tempfile import TemporaryFile
 
+from cryptography.x509 import Certificate as CryptoCertificate
 from zope.interface import implementer
 from klein import Klein
 from twisted.web import http
@@ -18,6 +19,7 @@ from twisted.internet.interfaces import (
     IStreamServerEndpoint,
     IPullProducer,
 )
+from twisted.internet.address import IPv4Address, IPv6Address
 from twisted.internet.defer import Deferred
 from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate
 from twisted.web.server import Site, Request
@@ -193,7 +195,12 @@ class UploadsInProgress(object):
 
     def remove_write_bucket(self, bucket: BucketWriter):
         """Stop tracking the given ``BucketWriter``."""
+        try:
             storage_index, share_number = self._bucketwriters.pop(bucket)
+        except KeyError:
+            # This is probably a BucketWriter created by Foolscap, so just
+            # ignore it.
+            return
         uploads_index = self._uploads[storage_index]
         uploads_index.shares.pop(share_number)
         uploads_index.upload_secrets.pop(share_number)
@@ -253,7 +260,7 @@ _SCHEMAS = {
     "allocate_buckets": Schema(
         """
     request = {
-      share-numbers: #6.258([*256 uint])
+      share-numbers: #6.258([0*256 uint])
       allocated-size: uint
     }
     """
@@ -269,15 +276,13 @@ _SCHEMAS = {
         """
         request = {
             "test-write-vectors": {
-                ; TODO Add length limit here, after
+                0*256 share_number : {
-                ; https://github.com/anweiss/cddl/issues/128 is fixed
+                    "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}]
-                * share_number => {
+                    "write": [0*30 {"offset": uint, "data": bstr}]
-                    "test": [*30 {"offset": uint, "size": uint, "specimen": bstr}]
-                    "write": [*30 {"offset": uint, "data": bstr}]
                     "new-length": uint / null
                 }
             }
-            "read-vector": [*30 {"offset": uint, "size": uint}]
+            "read-vector": [0*30 {"offset": uint, "size": uint}]
         }
         share_number = uint
         """
@@ -545,7 +550,7 @@ class HTTPServer(object):
 
     ##### Generic APIs #####
 
-    @_authorized_route(_app, set(), "/v1/version", methods=["GET"])
+    @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"])
     def version(self, request, authorization):
         """Return version information."""
         return self._send_encoded(request, self._storage_server.get_version())
@@ -555,7 +560,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD},
-        "/v1/immutable/<storage_index:storage_index>",
+        "/storage/v1/immutable/<storage_index:storage_index>",
         methods=["POST"],
     )
     def allocate_buckets(self, request, authorization, storage_index):
@@ -591,7 +596,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         {Secrets.UPLOAD},
-        "/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>/abort",
+        "/storage/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>/abort",
         methods=["PUT"],
     )
     def abort_share_upload(self, request, authorization, storage_index, share_number):
@@ -622,7 +627,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         {Secrets.UPLOAD},
-        "/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>",
+        "/storage/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>",
         methods=["PATCH"],
     )
     def write_share_data(self, request, authorization, storage_index, share_number):
@@ -665,7 +670,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         set(),
-        "/v1/immutable/<storage_index:storage_index>/shares",
+        "/storage/v1/immutable/<storage_index:storage_index>/shares",
         methods=["GET"],
     )
     def list_shares(self, request, authorization, storage_index):
@@ -678,7 +683,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         set(),
-        "/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>",
+        "/storage/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>",
         methods=["GET"],
     )
     def read_share_chunk(self, request, authorization, storage_index, share_number):
@@ -694,7 +699,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL},
-        "/v1/lease/<storage_index:storage_index>",
+        "/storage/v1/lease/<storage_index:storage_index>",
         methods=["PUT"],
     )
     def add_or_renew_lease(self, request, authorization, storage_index):
@@ -715,7 +720,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         set(),
-        "/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>/corrupt",
+        "/storage/v1/immutable/<storage_index:storage_index>/<int(signed=False):share_number>/corrupt",
         methods=["POST"],
     )
     def advise_corrupt_share_immutable(
@@ -736,7 +741,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.WRITE_ENABLER},
-        "/v1/mutable/<storage_index:storage_index>/read-test-write",
+        "/storage/v1/mutable/<storage_index:storage_index>/read-test-write",
         methods=["POST"],
     )
     def mutable_read_test_write(self, request, authorization, storage_index):
@@ -771,7 +776,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         set(),
-        "/v1/mutable/<storage_index:storage_index>/<int(signed=False):share_number>",
+        "/storage/v1/mutable/<storage_index:storage_index>/<int(signed=False):share_number>",
         methods=["GET"],
     )
     def read_mutable_chunk(self, request, authorization, storage_index, share_number):
@@ -795,7 +800,10 @@ class HTTPServer(object):
         return read_range(request, read_data, share_length)
 
     @_authorized_route(
-        _app, set(), "/v1/mutable/<storage_index:storage_index>/shares", methods=["GET"]
+        _app,
+        set(),
+        "/storage/v1/mutable/<storage_index:storage_index>/shares",
+        methods=["GET"],
     )
     def enumerate_mutable_shares(self, request, authorization, storage_index):
         """List mutable shares for a storage index."""
@@ -805,7 +813,7 @@ class HTTPServer(object):
     @_authorized_route(
         _app,
         set(),
-        "/v1/mutable/<storage_index:storage_index>/<int(signed=False):share_number>/corrupt",
+        "/storage/v1/mutable/<storage_index:storage_index>/<int(signed=False):share_number>/corrupt",
         methods=["POST"],
     )
     def advise_corrupt_share_mutable(
@@ -859,6 +867,29 @@ class _TLSEndpointWrapper(object):
         )
 
 
+def build_nurl(
+    hostname: str, port: int, swissnum: str, certificate: CryptoCertificate
+) -> DecodedURL:
+    """
+    Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509
+    certificate for the server.  Clients can then connect to the server using
+    this NURL.
+    """
+    return DecodedURL().replace(
+        fragment="v=1",  # how we know this NURL is HTTP-based (i.e. not Foolscap)
+        host=hostname,
+        port=port,
+        path=(swissnum,),
+        userinfo=(
+            str(
+                get_spki_hash(certificate),
+                "ascii",
+            ),
+        ),
+        scheme="pb",
+    )
+
+
 def listen_tls(
     server: HTTPServer,
     hostname: str,
@@ -878,22 +909,15 @@ def listen_tls(
     """
     endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path)
 
-    def build_nurl(listening_port: IListeningPort) -> DecodedURL:
+    def get_nurl(listening_port: IListeningPort) -> DecodedURL:
-        nurl = DecodedURL().replace(
+        address = cast(Union[IPv4Address, IPv6Address], listening_port.getHost())
-            fragment="v=1",  # how we know this NURL is HTTP-based (i.e. not Foolscap)
+        return build_nurl(
-            host=hostname,
+            hostname,
-            port=listening_port.getHost().port,
+            address.port,
-            path=(str(server._swissnum, "ascii"),),
+            str(server._swissnum, "ascii"),
-            userinfo=(
+            load_pem_x509_certificate(cert_path.getContent()),
-                str(
-                    get_spki_hash(load_pem_x509_certificate(cert_path.getContent())),
-                    "ascii",
-                ),
-            ),
-            scheme="pb",
         )
-        return nurl
 
     return endpoint.listen(Site(server.get_resource())).addCallback(
-        lambda listening_port: (build_nurl(listening_port), listening_port)
+        lambda listening_port: (get_nurl(listening_port), listening_port)
     )

src/allmydata/storage/immutable.py

@@ -397,7 +397,9 @@ class BucketWriter(object):
         """
         Write data at given offset, return whether the upload is complete.
         """
-        # Delay the timeout, since we received data:
+        # Delay the timeout, since we received data; if we get an
+        # AlreadyCancelled error, that means there's a bug in the client and
+        # write() was called after close().
         self._timeout.reset(30 * 60)
         start = self._clock.seconds()
         precondition(not self.closed)
@@ -419,14 +421,18 @@ class BucketWriter(object):
         self._already_written.set(True, offset, end)
         self.ss.add_latency("write", self._clock.seconds() - start)
         self.ss.count("write")
+        return self._is_finished()
 
-        # Return whether the whole thing has been written. See
+    def _is_finished(self):
-        # https://github.com/mlenzen/collections-extended/issues/169 and
+        """
-        # https://github.com/mlenzen/collections-extended/issues/172 for why
+        Return whether the whole thing has been written.
-        # it's done this way.
+        """
         return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size
 
     def close(self):
+        # This can't actually be enabled, because it's not backwards compatible
+        # with old Foolscap clients.
+        # assert self._is_finished()
         precondition(not self.closed)
         self._timeout.cancel()
         start = self._clock.seconds()

src/allmydata/test/cli/test_run.py

@@ -12,23 +12,19 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+import re
 from six.moves import (
     StringIO,
 )
 
-from testtools import (
+from hypothesis.strategies import text
-    skipIf,
+from hypothesis import given, assume
-)
 
 from testtools.matchers import (
     Contains,
     Equals,
-    HasLength,
 )
 
-from twisted.python.runtime import (
-    platform,
-)
 from twisted.python.filepath import (
     FilePath,
 )
@@ -44,6 +40,10 @@ from ...scripts.tahoe_run import (
     RunOptions,
     run,
 )
+from ...util.pid import (
+    check_pid_process,
+    InvalidPidFile,
+)
 
 from ...scripts.runner import (
     parse_options
@@ -151,7 +151,7 @@ class RunTests(SyncTestCase):
     """
     Tests for ``run``.
     """
-    @skipIf(platform.isWindows(), "There are no PID files on Windows.")
+
     def test_non_numeric_pid(self):
         """
         If the pidfile exists but does not contain a numeric value, a complaint to
@@ -159,7 +159,7 @@ class RunTests(SyncTestCase):
         """
         basedir = FilePath(self.mktemp()).asTextMode()
         basedir.makedirs()
-        basedir.child(u"twistd.pid").setContent(b"foo")
+        basedir.child(u"running.process").setContent(b"foo")
         basedir.child(u"tahoe-client.tac").setContent(b"")
 
         config = RunOptions()
@@ -168,17 +168,30 @@ class RunTests(SyncTestCase):
         config['basedir'] = basedir.path
         config.twistd_args = []
 
+        reactor = MemoryReactor()
+
         runs = []
-        result_code = run(config, runApp=runs.append)
+        result_code = run(reactor, config, runApp=runs.append)
         self.assertThat(
             config.stderr.getvalue(),
             Contains("found invalid PID file in"),
         )
-        self.assertThat(
+        # because the pidfile is invalid we shouldn't get to the
-            runs,
+        # .run() call itself.
-            HasLength(1),
+        self.assertThat(runs, Equals([]))
-        )
+        self.assertThat(result_code, Equals(1))
-        self.assertThat(
+
-            result_code,
+    good_file_content_re = re.compile(r"\w[0-9]*\w[0-9]*\w")
-            Equals(0),
+
-        )
+    @given(text())
+    def test_pidfile_contents(self, content):
+        """
+        invalid contents for a pidfile raise errors
+        """
+        assume(not self.good_file_content_re.match(content))
+        pidfile = FilePath("pidfile")
+        pidfile.setContent(content.encode("utf8"))
+
+        with self.assertRaises(InvalidPidFile):
+            with check_pid_process(pidfile):
+                pass

src/allmydata/test/cli_node_api.py

@@ -134,7 +134,7 @@ class CLINodeAPI(object):
 
     @property
     def twistd_pid_file(self):
-        return self.basedir.child(u"twistd.pid")
+        return self.basedir.child(u"running.process")
 
     @property
     def node_url_file(self):

src/allmydata/test/common_util.py

@@ -145,6 +145,7 @@ def run_cli_native(verb, *args, **kwargs):
     )
     d.addCallback(
         runner.dispatch,
+        reactor,
         stdin=stdin,
         stdout=stdout,
         stderr=stderr,

src/allmydata/test/test_istorageserver.py

@@ -18,21 +18,14 @@ from unittest import SkipTest
 from twisted.internet.defer import inlineCallbacks, returnValue, succeed
 from twisted.internet.task import Clock
 from twisted.internet import reactor
-from twisted.internet.endpoints import serverFromString
-from twisted.python.filepath import FilePath
 from foolscap.api import Referenceable, RemoteException
 
-from allmydata.interfaces import IStorageServer  # really, IStorageClient
+# A better name for this would be IStorageClient...
+from allmydata.interfaces import IStorageServer
+
 from .common_system import SystemTestMixin
-from .common import AsyncTestCase, SameProcessStreamEndpointAssigner
+from .common import AsyncTestCase
-from .certs import (
-    generate_certificate,
-    generate_private_key,
-    private_key_to_file,
-    cert_to_file,
-)
 from allmydata.storage.server import StorageServer  # not a IStorageServer!!
-from allmydata.storage.http_server import HTTPServer, listen_tls
 from allmydata.storage.http_client import StorageClient
 from allmydata.storage_client import _HTTPStorageServer
 
@@ -1084,40 +1077,17 @@ class _FoolscapMixin(_SharedMixin):
 class _HTTPMixin(_SharedMixin):
     """Run tests on the HTTP version of ``IStorageServer``."""
 
-    def setUp(self):
-        self._port_assigner = SameProcessStreamEndpointAssigner()
-        self._port_assigner.setUp()
-        self.addCleanup(self._port_assigner.tearDown)
-        return _SharedMixin.setUp(self)
-
-    @inlineCallbacks
     def _get_istorage_server(self):
-        swissnum = b"1234"
+        nurl = list(self.clients[0].storage_nurls)[0]
-        http_storage_server = HTTPServer(self.server, swissnum)
-
-        # Listen on randomly assigned port, using self-signed cert:
-        private_key = generate_private_key()
-        certificate = generate_certificate(private_key)
-        _, endpoint_string = self._port_assigner.assign(reactor)
-        nurl, listening_port = yield listen_tls(
-            http_storage_server,
-            "127.0.0.1",
-            serverFromString(reactor, endpoint_string),
-            private_key_to_file(FilePath(self.mktemp()), private_key),
-            cert_to_file(FilePath(self.mktemp()), certificate),
-        )
-        self.addCleanup(listening_port.stopListening)
 
         # Create HTTP client with non-persistent connections, so we don't leak
         # state across tests:
-        returnValue(
+        client: IStorageServer = _HTTPStorageServer.from_http_client(
-            _HTTPStorageServer.from_http_client(
             StorageClient.from_nurl(nurl, reactor, persistent=False)
         )
-        )
+        self.assertTrue(IStorageServer.providedBy(client))
 
-        # Eventually should also:
+        return succeed(client)
-        #  self.assertTrue(IStorageServer.providedBy(client))
 
 
 class FoolscapSharedAPIsTests(

src/allmydata/test/test_protocol_switch.py

@@ -0,0 +1,43 @@
+"""
+Unit tests for ``allmydata.protocol_switch``.
+
+By its nature, most of the testing needs to be end-to-end; essentially any test
+that uses real Foolscap (``test_system.py``, integration tests) ensures
+Foolscap still works.  ``test_istorageserver.py`` tests the HTTP support.
+"""
+
+from foolscap.negotiate import Negotiation
+
+from .common import TestCase
+from ..protocol_switch import _PretendToBeNegotiation
+
+
+class UtilityTests(TestCase):
+    """Tests for utilities in the protocol switch code."""
+
+    def test_metaclass(self):
+        """
+        A class that has the ``_PretendToBeNegotiation`` metaclass will support
+        ``isinstance()``'s normal semantics on its own instances, but will also
+        indicate that ``Negotiation`` instances are its instances.
+        """
+
+        class Parent(metaclass=_PretendToBeNegotiation):
+            pass
+
+        class Child(Parent):
+            pass
+
+        class Other:
+            pass
+
+        p = Parent()
+        self.assertIsInstance(p, Parent)
+        self.assertIsInstance(Negotiation(), Parent)
+        self.assertNotIsInstance(Other(), Parent)
+
+        c = Child()
+        self.assertIsInstance(c, Child)
+        self.assertIsInstance(c, Parent)
+        self.assertIsInstance(Negotiation(), Child)
+        self.assertNotIsInstance(Other(), Child)

src/allmydata/test/test_repairer.py

@@ -251,6 +251,12 @@ class Verifier(GridTestMixin, unittest.TestCase, RepairTestMixin):
                                       self.judge_invisible_corruption)
 
     def test_corrupt_ueb(self):
+        # Note that in some rare situations this might fail, specifically if
+        # the length of the UEB is corrupted to be a value that is bigger than
+        # the size but less than 2000, it might not get caught... But that's
+        # mostly because in that case it doesn't meaningfully corrupt it. See
+        # _get_uri_extension_the_old_way() in layout.py for where the 2000
+        # number comes from.
         self.basedir = "repairer/Verifier/corrupt_ueb"
         return self._help_test_verify(common._corrupt_uri_extension,
                                       self.judge_invisible_corruption)

src/allmydata/test/test_runner.py

@@ -42,16 +42,19 @@ from twisted.trial import unittest
 
 from twisted.internet import reactor
 from twisted.python import usage
+from twisted.python.runtime import platform
 from twisted.internet.defer import (
     inlineCallbacks,
     DeferredList,
 )
 from twisted.python.filepath import FilePath
-from twisted.python.runtime import (
-    platform,
-)
 from allmydata.util import fileutil, pollmixin
 from allmydata.util.encodingutil import unicode_to_argv
+from allmydata.util.pid import (
+    check_pid_process,
+    _pidfile_to_lockpath,
+    ProcessInTheWay,
+)
 from allmydata.test import common_util
 import allmydata
 from allmydata.scripts.runner import (
@@ -418,8 +421,6 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         tahoe.active()
 
-        # We don't keep track of PIDs in files on Windows.
-        if not platform.isWindows():
         self.assertTrue(tahoe.twistd_pid_file.exists())
         self.assertTrue(tahoe.node_url_file.exists())
 
@@ -493,8 +494,6 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         # change on restart
         storage_furl = fileutil.read(tahoe.storage_furl_file.path)
 
-        # We don't keep track of PIDs in files on Windows.
-        if not platform.isWindows():
         self.assertTrue(tahoe.twistd_pid_file.exists())
 
         # rm this so we can detect when the second incarnation is ready
@@ -513,7 +512,6 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
             fileutil.read(tahoe.storage_furl_file.path),
         )
 
-        if not platform.isWindows():
         self.assertTrue(
             tahoe.twistd_pid_file.exists(),
             "PID file ({}) didn't exist when we expected it to.  "
@@ -524,11 +522,13 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         )
         yield tahoe.stop_and_wait()
 
+        # twistd.pid should be gone by now -- except on Windows, where
+        # killing a subprocess immediately exits with no chance for
+        # any shutdown code (that is, no Twisted shutdown hooks can
+        # run).
         if not platform.isWindows():
-            # twistd.pid should be gone by now.
             self.assertFalse(tahoe.twistd_pid_file.exists())
 
-
     def _remove(self, res, file):
         fileutil.remove(file)
         return res
@@ -610,8 +610,9 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
             ),
         )
 
+        # It should not be running (but windows shutdown can't run
+        # code so the PID file still exists there).
         if not platform.isWindows():
-            # It should not be running.
             self.assertFalse(tahoe.twistd_pid_file.exists())
 
         # Wait for the operation to *complete*.  If we got this far it's
@@ -621,3 +622,42 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         # What's left is a perfect indicator that the process has exited and
         # we won't get blamed for leaving the reactor dirty.
         yield client_running
+
+
+class PidFileLocking(SyncTestCase):
+    """
+    Direct tests for allmydata.util.pid functions
+    """
+
+    def test_locking(self):
+        """
+        Fail to create a pidfile if another process has the lock already.
+        """
+        # this can't just be "our" process because the locking library
+        # allows the same process to acquire a lock multiple times.
+        pidfile = FilePath(self.mktemp())
+        lockfile = _pidfile_to_lockpath(pidfile)
+
+        with open("other_lock.py", "w") as f:
+            f.write(
+                "\n".join([
+                    "import filelock, time, sys",
+                    "with filelock.FileLock(sys.argv[1], timeout=1):",
+                    "    sys.stdout.write('.\\n')",
+                    "    sys.stdout.flush()",
+                    "    time.sleep(10)",
+                ])
+            )
+        proc = Popen(
+            [sys.executable, "other_lock.py", lockfile.path],
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        # make sure our subprocess has had time to acquire the lock
+        # for sure (from the "." it prints)
+        proc.stdout.read(2)
+
+        # acquiring the same lock should fail; it is locked by the subprocess
+        with self.assertRaises(ProcessInTheWay):
+            check_pid_process(pidfile)
+        proc.terminate()

src/allmydata/test/test_storage.py

@@ -463,7 +463,7 @@ class BucketProxy(unittest.TestCase):
                               block_size=10,
                               num_segments=5,
                               num_share_hashes=3,
-                              uri_extension_size_max=500)
+                              uri_extension_size=500)
         self.failUnless(interfaces.IStorageBucketWriter.providedBy(bp), bp)
 
     def _do_test_readwrite(self, name, header_size, wbp_class, rbp_class):
@@ -494,7 +494,7 @@ class BucketProxy(unittest.TestCase):
                        block_size=25,
                        num_segments=4,
                        num_share_hashes=3,
-                       uri_extension_size_max=len(uri_extension))
+                       uri_extension_size=len(uri_extension))
 
         d = bp.put_header()
         d.addCallback(lambda res: bp.put_block(0, b"a"*25))

src/allmydata/test/test_storage_http.py

@@ -255,7 +255,7 @@ class TestApp(object):
         else:
             return "BAD: {}".format(authorization)
 
-    @_authorized_route(_app, set(), "/v1/version", methods=["GET"])
+    @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"])
     def bad_version(self, request, authorization):
         """Return version result that violates the expected schema."""
         request.setHeader("content-type", CBOR_MIME_TYPE)
@@ -534,7 +534,7 @@ class GenericHTTPAPITests(SyncTestCase):
         lease_secret = urandom(32)
         storage_index = urandom(16)
         url = self.http.client.relative_url(
-            "/v1/immutable/" + _encode_si(storage_index)
+            "/storage/v1/immutable/" + _encode_si(storage_index)
         )
         message = {"bad-message": "missing expected keys"}
 
@@ -1418,7 +1418,7 @@ class SharedImmutableMutableTestsMixin:
             self.http.client.request(
                 "GET",
                 self.http.client.relative_url(
-                    "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
+                    "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
                 ),
             )
         )
@@ -1441,7 +1441,7 @@ class SharedImmutableMutableTestsMixin:
                 self.http.client.request(
                     "GET",
                     self.http.client.relative_url(
-                        "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
+                        "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
                     ),
                     headers=headers,
                 )

src/allmydata/test/test_storage_https.py

@@ -198,6 +198,10 @@ class PinningHTTPSValidation(AsyncTestCase):
             response = await self.request(url, certificate)
             self.assertEqual(await response.content(), b"YOYODYNE")
 
+        # We keep getting TLSMemoryBIOProtocol being left around, so try harder
+        # to wait for it to finish.
+        await deferLater(reactor, 0.001)
+
     # A potential attack to test is a private key that doesn't match the
     # certificate... but OpenSSL (quite rightly) won't let you listen with that
     # so I don't know how to test that! See

src/allmydata/test/test_upload.py

@@ -983,7 +983,7 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
         num_segments = encoder.get_param("num_segments")
         d = selector.get_shareholders(broker, sh, storage_index,
                                       share_size, block_size, num_segments,
-                                      10, 3, 4)
+                                      10, 3, 4, encoder.get_uri_extension_size())
         def _have_shareholders(upload_trackers_and_already_servers):
             (upload_trackers, already_servers) = upload_trackers_and_already_servers
             assert servers_to_break <= len(upload_trackers)

src/allmydata/util/pid.py

@@ -0,0 +1,120 @@
+import psutil
+
+# the docs are a little misleading, but this is either WindowsFileLock
+# or UnixFileLock depending upon the platform we're currently on
+from filelock import FileLock, Timeout
+
+
+class ProcessInTheWay(Exception):
+    """
+    our pidfile points at a running process
+    """
+
+
+class InvalidPidFile(Exception):
+    """
+    our pidfile isn't well-formed
+    """
+
+
+class CannotRemovePidFile(Exception):
+    """
+    something went wrong removing the pidfile
+    """
+
+
+def _pidfile_to_lockpath(pidfile):
+    """
+    internal helper.
+    :returns FilePath: a path to use for file-locking the given pidfile
+    """
+    return pidfile.sibling("{}.lock".format(pidfile.basename()))
+
+
+def parse_pidfile(pidfile):
+    """
+    :param FilePath pidfile:
+    :returns tuple: 2-tuple of pid, creation-time as int, float
+    :raises InvalidPidFile: on error
+    """
+    with pidfile.open("r") as f:
+        content = f.read().decode("utf8").strip()
+    try:
+        pid, starttime = content.split()
+        pid = int(pid)
+        starttime = float(starttime)
+    except ValueError:
+        raise InvalidPidFile(
+            "found invalid PID file in {}".format(
+                pidfile
+            )
+        )
+    return pid, starttime
+
+
+def check_pid_process(pidfile):
+    """
+    If another instance appears to be running already, raise an
+    exception.  Otherwise, write our PID + start time to the pidfile
+    and arrange to delete it upon exit.
+
+    :param FilePath pidfile: the file to read/write our PID from.
+
+    :raises ProcessInTheWay: if a running process exists at our PID
+    """
+    lock_path = _pidfile_to_lockpath(pidfile)
+
+    try:
+        # a short timeout is fine, this lock should only be active
+        # while someone is reading or deleting the pidfile .. and
+        # facilitates testing the locking itself.
+        with FileLock(lock_path.path, timeout=2):
+            # check if we have another instance running already
+            if pidfile.exists():
+                pid, starttime = parse_pidfile(pidfile)
+                try:
+                    # if any other process is running at that PID, let the
+                    # user decide if this is another legitimate
+                    # instance. Automated programs may use the start-time to
+                    # help decide this (if the PID is merely recycled, the
+                    # start-time won't match).
+                    psutil.Process(pid)
+                    raise ProcessInTheWay(
+                        "A process is already running as PID {}".format(pid)
+                    )
+                except psutil.NoSuchProcess:
+                    print(
+                        "'{pidpath}' refers to {pid} that isn't running".format(
+                            pidpath=pidfile.path,
+                            pid=pid,
+                        )
+                    )
+                    # nothing is running at that PID so it must be a stale file
+                    pidfile.remove()
+
+            # write our PID + start-time to the pid-file
+            proc = psutil.Process()
+            with pidfile.open("w") as f:
+                f.write("{} {}\n".format(proc.pid, proc.create_time()).encode("utf8"))
+    except Timeout:
+        raise ProcessInTheWay(
+            "Another process is still locking {}".format(pidfile.path)
+        )
+
+
+def cleanup_pidfile(pidfile):
+    """
+    Remove the pidfile specified (respecting locks). If anything at
+    all goes wrong, `CannotRemovePidFile` is raised.
+    """
+    lock_path = _pidfile_to_lockpath(pidfile)
+    with FileLock(lock_path.path):
+        try:
+            pidfile.remove()
+        except Exception as e:
+            raise CannotRemovePidFile(
+                "Couldn't remove '{pidfile}': {err}.".format(
+                    pidfile=pidfile.path,
+                    err=e,
+                )
+            )