Merge remote-tracking branch 'origin/master' into 3902-listen-storage-http

Makefile

@@ -224,3 +224,62 @@ src/allmydata/_version.py:
 
 .tox/create-venvs.log: tox.ini setup.py
 	tox --notest -p all | tee -a "$(@)"
+
+
+# to make a new release:
+# - create a ticket for the release in Trac
+# - ensure local copy is up-to-date
+# - create a branch like "XXXX.release" from up-to-date master
+# - in the branch, run "make release"
+# - run "make release-test"
+# - perform any other sanity-checks on the release
+# - run "make release-upload"
+# Note that several commands below hard-code "meejah"; if you are
+# someone else please adjust them.
+release:
+	@echo "Is checkout clean?"
+	git diff-files --quiet
+	git diff-index --quiet --cached HEAD --
+
+	@echo "Clean docs build area"
+	rm -rf docs/_build/
+
+	@echo "Install required build software"
+	python3 -m pip install --editable .[build]
+
+	@echo "Test README"
+	python3 setup.py check -r -s
+
+	@echo "Update NEWS"
+	python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag`
+	git add -u
+	git commit -m "update NEWS for release"
+
+# note that this always bumps the "middle" number, e.g. from 1.17.1 -> 1.18.0
+# and produces a tag into the Git repository
+	@echo "Bump version and create tag"
+	python3 misc/build_helpers/update-version.py
+
+	@echo "Build and sign wheel"
+	python3 setup.py bdist_wheel
+	gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl
+	ls dist/*`git describe | cut -b 12-`*
+
+	@echo "Build and sign source-dist"
+	python3 setup.py sdist
+	gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz
+	ls dist/*`git describe | cut -b 12-`*
+
+# basically just a bare-minimum smoke-test that it installs and runs
+release-test:
+	gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc
+	gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc
+	virtualenv testmf_venv
+	testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl
+	testmf_venv/bin/tahoe --version
+	rm -rf testmf_venv
+
+release-upload:
+	scp dist/*`git describe | cut -b 12-`* meejah@tahoe-lafs.org:/home/source/downloads
+	git push origin_push tahoe-lafs-`git describe | cut -b 12-`
+	twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc

NEWS.rst

@@ -5,6 +5,47 @@ User-Visible Changes in Tahoe-LAFS
 ==================================
 
 .. towncrier start line
+Release 1.18.0 (2022-10-02)
+'''''''''''''''''''''''''''
+
+Backwards Incompatible Changes
+------------------------------
+
+- Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. (`#3865 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3865>`_)
+- Python 3.7 or later is now required; Python 2 is no longer supported. (`#3873 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3873>`_)
+- Share corruption reports stored on disk are now always encoded in UTF-8. (`#3879 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3879>`_)
+- Record both the PID and the process creation-time:
+
+  a new kind of pidfile in `running.process` records both
+  the PID and the creation-time of the process. This facilitates
+  automatic discovery of a "stale" pidfile that points to a
+  currently-running process. If the recorded creation-time matches
+  the creation-time of the running process, then it is a still-running
+  `tahoe run` process. Otherwise, the file is stale.
+
+  The `twistd.pid` file is no longer present. (`#3926 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3926>`_)
+
+
+Features
+--------
+
+- The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification.
+
+  Some code existed to allow tests to shorten this and it's
+  conceptually possible a modified client produced mutables
+  with different key-sizes. However, the spec says that they
+  must be 2048 bits. If you happen to have a capability with
+  a key-size different from 2048 you may use 1.17.1 or earlier
+  to read the content. (`#3828 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3828>`_)
+- "make" based release automation (`#3846 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3846>`_)
+
+
+Misc/Other
+----------
+
+- `#3327 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3327>`_, `#3526 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3526>`_, `#3697 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3697>`_, `#3709 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3709>`_, `#3786 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3786>`_, `#3788 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3788>`_, `#3802 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3802>`_, `#3816 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3816>`_, `#3855 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3855>`_, `#3858 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3858>`_, `#3859 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3859>`_, `#3860 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860>`_, `#3867 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3867>`_, `#3868 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3868>`_, `#3871 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3871>`_, `#3872 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872>`_, `#3875 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3875>`_, `#3876 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3876>`_, `#3877 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3877>`_, `#3881 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3881>`_, `#3882 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3882>`_, `#3883 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3883>`_, `#3889 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3889>`_, `#3890 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3890>`_, `#3891 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3891>`_, `#3893 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3893>`_, `#3895 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3895>`_, `#3896 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3896>`_, `#3898 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3898>`_, `#3900 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3900>`_, `#3909 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3909>`_, `#3913 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3913>`_, `#3915 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3915>`_, `#3916 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3916>`_
+
+
 Release 1.17.1 (2022-01-07)
 '''''''''''''''''''''''''''
 

integration/test_i2p.py

@@ -55,7 +55,7 @@ def i2p_network(reactor, temp_dir, request):
         proto,
         which("docker"),
         (
-            "docker", "run", "-p", "7656:7656", "purplei2p/i2pd",
+            "docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.43.0",
             # Bad URL for reseeds, so it can't talk to other routers.
             "--reseed.urls", "http://localhost:1/",
         ),
@@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request):
 
     def cleanup():
         try:
-            proto.transport.signalProcess("KILL")
+            proto.transport.signalProcess("INT")
             util.block_with_timeout(proto.exited, reactor)
         except ProcessExitedAlready:
             pass

misc/build_helpers/update-version.py

@@ -0,0 +1,95 @@
+#
+# this updates the (tagged) version of the software
+#
+# Any "options" are hard-coded in here (e.g. the GnuPG key to use)
+#
+
+author = "meejah <meejah@meejah.ca>"
+
+
+import sys
+import time
+from datetime import datetime
+from packaging.version import Version
+
+from dulwich.repo import Repo
+from dulwich.porcelain import (
+    tag_list,
+    tag_create,
+    status,
+)
+
+from twisted.internet.task import (
+    react,
+)
+from twisted.internet.defer import (
+    ensureDeferred,
+)
+
+
+def existing_tags(git):
+    versions = sorted(
+        Version(v.decode("utf8").lstrip("tahoe-lafs-"))
+        for v in tag_list(git)
+        if v.startswith(b"tahoe-lafs-")
+    )
+    return versions
+
+
+def create_new_version(git):
+    versions = existing_tags(git)
+    biggest = versions[-1]
+
+    return Version(
+        "{}.{}.{}".format(
+            biggest.major,
+            biggest.minor + 1,
+            0,
+        )
+    )
+
+
+async def main(reactor):
+    git = Repo(".")
+
+    st = status(git)
+    if any(st.staged.values()) or st.unstaged:
+        print("unclean checkout; aborting")
+        raise SystemExit(1)
+
+    v = create_new_version(git)
+    if "--no-tag" in sys.argv:
+        print(v)
+        return
+
+    print("Existing tags: {}".format("\n".join(str(x) for x in existing_tags(git))))
+    print("New tag will be {}".format(v))
+
+    # the "tag time" is seconds from the epoch .. we quantize these to
+    # the start of the day in question, in UTC.
+    now = datetime.now()
+    s = now.utctimetuple()
+    ts = int(
+        time.mktime(
+            time.struct_time((s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0))
+        )
+    )
+    tag_create(
+        repo=git,
+        tag="tahoe-lafs-{}".format(str(v)).encode("utf8"),
+        author=author.encode("utf8"),
+        message="Release {}".format(v).encode("utf8"),
+        annotated=True,
+        objectish=b"HEAD",
+        sign=author.encode("utf8"),
+        tag_time=ts,
+        tag_timezone=0,
+    )
+
+    print("Tag created locally, it is not pushed")
+    print("To push it run something like:")
+    print("   git push origin {}".format(v))
+
+
+if __name__ == "__main__":
+    react(lambda r: ensureDeferred(main(r)))

newsfragments/3526.minor

@@ -1 +0,0 @@
-

newsfragments/3697.minor

@@ -1 +0,0 @@
-Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only).

newsfragments/3786.minor

@@ -1 +0,0 @@
-Added re-structured text documentation for the OpenMetrics format statistics endpoint.

newsfragments/3828.feature

@@ -1,8 +0,0 @@
-The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification.
-
-Some code existed to allow tests to shorten this and it's
-conceptually possible a modified client produced mutables
-with different key-sizes. However, the spec says that they
-must be 2048 bits. If you happen to have a capability with
-a key-size different from 2048 you may use 1.17.1 or earlier
-to read the content.

newsfragments/3865.incompat

@@ -1 +0,0 @@
-Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates.

newsfragments/3873.incompat

@@ -1 +0,0 @@
-Python 3.7 or later is now required; Python 2 is no longer supported.

newsfragments/3879.incompat

@@ -1 +0,0 @@
-Share corruption reports stored on disk are now always encoded in UTF-8.

newsfragments/3926.incompat

@@ -1,10 +0,0 @@
-Record both the PID and the process creation-time:
-
-a new kind of pidfile in `running.process` records both
-the PID and the creation-time of the process. This facilitates
-automatic discovery of a "stale" pidfile that points to a
-currently-running process. If the recorded creation-time matches
-the creation-time of the running process, then it is a still-running
-`tahoe run` process. Otherwise, the file is stale.
-
-The `twistd.pid` file is no longer present.

setup.py

@@ -380,8 +380,15 @@ setup(name="tahoe-lafs", # also set in __init__.py
           # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some
           # discussion.
           ':sys_platform=="win32"': ["pywin32 != 226"],
+          "build": [
+              "dulwich",
+              "gpg",
+          ],
           "test": [
               "flake8",
+              # On Python 3.7, importlib_metadata v5 breaks flake8.
+              # https://github.com/python/importlib_metadata/issues/407
+              "importlib_metadata<5; python_version < '3.8'",
               # Pin a specific pyflakes so we don't have different folks
               # disagreeing on what is or is not a lint issue.  We can bump
               # this version from time to time, but we will do it