From f435b54a8ab67fd7b05f807a05ced1014cd74eb5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:03:36 -0400 Subject: [PATCH 1/9] Make this test suite contingent on the availability of Conch --- src/allmydata/test/test_sftp.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index 06b6d1d7a..2d1e7730a 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -12,18 +12,15 @@ from allmydata.util import deferredutil conch_interfaces = None sftp = None sftpd = None -have_pycrypto = False -try: - from Crypto import Util - Util # hush pyflakes - have_pycrypto = True -except ImportError: - pass -if have_pycrypto: +try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd +except ImportError as e: + conch_unavailable_reason = e +else: + conch_unavailable_reason = None from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError @@ -38,8 +35,8 @@ from allmydata.test.common_util import ReallyEqualMixin class Handler(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, unittest.TestCase): """This is a no-network unit test of the SFTPUserHandler and the abstractions it uses.""" - if not have_pycrypto: - skip = "SFTP support requires pycrypto, which is not installed" + if conch_unavailable_reason: + skip = "SFTP support requires Twisted Conch which is not available: {}".format(e) def shouldFailWithSFTPError(self, expected_code, which, callable, *args, **kwargs): assert isinstance(expected_code, int), repr(expected_code) From faed622ee9e05b87d54fda052e06293ae500c155 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:04:20 -0400 Subject: [PATCH 2/9] Add an "sftp" extra and put the Conch dependency there --- setup.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 357b9cd41..02624c842 100644 --- a/setup.py +++ b/setup.py @@ -75,13 +75,7 @@ install_requires = [ # leftover timers) # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed # for coverage testing - - # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch - # extra, letting us use that extra instead of trying to duplicate its - # dependencies here. Twisted[conch] >18.7 introduces a dependency on - # bcrypt. It is nice to avoid that if the user ends up with an older - # version of Twisted. That's hard to express except by using the extra. - "Twisted[tls,conch] >= 16.6.0", + "Twisted[tls] >= 16.4.0", # We need Nevow >= 0.11.1 which can be installed using pip. "Nevow >= 0.11.1", @@ -108,6 +102,23 @@ setup_requires = [ 'setuptools >= 28.8.0', # for PEP-440 style versions ] +sftp_requires = [ + # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch + # extra, letting us use that extra instead of trying to duplicate its + # dependencies here. Twisted[conch] >18.7 introduces a dependency on + # bcrypt. It is nice to avoid that if the user ends up with an older + # version of Twisted. That's hard to express except by using the extra. + "twisted[conch] >= 16.6.0", +] + +tor_requires = [ + "foolscap[tor] >= 0.12.5", +] + +i2p_requires = [ + "foolscap[i2p] >= 0.12.6", +] + if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': del sys.argv[1] install_requires += ["fakedependency >= 1.0.0"] @@ -330,10 +341,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "coverage", "mock", "tox", - "foolscap[tor] >= 0.12.5", - "txtorcon >= 0.17.0", # in case pip's resolver doesn't work - "foolscap[i2p] >= 0.12.6", - "txi2p >= 0.3.2", # in case pip's resolver doesn't work "pytest", "pytest-twisted", "hypothesis >= 3.6.1", @@ -341,15 +348,10 @@ setup(name="tahoe-lafs", # also set in __init__.py "towncrier", "testtools", "fixtures", - ], - "tor": [ - "foolscap[tor] >= 0.12.5", - "txtorcon >= 0.17.0", # in case pip's resolver doesn't work - ], - "i2p": [ - "foolscap[i2p] >= 0.12.6", - "txi2p >= 0.3.2", # in case pip's resolver doesn't work - ], + ] + sftp_requires + tor_requires + i2p_requires, + "sftp": sftp_requires, + "tor": tor_requires, + "i2p": i2p_requires, }, package_data={"allmydata.web": ["*.xhtml", "static/*.js", "static/*.png", "static/*.css", From 6508038a74b85850478207ca4d83c362e891042d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:04:39 -0400 Subject: [PATCH 3/9] tor and i2p were already implied by test --- .circleci/populate-wheelhouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 80b684eba..75afb6f6f 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -40,7 +40,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ - "${PROJECT_ROOT}"[test,tor,i2p] \ + "${PROJECT_ROOT}"[test] \ ${BASIC_DEPS} \ ${TEST_DEPS} \ ${REPORTING_DEPS} From 079ffbd874e15d639d5e946d1ddbb6b75be559d0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:05:41 -0400 Subject: [PATCH 4/9] news fragment --- newsfragments/3240.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3240.installation diff --git a/newsfragments/3240.installation b/newsfragments/3240.installation new file mode 100644 index 000000000..eed82b8c1 --- /dev/null +++ b/newsfragments/3240.installation @@ -0,0 +1 @@ +The SFTP frontend is now contingent on the "sftp" Tahoe-LAFS extra. To use it, be sure to install at least "tahoe-lafs[sftp]". From 42f6a5609fa8190fbd816d147603c4b8db01fac9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:23:11 -0400 Subject: [PATCH 5/9] Use the better variable name --- src/allmydata/test/test_sftp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index 2d1e7730a..b6f1fbc8a 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -36,7 +36,9 @@ class Handler(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, unittest.TestCas """This is a no-network unit test of the SFTPUserHandler and the abstractions it uses.""" if conch_unavailable_reason: - skip = "SFTP support requires Twisted Conch which is not available: {}".format(e) + skip = "SFTP support requires Twisted Conch which is not available: {}".format( + conch_unavailable_reason, + ) def shouldFailWithSFTPError(self, expected_code, which, callable, *args, **kwargs): assert isinstance(expected_code, int), repr(expected_code) From 7511d605e821c201978f07528ee025ecd4a67979 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:27:48 -0400 Subject: [PATCH 6/9] Document the use of extras at install time --- docs/INSTALL.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index f85d5e124..9c7db2c47 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -193,6 +193,17 @@ You can also install directly from the source tarball URL:: tahoe-lafs: 1.13.0 ... +Extras +------ + +Tahoe-LAFS provides some functionality only when explicitly requested at installation time. +It does this using the "extras" feature of setuptools. +You can request these extra features when running the ``pip install`` command like this:: + + % venv/bin/pip install tahoe-lafs[sftp] + +This example enables the SFTP frontend. +The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested. Hacking On Tahoe-LAFS --------------------- From 0f283980ab540e1c4a5c1b4b3723022ba1bc8ac9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 09:29:55 -0400 Subject: [PATCH 7/9] Mention the sftp extra here --- docs/frontends/FTP-and-SFTP.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index dc348af34..cb1390967 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -37,6 +37,15 @@ We recommend SFTP over FTP, because the protocol is better, and the server implementation in Tahoe-LAFS is more complete. See `Known Issues`_, below, for details. +Software Dependencies +--------------------- + +The SFTP support in Tahoe-LAFS requires certain additional libraries not installed by default. +In order to install everything necessary, use the "sftp" extra when installing Tahoe-LAFS. +For example:: + + % pip install tahoe-lafs[sftp] + Tahoe-LAFS Support ================== From 739df66c80e857e4948e9bb00ae4f47c8f7719b3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 14:48:12 -0400 Subject: [PATCH 8/9] can't have nice things --- docs/INSTALL.rst | 4 ++-- docs/frontends/FTP-and-SFTP.rst | 9 -------- newsfragments/3240.installation | 1 - setup.py | 37 ++++++++++++++++++++------------- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index 9c7db2c47..5f9d72d1d 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -200,9 +200,9 @@ Tahoe-LAFS provides some functionality only when explicitly requested at install It does this using the "extras" feature of setuptools. You can request these extra features when running the ``pip install`` command like this:: - % venv/bin/pip install tahoe-lafs[sftp] + % venv/bin/pip install tahoe-lafs[tor] -This example enables the SFTP frontend. +This example enables support for listening and connecting using Tor. The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested. Hacking On Tahoe-LAFS diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index cb1390967..dc348af34 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -37,15 +37,6 @@ We recommend SFTP over FTP, because the protocol is better, and the server implementation in Tahoe-LAFS is more complete. See `Known Issues`_, below, for details. -Software Dependencies ---------------------- - -The SFTP support in Tahoe-LAFS requires certain additional libraries not installed by default. -In order to install everything necessary, use the "sftp" extra when installing Tahoe-LAFS. -For example:: - - % pip install tahoe-lafs[sftp] - Tahoe-LAFS Support ================== diff --git a/newsfragments/3240.installation b/newsfragments/3240.installation index eed82b8c1..e69de29bb 100644 --- a/newsfragments/3240.installation +++ b/newsfragments/3240.installation @@ -1 +0,0 @@ -The SFTP frontend is now contingent on the "sftp" Tahoe-LAFS extra. To use it, be sure to install at least "tahoe-lafs[sftp]". diff --git a/setup.py b/setup.py index 02624c842..f3b837d1d 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,22 @@ install_requires = [ # leftover timers) # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed # for coverage testing - "Twisted[tls] >= 16.4.0", + # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch + # extra, letting us use that extra instead of trying to duplicate its + # dependencies here. Twisted[conch] >18.7 introduces a dependency on + # bcrypt. It is nice to avoid that if the user ends up with an older + # version of Twisted. That's hard to express except by using the extra. + # + # In a perfect world, Twisted[conch] would be a dependency of an "sftp" + # extra. However, pip fails to resolve the dependencies all + # dependencies when asked for Twisted[tls] *and* Twisted[conch]. + # Specifically, "Twisted[conch]" (as the later requirement) is ignored. + # If there were an Tahoe-LAFS sftp extra that dependended on + # Twisted[conch] and install_requires only included Twisted[tls] then + # `pip install tahoe-lafs[sftp]` would not install requirements + # specified by Twisted[conch]. Since this would be the *whole point* of + # an sftp extra in Tahoe-LAFS, there is no point in having one. + "Twisted[tls,conch] >= 16.6.0", # We need Nevow >= 0.11.1 which can be installed using pip. "Nevow >= 0.11.1", @@ -102,21 +117,16 @@ setup_requires = [ 'setuptools >= 28.8.0', # for PEP-440 style versions ] -sftp_requires = [ - # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch - # extra, letting us use that extra instead of trying to duplicate its - # dependencies here. Twisted[conch] >18.7 introduces a dependency on - # bcrypt. It is nice to avoid that if the user ends up with an older - # version of Twisted. That's hard to express except by using the extra. - "twisted[conch] >= 16.6.0", -] - tor_requires = [ - "foolscap[tor] >= 0.12.5", + # This is exactly what `foolscap[tor]` means but pip resolves the pair of + # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose + # this if we don't declare it ourselves! + "txtorcon >= 0.17.0", ] i2p_requires = [ - "foolscap[i2p] >= 0.12.6", + # See the comment in tor_requires. + "txi2p >= 0.3.2", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': @@ -348,8 +358,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "towncrier", "testtools", "fixtures", - ] + sftp_requires + tor_requires + i2p_requires, - "sftp": sftp_requires, + ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, }, From deb72161073e1ad1216ce20dfaf3a3b4826b785c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Aug 2019 14:48:36 -0400 Subject: [PATCH 9/9] more appropriate like this now --- newsfragments/{3240.installation => 3240.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3240.installation => 3240.minor} (100%) diff --git a/newsfragments/3240.installation b/newsfragments/3240.minor similarity index 100% rename from newsfragments/3240.installation rename to newsfragments/3240.minor