diff --git a/CREDITS b/CREDITS index 1394d87d8..07ac1e476 100644 --- a/CREDITS +++ b/CREDITS @@ -206,4 +206,9 @@ D: various bug-fixes and features N: Viktoriia Savchuk W: https://twitter.com/viktoriiasvchk -D: Developer community focused improvements on the README file. \ No newline at end of file +D: Developer community focused improvements on the README file. + +N: Lukas Pirl +E: tahoe@lukas-pirl.de +W: http://lukas-pirl.de +D: Buildslaves (Debian, Fedora, CentOS; 2016-2021) diff --git a/docs/about.rst b/docs/about.rst index 626792d6b..120abb079 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -67,12 +67,12 @@ Here's how it works: A "storage grid" is made up of a number of storage servers. A storage server has direct attached storage (typically one or more hard disks). A "gateway" communicates with storage nodes, and uses them to provide access to the -grid over protocols such as HTTP(S), SFTP or FTP. +grid over protocols such as HTTP(S) and SFTP. Note that you can find "client" used to refer to gateway nodes (which act as a client to storage servers), and also to processes or programs connecting to a gateway node and performing operations on the grid -- for example, a CLI -command, Web browser, SFTP client, or FTP client. +command, Web browser, or SFTP client. Users do not rely on storage servers to provide *confidentiality* nor *integrity* for their data -- instead all of the data is encrypted and diff --git a/docs/configuration.rst b/docs/configuration.rst index 2c0746ba2..93c9aa0f1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -81,7 +81,6 @@ Client/server nodes provide one or more of the following services: * web-API service * SFTP service -* FTP service * helper service * storage service. @@ -708,12 +707,12 @@ CLI file store, uploading/downloading files, and creating/running Tahoe nodes. See :doc:`frontends/CLI` for details. -SFTP, FTP +SFTP - Tahoe can also run both SFTP and FTP servers, and map a username/password + Tahoe can also run SFTP servers, and map a username/password pair to a top-level Tahoe directory. See :doc:`frontends/FTP-and-SFTP` - for instructions on configuring these services, and the ``[sftpd]`` and - ``[ftpd]`` sections of ``tahoe.cfg``. + for instructions on configuring this service, and the ``[sftpd]`` + section of ``tahoe.cfg``. Storage Server Configuration diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index dc348af34..ee6371812 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -1,22 +1,21 @@ .. -*- coding: utf-8-with-signature -*- -================================= -Tahoe-LAFS SFTP and FTP Frontends -================================= +======================== +Tahoe-LAFS SFTP Frontend +======================== -1. `SFTP/FTP Background`_ +1. `SFTP Background`_ 2. `Tahoe-LAFS Support`_ 3. `Creating an Account File`_ 4. `Running An Account Server (accounts.url)`_ 5. `Configuring SFTP Access`_ -6. `Configuring FTP Access`_ -7. `Dependencies`_ -8. `Immutable and Mutable Files`_ -9. `Known Issues`_ +6. `Dependencies`_ +7. `Immutable and Mutable Files`_ +8. `Known Issues`_ -SFTP/FTP Background -=================== +SFTP Background +=============== FTP is the venerable internet file-transfer protocol, first developed in 1971. The FTP server usually listens on port 21. A separate connection is @@ -33,20 +32,18 @@ Both FTP and SFTP were developed assuming a UNIX-like server, with accounts and passwords, octal file modes (user/group/other, read/write/execute), and ctime/mtime timestamps. -We recommend SFTP over FTP, because the protocol is better, and the server -implementation in Tahoe-LAFS is more complete. See `Known Issues`_, below, -for details. +Previous versions of Tahoe-LAFS supported FTP, but now only the superior SFTP +frontend is supported. See `Known Issues`_, below, for details on the +limitations of SFTP. Tahoe-LAFS Support ================== All Tahoe-LAFS client nodes can run a frontend SFTP server, allowing regular SFTP clients (like ``/usr/bin/sftp``, the ``sshfs`` FUSE plugin, and many -others) to access the file store. They can also run an FTP server, so FTP -clients (like ``/usr/bin/ftp``, ``ncftp``, and others) can too. These -frontends sit at the same level as the web-API interface. +others) to access the file store. -Since Tahoe-LAFS does not use user accounts or passwords, the SFTP/FTP +Since Tahoe-LAFS does not use user accounts or passwords, the SFTP servers must be configured with a way to first authenticate a user (confirm that a prospective client has a legitimate claim to whatever authorities we might grant a particular user), and second to decide what directory cap @@ -173,39 +170,6 @@ clients and with the sshfs filesystem, see wiki:SftpFrontend_ .. _wiki:SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend -Configuring FTP Access -====================== - -To enable the FTP server with an accounts file, add the following lines to -the BASEDIR/tahoe.cfg file:: - - [ftpd] - enabled = true - port = tcp:8021:interface=127.0.0.1 - accounts.file = private/accounts - -The FTP server will listen on the given port number and on the loopback -interface only. The "accounts.file" pathname will be interpreted relative to -the node's BASEDIR. - -To enable the FTP server with an account server instead, provide the URL of -that server in an "accounts.url" directive:: - - [ftpd] - enabled = true - port = tcp:8021:interface=127.0.0.1 - accounts.url = https://example.com/login - -You can provide both accounts.file and accounts.url, although it probably -isn't very useful except for testing. - -FTP provides no security, and so your password or caps could be eavesdropped -if you connect to the FTP server remotely. The examples above include -":interface=127.0.0.1" in the "port" option, which causes the server to only -accept connections from localhost. - -Public key authentication is not supported for FTP. - Dependencies ============ @@ -216,7 +180,7 @@ separately: debian puts it in the "python-twisted-conch" package. Immutable and Mutable Files =========================== -All files created via SFTP (and FTP) are immutable files. However, files can +All files created via SFTP are immutable files. However, files can only be created in writeable directories, which allows the directory entry to be relinked to a different file. Normally, when the path of an immutable file is opened for writing by SFTP, the directory entry is relinked to another @@ -256,18 +220,3 @@ See also wiki:SftpFrontend_. .. _ticket #1059: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1059 .. _ticket #1089: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1089 - -Known Issues in the FTP Frontend --------------------------------- - -Mutable files are not supported by the FTP frontend (`ticket #680`_). - -Non-ASCII filenames are not supported by FTP (`ticket #682`_). - -The FTP frontend sometimes fails to report errors, for example if an upload -fails because it does meet the "servers of happiness" threshold (`ticket -#1081`_). - -.. _ticket #680: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/680 -.. _ticket #682: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/682 -.. _ticket #1081: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1081 diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index 99fa44979..417109213 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -2157,7 +2157,7 @@ When modifying the file, be careful to update it atomically, otherwise a request may arrive while the file is only halfway written, and the partial file may be incorrectly parsed. -The blacklist is applied to all access paths (including SFTP, FTP, and CLI +The blacklist is applied to all access paths (including SFTP and CLI operations), not just the web-API. The blacklist also applies to directories. If a directory is blacklisted, the gateway will refuse access to both that directory and any child files/directories underneath it, when accessed via diff --git a/docs/helper.rst b/docs/helper.rst index 0fcdf4601..55d302cac 100644 --- a/docs/helper.rst +++ b/docs/helper.rst @@ -122,7 +122,7 @@ Who should consider using a Helper? * clients who experience problems with TCP connection fairness: if other programs or machines in the same home are getting less than their fair share of upload bandwidth. If the connection is being shared fairly, then - a Tahoe upload that is happening at the same time as a single FTP upload + a Tahoe upload that is happening at the same time as a single SFTP upload should get half the bandwidth. * clients who have been given the helper.furl by someone who is running a Helper and is willing to let them use it diff --git a/docs/known_issues.rst b/docs/known_issues.rst index e040ffaf6..98bd1b35d 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -23,7 +23,7 @@ Known Issues in Tahoe-LAFS v1.10.3, released 30-Mar-2016 * `Disclosure of file through embedded hyperlinks or JavaScript in that file`_ * `Command-line arguments are leaked to other local users`_ * `Capabilities may be leaked to web browser phishing filter / "safe browsing" servers`_ - * `Known issues in the FTP and SFTP frontends`_ + * `Known issues in the SFTP frontend`_ * `Traffic analysis based on sizes of files/directories, storage indices, and timing`_ * `Privacy leak via Google Chart API link in map-update timing web page`_ @@ -213,8 +213,8 @@ To disable the filter in Chrome: ---- -Known issues in the FTP and SFTP frontends ------------------------------------------- +Known issues in the SFTP frontend +--------------------------------- These are documented in :doc:`frontends/FTP-and-SFTP` and on `the SftpFrontend page`_ on the wiki. diff --git a/docs/running.rst b/docs/running.rst index 6d82a97f2..82b0443f9 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -207,10 +207,10 @@ create a new directory and lose the capability to it, then you cannot access that directory ever again. -The SFTP and FTP frontends --------------------------- +The SFTP frontend +----------------- -You can access your Tahoe-LAFS grid via any SFTP_ or FTP_ client. See +You can access your Tahoe-LAFS grid via any SFTP_ client. See :doc:`frontends/FTP-and-SFTP` for how to set this up. On most Unix platforms, you can also use SFTP to plug Tahoe-LAFS into your computer's local filesystem via ``sshfs``, but see the `FAQ about performance @@ -220,7 +220,6 @@ The SftpFrontend_ page on the wiki has more information about using SFTP with Tahoe-LAFS. .. _SFTP: https://en.wikipedia.org/wiki/SSH_file_transfer_protocol -.. _FTP: https://en.wikipedia.org/wiki/File_Transfer_Protocol .. _FAQ about performance problems: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/FAQ#Q23_FUSE .. _SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend diff --git a/newsfragments/3577.minor b/newsfragments/3577.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3582.minor b/newsfragments/3582.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3583.removed b/newsfragments/3583.removed new file mode 100644 index 000000000..a3fce48be --- /dev/null +++ b/newsfragments/3583.removed @@ -0,0 +1 @@ +FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead. \ No newline at end of file diff --git a/newsfragments/3587.minor b/newsfragments/3587.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3587.minor @@ -0,0 +1 @@ + diff --git a/setup.py b/setup.py index 32581e293..952a921bc 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,8 @@ install_requires = [ # version of cryptography will *really* be installed. "cryptography >= 2.6", - # * We need Twisted 10.1.0 for the FTP frontend in order for - # Twisted's FTP server to support asynchronous close. # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server # rekeying bug - # * The FTP frontend depends on Twisted >= 11.1.0 for - # filepath.Permissions # * The SFTP frontend and manhole depend on the conch extra. However, we # can't explicitly declare that without an undesirable dependency on gmpy, # as explained in ticket #2740. diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 1ee507117..b7e1d0956 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os diff --git a/src/allmydata/client.py b/src/allmydata/client.py index bd744fe6a..f5e603490 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -86,12 +86,6 @@ _client_config = configutil.ValidConfiguration( "shares.total", "storage.plugins", ), - "ftpd": ( - "accounts.file", - "accounts.url", - "enabled", - "port", - ), "storage": ( "debug_discard", "enabled", @@ -656,7 +650,6 @@ class _Client(node.Node, pollmixin.PollMixin): raise ValueError("config error: helper is enabled, but tub " "is not listening ('tub.port=' is empty)") self.init_helper() - self.init_ftp_server() self.init_sftp_server() # If the node sees an exit_trigger file, it will poll every second to see @@ -1032,18 +1025,6 @@ class _Client(node.Node, pollmixin.PollMixin): ) ws.setServiceParent(self) - def init_ftp_server(self): - if self.config.get_config("ftpd", "enabled", False, boolean=True): - accountfile = self.config.get_config("ftpd", "accounts.file", None) - if accountfile: - accountfile = self.config.get_config_path(accountfile) - accounturl = self.config.get_config("ftpd", "accounts.url", None) - ftp_portstr = self.config.get_config("ftpd", "port", "8021") - - from allmydata.frontends import ftpd - s = ftpd.FTPServer(self, accountfile, accounturl, ftp_portstr) - s.setServiceParent(self) - def init_sftp_server(self): if self.config.get_config("sftpd", "enabled", False, boolean=True): accountfile = self.config.get_config("sftpd", "accounts.file", None) diff --git a/src/allmydata/frontends/ftpd.py b/src/allmydata/frontends/ftpd.py deleted file mode 100644 index 0b18df85b..000000000 --- a/src/allmydata/frontends/ftpd.py +++ /dev/null @@ -1,340 +0,0 @@ -from six import ensure_str - -from types import NoneType - -from zope.interface import implementer -from twisted.application import service, strports -from twisted.internet import defer -from twisted.internet.interfaces import IConsumer -from twisted.cred import portal -from twisted.python import filepath -from twisted.protocols import ftp - -from allmydata.interfaces import IDirectoryNode, ExistingChildError, \ - NoSuchChildError -from allmydata.immutable.upload import FileHandle -from allmydata.util.fileutil import EncryptedTemporaryFile -from allmydata.util.assertutil import precondition - -@implementer(ftp.IReadFile) -class ReadFile(object): - def __init__(self, node): - self.node = node - def send(self, consumer): - d = self.node.read(consumer) - return d # when consumed - -@implementer(IConsumer) -class FileWriter(object): - - def registerProducer(self, producer, streaming): - if not streaming: - raise NotImplementedError("Non-streaming producer not supported.") - # we write the data to a temporary file, since Tahoe can't do - # streaming upload yet. - self.f = EncryptedTemporaryFile() - return None - - def unregisterProducer(self): - # the upload actually happens in WriteFile.close() - pass - - def write(self, data): - self.f.write(data) - -@implementer(ftp.IWriteFile) -class WriteFile(object): - - def __init__(self, parent, childname, convergence): - self.parent = parent - self.childname = childname - self.convergence = convergence - - def receive(self): - self.c = FileWriter() - return defer.succeed(self.c) - - def close(self): - u = FileHandle(self.c.f, self.convergence) - d = self.parent.add_file(self.childname, u) - return d - - -class NoParentError(Exception): - pass - -# filepath.Permissions was added in Twisted-11.1.0, which we require. Twisted -# <15.0.0 expected an int, and only does '&' on it. Twisted >=15.0.0 expects -# a filepath.Permissions. This satisfies both. - -class IntishPermissions(filepath.Permissions): - def __init__(self, statModeInt): - self._tahoe_statModeInt = statModeInt - filepath.Permissions.__init__(self, statModeInt) - def __and__(self, other): - return self._tahoe_statModeInt & other - -@implementer(ftp.IFTPShell) -class Handler(object): - def __init__(self, client, rootnode, username, convergence): - self.client = client - self.root = rootnode - self.username = username - self.convergence = convergence - - def makeDirectory(self, path): - d = self._get_root(path) - d.addCallback(lambda root_and_path: - self._get_or_create_directories(root_and_path[0], root_and_path[1])) - return d - - def _get_or_create_directories(self, node, path): - if not IDirectoryNode.providedBy(node): - # unfortunately it is too late to provide the name of the - # blocking directory in the error message. - raise ftp.FileExistsError("cannot create directory because there " - "is a file in the way") - if not path: - return defer.succeed(node) - d = node.get(path[0]) - def _maybe_create(f): - f.trap(NoSuchChildError) - return node.create_subdirectory(path[0]) - d.addErrback(_maybe_create) - d.addCallback(self._get_or_create_directories, path[1:]) - return d - - def _get_parent(self, path): - # fire with (parentnode, childname) - path = [unicode(p) for p in path] - if not path: - raise NoParentError - childname = path[-1] - d = self._get_root(path) - def _got_root(root_and_path): - (root, path) = root_and_path - if not path: - raise NoParentError - return root.get_child_at_path(path[:-1]) - d.addCallback(_got_root) - def _got_parent(parent): - return (parent, childname) - d.addCallback(_got_parent) - return d - - def _remove_thing(self, path, must_be_directory=False, must_be_file=False): - d = defer.maybeDeferred(self._get_parent, path) - def _convert_error(f): - f.trap(NoParentError) - raise ftp.PermissionDeniedError("cannot delete root directory") - d.addErrback(_convert_error) - def _got_parent(parent_and_childname): - (parent, childname) = parent_and_childname - d = parent.get(childname) - def _got_child(child): - if must_be_directory and not IDirectoryNode.providedBy(child): - raise ftp.IsNotADirectoryError("rmdir called on a file") - if must_be_file and IDirectoryNode.providedBy(child): - raise ftp.IsADirectoryError("rmfile called on a directory") - return parent.delete(childname) - d.addCallback(_got_child) - d.addErrback(self._convert_error) - return d - d.addCallback(_got_parent) - return d - - def removeDirectory(self, path): - return self._remove_thing(path, must_be_directory=True) - - def removeFile(self, path): - return self._remove_thing(path, must_be_file=True) - - def rename(self, fromPath, toPath): - # the target directory must already exist - d = self._get_parent(fromPath) - def _got_from_parent(fromparent_and_childname): - (fromparent, childname) = fromparent_and_childname - d = self._get_parent(toPath) - d.addCallback(lambda toparent_and_tochildname: - fromparent.move_child_to(childname, - toparent_and_tochildname[0], toparent_and_tochildname[1], - overwrite=False)) - return d - d.addCallback(_got_from_parent) - d.addErrback(self._convert_error) - return d - - def access(self, path): - # we allow access to everything that exists. We are required to raise - # an error for paths that don't exist: FTP clients (at least ncftp) - # uses this to decide whether to mkdir or not. - d = self._get_node_and_metadata_for_path(path) - d.addErrback(self._convert_error) - d.addCallback(lambda res: None) - return d - - def _convert_error(self, f): - if f.check(NoSuchChildError): - childname = f.value.args[0].encode("utf-8") - msg = "'%s' doesn't exist" % childname - raise ftp.FileNotFoundError(msg) - if f.check(ExistingChildError): - msg = f.value.args[0].encode("utf-8") - raise ftp.FileExistsError(msg) - return f - - def _get_root(self, path): - # return (root, remaining_path) - path = [unicode(p) for p in path] - if path and path[0] == "uri": - d = defer.maybeDeferred(self.client.create_node_from_uri, - str(path[1])) - d.addCallback(lambda root: (root, path[2:])) - else: - d = defer.succeed((self.root,path)) - return d - - def _get_node_and_metadata_for_path(self, path): - d = self._get_root(path) - def _got_root(root_and_path): - (root,path) = root_and_path - if path: - return root.get_child_and_metadata_at_path(path) - else: - return (root,{}) - d.addCallback(_got_root) - return d - - def _populate_row(self, keys, childnode_and_metadata): - (childnode, metadata) = childnode_and_metadata - values = [] - isdir = bool(IDirectoryNode.providedBy(childnode)) - for key in keys: - if key == "size": - if isdir: - value = 0 - else: - value = childnode.get_size() or 0 - elif key == "directory": - value = isdir - elif key == "permissions": - # Twisted-14.0.2 (and earlier) expected an int, and used it - # in a rendering function that did (mode & NUMBER). - # Twisted-15.0.0 expects a - # twisted.python.filepath.Permissions , and calls its - # .shorthand() method. This provides both. - value = IntishPermissions(0o600) - elif key == "hardlinks": - value = 1 - elif key == "modified": - # follow sftpd convention (i.e. linkmotime in preference to mtime) - if "linkmotime" in metadata.get("tahoe", {}): - value = metadata["tahoe"]["linkmotime"] - else: - value = metadata.get("mtime", 0) - elif key == "owner": - value = self.username - elif key == "group": - value = self.username - else: - value = "??" - values.append(value) - return values - - def stat(self, path, keys=()): - # for files only, I think - d = self._get_node_and_metadata_for_path(path) - def _render(node_and_metadata): - (node, metadata) = node_and_metadata - assert not IDirectoryNode.providedBy(node) - return self._populate_row(keys, (node,metadata)) - d.addCallback(_render) - d.addErrback(self._convert_error) - return d - - def list(self, path, keys=()): - # the interface claims that path is a list of unicodes, but in - # practice it is not - d = self._get_node_and_metadata_for_path(path) - def _list(node_and_metadata): - (node, metadata) = node_and_metadata - if IDirectoryNode.providedBy(node): - return node.list() - return { path[-1]: (node, metadata) } # need last-edge metadata - d.addCallback(_list) - def _render(children): - results = [] - for (name, childnode) in children.iteritems(): - # the interface claims that the result should have a unicode - # object as the name, but it fails unless you give it a - # bytestring - results.append( (name.encode("utf-8"), - self._populate_row(keys, childnode) ) ) - return results - d.addCallback(_render) - d.addErrback(self._convert_error) - return d - - def openForReading(self, path): - d = self._get_node_and_metadata_for_path(path) - d.addCallback(lambda node_and_metadata: ReadFile(node_and_metadata[0])) - d.addErrback(self._convert_error) - return d - - def openForWriting(self, path): - path = [unicode(p) for p in path] - if not path: - raise ftp.PermissionDeniedError("cannot STOR to root directory") - childname = path[-1] - d = self._get_root(path) - def _got_root(root_and_path): - (root, path) = root_and_path - if not path: - raise ftp.PermissionDeniedError("cannot STOR to root directory") - return root.get_child_at_path(path[:-1]) - d.addCallback(_got_root) - def _got_parent(parent): - return WriteFile(parent, childname, self.convergence) - d.addCallback(_got_parent) - return d - -from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme - - -@implementer(portal.IRealm) -class Dispatcher(object): - def __init__(self, client): - self.client = client - - def requestAvatar(self, avatarID, mind, interface): - assert interface == ftp.IFTPShell - rootnode = self.client.create_node_from_uri(avatarID.rootcap) - convergence = self.client.convergence - s = Handler(self.client, rootnode, avatarID.username, convergence) - def logout(): pass - return (interface, s, None) - - -class FTPServer(service.MultiService): - def __init__(self, client, accountfile, accounturl, ftp_portstr): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) - service.MultiService.__init__(self) - - r = Dispatcher(client) - p = portal.Portal(r) - - if accountfile: - c = AccountFileChecker(self, accountfile) - p.registerChecker(c) - if accounturl: - c = AccountURLChecker(self, accounturl) - p.registerChecker(c) - if not accountfile and not accounturl: - # we could leave this anonymous, with just the /uri/CAP form - raise NeedRootcapLookupScheme("must provide some translation") - - f = ftp.FTPFactory(p) - # strports requires a native string. - ftp_portstr = ensure_str(ftp_portstr) - s = strports.service(ftp_portstr, f) - s.setServiceParent(self) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 342fe4af1..63a5ceaaa 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -51,7 +51,6 @@ from allmydata.nodemaker import ( NodeMaker, ) from allmydata.node import OldConfigError, UnescapedHashError, create_node_dir -from allmydata.frontends.auth import NeedRootcapLookupScheme from allmydata import client from allmydata.storage_client import ( StorageClientConfig, @@ -424,88 +423,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir) self.failUnlessReallyEqual(w.staticdir, expected) - # TODO: also test config options for SFTP. - - @defer.inlineCallbacks - def test_ftp_create(self): - """ - configuration for sftpd results in it being started - """ - root = FilePath(self.mktemp()) - root.makedirs() - accounts = root.child(b"sftp-accounts") - accounts.touch() - - data = FilePath(__file__).sibling(b"data") - privkey = data.child(b"openssh-rsa-2048.txt") - pubkey = data.child(b"openssh-rsa-2048.pub.txt") - - basedir = u"client.Basic.test_ftp_create" - create_node_dir(basedir, "testing") - with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: - f.write(( - '[sftpd]\n' - 'enabled = true\n' - 'accounts.file = {}\n' - 'host_pubkey_file = {}\n' - 'host_privkey_file = {}\n' - ).format(accounts.path, pubkey.path, privkey.path)) - - client_node = yield client.create_client( - basedir, - ) - sftp = client_node.getServiceNamed("frontend:sftp") - self.assertIs(sftp.parent, client_node) - - - @defer.inlineCallbacks - def test_ftp_auth_keyfile(self): - """ - ftpd accounts.file is parsed properly - """ - basedir = u"client.Basic.test_ftp_auth_keyfile" - os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), - (BASECONFIG + - "[ftpd]\n" - "enabled = true\n" - "port = tcp:0:interface=127.0.0.1\n" - "accounts.file = private/accounts\n")) - os.mkdir(os.path.join(basedir, "private")) - fileutil.write(os.path.join(basedir, "private", "accounts"), "\n") - c = yield client.create_client(basedir) # just make sure it can be instantiated - del c - - @defer.inlineCallbacks - def test_ftp_auth_url(self): - """ - ftpd accounts.url is parsed properly - """ - basedir = u"client.Basic.test_ftp_auth_url" - os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), - (BASECONFIG + - "[ftpd]\n" - "enabled = true\n" - "port = tcp:0:interface=127.0.0.1\n" - "accounts.url = http://0.0.0.0/\n")) - c = yield client.create_client(basedir) # just make sure it can be instantiated - del c - - @defer.inlineCallbacks - def test_ftp_auth_no_accountfile_or_url(self): - """ - ftpd requires some way to look up accounts - """ - basedir = u"client.Basic.test_ftp_auth_no_accountfile_or_url" - os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), - (BASECONFIG + - "[ftpd]\n" - "enabled = true\n" - "port = tcp:0:interface=127.0.0.1\n")) - with self.assertRaises(NeedRootcapLookupScheme): - yield client.create_client(basedir) + # TODO: also test config options for SFTP. See Git history for deleted FTP + # tests that could be used as basis for these tests. @defer.inlineCallbacks def _storage_dir_test(self, basedir, storage_path, expected_path): diff --git a/src/allmydata/test/test_ftp.py b/src/allmydata/test/test_ftp.py deleted file mode 100644 index 4eddef440..000000000 --- a/src/allmydata/test/test_ftp.py +++ /dev/null @@ -1,106 +0,0 @@ - -from twisted.trial import unittest - -from allmydata.frontends import ftpd -from allmydata.immutable import upload -from allmydata.mutable import publish -from allmydata.test.no_network import GridTestMixin -from allmydata.test.common_util import ReallyEqualMixin - -class Handler(GridTestMixin, ReallyEqualMixin, unittest.TestCase): - """ - This is a no-network unit test of ftpd.Handler and the abstractions - it uses. - """ - - FALL_OF_BERLIN_WALL = 626644800 - TURN_OF_MILLENIUM = 946684800 - - def _set_up(self, basedir, num_clients=1, num_servers=10): - self.basedir = "ftp/" + basedir - self.set_up_grid(num_clients=num_clients, num_servers=num_servers, - oneshare=True) - - self.client = self.g.clients[0] - self.username = "alice" - self.convergence = "" - - d = self.client.create_dirnode() - def _created_root(node): - self.root = node - self.root_uri = node.get_uri() - self.handler = ftpd.Handler(self.client, self.root, self.username, - self.convergence) - d.addCallback(_created_root) - return d - - def _set_metadata(self, name, metadata): - """Set metadata for `name', avoiding MetadataSetter's timestamp reset - behavior.""" - def _modifier(old_contents, servermap, first_time): - children = self.root._unpack_contents(old_contents) - children[name] = (children[name][0], metadata) - return self.root._pack_contents(children) - - return self.root._node.modify(_modifier) - - def _set_up_tree(self): - # add immutable file at root - immutable = upload.Data("immutable file contents", None) - d = self.root.add_file(u"immutable", immutable) - - # `mtime' and `linkmotime' both set - md_both = {'mtime': self.FALL_OF_BERLIN_WALL, - 'tahoe': {'linkmotime': self.TURN_OF_MILLENIUM}} - d.addCallback(lambda _: self._set_metadata(u"immutable", md_both)) - - # add link to root from root - d.addCallback(lambda _: self.root.set_node(u"loop", self.root)) - - # `mtime' set, but no `linkmotime' - md_just_mtime = {'mtime': self.FALL_OF_BERLIN_WALL, 'tahoe': {}} - d.addCallback(lambda _: self._set_metadata(u"loop", md_just_mtime)) - - # add mutable file at root - mutable = publish.MutableData("mutable file contents") - d.addCallback(lambda _: self.client.create_mutable_file(mutable)) - d.addCallback(lambda node: self.root.set_node(u"mutable", node)) - - # neither `mtime' nor `linkmotime' set - d.addCallback(lambda _: self._set_metadata(u"mutable", {})) - - return d - - def _compareDirLists(self, actual, expected): - actual_list = sorted(actual) - expected_list = sorted(expected) - - self.failUnlessReallyEqual(len(actual_list), len(expected_list), - "%r is wrong length, expecting %r" % ( - actual_list, expected_list)) - for (a, b) in zip(actual_list, expected_list): - (name, meta) = a - (expected_name, expected_meta) = b - self.failUnlessReallyEqual(name, expected_name) - self.failUnlessReallyEqual(meta, expected_meta) - - def test_list(self): - keys = ("size", "directory", "permissions", "hardlinks", "modified", - "owner", "group", "unexpected") - d = self._set_up("list") - - d.addCallback(lambda _: self._set_up_tree()) - d.addCallback(lambda _: self.handler.list("", keys=keys)) - - expected_root = [ - ('loop', - [0, True, ftpd.IntishPermissions(0o600), 1, self.FALL_OF_BERLIN_WALL, 'alice', 'alice', '??']), - ('immutable', - [23, False, ftpd.IntishPermissions(0o600), 1, self.TURN_OF_MILLENIUM, 'alice', 'alice', '??']), - ('mutable', - # timestamp should be 0 if no timestamp metadata is present - [0, False, ftpd.IntishPermissions(0o600), 1, 0, 'alice', 'alice', '??'])] - - d.addCallback(lambda root: self._compareDirLists(root, expected_root)) - - return d diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index e680acd04..12a04a6eb 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -1,6 +1,16 @@ """ Tests for ``allmydata.webish``. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from uuid import ( uuid4, @@ -96,7 +106,7 @@ class TahoeLAFSRequestTests(SyncTestCase): ]) self._fields_test( b"POST", - {b"content-type": b"multipart/form-data; boundary={}".format(boundary)}, + {b"content-type": b"multipart/form-data; boundary=" + bytes(boundary, 'ascii')}, form_data.encode("ascii"), AfterPreprocessing( lambda fs: { @@ -105,8 +115,8 @@ class TahoeLAFSRequestTests(SyncTestCase): in fs.keys() }, Equals({ - b"foo": b"bar", - b"baz": b"some file contents", + "foo": "bar", + "baz": b"some file contents", }), ), ) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index db2ceed03..14db70735 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -27,6 +27,7 @@ PORTED_MODULES = [ "allmydata.__main__", "allmydata._auto_deps", "allmydata._monkeypatch", + "allmydata.blacklist", "allmydata.codec", "allmydata.crypto", "allmydata.crypto.aes", @@ -112,6 +113,7 @@ PORTED_MODULES = [ "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", + "allmydata.webish", ] PORTED_TEST_MODULES = [ @@ -184,6 +186,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", - "allmydata.test.web.test_util", "allmydata.test.web.test_status", + "allmydata.test.web.test_util", + "allmydata.test.web.test_webish", ] diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index f32f56714..e90fa573a 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str import re, time, tempfile @@ -65,18 +77,24 @@ class TahoeLAFSRequest(Request, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - if self.method == 'POST': + if self.method == b'POST': # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. - self.fields = FieldStorage( - self.content, - { - name.lower(): value[-1] - for (name, value) - in self.requestHeaders.getAllRawHeaders() - }, - environ={'REQUEST_METHOD': 'POST'}) + + headers = { + ensure_str(name.lower()): ensure_str(value[-1]) + for (name, value) + in self.requestHeaders.getAllRawHeaders() + } + + if 'content-length' not in headers: + # Python 3's cgi module would really, really like us to set Content-Length. + self.content.seek(0, 2) + headers['content-length'] = str(self.content.tell()) + self.content.seek(0) + + self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'}) self.content.seek(0) self._tahoeLAFSSecurityPolicy()