#531: implement an SFTP frontend. Mostly works, still lots of debug messages. Still needs tests and auth-by-pubkey in accounts.file

This commit is contained in:
Brian Warner 2008-11-04 18:00:22 -07:00
parent 27bb9b88a5
commit 9f908de9e2
3 changed files with 651 additions and 0 deletions

74
docs/sftp.txt Normal file
View File

@ -0,0 +1,74 @@
= Tahoe SFTP Frontend =
All Tahoe client nodes can run a frontend SFTP server, allowing regular SFTP
clients to access the virtual filesystem.
Since Tahoe does not use user accounts or passwords, the FTP server must be
configured with a way to translate a username (and either a password or
public key) into a root directory cap. Two mechanisms are provided. The first
is a simple flat file with one account per line. The second is an HTTP-based
login mechanism, backed by simple PHP script and a database. The latter form
is used by allmydata.com to provide secure access to customer rootcaps.
The SFTP server must also be given a public/private host keypair.
== Configuring a Keypair ==
First, generate a keypair for your server:
% cd BASEDIR
% ssh-keygen -f private/ssh_host_rsa_key
You will then use the following lines in the tahoe.cfg file:
[sftpd]
sftp.host_pubkey_file = private/ssh_host_rsa_key.pub
sftp.host_privkey_file = private/ssh_host_rsa_key
== Configuring an Account File ==
To configure the first form, create a file (probably in
BASEDIR/private/sftp.accounts) in which each non-comment/non-blank line is a
space-separated line of (USERNAME, PASSWORD/PUBKEY, ROOTCAP), like so:
[TODO: the PUBKEY form is not yet supported]
% cat BASEDIR/private/sftp.accounts
# This is a password file, (username, password/pubkey, rootcap)
alice password URI:DIR2:ioej8xmzrwilg772gzj4fhdg7a:wtiizszzz2rgmczv4wl6bqvbv33ag4kvbr6prz3u6w3geixa6m6a
bob sekrit URI:DIR2:6bdmeitystckbl9yqlw7g56f4e:serp5ioqxnh34mlbmzwvkp3odehsyrr7eytt5f64we3k9hhcrcja
carol ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv2xHRVBoXnwxHLzthRD1wOWtyZ08b8n9cMZfJ58CBdBwAYP2NVNXc0XjRvswm5hnnAO+jyWPVNpXJjm9XllzYhODSNtSN+TXuJlUjhzA/T+ZwdgsgSAeHuuMQBoWt4Qc9HV6rHCdAeMhcnyqm6Q0sRAsfA/wfwiIgbvE7+cWpFa2anB6WeAnvK8+dMN0nvnkPE7GNyf/WFR1Ffuh9ifKdRB6yDNp17bQAqA3OWSFjch6fGPhp94y4g2jmTHlEUTyVsilgGqvGOutOVYnmOMnFijugU1Vu33G39GGzXWla6+fXwTk/oiVPiCYD7A7WFKes3nqMg8iVN6a6sxujrhnHQ== warner@fluxx URI:DIR2:6bdmeitystckbl9yqlw7g56f4e:serp5ioqxnh34mlbmzwvkp3odehsyrr7eytt5f64we3k9hhcrcja
Note that if the second word of the line is "ssh-rsa" or "ssh-dss", the rest
of the line is parsed differently, so users cannot have a password equal to
either of these strings.
Then add the following lines to the BASEDIR/tahoe.cfg file:
[sftpd]
enabled = true
sftp.port = 8022
sftp.host_pubkey_file = private/ssh_host_rsa_key.pub
sftp.host_privkey_file = private/ssh_host_rsa_key
sftp.accounts.file = private/sftp.accounts
The SFTP server will listen on the given port number. The sftp.accounts.file
pathname will be interpreted relative to the node's BASEDIR.
== Configuring an Account Server ==
Determine the URL of the account server, say https://example.com/login . Then
add the following lines to BASEDIR/tahoe.cfg:
[sftpd]
enabled = true
sftp.port = 8022
sftp.host_pubkey_file = private/ssh_host_rsa_key.pub
sftp.host_privkey_file = private/ssh_host_rsa_key
sftp.accounts.url = https://example.com/login
== Dependencies ==
The Tahoe SFTP server requires the Twisted "Conch" component, which itself
requires the pycrypto package (note that pycrypto is distinct from the
pycryptopp that Tahoe uses).

View File

@ -79,6 +79,7 @@ class Client(node.Node, pollmixin.PollMixin):
self.init_key_gen(key_gen_furl)
# ControlServer and Helper are attached after Tub startup
self.init_ftp_server()
self.init_sftp_server()
hotline_file = os.path.join(self.basedir,
self.SUICIDE_PREVENTION_HOTLINE_FILE)
@ -270,6 +271,19 @@ class Client(node.Node, pollmixin.PollMixin):
s = ftpd.FTPServer(self, accountfile, accounturl, ftp_portstr)
s.setServiceParent(self)
def init_sftp_server(self):
if self.get_config("sftpd", "enabled", False, boolean=True):
accountfile = self.get_config("sftpd", "sftp.accounts.file", None)
accounturl = self.get_config("sftpd", "sftp.accounts.url", None)
sftp_portstr = self.get_config("sftpd", "sftp.port", "8022")
pubkey_file = self.get_config("sftpd", "sftp.host_pubkey_file")
privkey_file = self.get_config("sftpd", "sftp.host_privkey_file")
from allmydata import sftpd
s = sftpd.SFTPServer(self, accountfile, accounturl,
sftp_portstr, pubkey_file, privkey_file)
s.setServiceParent(self)
def _check_hotline(self, hotline_file):
if os.path.exists(hotline_file):
mtime = os.stat(hotline_file)[stat.ST_MTIME]

563
src/allmydata/sftpd.py Normal file
View File

@ -0,0 +1,563 @@
import os
import tempfile
from zope.interface import implements
from twisted.python import components
from twisted.application import service, strports
from twisted.internet import defer
from twisted.internet.interfaces import IConsumer
from twisted.conch.ssh import factory, keys, session
from twisted.conch.interfaces import ISFTPServer, ISFTPFile, IConchUser
from twisted.conch.avatar import ConchUser
from twisted.conch.openssh_compat import primes
from twisted.conch import ls
from twisted.cred import error, portal, checkers, credentials
from twisted.web.client import getPage
from allmydata.interfaces import IDirectoryNode, ExistingChildError, \
NoSuchChildError
from allmydata.immutable.upload import FileHandle
from allmydata.util import base32
class MemoryConsumer:
implements(IConsumer)
def __init__(self):
self.chunks = []
self.done = False
def registerProducer(self, p, streaming):
if streaming:
# call resumeProducing once to start things off
p.resumeProducing()
else:
while not self.done:
p.resumeProducing()
def write(self, data):
self.chunks.append(data)
def unregisterProducer(self):
self.done = True
def download_to_data(n, offset=0, size=None):
d = n.read(MemoryConsumer(), offset, size)
d.addCallback(lambda mc: "".join(mc.chunks))
return d
class ReadFile:
implements(ISFTPFile)
def __init__(self, node):
self.node = node
def readChunk(self, offset, length):
d = download_to_data(self.node, offset, length)
def _got(data):
return data
d.addCallback(_got)
return d
def close(self):
pass
def getAttrs(self):
print "GETATTRS(file)"
raise NotImplementedError
def setAttrs(self, attrs):
print "SETATTRS(file)", attrs
raise NotImplementedError
class WriteFile:
implements(ISFTPFile)
def __init__(self, parent, childname, convergence):
self.parent = parent
self.childname = childname
self.convergence = convergence
self.f = tempfile.TemporaryFile()
def writeChunk(self, offset, data):
self.f.seek(offset)
self.f.write(data)
def close(self):
u = FileHandle(self.f, self.convergence)
d = self.parent.add_file(self.childname, u)
return d
def getAttrs(self):
print "GETATTRS(file)"
raise NotImplementedError
def setAttrs(self, attrs):
print "SETATTRS(file)", attrs
raise NotImplementedError
class NoParentError(Exception):
pass
class PermissionError(Exception):
pass
from twisted.conch.ssh.filetransfer import FileTransferServer, SFTPError, \
FX_NO_SUCH_FILE, FX_FILE_ALREADY_EXISTS, FX_OP_UNSUPPORTED, \
FX_PERMISSION_DENIED
from twisted.conch.ssh.filetransfer import FXF_READ, FXF_WRITE, FXF_APPEND, FXF_CREAT, FXF_TRUNC, FXF_EXCL
class SFTPUser(ConchUser):
def __init__(self, client, rootnode, username, convergence):
ConchUser.__init__(self)
self.channelLookup["session"] = session.SSHSession
self.subsystemLookup["sftp"] = FileTransferServer
self.client = client
self.root = rootnode
self.username = username
self.convergence = convergence
class StoppableList:
def __init__(self, items):
self.items = items
def __iter__(self):
for i in self.items:
yield i
def close(self):
pass
class FakeStat:
pass
class SFTPHandler:
implements(ISFTPServer)
def __init__(self, user):
print "Creating SFTPHandler from", user
self.client = user.client
self.root = user.root
self.username = user.username
self.convergence = user.convergence
def gotVersion(self, otherVersion, extData):
return {}
def openFile(self, filename, flags, attrs):
f = "|".join([(flags & FXF_READ) and "FXF_READ" or "",
(flags & FXF_WRITE) and "FXF_WRITE" or "",
(flags & FXF_APPEND) and "FXF_APPEND" or "",
(flags & FXF_CREAT) and "FXF_CREAT" or "",
(flags & FXF_TRUNC) and "FXF_TRUNC" or "",
(flags & FXF_EXCL) and "FXF_EXCL" or "",
])
print "OPENFILE", filename, flags, f, attrs
# this is used for both reading and writing.
# createPlease = False
# exclusive = False
# openFlags = 0
#
# if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0:
# openFlags = os.O_RDONLY
# if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0:
# createPlease = True
# openFlags = os.O_WRONLY
# if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ:
# createPlease = True
# openFlags = os.O_RDWR
# if flags & FXF_APPEND == FXF_APPEND:
# createPlease = True
# openFlags |= os.O_APPEND
# if flags & FXF_CREAT == FXF_CREAT:
# createPlease = True
# openFlags |= os.O_CREAT
# if flags & FXF_TRUNC == FXF_TRUNC:
# openFlags |= os.O_TRUNC
# if flags & FXF_EXCL == FXF_EXCL:
# exclusive = True
# /usr/bin/sftp 'get' gives us FXF_READ, while 'put' on a new file
# gives FXF_WRITE,FXF_CREAT,FXF_TRUNC . I'm guessing that 'put' on an
# existing file gives the same.
path = self._convert_sftp_path(filename)
if flags & FXF_READ:
if flags & FXF_WRITE:
raise NotImplementedError
d = self._get_node_and_metadata_for_path(path)
d.addCallback(lambda (node,metadata): ReadFile(node))
d.addErrback(self._convert_error)
return d
if flags & FXF_WRITE:
if not (flags & FXF_CREAT) or not (flags & FXF_TRUNC):
raise NotImplementedError
if not path:
raise PermissionError("cannot STOR to root directory")
childname = path[-1]
d = self._get_root(path)
def _got_root((root, path)):
if not path:
raise PermissionError("cannot STOR to root directory")
return root.get_child_at_path(path[:-1])
d.addCallback(_got_root)
def _got_parent(parent):
return WriteFile(parent, childname, self.convergence)
d.addCallback(_got_parent)
return d
raise NotImplementedError
def removeFile(self, path):
print "REMOVEFILE", path
path = self._convert_sftp_path(path)
return self._remove_thing(path, must_be_file=True)
def renameFile(self, oldpath, newpath):
print "RENAMEFILE", oldpath, newpath
fromPath = self._convert_sftp_path(oldpath)
toPath = self._convert_sftp_path(newpath)
# the target directory must already exist
d = self._get_parent(fromPath)
def _got_from_parent( (fromparent, childname) ):
d = self._get_parent(toPath)
d.addCallback(lambda (toparent, tochildname):
fromparent.move_child_to(childname,
toparent, tochildname,
overwrite=False))
return d
d.addCallback(_got_from_parent)
d.addErrback(self._convert_error)
return d
def makeDirectory(self, path, attrs):
print "MAKEDIRECTORY", path, attrs
# TODO: extract attrs["mtime"], use it to set the parent metadata.
# Maybe also copy attrs["ext_*"] .
path = self._convert_sftp_path(path)
d = self._get_root(path)
d.addCallback(lambda (root,path):
self._get_or_create_directories(root, path))
return d
def _get_or_create_directories(self, node, path):
if not IDirectoryNode.providedBy(node):
# unfortunately it is too late to provide the name of the
# blocking directory in the error message.
raise ExistingChildError("cannot create directory because there "
"is a file in the way") # close enough
if not path:
return defer.succeed(node)
d = node.get(path[0])
def _maybe_create(f):
f.trap(NoSuchChildError)
return node.create_empty_directory(path[0])
d.addErrback(_maybe_create)
d.addCallback(self._get_or_create_directories, path[1:])
return d
def removeDirectory(self, path):
print "REMOVEDIRECTORY", path
path = self._convert_sftp_path(path)
return self._remove_thing(path, must_be_directory=True)
def _remove_thing(self, path, must_be_directory=False, must_be_file=False):
d = defer.maybeDeferred(self._get_parent, path)
def _convert_error(f):
f.trap(NoParentError)
raise PermissionError("cannot delete root directory")
d.addErrback(_convert_error)
def _got_parent( (parent, childname) ):
d = parent.get(childname)
def _got_child(child):
if must_be_directory and not IDirectoryNode.providedBy(child):
raise RuntimeError("rmdir called on a file")
if must_be_file and IDirectoryNode.providedBy(child):
raise RuntimeError("rmfile called on a directory")
return parent.delete(childname)
d.addCallback(_got_child)
d.addErrback(self._convert_error)
return d
d.addCallback(_got_parent)
return d
def openDirectory(self, path):
print "OPENDIRECTORY", path
path = self._convert_sftp_path(path)
d = self._get_node_and_metadata_for_path(path)
d.addCallback(lambda (dirnode,metadata): dirnode.list())
def _render(children):
results = []
for filename, (node, metadata) in children.iteritems():
s = FakeStat()
if IDirectoryNode.providedBy(node):
s.st_mode = 040700
s.st_size = 0
else:
s.st_mode = 0100600
s.st_size = node.get_size()
s.st_nlink = 1
s.st_uid = 0
s.st_gid = 0
s.st_mtime = int(metadata.get("mtime", 0))
longname = ls.lsLine(filename.encode("utf-8"), s)
attrs = self._populate_attrs(node, metadata)
results.append( (filename.encode("utf-8"), longname, attrs) )
return StoppableList(results)
d.addCallback(_render)
return d
def getAttrs(self, path, followLinks):
print "GETATTRS", path, followLinks
# from ftp.stat
d = self._get_node_and_metadata_for_path(self._convert_sftp_path(path))
def _render((node,metadata)):
return self._populate_attrs(node, metadata)
d.addCallback(_render)
d.addErrback(self._convert_error)
def _done(res):
print " DONE", res
return res
d.addBoth(_done)
return d
def _convert_sftp_path(self, pathstring):
assert pathstring[0] == "/"
pathstring = pathstring.strip("/")
if pathstring == "":
path = []
else:
path = pathstring.split("/")
print "CONVERT", pathstring, path
path = [unicode(p) for p in path]
return path
def _get_node_and_metadata_for_path(self, path):
d = self._get_root(path)
def _got_root((root,path)):
print "ROOT", root
print "PATH", path
if path:
return root.get_child_and_metadata_at_path(path)
else:
return (root,{})
d.addCallback(_got_root)
return d
def _get_root(self, path):
# return (root, remaining_path)
path = [unicode(p) for p in path]
if path and path[0] == "uri":
d = defer.maybeDeferred(self.client.create_node_from_uri,
str(path[1]))
d.addCallback(lambda root: (root, path[2:]))
else:
d = defer.succeed((self.root,path))
return d
def _populate_attrs(self, childnode, metadata):
attrs = {}
attrs["uid"] = 1000
attrs["gid"] = 1000
attrs["atime"] = 0
attrs["mtime"] = int(metadata.get("mtime", 0))
isdir = bool(IDirectoryNode.providedBy(childnode))
if isdir:
attrs["size"] = 1
# the permissions must have the extra bits (040000 or 0100000),
# otherwise the client will not call openDirectory
attrs["permissions"] = 040700 # S_IFDIR
else:
attrs["size"] = childnode.get_size()
attrs["permissions"] = 0100600 # S_IFREG
return attrs
def _convert_error(self, f):
if f.check(NoSuchChildError):
childname = f.value.args[0].encode("utf-8")
raise SFTPError(FX_NO_SUCH_FILE, childname)
if f.check(ExistingChildError):
msg = f.value.args[0].encode("utf-8")
raise SFTPError(FX_FILE_ALREADY_EXISTS, msg)
if f.check(PermissionError):
raise SFTPError(FX_PERMISSION_DENIED, str(f.value))
if f.check(NotImplementedError):
raise SFTPError(FX_OP_UNSUPPORTED, str(f.value))
return f
def setAttrs(self, path, attrs):
print "SETATTRS", path, attrs
# ignored
return None
def readLink(self, path):
print "READLINK", path
raise NotImplementedError
def makeLink(self, linkPath, targetPath):
print "MAKELINK", linkPath, targetPath
raise NotImplementedError
def extendedRequest(self, extendedName, extendedData):
print "EXTENDEDREQUEST", extendedName, extendedData
# client 'df' command requires 'statvfs@openssh.com' extension
raise NotImplementedError
def realPath(self, path):
print "REALPATH", path
if path == ".":
return "/"
return path
def _get_parent(self, path):
# fire with (parentnode, childname)
path = [unicode(p) for p in path]
if not path:
raise NoParentError
childname = path[-1]
d = self._get_root(path)
def _got_root((root, path)):
if not path:
raise NoParentError
return root.get_child_at_path(path[:-1])
d.addCallback(_got_root)
def _got_parent(parent):
return (parent, childname)
d.addCallback(_got_parent)
return d
class FTPAvatarID:
def __init__(self, username, rootcap):
self.username = username
self.rootcap = rootcap
class AccountFileChecker:
implements(checkers.ICredentialsChecker)
credentialInterfaces = (credentials.IUsernamePassword,
credentials.IUsernameHashedPassword)
def __init__(self, client, accountfile):
self.client = client
self.passwords = {}
self.pubkeys = {}
self.rootcaps = {}
for line in open(os.path.expanduser(accountfile), "r"):
line = line.strip()
if line.startswith("#") or not line:
continue
name, passwd, rest = line.split(None, 2)
if passwd in ("ssh-dss", "ssh-rsa"):
bits = rest.split()
keystring = " ".join(bits[-1])
rootcap = bits[-1]
self.pubkeys[name] = keystring
else:
self.passwords[name] = passwd
rootcap = rest
self.rootcaps[name] = rootcap
def _cbPasswordMatch(self, matched, username):
if matched:
return FTPAvatarID(username, self.rootcaps[username])
raise error.UnauthorizedLogin
def requestAvatarId(self, credentials):
if credentials.username in self.passwords:
d = defer.maybeDeferred(credentials.checkPassword,
self.passwords[credentials.username])
d.addCallback(self._cbPasswordMatch, str(credentials.username))
return d
return defer.fail(error.UnauthorizedLogin())
class AccountURLChecker:
implements(checkers.ICredentialsChecker)
credentialInterfaces = (credentials.IUsernamePassword,)
def __init__(self, client, auth_url):
self.client = client
self.auth_url = auth_url
def _cbPasswordMatch(self, rootcap, username):
return FTPAvatarID(username, rootcap)
def post_form(self, username, password):
sepbase = base32.b2a(os.urandom(4))
sep = "--" + sepbase
form = []
form.append(sep)
fields = {"action": "authenticate",
"email": username,
"passwd": password,
}
for name, value in fields.iteritems():
form.append('Content-Disposition: form-data; name="%s"' % name)
form.append('')
assert isinstance(value, str)
form.append(value)
form.append(sep)
form[-1] += "--"
body = "\r\n".join(form) + "\r\n"
headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
}
return getPage(self.auth_url, method="POST",
postdata=body, headers=headers,
followRedirect=True, timeout=30)
def _parse_response(self, res):
rootcap = res.strip()
if rootcap == "0":
raise error.UnauthorizedLogin
return rootcap
def requestAvatarId(self, credentials):
# construct a POST to the login form. While this could theoretically
# be done with something like the stdlib 'email' package, I can't
# figure out how, so we just slam together a form manually.
d = self.post_form(credentials.username, credentials.password)
d.addCallback(self._parse_response)
d.addCallback(self._cbPasswordMatch, str(credentials.username))
return d
# if you have an SFTPUser, and you want something that provides ISFTPServer,
# then you get SFTPHandler(user)
components.registerAdapter(SFTPHandler, SFTPUser, ISFTPServer)
class Dispatcher:
implements(portal.IRealm)
def __init__(self, client):
self.client = client
def requestAvatar(self, avatarID, mind, interface):
assert interface == IConchUser
rootnode = self.client.create_node_from_uri(avatarID.rootcap)
convergence = self.client.convergence
s = SFTPUser(self.client, rootnode, avatarID.username, convergence)
def logout(): pass
return (interface, s, logout)
class SFTPServer(service.MultiService):
def __init__(self, client, accountfile, accounturl,
sftp_portstr, pubkey_file, privkey_file):
service.MultiService.__init__(self)
if accountfile:
c = AccountFileChecker(self, accountfile)
elif accounturl:
c = AccountURLChecker(self, accounturl)
else:
# we could leave this anonymous, with just the /uri/CAP form
raise RuntimeError("must provide some translation")
r = Dispatcher(client)
p = portal.Portal(r)
p.registerChecker(c)
pubkey = keys.Key.fromFile(pubkey_file)
privkey = keys.Key.fromFile(privkey_file)
class SSHFactory(factory.SSHFactory):
publicKeys = {pubkey.sshType(): pubkey}
privateKeys = {privkey.sshType(): privkey}
def getPrimes(self):
try:
# if present, this enables diffie-hellman-group-exchange
return primes.parseModuliFile("/etc/ssh/moduli")
except IOError:
return None
f = SSHFactory()
f.portal = p
s = strports.service(sftp_portstr, f)
s.setServiceParent(self)