diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 64b09c076..42c2ff384 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -16,6 +16,7 @@ from allmydata.control import ControlServer from allmydata.introducer.client import IntroducerClient from allmydata.util import hashutil, base32, pollmixin, log, keyutil, idlib from allmydata.util.encodingutil import get_filesystem_encoding +from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.stats import StatsProvider @@ -450,8 +451,8 @@ class Client(node.Node, pollmixin.PollMixin): from allmydata.webish import WebishServer nodeurl_path = os.path.join(self.basedir, "node.url") - staticdir = self.get_config("node", "web.static", "public_html") - staticdir = os.path.expanduser(staticdir) + staticdir_config = self.get_config("node", "web.static", "public_html").decode("utf-8") + staticdir = abspath_expanduser_unicode(staticdir_config, base=self.basedir) ws = WebishServer(self, webport, nodeurl_path, staticdir) self.add_service(ws) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index fa7fd3794..712a888a0 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -1,4 +1,5 @@ import os + from zope.interface import implements from twisted.web.client import getPage from twisted.internet import defer @@ -7,6 +8,8 @@ from twisted.conch import error as conch_error from twisted.conch.ssh import keys from allmydata.util import base32 +from allmydata.util.fileutil import abspath_expanduser_unicode + class NeedRootcapLookupScheme(Exception): """Accountname+Password-based access schemes require some kind of @@ -28,7 +31,7 @@ class AccountFileChecker: self.passwords = {} self.pubkeys = {} self.rootcaps = {} - for line in open(os.path.expanduser(accountfile), "r"): + for line in open(abspath_expanduser_unicode(accountfile), "r"): line = line.strip() if line.startswith("#") or not line: continue diff --git a/src/allmydata/frontends/drop_upload.py b/src/allmydata/frontends/drop_upload.py index 6f25625e0..0e2b48b49 100644 --- a/src/allmydata/frontends/drop_upload.py +++ b/src/allmydata/frontends/drop_upload.py @@ -1,5 +1,5 @@ -import os, sys +import sys from twisted.internet import defer from twisted.python.filepath import FilePath @@ -9,6 +9,7 @@ from foolscap.api import eventually from allmydata.interfaces import IDirectoryNode from allmydata.util.encodingutil import quote_output, get_filesystem_encoding +from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.immutable.upload import FileName @@ -19,7 +20,7 @@ class DropUploader(service.MultiService): service.MultiService.__init__(self) try: - local_dir_u = os.path.expanduser(local_dir_utf8.decode('utf-8')) + local_dir_u = abspath_expanduser_unicode(local_dir_utf8.decode('utf-8')) if sys.platform == "win32": local_dir = local_dir_u else: diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 43a026114..7031c3af6 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -6,7 +6,7 @@ from foolscap.api import Referenceable import allmydata from allmydata import node from allmydata.util import log, rrefutil -from allmydata.util.encodingutil import get_filesystem_encoding +from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.introducer.interfaces import \ RIIntroducerPublisherAndSubscriberService_v2 from allmydata.introducer.common import convert_announcement_v1_to_v2, \ @@ -21,7 +21,7 @@ class IntroducerNode(node.Node): NODETYPE = "introducer" GENERATED_FILES = ['introducer.furl'] - def __init__(self, basedir="."): + def __init__(self, basedir=u"."): node.Node.__init__(self, basedir) self.read_config() self.init_introducer() @@ -33,8 +33,8 @@ class IntroducerNode(node.Node): introducerservice = IntroducerService(self.basedir) self.add_service(introducerservice) - old_public_fn = os.path.join(self.basedir, "introducer.furl").encode(get_filesystem_encoding()) - private_fn = os.path.join(self.basedir, "private", "introducer.furl").encode(get_filesystem_encoding()) + old_public_fn = os.path.join(self.basedir, u"introducer.furl") + private_fn = os.path.join(self.basedir, u"private", u"introducer.furl") if os.path.exists(old_public_fn): if os.path.exists(private_fn): @@ -62,9 +62,9 @@ class IntroducerNode(node.Node): self.log("init_web(webport=%s)", args=(webport,), umid="2bUygA") from allmydata.webish import IntroducerWebishServer - nodeurl_path = os.path.join(self.basedir, "node.url") - staticdir = self.get_config("node", "web.static", "public_html") - staticdir = os.path.expanduser(staticdir) + nodeurl_path = os.path.join(self.basedir, u"node.url") + config_staticdir = self.get_config("node", "web.static", "public_html").decode('utf-8') + staticdir = abspath_expanduser_unicode(config_staticdir, base=self.basedir) ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir) self.add_service(ws) diff --git a/src/allmydata/manhole.py b/src/allmydata/manhole.py index 1d6121dc0..63b62a545 100644 --- a/src/allmydata/manhole.py +++ b/src/allmydata/manhole.py @@ -1,8 +1,8 @@ # this is adapted from my code in Buildbot -warner -import os.path import binascii, base64 + from twisted.python import log from twisted.application import service, strports from twisted.cred import checkers, portal @@ -12,6 +12,8 @@ from twisted.internet import protocol from zope.interface import implements +from allmydata.util.fileutil import precondition_abspath + # makeTelnetProtocol and _TelnetRealm are for the TelnetManhole class makeTelnetProtocol: @@ -63,7 +65,8 @@ class AuthorizedKeysChecker(conchc.SSHPublicKeyDatabase): """ def __init__(self, authorized_keys_file): - self.authorized_keys_file = os.path.expanduser(authorized_keys_file) + precondition_abspath(authorized_keys_file) + self.authorized_keys_file = authorized_keys_file def checkKey(self, credentials): f = open(self.authorized_keys_file) @@ -244,14 +247,12 @@ class AuthorizedKeysManhole(_BaseManhole): 'tcp:12345:interface=127.0.0.1'. Bare integers are treated as a simple tcp port. - @param keyfile: the name of a file (relative to the buildmaster's - basedir) that contains SSH public keys of authorized - users, one per line. This is the exact same format - as used by sshd in ~/.ssh/authorized_keys . + @param keyfile: the path of a file that contains SSH public keys of + authorized users, one per line. This is the exact + same format as used by sshd in ~/.ssh/authorized_keys . + The path should be absolute. """ - # TODO: expanduser this, and make it relative to the buildmaster's - # basedir self.keyfile = keyfile c = AuthorizedKeysChecker(keyfile) _BaseManhole.__init__(self, port, c) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index d29ad22ce..710ce7d50 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -93,12 +93,11 @@ class Node(service.MultiService): iputil.increase_rlimits() def init_tempdir(self): - local_tempdir_utf8 = "tmp" # default is NODEDIR/tmp/ - tempdir = self.get_config("node", "tempdir", local_tempdir_utf8).decode('utf-8') - tempdir = os.path.join(self.basedir, tempdir) + tempdir_config = self.get_config("node", "tempdir", "tmp").decode('utf-8') + tempdir = abspath_expanduser_unicode(tempdir_config, base=self.basedir) if not os.path.exists(tempdir): fileutil.make_dirs(tempdir) - tempfile.tempdir = abspath_expanduser_unicode(tempdir) + tempfile.tempdir = tempdir # this should cause twisted.web.http (which uses # tempfile.TemporaryFile) to put large request bodies in the given # directory. Without this, the default temp dir is usually /tmp/, @@ -220,11 +219,12 @@ class Node(service.MultiService): def setup_ssh(self): ssh_port = self.get_config("node", "ssh.port", "") if ssh_port: - ssh_keyfile = self.get_config("node", "ssh.authorized_keys_file").decode('utf-8') + ssh_keyfile_config = self.get_config("node", "ssh.authorized_keys_file").decode('utf-8') + ssh_keyfile = abspath_expanduser_unicode(ssh_keyfile_config, base=self.basedir) from allmydata import manhole - m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile.encode(get_filesystem_encoding())) + m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile) m.setServiceParent(self) - self.log("AuthorizedKeysManhole listening on %s" % ssh_port) + self.log("AuthorizedKeysManhole listening on %s" % (ssh_port,)) def get_app_versions(self): # TODO: merge this with allmydata.get_package_versions diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index b0e4e6dec..e240c9ecd 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -2,7 +2,7 @@ import os.path, re, fnmatch from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ DEFAULT_ALIAS, BaseOptions -from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output +from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_local_unicode_path NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?") @@ -140,15 +140,11 @@ class GetOptions(FilesystemOptions): # tahoe get FOO bar # write to local file # tahoe get tahoe:FOO bar # same + if arg2 == "-": + arg2 = None + self.from_file = argv_to_unicode(arg1) - - if arg2: - self.to_file = argv_to_unicode(arg2) - else: - self.to_file = None - - if self.to_file == "-": - self.to_file = None + self.to_file = None if arg2 is None else argv_to_abspath(arg2) def getSynopsis(self): return "Usage: %s [global-opts] get [options] REMOTE_FILE LOCAL_FILE" % (self.command_name,) @@ -180,17 +176,11 @@ class PutOptions(FilesystemOptions): def parseArgs(self, arg1=None, arg2=None): # see Examples below - if arg1 is not None and arg2 is not None: - self.from_file = argv_to_unicode(arg1) - self.to_file = argv_to_unicode(arg2) - elif arg1 is not None and arg2 is None: - self.from_file = argv_to_unicode(arg1) # might be "-" - self.to_file = None - else: - self.from_file = None - self.to_file = None - if self.from_file == u"-": - self.from_file = None + if arg1 == "-": + arg1 = None + + self.from_file = None if arg1 is None else argv_to_abspath(arg1) + self.to_file = None if arg2 is None else argv_to_unicode(arg2) if self['format']: if self['format'].upper() not in ("SDMF", "MDMF", "CHK"): @@ -347,7 +337,7 @@ class BackupOptions(FilesystemOptions): self['exclude'] = set() def parseArgs(self, localdir, topath): - self.from_dir = argv_to_unicode(localdir) + self.from_dir = argv_to_abspath(localdir) self.to_dir = argv_to_unicode(topath) def getSynopsis(self): @@ -368,7 +358,7 @@ class BackupOptions(FilesystemOptions): try: exclude_file = file(abs_filepath) except: - raise BackupConfigurationError('Error opening exclude file %s.' % quote_output(abs_filepath)) + raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath)) try: for line in exclude_file: self.opt_exclude(line) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 21481e35d..55cb3f08c 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -3,7 +3,8 @@ import os, sys, urllib import codecs from twisted.python import usage from allmydata.util.assertutil import precondition -from allmydata.util.encodingutil import unicode_to_url, quote_output, argv_to_abspath +from allmydata.util.encodingutil import unicode_to_url, quote_output, \ + quote_local_unicode_path, argv_to_abspath from allmydata.util.fileutil import abspath_expanduser_unicode @@ -40,7 +41,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" - % get_default_nodedir()], + % quote_local_unicode_path(_default_nodedir)], ] def parseArgs(self, basedir=None): @@ -196,5 +197,6 @@ def get_alias(aliases, path_unicode, default): return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] def escape_path(path): + # this always returns bytes, specifically US-ASCII, valid URL characters segments = path.split("/") return "/".join([urllib.quote(unicode_to_url(s)) for s in segments]) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 8ef82eca9..fedd69851 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -645,15 +645,15 @@ def find_shares(options): /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2 """ from allmydata.storage.server import si_a2b, storage_index_to_dir - from allmydata.util.encodingutil import listdir_unicode + from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path out = options.stdout sharedir = storage_index_to_dir(si_a2b(options.si_s)) for d in options.nodedirs: - d = os.path.join(d, "storage/shares", sharedir) + d = os.path.join(d, "storage", "shares", sharedir) if os.path.exists(d): for shnum in listdir_unicode(d): - print >>out, os.path.join(d, shnum) + print >>out, quote_local_unicode_path(os.path.join(d, shnum), quotemarks=False) return 0 @@ -832,7 +832,7 @@ def catalog_shares(options): err = options.stderr now = time.time() for d in options.nodedirs: - d = os.path.join(d, "storage/shares") + d = os.path.join(d, "storage", "shares") try: abbrevs = listdir_unicode(d) except EnvironmentError: diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 085967079..1a6de258e 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -6,7 +6,7 @@ from twisted.python import usage from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, startstop_node, cli, keygen, stats_gatherer, admin -from allmydata.util.encodingutil import quote_output, get_io_encoding +from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding def GROUP(s): # Usage.parseOptions compares argv[1] against command[0], so it will @@ -25,7 +25,7 @@ NODEDIR_HELP = ("Specify which Tahoe node directory should be used. The " "' which contains the mapping from alias name to root " "dirnode URI.") if _default_nodedir: - NODEDIR_HELP += " [default for most commands: " + quote_output(_default_nodedir) + "]" + NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]" class Options(usage.Options): # unit tests can override these to point at StringIO instances diff --git a/src/allmydata/scripts/startstop_node.py b/src/allmydata/scripts/startstop_node.py index 09cdc9373..45b7fd112 100644 --- a/src/allmydata/scripts/startstop_node.py +++ b/src/allmydata/scripts/startstop_node.py @@ -4,7 +4,7 @@ from allmydata.scripts.common import BasedirOptions from twisted.scripts import twistd from twisted.python import usage from allmydata.util import fileutil -from allmydata.util.encodingutil import listdir_unicode, quote_output +from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path class StartOptions(BasedirOptions): @@ -92,13 +92,14 @@ def identify_node_type(basedir): def start(config, out=sys.stdout, err=sys.stderr): basedir = config['basedir'] - print >>out, "STARTING", quote_output(basedir) + quoted_basedir = quote_local_unicode_path(basedir) + print >>out, "STARTING", quoted_basedir if not os.path.isdir(basedir): - print >>err, "%s does not look like a directory at all" % quote_output(basedir) + print >>err, "%s does not look like a directory at all" % quoted_basedir return 1 nodetype = identify_node_type(basedir) if not nodetype: - print >>err, "%s is not a recognizable node directory" % quote_output(basedir) + print >>err, "%s is not a recognizable node directory" % quoted_basedir return 1 # Now prepare to turn into a twistd process. This os.chdir is the point # of no return. @@ -108,7 +109,7 @@ def start(config, out=sys.stdout, err=sys.stderr): and "--nodaemon" not in config.twistd_args and "--syslog" not in config.twistd_args and "--logfile" not in config.twistd_args): - fileutil.make_dirs(os.path.join(basedir, "logs")) + fileutil.make_dirs(os.path.join(basedir, u"logs")) twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")]) twistd_args.extend(config.twistd_args) twistd_args.append("StartTahoeNode") # point at our StartTahoeNodePlugin @@ -154,17 +155,18 @@ def start(config, out=sys.stdout, err=sys.stderr): else: verb = "starting" - print >>out, "%s node in %s" % (verb, basedir) + print >>out, "%s node in %s" % (verb, quoted_basedir) twistd.runApp(twistd_config) # we should only reach here if --nodaemon or equivalent was used return 0 def stop(config, out=sys.stdout, err=sys.stderr): basedir = config['basedir'] - print >>out, "STOPPING", quote_output(basedir) - pidfile = os.path.join(basedir, "twistd.pid") + quoted_basedir = quote_local_unicode_path(basedir) + print >>out, "STOPPING", quoted_basedir + pidfile = os.path.join(basedir, u"twistd.pid") if not os.path.exists(pidfile): - print >>err, "%s does not look like a running node directory (no twistd.pid)" % quote_output(basedir) + print >>err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir # we define rc=2 to mean "nothing is running, but it wasn't me who # stopped it" return 2 diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index ef3da34b4..f12b31713 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -10,9 +10,9 @@ from allmydata.scripts.common_http import do_http, HTTPError, format_http_error from allmydata.util import time_format from allmydata.scripts import backupdb from allmydata.util.encodingutil import listdir_unicode, quote_output, \ - to_str, FilenameEncodingError, unicode_to_url + quote_local_unicode_path, to_str, FilenameEncodingError, unicode_to_url from allmydata.util.assertutil import precondition -from allmydata.util.fileutil import abspath_expanduser_unicode +from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath def get_local_metadata(path): @@ -160,10 +160,11 @@ class BackerUpper: print >>self.options.stderr, msg def process(self, localpath): - precondition(isinstance(localpath, unicode), localpath) + precondition_abspath(localpath) # returns newdircap - self.verboseprint("processing %s" % quote_output(localpath)) + quoted_path = quote_local_unicode_path(localpath) + self.verboseprint("processing %s" % (quoted_path,)) create_contents = {} # childname -> (type, rocap, metadata) compare_contents = {} # childname -> rocap @@ -171,11 +172,11 @@ class BackerUpper: children = listdir_unicode(localpath) except EnvironmentError: self.directories_skipped += 1 - self.warn("WARNING: permission denied on directory %s" % quote_output(localpath)) + self.warn("WARNING: permission denied on directory %s" % (quoted_path,)) children = [] except FilenameEncodingError: self.directories_skipped += 1 - self.warn("WARNING: could not list directory %s due to a filename encoding error" % quote_output(localpath)) + self.warn("WARNING: could not list directory %s due to a filename encoding error" % (quoted_path,)) children = [] for child in self.options.filter_listdir(children): @@ -197,17 +198,17 @@ class BackerUpper: compare_contents[child] = childcap except EnvironmentError: self.files_skipped += 1 - self.warn("WARNING: permission denied on file %s" % quote_output(childpath)) + self.warn("WARNING: permission denied on file %s" % quote_local_unicode_path(childpath)) else: self.files_skipped += 1 if os.path.islink(childpath): - self.warn("WARNING: cannot backup symlink %s" % quote_output(childpath)) + self.warn("WARNING: cannot backup symlink %s" % quote_local_unicode_path(childpath)) else: - self.warn("WARNING: cannot backup special file %s" % quote_output(childpath)) + self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath)) must_create, r = self.check_backupdb_directory(compare_contents) if must_create: - self.verboseprint(" creating directory for %s" % quote_output(localpath)) + self.verboseprint(" creating directory for %s" % quote_local_unicode_path(localpath)) newdircap = mkdir(create_contents, self.options) assert isinstance(newdircap, str) if r: @@ -215,7 +216,7 @@ class BackerUpper: self.directories_created += 1 return newdircap else: - self.verboseprint(" re-using old directory for %s" % quote_output(localpath)) + self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(localpath)) self.directories_reused += 1 return r.was_created() @@ -288,16 +289,16 @@ class BackerUpper: # This function will raise an IOError exception when called on an unreadable file def upload(self, childpath): - precondition(isinstance(childpath, unicode), childpath) + precondition_abspath(childpath) - #self.verboseprint("uploading %s.." % quote_output(childpath)) + #self.verboseprint("uploading %s.." % quote_local_unicode_path(childpath)) metadata = get_local_metadata(childpath) # we can use the backupdb here must_upload, bdb_results = self.check_backupdb_file(childpath) if must_upload: - self.verboseprint("uploading %s.." % quote_output(childpath)) + self.verboseprint("uploading %s.." % quote_local_unicode_path(childpath)) infileobj = open(childpath, "rb") url = self.options['node-url'] + "uri" resp = do_http("PUT", url, infileobj) @@ -305,7 +306,7 @@ class BackerUpper: raise HTTPError("Error during file PUT", resp) filecap = resp.read().strip() - self.verboseprint(" %s -> %s" % (quote_output(childpath, quotemarks=False), + self.verboseprint(" %s -> %s" % (quote_local_unicode_path(childpath, quotemarks=False), quote_output(filecap, quotemarks=False))) #self.verboseprint(" metadata: %s" % (quote_output(metadata, quotemarks=False),)) @@ -316,7 +317,7 @@ class BackerUpper: return filecap, metadata else: - self.verboseprint("skipping %s.." % quote_output(childpath)) + self.verboseprint("skipping %s.." % quote_local_unicode_path(childpath)) self.files_reused += 1 return bdb_results.was_uploaded(), metadata diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 965aa26f3..756db5bae 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -9,14 +9,15 @@ from allmydata.scripts.common import get_alias, escape_path, \ from allmydata.scripts.common_http import do_http, HTTPError from allmydata import uri from allmydata.util import fileutil -from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, to_str +from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath +from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \ + quote_local_unicode_path, to_str from allmydata.util.assertutil import precondition class MissingSourceError(TahoeError): - def __init__(self, name): - TahoeError.__init__(self, "No such file or directory %s" % quote_output(name)) + def __init__(self, name, quotefn=quote_output): + TahoeError.__init__(self, "No such file or directory %s" % quotefn(name)) def GET_to_file(url): @@ -61,37 +62,34 @@ def make_tahoe_subdirectory(nodeurl, parent_writecap, name): class LocalFileSource: def __init__(self, pathname): - precondition(isinstance(pathname, unicode), pathname) + precondition_abspath(pathname) self.pathname = pathname def need_to_copy_bytes(self): return True def open(self, caps_only): - return open(os.path.expanduser(self.pathname), "rb") - + return open(self.pathname, "rb") class LocalFileTarget: def __init__(self, pathname): - precondition(isinstance(pathname, unicode), pathname) + precondition_abspath(pathname) self.pathname = pathname def put_file(self, inf): fileutil.put_file(self.pathname, inf) - class LocalMissingTarget: def __init__(self, pathname): - precondition(isinstance(pathname, unicode), pathname) + precondition_abspath(pathname) self.pathname = pathname def put_file(self, inf): fileutil.put_file(self.pathname, inf) - class LocalDirectorySource: def __init__(self, progressfunc, pathname): - precondition(isinstance(pathname, unicode), pathname) + precondition_abspath(pathname) self.progressfunc = progressfunc self.pathname = pathname @@ -119,7 +117,7 @@ class LocalDirectorySource: class LocalDirectoryTarget: def __init__(self, progressfunc, pathname): - precondition(isinstance(pathname, unicode), pathname) + precondition_abspath(pathname) self.progressfunc = progressfunc self.pathname = pathname @@ -132,7 +130,6 @@ class LocalDirectoryTarget: children = listdir_unicode(self.pathname) for i,n in enumerate(children): self.progressfunc("examining %d of %d" % (i+1, len(children))) - n = unicode(n) pn = os.path.join(self.pathname, n) if os.path.isdir(pn): child = LocalDirectoryTarget(self.progressfunc, pn) @@ -144,6 +141,7 @@ class LocalDirectoryTarget: self.children[n] = LocalFileTarget(pn) def get_child_target(self, name): + precondition(isinstance(name, unicode), name) if self.children is None: self.populate(False) if name in self.children: @@ -160,6 +158,7 @@ class LocalDirectoryTarget: def set_children(self): pass + class TahoeFileSource: def __init__(self, nodeurl, mutable, writecap, readcap): self.nodeurl = nodeurl @@ -371,6 +370,7 @@ class TahoeDirectoryTarget: def get_child_target(self, name): # return a new target for a named subdirectory of this dir + precondition(isinstance(name, unicode), name) if self.children is None: self.populate(False) if name in self.children: @@ -383,6 +383,7 @@ class TahoeDirectoryTarget: return child def put_file(self, name, inf): + precondition(isinstance(name, unicode), name) url = self.nodeurl + "uri" if not hasattr(inf, "seek"): inf = inf.read() @@ -402,6 +403,7 @@ class TahoeDirectoryTarget: self.new_children[name] = filecap def put_uri(self, name, filecap): + precondition(isinstance(name, unicode), name) self.new_children[name] = filecap def set_children(self): @@ -518,16 +520,23 @@ class Copier: def to_stderr(self, text): print >>self.stderr, text + # FIXME reduce the amount of near-duplicate code between get_target_info + # and get_source_info. + def get_target_info(self, destination_spec): - rootcap, path = get_alias(self.aliases, destination_spec, None) + precondition(isinstance(destination_spec, unicode), destination_spec) + rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None) + path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file - pathname = abspath_expanduser_unicode(path.decode('utf-8')) + pathname = abspath_expanduser_unicode(path) if not os.path.exists(pathname): t = LocalMissingTarget(pathname) elif os.path.isdir(pathname): t = LocalDirectoryTarget(self.progress, pathname) else: + # TODO: should this be _assert? what happens if the target is + # a special file? assert os.path.isfile(pathname), pathname t = LocalFileTarget(pathname) # non-empty else: @@ -559,13 +568,15 @@ class Copier: return t def get_source_info(self, source_spec): - rootcap, path = get_alias(self.aliases, source_spec, None) + precondition(isinstance(source_spec, unicode), source_spec) + rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) + path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file - pathname = abspath_expanduser_unicode(path.decode('utf-8')) + pathname = abspath_expanduser_unicode(path) name = os.path.basename(pathname) if not os.path.exists(pathname): - raise MissingSourceError(source_spec) + raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname) else: @@ -577,9 +588,9 @@ class Copier: name = None if path: url += "/" + escape_path(path) - last_slash = path.rfind("/") + last_slash = path.rfind(u"/") name = path - if last_slash: + if last_slash != -1: name = path[last_slash+1:] resp = do_http("GET", url + "?t=json") @@ -598,8 +609,13 @@ class Copier: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it - if source_spec.rfind('/') != -1: - name = source_spec[source_spec.rfind('/')+1:] + + last_slash = source_spec.rfind(u"/") + if last_slash != -1: + # TODO: this looks funny and redundant with the 'name' + # assignment above. cf #2329 + name = source_spec[last_slash+1:] + t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) return name, t @@ -679,6 +695,7 @@ class Copier: return self.announce_success("files copied") def attach_to_target(self, source, name, target): + precondition(isinstance(name, unicode), name) if target not in self.targetmap: self.targetmap[target] = {} self.targetmap[target][name] = source @@ -686,7 +703,7 @@ class Copier: def assign_targets(self, source, target): # copy everything in the source into the target - assert isinstance(source, (LocalDirectorySource, TahoeDirectorySource)) + precondition(isinstance(source, (LocalDirectorySource, TahoeDirectorySource)), source) for name, child in source.children.items(): if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)): @@ -694,14 +711,12 @@ class Copier: subtarget = target.get_child_target(name) self.assign_targets(child, subtarget) else: - assert isinstance(child, (LocalFileSource, TahoeFileSource)) + precondition(isinstance(child, (LocalFileSource, TahoeFileSource)), child) self.attach_to_target(child, name, target) - - def copy_files_to_target(self, targetmap, target): for name, source in targetmap.items(): - assert isinstance(source, (LocalFileSource, TahoeFileSource)) + precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source) self.copy_file_into(source, name, target) self.files_copied += 1 self.progress("%d/%d files, %d/%d directories" % @@ -723,9 +738,9 @@ class Copier: return 0 def copy_file(self, source, target): - assert isinstance(source, (LocalFileSource, TahoeFileSource)) - assert isinstance(target, (LocalFileTarget, TahoeFileTarget, - LocalMissingTarget, TahoeMissingTarget)) + precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source) + precondition(isinstance(target, (LocalFileTarget, TahoeFileTarget, + LocalMissingTarget, TahoeMissingTarget)), target) if self.need_to_copy_bytes(source, target): # if the target is a local directory, this will just write the # bytes to disk. If it is a tahoe directory, it will upload the @@ -740,8 +755,9 @@ class Copier: return self.announce_success("file linked") def copy_file_into(self, source, name, target): - assert isinstance(source, (LocalFileSource, TahoeFileSource)) - assert isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)) + precondition(isinstance(source, (LocalFileSource, TahoeFileSource)), source) + precondition(isinstance(target, (LocalDirectoryTarget, TahoeDirectoryTarget)), target) + precondition(isinstance(name, unicode), name) if self.need_to_copy_bytes(source, target): # if the target is a local directory, this will just write the # bytes to disk. If it is a tahoe directory, it will upload the diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index 280d8c052..73ef67b45 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -1,5 +1,5 @@ -import os, urllib +import urllib from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error @@ -26,7 +26,7 @@ def get(options): resp = do_http("GET", url) if resp.status in (200, 201,): if to_file: - outf = open(os.path.expanduser(to_file), "wb") + outf = open(to_file, "wb") else: outf = stdout while True: diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index a85539efe..91b71c814 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,7 +1,7 @@ -import os from cStringIO import StringIO import urllib + from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -73,7 +73,7 @@ def put(options): url += "?" + "&".join(queryargs) if from_file: - infileobj = open(os.path.expanduser(from_file), "rb") + infileobj = open(from_file, "rb") else: # do_http() can't use stdin directly: for one thing, we need a # Content-Length field. So we currently must copy it. diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 7db323ba5..22f018124 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -11,8 +11,8 @@ from twisted.application.internet import TimerService from zope.interface import implements from foolscap.api import eventually, DeadReferenceError, Referenceable, Tub -from allmydata.util import log -from allmydata.util.encodingutil import quote_output +from allmydata.util import log, fileutil +from allmydata.util.encodingutil import quote_local_unicode_path from allmydata.interfaces import RIStatsProvider, RIStatsGatherer, IStatsProducer class LoadMonitor(service.MultiService): @@ -246,7 +246,7 @@ class StdOutStatsGatherer(StatsGatherer): class PickleStatsGatherer(StdOutStatsGatherer): # inherit from StdOutStatsGatherer for connect/disconnect notifications - def __init__(self, basedir=".", verbose=True): + def __init__(self, basedir=u".", verbose=True): self.verbose = verbose StatsGatherer.__init__(self, basedir) self.picklefile = os.path.join(basedir, "stats.pickle") @@ -258,7 +258,7 @@ class PickleStatsGatherer(StdOutStatsGatherer): except Exception: print ("Error while attempting to load pickle file %s.\n" "You may need to restore this file from a backup, or delete it if no backup is available.\n" % - quote_output(os.path.abspath(self.picklefile))) + quote_local_unicode_path(self.picklefile)) raise f.close() else: @@ -311,7 +311,7 @@ class StatsGathererService(service.MultiService): def save_portnum(self, junk): portnum = self.listener.getPortnum() portnumfile = os.path.join(self.basedir, 'portnum') - open(portnumfile, 'wb').write('%d\n' % (portnum,)) + fileutil.write(portnumfile, '%d\n' % (portnum,)) def tub_ready(self, ignored): ff = os.path.join(self.basedir, self.furl_file) diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index b61531b1b..2b52f00af 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -5,6 +5,8 @@ from twisted.conch import error as conch_error from twisted.conch.ssh import keys from allmydata.frontends import auth +from allmydata.util.fileutil import abspath_expanduser_unicode + DUMMY_KEY = keys.Key.fromString("""\ -----BEGIN RSA PRIVATE KEY----- @@ -37,7 +39,8 @@ class AccountFileCheckerKeyTests(unittest.TestCase): def setUp(self): self.account_file = filepath.FilePath(self.mktemp()) self.account_file.setContent(DUMMY_ACCOUNTS) - self.checker = auth.AccountFileChecker(None, self.account_file.path) + abspath = abspath_expanduser_unicode(unicode(self.account_file.path)) + self.checker = auth.AccountFileChecker(None, abspath) def test_unknown_user(self): """ diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index 6bc5fc1c6..b39cafc49 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -2509,6 +2509,28 @@ starting copy, 2 files, 1 directories d.addCallback(_check_local_fs) return d + def test_ticket_2027(self): + # This test ensures that tahoe will copy a file from the grid to + # a local directory without a specified file name. + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2027 + self.basedir = "cli/Cp/cp_verbose" + self.set_up_grid() + + # Write a test file, which we'll copy to the grid. + test1_path = os.path.join(self.basedir, "test1") + fileutil.write(test1_path, "test1") + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda ign: + self.do_cli("cp", test1_path, "tahoe:")) + d.addCallback(lambda ign: + self.do_cli("cp", "tahoe:test1", self.basedir)) + def _check(res): + (rc, out, err) = res + self.failUnlessIn("Success: file copied", out, str(res)) + return d + + class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def writeto(self, path, data): @@ -2854,7 +2876,8 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def _check((rc, out, err)): self.failUnlessReallyEqual(rc, 2) foo2 = os.path.join(source, "foo2.txt") - self.failUnlessReallyEqual(err, "WARNING: cannot backup symlink '%s'\n" % foo2) + self.failUnlessIn("WARNING: cannot backup symlink ", err) + self.failUnlessIn(foo2, err) fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt @@ -3779,7 +3802,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): raise return d -class Options(unittest.TestCase): +class Options(ReallyEqualMixin, unittest.TestCase): # this test case only looks at argument-processing and simple stuff. def parse(self, args, stdout=None): @@ -3861,17 +3884,17 @@ class Options(unittest.TestCase): # option after, or a basedir argument after, but none in the wrong # place, and not more than one of the three. o = self.parse(["start"]) - self.failUnlessEqual(o["basedir"], os.path.join(os.path.expanduser("~"), - ".tahoe")) + self.failUnlessReallyEqual(o["basedir"], os.path.join(fileutil.abspath_expanduser_unicode(u"~"), + u".tahoe")) o = self.parse(["start", "here"]) - self.failUnlessEqual(o["basedir"], os.path.abspath("here")) + self.failUnlessReallyEqual(o["basedir"], fileutil.abspath_expanduser_unicode(u"here")) o = self.parse(["start", "--basedir", "there"]) - self.failUnlessEqual(o["basedir"], os.path.abspath("there")) + self.failUnlessReallyEqual(o["basedir"], fileutil.abspath_expanduser_unicode(u"there")) o = self.parse(["--node-directory", "there", "start"]) - self.failUnlessEqual(o["basedir"], os.path.abspath("there")) + self.failUnlessReallyEqual(o["basedir"], fileutil.abspath_expanduser_unicode(u"there")) o = self.parse(["start", "here", "--nodaemon"]) - self.failUnlessEqual(o["basedir"], os.path.abspath("here")) + self.failUnlessReallyEqual(o["basedir"], fileutil.abspath_expanduser_unicode(u"here")) self.failUnlessRaises(usage.UsageError, self.parse, ["--basedir", "there", "start"]) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 531215f61..94bfbdf53 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,4 +1,4 @@ -import os +import os, sys from twisted.trial import unittest from twisted.application import service @@ -6,6 +6,7 @@ import allmydata from allmydata.node import Node, OldConfigError, OldConfigOptionError, MissingConfigEntry, UnescapedHashError from allmydata import client from allmydata.storage_client import StorageFarmBroker +from allmydata.manhole import AuthorizedKeysManhole from allmydata.util import base32, fileutil from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode @@ -68,10 +69,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): fileutil.write(os.path.join(basedir, "debug_discard_storage"), "") e = self.failUnlessRaises(OldConfigError, client.Client, basedir) - self.failUnlessIn(os.path.abspath(os.path.join(basedir, "introducer.furl")), e.args[0]) - self.failUnlessIn(os.path.abspath(os.path.join(basedir, "no_storage")), e.args[0]) - self.failUnlessIn(os.path.abspath(os.path.join(basedir, "readonly_storage")), e.args[0]) - self.failUnlessIn(os.path.abspath(os.path.join(basedir, "debug_discard_storage")), e.args[0]) + abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding()) + self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0]) + self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0]) + self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0]) + self.failUnlessIn(os.path.join(abs_basedir, "debug_discard_storage"), e.args[0]) for oldfile in ['introducer.furl', 'no_storage', 'readonly_storage', 'debug_discard_storage']: @@ -173,6 +175,34 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): "reserved_space = bogus\n") self.failUnlessRaises(ValueError, client.Client, basedir) + def test_web_staticdir(self): + basedir = u"client.Basic.test_web_staticdir" + os.mkdir(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[node]\n" + + "web.port = tcp:0:interface=127.0.0.1\n" + + "web.static = relative\n") + c = client.Client(basedir) + w = c.getServiceNamed("webish") + abs_basedir = fileutil.abspath_expanduser_unicode(basedir) + expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir) + self.failUnlessReallyEqual(w.staticdir, expected) + + def test_manhole_keyfile(self): + basedir = u"client.Basic.test_manhole_keyfile" + os.mkdir(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + BASECONFIG + + "[node]\n" + + "ssh.port = tcp:0:interface=127.0.0.1\n" + + "ssh.authorized_keys_file = relative\n") + c = client.Client(basedir) + m = [s for s in c if isinstance(s, AuthorizedKeysManhole)][0] + abs_basedir = fileutil.abspath_expanduser_unicode(basedir) + expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir) + self.failUnlessReallyEqual(m.keyfile, expected) + def _permute(self, sb, key): return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index abd3d8cb5..926c3659d 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -61,10 +61,11 @@ from mock import patch import os, sys, locale from allmydata.test.common_util import ReallyEqualMixin -from allmydata.util import encodingutil +from allmydata.util import encodingutil, fileutil from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \ - unicode_to_output, quote_output, unicode_platform, listdir_unicode, \ - FilenameEncodingError, get_io_encoding, get_filesystem_encoding, _reload + unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \ + unicode_platform, listdir_unicode, FilenameEncodingError, get_io_encoding, \ + get_filesystem_encoding, _reload from allmydata.dirnode import normalize from twisted.python import usage @@ -274,8 +275,8 @@ class StdlibUnicode(unittest.TestCase): # to lumiere_nfc (on Mac OS X, it will be the NFD equivalent). self.failUnlessIn(lumiere_nfc + ".txt", set([normalize(fname) for fname in filenames])) - expanded = os.path.expanduser("~/" + lumiere_nfc) - self.failIfIn("~", expanded) + expanded = fileutil.expanduser(u"~/" + lumiere_nfc) + self.failIfIn(u"~", expanded) self.failUnless(expanded.endswith(lumiere_nfc), expanded) def test_open_unrepresentable(self): @@ -395,6 +396,19 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): self.test_quote_output_utf8(None) +class QuotePaths(ReallyEqualMixin, unittest.TestCase): + def test_quote_path(self): + self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'") + self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'") + self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), '"foo/\\x0abar"') + self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"') + + self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"), + "'C:\\foo'" if sys.platform == "win32" else "'\\\\?\\C:\\foo'") + self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"), + "'\\\\foo\\bar'" if sys.platform == "win32" else "'\\\\?\\UNC\\foo\\bar'") + + class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' argv = 'lumi\xc3\xa8re' diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 21adb8140..475b04dba 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -28,7 +28,7 @@ class LoggingMultiService(service.MultiService): def log(self, msg, **kw): log.msg(msg, **kw) -class Node(testutil.SignalMixin, unittest.TestCase): +class Node(testutil.SignalMixin, testutil.ReallyEqualMixin, unittest.TestCase): def test_furl(self): basedir = "introducer.IntroducerNode.test_furl" os.mkdir(basedir) @@ -74,6 +74,20 @@ class Node(testutil.SignalMixin, unittest.TestCase): d.addCallback(_check_furl) return d + def test_web_static(self): + basedir = u"introducer.Node.test_web_static" + os.mkdir(basedir) + fileutil.write(os.path.join(basedir, "tahoe.cfg"), + "[node]\n" + + "web.port = tcp:0:interface=127.0.0.1\n" + + "web.static = relative\n") + c = IntroducerNode(basedir) + w = c.getServiceNamed("webish") + abs_basedir = fileutil.abspath_expanduser_unicode(basedir) + expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir) + self.failUnlessReallyEqual(w.staticdir, expected) + + class ServiceMixin: def setUp(self): self.parent = LoggingMultiService() diff --git a/src/allmydata/test/test_mutable.py b/src/allmydata/test/test_mutable.py index fb39af72e..26c7f244d 100644 --- a/src/allmydata/test/test_mutable.py +++ b/src/allmydata/test/test_mutable.py @@ -1,10 +1,13 @@ import os, re, base64 from cStringIO import StringIO + from twisted.trial import unittest from twisted.internet import defer, reactor + from allmydata import uri, client from allmydata.nodemaker import NodeMaker from allmydata.util import base32, consumer, fileutil, mathutil +from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.hashutil import tagged_hash, ssk_writekey_hash, \ ssk_pubkey_fingerprint_hash from allmydata.util.consumer import MemoryConsumer @@ -3110,7 +3113,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ fso = debug.FindSharesOptions() storage_index = base32.b2a(n.get_storage_index()) fso.si_s = storage_index - fso.nodedirs = [unicode(os.path.dirname(os.path.abspath(storedir))) + fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(unicode(storedir))) for (i,ss,storedir) in self.iterate_servers()] fso.stdout = StringIO() diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 2a5ba1997..d907c118a 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,6 +15,8 @@ from allmydata.util import limiter, time_format, pollmixin, cachedir from allmydata.util import statistics, dictutil, pipeline from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans +from allmydata.test.common_util import ReallyEqualMixin + class Base32(unittest.TestCase): def test_b2a_matches_Pythons(self): @@ -370,7 +372,7 @@ class Asserts(unittest.TestCase): m = self.should_assert(f, False, othermsg="message2") self.failUnlessEqual("postcondition: othermsg: 'message2' ", m) -class FileUtil(unittest.TestCase): +class FileUtil(ReallyEqualMixin, unittest.TestCase): def mkdir(self, basedir, path, mode=0777): fn = os.path.join(basedir, path) fileutil.make_dirs(fn, mode) @@ -429,22 +431,6 @@ class FileUtil(unittest.TestCase): fileutil.write_atomically(fn, "two", mode="") # non-binary self.failUnlessEqual(fileutil.read(fn), "two") - def test_open_or_create(self): - basedir = "util/FileUtil/test_open_or_create" - fileutil.make_dirs(basedir) - fn = os.path.join(basedir, "here") - f = fileutil.open_or_create(fn) - f.write("stuff.") - f.close() - f = fileutil.open_or_create(fn) - f.seek(0, os.SEEK_END) - f.write("more.") - f.close() - f = open(fn, "r") - data = f.read() - f.close() - self.failUnlessEqual(data, "stuff.more.") - def test_NamedTemporaryDirectory(self): basedir = "util/FileUtil/test_NamedTemporaryDirectory" fileutil.make_dirs(basedir) @@ -488,7 +474,16 @@ class FileUtil(unittest.TestCase): abspath_cwd = fileutil.abspath_expanduser_unicode(u".") self.failUnless(isinstance(saved_cwd, unicode), saved_cwd) self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd) - self.failUnlessEqual(abspath_cwd, saved_cwd) + if sys.platform == "win32": + self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd)) + else: + self.failUnlessReallyEqual(abspath_cwd, saved_cwd) + + self.failUnlessReallyEqual(fileutil.to_windows_long_path(u"\\\\?\\foo"), u"\\\\?\\foo") + self.failUnlessReallyEqual(fileutil.to_windows_long_path(u"\\\\.\\foo"), u"\\\\.\\foo") + self.failUnlessReallyEqual(fileutil.to_windows_long_path(u"\\\\server\\foo"), u"\\\\?\\UNC\\server\\foo") + self.failUnlessReallyEqual(fileutil.to_windows_long_path(u"C:\\foo"), u"\\\\?\\C:\\foo") + self.failUnlessReallyEqual(fileutil.to_windows_long_path(u"C:\\foo/bar"), u"\\\\?\\C:\\foo\\bar") # adapted from @@ -512,6 +507,34 @@ class FileUtil(unittest.TestCase): finally: os.chdir(saved_cwd) + def test_create_long_path(self): + workdir = u"test_create_long_path" + fileutil.make_dirs(workdir) + long_path = fileutil.abspath_expanduser_unicode(os.path.join(workdir, u'x'*255)) + def _cleanup(): + fileutil.remove(long_path) + self.addCleanup(_cleanup) + + fileutil.write(long_path, "test") + self.failUnless(os.path.exists(long_path)) + self.failUnlessEqual(fileutil.read(long_path), "test") + _cleanup() + self.failIf(os.path.exists(long_path)) + + def test_windows_expanduser(self): + def call_windows_getenv(name): + if name == u"HOMEDRIVE": return u"C:" + if name == u"HOMEPATH": return u"\\Documents and Settings\\\u0100" + self.fail("unexpected argument to call_windows_getenv") + self.patch(fileutil, 'windows_getenv', call_windows_getenv) + + self.failUnlessReallyEqual(fileutil.windows_expanduser(u"~"), os.path.join(u"C:", u"\\Documents and Settings\\\u0100")) + self.failUnlessReallyEqual(fileutil.windows_expanduser(u"~\\foo"), os.path.join(u"C:", u"\\Documents and Settings\\\u0100", u"foo")) + self.failUnlessReallyEqual(fileutil.windows_expanduser(u"~/foo"), os.path.join(u"C:", u"\\Documents and Settings\\\u0100", u"foo")) + self.failUnlessReallyEqual(fileutil.windows_expanduser(u"a"), u"a") + self.failUnlessReallyEqual(fileutil.windows_expanduser(u"a~"), u"a~") + self.failUnlessReallyEqual(fileutil.windows_expanduser(u"a\\~\\foo"), u"a\\~\\foo") + def test_disk_stats(self): avail = fileutil.get_available_space('.', 2**14) if avail == 0: diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 3ceb1a919..feafd8f5c 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -230,6 +230,16 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): def quote_path(path, quotemarks=True): return quote_output("/".join(map(to_str, path)), quotemarks=quotemarks, quote_newlines=True) +def quote_local_unicode_path(path, quotemarks=True): + precondition(isinstance(path, unicode), path) + + if sys.platform == "win32" and path.startswith(u"\\\\?\\"): + path = path[4 :] + if path.startswith(u"UNC\\"): + path = u"\\\\" + path[4 :] + + return quote_output(path, quotemarks=quotemarks, quote_newlines=True) + def unicode_platform(): """ diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 8eb8e9fb6..7bb25e8e7 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -224,12 +224,6 @@ def remove_if_possible(f): except: pass -def open_or_create(fname, binarymode=True): - try: - return open(fname, binarymode and "r+b" or "r+") - except EnvironmentError: - return open(fname, binarymode and "w+b" or "w+") - def du(basedir): size = 0 @@ -269,9 +263,11 @@ def read(path): finally: rf.close() -def put_file(pathname, inf): +def put_file(path, inf): + precondition_abspath(path) + # TODO: create temporary file and move into place? - outf = open(os.path.expanduser(pathname), "wb") + outf = open(path, "wb") try: while True: data = inf.read(32768) @@ -282,6 +278,20 @@ def put_file(pathname, inf): outf.close() +def precondition_abspath(path): + if not isinstance(path, unicode): + raise AssertionError("an abspath must be a Unicode string") + + if sys.platform == "win32": + # This intentionally doesn't view absolute paths starting with a drive specification, or + # paths relative to the current drive, as acceptable. + if not path.startswith("\\\\"): + raise AssertionError("an abspath should be normalized using abspath_expanduser_unicode") + else: + # This intentionally doesn't view the path '~' or paths starting with '~/' as acceptable. + if not os.path.isabs(path): + raise AssertionError("an abspath should be normalized using abspath_expanduser_unicode") + # Work around . This code is adapted from # # with some simplifications. @@ -292,11 +302,20 @@ try: except ImportError: pass -def abspath_expanduser_unicode(path): - """Return the absolute version of a path.""" - assert isinstance(path, unicode), path +def abspath_expanduser_unicode(path, base=None): + """ + Return the absolute version of a path. If 'base' is given and 'path' is relative, + the path will be expanded relative to 'base'. + 'path' must be a Unicode string. 'base', if given, must be a Unicode string + corresponding to an absolute path as returned by a previous call to + abspath_expanduser_unicode. + """ + if not isinstance(path, unicode): + raise AssertionError("paths must be Unicode strings") + if base is not None: + precondition_abspath(base) - path = os.path.expanduser(path) + path = expanduser(path) if _getfullpathname: # On Windows, os.path.isabs will return True for paths without a drive letter, @@ -307,19 +326,50 @@ def abspath_expanduser_unicode(path): pass if not os.path.isabs(path): - path = os.path.join(os.getcwdu(), path) + if base is None: + path = os.path.join(os.getcwdu(), path) + else: + path = os.path.join(base, path) # We won't hit because # there is always at least one Unicode path component. - return os.path.normpath(path) + path = os.path.normpath(path) + + if sys.platform == "win32": + path = to_windows_long_path(path) + + return path + +def to_windows_long_path(path): + # '/' is normally a perfectly valid path component separator in Windows. + # However, when using the "\\?\" syntax it is not recognized, so we + # replace it with '\' here. + path = path.replace(u"/", u"\\") + + # Note that other normalizations such as removing '.' and '..' should + # be done outside this function. + + if path.startswith(u"\\\\?\\") or path.startswith(u"\\\\.\\"): + return path + elif path.startswith(u"\\\\"): + return u"\\\\?\\UNC\\" + path[2 :] + else: + return u"\\\\?\\" + path have_GetDiskFreeSpaceExW = False if sys.platform == "win32": - try: - from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_ulonglong - from ctypes.wintypes import BOOL, DWORD, LPCWSTR + from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_ulonglong, create_unicode_buffer + from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPWSTR + # + GetLastError = WINFUNCTYPE(DWORD)(("GetLastError", windll.kernel32)) + + # + GetEnvironmentVariableW = WINFUNCTYPE(DWORD, LPCWSTR, LPWSTR, DWORD)( + ("GetEnvironmentVariableW", windll.kernel32)) + + try: # PULARGE_INTEGER = POINTER(c_ulonglong) @@ -327,14 +377,57 @@ if sys.platform == "win32": GetDiskFreeSpaceExW = WINFUNCTYPE(BOOL, LPCWSTR, PULARGE_INTEGER, PULARGE_INTEGER, PULARGE_INTEGER)( ("GetDiskFreeSpaceExW", windll.kernel32)) - # - GetLastError = WINFUNCTYPE(DWORD)(("GetLastError", windll.kernel32)) - have_GetDiskFreeSpaceExW = True except Exception: import traceback traceback.print_exc() +def expanduser(path): + # os.path.expanduser is hopelessly broken for Unicode paths on Windows (ticket #1674). + if sys.platform == "win32": + return windows_expanduser(path) + else: + return os.path.expanduser(path) + +def windows_expanduser(path): + if not path.startswith('~'): + return path + home_drive = windows_getenv(u'HOMEDRIVE') + home_path = windows_getenv(u'HOMEPATH') + if path == '~': + return os.path.join(home_drive, home_path) + elif path.startswith('~/') or path.startswith('~\\'): + return os.path.join(home_drive, home_path, path[2 :]) + else: + return path + +def windows_getenv(name): + # Based on , + # with improved error handling. + if not isinstance(name, unicode): + raise AssertionError("name must be Unicode") + + n = GetEnvironmentVariableW(name, None, 0) + if n == 0: + err = GetLastError() + raise OSError("Windows error %d attempting to read size of environment variable %r" + % (err, name)) + elif n < 0: + raise OSError("Unexpected result %d from GetEnvironmentVariableW attempting to read size of environment variable %r" + % (n, name)) + + buf = create_unicode_buffer(u'\0'*n) + retval = GetEnvironmentVariableW(name, buf, n) + if retval == 0: + err = GetLastError() + raise OSError("Windows error %d attempting to read environment variable %r" + % (err, name)) + elif retval != n-1: + raise OSError("Unexpected result %d from GetEnvironmentVariableW attempting to read environment variable %r" + % (n, name)) + + return buf.value + def get_disk_stats(whichdir, reserved_space=0): """Return disk statistics for the storage disk, in the form of a dict with the following fields. diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 813856cb0..e2029feec 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -147,6 +147,7 @@ class WebishServer(service.MultiService): self.site = site = appserver.NevowSite(self.root) self.site.requestFactory = MyRequest self.site.remember(MyExceptionHandler(), inevow.ICanHandleException) + self.staticdir = staticdir # so tests can check if staticdir: self.root.putChild("static", static.File(staticdir)) if re.search(r'^\d', webport):