Merge remote-tracking branch 'origin/master' into 2879.magic-folder-create-and-umask.0

.lgtm.yml

@@ -0,0 +1,22 @@
+extraction:
+  python:
+    after_prepare:
+      - |
+        # https://discuss.lgtm.com/t/determination-of-python-requirements/974/4
+        sed -i 's/\("pyOpenSSL\)/\# Dependency removed for lgtm  (see .lgtm.yml): \1/g' src/allmydata/_auto_deps.py
+
+queries:
+  # This generates spurious errors for calls by interface because of the
+  # zope.interface choice to exclude self from method signatures.  So, turn it
+  # off.
+  - exclude: "py/call/wrong-arguments"
+
+  # The premise of this query is broken.  The errors it produces are nonsense.
+  # There is no such thing as a "procedure" in Python and "None" is not
+  # meaningless.
+  - exclude: "py/procedure-return-value-used"
+
+  # It is true that this query identifies things which are sometimes mistakes.
+  # However, it also identifies things which are entirely valid.  Therefore,
+  # it produces noisy results.
+  - exclude: "py/implicit-string-concatenation-in-list"

misc/coding_tools/make-canary-files.py

@@ -50,7 +50,7 @@ system where Tahoe is installed, or in a source tree with setup.py like this:
  setup.py run_with_pythonpath -p -c 'misc/make-canary-files.py ARGS..'
 """
 
-import os, sha
+import os, hashlib
 from twisted.python import usage
 from allmydata.immutable import upload
 from allmydata.util import base32
@@ -96,7 +96,7 @@ convergence = base32.a2b(convergence_s)
 def get_permuted_peers(key):
     results = []
     for nodeid in nodes:
-        permuted = sha.new(key + nodeid).digest()
+        permuted = hashlib.sha1(key + nodeid).digest()
         results.append((permuted, nodeid))
     results.sort(lambda a,b: cmp(a[0], b[0]))
     return [ r[1] for r in results ]

misc/operations_helpers/munin/tahoe_nodememory

@@ -6,10 +6,9 @@
 
 import os, sys, re
 
-if 0:
-    # for testing
-    os.environ["nodememory_warner1"] = "run/warner1"
-    os.environ["nodememory_warner2"] = "run/warner2"
+# for testing
+# os.environ["nodememory_warner1"] = "run/warner1"
+# os.environ["nodememory_warner2"] = "run/warner2"
 
 nodedirs = []
 for k,v in os.environ.items():

misc/simulators/simulator.py

@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 
-import sha as shamodule
+import hashlib
 import os, random
 
 from pkg_resources import require
@@ -10,7 +10,7 @@ from pyrrd.rrd import DataSource, RRD, RRA
 
 
 def sha(s):
-    return shamodule.new(s).digest()
+    return hashlib.sha1(s).digest()
 
 def randomid():
     return os.urandom(20)
@@ -70,10 +70,7 @@ class Node:
             return False
 
     def decide(self, sharesize):
-        if sharesize > self.capacity:
-            return False
         return False
-        return random.random() > 0.5
 
     def make_space(self, sharesize):
         assert sharesize <= self.capacity

src/allmydata/frontends/sftpd.py

@@ -38,24 +38,9 @@ from allmydata.dirnode import update_metadata
 from allmydata.util.fileutil import EncryptedTemporaryFile
 
 noisy = True
-use_foolscap_logging = True
 
 from allmydata.util.log import NOISY, OPERATIONAL, WEIRD, \
-    msg as _msg, err as _err, PrefixingLogMixin as _PrefixingLogMixin
-
-if use_foolscap_logging:
-    (logmsg, logerr, PrefixingLogMixin) = (_msg, _err, _PrefixingLogMixin)
-else:  # pragma: no cover
-    def logmsg(s, level=None):
-        print s
-    def logerr(s, level=None):
-        print s
-    class PrefixingLogMixin:
-        def __init__(self, facility=None, prefix=''):
-            self.prefix = prefix
-        def log(self, s, level=None):
-            print "%r %s" % (self.prefix, s)
-
+    msg as logmsg, PrefixingLogMixin
 
 def eventually_callback(d):
     return lambda res: eventually(d.callback, res)

src/allmydata/immutable/layout.py

@@ -438,9 +438,7 @@ class ReadBucketProxy(object):
     def _get_share_hashes(self, unused=None):
         if hasattr(self, '_share_hashes'):
             return self._share_hashes
-        else:
-            return self._get_share_hashes_the_old_way()
-        return self._share_hashes
+        return self._get_share_hashes_the_old_way()
 
     def get_share_hashes(self):
         d = self._start_if_needed()

src/allmydata/immutable/upload.py

@@ -533,7 +533,6 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
                 # we haven't improved over the last iteration; give up
                 break;
             if errors_before == self._query_stats.bad:
-                if False: print("no more errors; break")
                 break;
             last_happiness = effective_happiness
             # print("write trackers left: {}".format(len(write_trackers)))

src/allmydata/mutable/publish.py

@@ -909,9 +909,7 @@ class Publish:
                  level=log.NOISY)
 
     def update_goal(self):
-        # if log.recording_noisy
-        if True:
-            self.log_goal(self.goal, "before update: ")
+        self.log_goal(self.goal, "before update: ")
 
         # first, remove any bad servers from our goal
         self.goal = set([ (server, shnum)
@@ -976,8 +974,7 @@ class Publish:
             i += 1
             if i >= len(serverlist):
                 i = 0
-        if True:
-            self.log_goal(self.goal, "after update: ")
+        self.log_goal(self.goal, "after update: ")
 
 
     def _got_write_answer(self, answer, writer, started):

src/allmydata/scripts/runner.py

@@ -47,7 +47,6 @@ class Options(usage.Options):
     stdout = sys.stdout
     stderr = sys.stderr
 
-    synopsis = "\nUsage:  tahoe <command> [command options]"
     subCommands = ( GROUP("Administration")
                     +   create_node.subCommands
                     +   stats_gatherer.subCommands

src/allmydata/util/encodingutil.py

@@ -335,8 +335,8 @@ def listdir_unicode_fallback(path):
 
     try:
         return [unicode(fn, filesystem_encoding) for fn in os.listdir(byte_path)]
-    except UnicodeDecodeError:
-        raise FilenameEncodingError(fn)
+    except UnicodeDecodeError as e:
+        raise FilenameEncodingError(e.object)
 
 def listdir_unicode(path):
     """

src/allmydata/util/tor_provider.py

@@ -72,13 +72,9 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
     tor_config = txtorcon.TorConfig()
     tor_config.DataDirectory = data_directory(private_dir)
 
-    if True: # unix-domain control socket
-        tor_config.ControlPort = "unix:" + os.path.join(private_dir, "tor.control")
-        tor_control_endpoint_desc = tor_config.ControlPort
-    else:
-        # we allocate a new TCP control port each time
-        tor_config.ControlPort = allocate_tcp_port()
-        tor_control_endpoint_desc = "tcp:127.0.0.1:%d" % tor_config.ControlPort
+    # unix-domain control socket
+    tor_config.ControlPort = "unix:" + os.path.join(private_dir, "tor.control")
+    tor_control_endpoint_desc = tor_config.ControlPort
 
     tor_config.SOCKSPort = allocate_tcp_port()
 

src/allmydata/util/verlib.py

@@ -254,7 +254,7 @@ def suggest_normalized_version(s):
     # if we have something like "b-2" or "a.2" at the end of the
     # version, that is pobably beta, alpha, etc
     # let's remove the dash or dot
-    rs = re.sub(r"([abc|rc])[\-\.](\d+)$", r"\1\2", rs)
+    rs = re.sub(r"([abc]|rc)[\-\.](\d+)$", r"\1\2", rs)
 
     # 1.0-dev-r371 -> 1.0.dev371
     # 0.1-dev-r79 -> 0.1.dev79
@@ -324,4 +324,3 @@ def suggest_normalized_version(s):
     except IrrationalVersionError:
         pass
     return None
-

src/allmydata/web/directory.py

@@ -74,8 +74,6 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         return d
 
     def got_child(self, node_or_failure, ctx, name):
-        DEBUG = False
-        if DEBUG: print "GOT_CHILD", name, node_or_failure
         req = IRequest(ctx)
         method = req.method
         nonterminal = len(req.postpath) > 1
@@ -84,24 +82,18 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
             f = node_or_failure
             f.trap(NoSuchChildError)
             # No child by this name. What should we do about it?
-            if DEBUG: print "no child", name
-            if DEBUG: print "postpath", req.postpath
             if nonterminal:
-                if DEBUG: print " intermediate"
                 if should_create_intermediate_directories(req):
                     # create intermediate directories
-                    if DEBUG: print " making intermediate directory"
                     d = self.node.create_subdirectory(name)
                     d.addCallback(make_handler_for,
                                   self.client, self.node, name)
                     return d
             else:
-                if DEBUG: print " terminal"
                 # terminal node
                 if (method,t) in [ ("POST","mkdir"), ("PUT","mkdir"),
                                    ("POST", "mkdir-with-children"),
                                    ("POST", "mkdir-immutable") ]:
-                    if DEBUG: print " making final directory"
                     # final directory
                     kids = {}
                     if t in ("mkdir-with-children", "mkdir-immutable"):
@@ -122,14 +114,12 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
                                   self.client, self.node, name)
                     return d
                 if (method,t) in ( ("PUT",""), ("PUT","uri"), ):
-                    if DEBUG: print " PUT, making leaf placeholder"
                     # we were trying to find the leaf filenode (to put a new
                     # file in its place), and it didn't exist. That's ok,
                     # since that's the leaf node that we're about to create.
                     # We make a dummy one, which will respond to the PUT
                     # request by replacing itself.
                     return PlaceHolderNodeHandler(self.client, self.node, name)
-            if DEBUG: print " 404"
             # otherwise, we just return a no-such-child error
             return f
 
@@ -138,11 +128,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
             if not IDirectoryNode.providedBy(node):
                 # we would have put a new directory here, but there was a
                 # file in the way.
-                if DEBUG: print "blocking"
                 raise WebError("Unable to create directory '%s': "
                                "a file was in the way" % name,
                                http.CONFLICT)
-        if DEBUG: print "good child"
         return make_handler_for(node, self.client, self.node, name)
 
     def render_DELETE(self, ctx):

src/allmydata/webish.py

@@ -2,6 +2,10 @@ import re, time
 from twisted.application import service, strports, internet
 from twisted.web import http, static
 from twisted.internet import defer
+from twisted.internet.address import (
+    IPv4Address,
+    IPv6Address,
+)
 from nevow import appserver, inevow
 from allmydata.util import log, fileutil
 
@@ -114,16 +118,32 @@ class MyRequest(appserver.NevowRequest):
         if self._tahoe_request_had_error:
             error = " [ERROR]"
 
-        log.msg(format="web: %(clientip)s %(method)s %(uri)s %(code)s %(length)s%(error)s",
-                clientip=self.getClientIP(),
-                method=self.method,
-                uri=uri,
-                code=self.code,
-                length=(self.sentLength or "-"),
-                error=error,
-                facility="tahoe.webish",
-                level=log.OPERATIONAL,
-                )
+        log.msg(
+            format=(
+                "web: %(clientip)s %(method)s %(uri)s %(code)s "
+                "%(length)s%(error)s"
+            ),
+            clientip=_get_client_ip(self),
+            method=self.method,
+            uri=uri,
+            code=self.code,
+            length=(self.sentLength or "-"),
+            error=error,
+            facility="tahoe.webish",
+            level=log.OPERATIONAL,
+        )
+
+
+def _get_client_ip(request):
+    try:
+        get = request.getClientAddress
+    except AttributeError:
+        return request.getClientIP()
+    else:
+        client_addr = get()
+        if isinstance(client_addr, (IPv4Address, IPv6Address)):
+            return client_addr.host
+        return None
 
 
 class WebishServer(service.MultiService):
@@ -218,4 +238,3 @@ class IntroducerWebishServer(WebishServer):
         service.MultiService.__init__(self)
         self.root = introweb.IntroducerRoot(introducer)
         self.buildServer(webport, nodeurl_path, staticdir)
-

src/allmydata/windows/registry.py

@@ -5,9 +5,9 @@ _AMD_KEY = r"Software\Allmydata"
 _BDIR_KEY = 'Base Dir Path'
 
 if sys.platform not in ('win32'):
-    raise ImportError, "registry cannot be used on non-windows systems"
     class WindowsError(Exception): # stupid voodoo to appease pyflakes
         pass
+    raise ImportError("registry cannot be used on non-windows systems")
 
 def get_registry_setting(key, name, _topkey=None):
     """