From 124b2160b930c180fc2abc70497be35c4728a71e Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Tue, 12 Feb 2008 21:28:52 -0700 Subject: [PATCH] webish.py: fix for #237: when listing large directories, insert a turn break once every 100 children, to work around non-optimized tail recursion Deferreds --- src/allmydata/webish.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index a77288638..fa374e904 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -18,6 +18,7 @@ from allmydata import get_package_versions_string from zope.interface import implements, Interface import urllib from formless import webform +from foolscap.eventual import fireEventually from nevow.util import resource_filename @@ -209,6 +210,25 @@ class Directory(rend.Page): def data_children(self, ctx, data): d = self._dirnode.list() d.addCallback(lambda dict: sorted(dict.items())) + def _stall_some(items): + # Deferreds don't optimize out tail recursion, and the way + # Nevow's flattener handles Deferreds doesn't take this into + # account. As a result, large lists of Deferreds that fire in the + # same turn (i.e. the output of defer.succeed) will cause a stack + # overflow. To work around this, we insert a turn break after + # every 100 items, using foolscap's fireEventually(). This gives + # the stack a chance to be popped. It would also work to put + # every item in its own turn, but that'd be a lot more + # inefficient. This addresses ticket #237, for which I was never + # able to create a failing unit test. + output = [] + for i,item in enumerate(items): + if i % 100 == 0: + output.append(fireEventually(item)) + else: + output.append(item) + return output + d.addCallback(_stall_some) return d def render_row(self, ctx, data):