ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-12 05:55:35 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

dirnode: use the concurrency limiter in t=manifest and t=deep-size, allow 10 retrievals in parallel

Browse Source
This commit is contained in:
Brian Warner 2008-05-07 18:36:37 -07:00
parent d76cf6f748
commit 3cb361e233
1 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/allmydata/dirnode.py
View File

@ -10,6 +10,7 @@ from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\
IURI, IFileNode, IMutableFileURI, IVerifierURI, IFilesystemNode IURI, IFileNode, IMutableFileURI, IVerifierURI, IFilesystemNode
from allmydata.util import hashutil from allmydata.util import hashutil
from allmydata.util.hashutil import netstring from allmydata.util.hashutil import netstring
from allmydata.util.limiter import ConcurrencyLimiter
from allmydata.uri import NewDirectoryURI from allmydata.uri import NewDirectoryURI
from pycryptopp.cipher.aes import AES from pycryptopp.cipher.aes import AES
@ -425,12 +426,14 @@ class NewDirectoryNode:
(directories and files) reachable from this one.""" (directories and files) reachable from this one."""
# this is just a tree-walker, except that following each edge # this is just a tree-walker, except that following each edge
# requires a Deferred. # requires a Deferred. We use a ConcurrencyLimiter to make sure the
# fan-out doesn't cause problems.
manifest = set() manifest = set()
manifest.add(self.get_verifier()) manifest.add(self.get_verifier())
limiter = ConcurrencyLimiter(10) # allow 10 in parallel
d = self._build_manifest_from_node(self, manifest) d = self._build_manifest_from_node(self, manifest, limiter)
def _done(res): def _done(res):
# LIT nodes have no verifier-capability: their data is stored # LIT nodes have no verifier-capability: their data is stored
# inside the URI itself, so there is no need to refresh anything. # inside the URI itself, so there is no need to refresh anything.
@ -443,8 +446,8 @@ class NewDirectoryNode:
d.addCallback(_done) d.addCallback(_done)
return d return d
def _build_manifest_from_node(self, node, manifest): def _build_manifest_from_node(self, node, manifest, limiter):
d = node.list() d = limiter.add(node.list)
def _got_list(res): def _got_list(res):
dl = [] dl = []
for name, (child, metadata) in res.iteritems(): for name, (child, metadata) in res.iteritems():
@ -453,7 +456,8 @@ class NewDirectoryNode:
manifest.add(verifier) manifest.add(verifier)
if IDirectoryNode.providedBy(child): if IDirectoryNode.providedBy(child):
dl.append(self._build_manifest_from_node(child, dl.append(self._build_manifest_from_node(child,
manifest)) manifest,
limiter))
if dl: if dl:
return defer.DeferredList(dl) return defer.DeferredList(dl)
d.addCallback(_got_list) d.addCallback(_got_list)