ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-25 07:31:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

encode.py: add a reactor turn barrier between segments, to allow Deferreds to retire and free their arguments, all in the name of reducing memory footprint

Browse Source
This commit is contained in:
Brian Warner 2007-08-09 18:26:17 -07:00
parent c0c6417e95
commit 684966d103
1 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/allmydata/encode.py
View File

@ -3,6 +3,7 @@
from zope.interface import implements from zope.interface import implements
from twisted.internet import defer from twisted.internet import defer
from twisted.python import log from twisted.python import log
from foolscap import eventual
from allmydata import uri from allmydata import uri
from allmydata.hashtree import HashTree from allmydata.hashtree import HashTree
from allmydata.util import mathutil, hashutil from allmydata.util import mathutil, hashutil
@ -212,8 +213,9 @@ class Encoder(object):
# that we sent to that landlord. # that we sent to that landlord.
self.share_root_hashes = [None] * self.num_shares self.share_root_hashes = [None] * self.num_shares
d = defer.maybeDeferred(self._uploadable.set_segment_size, d = eventual.fireEventually()
self.segment_size) d.addCallback(lambda res:
self._uploadable.set_segment_size(self.segment_size))
for l in self.landlords.values(): for l in self.landlords.values():
d.addCallback(lambda res, l=l: l.start()) d.addCallback(lambda res, l=l: l.start())
@ -225,9 +227,11 @@ class Encoder(object):
# use this form instead: # use this form instead:
d.addCallback(lambda res, i=i: self._encode_segment(i)) d.addCallback(lambda res, i=i: self._encode_segment(i))
d.addCallback(self._send_segment, i) d.addCallback(self._send_segment, i)
d.addCallback(self._turn_barrier)
last_segnum = self.num_segments - 1 last_segnum = self.num_segments - 1
d.addCallback(lambda res: self._encode_tail_segment(last_segnum)) d.addCallback(lambda res: self._encode_tail_segment(last_segnum))
d.addCallback(self._send_segment, last_segnum) d.addCallback(self._send_segment, last_segnum)
d.addCallback(self._turn_barrier)
d.addCallback(lambda res: self.finish_hashing()) d.addCallback(lambda res: self.finish_hashing())
@ -243,6 +247,16 @@ class Encoder(object):
d.addCallbacks(lambda res: self.done(), self.err) d.addCallbacks(lambda res: self.done(), self.err)
return d return d
def _turn_barrier(self, res):
# putting this method in a Deferred chain imposes a guaranteed
# reactor turn between the pre- and post- portions of that chain.
# This can be useful to limit memory consumption: since Deferreds do
# not do tail recursion, code which uses defer.succeed(result) for
# consistency will cause objects to live for longer than you might
# normally expect.
return eventual.fireEventually(res)
def _encode_segment(self, segnum): def _encode_segment(self, segnum):
codec = self._codec codec = self._codec