check_memory: add download, download-GET

This commit is contained in:
Brian Warner 2007-09-18 18:56:05 -07:00
parent 38946439f3
commit f6acf5a0c3
2 changed files with 82 additions and 47 deletions

View File

@ -177,6 +177,10 @@ check-memory: build
$(PYTHON) src/allmydata/test/check_memory.py upload-self $(PYTHON) src/allmydata/test/check_memory.py upload-self
$(PP) \ $(PP) \
$(PYTHON) src/allmydata/test/check_memory.py upload-POST $(PYTHON) src/allmydata/test/check_memory.py upload-POST
$(PP) \
$(PYTHON) src/allmydata/test/check_memory.py download
$(PP) \
$(PYTHON) src/allmydata/test/check_memory.py download-GET
test-darcs-boringfile: test-darcs-boringfile:

View File

@ -1,10 +1,10 @@
#! /usr/bin/env python #! /usr/bin/env python
import os, shutil, sys import os, shutil, sys, urllib
from cStringIO import StringIO from cStringIO import StringIO
from twisted.internet import defer, reactor, protocol, error from twisted.internet import defer, reactor, protocol, error
from twisted.application import service, internet from twisted.application import service, internet
from twisted.web.client import getPage from twisted.web.client import getPage, downloadPage
from allmydata import client, introducer_and_vdrive from allmydata import client, introducer_and_vdrive
from allmydata.scripts import create_node from allmydata.scripts import create_node
from allmydata.util import testutil, fileutil from allmydata.util import testutil, fileutil
@ -30,6 +30,8 @@ class SystemFramework(testutil.PollMixin):
self.tub.setServiceParent(self.sparent) self.tub.setServiceParent(self.sparent)
self.discard_shares = True self.discard_shares = True
self.mode = mode self.mode = mode
if mode in ("download", "download-GET"):
self.discard_shares = False
self.failed = False self.failed = False
def run(self): def run(self):
@ -169,7 +171,7 @@ this file are ignored.
f = open(os.path.join(clientdir, "debug_no_storage"), "w") f = open(os.path.join(clientdir, "debug_no_storage"), "w")
f.write("no_storage\n") f.write("no_storage\n")
f.close() f.close()
if self.mode == "upload-self": if self.mode in ("upload-self", "download", "download-GET"):
f = open(os.path.join(clientdir, "push_to_ourselves"), "w") f = open(os.path.join(clientdir, "push_to_ourselves"), "w")
f.write("push_to_ourselves\n") f.write("push_to_ourselves\n")
f.close() f.close()
@ -269,18 +271,13 @@ this file are ignored.
return getPage(url, method="POST", postdata=body, return getPage(url, method="POST", postdata=body,
headers=headers, followRedirect=False) headers=headers, followRedirect=False)
def do_test(self): def GET_discard(self, urlpath):
#print "CLIENT STARTED" # TODO: Slow
#print "FURL", self.control_furl url = self.webish_url + urlpath + "?filename=dummy-get.out"
#print "RREF", self.control_rref return downloadPage(url, os.path.join(self.basedir, "dummy-get.out"))
#print
kB = 1000; MB = 1000*1000
files = {}
uris = {}
control = self.control_rref
def _print_usage(res=None): def _print_usage(self, res=None):
d = control.callRemote("get_memory_usage") d = self.control_rref.callRemote("get_memory_usage")
def _print(stats): def _print(stats):
print "VmSize: %9d VmPeak: %9d" % (stats["VmSize"], print "VmSize: %9d VmPeak: %9d" % (stats["VmSize"],
stats["VmPeak"]) stats["VmPeak"])
@ -288,13 +285,14 @@ this file are ignored.
d.addCallback(_print) d.addCallback(_print)
return d return d
def _do_upload(res, size): def _do_upload(self, res, size, files, uris):
name = '%d' % size name = '%d' % size
print print
print "uploading %s" % name print "uploading %s" % name
if self.mode in ("upload", "upload-self"): if self.mode in ("upload", "upload-self"):
files[name] = self.create_data(name, size) files[name] = self.create_data(name, size)
d = control.callRemote("upload_from_file_to_uri", files[name]) d = self.control_rref.callRemote("upload_from_file_to_uri",
files[name])
def _done(uri): def _done(uri):
os.remove(files[name]) os.remove(files[name])
del files[name] del files[name]
@ -304,6 +302,12 @@ this file are ignored.
data = "a" * size data = "a" * size
url = "/vdrive/global" url = "/vdrive/global"
d = self.POST(url, t="upload", file=("%d.data" % size, data)) d = self.POST(url, t="upload", file=("%d.data" % size, data))
elif self.mode in ("download", "download-GET"):
# upload the data from a local peer, then have the
# client-under-test download it.
files[name] = self.create_data(name, size)
u = self.nodes[0].getServiceNamed("uploader")
d = u.upload_filename(files[name])
else: else:
raise RuntimeError("unknown mode=%s" % self.mode) raise RuntimeError("unknown mode=%s" % self.mode)
def _complete(uri): def _complete(uri):
@ -312,27 +316,54 @@ this file are ignored.
d.addCallback(_complete) d.addCallback(_complete)
return d return d
d = _print_usage() def _do_download(self, res, size, uris):
if self.mode not in ("download", "download-GET"):
return
name = '%d' % size
uri = uris[name]
if self.mode == "download":
d = self.control_rref.callRemote("download_from_uri_to_file",
uri, "dummy.out")
if self.mode == "download-GET":
url = "/uri/%s" % uri
d = self.GET_discard(urllib.quote(url))
return d
def do_test(self):
#print "CLIENT STARTED"
#print "FURL", self.control_furl
#print "RREF", self.control_rref
#print
kB = 1000; MB = 1000*1000
files = {}
uris = {}
d = self._print_usage()
d.addCallback(self.stash_stats, "0B") d.addCallback(self.stash_stats, "0B")
for i in range(10): for i in range(10):
d.addCallback(_do_upload, size=10*kB+i) d.addCallback(self._do_upload, 10*kB+i, files, uris)
d.addCallback(_print_usage) d.addCallback(self._do_download, 10*kB+i, uris)
d.addCallback(self._print_usage)
d.addCallback(self.stash_stats, "10kB") d.addCallback(self.stash_stats, "10kB")
for i in range(3): for i in range(3):
d.addCallback(_do_upload, size=10*MB+i) d.addCallback(self._do_upload, 10*MB+i, files, uris)
d.addCallback(_print_usage) d.addCallback(self._do_download, 10*MB+i, uris)
d.addCallback(self._print_usage)
d.addCallback(self.stash_stats, "10MB") d.addCallback(self.stash_stats, "10MB")
for i in range(3): for i in range(1):
d.addCallback(_do_upload, size=50*MB+i) d.addCallback(self._do_upload, 50*MB+i, files, uris)
d.addCallback(_print_usage) d.addCallback(self._do_download, 50*MB+i, uris)
d.addCallback(self._print_usage)
d.addCallback(self.stash_stats, "50MB") d.addCallback(self.stash_stats, "50MB")
#for i in range(1): #for i in range(1):
# d.addCallback(_do_upload, size=100*MB+i) # d.addCallback(self._do_upload, 100*MB+i, files, uris)
# d.addCallback(_print_usage) # d.addCallback(self._do_download, 100*MB+i, uris)
# d.addCallback(self._print_usage)
#d.addCallback(self.stash_stats, "100MB") #d.addCallback(self.stash_stats, "100MB")
#d.addCallback(self.stall) #d.addCallback(self.stall)