check_speed: upload multiple files, measure Ax+B

Makefile

@@ -212,6 +212,7 @@ check-memory-once: .built
 check-speed: .built
 	if [ -z '$(TESTCLIENTDIR)' ]; then exit 1; fi
 	$(PYTHON) bin/allmydata-tahoe restart -f $(TESTCLIENTDIR)
+	sleep 1
 	$(PYTHON) src/allmydata/test/check_speed.py $(TESTCLIENTDIR)
 
 test-darcs-boringfile:

src/allmydata/control.py

@@ -2,9 +2,10 @@
 import os, time
 from zope.interface import implements
 from twisted.application import service
+from twisted.internet import defer
 from foolscap import Referenceable
 from allmydata.interfaces import RIControlClient
-from allmydata.util import testutil, idlib
+from allmydata.util import testutil, fileutil
 from twisted.python import log
 
 def get_memory_usage():
@@ -46,26 +47,43 @@ class ControlServer(Referenceable, service.Service, testutil.PollMixin):
         d.addCallback(lambda res: filename)
         return d
 
-    def remote_upload_speed_test(self, size):
+    def remote_upload_speed_test(self, count, size):
         assert size > 8
-        fn = os.path.join(self.parent.basedir, idlib.b2a(os.urandom(8)))
-        f = open(fn, "w")
-        f.write(os.urandom(8))
-        size -= 8
-        while size > 0:
-            chunk = min(size, 4096)
-            f.write("\x00" * chunk)
-            size -= chunk
-        f.close()
+        basedir = os.path.join(self.parent.basedir, "_speed_test_data")
+        log.msg("speed_test: count=%d, size=%d" % (count, size))
+        fileutil.make_dirs(basedir)
+        for i in range(count):
+            s = size
+            fn = os.path.join(basedir, str(i))
+            if os.path.exists(fn):
+                os.unlink(fn)
+            f = open(fn, "w")
+            f.write(os.urandom(8))
+            s -= 8
+            while s > 0:
+                chunk = min(s, 4096)
+                f.write("\x00" * chunk)
+                s -= chunk
+            f.close()
         uploader = self.parent.getServiceNamed("uploader")
         start = time.time()
-        d = uploader.upload_filename(fn)
-        def _done(uri):
+        d = defer.succeed(None)
+        def _do_one_file(uri, i):
+            if i >= count:
+                return
+            fn = os.path.join(basedir, str(i))
+            d1 = uploader.upload_filename(fn)
+            d1.addCallback(_do_one_file, i+1)
+            return d1
+        d.addCallback(_do_one_file, 0)
+        def _done(ignored):
             stop = time.time()
             return stop - start
         d.addCallback(_done)
         def _cleanup(res):
-            os.unlink(fn)
+            for i in range(count):
+                fn = os.path.join(basedir, str(i))
+                os.unlink(fn)
             return res
         d.addBoth(_cleanup)
         return d

src/allmydata/interfaces.py

@@ -1054,8 +1054,8 @@ class RIControlClient(RemoteInterface):
         measuring memory consupmtion in bytes."""
         return DictOf(str, int)
 
-    def upload_speed_test(size=int):
-        """Write a tempfile to disk of the given size. Measure how long
-        it takes to upload it to the servers.
+    def upload_speed_test(count=int, size=int):
+        """Write 'count' tempfiles to disk, all of the given size. Measure
+        how long (in seconds) it takes to upload them all to the servers.
         """
         return float

src/allmydata/test/check_speed.py

@@ -6,6 +6,8 @@ from twisted.python import log
 from twisted.application import service
 from foolscap import Tub, eventual
 
+MB = 1000000
+
 class SpeedTest:
     def __init__(self, test_client_dir):
         #self.real_stderr = sys.stderr
@@ -59,6 +61,11 @@ class SpeedTest:
         print "TIME (%s): %s" % (key, time)
         self.times[key] = time
 
+    def one_test(self, res, name, count, size):
+        d = self.client_rref.callRemote("upload_speed_test", count, size)
+        d.addCallback(self.record_time, name)
+        return d
+
     def do_test(self):
         print "doing test"
         rr = self.client_rref
@@ -66,14 +73,48 @@ class SpeedTest:
         def _got(res):
             print "MEMORY USAGE:", res
         d.addCallback(_got)
-        d.addCallback(lambda res: rr.callRemote("upload_speed_test", 1000))
-        d.addCallback(self.record_time, "startup")
-        d.addCallback(lambda res: rr.callRemote("upload_speed_test", int(1e6)))
-        d.addCallback(self.record_time, "1MB.1")
-        d.addCallback(lambda res: rr.callRemote("upload_speed_test", int(1e6)))
-        d.addCallback(self.record_time, "1MB.2")
+        d.addCallback(self.one_test, "startup", 1, 1000) # ignore this one
+        d.addCallback(self.one_test, "1x 200B", 1, 200)
+        d.addCallback(self.one_test, "10x 200B", 10, 200)
+        #d.addCallback(self.one_test, "100x 200B", 100, 200)
+        d.addCallback(self.one_test, "1MB", 1, 1*MB)
+        d.addCallback(self.one_test, "10MB", 1, 10*MB)
+        d.addCallback(self.calculate_speed)
         return d
 
+    def calculate_speed(self, res):
+        perfile = self.times["1x 200B"]
+        # time = A*size+B
+        # we assume that A*200bytes is negligible
+        B = self.times["10x 200B"] / 10
+        print "per-file time: %.3fs" % B
+        A1 = 1*MB / (self.times["1MB"] - B) # in bytes per second
+        print "speed (1MB):", self.number(A1, "Bps")
+        A2 = 10*MB / (self.times["10MB"] - B)
+        print "speed (10MB):", self.number(A2, "Bps")
+
+    def number(self, value, suffix=""):
+        scaling = 1
+        if value < 1:
+            fmt = "%1.2g%s"
+        elif value < 100:
+            fmt = "%.1f%s"
+        elif value < 1000:
+            fmt = "%d%s"
+        elif value < 1e6:
+            fmt = "%.2fk%s"; scaling = 1e3
+        elif value < 1e9:
+            fmt = "%.2fM%s"; scaling = 1e6
+        elif value < 1e12:
+            fmt = "%.2fG%s"; scaling = 1e9
+        elif value < 1e15:
+            fmt = "%.2fT%s"; scaling = 1e12
+        elif value < 1e18:
+            fmt = "%.2fP%s"; scaling = 1e15
+        else:
+            fmt = "huge! %g%s"
+        return fmt % (value / scaling, suffix)
+
     def tearDown(self, res):
         d = self.base_service.stopService()
         d.addCallback(lambda ignored: res)