tahoe-lafs/misc/checkers/check_load.py

from __future__ import print_function

"""
this is a load-generating client program. It does all of its work through a
given tahoe node (specified by URL), and performs random reads and writes
to the target.

Run this in a directory with the following files:
 server-URLs : a list of tahoe node URLs (one per line). Each operation
               will use a randomly-selected server.
 root.cap: (string) the top-level directory rwcap to use
 delay: (float) seconds to delay between operations
 operation-mix: "R/W": two ints, relative frequency of read and write ops
 #size:?

Set argv[1] to a per-client stats-NN.out file. This will will be updated with
running totals of bytes-per-second and operations-per-second. The stats from
multiple clients can be totalled together and averaged over time to compute
the traffic being accepted by the grid.

Each time a 'read' operation is performed, the client will begin at the root
and randomly choose a child. If the child is a directory, the client will
recurse. If the child is a file, the client will read the contents of the
file.

Each time a 'write' operation is performed, the client will generate a target
filename (a random string). 90% of the time, the file will be written into
the same directory that was used last time (starting at the root). 10% of the
time, a new directory is created by assembling 1 to 5 pathnames chosen at
random. The client then writes a certain number of zero bytes to this file.
The filesize is determined with something like a power-law distribution, with
a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)


"""

import os, sys, httplib, binascii
import urllib, json, random, time, urlparse

try:
    from typing import Dict
except ImportError:
    pass

# Python 2 compatibility
from future.utils import PY2
if PY2:
    from future.builtins import str  # noqa: F401

if sys.argv[1] == "--stats":
    statsfiles = sys.argv[2:]
    # gather stats every 10 seconds, do a moving-window average of the last
    # 60 seconds
    DELAY = 10
    MAXSAMPLES = 6
    totals = []
    last_stats = {}  # type: Dict[str, float]
    while True:
        stats = {}  # type: Dict[str, float]
        for sf in statsfiles:
            for line in open(sf, "r").readlines():
                name, str_value = line.split(":")
                value = int(str_value.strip())
                if name not in stats:
                    stats[name] = 0
                stats[name] += float(value)
        del name
        if last_stats:
            delta = dict( [ (n,stats[n]-last_stats[n])
                            for n in stats ] )
            print("THIS SAMPLE:")
            for name in sorted(delta.keys()):
                avg = float(delta[name]) / float(DELAY)
                print("%20s: %0.2f per second" % (name, avg))
            totals.append(delta)
            while len(totals) > MAXSAMPLES:
                totals.pop(0)

            # now compute average
            print()
            print("MOVING WINDOW AVERAGE:")
            for name in sorted(delta.keys()):
                avg = sum([ s[name] for s in totals]) / (DELAY*len(totals))
                print("%20s %0.2f per second" % (name, avg))

        last_stats = stats
        print()
        print()
        time.sleep(DELAY)

stats_out = sys.argv[1]

server_urls = []
for url in open("server-URLs", "r").readlines():
    url = url.strip()
    if url:
        server_urls.append(url)
root = open("root.cap", "r").read().strip()
delay = float(open("delay", "r").read().strip())
readfreq, writefreq = (
    [int(x) for x in open("operation-mix", "r").read().strip().split("/")])


files_uploaded = 0
files_downloaded = 0
bytes_uploaded = 0
bytes_downloaded = 0
directories_read = 0
directories_written = 0

def listdir(nodeurl, root, remote_pathname):
    if nodeurl[-1] != "/":
        nodeurl += "/"
    url = nodeurl + "uri/%s/" % urllib.quote(root)
    if remote_pathname:
        url += urllib.quote(remote_pathname)
    url += "?t=json"
    data = urllib.urlopen(url).read()
    try:
        parsed = json.loads(data)
    except ValueError:
        print("URL was", url)
        print("DATA was", data)
        raise
    nodetype, d = parsed
    assert nodetype == "dirnode"
    global directories_read
    directories_read += 1
    children = dict( [(str(name),value)
                      for (name,value)
                      in d["children"].iteritems()] )
    return children


def choose_random_descendant(server_url, root, pathname=""):
    children = listdir(server_url, root, pathname)
    name = random.choice(children.keys())
    child = children[name]
    if pathname:
        new_pathname = pathname + "/" + name
    else:
        new_pathname = name
    if child[0] == "filenode":
        return new_pathname
    return choose_random_descendant(server_url, root, new_pathname)

def read_and_discard(nodeurl, root, pathname):
    if nodeurl[-1] != "/":
        nodeurl += "/"
    url = nodeurl + "uri/%s/" % urllib.quote(root)
    if pathname:
        url += urllib.quote(pathname)
    f = urllib.urlopen(url)
    global bytes_downloaded
    while True:
        data = f.read(4096)
        if not data:
            break
        bytes_downloaded += len(data)


directories = [
    "dreamland/disengaging/hucksters",
    "dreamland/disengaging/klondikes",
    "dreamland/disengaging/neatly",
    "dreamland/cottages/richmond",
    "dreamland/cottages/perhaps",
    "dreamland/cottages/spies",
    "dreamland/finder/diversion",
    "dreamland/finder/cigarette",
    "dreamland/finder/album",
    "hazing/licences/comedian",
    "hazing/licences/goat",
    "hazing/licences/shopkeeper",
    "hazing/regiment/frigate",
    "hazing/regiment/quackery",
    "hazing/regiment/centerpiece",
    "hazing/disassociate/mob",
    "hazing/disassociate/nihilistic",
    "hazing/disassociate/bilbo",
    ]

def create_random_directory():
    d = random.choice(directories)
    pieces = d.split("/")
    numsegs = random.randint(1, len(pieces))
    return "/".join(pieces[0:numsegs])

def generate_filename():
    fn = binascii.hexlify(os.urandom(4))
    return fn

def choose_size():
    mean = 10e3
    size = random.expovariate(1.0 / mean)
    return int(min(size, 100e6))

# copied from twisted/web/client.py
def parse_url(url, defaultPort=None):
    url = url.strip()
    parsed = urlparse.urlparse(url)
    scheme = parsed[0]
    path = urlparse.urlunparse(('','')+parsed[2:])
    if defaultPort is None:
        if scheme == 'https':
            defaultPort = 443
        else:
            defaultPort = 80
    host, port = parsed[1], defaultPort
    if ':' in host:
        host, port = host.split(':')
        port = int(port)
    if path == "":
        path = "/"
    return scheme, host, port, path

def generate_and_put(nodeurl, root, remote_filename, size):
    if nodeurl[-1] != "/":
        nodeurl += "/"
    url = nodeurl + "uri/%s/" % urllib.quote(root)
    url += urllib.quote(remote_filename)

    scheme, host, port, path = parse_url(url)
    if scheme == "http":
        c = httplib.HTTPConnection(host, port)
    elif scheme == "https":
        c = httplib.HTTPSConnection(host, port)
    else:
        raise ValueError("unknown scheme '%s', need http or https" % scheme)
    c.putrequest("PUT", path)
    c.putheader("Hostname", host)
    c.putheader("User-Agent", "tahoe-check-load")
    c.putheader("Connection", "close")
    c.putheader("Content-Length", "%d" % size)
    c.endheaders()
    global bytes_uploaded
    while size:
        chunksize = min(size, 4096)
        size -= chunksize
        c.send("\x00" * chunksize)
        bytes_uploaded += chunksize
    return c.getresponse()


current_writedir = ""

while True:
    time.sleep(delay)
    if random.uniform(0, readfreq+writefreq) < readfreq:
        op = "read"
    else:
        op = "write"
    print("OP:", op)
    server = random.choice(server_urls)
    if op == "read":
        pathname = choose_random_descendant(server, root)
        print("  reading", pathname)
        read_and_discard(server, root, pathname)
        files_downloaded += 1
    elif op == "write":
        if random.uniform(0, 100) < 10:
            current_writedir = create_random_directory()
        filename = generate_filename()
        if current_writedir:
            pathname = current_writedir + "/" + filename
        else:
            pathname = filename
        print("  writing", pathname)
        size = choose_size()
        print("   size", size)
        generate_and_put(server, root, pathname, size)
        files_uploaded += 1

    f = open(stats_out+".tmp", "w")
    f.write("files-uploaded: %d\n" % files_uploaded)
    f.write("files-downloaded: %d\n" % files_downloaded)
    f.write("bytes-uploaded: %d\n" % bytes_uploaded)
    f.write("bytes-downloaded: %d\n" % bytes_downloaded)
    f.write("directories-read: %d\n" % directories_read)
    f.write("directories-written: %d\n" % directories_written)
    f.close()
    os.rename(stats_out+".tmp", stats_out)