[PATCH 3 of 7] Added graph example

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226015460 -3600
# Node ID 0985564470de2bb2c5247effd30dc40f74048f17
# Parent aa8ba57c7833f0d68792db7c36782e3ba9fbc194
Added graph example.

diff -r aa8ba57c7833 -r 0985564470de apps/benchmark/graph.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/graph.py Fri Nov 07 00:51:00 2008 +0100
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+""" A rudimentary script that generates graphs of some benchmark results.
+It could be executed periodically or triggered by a benchmark run, a
+viff commit or something else.
+
+All sorts of statistics could be generated by scripts like this.
+"""
+
+import sys
+from database import Database
+import numpy as np
+import matplotlib.pyplot as plt
+
+def example1(database):
+ """Create a graph showing for each suite the revision on the x-axis and
+ the average execution time of Toft05 comparison for three players on the
+ y-axis (timings from host with player_id one is used).
+ """
+ vals = database.sql_query("""
+ SELECT Suite.revision, Suite.starttime, AVG(Result.value)
+ FROM Suite
+ INNER JOIN VIFFBenchmark AS b ON Suite.id = b.suite
+ INNER JOIN IntegerBenchmarkAttribute AS iba ON b.id = iba.benchmark
+ INNER JOIN Result ON b.id = Result.benchmark
+ WHERE b.name = 'ComparisonToft05'
+ AND iba.attribute = 'n'
+ AND iba.value = 3
+ AND Result.host = 1
+ AND Result.attribute = 'execution_time'
+ GROUP BY Suite.id, b.id;
+ """)
+
+ toft05_avgs = []
+ revisions = []
+ for revision, timestamp, average in vals:
+ print revision, timestamp, average
+ toft05_avgs.append(int(average/1000))
+ revisions.append(revision)
+ N = len(revisions)
+ ind = np.arange(N)
+ width = 0.35 # the width of the bars: can also be len(x) sequence
+ p1 = plt.bar(ind, toft05_avgs, width, color='r')
+ plt.ylabel('execution time / ms')
+ plt.title('VIFF Benchmark Results')
+ plt.xticks(ind+width/2., revisions )
+ plt.legend( (p1[0],), ('Toft05Comparison',) )
+ plt.show()
+ #savefig("fig1.png",dpi=(640/8))
+
+if __name__ == "__main__":
+ database = Database(host="localhost",
+ db="viff_benchmark",
+ user="root",
+ passwd=sys.argv[1])
+
+ example1(database)
+
+ # TODO: We would like hg commit dates to exist in database as well
+ # as the suite execution date. How?
+
+ # TODO: Consider replacing the raw SQL with calls to general methods
+ # in database.py.
+
+
+

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226015319 -3600
# Node ID dd640eac89c544273c9e1a7230af6b44c975bf01
# Parent ed614874c220220495746906180caf9a2d5f258d
Added example benchmarks.

diff -r ed614874c220 -r dd640eac89c5 apps/benchmark/examples/ComparisonToft05.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/examples/ComparisonToft05.py Fri Nov 07 00:48:39 2008 +0100
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import sys, time
+from viff_benchmark import VIFFBenchmark
+from benchmark import parse_args
+from viff.runtime import gather_shares
+from viff.comparison import Toft05Runtime
+from viff.field import GF
+
+class ComparisonToft05(VIFFBenchmark):
+ """ This is an example how to use the benchmark application.
+
+ In the get_runtime_class method we specify that this benchmark should
+ use the Toft05Runtime. The protocol method does the actual
+ benchmarking. The protocol method is invoked on each benchmark
+ host given the appropriate runtime and a dictionary of attributes.
+ This dictionary contains some predefined attributes like player_id,
+ n (number of players), etc. as well as the custom attributes that was
+ specified in the benchmark constructor when adding the benchmark to
+ the suite, e.g. modulus in this case.
+
+ The protocol method schedules the setup of some random secret sharings.
+ Then when they are ready, a timer is started, a comparison is done,
+ and when that comparison finishes, the timer is stopped.
+
+ The protocol method must return a dictionary of the form
+
+ { attribute_name_1: value_1, attribute_name_2: value_2, ...}
+
+ in this case { 'execution_time': x }.
+
+ It is thus possible for one benchmark to measure values for
+ multiple attributes. TODO: It is not clear now whether this
+ is a desired feature.
+
+ """
+
+ def get_runtime_class(self):
+ return Toft05Runtime
+
+ def start_timer(self, _):
+ global start_time
+ start_time = time.time()
+
+ def stop_timer(self, _):
+ # Execution time is reported as whole microseconds.
+ # TODO: Don't know if this is the best way measure the time.
+ stop_time = time.time()
+ millis = int((stop_time - start_time) * 1000000)
+ return {'execution_time': millis}
+
+ def protocol(self, rt, attr):
+ Zp = GF(int(attr['modulus']))
+ a = rt.prss_share_random(Zp)
+ b = rt.prss_share_random(Zp)
+ c = gather_shares([a,b])
+ c.addCallback(self.start_timer)
+ d = (a <= b)
+ d.addCallback(self.stop_timer)
+ return d
+
+
+# TODO: This is always needed. Make this more intuitive.
+if __name__ == "__main__":
+ ComparisonToft05(parse_args(sys.argv)).run_on_slave()
+
\ No newline at end of file
diff -r ed614874c220 -r dd640eac89c5 apps/benchmark/examples/LocalMult.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/examples/LocalMult.py Fri Nov 07 00:48:39 2008 +0100
@@ -0,0 +1,53 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import time
+import random
+import timeit
+from benchmark import Benchmark, parse_args
+
+class LocalMult(Benchmark):
+ """This benchmark measures the time it takes to do a local multiplication
+ of two Zp elements.
+
+ It serves as an example of how to use the benchmark application to
+ benchmark something that does not involve execution of a VIFF protocol.
+ """
+
+ def setup(self, attr):
+ random.seed()
+
+ # TODO: I'm not sure at all whether this is the right way of timing
+ # local multiplication.
+ def benchmark(self, attr):
+ a = random.randint(0, int(attr['modulus']))
+ b = random.randint(0, int(attr['modulus']))
+ pre = "from viff.field import GF\nZp=GF(%s)\na = Zp(%s)\nb = Zp(%s)" %\
+ (attr['modulus'], str(a), str(b))
+ t = timeit.Timer(stmt="c = a * b", setup=pre)
+ time = t.timeit()
+ micro_seconds = int(time * 1000)
+ return {'execution_time': micro_seconds}
+
+
+# TODO: This is always needed. Make this more intuitive.
+if __name__ == "__main__":
+ LocalMult(parse_args(sys.argv)).run_on_slave()
+
\ No newline at end of file
diff -r ed614874c220 -r dd640eac89c5 apps/benchmark/examples/__init__.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/examples/__init__.py Fri Nov 07 00:48:39 2008 +0100
@@ -0,0 +1,17 @@
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+

[Martin Geisler]

Thomas Pelle Jakobsen <***@daimi.au.dk> writes:

Hi Thomas,

I figured that I might as well start looking at this -- it doesn't seem
that anybody else will jump in :-)

Post by Thomas Pelle Jakobsen
# HG changeset patch
# Date 1226015319 -3600
# Node ID dd640eac89c544273c9e1a7230af6b44c975bf01
# Parent ed614874c220220495746906180caf9a2d5f258d
Added example benchmarks.
diff -r ed614874c220 -r dd640eac89c5 apps/benchmark/examples/ComparisonToft05.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/examples/ComparisonToft05.py Fri Nov 07 00:48:39 2008 +0100
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.

This is new code -- the copyright header should mention the years in
which the software is released. Mentioning the years where the software
has been edited is normally a good approximation.

I guess you don't use Emacs, but let me just note anyway that I use

(add-hook 'before-save-hook 'copyright-update)

in my ~/.emacs file to automatically update any such copyright notices
whenever I save a file. So in a month the copyright headers will start
being updated as we work on the different files.

Post by Thomas Pelle Jakobsen
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import sys, time
+from viff_benchmark import VIFFBenchmark
+from benchmark import parse_args
+from viff.runtime import gather_shares
+from viff.comparison import Toft05Runtime
+from viff.field import GF
+
+ """ This is an example how to use the benchmark application.
+
+ In the get_runtime_class method we specify that this benchmark should
+ use the Toft05Runtime. The protocol method does the actual
+ benchmarking. The protocol method is invoked on each benchmark
+ host given the appropriate runtime and a dictionary of attributes.
+ This dictionary contains some predefined attributes like player_id,
+ n (number of players), etc. as well as the custom attributes that was

The three precedings lines end with an extra space.

Post by Thomas Pelle Jakobsen
+ specified in the benchmark constructor when adding the benchmark to
+ the suite, e.g. modulus in this case.
+
+ The protocol method schedules the setup of some random secret sharings.
+ Then when they are ready, a timer is started, a comparison is done,
+ and when that comparison finishes, the timer is stopped.
+
+ The protocol method must return a dictionary of the form
+
+ { attribute_name_1: value_1, attribute_name_2: value_2, ...}
+
+ in this case { 'execution_time': x }.
+
+ It is thus possible for one benchmark to measure values for

Another trailing space.

Post by Thomas Pelle Jakobsen
+ multiple attributes. TODO: It is not clear now whether this
+ is a desired feature.
+
+ """
+
+ return Toft05Runtime
+
+ global start_time
+ start_time = time.time()
+
+ # Execution time is reported as whole microseconds.
+ # TODO: Don't know if this is the best way measure the time.
+ stop_time = time.time()

The time module says that the clock function is the most accurate time
source:

http://docs.python.org/library/time.html#time.clock

But when I used it last, it kept giving me strage numbers. Maybe it was
because the timespan I tried to measure was so long that it would
overflow an float?

So I guess time.time is okay to use.

Post by Thomas Pelle Jakobsen
+ millis = int((stop_time - start_time) * 1000000)

Should that not be micros? And why not keep it as seconds and as a
floating point number?

Post by Thomas Pelle Jakobsen
+ return {'execution_time': millis}
+
+ Zp = GF(int(attr['modulus']))
+ a = rt.prss_share_random(Zp)
+ b = rt.prss_share_random(Zp)
+ c = gather_shares([a,b])
+ c.addCallback(self.start_timer)
+ d = (a <= b)
+ d.addCallback(self.stop_timer)
+ return d
+
+
+# TODO: This is always needed. Make this more intuitive.
+ ComparisonToft05(parse_args(sys.argv)).run_on_slave()

I guess this means that each benchmark becomes its own executable file?
Maybe it would be better to make one single executable file and then let
this file run the benchmark?

It could import the given file with

bench = __import__(file)
execute_benchmark_module(benc)

or something like that.

Post by Thomas Pelle Jakobsen
\ No newline at end of file

You'll probably want a newline here.

Post by Thomas Pelle Jakobsen
diff -r ed614874c220 -r dd640eac89c5 apps/benchmark/examples/LocalMult.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/examples/LocalMult.py Fri Nov 07 00:48:39 2008 +0100
@@ -0,0 +1,53 @@
+ """This benchmark measures the time it takes to do a local multiplication
+ of two Zp elements.
+
+ It serves as an example of how to use the benchmark application to
+ benchmark something that does not involve execution of a VIFF protocol.
+ """
+
+ random.seed()

I believe the random module seeds itself as necessary.

Post by Thomas Pelle Jakobsen
+ # TODO: I'm not sure at all whether this is the right way of timing
+ # local multiplication.

Hmm, well, ... Using timeit it cool, but having to define the benchmark
as a text string is not. Maybe we could see how timeit works and then do
something similar, but on real code?

Post by Thomas Pelle Jakobsen
+ a = random.randint(0, int(attr['modulus']))
+ b = random.randint(0, int(attr['modulus']))
+ pre = "from viff.field import GF\nZp=GF(%s)\na = Zp(%s)\nb = Zp(%s)" %\
+ (attr['modulus'], str(a), str(b))
+ t = timeit.Timer(stmt="c = a * b", setup=pre)
+ time = t.timeit()
+ micro_seconds = int(time * 1000)
+ return {'execution_time': micro_seconds}

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226015502 -3600
# Node ID 75e5113f27777649c2001b1221c9717d8d375423
# Parent 0985564470de2bb2c5247effd30dc40f74048f17
Added benchmark suite class.

diff -r 0985564470de -r 75e5113f2777 apps/benchmark/suite.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/suite.py Fri Nov 07 00:51:42 2008 +0100
@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+"""
+ Description:
+ Main goal is to make it easy to write and run VIFF benchmarks while
+ at the same time not limiting the kind of benchmarks that could be
+ made. Also, the solution should scale as the number of benchmarks
+ increase. Finally, the benchmark data should be collected in a central
+ database rather than as a bunch of text files.
+
+ A benchmark is created by subclassing either Benchmark or
+ VIFFBenchmark. By subclassing VIFFBenchmark, you need only specify the
+ runtime that is to be used, the protocol that is to be benchmarked, and
+ the code that does the measure. See the examples.
+
+
+ Features:
+ When setting up a suite, one can specify which revision to benchmark.
+ Uses ssh and thus benefits from key agents like ssh-agent.
+ Non-viff benchmarks can be done by subclassing Benchmark while VIFF
+ benchmarks can easily be benchmarked by subclassing VIFFBenchmark.
+ Data is automatically stored in central database.
+ Benchmark results are stored for each individual run. This enables all
+ kind of statistics to be applied at a later time, since no info is
+ lost. E.g. storing only mean values would prevent calculating
+ conficence intervals and other statistical stuff.
+ Dynamic approach: All parameter names are available except a few
+ pre-defined, e.g. work_dir, player_id, ... (TODO: List them here.)
+
+
+ Issues:
+ Looks like the first run takes longer time than the rest. Is that ok?
+ Results for all runs for a benchmark is kept in memory at once.
+ Problem?
+ Currently master only works on *nix since we use select.
+ Benchmark slaves works only on *nix now due to small issues with pipes,
+ etc.
+ It is assumed that all benchmark hosts have access to the same file
+ system and that the dir supplied to the suite is on this shared
+ file system.
+ Can we run multiple benchmark slaves on one host?
+ Currently, default parameters are always used, e.g. use_ssl = no.
+ Support for string, float and boolean results. Currently only integers
+ (up to Mysql.BIGNUM) are supported.
+ We assume that python setup.py install --home=$HOME/opt overwrites old
+ installation nicely.
+ Log err and out nicely to various files and/or use python logging.
+ Write unit tests!
+ All random seeds should come in as parameters.
+ Currently, the runs are done in sequence. Maybe also support parallel
+ runs; this is ok if benchmark doesn't measure bandwith or time, and
+ it is (presumably) faster.
+ The table TimedResults cannot be used yet.
+ The benchmark suite must be run in the viff/benchmark dir due to scp
+ copy of the benchmark classes, ok?
+ Better documentation.
+ Security. Currently, password to the mysql db is supplied on the
+ command line both in example.py and when executing benchmark.py on
+ each host. This is not secure as everyone having access to the
+ benchmark computers will be able to read off the password using
+ e.g. ps -eLf | grep benchmark.py.
+ Can't get the popen objects to finish when ssh finishes, so I've done a
+ hack by letting each benchmark write COMPLETED WITH RETURNCODE x as
+ the last thing. Not pretty, but it works (at least now..)
+ Non-VIFF benchmarks are always run on the same host that starts the
+ suite. It probably would be nice to be able to specify which host
+ to use for the benchmark.
+ VIFFBenchmark reports back to master on the fly. Would also be nice
+ if non-VIFF benchmarks did that.
+
+ Whishlist:
+ Make it even easier for user by letting him specify that this protocol
+ should be timed, or this protocol should be measured w.r.t memory
+ usage, etc. This should be done in a way that doesn't prohibit user
+ from writing his own benchmarks measuring things in his own way. This
+ will also help make the typical benchmarks of timing, memory etc. more
+ comparable.
+
+ Support for other databases than mysql.
+
+ Code would maybe benefit from using a ORM (Object-Relational Mapping)
+ instead of using raw SQL statements.
+
+"""
+
+import time
+import copy
+import os
+import select
+import sys
+import re
+from viff.config import load_config
+from twisted.internet import reactor
+from database import Database
+from util import exec_on_host
+
+def parse(args):
+ """ Creates a dict form a list of strings a la k1=val1. """
+ res = {}
+ for a in args[1:]:
+ # Note: This allows attribute names to contain '='
+ s = a.rsplit("=",1)
+ # TODO: Info about attribute types should be included. Here, we
+ # simply treat an attribute as int if possible and otherwise
+ # as a string.
+ try:
+ res[s[0]] = int(s[1])
+ except ValueError:
+ res[s[0]] = s[1]
+ return res
+
+
+class Suite:
+
+ # TODO: Change revision to hg_revision.
+
+ """
+ hosts: A list of (hostname, port). This is the list of all available hosts
+ that can be used for the benchmark and the port numbers that should be
+ used on each host. As hostname you can either supply a real hostname,
+ e.g. camel17.daimi.au.dk, or an integer which then refers to the
+ host_id in the benchmark database. This is useful if one host like
+ camel17.daimi.au.dk is used with several configurations and thus should
+ be treated as multiple "hosts". If multiple such host configurations
+ exists in the database and only the string hostname is given, an
+ exception is thrown.
+
+ Note that this should be the complete list of hosts that can be used
+ for the benchmark. Some of the benchmarks will only use a subset of the
+ hosts listed.
+
+ Note also that the hosts are not nescessarily given protocol ids in the
+ same order as this list.
+
+ user: The username that should be used to log into the benchmark hosts.
+ TODO: Defaults to the username of the user executing this script.
+
+ work_dir: A directory on a shared filesystem that all the benchmark slaves
+ have access to. Here, viff will be checked out and temporary files will
+ possibly be written. TODO: Defaults to same directory as on the master
+ host (e.g. where this script is executed).
+
+ database: The benchmark database. Note that the credentials of this
+ database should be set up to provide write access to some of the
+ database tables.
+
+ revision: The VIFF revision which should be benchmarked. Defaults to tip.
+
+ hg_repository: The repository from where the revision is checked out. This
+ could be the main repository http://hg.viff.dk/viff, but you can also
+ use your own clone such as ssh://***@fh.daimi.au.dk/viff-benchmark. If
+ no hg_repository is supplied it is assumed that a hg repository clone
+ already exists in work_dir/viff and the needed revision is simply
+ checked out from this clone. TODO: Add support for pull.
+
+ viff_dir: Where VIFF should be installed. When the appropriate VIFF
+ revision has been checked out, the suite executes
+
+ python setup.py install --home=viff_dir
+
+ on one of the available hosts. Make sure that your PATH and PYTHONPATH
+ are set up correspondingly, e.g. as described in "Installing from
+ Source" at http://viff.dk/doc/install.html. One example could be to use
+
+ viff_dir = $HOME/opt
+
+ and to include these in your .bashrc file:
+
+ export PYTHONPATH=$PYTHONPATH:$HOME/opt/lib/python
+ export PATH=$PATH:$HOME/opt/bin
+
+ """
+ def __init__(self, database, hosts, user, work_dir, viff_dir,
+ revision=None,
+ hg_repository=None):
+ self.user = user
+ self.viff_dir = viff_dir
+ self.revision = revision
+ self.benchmarks = {}
+ self.host_name = {}
+ self.host_port = {}
+ self.database = database
+ self.hg_repository = hg_repository
+ self.suite_id = self.database.create_suite(revision)
+ self.work_dir = work_dir
+ for hostname, port in hosts:
+ if type(hostname) is str:
+ host_id = database.get_host_id(hostname)
+ self.host_name[host_id] = hostname
+ self.host_port[host_id] = port
+ else:
+ self.host_name[hostname] = database.get_host_name(hostname)
+ self.host_port[hostname] = port
+
+ def setup(self):
+ print "Setting up Suite"
+ somehost = self.host_name.values()[0]
+
+ # If user supplied a hg_repository, then check out viff from there.
+ # TODO: Take care using rm -rf in a script like this!!!
+ if self.hg_repository:
+ exec_on_host(self.user, somehost,
+ ["rm -rf %s/viff; cd %s; hg clone %s viff" %
+ (self.work_dir, self.work_dir, self.hg_repository)])
+
+ # If user supplied revision, check it out. Otherwise, check out the tip.
+ if self.revision:
+ rev = "--rev %s" % self.revision
+ else:
+ rev = ""
+ exec_on_host(self.user, somehost,
+ ["cd %s/viff; hg update --clean %s" %
+ (self.work_dir, rev)])
+
+ # Build VIFF.
+ exec_on_host(self.user, somehost,
+ ["cd %s/viff; python setup.py install --home=%s" %
+ (self.work_dir, self.viff_dir)])
+
+ def teardown(self):
+ print "Tearing down Suite"
+ # TODO: Remove local checkout but not the hg clone?
+
+ def add_benchmark(self, benchmark):
+ """ Note that if the benchmark has already database parameters, e.g.
+ db_host, db_user, db_password, db_port, db_name, these are used to
+ report the result. If they are not set, the same database parameters
+ are used as those given when creating the Suite."""
+
+ # TODO: Hack -> Benchmarks name is derived from class name.
+ benchmark_name = str(benchmark.__class__).split('.')[-1]
+ benchmark_id = self.database.add_benchmark(self.suite_id,
+ benchmark_name)
+
+ # Database host, port, name, user and passwd supplied to the benchmark
+ # overrides those supplied to the suite.
+ #
+ # This makes it possible for at benchmark to report back to another
+ # database or using other credentials than the administrator
+ # credentials that must be supplied in the database given when
+ # initializing the Suite. Normally,it should be enough to use a
+ # 'benchmark' user in the database that has only write access to the
+ # Result and TimedResult tables.
+ benchmark.attr['benchmark_id'] = benchmark_id
+ if not 'db_host' in benchmark.attr.keys():
+ benchmark.attr['db_host'] = self.database.db_host
+ if not 'db_name' in benchmark.attr.keys():
+ benchmark.attr['db_name'] = self.database.db_name
+ if not 'db_user' in benchmark.attr.keys():
+ benchmark.attr['db_user'] = self.database.db_user
+ if not 'db_passwd' in benchmark.attr.keys():
+ benchmark.attr['db_passwd'] = self.database.db_passwd
+ if not 'db_port' in benchmark.attr.keys():
+ benchmark.attr['db_port'] = self.database.db_port
+ self.benchmarks[benchmark_id] = benchmark
+ for atr, val in benchmark.attr.items():
+ self.database.add_benchmark_attribute(benchmark_id, atr, val)
+
+ def run(self):
+ self.setup()
+ for benchmark in self.benchmarks.values():
+ benchmark.run_on_master(self)
+ self.teardown()

[Thomas Jakobsen]

Just realized that some of the comments in the suite.py are outdated.
This is confusing. Here is an update:


diff -r e01382aea278 apps/benchmark/suite.py
--- a/apps/benchmark/suite.py Fri Nov 07 01:11:24 2008 +0100
+++ b/apps/benchmark/suite.py Fri Nov 07 09:45:58 2008 +0100
@@ -45,7 +45,6 @@


Issues:
- Looks like the first run takes longer time than the rest. Is that ok?
Results for all runs for a benchmark is kept in memory at once.
Problem?
Currently master only works on *nix since we use select.
@@ -55,7 +54,6 @@
system and that the dir supplied to the suite is on this shared
file system.
Can we run multiple benchmark slaves on one host?
- Currently, default parameters are always used, e.g. use_ssl = no.
Support for string, float and boolean results. Currently only integers
(up to Mysql.BIGNUM) are supported.
We assume that python setup.py install --home=$HOME/opt overwrites old
@@ -67,8 +65,8 @@
runs; this is ok if benchmark doesn't measure bandwith or time, and
it is (presumably) faster.
The table TimedResults cannot be used yet.
- The benchmark suite must be run in the viff/benchmark dir due to scp
- copy of the benchmark classes, ok?
+ The benchmark suite must be run in the viff/apps/benchmark dir due to
+ scp copy of the benchmark classes, ok?
Better documentation.
Security. Currently, password to the mysql db is supplied on the
command line both in example.py and when executing benchmark.py on
@@ -78,9 +76,6 @@
Can't get the popen objects to finish when ssh finishes, so I've done a
hack by letting each benchmark write COMPLETED WITH RETURNCODE x as
the last thing. Not pretty, but it works (at least now..)
- Non-VIFF benchmarks are always run on the same host that starts the
- suite. It probably would be nice to be able to specify which host
- to use for the benchmark.
VIFFBenchmark reports back to master on the fly. Would also be nice
if non-VIFF benchmarks did that.

[Martin Geisler]

Post by Thomas Pelle Jakobsen
# HG changeset patch
# Date 1226015502 -3600
# Node ID 75e5113f27777649c2001b1221c9717d8d375423
# Parent 0985564470de2bb2c5247effd30dc40f74048f17
Added benchmark suite class.
diff -r 0985564470de -r 75e5113f2777 apps/benchmark/suite.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/suite.py Fri Nov 07 00:51:42 2008 +0100
@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-

Are there any non-ASCII characters here that require an explicit
encoding?

Post by Thomas Pelle Jakobsen
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+"""
+ Main goal is to make it easy to write and run VIFF benchmarks while
+ at the same time not limiting the kind of benchmarks that could be
+ made. Also, the solution should scale as the number of benchmarks
+ increase. Finally, the benchmark data should be collected in a central
+ database rather than as a bunch of text files.
+
+ A benchmark is created by subclassing either Benchmark or
+ VIFFBenchmark. By subclassing VIFFBenchmark, you need only specify the
+ runtime that is to be used, the protocol that is to be benchmarked, and
+ the code that does the measure. See the examples.
+
+
+ When setting up a suite, one can specify which revision to benchmark.
+ Uses ssh and thus benefits from key agents like ssh-agent.
+ Non-viff benchmarks can be done by subclassing Benchmark while VIFF
+ benchmarks can easily be benchmarked by subclassing VIFFBenchmark.
+ Data is automatically stored in central database.
+ Benchmark results are stored for each individual run. This enables all
+ kind of statistics to be applied at a later time, since no info is
+ lost. E.g. storing only mean values would prevent calculating
+ conficence intervals and other statistical stuff.
+ Dynamic approach: All parameter names are available except a few
+ pre-defined, e.g. work_dir, player_id, ... (TODO: List them here.)
+
+
+ Looks like the first run takes longer time than the rest. Is that ok?
+ Results for all runs for a benchmark is kept in memory at once.
+ Problem?
+ Currently master only works on *nix since we use select.
+ Benchmark slaves works only on *nix now due to small issues with pipes,
+ etc.
+ It is assumed that all benchmark hosts have access to the same file
+ system and that the dir supplied to the suite is on this shared
+ file system.
+ Can we run multiple benchmark slaves on one host?
+ Currently, default parameters are always used, e.g. use_ssl = no.
+ Support for string, float and boolean results. Currently only integers
+ (up to Mysql.BIGNUM) are supported.
+ We assume that python setup.py install --home=$HOME/opt overwrites old
+ installation nicely.
+ Log err and out nicely to various files and/or use python logging.
+ Write unit tests!
+ All random seeds should come in as parameters.
+ Currently, the runs are done in sequence. Maybe also support parallel
+ runs; this is ok if benchmark doesn't measure bandwith or time, and
+ it is (presumably) faster.
+ The table TimedResults cannot be used yet.
+ The benchmark suite must be run in the viff/benchmark dir due to scp
+ copy of the benchmark classes, ok?
+ Better documentation.
+ Security. Currently, password to the mysql db is supplied on the
+ command line both in example.py and when executing benchmark.py on
+ each host. This is not secure as everyone having access to the
+ benchmark computers will be able to read off the password using
+ e.g. ps -eLf | grep benchmark.py.
+ Can't get the popen objects to finish when ssh finishes, so I've done a
+ hack by letting each benchmark write COMPLETED WITH RETURNCODE x as
+ the last thing. Not pretty, but it works (at least now..)
+ Non-VIFF benchmarks are always run on the same host that starts the
+ suite. It probably would be nice to be able to specify which host
+ to use for the benchmark.
+ VIFFBenchmark reports back to master on the fly. Would also be nice
+ if non-VIFF benchmarks did that.

Wow, you've made a big system... I cannot help but think that it does
half of what buildbot already does.

Post by Thomas Pelle Jakobsen
+ """ Creates a dict form a list of strings a la k1=val1. """
+ res = {}

Why not the first item? I have a hunch that you use this for command
line parsing -- would the optparse module not be better for this? At
least add some doctests that show what this function really does.

Post by Thomas Pelle Jakobsen
+ # Note: This allows attribute names to contain '='
+ s = a.rsplit("=",1)
+ # TODO: Info about attribute types should be included. Here, we
+ # simply treat an attribute as int if possible and otherwise
+ # as a string.
+ res[s[0]] = int(s[1])
+ res[s[0]] = s[1]
+ return res
+
+
+
+ # TODO: Change revision to hg_revision.
+
+ """
+ hosts: A list of (hostname, port). This is the list of all available hosts
+ that can be used for the benchmark and the port numbers that should be
+ used on each host. As hostname you can either supply a real hostname,
+ e.g. camel17.daimi.au.dk, or an integer which then refers to the
+ host_id in the benchmark database. This is useful if one host like
+ camel17.daimi.au.dk is used with several configurations and thus should
+ be treated as multiple "hosts". If multiple such host configurations
+ exists in the database and only the string hostname is given, an
+ exception is thrown.
+
+ Note that this should be the complete list of hosts that can be used
+ for the benchmark. Some of the benchmarks will only use a subset of the
+ hosts listed.
+
+ Note also that the hosts are not nescessarily given protocol ids in the
+ same order as this list.
+
+ user: The username that should be used to log into the benchmark hosts.
+ TODO: Defaults to the username of the user executing this script.
+
+ work_dir: A directory on a shared filesystem that all the benchmark slaves
+ have access to. Here, viff will be checked out and temporary files will
+ possibly be written. TODO: Defaults to same directory as on the master
+ host (e.g. where this script is executed).
+
+ database: The benchmark database. Note that the credentials of this
+ database should be set up to provide write access to some of the
+ database tables.
+
+ revision: The VIFF revision which should be benchmarked. Defaults to tip.
+
+ hg_repository: The repository from where the revision is checked out. This
+ could be the main repository http://hg.viff.dk/viff, but you can also
+ no hg_repository is supplied it is assumed that a hg repository clone
+ already exists in work_dir/viff and the needed revision is simply
+ checked out from this clone. TODO: Add support for pull.
+
+ viff_dir: Where VIFF should be installed. When the appropriate VIFF
+ revision has been checked out, the suite executes
+
+ python setup.py install --home=viff_dir

VIFF does not really need to be installed to be used. If you put the
root of a VIFF checkout in the PYTHONPATH, then it just works. That
might be easier and simpler than installing it.

In any case, it would be cool if the directories were created with the
tempfile module:

http://docs.python.org/library/tempfile.html

That would make it unnecessary for the user to specify them and easier
to cleanup since they can just be nuked afterwards.

Post by Thomas Pelle Jakobsen
+ on one of the available hosts. Make sure that your PATH and PYTHONPATH
+ are set up correspondingly, e.g. as described in "Installing from
+ Source" at http://viff.dk/doc/install.html. One example could be to use
+
+ viff_dir = $HOME/opt
+
+
+ export PYTHONPATH=$PYTHONPATH:$HOME/opt/lib/python
+ export PATH=$PATH:$HOME/opt/bin

Please don't mention bash here -- not everybody used it. It is bad
enough that we do it in the installation guide :-)

Post by Thomas Pelle Jakobsen
+ """
+ def __init__(self, database, hosts, user, work_dir, viff_dir,
+ revision=None,
+ self.user = user
+ self.viff_dir = viff_dir
+ self.revision = revision
+ self.benchmarks = {}
+ self.host_name = {}
+ self.host_port = {}
+ self.database = database
+ self.hg_repository = hg_repository
+ self.suite_id = self.database.create_suite(revision)
+ self.work_dir = work_dir
+ host_id = database.get_host_id(hostname)
+ self.host_name[host_id] = hostname
+ self.host_port[host_id] = port
+ self.host_name[hostname] = database.get_host_name(hostname)
+ self.host_port[hostname] = port
+
+ print "Setting up Suite"
+ somehost = self.host_name.values()[0]
+
+ # If user supplied a hg_repository, then check out viff from there.
+ # TODO: Take care using rm -rf in a script like this!!!
+ exec_on_host(self.user, somehost,
+ ["rm -rf %s/viff; cd %s; hg clone %s viff" %
+ (self.work_dir, self.work_dir, self.hg_repository)])
+
+ # If user supplied revision, check it out. Otherwise, check out the tip.
+ rev = "--rev %s" % self.revision
+ rev = ""
+ exec_on_host(self.user, somehost,
+ ["cd %s/viff; hg update --clean %s" %
+ (self.work_dir, rev)])
+
+ # Build VIFF.
+ exec_on_host(self.user, somehost,
+ ["cd %s/viff; python setup.py install --home=%s" %
+ (self.work_dir, self.viff_dir)])
+
+ print "Tearing down Suite"
+ # TODO: Remove local checkout but not the hg clone?
+
+ """ Note that if the benchmark has already database parameters, e.g.
+ db_host, db_user, db_password, db_port, db_name, these are used to
+ report the result. If they are not set, the same database parameters
+ are used as those given when creating the Suite."""
+
+ # TODO: Hack -> Benchmarks name is derived from class name.
+ benchmark_name = str(benchmark.__class__).split('.')[-1]

Hmm, well. Is this related to the scp-stuff I think I've seen somewhere
else? If so, then since everybody shares the same filesystem, there
might be an easier and nicer way... I don't know.

Post by Thomas Pelle Jakobsen
+ # initializing the Suite. Normally,it should be enough to use a
+ # 'benchmark' user in the database that has only write access to the
+ # Result and TimedResult tables.
+ benchmark.attr['benchmark_id'] = benchmark_id

I think I said this previously, but 'x in d' == 'x in d.keys()'.

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226015530 -3600
# Node ID 693761d8181f39ccbbf699b222d97c761127b461
# Parent 75e5113f27777649c2001b1221c9717d8d375423
Added basic benchmark class.

diff -r 75e5113f2777 -r 693761d8181f apps/benchmark/benchmark.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/benchmark.py Fri Nov 07 00:52:10 2008 +0100
@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import time
+import copy
+import os
+import sys
+import subprocess
+import re
+from database import Database
+from util import exec_on_host
+
+
+def parse_args(args):
+ """ Creates a dict from a list of strings a la k1=val1. """
+ res = {}
+ for a in args[1:]:
+ # Note: This allows attribute names to contain '='
+ s = a.rsplit("=",1)
+ # TODO: Info about attribute types should be included. Here, we
+ # simply treat an attribute as int if possible and otherwise
+ # as a string.
+ try:
+ res[s[0]] = int(s[1])
+ except ValueError:
+ res[s[0]] = s[1]
+ return res
+
+class Benchmark:
+ """Extend this class and override its benchmark method with code that
+ produces benchmark results, you
+
+ The benchmark is run on an arbitrary host from the list of hosts that is
+ supplied when creating the Suite.
+
+ These are reserved attributes that shouldn't be overridden or removed:
+
+ benchmark_id:
+ Internal database id for the current benchmark.
+ host_id:
+ Internal database id for the current host.
+ db_host, db_port, db_name, db_user, db_passwd:
+ Credentials and info for the database. Same as was given to the
+ database passed to the suite constructor.
+
+ and these must always be set manually:
+
+ runs:
+ Number of times to repeat the benchmark.
+
+ and additionally, when extending VIFFBenchmark,
+
+ n:
+ The number of players that should participate in current
+ benchmark.
+ threshold: The security threshold for the current benchmar.
+ player_id:
+ Unique non-zero integer assigned to the current host.
+ use_ssl:
+ This is True by default, but it should still not be removed or
+ overwritten.
+
+ """
+ def __init__(self, attr):
+ self.attr = attr
+
+ def attribute(self, name, val):
+ self.attr[name] = val
+
+ def report_result(self, result):
+ print "Reporting to:"
+ print " db_host=" + self.attr['db_host']
+ print " db_name=" + self.attr['db_name']
+ print " db_port=" + str(self.attr['db_port'])
+ print " db_user=" + self.attr['db_user']
+ print "Result:"
+ print str(result)
+ database = Database(host=self.attr['db_host'],
+ db=self.attr['db_name'],
+ user=self.attr['db_user'],
+ passwd=self.attr['db_passwd'],
+ port=self.attr['db_port'])
+ database.report_result(self.attr['benchmark_id'], self.attr['host_id'],
+ result)
+
+ # This method is invoked once at each host
+ # - order: ?
+ def setup(self, attr):
+ print "Setting up Benchmark"
+
+ # This method is invoked only once for a benchmark
+ # - on the host that starts it all
+ def setup_once(self, suite):
+ print "Setup once in Benchmark"
+ # TODO: Hack -> Benchmarks must then exist only in files with same name
+ filename = str(self.__class__).split('.')[-1]
+ if sys.platform == 'win32':
+ scp = "pscp"
+ else:
+ scp = "scp"
+ # Note that we use the benchmark classes from the repository in which
+ # the suite is run. It would be confusing to use the classes in the
+ # revisions that are checked out for benchmarking.
+ #
+ # TODO: However, I'm not sure whether generate-config-files and
+ # generate-certificates from the "master" repository or from the
+ # benchmark repositories should be used?!
+ #
+ hostname = suite.host_name.values()[0] # Execute this on arbitrary host
+
+ # TODO: This is hacky. Not sure whether to copy these or to
+ # supply work_dir and assume that they are present at work_dir:
+ # Currently, only benchmarks in the example folder is copied.
+ # Would be better to get the absolute path for a class from python.
+ subprocess.call([scp, "benchmark.py", suite.user + "@" + hostname +
+ ":" + suite.work_dir])
+ subprocess.call([scp, "database.py", suite.user + "@" + hostname +
+ ":" + suite.work_dir])
+ subprocess.call([scp, "suite.py", suite.user + "@" + hostname +
+ ":" + suite.work_dir])
+ subprocess.call([scp, "util.py", suite.user + "@" + hostname +
+ ":" + suite.work_dir])
+ subprocess.call([scp, "examples/" + filename + ".py", suite.user + "@" +
+ hostname + ":" + suite.work_dir])
+
+ def teardown(self, attr):
+ # Remove config files, etc.
+ print "Tearing down Benchmark"
+
+ # This is the part that actually is to be benchmarked
+ # Return: Dict containing attribute->value pairs
+ def benchmark(self, attr):
+ raise NotImplemented("Override this abstract method in a subclass.")
+
+ def run_on_master(self, suite):
+ self.setup_once(suite)
+ # TODO: Hack -> Benchmarks must then exist only in files with same name
+ filename = str(self.__class__).split('.')[-1] + ".py"
+ # TODO: Would like to run ./benchmarkname.py instead, but that
+ # doesn't work on Windows.
+ host_id = suite.host_name.keys()[0] # Use arbitrary host.
+ command = "cd %s; python -u %s" % (suite.work_dir, filename)
+ command = command + " host_id=%d" % host_id
+ for attr, val in self.attr.items():
+ command = command + " " + attr + "=" + str(val)
+ exec_on_host(suite.user, suite.host_name[host_id], [command], sync=True)
+
+ def run_on_slave(self):
+ self.setup(self.attr)
+ # TODO: Might also write to text file and parse - more robust?
+ results = []
+ for run in range(int(self.attr['runs'])):
+
+ print "**** Run %d" % run
+ res = self.benchmark(self.attr)
+ print "**** Result: " + str(res)
+ results.append(res)
+ self.report_result(results)
+ self.teardown(self.attr)
+ sys.exit(0)
diff -r 75e5113f2777 -r 693761d8181f apps/benchmark/util.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/util.py Fri Nov 07 00:52:10 2008 +0100
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import time
+import copy
+import os
+import select
+import sys
+import subprocess
+import re
+from viff.config import load_config
+from twisted.internet import reactor
+from database import Database
+
+
+#
+# TODO: Some clean-up nessecary, e.g. on remote host?
+#
+def exec_on_host(user, host, commands, sync=True, verbose=True):
+ """If mode = sync, the method blocks until command has finished and returns
+ none (throws exception if errorcode != 0). If mode = async, a
+ subprocess.Popen object, p, containing the running command is returned.
+ """
+ if verbose:
+ print "Starting command on " + user + "@" + host + ":"
+ print ''.join([command.strip() + " " for command in commands])
+ if sys.platform == 'win32':
+ cmd = ["plink", "-x", "-a"] + [user + "@" + host] + commands
+ else:
+ cmd = ["ssh", "-x", "-a"] + [user + "@" + host] + commands
+
+ proc = subprocess.Popen(cmd,
+ shell=False,
+ #stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ bufsize=1 # Line-buffered.
+ )
+
+ if sync:
+ stdout, stderr = proc.communicate()
+ retval = proc.returncode
+ if verbose and stdout:
+ print host + "(stdout): " + stdout
+ if verbose and stderr:
+ print host + "(stderr): " + stderr
+ if not retval == 0:
+ raise Exception("Process " + str(cmd) +
+ " failed with return code " + str(retval))
+ else:
+ return proc
+

[Martin Geisler]

Post by Thomas Pelle Jakobsen
# HG changeset patch
# Date 1226015530 -3600
# Node ID 693761d8181f39ccbbf699b222d97c761127b461
# Parent 75e5113f27777649c2001b1221c9717d8d375423
Added basic benchmark class.
diff -r 75e5113f2777 -r 693761d8181f apps/benchmark/benchmark.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/benchmark.py Fri Nov 07 00:52:10 2008 +0100
@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import time
+import copy
+import os
+import sys
+import subprocess
+import re
+from database import Database
+from util import exec_on_host
+
+

Is this not just like the parse function I found in the patch before?

Post by Thomas Pelle Jakobsen
+ """ Creates a dict from a list of strings a la k1=val1. """
+ res = {}
+ # Note: This allows attribute names to contain '='
+ s = a.rsplit("=",1)
+ # TODO: Info about attribute types should be included. Here, we
+ # simply treat an attribute as int if possible and otherwise
+ # as a string.
+ res[s[0]] = int(s[1])
+ res[s[0]] = s[1]
+ return res
+
+ """Extend this class and override its benchmark method with code that
+ produces benchmark results, you

Cut-off sentence?

Post by Thomas Pelle Jakobsen
+ The benchmark is run on an arbitrary host from the list of hosts that is
+ supplied when creating the Suite.
+
+
+ Internal database id for the current benchmark.
+ Internal database id for the current host.
+ Credentials and info for the database. Same as was given to the
+ database passed to the suite constructor.
+
+
+ Number of times to repeat the benchmark.
+
+ and additionally, when extending VIFFBenchmark,
+
+ The number of players that should participate in current
+ benchmark.
+ threshold: The security threshold for the current benchmar.
+ Unique non-zero integer assigned to the current host.
+ This is True by default, but it should still not be removed or
+ overwritten.
+
+ """
+ self.attr = attr
+
+ self.attr[name] = val

Hmm, this is not Java :-) You might want to look at this nice attrdict
class: (first patch hunk)

http://www.bitbucket.org/mg/hgchart/changeset/01bd2a3a32ea/

It allows you to use self.attr.db_host instead of self.attr['db_host'],
and if you let the Benchmark class inherit from attrdict you could skip
the 'attr' layer.

Post by Thomas Pelle Jakobsen
+ print "Reporting to:"
+ print " db_host=" + self.attr['db_host']
+ print " db_name=" + self.attr['db_name']
+ print " db_port=" + str(self.attr['db_port'])
+ print " db_user=" + self.attr['db_user']

The print statement allows you to write

print " foo:", foo
print " bar:", bar

and it will automatically insert a space.

Post by Thomas Pelle Jakobsen
+ print "Result:"
+ print str(result)
+ database = Database(host=self.attr['db_host'],
+ db=self.attr['db_name'],
+ user=self.attr['db_user'],
+ passwd=self.attr['db_passwd'],
+ port=self.attr['db_port'])
+ database.report_result(self.attr['benchmark_id'], self.attr['host_id'],
+ result)
+
+ # This method is invoked once at each host
+ # - order: ?
+ print "Setting up Benchmark"
+
+ # This method is invoked only once for a benchmark
+ # - on the host that starts it all
+ print "Setup once in Benchmark"
+ # TODO: Hack -> Benchmarks must then exist only in files with same name
+ filename = str(self.__class__).split('.')[-1]
+ scp = "pscp"
+ scp = "scp"
+ # Note that we use the benchmark classes from the repository in which
+ # the suite is run. It would be confusing to use the classes in the
+ # revisions that are checked out for benchmarking.
+ #
+ # TODO: However, I'm not sure whether generate-config-files and
+ # generate-certificates from the "master" repository or from the
+ # benchmark repositories should be used?!
+ #
+ hostname = suite.host_name.values()[0] # Execute this on arbitrary host
+
+ # TODO: This is hacky. Not sure whether to copy these or to
+ # Currently, only benchmarks in the example folder is copied.
+ # Would be better to get the absolute path for a class from python.
+ ":" + suite.work_dir])

When building strings, the Pythonic way is to do

"%s@%s:%s" % (suite.user, hostname, suite.work_dir)

which created fewer intermediate str objects and gives you more ways to
format the inserted strings (not that you need that here).

Post by Thomas Pelle Jakobsen
+ ":" + suite.work_dir])
+ ":" + suite.work_dir])
+ ":" + suite.work_dir])
+ hostname + ":" + suite.work_dir])

Who creates the "examples/" folder on the receiving side? I guess the
copy fails otherwise.

Post by Thomas Pelle Jakobsen
+ # Remove config files, etc.
+ print "Tearing down Benchmark"
+
+ # This is the part that actually is to be benchmarked
+ # Return: Dict containing attribute->value pairs

The type is called 'dict', always with small 'd'.

Post by Thomas Pelle Jakobsen
+ raise NotImplemented("Override this abstract method in a subclass.")
+
+ self.setup_once(suite)
+ # TODO: Hack -> Benchmarks must then exist only in files with same name
+ filename = str(self.__class__).split('.')[-1] + ".py"
+ # TODO: Would like to run ./benchmarkname.py instead, but that
+ # doesn't work on Windows.
+ host_id = suite.host_name.keys()[0] # Use arbitrary host.
+ command = "cd %s; python -u %s" % (suite.work_dir, filename)

Why is '-u' needed when executing Python?

Post by Thomas Pelle Jakobsen
+ command = command + " host_id=%d" % host_id
+ command = command + " " + attr + "=" + str(val)

This applied to the immutable strings in Java too: this could require
copying O(n^2) bytes. The problem is the intermediate copies created in
most Python versions:

http://wiki.python.org/moin/PythonSpeed/PerformanceTips#StringConcatenation

Post by Thomas Pelle Jakobsen
+ exec_on_host(suite.user, suite.host_name[host_id], [command], sync=True)
+
+ self.setup(self.attr)
+ # TODO: Might also write to text file and parse - more robust?
+ results = []
+
+ print "**** Run %d" % run
+ res = self.benchmark(self.attr)
+ print "**** Result: " + str(res)
+ results.append(res)
+ self.report_result(results)
+ self.teardown(self.attr)
+ sys.exit(0)
diff -r 75e5113f2777 -r 693761d8181f apps/benchmark/util.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/util.py Fri Nov 07 00:52:10 2008 +0100
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-

[...]

Post by Thomas Pelle Jakobsen
+# TODO: Some clean-up nessecary, e.g. on remote host?
+#
+ """If mode = sync, the method blocks until command has finished and returns
+ none (throws exception if errorcode != 0). If mode = async, a
+ subprocess.Popen object, p, containing the running command is returned.
+ """
+ print ''.join([command.strip() + " " for command in commands])

This should be the same and will not end with a trailing space:

print ' '.join([command.strip() for command in commands])

you could even write it in functional style:

print ' '.join(map(str.strip, commands))
This kind of platform dependent code is scary... I'm even considering
using the SSH client in Twisted instead to get something platform
independent...

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226015362 -3600
# Node ID aa8ba57c7833f0d68792db7c36782e3ba9fbc194
# Parent dd640eac89c544273c9e1a7230af6b44c975bf01
Added example benchmark suite.

diff -r dd640eac89c5 -r aa8ba57c7833 apps/benchmark/example.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/example.py Fri Nov 07 00:49:22 2008 +0100
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+from database import Database
+from suite import Suite
+from examples.ComparisonToft05 import ComparisonToft05
+from examples.LocalMult import LocalMult
+import sys
+
+if __name__ == "__main__":
+ database = Database(host="localhost",
+ db="viff_benchmark",
+ user="root",
+ passwd=sys.argv[1])
+
+ """
+ These statements reset the database and define the needed attributes
+ and hosts. They should only be executed once for a certain benchmark
+ database.
+
+ TODO: Move the predefined attributes, e.g. n, threshold, etc. to
+ constructors of Benchmark and VIFFBenchmark. Ideally, user should only
+ need to add the attributes that are specific for his benchmarks.
+
+ TODO: Make sure that adding an already existing host causes no harm.
+ """
+ database.reset()
+ database.add_host("camel17.daimi.au.dk")
+ database.add_host("camel18.daimi.au.dk")
+ database.add_host("camel19.daimi.au.dk")
+ database.add_host("camel21.daimi.au.dk")
+ database.add_host("camel22.daimi.au.dk")
+ database.add_attribute(name="execution_time", label="microseconds")
+ database.add_attribute(name="runs")
+ database.add_attribute(name="benchmark_id") # Redundant, but still need it.
+ database.add_attribute(name="db_user", type=Database.AttributeType.String)
+ database.add_attribute(name="db_passwd", type=Database.AttributeType.String)
+ database.add_attribute(name="db_name", type=Database.AttributeType.String)
+ database.add_attribute(name="db_host", type=Database.AttributeType.String)
+ database.add_attribute(name="db_port")
+ database.add_attribute(name="n")
+ database.add_attribute(name="threshold")
+ database.add_attribute(name="modulus", type=Database.AttributeType.String)
+ database.add_attribute(name="use_ssl", enum=['False', 'True'],
+ type=Database.AttributeType.Enumeration)
+
+
+
+ s = Suite(#hg_repository="http://hg.viff.dk/viff",
+ revision="383e6bfb4863",
+ database=database,
+ user="mas",
+ viff_dir="$HOME/opt",
+ work_dir="/users/mas/temp/viff-benchmark",
+ hosts=[("camel17.daimi.au.dk", 9734),
+ ("camel18.daimi.au.dk", 9843),
+ ("camel19.daimi.au.dk", 9962),
+ ("camel21.daimi.au.dk", 9963),
+ ("camel22.daimi.au.dk", 9964)])
+
+ s.add_benchmark(ComparisonToft05({\
+ 'runs': 10, \
+ 'n': 3, \
+ 'threshold': 1, \
+ 'modulus': 30916444023318367583, \
+ 'db_user': 'viff_benchmark',
+ 'db_host': 'tpj.dyndns.org',
+ 'db_passwd': sys.argv[2],
+ 'use_ssl': "True"}))
+
+ s.add_benchmark(ComparisonToft05({\
+ 'runs': 10, \
+ 'n': 5, \
+ 'threshold': 2, \
+ 'modulus': 30916444023318367583, \
+ 'db_user': 'viff_benchmark',
+ 'db_host': 'tpj.dyndns.org',
+ 'db_passwd': sys.argv[2],
+ 'use_ssl': "True"}))
+
+ s.add_benchmark(LocalMult({
+ 'runs': 5,
+ 'modulus': 30916444023318367583,
+ 'db_user': 'viff_benchmark',
+ 'db_host': 'tpj.dyndns.org',
+ 'db_passwd': sys.argv[2]}))
+
+ s.run()

[Martin Geisler]

Post by Thomas Pelle Jakobsen
# HG changeset patch
# Date 1226015362 -3600
# Node ID aa8ba57c7833f0d68792db7c36782e3ba9fbc194
# Parent dd640eac89c544273c9e1a7230af6b44c975bf01
Added example benchmark suite.
diff -r dd640eac89c5 -r aa8ba57c7833 apps/benchmark/example.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/example.py Fri Nov 07 00:49:22 2008 +0100
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+from database import Database
+from suite import Suite
+from examples.ComparisonToft05 import ComparisonToft05
+from examples.LocalMult import LocalMult

Modules are welcome to contain more than one class and should be named
in lowercase.

Post by Thomas Pelle Jakobsen
+import sys
+
+ database = Database(host="localhost",
+ db="viff_benchmark",
+ user="root",

Ahem... root?

Post by Thomas Pelle Jakobsen
+ passwd=sys.argv[1])

We should eventually put the password somewhere else since the command
line arguments are visible to all applications on the machine.

Post by Thomas Pelle Jakobsen
+
+ """
+ These statements reset the database and define the needed attributes
+ and hosts. They should only be executed once for a certain benchmark
+ database.
+
+ TODO: Move the predefined attributes, e.g. n, threshold, etc. to
+ constructors of Benchmark and VIFFBenchmark. Ideally, user should only
+ need to add the attributes that are specific for his benchmarks.
+
+ TODO: Make sure that adding an already existing host causes no harm.
+ """
+ database.reset()
+ database.add_host("camel17.daimi.au.dk")
+ database.add_host("camel18.daimi.au.dk")
+ database.add_host("camel19.daimi.au.dk")
+ database.add_host("camel21.daimi.au.dk")
+ database.add_host("camel22.daimi.au.dk")

I guess these host names should be read from a file...

Post by Thomas Pelle Jakobsen
+ database.add_attribute(name="execution_time", label="microseconds")
+ database.add_attribute(name="runs")
+ database.add_attribute(name="benchmark_id") # Redundant, but still need it.

Why do we need it? I guess the comment should be expanded to explain it.

Post by Thomas Pelle Jakobsen
+ database.add_attribute(name="db_user", type=Database.AttributeType.String)
+ database.add_attribute(name="db_passwd", type=Database.AttributeType.String)
+ database.add_attribute(name="db_name", type=Database.AttributeType.String)
+ database.add_attribute(name="db_host", type=Database.AttributeType.String)
+ database.add_attribute(name="db_port")
+ database.add_attribute(name="n")
+ database.add_attribute(name="threshold")
+ database.add_attribute(name="modulus", type=Database.AttributeType.String)
+ database.add_attribute(name="use_ssl", enum=['False', 'True'],
+ type=Database.AttributeType.Enumeration)
+
+
+
+ s = Suite(#hg_repository="http://hg.viff.dk/viff",
+ revision="383e6bfb4863",
+ database=database,
+ user="mas",
+ viff_dir="$HOME/opt",
+ work_dir="/users/mas/temp/viff-benchmark",
+ hosts=[("camel17.daimi.au.dk", 9734),
+ ("camel18.daimi.au.dk", 9843),
+ ("camel19.daimi.au.dk", 9962),
+ ("camel21.daimi.au.dk", 9963),
+ ("camel22.daimi.au.dk", 9964)])

We have to put this kind of configuration options in a .ini file or
similar. Or maybe a YAML file -- I like that format and it is extremely
simple to work with:

http://pyyaml.org/wiki/PyYAML

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226015551 -3600
# Node ID 1e8dec2648c1486537076da790745074218cdc78
# Parent 693761d8181f39ccbbf699b222d97c761127b461
Added VIFF benchmark class.

diff -r 693761d8181f -r 1e8dec2648c1 apps/benchmark/viff_benchmark.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/viff_benchmark.py Fri Nov 07 00:52:31 2008 +0100
@@ -0,0 +1,258 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import time
+import copy
+import os
+import select
+import sys
+import subprocess
+import re
+from optparse import OptionParser
+from viff.runtime import Runtime, create_runtime
+from viff.config import load_config
+from twisted.internet import reactor
+from database import Database
+from util import exec_on_host
+from benchmark import Benchmark, parse_args
+
+class VIFFBenchmark(Benchmark):
+ """Extend VIFFBenchmark to benchmark distributed VIFF protocols. The needed
+ VIFF players are then automatically set up on remote hosts.
+
+ A VIFFBenchmark object has predefined methods for generating config
+ files, certificates, and other VIFF-specific tasks.
+
+ See examples/ComparisonToft05.py for example usage.
+ """
+
+ def setup_once(self, suite):
+ """Do VIFF related setup for whole suite."""
+ Benchmark.setup_once(self, suite)
+ print "Setup once in VIFFBenchmark"
+
+ # TODO: This is code duplicated from Benchmark. How to avoid that?
+ filename = str(self.__class__).split('.')[-1]
+ if sys.platform == 'win32':
+ scp = "pscp"
+ else:
+ scp = "scp"
+ hostname = suite.host_name.values()[0] # Execute this on arbitrary host
+
+ subprocess.call([scp, "viff_benchmark.py", suite.user + "@" + hostname +
+ ":" + suite.work_dir])
+ subprocess.call([scp, "../generate-config-files.py", suite.user +
+ "@" + hostname + ":" + suite.work_dir])
+ subprocess.call([scp, "../generate-certificates.py", suite.user +
+ "@" + hostname + ":" + suite.work_dir])
+ if self.attr['use_ssl'] == "True":
+ self.generateCertificates(suite)
+ self.generateConfigFiles(suite)
+
+
+ def viff_options(self):
+ """ Util function that returns the basic viff options,
+ e.g. use-ssl, profile, etc."""
+ parser = OptionParser()
+ Runtime.add_options(parser)
+ args = []
+ if self.attr['use_ssl']:
+ args.append('--ssl')
+ else:
+ args.append('--no-ssl')
+ options, _ = parser.parse_args(args)
+ return options
+
+ def create_pre_runtime(self, player_id, players):
+ return create_runtime(player_id, players, self.attr['threshold'],
+ self.viff_options(),
+ runtime_class=self.get_runtime_class())
+
+ def get_runtime_class(self):
+ """ Override this method to create other than the basic runtime.
+ Returning None will result in the standard runtime being used."""
+ return Runtime
+
+ def protocol(self, rt, attr):
+ """ Returns a deferred where previous callback will return a
+ dict with attr name as keys and attr values as values."""
+ raise NotImplemented("Override this abstract method in a subclass.")
+
+ def err_handler(self, err):
+ print "Exception:", err.__class__
+ print str(err)
+ print "Stopping reactor in an unclean fashion"
+ reactor.stop()
+ sys.exit(42) # TODO: This doesn't work either!
+ # TODO: If rt has been created, then rather call
+ # rt.shutdown()
+
+ # Convenience method called from setup_once, e.g. on 'master' host
+ def generateCertificates(self, suite):
+ cmd = ["hostname; cd %s; python ./generate-certificates.py -n %d" %
+ (suite.work_dir, self.attr['n'])]
+ hostname = suite.host_name.values()[0] # Execute this on arbitrary host
+ exec_on_host(suite.user, hostname, cmd)
+
+ # Convenience method called from setup_once, e.g. on 'master' host
+ def generateConfigFiles(self, suite):
+ cmd = ["echo $PYTHONPATH; echo $HOME; hostname; python -V; " \
+ "cd %s; python ./generate-config-files.py" % suite.work_dir, \
+ "-n", str(self.attr['n']), "-t", str(self.attr['threshold'])]
+ for host_id in suite.host_name.keys()[:self.attr['n']]:
+ cmd.append("%s:%d" %
+ (suite.host_name[host_id],suite.host_port[host_id]))
+ hostname = suite.host_name.values()[0] # Execute this on arbitrary host
+ exec_on_host(suite.user, hostname, cmd)
+
+
+
+ def run_on_slave(self):
+ self.setup(self.attr)
+ player_id, players = load_config("player-%d.ini" %
+ self.attr['player_id'])
+ pre_runtime = self.create_pre_runtime(self.attr['player_id'], players)
+ results = []
+
+ # We schedule all protocol runs in one reactor.
+ # TODO: This might fill up stack with deferreds,
+ # so we need some way of inserting a break in Twisted?!
+
+ def schedule_runs(rt, attr):
+ sync = rt.synchronize()
+ for run in range(attr['runs']):
+ sync.addCallback(lambda _: self.protocol(rt, attr))
+ sync.addCallback(lambda res: results.append(res))
+ # TODO: Only need synchronize when doing tests of time,
+ # bandwidth, etc.
+ sync.addCallback(lambda _: rt.synchronize())
+ sync.addCallback(lambda _: rt.shutdown())
+
+ pre_runtime.addCallback(schedule_runs, self.attr)
+ pre_runtime.addErrback(self.err_handler) # Need this?
+ reactor.run()
+ self.report_result(results)
+ self.teardown(self.attr)
+ sys.exit(0)
+
+ def cleanup(self, suite, procs):
+ """If any process is still alive, kill it."""
+ # TODO: Hack -> All python processes owned by user is killed.
+ # Rather, save pid and use it here.
+ for host_id, p in procs.items():
+ if not p.returncode:
+ print "Sending SIGKILL to " + suite.host_name[host_id]
+ exec_on_host(suite.user,
+ suite.host_name[host_id],
+ ["hostname; pkill -9 -u " + suite.user +
+ " -x python; ps -u " + suite.user])
+ sys.exit(117) # TODO: What is the exit code strategy?
+
+ def wait_for_benchmark(self, suite, procs, poll_interval=1):
+ """ Takes as input a dictionary with host_ids as keys and popen objects
+ for the running processes as values. Blocks until all remote processes
+ have finished with returncode zero. All output from the remote
+ processes is forwarded to stdout (incl. errors). If at least one remote
+ process returns a non-zero returncode, an exception is raised
+ containing a list of hostnames that have not yet finished."""
+ # Take a shallow copy of the dict so we can remove objects
+ jobs = copy.copy(procs)
+ hostname = {} # host process id -> host name
+ hostid = {} # host process id -> host id
+ for host_id, job in jobs.items():
+ hostname[job.stdout.fileno()] = suite.host_name[host_id]
+ hostid[job.stdout.fileno()] = host_id
+ while len(jobs) > 0:
+ # TODO: This will not work on Windows? In order to fix this,
+ # have a look at http://code.activestate.com/recipes/440554/.
+ active = [job.stdout.fileno() for job in jobs.values()]
+ ready = select.select(active, [], [], poll_interval)[0]
+ for fd in ready:
+ # Don't use out.stdout.read() since select() is not aware of
+ # its buffers.
+ output = os.read(fd, 4096) # TODO: Too low max value here?
+ print "-----" + hostname[fd] + "--------------------:"
+ print output
+ print "-------------------------------------------"
+ # TODO: Do we need to dispose the read data somehow?
+ # TODO: We assume here that whole lines are read!
+ m = re.search('COMPLETED WITH RETURNCODE (\d)$', output,
+ re.MULTILINE)
+ if m:
+ print "removing host " + hostname[fd]
+ del jobs[hostid[fd]]
+ returncode = int(m.group(1))
+ if not returncode == 0:
+ raise Exception(hostname[fd] +
+ " failed with returncode " +
+ str(returncode), jobs)
+
+
+ # TODO: Would be nice to refactor by having
+ # Benchmark -> DistributedBenchmark -> VIFFBenchmark
+ # and to move the logic like this for running the benchmarks to
+ # the respective classes.
+ def run_benchmark(self, suite):
+ """ Starts a benchmark on the remote hosts and returns a dictionary
+ with host_id as keys and running popen objects as values."""
+ player_id = self.attr['n']
+ procs = {}
+ host_ids = suite.host_name.keys()[:self.attr['n']]
+ host_ids.reverse()
+ for host_id in host_ids:
+ # TODO: Hack -> Benchmarks must then exist only in files with same
+ # name.
+ filename = str(self.__class__).split('.')[-1] + ".py"
+ # TODO: Would like to run ./benchmarkname.py instead, but that
+ # doesn't work on Windows
+
+ command = "cd %s; python -u %s" % (suite.work_dir, filename)
+
+ # Hosts are given player_id's from 1 and up
+ # TODO: How to append in a nice way to strings?
+ command = command + " player_id=%d" % player_id
+ command = command + " host_id=%d" % host_id
+
+ # TTP with highest player_id must start, etc.
+ player_id = player_id - 1
+ for attr, val in self.attr.items():
+ command = command + " " + attr + "=" + str(val)
+ # TODO: Ugly HACK! Need this since ssh via subprocess doesn't
+ # stop.
+ command = command + "; echo COMPLETED WITH RETURNCODE $?"
+ procs[host_id] = exec_on_host(suite.user, suite.host_name[host_id],
+ [command], sync=False)
+ # TODO: Hack -> Two second to start twisted... should
+ # rather wait for some safe sign from process showing that
+ # the reactor has started!
+ time.sleep(2)
+ return procs
+
+ def run_on_master(self, suite):
+ self.setup_once(suite)
+ procs = self.run_benchmark(suite)
+ try:
+ self.wait_for_benchmark(suite, procs)
+ except KeyboardInterrupt:
+ print "Ctrl-C detected, cleaning up..."
+ self.cleanup(suite, procs)
+ except Exception, ex:
+ print str(ex)
+ print "Cleaning up..."
+ self.cleanup(suite, procs)

[Thomas Pelle Jakobsen]

# HG changeset patch
# User Thomas Pelle Jakobsen <***@daimi.au.dk>
# Date 1226016684 -3600
# Node ID e01382aea278689dde142f6461403e25b852f6d8
# Parent 1e8dec2648c1486537076da790745074218cdc78
Added database wrapper class.

diff -r 1e8dec2648c1 -r e01382aea278 apps/benchmark/database.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/database.py Fri Nov 07 01:11:24 2008 +0100
@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+import MySQLdb
+from enum import Enum
+
+# TODO: Need to secure this against injection attacks?
+# TODO: What about trancactions and auto_commit?
+# TODO: Do rollback when catching exceptions if multiple inserts, updates, etc.
+class Database:
+
+ # TODO: Is the assignment of integers to strings for
+ # the Enum class "implementation independent"?
+ AttributeType = Enum('Integer', 'String', 'Enumeration')
+
+ def __init__(self, host="localhost", db="benchmark", passwd="", user="root",
+ port=3306):
+ self.db_host = host
+ self.db_port = port
+ self.db_name = db
+ self.db_user = user
+ self.db_passwd = passwd
+ self.connection = MySQLdb.connect(host=host, db=db, passwd=passwd,
+ user=user, port=port)
+ self.cursor = self.connection.cursor()
+
+ def sql_query(self, query):
+ self.cursor.execute(query)
+ self.connection.commit()
+ # TODO: Doesn't scale to large result sets.
+ return self.cursor.fetchall()
+
+
+
+ """ Adds a host and returns it's id in the database as a long value."""
+ def add_host(self, name):
+ self.cursor.execute("INSERT INTO Host(name) VALUES('%s')" % name)
+ host_id = self.connection.insert_id()
+ self.connection.commit()
+ return host_id
+
+ def get_host_name(self, id):
+ self.cursor.execute("SELECT name FROM Host WHERE id = '%d'" % int(id))
+ res = self.cursor.fetchone()
+ if not res:
+ raise Exception("No host exists with id %d" % int(id))
+ return res[0]
+
+ def get_host_id(self, name):
+ self.cursor.execute("SELECT id FROM Host WHERE name = '%s'" % name)
+ res = self.cursor.fetchone()
+ if not res:
+ raise Exception("No host exists with name %s" % name)
+ if len(res) > 1:
+ raise Exception("More than one host exists with name %s" % name)
+ return res[0]
+
+ def report_result(self, benchmark_id, host_id, result):
+ for run in range(len(result)):
+ for attr, value in result[run].items():
+ self.cursor.execute("""
+ INSERT INTO Result(benchmark, host, run, attribute, value)
+ VALUES (%d, %d, %d, '%s', %d)""" %
+ (benchmark_id, host_id, run, attr, value))
+ self.connection.commit()
+
+ def create_suite(self, revision):
+ self.cursor.execute("INSERT INTO Suite(revision) VALUES ('%s')" %
+ (revision,))
+ suite_id = self.connection.insert_id()
+ self.connection.commit()
+ return suite_id
+
+ def add_benchmark(self, suite_id, benchmark_name):
+ self.cursor.execute("INSERT INTO VIFFBenchmark(suite, name) " +
+ "VALUES ('%s', '%s')" %
+ (suite_id, benchmark_name))
+ benchmark_id = self.connection.insert_id()
+ self.connection.commit()
+ return benchmark_id
+
+ def add_benchmark_attribute(self, benchmark_id, attribute, value):
+
+ # Find out whether attribute type is integer or string
+ query = "SELECT type FROM Attribute WHERE name = '%s'" % attribute
+ self.cursor.execute(query)
+ res = self.cursor.fetchone()
+ if not res:
+ raise Exception("Unknown attribute: %s" % attribute)
+ type = int(res[0])
+ if type == self.AttributeType.String.index:
+ self.cursor.execute("INSERT INTO StringBenchmarkAttribute(" \
+ "benchmark, attribute, value) VALUES " \
+ "('%d','%s','%s')" %
+ (benchmark_id, attribute, value))
+ elif type == self.AttributeType.Integer.index:
+ self.cursor.execute("INSERT INTO IntegerBenchmarkAttribute(" \
+ "benchmark, attribute, value) VALUES "
+ "('%d','%s','%d')" %
+ (benchmark_id, attribute, value))
+ elif type == self.AttributeType.Enumeration.index:
+ query = "SELECT value FROM Enumeration WHERE name = " \
+ "'%s' AND attribute = '%s'" % (value, attribute)
+ self.cursor.execute(query)
+ res = self.cursor.fetchone()
+ if not res:
+ raise Exception("Unknown enumeration value for %s: %s" %
+ (attribute, value))
+ enum_value = int(res[0])
+ self.cursor.execute("INSERT INTO IntegerBenchmarkAttribute(" \
+ "benchmark, attribute, value) VALUES " \
+ "('%d','%s','%d')" %
+ (benchmark_id, attribute, enum_value))
+ else:
+ raise Exception("Unknown attribute type: " + str(type))
+ self.connection.commit()
+
+
+ def add_attribute(self, name, type=AttributeType.Integer, label=None,
+ enum=[]):
+ try:
+ if not label:
+ label = "NULL"
+ self.cursor.execute("INSERT INTO Attribute(name, " \
+ "type, label) VALUES ('%s','%d','%s')" %
+ (name, type.index, label))
+ if type == self.AttributeType.Enumeration:
+ for i in range(len(enum)):
+ self.cursor.execute("INSERT INTO Enumeration(attribute, " \
+ "value, name) VALUES ('%s','%d','%s')" %
+ (name, i, enum[i]))
+ except:
+ self.connection.rollback()
+ raise
+ self.connection.commit()
+
+ def reset(self):
+ """Rebuilds the complete database. This causes all data to be lost!"""
+
+ self.cursor.execute("DROP DATABASE " + self.db_name)
+ self.cursor.execute("CREATE DATABASE " + self.db_name)
+ self.cursor.execute("USE " + self.db_name)
+
+ """A suite is a collection of benchmarks that is executed in one
+ batch."""
+ self.cursor.execute("CREATE TABLE Suite ("\
+ "id INTEGER UNSIGNED PRIMARY KEY AUTO_INCREMENT,"\
+ "revision VARCHAR(12) NOT NULL,"\
+ "starttime TIMESTAMP NOT NULL DEFAULT NOW()) "\
+ "ENGINE=INNODB")
+
+ """A benchmark is one piece of code or a protocol that is to be tested
+ individually with regard to time, memory usage, etc."""
+ # TODO: This should be called just Benchmark, but this seems not to
+ # be allowed in some SQL implementations if a field of the same name
+ # exists?!
+ self.cursor.execute("CREATE TABLE VIFFBenchmark ("\
+ "id INTEGER UNSIGNED PRIMARY KEY NOT NULL "
+ " AUTO_INCREMENT,"\
+ "suite INTEGER UNSIGNED NOT NULL,"\
+ "name VARCHAR(50) NOT NULL,"\
+ "INDEX baz(id),"\
+ "FOREIGN KEY (suite) REFERENCES Suite(id) "\
+ "ON UPDATE CASCADE ON DELETE CASCADE ) ENGINE=INNODB")
+
+ """An attribute can either have integer values (type 0), string values
+ (type 1), or enumerations (type 2). Enumerations act like
+ integers, but the corresponding names can be looked up in the
+ Enumeration table. Label is e.g. 'seconds', 'bytes',
+ 'hours', 'name' etc. A NULL label means just a number."""
+ self.cursor.execute("CREATE TABLE Attribute ("\
+ "name VARCHAR(32) PRIMARY KEY NOT NULL,"\
+ "type INTEGER UNSIGNED NOT NULL,"\
+ "label VARCHAR(40)) ENGINE=INNODB")
+
+ self.cursor.execute("CREATE TABLE Enumeration ("\
+ "attribute VARCHAR(32) NOT NULL,"\
+ "value SMALLINT NOT NULL,"\
+ "name VARCHAR(50),"\
+ "INDEX nsdfii (name),"\
+ "PRIMARY KEY(attribute, value),"\
+ "FOREIGN KEY(attribute) REFERENCES Attribute(name) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE"\
+ ") ENGINE=INNODB")
+
+ """Each benchmark can have a number of integer attributes associated."""
+ self.cursor.execute("CREATE TABLE IntegerBenchmarkAttribute ("\
+ "benchmark INTEGER UNSIGNED NOT NULL,"\
+ "attribute VARCHAR(32) NOT NULL,"\
+ "value BIGINT NOT NULL,"\
+ "PRIMARY KEY(benchmark, attribute),"\
+ "INDEX foo(benchmark),"\
+ "INDEX bar(attribute),"\
+ "FOREIGN KEY(benchmark) REFERENCES VIFFBenchmark(id) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE, "\
+ "FOREIGN KEY(attribute) REFERENCES Attribute(name) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE"\
+ ") ENGINE=INNODB")
+
+ """Each benchmark can have a number of string attributes associated."""
+ self.cursor.execute("CREATE TABLE StringBenchmarkAttribute ("\
+ "benchmark INTEGER UNSIGNED NOT NULL,"\
+ "attribute VARCHAR(32) NOT NULL,"\
+ "value TEXT,"\
+ "PRIMARY KEY(benchmark, attribute),"\
+ "INDEX foo(benchmark),"\
+ "INDEX bar(attribute),"\
+ "FOREIGN KEY(benchmark) REFERENCES VIFFBenchmark(id) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE, "\
+ "FOREIGN KEY(attribute) REFERENCES Attribute(name) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE"\
+ ") ENGINE=INNODB")
+
+ """ Defines the hosts on which the benchmarks are run."""
+ self.cursor.execute("CREATE TABLE Host ("\
+ "id INTEGER UNSIGNED PRIMARY KEY AUTO_INCREMENT,"\
+ "name VARCHAR(50) NOT NULL) ENGINE=INNODB")
+
+ """ Attributes for individual hosts, such as CPU frequency, number of
+ cores, RAM size, IP, etc."""
+ self.cursor.execute("CREATE TABLE HostAttribute ("\
+ "host INTEGER UNSIGNED NOT NULL,"\
+ "attribute VARCHAR(32) NOT NULL,"\
+ "value TEXT,"\
+ "PRIMARY KEY(host, attribute),"\
+ "INDEX foobar (host), INDEX sdflkj (attribute),"\
+ "FOREIGN KEY(host) REFERENCES Host(id) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE,"\
+ "FOREIGN KEY(attribute) REFERENCES Attribute(name) "\
+ "ON DELETE CASCADE ON UPDATE CASCADE"\
+ ") ENGINE=INNODB")
+
+ """ Each benchmark involves a number of hosts."""
+ self.cursor.execute("CREATE TABLE BenchmarkHost ("\
+ "host INTEGER UNSIGNED NOT NULL,"\
+ "benchmark INTEGER UNSIGNED NOT NULL,"\
+ "id INTEGER,"\
+ "PRIMARY KEY(host, benchmark),"\
+ "INDEX enskef (host), INDEX sewerwe (benchmark),"\
+ "FOREIGN KEY(host) REFERENCES Host(id) "\
+ "ON UPDATE CASCADE ON DELETE RESTRICT,"\
+ "FOREIGN KEY(benchmark) REFERENCES VIFFBenchmark(id) "\
+ "ON UPDATE CASCADE ON DELETE CASCADE"\
+ ") ENGINE=INNODB")
+
+ """ Contains the results of an benchmark on a specific host. Runs should
+ increase continously from 0 to the number of runs."""
+ self.cursor.execute("CREATE TABLE Result ("\
+ "benchmark INTEGER UNSIGNED NOT NULL,"\
+ "host INTEGER UNSIGNED NOT NULL,"\
+ "run INTEGER NOT NULL,"\
+ "attribute VARCHAR(32) NOT NULL,"\
+ "value BIGINT NOT NULL,"\
+ "PRIMARY KEY(benchmark, host, run, attribute),"\
+ "INDEX nslnln (benchmark), INDEX nlslsqwe (host), "\
+ "INDEX nslenkeb (attribute),"\
+ "FOREIGN KEY(benchmark) REFERENCES VIFFBenchmark(id) "\
+ "ON UPDATE CASCADE ON DELETE CASCADE,"\
+ "FOREIGN KEY(host) REFERENCES Host(id) "\
+ "ON UPDATE CASCADE ON DELETE RESTRICT,"\
+ "FOREIGN KEY(attribute) REFERENCES Attribute(name) "\
+ "ON UPDATE CASCADE ON DELETE CASCADE"\
+ ") ENGINE=INNODB")
+
+ """ This table is for benchmarks where there are multiple measurements
+ during one benchmark, e.g. the CPU load or memory usage during a
+ double auction. Time is milliseconds since the start of the
+ benchmark on the specific host."""
+ self.cursor.execute("CREATE TABLE TimedResult ("\
+ "benchmark INTEGER UNSIGNED NOT NULL,"\
+ "host INTEGER UNSIGNED NOT NULL,"\
+ "run INTEGER NOT NULL,"\
+ "attribute VARCHAR(32) NOT NULL,"\
+ "time BIGINT UNSIGNED NOT NULL,"\
+ "value BIGINT NOT NULL,"\
+ "PRIMARY KEY(benchmark, host, run, attribute, time),"\
+ "INDEX ndgdgnln (benchmark), INDEX dgdgdg (host), "\
+ "INDEX llrollo (attribute),"\
+ "FOREIGN KEY(benchmark) REFERENCES VIFFBenchmark(id) "\
+ "ON UPDATE CASCADE ON DELETE CASCADE,"\
+ "FOREIGN KEY(host) REFERENCES Host(id) "\
+ "ON UPDATE CASCADE ON DELETE RESTRICT,"\
+ "FOREIGN KEY(attribute) REFERENCES Attribute(name) "\
+ "ON UPDATE CASCADE ON DELETE CASCADE"\
+ ") ENGINE=INNODB")
+
+ self.connection.commit()
+

[Martin Geisler]

Thomas Pelle Jakobsen <***@daimi.au.dk> writes:

Hi Thomas,

I've just had a quick look through the code and wow, it looks very
impressive!

But are the patches not ordered in the wrong order? I mean, the first
patch imports code which is only introduced in patch 5.

Here's my initial attempt at automating benchmarks and graph
generation in VIFF. Some parts of it is borrowed from the
benchmark.py script that's alread in the repository.
The main goal is to make it easy to write, run, and generate graphs
for distributed VIFF benchmarks while at the same time not limiting
the kind of benchmarks that could be made. Another design goal is
that the code should scale as the number of benchmarks increase. An
approah is chosen where benchmark data is collected in a central
database rather than as a bunch of text files. My feeling is that
this eases data maintainability and production of more complex
statistics.

Sure -- I have never believed that tons of text files were the best
way to store long term benchmark data. It was simply the easiest and
fastest way for me to collect data.

The code is by no means finished. Lots of TODO's are still there,
documentation can be improved, and I haven't done much to remove
trailing whitespaces, etc. However, I've reached a state where I'm
actually able to use it to run benchmarks on the DAIMI hosts while
data is reported to a MySQL database at my computer at home.

Cool! I saw you write somewhere that an ORM would be nice and I agree.
After having used such an interface I cringe when I see the raw SQL
strings in the code -- there are just so many possibilities for
putting MySQL specific stuff there, not to talk about the possible
robustness and security problems.

But we should not make that stop us from starting out like this.

I post the patch now hoping that an early review will result in time
saved, so please feel free to comment on the code. I'd be happy to
hear about bugs and ideas for improving it. I'm quite a newbie in
Python, so if I've done something very non-pythonic, please let me
know, too :-)

I saw that you're using strings here and there as big comments:

"""bla bla, my nice
explaination for
something complex"""

instead of the more natural

# bla bla, my nice
# explaination for
# something complex

It is only the very first string literal that is treated as a
doccomment in Python -- the others are just string literals.

Also, I saw you do

if 'foo' in some_dict.keys():

which is the same as

if 'foo' in some_dict

When you do

str(benchmark.__class__).split('.')[-1]

you could instead do

benchmark.__class__.__name__

which is still sort of a hack, but is easier to understand.

Lastly, since the files are introduced this year, the copyright
statements should simply read

# Copyright 2008 VIFF Development Team.


I hope I'll have time to look at and play with the code some more.
Mikkel, Jakob, Sigurd, ...: please jump in and let me/Thomas know what
you think of the code and design!

[Thomas Jakobsen]

Post by Martin Geisler
But are the patches not ordered in the wrong order? I mean, the first
patch imports code which is only introduced in patch 5.

You're right. I'll send a new patch with the right order when I've
done further improvements based on your comments.

Post by Martin Geisler

The code is by no means finished. Lots of TODO's are still there,
documentation can be improved, and I haven't done much to remove
trailing whitespaces, etc. However, I've reached a state where I'm
actually able to use it to run benchmarks on the DAIMI hosts while
data is reported to a MySQL database at my computer at home.

Cool! I saw you write somewhere that an ORM would be nice and I agree.
After having used such an interface I cringe when I see the raw SQL
strings in the code -- there are just so many possibilities for
putting MySQL specific stuff there, not to talk about the possible
robustness and security problems.
But we should not make that stop us from starting out like this.

No. I chose the second best solution, namely to isolate the SQL-stuff
in a single class. I actually had a look at using an ORM, but i
realized that it would take me quite some time to get used to. So I
did the usual SQL-stuff in order to get something going quickly. Not
the best motive, I admit. But I hope that we will be able to refactor
to ORM-stuff at a later time.

Post by Martin Geisler
"""bla bla, my nice
explaination for
something complex"""
instead of the more natural
# bla bla, my nice
# explaination for
# something complex
It is only the very first string literal that is treated as a
doccomment in Python -- the others are just string literals.
Also, I saw you do
which is the same as
if 'foo' in some_dict
When you do
str(benchmark.__class__).split('.')[-1]
you could instead do
benchmark.__class__.__name__
which is still sort of a hack, but is easier to understand.
Lastly, since the files are introduced this year, the copyright
statements should simply read
# Copyright 2008 VIFF Development Team.

Thanks. I'll wait for more corrections and then send a new updated
patch at some time.

If you want to play with the code and have problems getting it to run,
I'll be happy to help. Maybe the problems are due to some bug that I
haven't stumbled over yet.

Regards,
Thomas

[Martin Geisler]

"Thomas Jakobsen" <***@daimi.au.dk> writes:

Hi Thomas

Post by Thomas Jakobsen

Post by Martin Geisler
[...]
# Copyright 2008 VIFF Development Team.

Thanks. I'll wait for more corrections and then send a new updated
patch at some time.

Sounds good. I'm sorry to have let you wait -- I wanted to see if some
ofthe others would give you some feedback over the weekend. I've put
Mikkel and Sigurd on CC since they know Python too, but I guess they
are busy with other stuff.

I wont be looking at the patches right now, instead I'm looking at the
VIFF VM spec for CACE today.

[Thomas Jakobsen]

Post by Martin Geisler
Sounds good. I'm sorry to have let you wait -- I wanted to see if some
ofthe others would give you some feedback over the weekend. I've put
Mikkel and Sigurd on CC since they know Python too, but I guess they
are busy with other stuff.
I wont be looking at the patches right now, instead I'm looking at the
VIFF VM spec for CACE today.

No problem. Next thing I'll do is probably to set up some unit tests.

Regards,
Thomas

[Thomas Jakobsen]

By the way, I think question to consider first is: Does my benchmark
code support the various kinds of benchmarks that we would like to
carry out and the statistics and stuff that we would like to do on the
results? If yes, I'll go on and improve on the details. If not, what
could be done instead.

Regards,
Thomas

Post by Thomas Jakobsen

Post by Martin Geisler
Sounds good. I'm sorry to have let you wait -- I wanted to see if some
ofthe others would give you some feedback over the weekend. I've put
Mikkel and Sigurd on CC since they know Python too, but I guess they
are busy with other stuff.
I wont be looking at the patches right now, instead I'm looking at the
VIFF VM spec for CACE today.

No problem. Next thing I'll do is probably to set up some unit tests.
Regards,
Thomas

[Martin Geisler]

Post by Thomas Pelle Jakobsen
# HG changeset patch
# Date 1226015460 -3600
# Node ID 0985564470de2bb2c5247effd30dc40f74048f17
# Parent aa8ba57c7833f0d68792db7c36782e3ba9fbc194
Added graph example.
diff -r aa8ba57c7833 -r 0985564470de apps/benchmark/graph.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/apps/benchmark/graph.py Fri Nov 07 00:51:00 2008 +0100
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+#
+# Copyright 2007, 2008 VIFF Development Team.
+#
+# This file is part of VIFF, the Virtual Ideal Functionality Framework.
+#
+# VIFF is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License (LGPL) as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# VIFF is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
+# Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
+
+""" A rudimentary script that generates graphs of some benchmark results.
+It could be executed periodically or triggered by a benchmark run, a
+viff commit or something else.

VIFF should be in capitals, the leading and trailing spaces should go.

Post by Thomas Pelle Jakobsen
+
+All sorts of statistics could be generated by scripts like this.
+"""
+
+import sys
+from database import Database
+import numpy as np
+import matplotlib.pyplot as plt
+
+ """Create a graph showing for each suite the revision on the x-axis and
+ the average execution time of Toft05 comparison for three players on the
+ y-axis (timings from host with player_id one is used).
+ """
+ vals = database.sql_query("""
+ SELECT Suite.revision, Suite.starttime, AVG(Result.value)
+ FROM Suite
+ INNER JOIN VIFFBenchmark AS b ON Suite.id = b.suite
+ INNER JOIN IntegerBenchmarkAttribute AS iba ON b.id = iba.benchmark
+ INNER JOIN Result ON b.id = Result.benchmark
+ WHERE b.name = 'ComparisonToft05'
+ AND iba.attribute = 'n'
+ AND iba.value = 3
+ AND Result.host = 1
+ AND Result.attribute = 'execution_time'
+ GROUP BY Suite.id, b.id;
+ """)

<joke>Argh, raw SQL! I thought that had been forbidden years ago. I
cannot even remember when I last had to write SQL...</joke>

Post by Thomas Pelle Jakobsen
+ toft05_avgs = []
+ revisions = []
+ print revision, timestamp, average
+ toft05_avgs.append(int(average/1000))
+ revisions.append(revision)
+ N = len(revisions)
+ ind = np.arange(N)
+ width = 0.35 # the width of the bars: can also be len(x) sequence
+ p1 = plt.bar(ind, toft05_avgs, width, color='r')
+ plt.ylabel('execution time / ms')
+ plt.title('VIFF Benchmark Results')
+ plt.xticks(ind+width/2., revisions )
+ plt.legend( (p1[0],), ('Toft05Comparison',) )
+ plt.show()
+ #savefig("fig1.png",dpi=(640/8))
+
+ database = Database(host="localhost",
+ db="viff_benchmark",
+ user="root",
+ passwd=sys.argv[1])
+
+ example1(database)
+
+ # TODO: We would like hg commit dates to exist in database as well
+ # as the suite execution date. How?

You mean the date of the commit that defines the suite? Something like

hg log -r 123 --template '{date|isodate}'

will give you what you want, replace 123 with the revision hash. There
are more about the template filters here:

http://hgbook.red-bean.com/hgbookch11.html#x15-25800011.6