From: Mark Wooding <mdw@distorted.org.uk>
Date: Sun, 12 Mar 2006 18:20:39 +0000 (+0000)
Subject: Add testing scripts.
X-Git-Tag: 0.9.0-dev~4
X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~mdw/git/mm/commitdiff_plain/c2df194c52c1726d49cf054fe0923c16244b8114

Add testing scripts.

The trial script compares a number of implementations with a random
selection of games, and reports on how good or bad they are.
---

diff --git a/trial b/trial
new file mode 100755
index 0000000..fc8a0ec
--- /dev/null
+++ b/trial
@@ -0,0 +1,68 @@
+#! /usr/bin/python
+
+import sre as R
+import os as OS
+from popen2 import Popen3
+from sys import argv
+import catacomb as C
+
+r_score = R.compile(r'^Solved in ([\d]+) guesses$')
+r_time = R.compile(r'^time:([\d.]+),([\d.]+)$')
+def run(prog):
+  proc = Popen3(['time', '-f', 'time:%U,%S'] + prog, capturestderr = True)
+  guesses = -1
+  time = -1
+  proc.tochild.close()
+  for l in proc.fromchild:
+    m = r_score.match(l)
+    if m: guesses = int(m.group(1))
+  for l in proc.childerr:
+    m = r_time.match(l)
+    if m:
+      u, s = map(float, m.groups([1, 2]))
+      time = u + s
+  return guesses, time
+
+impl = argv[1:]
+def compare(h, c, scores):
+  code = [C.rand.range(c) for i in xrange(h)]
+  perf = {}
+  best, bestperf = [None, None], [-1, -1]
+  for i in impl:
+    p = perf[i] = run(i + map(str, [h, c] + code))
+    for j in [0, 1]:
+      if bestperf[j] == -1 or p[j] < bestperf[j]:
+        bestperf[j] = p[j]
+        best[j] = i
+      scores[i][j] += p[j]
+  st = ''
+  for i in [0, 1]:
+    st += '%s:' % ['G', 'T'][i]
+    for j in impl:
+      if best[i] == j:
+        st += '+'
+      else:
+        st += '.'
+    st += ' '
+  print '%s%s' % (st, code)
+  ind = (len(impl) + 3) * len(best)
+  for i in impl:
+    ##print '%s%s: %s' % (' ' * ind, i, ' '.join(map(str, perf[i])))
+    pass
+  ##print
+
+for n, h, c in [(100, 4, 6),
+                (50, 4, 10),
+                (20, 5, 6),
+                (20, 5, 8),
+                (20, 5, 10)]:
+  print '************************** %d %d' % (h, c)
+  scores = {}
+  for i in impl: scores[i] = [0, 0]
+  for i in xrange(n):
+    compare(h, c, scores)
+  print 'Summary, %d %d' % (h, c)
+  for i in impl:
+    print '  %s: %s' % (i, ' '.join(['%s:%s' % (['G', 'T'][j], scores[i][j])
+                                   for j in [0, 1]]))
+  print