merge?

.gitignore

@@ -3,4 +3,5 @@
 /build
 /dist
 medleysolver.egg-info
-/tmp
+*.smt2
+.DS_STORE

bin/medley

@@ -5,7 +5,7 @@ import glob
 import dill
 from medleysolver.runner import execute
 from medleysolver.constants import SOLVERS
-from medleysolver.timers import Exponential, Constant, NearestExponential
+from medleysolver.timers import Exponential, Constant, NearestExponential, PerfectTimer
 from medleysolver.classifiers import *
 
 def main():
@@ -26,7 +26,7 @@ def main():
         "-e",
         help="extra time to first? Add to last if not",
         type=bool,
-        default=False,
+        default=True,
     )
     global_parser.add_argument(
         "--kind",
@@ -52,7 +52,7 @@ def main():
         help="select mechanism for choosing orderings of solvers",
         type=str,
         default="neighbor",
-        choices=["neighbor", "random", "MLP", "thompson", "linear", "preset", "exp3", "knearest"]
+        choices=["neighbor", "random", "MLP", "thompson", "linear", "preset", "exp3", "knearest", "perfect"]
     )
 
     global_parser.add_argument(
@@ -69,7 +69,7 @@ def main():
         help="choose how timeout is distributed amongst solvers",
         type=str,
         default="expo",
-        choices=["expo", "const", "nearest"]
+        choices=["expo", "const", "nearest", "perfect"]
     )
 
     global_parser.add_argument(
@@ -153,7 +153,7 @@ def main():
         "--time_k",
         help="set k value for knearest neighbor",
         type=int,
-        default=20
+        default=40
     )
 
     global_parser.add_argument(
@@ -199,26 +199,30 @@ def main():
         if not args.set_lambda:
             args.set_lambda = 1 / (args.timeout / len(SOLVERS))
         timeout_manager = NearestExponential(args.set_lambda, args.confidence, args.timeout)
+    elif args.timeout_manager == "perfect":
+        timeout_manager = PerfectTimer()
     else:
         raise RuntimeError("timeout_manager not properly set")
 
     if args.classifier == "neighbor":
-        classifier = NearestNeighbor(args.epsilon, args.epsilon_decay, args.kind)
+        classifier = NearestNeighbor(args.epsilon, args.epsilon_decay, args.kind, args.time_k)
     elif args.classifier == "random":
-        classifier = Random()
+        classifier = Random(args.time_k)
     elif args.classifier == "MLP":
-        classifier = MLP()
+        classifier = MLP(args.time_k)
     elif args.classifier == "thompson":
-        classifier = Thompson(args.kind)
+        classifier = Thompson(args.kind, args.time_k)
     elif args.classifier == "linear":
-        classifier = LinearBandit()
+        classifier = LinearBandit(args.time_k)
     elif args.classifier == "preset":
         classifier = Preset(args.preset)
         timeout_manager = Constant(args.timeout)
     elif args.classifier == "exp3":
-        classifier = Exp3(args.gamma)
+        classifier = Exp3(args.gamma, args.time_k)
     elif args.classifier == "knearest":
         classifier = KNearest(args.k, args.epsilon, args.epsilon_decay, args.time_k)
+    elif args.classifier == "perfect":
+        classifier = PerfectSelector(args.time_k)
     else:
         raise RuntimeError("classifier not properly set")
 

machine1.sh

@@ -1,4 +1,4 @@
-for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c exp3d; do
+for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c; do
     sh ./runlearner.sh tmp/BV $learner
     sh ./runlearner.sh tmp/QF_ABV $learner
     sh ./runlearner.sh tmp/QF_AUFBV $learner

machine6.sh

@@ -1,3 +1,3 @@
-for learner in exp3c exp3d; do
+for learner in exp3c; do
     sh ./runlearner.sh tmp/uclid $learner
 done

machine7.sh

@@ -1,3 +1,3 @@
-for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c exp3d; do
+for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c; do
     sh ./runlearner.sh tmp/Sage2 $learner
 done

ordermisses.py

@@ -27,7 +27,9 @@
             r = r if row[4] != "error" and "timeout" not in row[4] else 60
             cols[solver][row[0]] = r
 
-print("choice, answer, problem")
+count = 0
+
+# print("choice, answer, problem")
 # Load nearest, zip times with solvers
 with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile:
     spamreader = list(csv.reader(csvfile))
@@ -36,11 +38,14 @@
         runtimes = eval(row[6])
         solvers = eval(row[5])[:len(runtimes)]
 
-        solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] > 0 and solvers[i] in cols and row[0] in cols[solvers[i]]]
+        solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] >= 0 and solvers[i] in cols and row[0] in cols[solvers[i]]]
 
         times = [cols[a][row[0]] for a in solvers]
 
         index_min = min(range(len(times)), key=times.__getitem__)
 
-        if index_min != 0:
-            print(",".join([str(solvers[0]), str(solvers[index_min], str(row[0]))]))
+        if index_min != 0 and times[index_min] < times[0]:
+            # print(",".join([str(solvers[0]), str(solvers[index_min]), str(row[0])]))
+            count += 1
+
+print(count)

runlearner.sh

@@ -1,16 +1,13 @@
 for seed in 0 1; do
-    if [ $2 == exp3a ]
+    if [ $2 = exp3a ]
     then
         learnconfig="exp3 --gamma 0.07"
-    elif [ $2 == exp3b ]
+    elif [ $2 = exp3b ]
     then
         learnconfig="exp3 --gamma 0.1"
-    elif [ $2 == exp3c ]
+    elif [ $2 = exp3c ]
     then
         learnconfig="exp3 --gamma 0.25"
-    elif [ $2 == exp3d ]
-    then
-        learnconfig="exp3 --gamma 0.5"
     else 
         learnconfig=$2
     fi

timemisses.py

@@ -27,7 +27,8 @@
             r = r if row[4] != "error" and "timeout" not in row[4] else 60
             cols[solver][row[0]] = r
 
-print("solver, answer, guess, problem")
+total = 0
+# print("solver, answer, guess, problem")
 # Load nearest, zip times with solvers
 with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile:
     spamreader = list(csv.reader(csvfile))
@@ -38,6 +39,9 @@
         zipped = zip(solvers, times)
 
 
-        for (a, b) in zipped:
+        for (a, b) in list(zipped)[:1]:
             if b > 0 and a in cols and cols[a][row[0]] > b and cols[a][row[0]] < 60:
-                print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])]))
+                # print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])]))
+                total += 1
+
+print(total)