fixes to scripts

bin/medley

@@ -147,7 +147,7 @@ def main():
         type=int,
         default=20
     )
-    
+
     global_parser.add_argument(
         "--feature_setting",
         "-f",
@@ -183,7 +183,7 @@ def main():
     elif args.timeout_manager == "nearest":
         if not args.set_lambda:
             args.set_lambda = 1 / (args.timeout / len(SOLVERS))
-        timeout_manager = NearestExponential(args.set_lambda, args.confidence)
+        timeout_manager = NearestExponential(args.set_lambda, args.confidence, args.timeout)
     else:
         raise RuntimeError("timeout_manager not properly set")
 

medleysolver/classifiers.py

@@ -223,6 +223,7 @@ def get_ordering(self, point, count):
             order = sorted(ss, key= lambda x: -1 * methods.count(x))
         else:
             order = Random.get_ordering(self, point, count)
+        return order
 
     def update(self, solved_prob, rewards):
         #TODO: Implement pruning

medleysolver/dispatch.py

@@ -4,7 +4,7 @@
 import csv 
 
 def run_problem(solver, invocation, problem, timeout):
-    instance = problem.split("/", 2)[-1]
+    instance = problem.split("/")[-1]
     directory = problem[:-len(instance)]
 
     try:

medleysolver/timers.py

@@ -36,13 +36,14 @@ def update(self, solver, time, timeout, success, error):
                 self.timers[solver].add_timeout()
 
 class NearestExponential(TimerInterface):
-    def __init__(self, init_lambda, confidence):
+    def __init__(self, init_lambda, confidence, T):
         self.init_lambda = init_lambda
         self.confidence = confidence
+        self.T = T
 
     def get_timeout(self, solver, times):
         # want time based on times for same solver at nearby points
-        timer = ExponentialDist(self.init_lambda, self.confidence)
+        timer = ExponentialDist(self.init_lambda, self.confidence, self.T)
         for (s, t) in times:
             if s == solver:
                 timer.add_sample(t)

ordermisses.py

@@ -15,7 +15,7 @@
 
 cols = {}
 for r in result:
-    cols[r[:-len(".csv")]] = [] 
+    cols[r[:-len(".csv")]] = {}
 
 for r in result:
     solver = r[:-len(".csv")]
@@ -24,9 +24,10 @@
         for row in spamreader:
             r = float(row[3]) if isinstance(row[3], str) else row[3]
             r = r if r < 60 else 60
-            cols[solver].append(r)
+            r = r if row[4] != "error" else 60
+            cols[solver][row[0]] = r
 
-print("problem, choice, answer")
+print("choice, answer, problem")
 # Load nearest, zip times with solvers
 with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile:
     spamreader = list(csv.reader(csvfile))
@@ -35,11 +36,11 @@
         runtimes = eval(row[6])
         solvers = eval(row[5])[:len(runtimes)]
 
-        solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] > 0 and solvers[i] in cols]
+        solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] > 0 and solvers[i] in cols and row[0] in cols[solvers[i]]]
 
-        times = [cols[a][r] for a in solvers]
+        times = [cols[a][row[0]] for a in solvers]
 
         index_min = min(range(len(times)), key=times.__getitem__)
 
         if index_min != 0:
-            print(",".join([str(r), str(solvers[0]), str(solvers[index_min])]))
+            print(",".join([str(solvers[0]), str(solvers[index_min], str(row[0]))]))

process.py

@@ -27,20 +27,6 @@
 totaltimes = [str(sum(time)) for time in times]
 totalcount = [str(sum(map(lambda x: 1 if x == "sat" or x == "unsat" else 0, answer))) for answer in answers]
 
-virtualbesttime = 0
-virtualbestcount = 0
-for j in range(len(times[0])):
-    choices = []
-    for i in range(len(solvers)):
-        if "sat" in answers[i][j]:
-            choices.append(times[i][j])
-
-    if len(choices) > 0:
-        virtualbestcount += 1
-
-    virtualbesttime += min(*(choices + [TIMEOUT, TIMEOUT]))
-
-
-print(",".join(solvers + ["Virtual Best"]))
-print(",".join(totaltimes + [str(virtualbesttime)]))
-print(",".join(totalcount + [str(virtualbestcount)]))
+print(",".join(solvers))
+print(",".join(totaltimes))
+print(",".join(totalcount))

runmedley.sh

@@ -1,6 +1,17 @@
 medley ./$1/ ./$1/thompson.csv -c thompson
 medley ./$1/ ./$1/nearest.csv -c neighbor
-medley ./$1/ ./$1/random.csv -c random
-medley ./$1/ ./$1/MLP.csv -c MLP
-medley ./$1/ ./$1/linear.csv -c linear
-medley ./$1/ ./$1/exp3.csv -c exp3
+medley ./$1/ ./$1/knearest.csv -c knearest --k 10
+medley ./$1/ ./$1/random.csv -c random 
+medley ./$1/ ./$1/MLP.csv -c MLP 
+medley ./$1/ ./$1/linear.csv -c linear 
+medley ./$1/ ./$1/exp3.csv -c exp3 
+
+medley ./$1/ ./$1/thompson_only.csv -c thompson --timeout_manager const --set_const 60
+medley ./$1/ ./$1/nearest_only.csv -c neighbor --timeout_manager const --set_const 60
+medley ./$1/ ./$1/knearest_only.csv -c knearest --k 10 --timeout_manager const --set_const 60
+medley ./$1/ ./$1/random_only.csv -c random --timeout_manager const --set_const 60
+medley ./$1/ ./$1/MLP_only.csv -c MLP --timeout_manager const --set_const 60
+medley ./$1/ ./$1/linear_only.csv -c linear --timeout_manager const --set_const 60
+medley ./$1/ ./$1/exp3_only.csv -c exp3 --timeout_manager const --set_const 60
+
+medley ./$1/ ./$1/timenearest.csv -c knearest --k 10 --timeout_manager nearest --time_k 20

timemisses.py

@@ -15,7 +15,7 @@
 
 cols = {}
 for r in result:
-    cols[r[:-len(".csv")]] = [] 
+    cols[r[:-len(".csv")]] = {}
 
 for r in result:
     solver = r[:-len(".csv")]
@@ -24,9 +24,10 @@
         for row in spamreader:
             r = float(row[3]) if isinstance(row[3], str) else row[3]
             r = r if r < 60 else 60
-            cols[solver].append(r)
+            r = r if row[4] != "error" else 60
+            cols[solver][row[0]] = r
 
-print("problem, solver, answer, guess")
+print("solver, answer, guess, problem")
 # Load nearest, zip times with solvers
 with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile:
     spamreader = list(csv.reader(csvfile))
@@ -38,5 +39,5 @@
 
 
         for (a, b) in zipped:
-            if b > 0 and a in cols and cols[a][r] > b and cols[a][r] < 60:
-                print(",".join([str(r), str(a), str(cols[a][r]), str(b)]))
+            if b > 0 and a in cols and cols[a][row[0]] > b and cols[a][row[0]] < 60:
+                print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])]))