no idea why I need to recommit everything

runmedley.sh

@@ -1,25 +1,22 @@
-for seed in 0 1; do
-    for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c exp3d; do
-        if [ $learner == exp3a ]
+for seed in 0; do #1; do
+    for learner in random; do #thompson knearest MLP linear exp3a; do
+        if [ $learner = exp3a ]
         then
             learnconfig="exp3 --gamma 0.07"
-        elif [ $learner == exp3b ]
+        elif [ $learner = exp3b ]
         then
             learnconfig="exp3 --gamma 0.1"
-        elif [ $learner == exp3c ]
+        elif [ $learner = exp3c ]
         then
             learnconfig="exp3 --gamma 0.25"
-        elif [ $learner == exp3d ]
-        then
-            learnconfig="exp3 --gamma 0.5"
         else 
             learnconfig=$learner
         fi
-        for feature in both probes bow; do
-            for reward in binary bump exp; do
-                medley ./$1/ ./$1/${learner}_${feature}_${reward}_const_${seed}.csv   --classifier $learnconfig --seed $seed --feature_setting $feature --reward $reward --timeout_manager const --set_const 60
-                medley ./$1/ ./$1/${learner}_${feature}_${reward}_expo_${seed}.csv    --classifier $learnconfig --seed $seed --feature_setting $feature --reward $reward --timeout_manager expo
-                medley ./$1/ ./$1/${learner}_${feature}_${reward}_nearest_${seed}.csv --classifier $learnconfig --seed $seed --feature_setting $feature --reward $reward --timeout_manager nearest --time_k 20
+        for feature in both; do #probes bow; do
+            for reward in bump; do # binary exp; do
+                # medley ./$1/ ./$1/${learner}_${feature}_${reward}_const_${seed}.csv   --classifier $learnconfig --seed $seed --feature_setting $feature --reward $reward --timeout_manager const --set_const 60
+                medley ./$1/ ./$1/${learner}_${feature}_${reward}_perfect_${seed}.csv    --classifier $learnconfig --seed $seed --feature_setting $feature --reward $reward --timeout_manager perfect
+                # medley ./$1/ ./$1/${learner}_${feature}_${reward}_5_nearest_${seed}.csv --classifier $learnconfig --seed $seed --feature_setting $feature --reward $reward --timeout_manager nearest --time_k 20
             done 
         done 
     done 

runordermisses.sh

@@ -0,0 +1,26 @@
+total=0
+
+for seed in 0; do #1; do
+    for learner in random; do #thompson knearest MLP linear exp3a; do
+        if [ $learner = exp3a ]
+        then
+            learnconfig="exp3 --gamma 0.07"
+        elif [ $learner = exp3b ]
+        then
+            learnconfig="exp3 --gamma 0.1"
+        elif [ $learner = exp3c ]
+        then
+            learnconfig="exp3 --gamma 0.25"
+        else 
+            learnconfig=$learner
+        fi
+        for feature in both; do #probes bow; do
+            for reward in bump; do # binary exp; do
+                val=$(./ordermisses.py ./$1/ ${learner}_${feature}_${reward}_perfect_${seed})
+                total=$total+$val
+            done 
+        done 
+    done 
+done 
+
+echo \($total\)/5

runtimebenfit.sh

@@ -0,0 +1,26 @@
+total=0
+
+for seed in 0; do #1; do
+    for learner in thompson knearest MLP linear exp3a; do
+        if [ $learner = exp3a ]
+        then
+            learnconfig="exp3 --gamma 0.07"
+        elif [ $learner = exp3b ]
+        then
+            learnconfig="exp3 --gamma 0.1"
+        elif [ $learner = exp3c ]
+        then
+            learnconfig="exp3 --gamma 0.25"
+        else 
+            learnconfig=$learner
+        fi
+        for feature in both; do #probes bow; do
+            for reward in bump; do # binary exp; do
+                val=$(./timebenefit.py ./$1/ ${learner}_${feature}_${reward}_${2}_${seed})
+                total=$total+$val
+            done 
+        done 
+    done 
+done 
+
+echo \($total\)/5

runtimecost.sh

@@ -0,0 +1,26 @@
+total=0
+
+for seed in 0; do #1; do
+    for learner in thompson knearest MLP linear exp3a; do
+        if [ $learner = exp3a ]
+        then
+            learnconfig="exp3 --gamma 0.07"
+        elif [ $learner = exp3b ]
+        then
+            learnconfig="exp3 --gamma 0.1"
+        elif [ $learner = exp3c ]
+        then
+            learnconfig="exp3 --gamma 0.25"
+        else 
+            learnconfig=$learner
+        fi
+        for feature in both; do #probes bow; do
+            for reward in bump; do # binary exp; do
+                val=$(./timemisses.py ./$1/ ${learner}_${feature}_${reward}_${2}_${seed})
+                total=$total+$val
+            done 
+        done 
+    done 
+done 
+
+echo \($total\)/5

timebenefit.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+import csv 
+import os
+import glob
+import sys
+
+path = os.getcwd()
+
+dataset = sys.argv[1]
+learner = sys.argv[2]
+extension = 'csv'
+os.chdir(dataset)
+result = glob.glob('*.{}'.format(extension))
+
+cols = {}
+for r in result:
+    cols[r[:-len(".csv")]] = {}
+
+for r in result:
+    solver = r[:-len(".csv")]
+    with open(r) as csvfile:
+        spamreader = csv.reader(csvfile)
+        for row in spamreader:
+            r = float(row[3]) if isinstance(row[3], str) else row[3]
+            r = r if r < 60 else 60
+            r = r if row[4] != "error" and "timeout" not in row[4] else 60
+            cols[solver][row[0]] = r
+benefit = 0
+# print("solver, answer, guess, problem")
+# Load nearest, zip times with solvers
+with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile:
+    spamreader = list(csv.reader(csvfile))
+    for r in range(len(spamreader)):
+        row = spamreader[r]
+        times = eval(row[6])
+        solvers = [s for s in eval(row[5])[:len(times)]]
+        zipped = zip(solvers, times)
+
+
+        for (a, b) in list(zipped)[:1]:
+            if b > 0 and a in cols and cols[a][row[0]] >= 60 and b < 60:
+                # print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])]))
+                # benefit += 60 - b
+                benefit += 1
+
+print(benefit)

uclid500.py

@@ -0,0 +1,36 @@
+#!/usr/bin/python3
+
+import csv 
+import os
+import glob
+import sys
+
+path = os.getcwd()
+
+dataset = sys.argv[1]
+learner = sys.argv[2]
+extension = 'csv'
+os.chdir(dataset)
+result = glob.glob('*.{}'.format(extension))
+
+to_keep = []
+with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile:
+    spamreader = list(csv.reader(csvfile))
+    for r in range(len(spamreader)):
+        row = spamreader[r]
+        name = row[0]
+        to_keep.append(name)
+
+
+for r in result:
+    solver = r[:-len(".csv")]
+    out = r.split("/")[-1]
+    out = "%s/%s500/%s"%(path, dataset, out)
+    # Open both files
+    with open(r) as f_in, open(out, 'w') as f_out:
+        spamreader = list(csv.reader(f_in))
+        spamwriter = csv.writer(f_out)
+        for line in spamreader:
+            name = line[0]
+            if name in to_keep:
+                spamwriter.writerow(line)

virtualbest.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+
+import csv 
+import os
+import glob
+import sys
+
+path = os.getcwd()
+
+dataset = sys.argv[1]
+extension = 'csv'
+os.chdir(dataset)
+result = glob.glob('*.{}'.format(extension))
+
+result = [r for r in result if "_" not in r]
+
+cols = {}
+for r in result:
+    cols[r[:-len(".csv")]] = {}
+
+solvers = []
+
+for r in result:
+    solver = r[:-len(".csv")]
+    solvers.append(solver)
+    with open(r) as csvfile:
+        spamreader = csv.reader(csvfile)
+        for row in spamreader:
+            r = float(row[3]) if isinstance(row[3], str) else row[3]
+            r = r if r < 60 else 60
+            r = r if row[4] != "error" and "timeout" not in row[4] else 60
+            cols[solver][row[0]] = r
+
+vbtime = 0
+vbsolved = 0
+spread = {}
+
+for s in solvers:
+    spread[s] = 0
+
+for q in cols[solvers[0]]:
+    best_time = 60
+    best_solver = ""
+
+    for s in solvers:
+        if cols[s][q] < best_time:
+            best_time = cols[s][q]
+            best_solver = s
+
+    if best_solver != "":
+        spread[best_solver] += 1
+        vbsolved += 1
+
+    vbtime += best_time
+
+print(vbtime, vbsolved)
+
+for s in spread:
+    print(s, spread[s])