diff --git a/.gitignore b/.gitignore index 746adcb..0884f81 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /build /dist medleysolver.egg-info -/tmp \ No newline at end of file +*.smt2 +.DS_STORE diff --git a/bin/medley b/bin/medley index f377a01..b778c52 100755 --- a/bin/medley +++ b/bin/medley @@ -5,7 +5,7 @@ import glob import dill from medleysolver.runner import execute from medleysolver.constants import SOLVERS -from medleysolver.timers import Exponential, Constant, NearestExponential +from medleysolver.timers import Exponential, Constant, NearestExponential, PerfectTimer from medleysolver.classifiers import * def main(): @@ -26,7 +26,7 @@ def main(): "-e", help="extra time to first? Add to last if not", type=bool, - default=False, + default=True, ) global_parser.add_argument( "--kind", @@ -52,7 +52,7 @@ def main(): help="select mechanism for choosing orderings of solvers", type=str, default="neighbor", - choices=["neighbor", "random", "MLP", "thompson", "linear", "preset", "exp3", "knearest"] + choices=["neighbor", "random", "MLP", "thompson", "linear", "preset", "exp3", "knearest", "perfect"] ) global_parser.add_argument( @@ -69,7 +69,7 @@ def main(): help="choose how timeout is distributed amongst solvers", type=str, default="expo", - choices=["expo", "const", "nearest"] + choices=["expo", "const", "nearest", "perfect"] ) global_parser.add_argument( @@ -153,7 +153,7 @@ def main(): "--time_k", help="set k value for knearest neighbor", type=int, - default=20 + default=40 ) global_parser.add_argument( @@ -199,26 +199,30 @@ def main(): if not args.set_lambda: args.set_lambda = 1 / (args.timeout / len(SOLVERS)) timeout_manager = NearestExponential(args.set_lambda, args.confidence, args.timeout) + elif args.timeout_manager == "perfect": + timeout_manager = PerfectTimer() else: raise RuntimeError("timeout_manager not properly set") if args.classifier == "neighbor": - classifier = NearestNeighbor(args.epsilon, args.epsilon_decay, args.kind) + classifier = NearestNeighbor(args.epsilon, args.epsilon_decay, args.kind, args.time_k) elif args.classifier == "random": - classifier = Random() + classifier = Random(args.time_k) elif args.classifier == "MLP": - classifier = MLP() + classifier = MLP(args.time_k) elif args.classifier == "thompson": - classifier = Thompson(args.kind) + classifier = Thompson(args.kind, args.time_k) elif args.classifier == "linear": - classifier = LinearBandit() + classifier = LinearBandit(args.time_k) elif args.classifier == "preset": classifier = Preset(args.preset) timeout_manager = Constant(args.timeout) elif args.classifier == "exp3": - classifier = Exp3(args.gamma) + classifier = Exp3(args.gamma, args.time_k) elif args.classifier == "knearest": classifier = KNearest(args.k, args.epsilon, args.epsilon_decay, args.time_k) + elif args.classifier == "perfect": + classifier = PerfectSelector(args.time_k) else: raise RuntimeError("classifier not properly set") diff --git a/machine1.sh b/machine1.sh index ad0bbb5..a7b0be1 100644 --- a/machine1.sh +++ b/machine1.sh @@ -1,4 +1,4 @@ -for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c exp3d; do +for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c; do sh ./runlearner.sh tmp/BV $learner sh ./runlearner.sh tmp/QF_ABV $learner sh ./runlearner.sh tmp/QF_AUFBV $learner diff --git a/machine6.sh b/machine6.sh index 97f2676..17f8666 100644 --- a/machine6.sh +++ b/machine6.sh @@ -1,3 +1,3 @@ -for learner in exp3c exp3d; do +for learner in exp3c; do sh ./runlearner.sh tmp/uclid $learner done \ No newline at end of file diff --git a/machine7.sh b/machine7.sh index f4271b7..b60e21e 100644 --- a/machine7.sh +++ b/machine7.sh @@ -1,3 +1,3 @@ -for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c exp3d; do +for learner in thompson neighbor knearest random MLP linear exp3a exp3b exp3c; do sh ./runlearner.sh tmp/Sage2 $learner done \ No newline at end of file diff --git a/ordermisses.py b/ordermisses.py index b157a11..337a7eb 100755 --- a/ordermisses.py +++ b/ordermisses.py @@ -27,7 +27,9 @@ r = r if row[4] != "error" and "timeout" not in row[4] else 60 cols[solver][row[0]] = r -print("choice, answer, problem") +count = 0 + +# print("choice, answer, problem") # Load nearest, zip times with solvers with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile: spamreader = list(csv.reader(csvfile)) @@ -36,11 +38,14 @@ runtimes = eval(row[6]) solvers = eval(row[5])[:len(runtimes)] - solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] > 0 and solvers[i] in cols and row[0] in cols[solvers[i]]] + solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] >= 0 and solvers[i] in cols and row[0] in cols[solvers[i]]] times = [cols[a][row[0]] for a in solvers] index_min = min(range(len(times)), key=times.__getitem__) - if index_min != 0: - print(",".join([str(solvers[0]), str(solvers[index_min], str(row[0]))])) + if index_min != 0 and times[index_min] < times[0]: + # print(",".join([str(solvers[0]), str(solvers[index_min]), str(row[0])])) + count += 1 + +print(count) diff --git a/runlearner.sh b/runlearner.sh index 1d7ae18..0550286 100644 --- a/runlearner.sh +++ b/runlearner.sh @@ -1,16 +1,13 @@ for seed in 0 1; do - if [ $2 == exp3a ] + if [ $2 = exp3a ] then learnconfig="exp3 --gamma 0.07" - elif [ $2 == exp3b ] + elif [ $2 = exp3b ] then learnconfig="exp3 --gamma 0.1" - elif [ $2 == exp3c ] + elif [ $2 = exp3c ] then learnconfig="exp3 --gamma 0.25" - elif [ $2 == exp3d ] - then - learnconfig="exp3 --gamma 0.5" else learnconfig=$2 fi diff --git a/timemisses.py b/timemisses.py index cd11381..b127196 100755 --- a/timemisses.py +++ b/timemisses.py @@ -27,7 +27,8 @@ r = r if row[4] != "error" and "timeout" not in row[4] else 60 cols[solver][row[0]] = r -print("solver, answer, guess, problem") +total = 0 +# print("solver, answer, guess, problem") # Load nearest, zip times with solvers with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile: spamreader = list(csv.reader(csvfile)) @@ -38,6 +39,9 @@ zipped = zip(solvers, times) - for (a, b) in zipped: + for (a, b) in list(zipped)[:1]: if b > 0 and a in cols and cols[a][row[0]] > b and cols[a][row[0]] < 60: - print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])])) + # print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])])) + total += 1 + +print(total)