From aab2f8b7605105cefef22f574253be904401d218 Mon Sep 17 00:00:00 2001 From: FedericoAureliano Date: Tue, 13 Oct 2020 12:59:00 -0700 Subject: [PATCH] fixes to scripts --- bin/medley | 4 ++-- medleysolver/classifiers.py | 1 + medleysolver/dispatch.py | 2 +- medleysolver/timers.py | 5 +++-- ordermisses.py | 13 +++++++------ process.py | 20 +++----------------- runmedley.sh | 19 +++++++++++++++---- timemisses.py | 11 ++++++----- 8 files changed, 38 insertions(+), 37 deletions(-) diff --git a/bin/medley b/bin/medley index e3ea66c..936fafe 100755 --- a/bin/medley +++ b/bin/medley @@ -147,7 +147,7 @@ def main(): type=int, default=20 ) - + global_parser.add_argument( "--feature_setting", "-f", @@ -183,7 +183,7 @@ def main(): elif args.timeout_manager == "nearest": if not args.set_lambda: args.set_lambda = 1 / (args.timeout / len(SOLVERS)) - timeout_manager = NearestExponential(args.set_lambda, args.confidence) + timeout_manager = NearestExponential(args.set_lambda, args.confidence, args.timeout) else: raise RuntimeError("timeout_manager not properly set") diff --git a/medleysolver/classifiers.py b/medleysolver/classifiers.py index 927d713..d0f5060 100644 --- a/medleysolver/classifiers.py +++ b/medleysolver/classifiers.py @@ -223,6 +223,7 @@ def get_ordering(self, point, count): order = sorted(ss, key= lambda x: -1 * methods.count(x)) else: order = Random.get_ordering(self, point, count) + return order def update(self, solved_prob, rewards): #TODO: Implement pruning diff --git a/medleysolver/dispatch.py b/medleysolver/dispatch.py index d2492bd..624fb14 100644 --- a/medleysolver/dispatch.py +++ b/medleysolver/dispatch.py @@ -4,7 +4,7 @@ import csv def run_problem(solver, invocation, problem, timeout): - instance = problem.split("/", 2)[-1] + instance = problem.split("/")[-1] directory = problem[:-len(instance)] try: diff --git a/medleysolver/timers.py b/medleysolver/timers.py index 2c53fa3..5fffa20 100644 --- a/medleysolver/timers.py +++ b/medleysolver/timers.py @@ -36,13 +36,14 @@ def update(self, solver, time, timeout, success, error): self.timers[solver].add_timeout() class NearestExponential(TimerInterface): - def __init__(self, init_lambda, confidence): + def __init__(self, init_lambda, confidence, T): self.init_lambda = init_lambda self.confidence = confidence + self.T = T def get_timeout(self, solver, times): # want time based on times for same solver at nearby points - timer = ExponentialDist(self.init_lambda, self.confidence) + timer = ExponentialDist(self.init_lambda, self.confidence, self.T) for (s, t) in times: if s == solver: timer.add_sample(t) diff --git a/ordermisses.py b/ordermisses.py index d5de896..1859a63 100755 --- a/ordermisses.py +++ b/ordermisses.py @@ -15,7 +15,7 @@ cols = {} for r in result: - cols[r[:-len(".csv")]] = [] + cols[r[:-len(".csv")]] = {} for r in result: solver = r[:-len(".csv")] @@ -24,9 +24,10 @@ for row in spamreader: r = float(row[3]) if isinstance(row[3], str) else row[3] r = r if r < 60 else 60 - cols[solver].append(r) + r = r if row[4] != "error" else 60 + cols[solver][row[0]] = r -print("problem, choice, answer") +print("choice, answer, problem") # Load nearest, zip times with solvers with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile: spamreader = list(csv.reader(csvfile)) @@ -35,11 +36,11 @@ runtimes = eval(row[6]) solvers = eval(row[5])[:len(runtimes)] - solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] > 0 and solvers[i] in cols] + solvers = [solvers[i] for i in range(len(solvers)) if runtimes[i] > 0 and solvers[i] in cols and row[0] in cols[solvers[i]]] - times = [cols[a][r] for a in solvers] + times = [cols[a][row[0]] for a in solvers] index_min = min(range(len(times)), key=times.__getitem__) if index_min != 0: - print(",".join([str(r), str(solvers[0]), str(solvers[index_min])])) + print(",".join([str(solvers[0]), str(solvers[index_min], str(row[0]))])) diff --git a/process.py b/process.py index f22481b..692a722 100755 --- a/process.py +++ b/process.py @@ -27,20 +27,6 @@ totaltimes = [str(sum(time)) for time in times] totalcount = [str(sum(map(lambda x: 1 if x == "sat" or x == "unsat" else 0, answer))) for answer in answers] -virtualbesttime = 0 -virtualbestcount = 0 -for j in range(len(times[0])): - choices = [] - for i in range(len(solvers)): - if "sat" in answers[i][j]: - choices.append(times[i][j]) - - if len(choices) > 0: - virtualbestcount += 1 - - virtualbesttime += min(*(choices + [TIMEOUT, TIMEOUT])) - - -print(",".join(solvers + ["Virtual Best"])) -print(",".join(totaltimes + [str(virtualbesttime)])) -print(",".join(totalcount + [str(virtualbestcount)])) +print(",".join(solvers)) +print(",".join(totaltimes)) +print(",".join(totalcount)) diff --git a/runmedley.sh b/runmedley.sh index e22fd2f..14a0c6c 100644 --- a/runmedley.sh +++ b/runmedley.sh @@ -1,6 +1,17 @@ medley ./$1/ ./$1/thompson.csv -c thompson medley ./$1/ ./$1/nearest.csv -c neighbor -medley ./$1/ ./$1/random.csv -c random -medley ./$1/ ./$1/MLP.csv -c MLP -medley ./$1/ ./$1/linear.csv -c linear -medley ./$1/ ./$1/exp3.csv -c exp3 +medley ./$1/ ./$1/knearest.csv -c knearest --k 10 +medley ./$1/ ./$1/random.csv -c random +medley ./$1/ ./$1/MLP.csv -c MLP +medley ./$1/ ./$1/linear.csv -c linear +medley ./$1/ ./$1/exp3.csv -c exp3 + +medley ./$1/ ./$1/thompson_only.csv -c thompson --timeout_manager const --set_const 60 +medley ./$1/ ./$1/nearest_only.csv -c neighbor --timeout_manager const --set_const 60 +medley ./$1/ ./$1/knearest_only.csv -c knearest --k 10 --timeout_manager const --set_const 60 +medley ./$1/ ./$1/random_only.csv -c random --timeout_manager const --set_const 60 +medley ./$1/ ./$1/MLP_only.csv -c MLP --timeout_manager const --set_const 60 +medley ./$1/ ./$1/linear_only.csv -c linear --timeout_manager const --set_const 60 +medley ./$1/ ./$1/exp3_only.csv -c exp3 --timeout_manager const --set_const 60 + +medley ./$1/ ./$1/timenearest.csv -c knearest --k 10 --timeout_manager nearest --time_k 20 \ No newline at end of file diff --git a/timemisses.py b/timemisses.py index 57ff514..f1e05b2 100755 --- a/timemisses.py +++ b/timemisses.py @@ -15,7 +15,7 @@ cols = {} for r in result: - cols[r[:-len(".csv")]] = [] + cols[r[:-len(".csv")]] = {} for r in result: solver = r[:-len(".csv")] @@ -24,9 +24,10 @@ for row in spamreader: r = float(row[3]) if isinstance(row[3], str) else row[3] r = r if r < 60 else 60 - cols[solver].append(r) + r = r if row[4] != "error" else 60 + cols[solver][row[0]] = r -print("problem, solver, answer, guess") +print("solver, answer, guess, problem") # Load nearest, zip times with solvers with open("%s/%s/%s.csv"%(path, dataset, learner)) as csvfile: spamreader = list(csv.reader(csvfile)) @@ -38,5 +39,5 @@ for (a, b) in zipped: - if b > 0 and a in cols and cols[a][r] > b and cols[a][r] < 60: - print(",".join([str(r), str(a), str(cols[a][r]), str(b)])) + if b > 0 and a in cols and cols[a][row[0]] > b and cols[a][row[0]] < 60: + print(",".join([str(a), str(cols[a][row[0]]), str(b), str(row[0])]))