-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathreproduce_ablation.py
96 lines (83 loc) · 4.74 KB
/
reproduce_ablation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
from util.helper_classes import Reproduce
from models.ensemble import Ensemble
import numpy as np
from util.helper_funcs import compute_confidence_interval
fb15k237_path = 'KGs/FB15k-237'
no_input_dropout = True
no_label_smoothing = True
no_dropout = True
use_rmsprop = True
lp_per_relation_results = False
if no_input_dropout:
print('###########################################{0}##################################################'.format(
fb15k237_path))
print('NO input')
report1 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/NoDropout/1', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
report2 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/NoDropout/2', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
hit10 = np.array([report1['Hits@10'], report2['Hits@10']])
hit3 = np.array([report1['Hits@3'], report2['Hits@3']])
hit1 = np.array([report1['Hits@1'], report2['Hits@1']])
mrr = np.array([report1['MRR'], report2['MRR']])
results = {'MRR': mrr,
'Hits@1': hit1,
'Hits@3': hit3,
'Hits@10': hit10}
compute_confidence_interval(results)
print('###########################################{0}##################################################'.format(
fb15k237_path))
if no_label_smoothing:
print('###########################################{0}##################################################'.format(
fb15k237_path))
report1 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/NpLS/1', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
report2 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/NpLS/2', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
hit10 = np.array([report1['Hits@10'], report2['Hits@10']])
hit3 = np.array([report1['Hits@3'], report2['Hits@3']])
hit1 = np.array([report1['Hits@1'], report2['Hits@1']])
mrr = np.array([report1['MRR'], report2['MRR']])
results = {'MRR': mrr,
'Hits@1': hit1,
'Hits@3': hit3,
'Hits@10': hit10}
compute_confidence_interval(results)
print('###########################################{0}##################################################'.format(
fb15k237_path))
if no_dropout:
print('###########################################{0}##################################################'.format(
fb15k237_path))
report1 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/NoDropout/1', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
report2 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/NoDropout/2', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
hit10 = np.array([report1['Hits@10'], report2['Hits@10']])
hit3 = np.array([report1['Hits@3'], report2['Hits@3']])
hit1 = np.array([report1['Hits@1'], report2['Hits@1']])
mrr = np.array([report1['MRR'], report2['MRR']])
results = {'MRR': mrr,
'Hits@1': hit1,
'Hits@3': hit3,
'Hits@10': hit10}
compute_confidence_interval(results)
print('###########################################{0}##################################################'.format(
fb15k237_path))
if use_rmsprop:
print('###########################################{0}##################################################'.format(
fb15k237_path))
report1 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/Optim/1', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
report2 = Reproduce().reproduce(model_path='PretrainedModels/Ablation/Optim/2', data_path="%s/" % fb15k237_path,
model_name='ConEx', per_rel_flag_=lp_per_relation_results)
hit10 = np.array([report1['Hits@10'], report2['Hits@10']])
hit3 = np.array([report1['Hits@3'], report2['Hits@3']])
hit1 = np.array([report1['Hits@1'], report2['Hits@1']])
mrr = np.array([report1['MRR'], report2['MRR']])
results = {'MRR': mrr,
'Hits@1': hit1,
'Hits@3': hit3,
'Hits@10': hit10}
compute_confidence_interval(results)
print('###########################################{0}##################################################'.format(
fb15k237_path))