-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathla_classifier.py
107 lines (83 loc) · 3.04 KB
/
la_classifier.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
import os
import pandas as pd
import csv
import utils
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from metrices_calculator import calculate_effort, calculate_normalized_effort
dataset_name = 'ase_dataset_sept_19_2021.csv'
def get_loc_add():
print("Reading dataset...")
df = pd.read_csv(dataset_name)
df = df[['commit_id', 'repo', 'partition', 'diff', 'label', 'PL', 'LOC_ADD']]
items = df.to_numpy().tolist()
url_to_diff = {}
url_to_label = {}
url_to_pl = {}
url_to_loc_add = {}
for item in items:
commit_id = item[0]
repo = item[1]
url = repo + '/commit/' + commit_id
label = item[4]
pl = item[5]
loc_add = item[6]
if url not in url_to_diff:
url_to_diff[url] = ''
url_to_loc_add[url] = 0
url_to_label[url] = label
url_to_pl[url] = pl
url_to_loc_add[url] += loc_add
return url_to_label, url_to_loc_add
def do_train():
print("Reading data...")
url_data, label_data = utils.get_data(dataset_name)
url_to_label, url_to_loc_add = get_loc_add()
X_train = []
y_train = []
for url in url_data['train']:
X_train.append([url_to_loc_add[url]])
y_train.append(url_to_label[url])
print("Training classifier...")
clf = LogisticRegression(random_state=109).fit(X_train, y_train)
# Testing on Java
print("Testing on Java...")
X_test_java = []
y_test_java = []
for url in url_data['test_java']:
X_test_java.append([url_to_loc_add[url]])
y_test_java.append(url_to_label[url])
pred_probs = clf.predict_proba(X_test_java)
y_pred_java = []
for prob in pred_probs:
y_pred_java.append(prob[1])
auc = metrics.roc_auc_score(y_true=y_test_java, y_score=y_pred_java)
print("AUC on java: {}".format(auc))
la_java_prob_path = 'probs/la_prob_java.txt'
with open(la_java_prob_path, 'w') as file:
writer = csv.writer(file)
for i, url in enumerate(url_data['test_java']):
writer.writerow([url, pred_probs[i][1]])
calculate_effort(la_java_prob_path, 'java')
calculate_normalized_effort(la_java_prob_path, 'java')
# Testing on Python
X_test_python = []
y_test_python = []
for url in url_data['test_python']:
X_test_python.append([url_to_loc_add[url]])
y_test_python.append(url_to_label[url])
pred_probs = clf.predict_proba(X_test_python)
y_pred_python = []
for prob in pred_probs:
y_pred_python.append(prob[1])
auc = metrics.roc_auc_score(y_true=y_test_python, y_score=y_pred_python)
print("AUC on python: {}".format(auc))
la_python_prob_path = 'probs/la_prob_python.txt'
with open(la_python_prob_path, 'w') as file:
writer = csv.writer(file)
for i, url in enumerate(url_data['test_python']):
writer.writerow([url, pred_probs[i][1]])
calculate_effort(la_java_prob_path, 'python')
calculate_normalized_effort(la_java_prob_path, 'python')
if __name__ == '__main__':
do_train()