-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathnaive_baseline.py
187 lines (152 loc) · 7.68 KB
/
naive_baseline.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
#!/usr/bin/env python
# -*- coding: utf-8 -*-
################################################################################
# Copyright (c) 2019. Vincenzo Lomonaco, Massimo Caccia, Pau Rodriguez, #
# Lorenzo Pellegrini. All rights reserved. #
# Copyrights licensed under the CC BY 4.0 License. #
# See the accompanying LICENSE file for terms. #
# #
# Date: 1-02-2019 #
# Author: Vincenzo Lomonaco #
# E-mail: [email protected] #
# Website: vincenzolomonaco.com #
################################################################################
"""
Getting Started example for the CVPR 2020 CLVision Challenge. It will load the data and create the submission file
for you in the cvpr_clvision_challenge/submissions directory.
"""
# Python 2-3 compatible
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import argparse # The argparse module also automatically generates help and usage messages, and issues errors when users give the program invalid arguments.
import os # Usefull for file system and os manipulation
import time # for tracking time it takes to complete work
import copy # provides for shallow and deep copy opperations
from core50.dataset import CORE50 # Imports the custom module for dealing with the core50 dataset
import torch # Imports the torch library
import numpy as np # imports the numpy library
from utils.train_test import train_net, test_multitask, preprocess_imgs # custom utils in the util module inside the sub utils folder
import torchvision.models as models # provides access to pytorch compatable datasets
from utils.common import create_code_snapshot # custom utils in the util module inside the sub utils folder
def main(args):
# print args recap
print(args, end="\n\n")
# do not remove this line
start = time.time()
# Create the dataset object for example with the "ni, multi-task-nc, or nic tracks"
# and assuming the core50 location in ./core50/data/
# ???review CORE50 to see if there is a way to shorten dataset and runtime for testing
# Original call to dataset. Takes a long time.
# dataset = CORE50(root='core50/data/', scenario=args.scenario,
# preload=args.preload_data)
#
# custom call to create CORE50 custom object
# using train=True uses training set and allows more control over batches and other stuff.
dataset = CORE50(root='core50/data/', scenario=args.scenario,
preload=args.preload_data)
# Get the validation set
print("Recovering validation set...")
# default full validation set
# full_valdidset = dataset.get_full_valid_set()
# reduced validation set
full_valdidset = dataset.get_full_valid_set(reduced=True)
# model
if args.classifier == 'ResNet18':
classifier = models.resnet18(pretrained=True) # classifier is a pretrained model
classifier.fc = torch.nn.Linear(512, args.n_classes) # in features: 512 # out features: set below -> args.n_classes = 50 # Applies a linear transformation to the incoming data
opt = torch.optim.SGD(classifier.parameters(), lr=args.lr) # Implements stochastic gradient descent
criterion = torch.nn.CrossEntropyLoss() # This criterion combines nn.LogSoftmax() and nn.NLLLoss() in one single class.
# vars to update over time
valid_acc = []
ext_mem_sz = []
ram_usage = []
heads = []
# loop over the training incremental batches (x, y, t)
for i, train_batch in enumerate(dataset):
train_x, train_y, t = train_batch
# Print current batch number
print("----------- batch {0} -------------".format(i))
# Print current batch shape
print("x shape: {0}, y shape: {1}"
.format(train_x.shape, train_y.shape))
# print task label type
print("Task Label: ", t)
# utils.train_net: a custom function to train neural network. returns stats.
_, _, stats = train_net(
opt, classifier, criterion, args.batch_size, train_x, train_y, t,
args.epochs, preproc=preprocess_imgs
)
# if multi-task-nc: make deep copy in list heads (aka nn brains)
if args.scenario == "multi-task-nc":
heads.append(copy.deepcopy(classifier.fc))
ext_mem_sz += stats['disk']
ram_usage += stats['ram']
# test all nn models in list heads for performance. return stats for each.
stats, _ = test_multitask(
classifier, full_valdidset, args.batch_size, preproc=preprocess_imgs, multi_heads=heads
)
# print new stats on performance
valid_acc += stats['acc']
print("------------------------------------------")
print("Avg. acc: {}".format(stats['acc']))
print("------------------------------------------")
# Generate submission.zip
# directory with the code snapshot to generate the results
sub_dir = 'submissions/' + args.sub_dir
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
# copy code
# custom function in utils folder to deal with possible file path issues
create_code_snapshot(".", sub_dir + "/code_snapshot")
# generating metadata.txt: with all the data used for the CLScore
elapsed = (time.time() - start) / 60
print("Training Time: {}m".format(elapsed))
with open(sub_dir + "/metadata.txt", "w") as wf:
for obj in [
np.average(valid_acc), elapsed, np.average(ram_usage),
np.max(ram_usage), np.average(ext_mem_sz), np.max(ext_mem_sz)
]:
wf.write(str(obj) + "\n")
# run final full test
# test_preds.txt: with a list of labels separated by "\n"
print("Final inference on test set...")
full_testset = dataset.get_full_test_set()
stats, preds = test_multitask(
classifier, full_testset, args.batch_size, preproc=preprocess_imgs
)
with open(sub_dir + "/test_preds.txt", "w") as wf:
for pred in preds:
wf.write(str(pred) + "\n")
print("Experiment completed.")
# Code Starts running here
if __name__ == "__main__":
parser = argparse.ArgumentParser('CVPR Continual Learning Challenge')
# Setup module to recieve arguments
# General
parser.add_argument('--scenario', type=str, default="multi-task-nc",
choices=['ni', 'multi-task-nc', 'nic'])
parser.add_argument('--preload_data', type=bool, default=True,
help='preload data into RAM')
# Model
parser.add_argument('-cls', '--classifier', type=str, default='ResNet18',
choices=['ResNet18'])
# Optimization
# --lr: Set Learning rate
parser.add_argument('--lr', type=float, default=0.01,
help='learning rate')
# --batch_size: set batch size
parser.add_argument('--batch_size', type=int, default=32,
help='batch_size')
# --epochs:
parser.add_argument('--epochs', type=int, default=1,
help='number of epochs')
# Misc
parser.add_argument('--sub_dir', type=str, default="multi-task-nc",
help='directory of the submission file for this exp.')
args = parser.parse_args()
args.n_classes = 50
args.input_size = [3, 128, 128]
args.cuda = torch.cuda.is_available()
args.device = 'cuda:0' if args.cuda else 'cpu'
main(args)