-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathkeras_nets.py
117 lines (103 loc) · 6.08 KB
/
keras_nets.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#==============================================================================
# Pre-defined nets for Keras
# Each returns a Keras model and an optimizer function
# Sourya Dey, USC
#==============================================================================
#==============================================================================
#==============================================================================
# # Imports
#==============================================================================
#==============================================================================
import numpy as np
from keras.models import Sequential
from keras.layers import Conv2D,MaxPooling2D,Flatten,Dense,Dropout,Activation
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l2
#==============================================================================
#==============================================================================
#==============================================================================
# # General CL functions
#==============================================================================
#==============================================================================
def any_cl_only(config, activation='relu', output_activation='softmax', kernel_initializer='he_normal', bias_initializer='zeros', kernel_regularizer=l2(0.)):
'''
Any MLP network
lr and decay are set to defaults for Adam as in Keras
'''
model = Sequential()
model = add_cls(model, config, activation=activation, output_activation=output_activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer)
return model
def add_cls(model, config, activation='relu', output_activation='softmax', kernel_initializer='he_normal', bias_initializer='zeros', kernel_regularizer=l2(0.)):
'''
Helper function to add CLs to any EXISTING model
Inputs:
model: Existing model
config: Must know what the exact shape of CL portion is
Eg: If adding CLs to a CIFAR net with CNNs, then we must know what the number of neurons after flattening is
Eg: cifar_gitblog1_example gives 4096 neurons after flattening, so maybe config = np.array([4096,512,10])
kernel_initializer, bias_initializer, kernel_regularizer: Use the same for all layers
Use activation for all hidden layers, output_activation for output layer
Output:
Model with CLs attached
Possible improvement:
Add dropout as as ndarray input with size = len(config)-1
'''
for i in range(1,len(config)):
if i==len(config)-1: #Use output_activation for output layer and name it 'output'
model.add(Dense(config[i], input_shape=(config[i-1],), activation=output_activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, name='output'))
else: #Standard hidden layers
model.add(Dense(config[i], input_shape=(config[i-1],), activation=activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer))
return model
#==============================================================================
def cifar_deep(config=np.array([4096,512,10]), activation='relu', output_activation='softmax', kernel_initializer='he_normal',
bias_initializer='zeros', kernel_regularizer=l2(0.), dropout=0.5):
'''
dropout: Fraction of units to DROP, i.e. set to 0. for no dropout
'''
model = Sequential()
model.add(Conv2D(60, (3, 3), padding='same', input_shape=(32,32,3))) #note that conv layers have no regularizer, but they do have dropout
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(Conv2D(60, (3, 3), padding='same'))
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Conv2D(125, (3, 3), padding='same'))
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(Conv2D(125, (3, 3), padding='same'))
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Conv2D(250, (3, 3), padding='same'))
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(Conv2D(250, (3, 3), padding='same'))
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Flatten(name='flatten_before_mlp'))
model = add_cls(model, config, activation=activation, output_activation=output_activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer)
# NOTE: Original code uses dropout(0.5) in between any 2 CLs
#==============================================================================
# opt = SGD(lr=lr,decay=decay,momentum=0.9)
# opt = Adam(lr=lr, decay=decay)
#==============================================================================
return model
def cifar_shallow(config=np.array([4000,500,100]), activation='relu', output_activation='softmax', kernel_initializer='he_normal',
bias_initializer='zeros', kernel_regularizer=l2(0.), dropout=0.25):
'''
Intentionally make it harder for MLP by reducing the amount of feature extraction via CNN
'''
model = Sequential()
model.add(Conv2D(250, (5, 5), padding='same', input_shape=(32,32,3)))
model.add(BatchNormalization(axis=3))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(8, 8)))
model.add(Dropout(dropout))
model.add(Flatten(name='flatten_before_mlp'))
model = add_cls(model, config, activation=activation, output_activation=output_activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer)
return model