Contextual training

config.py

@@ -1,13 +1,13 @@
 import keras.backend as K
 
 MODEL_PATH = "glstm.hdf5"
-EPOCHS = 400
+EPOCHS = 250
 RAW_MODEL_PATH = "glstm_raw.hdf5"
 VALIDATION_MODEL = "../data/checkpoints/model_340_0.15_0.13_0.83_0.85.hdf5"
 IMAGE_SHAPE = (250, 250, 3)
 SLIC_SHAPE = (IMAGE_SHAPE[0], IMAGE_SHAPE[1])
 N_SUPERPIXELS = 100
-N_FEATURES = 3
+N_FEATURES = 4
 INPUT_PATHS = 2
 
 SLIC_SIGMA = 0

test/generate_model.py

@@ -1,7 +1,7 @@
 from keras import Input, Model
 from keras.initializers import RandomUniform
 from keras.layers import Softmax, Dense, Concatenate, Dropout
-from keras.optimizers import SGD
+from keras.optimizers import SGD, RMSprop
 from keras.utils import plot_model
 
 from config import IMAGE_SHAPE, TRAIN_BATCH_SIZE, N_SUPERPIXELS, N_FEATURES, \
@@ -12,64 +12,67 @@
 
 
 def generate_model():
-    init = RandomUniform(minval=-0.1, maxval=0.1, seed=None)
-
-    vertices = [Input(shape=(N_SUPERPIXELS, IMAGE_SHAPE[2]),
-                      name="Vertices",
-                      batch_shape=(TRAIN_BATCH_SIZE,
-                                   N_SUPERPIXELS,
-                                   IMAGE_SHAPE[2]))]
-    neighbors = [Input(shape=(N_SUPERPIXELS, N_SUPERPIXELS),
-                       name="Neighborhood",
-                       batch_shape=(TRAIN_BATCH_SIZE,
-                                    N_SUPERPIXELS,
-                                    N_SUPERPIXELS))]
-    inputs = [Input(shape=(N_SUPERPIXELS, IMAGE_SHAPE[2]),
-                    name="Vertices_{0!s}".format(i),
-                    batch_shape=(TRAIN_BATCH_SIZE,
-                                 N_SUPERPIXELS,
-                                 IMAGE_SHAPE[2]))
-              for i in range(INPUT_PATHS)]
-    indexes = [Input(shape=(N_SUPERPIXELS,),
-                     batch_shape=(TRAIN_BATCH_SIZE, N_SUPERPIXELS),
-                     name="Index_{0!s}".format(i), dtype="int32")
-               for i in range(INPUT_PATHS)]
-    r_indexes = [Input(shape=(N_SUPERPIXELS,),
-                       batch_shape=(TRAIN_BATCH_SIZE, N_SUPERPIXELS),
-                       name="ReverseIndex_{0!s}".format(i), dtype="int32")
+    for k in [2, 3, 5, 7, 9]:
+        INPUT_PATHS = k
+        init = RandomUniform(minval=-0.1, maxval=0.1, seed=None)
+
+        vertices = [Input(shape=(N_SUPERPIXELS, IMAGE_SHAPE[2]),
+                          name="Vertices",
+                          batch_shape=(TRAIN_BATCH_SIZE,
+                                       N_SUPERPIXELS,
+                                       IMAGE_SHAPE[2]))]
+        neighbors = [Input(shape=(N_SUPERPIXELS, N_SUPERPIXELS),
+                           name="Neighborhood",
+                           batch_shape=(TRAIN_BATCH_SIZE,
+                                        N_SUPERPIXELS,
+                                        N_SUPERPIXELS))]
+        inputs = [Input(shape=(N_SUPERPIXELS, IMAGE_SHAPE[2]),
+                        name="Vertices_{0!s}".format(i),
+                        batch_shape=(TRAIN_BATCH_SIZE,
+                                     N_SUPERPIXELS,
+                                     IMAGE_SHAPE[2]))
+                  for i in range(INPUT_PATHS)]
+        indexes = [Input(shape=(N_SUPERPIXELS,),
+                         batch_shape=(TRAIN_BATCH_SIZE, N_SUPERPIXELS),
+                         name="Index_{0!s}".format(i), dtype="int32")
+                   for i in range(INPUT_PATHS)]
+        r_indexes = [Input(shape=(N_SUPERPIXELS,),
+                           batch_shape=(TRAIN_BATCH_SIZE, N_SUPERPIXELS),
+                           name="ReverseIndex_{0!s}".format(i), dtype="int32")
+                     for i in range(INPUT_PATHS)]
+
+        cells = [GraphLSTMCell(N_FEATURES, kernel_initializer=init,
+                               recurrent_initializer=init,
+                               bias_initializer=init) for _ in range(INPUT_PATHS)]
+        lstms = [GraphLSTM(cells[i], return_sequences=True,
+                           name="G-LSTM_{0!s}".format(i), stateful=False)
+                 ([inputs[i], vertices[0], neighbors[0], indexes[i], r_indexes[i]])
                  for i in range(INPUT_PATHS)]
 
-    cells = [GraphLSTMCell(N_FEATURES, kernel_initializer=init,
-                           recurrent_initializer=init,
-                           bias_initializer=init) for _ in range(INPUT_PATHS)]
-    lstms = [GraphLSTM(cells[i], return_sequences=True,
-                       name="G-LSTM_{0!s}".format(i), stateful=False)
-             ([inputs[i], vertices[0], neighbors[0], indexes[i], r_indexes[i]])
-             for i in range(INPUT_PATHS)]
+        inverse = [InverseGraphPropagation()([lstms[i], r_indexes[i]]) for i in range(INPUT_PATHS)]
 
-    inverse = [InverseGraphPropagation()([lstms[i], r_indexes[i]]) for i in range(INPUT_PATHS)]
+        concat = Concatenate(axis=-1)(inverse)
+        # drop0 = Dropout(0.5)(lstms[0])
+        # d1 = Dense(int(INPUT_PATHS * N_FEATURES))(drop0)
+        # drop1 = Dropout(0.4)(concat)
+        # d2 = Dense(int(INPUT_PATHS * N_FEATURES))(drop1)
+        d3 = Dense(N_FEATURES)(concat)
 
-    concat = Concatenate(axis=-1)(inverse)
-    # drop0 = Dropout(0.5)(lstms[0])
-    # d1 = Dense(int(INPUT_PATHS * N_FEATURES))(drop0)
-    # drop1 = Dropout(0.4)(concat)
-    # d2 = Dense(int(INPUT_PATHS * N_FEATURES))(drop1)
-    d3 = Dense(N_FEATURES)(concat)
+        soft = Softmax()(d3)
 
-    soft = Softmax()(d3)
+        model = Model(inputs=vertices + neighbors + inputs + indexes + r_indexes,
+                      outputs=[soft])
 
-    model = Model(inputs=vertices + neighbors + inputs + indexes + r_indexes,
-                  outputs=[soft])
+        model.summary()
 
-    model.summary()
+        # PLOT
+        # plot_model(model, show_shapes=True)
 
-    # PLOT
-    plot_model(model, show_shapes=True)
-
-    # OPTIMIZER
-    sgd = SGD(lr=0.01, momentum=0.9, decay=0.005, nesterov=False)
-    model.compile(sgd, loss="categorical_crossentropy", metrics=["acc"])
-    model.save(RAW_MODEL_PATH)
+        # OPTIMIZER
+        # sgd = SGD(lr=0.001, momentum=0.9, decay=0.005, nesterov=False)
+        rms = RMSprop()
+        model.compile(rms, loss="categorical_crossentropy", metrics=["acc"])
+        model.save("glstm_raw{0!s}.hdf5".format(k))
 
 
 # return model

test/train.py

@@ -1,3 +1,4 @@
+import copy
 import random
 import matplotlib.pyplot as plt
 import numpy
@@ -15,24 +16,25 @@
     get_superpixels_index_for_hot_areas
 
 
-def init_callbacks():
+def init_callbacks(k):
     terminator = TerminateOnNaN()
     checkpointer = ModelCheckpoint(
-        "../data/checkpoints/model_{epoch:02d}_{loss:.2f}_{val_loss:.2f}_{acc:.2f}_{val_acc:.2f}.hdf5",
+        "../data/checkpoints/model"+str(k)+"_{epoch:02d}_{loss:.2f}_{val_loss:.2f}_{acc:.2f}_{val_acc:.2f}.hdf5",
         monitor="val_loss", save_weights_only=False, mode="min", period=5)
     return [terminator, checkpointer]
 
 
-def generator(image_list, images_path, expected_images, size=1):
+def generator(image_list, images_path, expected_images, k, size=1):
     while True:
         batch_names = numpy.random.choice(image_list, size=size)
 
         batch_vertices = []
         batch_neighbors = []
+        batch_expected = []
 
-        batch_inputs = [[] for _ in range(INPUT_PATHS)]
-        batch_indexes = [[] for _ in range(INPUT_PATHS)]
-        batch_r_indexes = [[] for _ in range(INPUT_PATHS)]
+        batch_inputs = [[] for _ in range(k)]
+        batch_indexes = [[] for _ in range(k)]
+        batch_r_indexes = [[] for _ in range(k)]
 
         for image_name in batch_names:
             # LOAD IMAGES
@@ -43,8 +45,13 @@ def generator(image_list, images_path, expected_images, size=1):
             slic = obtain_superpixels(img, N_SUPERPIXELS, SLIC_SIGMA)
             vertices = average_rgb_for_superpixels(img, slic)
             neighbors = get_neighbors(slic, N_SUPERPIXELS)
+
+            expected = copy.deepcopy(vertices)
+            for i in expected:
+                i.append(0.0)
+
             areas = get_superpixels_index_for_hot_areas(slic)
-            for paths in range(INPUT_PATHS):
+            for paths in range(k):
                 vertex_index = areas[paths]
                 path, mapping, r_mapping = sort_values(vertices, neighbors,
                                                        vertex_index,
@@ -53,12 +60,26 @@ def generator(image_list, images_path, expected_images, size=1):
                 batch_indexes[paths].append(mapping)
                 batch_r_indexes[paths].append(r_mapping)
 
+            green_near_red_indices = []
+            for i, v in enumerate(vertices):
+                if v[0] != 1.0:
+                    continue
+                neighborhood_indexes = numpy.where(neighbors[i] == 1)[0]
+                if any(vertices[n][1] == 1.0 for n in neighborhood_indexes):
+                    green_near_red_indices.append(i)
+
+            for i in green_near_red_indices:
+                expected[i] = [0.0] * len(expected[i])
+                expected[i][-1] = 1.0
+
             # ADD TO BATCH
             batch_vertices += [vertices]
             batch_neighbors += [neighbors]
+            batch_expected += [expected]
 
         batch_vertices = numpy.array(batch_vertices)
         batch_neighbors = numpy.array(batch_neighbors)
+        batch_expected = numpy.array(batch_expected)
         batch_inputs = [numpy.array(i) for i in batch_inputs]
         batch_indexes = [numpy.array(i) for i in batch_indexes]
         batch_r_indexes = [numpy.array(i) for i in batch_r_indexes]
@@ -67,47 +88,48 @@ def generator(image_list, images_path, expected_images, size=1):
                batch_inputs +
                batch_indexes +
                batch_r_indexes,
-               [batch_vertices])
+               [batch_expected])
 
 
 if __name__ == '__main__':
-    callbacks = init_callbacks()
-    val_image_list = image_list[:VALIDATION_ELEMS]
-    train_image_list = image_list[VALIDATION_ELEMS:]
-
-    model = load_model(RAW_MODEL_PATH,
-                       custom_objects={'GraphLSTM': GraphLSTM,
-                                       'GraphLSTMCell': GraphLSTMCell,
-                                       'InverseGraphPropagation': InverseGraphPropagation})
-    # model = create_model()
-    history = model.fit_generator(
-        generator(train_image_list, "../data/test/", "../data/test/",
-                  TRAIN_BATCH_SIZE),
-        steps_per_epoch=numpy.ceil(TRAIN_ELEMS / TRAIN_BATCH_SIZE),
-        epochs=EPOCHS,
-        verbose=1,
-        callbacks=callbacks,
-        validation_data=generator(val_image_list, "../data/test/",
-                                  "../data/test/", VALIDATION_BATCH_SIZE),
-        validation_steps=numpy.ceil(VALIDATION_ELEMS / VALIDATION_BATCH_SIZE),
-        max_queue_size=10,
-        shuffle=True)
-    model.save(MODEL_PATH)
-
-    plt.plot(history.history['acc'], color="#FF3864")
-    plt.plot(history.history['val_acc'], color="#261447")
-    plt.title('Accuracy')
-    plt.ylabel('accuracy')
-    plt.xlabel('epoch')
-    plt.legend(['train', 'test'], loc='upper left')
-    plt.show()
-    plt.savefig('model_accuracy.png')
-    # summarize history for loss
-    plt.plot(history.history['loss'], color="#FF3864")
-    plt.plot(history.history['val_loss'], color="#261447")
-    plt.title('Loss')
-    plt.ylabel('loss')
-    plt.xlabel('epoch')
-    plt.legend(['train', 'test'], loc='upper left')
-    plt.show()
-    plt.savefig('model_loss.png')
+    for k in [2, 3, 5, 7, 9]:
+        callbacks = init_callbacks(k)
+        val_image_list = image_list[:VALIDATION_ELEMS]
+        train_image_list = image_list[VALIDATION_ELEMS:]
+
+        model = load_model("glstm_raw{0!s}.hdf5".format(k),
+                           custom_objects={'GraphLSTM': GraphLSTM,
+                                           'GraphLSTMCell': GraphLSTMCell,
+                                           'InverseGraphPropagation': InverseGraphPropagation})
+        # model = create_model()
+        history = model.fit_generator(
+            generator(train_image_list, "../data/test/", "../data/test/", k,
+                      TRAIN_BATCH_SIZE),
+            steps_per_epoch=numpy.ceil(TRAIN_ELEMS / TRAIN_BATCH_SIZE),
+            epochs=EPOCHS,
+            verbose=1,
+            callbacks=callbacks,
+            validation_data=generator(val_image_list, "../data/test/",
+                                      "../data/test/", k, VALIDATION_BATCH_SIZE),
+            validation_steps=numpy.ceil(VALIDATION_ELEMS / VALIDATION_BATCH_SIZE),
+            max_queue_size=10,
+            shuffle=True)
+        model.save("glstm{0!s}.hdf5".format(k))
+
+        plt.plot(history.history['acc'], color="#FF3864")
+        plt.plot(history.history['val_acc'], color="#261447")
+        plt.title('Accuracy - {0!s} paths'.format(k))
+        plt.ylabel('accuracy')
+        plt.xlabel('epoch')
+        plt.legend(['train', 'test'], loc='upper left')
+        plt.show()
+        plt.savefig('model_accuracy_{0!s}_paths.png'.format(k))
+        # summarize history for loss
+        plt.plot(history.history['loss'], color="#FF3864")
+        plt.plot(history.history['val_loss'], color="#261447")
+        plt.title('Loss - {0!s} paths'.format(k))
+        plt.ylabel('loss')
+        plt.xlabel('epoch')
+        plt.legend(['train', 'test'], loc='upper left')
+        plt.show()
+        plt.savefig('model_loss_{0!s}_paths.png'.format(k))

utils/utils.py

@@ -16,6 +16,9 @@ def average_rgb_for_superpixels(image, segments):
         av_local = []
         for c in range(filtered.shape[-1]):
             av_local.append(np.sum(filtered[:, :, c], axis=1).sum() / non_zero_pixels_amount)
+        i = av_local.index(max(av_local))
+        av_local = [0.0] * len(av_local)
+        av_local[i] = 1.0
         averages.append(av_local)
     while len(averages) != N_SUPERPIXELS:
         averages.append([0.0] * IMAGE_SHAPE[-1])