iterative_replacement-imagenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/job:localhost/replica:0/task:0/device:CPU:0', '/job:localhost/replica:0/task:0/device:XLA_GPU:0', '/job:localhost/replica:0/task:0/device:XLA_CPU:0', '/job:localhost/replica:0/task:0/device:GPU:0']\n"
     ]
    }
   ],
   "source": [
    "import tensorflow.keras as keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.models import load_model\n",
    "import tensorflow.keras.layers as layers\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "print(tf.config.experimental_list_devices())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0-dev20190506'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.executing_eagerly()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "\n",
    "AUTOTUNE = tf.data.experimental.AUTOTUNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.applications.VGG16(weights='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./baseline_vgg_imagenet.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = [i for i, layer in enumerate(model.layers) if layer.__class__.__name__ == 'Conv2D']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'target' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-1d201b8bcf8d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m get_output = tf.keras.Model(inputs=model.input, \n\u001b[0;32m----> 3\u001b[0;31m                             outputs=[model.layers[target - 1].output,\n\u001b[0m\u001b[1;32m      4\u001b[0m                                      model.layers[target].output])\n",
      "\u001b[0;31mNameError\u001b[0m: name 'target' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "get_output = tf.keras.Model(inputs=model.input, \n",
    "                            outputs=[model.layers[target - 1].output,\n",
    "                                     model.layers[target].output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'get_output' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-a656d58f4b68>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'get_output' is not defined"
     ]
    }
   ],
   "source": [
    "get_output.output[1].shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'x_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-31ef7fe2146a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_tensor_slices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepeat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'x_train' is not defined"
     ]
    }
   ],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
    "\n",
    "dataset = dataset.batch(batch_size)\n",
    "dataset = dataset.repeat()\n",
    "\n",
    "dataset_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
    "\n",
    "dataset_test = dataset.batch(batch_size)\n",
    "dataset_test = dataset.repeat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_replacement(get_output):\n",
    "    inputs = tf.keras.Input(shape=get_output.output[0].shape[1::])\n",
    "    X = tf.keras.layers.SeparableConv2D(name=f'sep_conv_{build_replacement.counter}', filters=get_output.output[1].shape[-1], \n",
    "                                        kernel_size= (3,3),\n",
    "                                        padding='Same')(inputs)\n",
    "    X = tf.keras.layers.BatchNormalization(name=f'batch_norm_{build_replacement.counter}')(X)\n",
    "    X = tf.keras.layers.ReLU(name=f'relu_{build_replacement.counter}')(X)\n",
    "    \n",
    "    build_replacement.counter += 1\n",
    "    \n",
    "    X = tf.keras.layers.SeparableConv2D(name=f'sep_conv_{build_replacement.counter}', filters=get_output.output[1].shape[-1],\n",
    "                                        kernel_size=(3,3), \n",
    "                                        padding='Same')(X)\n",
    "    X = tf.keras.layers.BatchNormalization(name=f'batch_norm_{build_replacement.counter}')(X)\n",
    "    X = tf.keras.layers.ReLU(name=f'relu_{build_replacement.counter}')(X)\n",
    "    replacement_layers = tf.keras.Model(inputs=inputs, outputs=X)\n",
    "    \n",
    "    build_replacement.counter += 1\n",
    "    \n",
    "    return replacement_layers\n",
    "\n",
    "build_replacement.counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "class LayerBatch(tf.keras.utils.Sequence):\n",
    "    \n",
    "    def __init__(self, input_model, dataset):\n",
    "        self.input_model = input_model\n",
    "        self.dataset = dataset.__iter__()\n",
    "        \n",
    "    def __len__(self):\n",
    "        return math.ceil(50000 / 32)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        X, y = self.input_model(next(self.dataset))\n",
    "        return X, y\n",
    "    \n",
    "import math\n",
    "class LayerTest(tf.keras.utils.Sequence):\n",
    "    \n",
    "    def __init__(self, input_model, dataset):\n",
    "        self.input_model = input_model\n",
    "        self.dataset = dataset.__iter__()\n",
    "        \n",
    "    def __len__(self):\n",
    "        return math.ceil(10000 / 32)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        X, y = self.input_model(next(self.dataset))\n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.keras as keras\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.layers as layers\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.models import load_model\n",
    "from numba import cuda\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "targets [1, 2, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17]\n",
      "taking target\n",
      "making output for target layer 2\n",
      "making replacement layers for target layer 2\n",
      "starting fit generator for target layer 2\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,318,824\n",
      "Trainable params: 138,318,824\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,330,600\n",
      "Trainable params: 78,848\n",
      "Non-trainable params: 138,251,752\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0506 19:57:13.408628 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,330,600\n",
      "Trainable params: 78,848\n",
      "Non-trainable params: 138,251,752\n",
      "_________________________________________________________________\n",
      "targets [1, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22]\n",
      "taking target\n",
      "making output for target layer 9\n",
      "making replacement layers for target layer 9\n",
      "starting fit generator for target layer 9\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,244,968\n",
      "Trainable params: 0\n",
      "Non-trainable params: 138,244,968\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,284,328\n",
      "Trainable params: 18,176\n",
      "Non-trainable params: 138,266,152\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:57:19.759829 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,284,328\n",
      "Trainable params: 18,176\n",
      "Non-trainable params: 138,266,152\n",
      "_________________________________________________________________\n",
      "targets [1, 15, 17, 18, 19, 21, 22, 23, 25, 26, 27]\n",
      "taking target\n",
      "making output for target layer 15\n",
      "making replacement layers for target layer 15\n",
      "starting fit generator for target layer 15\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,097,384\n",
      "Trainable params: 0\n",
      "Non-trainable params: 138,097,384\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,173,096\n",
      "Trainable params: 18,176\n",
      "Non-trainable params: 138,154,920\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:57:26.393838 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,173,096\n",
      "Trainable params: 18,176\n",
      "Non-trainable params: 138,154,920\n",
      "_________________________________________________________________\n",
      "targets [1, 22, 23, 24, 26, 27, 28, 30, 31, 32]\n",
      "taking target\n",
      "making output for target layer 22\n",
      "making replacement layers for target layer 22\n",
      "starting fit generator for target layer 22\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,802,216\n",
      "Trainable params: 0\n",
      "Non-trainable params: 137,802,216\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,982,248\n",
      "Trainable params: 69,120\n",
      "Non-trainable params: 137,913,128\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:57:33.128500 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,982,248\n",
      "Trainable params: 69,120\n",
      "Non-trainable params: 137,913,128\n",
      "_________________________________________________________________\n",
      "targets [1, 28, 29, 31, 32, 33, 35, 36, 37]\n",
      "taking target\n",
      "making output for target layer 28\n",
      "making replacement layers for target layer 28\n",
      "starting fit generator for target layer 28\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,212,136\n",
      "Trainable params: 0\n",
      "Non-trainable params: 137,212,136\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,530,408\n",
      "Trainable params: 69,120\n",
      "Non-trainable params: 137,461,288\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:57:40.133135 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,530,408\n",
      "Trainable params: 69,120\n",
      "Non-trainable params: 137,461,288\n",
      "_________________________________________________________________\n",
      "targets [1, 34, 36, 37, 38, 40, 41, 42]\n",
      "taking target\n",
      "making output for target layer 34\n",
      "making replacement layers for target layer 34\n",
      "starting fit generator for target layer 34\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 136,622,056\n",
      "Trainable params: 0\n",
      "Non-trainable params: 136,622,056\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,078,568\n",
      "Trainable params: 69,120\n",
      "Non-trainable params: 137,009,448\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:57:47.328646 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 137,078,568\n",
      "Trainable params: 69,120\n",
      "Non-trainable params: 137,009,448\n",
      "_________________________________________________________________\n",
      "targets [1, 41, 42, 43, 45, 46, 47]\n",
      "taking target\n",
      "making output for target layer 41\n",
      "making replacement layers for target layer 41\n",
      "starting fit generator for target layer 41\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 135,441,896\n",
      "Trainable params: 0\n",
      "Non-trainable params: 135,441,896\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 136,303,656\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 136,034,344\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:57:54.704124 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 136,303,656\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 136,034,344\n",
      "_________________________________________________________________\n",
      "targets [1, 47, 48, 50, 51, 52]\n",
      "taking target\n",
      "making output for target layer 47\n",
      "making replacement layers for target layer 47\n",
      "starting fit generator for target layer 47\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 133,082,088\n",
      "Trainable params: 0\n",
      "Non-trainable params: 133,082,088\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 134,482,472\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 134,213,160\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:58:02.308291 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 134,482,472\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 134,213,160\n",
      "_________________________________________________________________\n",
      "targets [1, 53, 55, 56, 57]\n",
      "taking target\n",
      "making output for target layer 53\n",
      "making replacement layers for target layer 53\n",
      "starting fit generator for target layer 53\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 130,722,280\n",
      "Trainable params: 0\n",
      "Non-trainable params: 130,722,280\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 132,661,288\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 132,391,976\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:58:10.122358 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 132,661,288\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 132,391,976\n",
      "_________________________________________________________________\n",
      "targets [1, 60, 61, 62]\n",
      "taking target\n",
      "making output for target layer 60\n",
      "making replacement layers for target layer 60\n",
      "starting fit generator for target layer 60\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 128,362,472\n",
      "Trainable params: 0\n",
      "Non-trainable params: 128,362,472\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_18 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_18 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_18 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_19 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_19 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_19 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 130,840,104\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 130,570,792\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:58:18.137063 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_18 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_18 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_18 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_19 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_19 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_19 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 130,840,104\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 130,570,792\n",
      "_________________________________________________________________\n",
      "targets [1, 66, 67]\n",
      "taking target\n",
      "making output for target layer 66\n",
      "making replacement layers for target layer 66\n",
      "starting fit generator for target layer 66\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 126,002,664\n",
      "Trainable params: 0\n",
      "Non-trainable params: 126,002,664\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_18 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_18 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_18 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_19 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_19 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_19 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_20 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_20 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_20 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_21 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_21 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_21 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 129,018,920\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 128,749,608\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:58:26.406017 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_18 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_18 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_18 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_19 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_19 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_19 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_20 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_20 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_20 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_21 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_21 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_21 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 129,018,920\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 128,749,608\n",
      "_________________________________________________________________\n",
      "targets [1, 72]\n",
      "taking target\n",
      "making output for target layer 72\n",
      "making replacement layers for target layer 72\n",
      "starting fit generator for target layer 72\n",
      "saving replacement layers to json\n",
      "building top half of model\n",
      "building middle of model with replacement layers\n",
      "building bottom of model\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 123,642,856\n",
      "Trainable params: 0\n",
      "Non-trainable params: 123,642,856\n",
      "_________________________________________________________________\n",
      "combining model\n",
      "refactoring model\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "freezing first half of layers\n",
      "freezing last half of layers\n",
      "fine tuning combined model\n",
      "loading best weights from fine tune\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_18 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_18 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_18 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_19 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_19 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_19 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_20 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_20 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_20 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_21 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_21 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_21 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_22 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_22 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_22 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_23 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_23 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_23 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 127,197,736\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 126,928,424\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0506 19:58:35.000385 140377590093632 hdf5_format.py:221] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new summary\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "sep_conv_0 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_0 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_0 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_1 (SeparableConv2D) (None, 224, 224, 64)      4736      \n",
      "_________________________________________________________________\n",
      "batch_norm_1 (BatchNormaliza (None, 224, 224, 64)      256       \n",
      "_________________________________________________________________\n",
      "relu_1 (ReLU)                (None, 224, 224, 64)      0         \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "sep_conv_2 (SeparableConv2D) (None, 112, 112, 128)     8896      \n",
      "_________________________________________________________________\n",
      "batch_norm_2 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_2 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_3 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_3 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_3 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_4 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_4 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_4 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "sep_conv_5 (SeparableConv2D) (None, 112, 112, 128)     17664     \n",
      "_________________________________________________________________\n",
      "batch_norm_5 (BatchNormaliza (None, 112, 112, 128)     512       \n",
      "_________________________________________________________________\n",
      "relu_5 (ReLU)                (None, 112, 112, 128)     0         \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_6 (SeparableConv2D) (None, 56, 56, 256)       34176     \n",
      "_________________________________________________________________\n",
      "batch_norm_6 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_6 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_7 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_7 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_7 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_8 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_8 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_8 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_9 (SeparableConv2D) (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_9 (BatchNormaliza (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_9 (ReLU)                (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_10 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_10 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_10 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_11 (SeparableConv2D (None, 56, 56, 256)       68096     \n",
      "_________________________________________________________________\n",
      "batch_norm_11 (BatchNormaliz (None, 56, 56, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu_11 (ReLU)               (None, 56, 56, 256)       0         \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_12 (SeparableConv2D (None, 28, 28, 512)       133888    \n",
      "_________________________________________________________________\n",
      "batch_norm_12 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_12 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_13 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_13 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_13 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_14 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_14 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_14 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_15 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_15 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_15 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_16 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_16 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_16 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_17 (SeparableConv2D (None, 28, 28, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_17 (BatchNormaliz (None, 28, 28, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_17 (ReLU)               (None, 28, 28, 512)       0         \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_18 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_18 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_18 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_19 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_19 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_19 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_20 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_20 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_20 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_21 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_21 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_21 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_22 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_22 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_22 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "sep_conv_23 (SeparableConv2D (None, 14, 14, 512)       267264    \n",
      "_________________________________________________________________\n",
      "batch_norm_23 (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu_23 (ReLU)               (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 127,197,736\n",
      "Trainable params: 269,312\n",
      "Non-trainable params: 126,928,424\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "import gc\n",
    "while len(targets) > 1:\n",
    "    \n",
    "    print(f'targets {targets}')\n",
    "    print(\"taking target\")\n",
    "    target = targets[1]\n",
    "    \n",
    "    print(f'making output for target layer {target}')\n",
    "    \n",
    "    get_output = tf.keras.Model(inputs=model.input, \n",
    "                                outputs=[model.layers[target - 1].output,\n",
    "                                         model.layers[target].output])\n",
    "    \n",
    "    print(f'making replacement layers for target layer {target}')\n",
    "    replacement_layers = build_replacement(get_output)\n",
    "    \n",
    "    replacement_len = len(replacement_layers.layers)\n",
    "    \n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=.001)\n",
    "\n",
    "    loss_object = tf.losses.MeanSquaredError()\n",
    "    \n",
    "    replacement_layers.compile(loss=loss_object, optimizer=optimizer)\n",
    "    \n",
    "    save = tf.keras.callbacks.ModelCheckpoint('./replacement_layer.h5', \n",
    "                                              verbose=0, \n",
    "                                              save_weights_only=True,\n",
    "                                              save_best_only=True)\n",
    "\n",
    "    \n",
    "    print(f'starting fit generator for target layer {target}')\n",
    "#     replacement_layers.fit_generator(generator=train_gen, \n",
    "#                                      epochs=50, \n",
    "#                                      validation_data=test_gen ,\n",
    "#                                      verbose=0, callbacks=[save])\n",
    "    \n",
    "    print('saving replacement layers to json')\n",
    "    \n",
    "#     replacement_json = replacement_layers.to_json()\n",
    "#     with open('replacement_layer.json', 'w') as json_file:\n",
    "#         json_file.write(replacement_json)\n",
    "        \n",
    "#     del replacement_layers\n",
    "    \n",
    "#     with open('replacement_layer.json', 'r') as json_file:\n",
    "#         replacement_layers = tf.keras.models.model_from_json(json_file.read())\n",
    "\n",
    "#     print('loading replacement layers weights')\n",
    "#     replacement_layers.load_weights('replacement_layer.h5')\n",
    "#     replacement_layers.compile(loss=loss_object, optimizer=optimizer)\n",
    "#     replacement_layers.evaluate_generator(test_gen)\n",
    "    # build top half of model\n",
    "    print('building top half of model')\n",
    "    get_output = tf.keras.Model(inputs=model.input, outputs=[model.layers[target - 1].output])\n",
    "    # add in replacement layers\n",
    "    print('building middle of model with replacement layers')\n",
    "    new_joint = tf.keras.Model(inputs=get_output.input, outputs=replacement_layers(get_output.output))\n",
    "    \n",
    "    #new_joint.summary()\n",
    "    \n",
    "    # build bottom of model\n",
    "    bottom_half = tf.keras.Sequential()\n",
    "    for layer in model.layers[target + 1::]:\n",
    "        bottom_half.add(layer)\n",
    "        \n",
    "    \n",
    "    \n",
    "    print('building bottom of model')\n",
    "    bottom_half.build(input_shape=new_joint.output.shape)\n",
    "    bottom_half.summary()\n",
    "    print('combining model')\n",
    "    combined = tf.keras.Model(inputs=new_joint.input, outputs=bottom_half(new_joint.output))\n",
    "    \n",
    "    combined.layers[-1].trainable=False\n",
    "    opt = keras.optimizers.RMSprop(lr=0.00005, decay=1e-6)\n",
    "    combined.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "    \n",
    "    #combined.summary()\n",
    "    del bottom_half, new_joint, replacement_layers, model\n",
    "    gc.collect()\n",
    "    \n",
    "#     print('testing combined model')\n",
    "#     scores = combined.evaluate(x_test, y_test, verbose=1)\n",
    "#     print('Test loss:', scores[0])\n",
    "#     print('Test accuracy:', scores[1])\n",
    "    \n",
    "    new_combined = tf.keras.Sequential()\n",
    "    new_layers = []\n",
    "    new_combined.add(tf.keras.layers.Input(shape=(224,224,3)))\n",
    "    accum = 0\n",
    "    print('refactoring model')\n",
    "    for layer in combined.layers:\n",
    "        #print(layer.__class__.__name__)\n",
    "        if hasattr(layer, 'layers'):\n",
    "            \n",
    "            for sublayer in layer.layers:\n",
    "                #print(sublayer.__class__)\n",
    "                if(sublayer.__class__.__name__ != 'InputLayer'): \n",
    "                    new_layers.append((sublayer.__class__.__name__, sublayer.get_config(), accum))\n",
    "\n",
    "                accum += 1\n",
    "        elif layer.__class__.__name__ != 'InputLayer':          \n",
    "            new_layers.append((layer.__class__.__name__, layer.get_config(), accum))\n",
    "                \n",
    "            accum += 1 \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    for i, layer in enumerate(new_layers):\n",
    "        new_combined.add(keras.layers.deserialize(\n",
    "                                {'class_name': layer[0], \n",
    "                                 'config': layer[1]}))\n",
    "\n",
    "    new_combined.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "   \n",
    "\n",
    "    accum = 0\n",
    "    for i, layer in enumerate(combined.layers):\n",
    "        if hasattr(layer, 'layers'):\n",
    "\n",
    "            for sublayer in layer.layers:\n",
    "                #print(f'{accum} sub is {sublayer} new is {new_combined.layers[accum]}')\n",
    "                if(sublayer.__class__.__name__ != 'InputLayer'): \n",
    "                    #print(sublayer.__class__)\n",
    "                    new_combined.layers[accum].set_weights(sublayer.get_weights())\n",
    "                    accum += 1\n",
    "    #             else:\n",
    "    #                 accum += 1\n",
    "            continue\n",
    "        else:\n",
    "            #print(layer)\n",
    "            if(layer.__class__.__name__ != 'InputLayer'):\n",
    "                print()\n",
    "                new_combined.layers[accum].set_weights(layer.get_weights())\n",
    "                accum +=1 \n",
    "            elif(layer.__class__.__name__ == 'Flatten'):\n",
    "                accum += 1\n",
    "\n",
    "    print('freezing first half of layers')\n",
    "    for i in range(target):\n",
    "        new_combined.layers[i].trainable = False\n",
    "    \n",
    "    print('freezing last half of layers')\n",
    "    for i in range(target +  replacement_len, len(new_combined.layers)):\n",
    "        new_combined.layers[i].trainable = False\n",
    "        \n",
    "    new_combined.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=opt,\n",
    "                  metrics=['accuracy'])\n",
    "    del combined\n",
    "    gc.collect()\n",
    "\n",
    "    new_save=tf.keras.callbacks.ModelCheckpoint('./refactor_finetune.h5', \n",
    "                                                verbose=1, \n",
    "                                                save_weights_only=False, \n",
    "                                                save_best_only=True)\n",
    "    print('fine tuning combined model')\n",
    "    #new_combined.fit(x=x_train, y=y_train, validation_data=(x_test, y_test), epochs=5, callbacks=[new_save])\n",
    "#     new_combined.fit_generator(datagen.flow(x_train, y_train,\n",
    "#                                      batch_size=batch_size),\n",
    "#                         epochs=5,\n",
    "#                         validation_data=(x_test, y_test),\n",
    "#                         #workers=5,\n",
    "#                         callbacks=[new_save])\n",
    "    \n",
    "    print('loading best weights from fine tune')\n",
    "    #new_combined.load_weights('./refactor_finetune.h5')\n",
    "    #new_combined.save('.refactor_finetune.h5')\n",
    "\n",
    "    model = tf.keras.Model(inputs=new_combined.input, \n",
    "                                outputs=new_combined.output)\n",
    "    model.summary()\n",
    "    del new_combined\n",
    "    gc.collect()\n",
    "    model.save('./refactor_finetune.h5')\n",
    "    tf.keras.backend.clear_session()\n",
    "    model = tf.keras.models.load_model('./refactor_finetune.h5')\n",
    "    print('new summary')\n",
    "    model.summary()\n",
    "    targets = [i for i, layer in enumerate(model.layers) if layer.__class__.__name__ == 'Conv2D']\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}