part5-CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 卷积神经网络Convolution Neural Network\n",
    "** 简单介绍 **<br>\n",
    "* CNN是一个模仿人类视觉原理的一个神经网络。\n",
    "* 与全连接网络相比，CNN可以更大效率的降低参数的数量。\n",
    "* 与全连接网络相比，CNN可以更好的提取图像的特征。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# 导入数据\n",
    "import numpy as np\n",
    "from keras.utils import to_categorical\n",
    "from keras.datasets import mnist\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(train_data, train_labels), (test_data, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 数据预处理\n",
    "train_data = train_data.reshape(60000, 28, 28, 1)\n",
    "train_data = train_data.astype(\"float32\") / 255.\n",
    "test_data = test_data.reshape(10000, 28, 28, 1)\n",
    "x_test = test_data.astype(\"float32\") / 255.\n",
    "\n",
    "train_labels = to_categorical(train_labels)\n",
    "y_test = to_categorical(test_labels)\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(train_data, train_labels, random_state=42, test_size=0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense\n",
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "# 第一层使用32个3*3的卷积核\n",
    "# 要注意CNN的input层要注明通道数的\n",
    "model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
    "# 使用2*2的Maxpooling池化层\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))\n",
    "# Flatten层指的是将上面的卷积层的输出展平，以便后面可以作为全连接层的输入\n",
    "model.add(Flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 11, 11, 64)        18496     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 7744)              0         \n",
      "=================================================================\n",
      "Total params: 18,816\n",
      "Trainable params: 18,816\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 先查看以上部分的网络的示意图\n",
    "# 可得 7744=11*11*64是正确的，并且总共参数数量为18816\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 继续添加全连接层\n",
    "model.add(Dense(units=64, activation='relu'))\n",
    "model.add(Dense(units=10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 11, 11, 64)        18496     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 7744)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                495680    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 515,146\n",
      "Trainable params: 515,146\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 很明显，我们只是添加了两层全连接层，一个64节点，一个10节点\n",
    "# 然后参数已经到达了515,146的数量，远远大于卷积层的参数数量\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 45000 samples, validate on 15000 samples\n",
      "Epoch 1/20\n",
      "45000/45000 [==============================] - 8s 168us/step - loss: 0.2129 - acc: 0.9336 - val_loss: 0.0730 - val_acc: 0.9779\n",
      "Epoch 2/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0575 - acc: 0.9820 - val_loss: 0.0650 - val_acc: 0.9809\n",
      "Epoch 3/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0367 - acc: 0.9886 - val_loss: 0.0565 - val_acc: 0.9843\n",
      "Epoch 4/20\n",
      "45000/45000 [==============================] - 5s 117us/step - loss: 0.0259 - acc: 0.9920 - val_loss: 0.0560 - val_acc: 0.9863\n",
      "Epoch 5/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0182 - acc: 0.9944 - val_loss: 0.0514 - val_acc: 0.9866\n",
      "Epoch 6/20\n",
      "45000/45000 [==============================] - 6s 130us/step - loss: 0.0134 - acc: 0.9960 - val_loss: 0.0450 - val_acc: 0.9885\n",
      "Epoch 7/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0097 - acc: 0.9968 - val_loss: 0.0562 - val_acc: 0.9875\n",
      "Epoch 8/20\n",
      "45000/45000 [==============================] - 5s 122us/step - loss: 0.0079 - acc: 0.9976 - val_loss: 0.0550 - val_acc: 0.9875\n",
      "Epoch 9/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0062 - acc: 0.9982 - val_loss: 0.0568 - val_acc: 0.9887\n",
      "Epoch 10/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0045 - acc: 0.9986 - val_loss: 0.0578 - val_acc: 0.9886\n",
      "Epoch 11/20\n",
      "45000/45000 [==============================] - 5s 117us/step - loss: 0.0040 - acc: 0.9987 - val_loss: 0.0632 - val_acc: 0.9897\n",
      "Epoch 12/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0029 - acc: 0.9992 - val_loss: 0.0718 - val_acc: 0.9887\n",
      "Epoch 13/20\n",
      "45000/45000 [==============================] - 5s 118us/step - loss: 0.0021 - acc: 0.9994 - val_loss: 0.0888 - val_acc: 0.9881\n",
      "Epoch 14/20\n",
      "45000/45000 [==============================] - 5s 118us/step - loss: 0.0024 - acc: 0.9994 - val_loss: 0.0839 - val_acc: 0.9874\n",
      "Epoch 15/20\n",
      "45000/45000 [==============================] - 5s 118us/step - loss: 0.0018 - acc: 0.9995 - val_loss: 0.0854 - val_acc: 0.9881\n",
      "Epoch 16/20\n",
      "45000/45000 [==============================] - 5s 119us/step - loss: 0.0014 - acc: 0.9996 - val_loss: 0.0825 - val_acc: 0.9888\n",
      "Epoch 17/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 0.0018 - acc: 0.9994 - val_loss: 0.0887 - val_acc: 0.9899\n",
      "Epoch 18/20\n",
      "45000/45000 [==============================] - 5s 119us/step - loss: 0.0017 - acc: 0.9995 - val_loss: 0.0920 - val_acc: 0.9886\n",
      "Epoch 19/20\n",
      "45000/45000 [==============================] - 5s 117us/step - loss: 0.0013 - acc: 0.9996 - val_loss: 0.0893 - val_acc: 0.9881\n",
      "Epoch 20/20\n",
      "45000/45000 [==============================] - 5s 116us/step - loss: 8.0244e-04 - acc: 0.9999 - val_loss: 0.0968 - val_acc: 0.9878\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x13b2a50de80>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model.fit(x=x_train, y=y_train, validation_data=(x_val, y_val), batch_size=128, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 146us/step\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.989\n"
     ]
    }
   ],
   "source": [
    "# 经过了20个epoch，模型很快的达到了98.9%的准确率，如果更多点epoch或更深点的模型，达到100%也并非不可能\n",
    "print(test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 卷积网络是如何提取图片特征的？\n",
    "* CNN通过局部视野，提取局部特征<br>\n",
    "![title](images/extract_features.JPG)\n",
    "* CNN据具有平移不变性，是全连接网络不具备的，即当一个物体的位置或形状发送了变化，DNN可能就无法准确的检测出来。\n",
    "* CNN可以学习到空间层次的特征，可能在较浅的卷积层，学习到的是一些边缘线条，再之后更深的卷积层，学习到更多抽象的特征。\n",
    "![title](images/spatial_hierarchies.JPG)\n",
    "* 卷积是如何计算的，这个其实不难，可以看一下Andrew老师的CNN章节即可看懂。<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 我们来查看模型中间的卷积层学习的特征\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 获取最后一个卷积层\n",
    "conv2d_2_model = Model(inputs=model.input, outputs=model.get_layer(\"conv2d_2\").output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 28, 1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADeNJREFUeJzt3X+MHPV5x/HP4+vZRwwE25CTSyyME5OGH8VRtzYQlEKd\nEECJDPnDiqtGbkI4p3JQUPkjLq1Uqv4QSsuPFKVQp3ZiEAGiEISVECrspHVoqcNBjG1wwNQ1xdbh\nA9mqbVzOZ+7pHzeODnPz3WV3dmfunvdLWt3uPDM7j1b+eHbnuztfc3cBiGdK2Q0AKAfhB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgCD8Q1G90cmdTbZr3aHondwmE8pbe1FEfskbWbSn8ZnalpG9K6pL0\nz+5+a2r9Hk3XIlvcyi4BJGz2jQ2v2/TbfjPrkvQtSVdJOlfSMjM7t9nnA9BZrXzmXyjpZXff5e5H\nJT0oaUkxbQFot1bCf6akV8c83pMtewcz6zOzfjPrH9ZQC7sDUKS2n+1399XuXnP3WremtXt3ABrU\nSvj3Spoz5vEHs2UAJoBWwv+0pPlmdraZTZX0eUnri2kLQLs1PdTn7sfM7KuS/kWjQ31r3f35wjoD\n0FYtjfO7+2OSHiuoFwAdxNd7gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiqo1N0o3q6Tnt/sv6rv/xosn75ou3J\n+pMbLsitzfvrXya3HXnrrWQdreHIDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBtTTOb2a7JR2S9Lak\nY+5eK6IpFKdrxoxk/X8fSNdfvOBbyfoUWbI+8sV/y6198fcXJ7d9Y+WHknX/JTPCt6KIL/lc7u5v\nFPA8ADqIt/1AUK2G3yVtMLNnzKyviIYAdEarb/svdfe9ZvYBSU+Y2a/cfdPYFbL/FPokqUfva3F3\nAIrS0pHf3fdmfwclPSJp4TjrrHb3mrvXujWtld0BKFDT4Tez6WZ2yvH7kq6QlP6JF4DKaOVtf6+k\nR8zs+PN8z90fL6QrAG3XdPjdfZekCwvsBW2w42/nJ+svXXB3W/d/54FzcmvfOWtjctuHHzw9WV+3\n9MpkfeS5Hcl6dAz1AUERfiAowg8ERfiBoAg/EBThB4Li0t2TwOGlF+XWtn32zjpbdyery3Z9Oll/\n4fH8oTxJOuu2Z3Nr961MP/czf3JXsv6NS05L1s94LlkOjyM/EBThB4Ii/EBQhB8IivADQRF+ICjC\nDwTFOP8kMPg7+ZfPnmbpcfx63vzj9M9q52z/j2R9JFGbfVt624Gv/V+yPvTJg8m62vtr5QmPIz8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBMU4/yTw5as3NL1t36uXpVfY9T9NP3er1hxYlKz/qPZPyfqK\ni1fm1uwpfuzPkR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqo7zm9mayV9RtKgu5+fLZsp6SFJcyXt\nlrTU3Q+0r83YXv/Kxcn6jTP/Ibc27J7c9sl/PT9Zn3fkqWS9ne77+aXJ+p9fuzVZH5o1LbfW01RH\nk0sjR/7vSjpxIvRVkja6+3xJG7PHACaQuuF3902S9p+weImkddn9dZKuKbgvAG3W7Gf+XncfyO6/\nJqm3oH4AdEjLJ/zc3SXlfrA0sz4z6zez/mENtbo7AAVpNvz7zGy2JGV/B/NWdPfV7l5z91q38k/A\nAOisZsO/XtLy7P5ySY8W0w6ATqkbfjN7QNJTkj5iZnvM7DpJt0r6lJntlPTJ7DGACaTuOL+7L8sp\nLS64F+T4xPVPJ+tTEv+H33ngt5LbzltV3jh+ux28Pv+6/j0/6mAjFcU3/ICgCD8QFOEHgiL8QFCE\nHwiK8ANBcenuKrjot5PlFbPuqfME+d+c/MnAeXW23F3nucsz/ZWusluY1DjyA0ERfiAowg8ERfiB\noAg/EBThB4Ii/EBQjPNXwJ7LT07WP9ydvgLSET+aW7Pbz6iz99116uU55dWRsluY1DjyA0ERfiAo\nwg8ERfiBoAg/EBThB4Ii/EBQjPNXQM8lbyTrU2TJev9Q/vcEpj6evux3lS24aUuyXu91OXT4pNza\nB5rqaHLhyA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQdUd5zeztZI+I2nQ3c/Plt0i6XpJr2er3ezu\nj7WryYmuqzc9qvw3H300WR+RJ+vX//jLubX52pzctspW9W5I1keUP44vSXPu5br/KY0c+b8r6cpx\nlt/h7guyG8EHJpi64Xf3TZL2d6AXAB3Uymf+G8xsq5mtNbMZhXUEoCOaDf/dkuZJWiBpQNJteSua\nWZ+Z9ZtZ/7CGmtwdgKI1FX533+fub7v7iKRvS1qYWHe1u9fcvdadmFASQGc1FX4zmz3m4bWSthfT\nDoBOaWSo7wFJl0k63cz2SPoLSZeZ2QJJrtFrP69oY48A2qBu+N192TiL17Shl0nLTupJ1hefdKRD\nnVSL/e4Fyfr77N+T9V8MpX/P37P3cG6NGQH4hh8QFuEHgiL8QFCEHwiK8ANBEX4gKC7djdK8ctUp\nyfr7p6SHSP9wQ/rrJedsn7iXLe8EjvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBTj/Ggrv+TC3NoP\nvpR79TdJ0pH0Fcv1mxu4NHcrOPIDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM86MlRz63KFlf+leP\n59bO6Z6a3Pa8TV9K1s/+/n8m60jjyA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQdUd5zezOZLuldQr\nySWtdvdvmtlMSQ9Jmitpt6Sl7n6gfa2iDF2nnpqsX/inW5L1r5y2K7d2xI8mt/3wiv9O1t9OVlFP\nI0f+Y5JucvdzJV0kaaWZnStplaSN7j5f0sbsMYAJom743X3A3Z/N7h+StEPSmZKWSFqXrbZO0jXt\nahJA8d7TZ34zmyvpY5I2S+p194Gs9JpGPxYAmCAaDr+ZnSzpYUk3uvvBsTV3d42eDxhvuz4z6zez\n/mENtdQsgOI0FH4z69Zo8O939x9mi/eZ2eysPlvS4Hjbuvtqd6+5e61b04roGUAB6obfzEzSGkk7\n3P32MaX1kpZn95dLerT49gC0SyM/6f24pC9I2mZmx8d1bpZ0q6Tvm9l1kl6RtLQ9LU4Cw8eS5ZeG\n00Ne9X76uvz3fp5bu//BWnLben5y8T8m6x/qPjlZv+vAvNzaD77+6eS2PQd/kayjNXXD7+5PSrKc\n8uJi2wHQKXzDDwiK8ANBEX4gKMIPBEX4gaAIPxCUjX4ztzNOtZm+yBgdPNHOu9KXv37xc+mx9jLd\neeCcZP2hO67Irc1a81TR7YS32TfqoO/PG5p/B478QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAUU3RX\nwEdWbU/Wz3vzq8n6T//g73JrvV0nNdXTcRfec0OyPveencn6rNcZy68qjvxAUIQfCIrwA0ERfiAo\nwg8ERfiBoAg/EBS/5wcmEX7PD6Auwg8ERfiBoAg/EBThB4Ii/EBQhB8Iqm74zWyOmf3MzF4ws+fN\n7GvZ8lvMbK+ZbcluV7e/XQBFaeRiHsck3eTuz5rZKZKeMbMnstod7v737WsPQLvUDb+7D0gayO4f\nMrMdks5sd2MA2us9feY3s7mSPiZpc7boBjPbamZrzWxGzjZ9ZtZvZv3DGmqpWQDFaTj8ZnaypIcl\n3ejuByXdLWmepAUafWdw23jbuftqd6+5e61b0wpoGUARGgq/mXVrNPj3u/sPJcnd97n72+4+Iunb\nkha2r00ARWvkbL9JWiNph7vfPmb57DGrXSspfQlaAJXSyNn+j0v6gqRtZrYlW3azpGVmtkCSS9ot\naUVbOgTQFo2c7X9S0ni/D36s+HYAdArf8AOCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwTV0Sm6zex1Sa+MWXS6pDc61sB7U9XeqtqXRG/NKrK3s9z9jEZW7Gj4\n37Vzs353r5XWQEJVe6tqXxK9Naus3njbDwRF+IGgyg7/6pL3n1LV3qral0RvzSqlt1I/8wMoT9lH\nfgAlKSX8Znalmb1oZi+b2aoyeshjZrvNbFs283B/yb2sNbNBM9s+ZtlMM3vCzHZmf8edJq2k3iox\nc3NiZulSX7uqzXjd8bf9ZtYl6SVJn5K0R9LTkpa5+wsdbSSHme2WVHP30seEzewTkg5Lutfdz8+W\nfUPSfne/NfuPc4a7f70ivd0i6XDZMzdnE8rMHjuztKRrJP2RSnztEn0tVQmvWxlH/oWSXnb3Xe5+\nVNKDkpaU0EflufsmSftPWLxE0rrs/jqN/uPpuJzeKsHdB9z92ez+IUnHZ5Yu9bVL9FWKMsJ/pqRX\nxzzeo2pN+e2SNpjZM2bWV3Yz4+jNpk2XpNck9ZbZzDjqztzcSSfMLF2Z166ZGa+Lxgm/d7vU3RdI\nukrSyuztbSX56Ge2Kg3XNDRzc6eMM7P0r5X52jU743XRygj/Xklzxjz+YLasEtx9b/Z3UNIjqt7s\nw/uOT5Ka/R0suZ9fq9LMzePNLK0KvHZVmvG6jPA/LWm+mZ1tZlMlfV7S+hL6eBczm56diJGZTZd0\nhao3+/B6Scuz+8slPVpiL+9QlZmb82aWVsmvXeVmvHb3jt8kXa3RM/7/JenPyughp695kp7Lbs+X\n3ZukBzT6NnBYo+dGrpM0S9JGSTslbZA0s0K93Sdpm6StGg3a7JJ6u1Sjb+m3StqS3a4u+7VL9FXK\n68Y3/ICgOOEHBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCo/wfFhyrTAkApwAAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13bc026fb38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 找一个样本\n",
    "plt.imshow(x_train[0].reshape(28, 28))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_features = conv2d_2_model.predict(x_train[0].reshape(1, 28, 28, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 11, 11, 64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看特征的shape, 是一个width=11, height=11, channel=64的feature map\n",
    "pred_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAADTCAYAAACRDeixAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF4RJREFUeJzt3XlwnPV5B/DvszpWvmRLxhgfsvEhDkMaCoohIQcJQyE0\nLUnT1JCmddMQt83QNDSZQCdpydFOOs00k3RCM+N2GJNpCCHksAkOJjgkToMBCwypDRgrvm35ki1L\ntnXuPv1DK7Or57fW7r7Han/7/cxorH1m32O/2n142d/7/l5RVRARUeVLlHsHiIgoHGzoRESeYEMn\nIvIEGzoRkSfY0ImIPMGGTkTkCTZ0IiJPsKETEXkiUEMXkVtEZIeIdIjIvWHtVCVjJm7MxWImFjMJ\nprbUBUWkBsD9AG4CcADAFhFZp6qv5FumXpLagCmlbhIQW0pPn2xqU+acdS4+vabP1PaduMDU6g+d\nKX7fACgUggQU6RMA5iCOTFym2UxmLuh2PvXQqSZTSx6wr19qaxxLO/4gAHR4OPcxFADSAFpRwHsl\nkkxiIjXuYyRNpXMfF5kJEE0u6vj8pOrdf1dxXFRec7y0z4pzX3BuA0tQxkwkWW/3bWAw1G0Uqxcn\nj6vqrPGeV3JDB7AcQIeq7gIAEXkYwG0A8obfgCm4Vm4seYNSa3f37A1Xm9rV//Sic/lbp79sap98\n6E5TW/iF5+3C6dS4+9etXdiFV3ACR3ar6mAcmbik3mIz+fNvrXM+94vr/9jUlv79s6ZW0zTTLlzj\navJA6sjRnMfd2oV2PH260PdKFJnEpWZqo7Oe6unJeVxsJkAIuSTs36v/ndeYWs8Cd1uo7bcdvfmB\nzaXvzxjd2oWt+BWGdSi+TBxqFiwytVTH7lC3Uayn9NG9hTwvyFcu8wDsz3p8IFOrWgPoQwMmZZeq\nPhNgJBcA2Yc4VZ8LM7EG0IdEbkuq+kyKFeQIvSAisgrAKgBogP3fu2rETCxm4sZcLGaSX5Aj9IMA\nWrIez8/UcqjqalVtU9W2OiQDbG7iS2IS+pHzPX3VZwKM5AIg+4tJkwsz4XsliUlII2esoeozKVaQ\nI/QtAFpFZBFGQr8dwIdD2as8xg62AcCktfb77sduuM65/NdXtJvaP654xNT++9cfMLX6DXbZsRrR\nhD6cBoB6EalHDJm4LPzK66b2Px/9fedzl26235e7pI53lbw/jWgCgIY43yvlkjqdZ5BQcgcaG9EM\naLyZ1Fy2xNQafmI/Pw15lj/8qbeZ2oZDL5narVe829RSJ0+Ou3+NaEIaaZT7fbJnxRxTa/mX8n6H\nXqiSj9BVdRjAXQA2AHgVwCOquj2sHatECUngUlwFAJeAmZyTkAQA7APfK+cwEyshidGvUJhJiQJ9\nh66q6wGsD2lfvHCBzAEU21S1rdz7MsGcYiYGMxmjFnVQ1UvKvR+VileKEhF5gg2diMgTbOhERJ6I\n/Dz0clh6t/vMjZvvvqqg5esx/hktE0XtfHvdRecf2MuU5Zi9SjYqknScStYf2+YjIW95k6klzjpy\n7nGf5ZI+4TjL43Tg3SqK1gY7frvo68+Y2uXTP2Fqw/9gryid/Xza1ABgyqPPBdqnoPZ/3p65MzzV\nMcdBheAROhGRJ9jQiYg8wYZOROQJNnQiIk94OSgaJ6mzcycjxqmTD3xwoald9A07eBWr1PhTDVca\n3fJ/plbMq3QOFEcoMW2aqWmyrqBlaxe2OOvDe/eb2oIv2vdazaVLTa3zpgud6yz3rPfLbrbTZPS+\n43igdc78tb3HQNf14099EAYeoRMReYINnYjIE2zoRESeYEMnIvIEB0UDkgbHYFeMg6JlHwB1cM1b\nX+0S06baYoRXz0qLndM73b6toGUHFrvvRZwcHDK14c7Ddjt77ODpwHT3oGic5JorTO3RJd8xtZtR\n2BXl+Xx27hOmdg+uDbTOQvEInYjIE2zoRESeYEMnIvIEGzoRkSc4KBqQDsY4AkqhSkxxX6eYPpPn\nRs8BxH2l6MGbLjC1i16xV0W6aEKc9cGldqA14RgU1YEBU6vvybOxRE3u4wgvMt79R42mdtdB12Cl\n3f9i9Gi8f+tsPEInIvIEGzoRkSfY0ImIPMGGTkTkCTZ0IiJP8CyXoCp87u9EQ4Oppfsr/I7OBYri\nbBbXPORAGaZDKPBQreaKS02td5Z73vQZG39raoW++6d25nlmOr7Pz8XX2SkJHt/yZlO7BM8XvM7B\nm9tM7T8O2jOMgGBzrBeKR+hERJ5gQyci8gQbOhGRJ9jQiYg8wUHRgLTCB0V/uutZU7t5brD5oKuZ\n9rsvG0/39sa6H7NeKmxge88HZpralEPqfG7q2LGS92fq2q3OuntL0Zg/pdvUDu9cEGidWmOnSQh6\nk+kgeIROROQJNnQiIk+woRMReYINnYjIExwUDUrjHNYp3bqDW5z1j+57j6Ma7wBexRo7lzeAxMJ5\nzqemOnZHvTc5krvtwNzw8jeZWt/Fdj7/ln9uD31/dKj89w34+cuXm9q8fcFOakiud3+uyoVH6ERE\nnmBDJyLyBBs6EZEn2NCJiDzh5aDo8I3XOOu1G1+IeU9KU7uwxVkf3mun/yxU29f/zlmf+9VnSl6n\nr2rnzTW1dJO9wbAc6DS1uAc/8xnes88W99jSnEXXRb4vE8Xi76VNrWGn/RvGPNFxqHiETkTkCTZ0\nIiJPsKETEXmCDZ2IyBPjNnQReUBEjorItqxas4j8TER2Zv5tinY3J57t2o5f6mPYrE+eqw3pIF7U\nTQBwZTXmwkwsZmKdL5PTOIVqzCQshZzlsgbANwF8O6t2L4CNqvqvInJv5vE94e9eaeI4m2UuFqIF\nS7Adb1z6uwevoRkX4gSObgOwESXmEuRslnziOJslykwgdt5pABh+z9Wm1vnWpKmtumO9qX2qaY9z\nnYt/eK2pXfaFDlNLdZ9yLp8t0kxCMO17dj78qJ0vkyEMoRcnI8mk9ue2L1TyGS0u4x6hq+omACfG\nlG8D8GDm9wcBvD/k/ZrwmmQW6lCfUzuGQ5iDhaMPqy4XZmIxE4uZRKfU79Bnq+roCZyHAcwOaX8q\n2iAGkJRJow+ZC5iJCzOxmEk4Ag+KqqriPHeSEpFVItIuIu1DcN+ey0fny4WZWMzErRpzYSalK7Wh\nHxGROQCQ+fdovieq6mpVbVPVtjrY7zZ9Uo8kBrQPwPlzYSYWM3GrllyYSThEC5jPW0QuBvATVb0y\n8/irALqyBkWbVfWzBaznGIC9mYcXACjf3VTDUQ+gFcB2jLyeBoyMsyQB/DsKyIWZWJ5ngszvvSgi\nEyAnF2aS4eF7JVv261moqrPGXUJVz/sD4LsAOgEMATgA4GMAZmJkdH4ngKcwEv646xqz3vZil5lI\nP45c9gTNhZlURSYfA7CVmYSbiQ+5hPF6CjpCj4KItKtqW1k2HoEwXg8ziWYdE03Q18RMolvHRFLK\n6+GVokREnihnQ19dxm1HIYzXw0yiWcdEE/Q1MZPo1jGRFP16yvaVCxERhYtfuRAReSL2hi4it4jI\nDhHpyJzyWHHCnrCMmeRdZ0Xnwkzc+Pmxwsok1oYuIjUA7gfwXgDLANwhIsvi3IeQrAFwy5ja6IRl\nrXhjwqVxMRM3T3JZA2bisgb8/Iy1BiFkEvcR+nIAHaq6S1UHATyMkYm+KoqGO2EZM3Gr+FyYiRs/\nP1ZYmcTd0OcByJ4b9kCm5oNSJyxjJm6+5sJM3Pj5sYrOhIOiEdCRU4d4+lAWZmIxEzfmYhWaSdwN\n/SCAlqzH8zM1HxQ8YdkYzMTN11yYiRs/P1bRmcTd0LcAaBWRRSJSD+B2AOti3oeorAOwMvP7SgBr\nC1yOmbj5mgszcePnxyo+kzJMOHMrgNcB/BbA58o9AU6JryHUCcuYiZ+5MJN4cmEmb/zwSlEiIk9w\nUJSIyBNs6EREnmBDJyLyBBs6EZEn2NCJiDzBhk5E5Ak2dCIiT7ChExF5gg2diMgTbOhERJ5gQyci\n8gQbOhGRJ9jQiYg8wYZOROQJNnQiIk+woRMReYINnYjIE2zoRESeYEMnIvIEGzoRkSfY0ImIPMGG\nTkTkCTZ0IiJPsKETEXmCDZ2IyBNs6EREnmBDJyLyBBs6EZEn2NCJiDzBhk5E5Ak2dCIiT7ChExF5\ngg2diMgTbOhERJ5gQyci8gQbOhGRJ9jQiYg8wYZOROQJNnQiIk+woRMReYINnYjIE2zoRESeYEMn\nIvIEGzoRkScCNXQRuUVEdohIh4jcG9ZOVTJm4sZcLGZiMZNgaktdUERqANwP4CYABwBsEZF1qvpK\nvmXqJakNmFLqJiENSVNLJ2vs806dLXkbQSgUggQU6RMA5qBMmWj/QMnrA4D0jMmmlmoQU6vrSblX\ncLY/d3+gAJAG0IoC3itBM3GRGvs+GZ7R4HxuTdeZULftUmwmQAi5THa83jF/q3LKZAIASxBXJhWi\nFyePq+qs8Z5XckMHsBxAh6ruAgAReRjAbQDyht+AKbhWbix5gzVLLzW1sxc3mlry8S0lbyOIbu3C\nLryCEziyW1UHy5VJavuOktcHAH03LDe1rmX2rdLyxCnn8rp1e87jbu1CO54+Xeh7JWgmLjWN002t\n633LnM+d8e3NoW7bpdhMgOC5yLIrTE1fdGxO1dZi0K1d2IpfYViHYsukUjylj+4t5HlBvnKZB2B/\n1uMDmVrVGkAfGjApu1T1mQAjuQAYzCpVfS7MxBpAHxK5LanqMylWkCP0gojIKgCrAKAB9n/lqxEz\nsZiJG3OxmEl+QY7QDwJoyXo8P1PLoaqrVbVNVdvqYL/v9UkSk9A/cuQ1quozAUZyAVCfVTK5MBO+\nV5KYhDTS2aWqz6RYQY7QtwBoFZFFGAn9dgAfDmWvAEjS8Yc6esKUkgG/Lw5TI5rQh9MAUC8i9Qg5\nk5pGO17gHEPYbkpFmbT2eVO748t28Ozb09/tXH7R1tzHjWgCgIao3isFmW3Hk6bvdA+en/ngtaY2\n5QfPhbo7UWfi+vwkHCcLpMr0fblLI5qQRhplfZ8AuO7lIVP7zlPvcD53yaefjXp3ilLyEbqqDgO4\nC8AGAK8CeERVA7aSypaQBC7FVQBwCZjJOQlJAMA+8L1yDjOxEpIY/QqFmZQo0HfoqroewPqQ9sUL\nF8gcQLFNVdvKvS8TzClmYjCTMWpRB1W9pNz7Ual4pSgRkSfY0ImIPMGGTkTkicjPQy9VYoa9si91\n5GgEG7KXhCOd55L2Mjv97stMzXVGShSeP3mxqd16k/uK3B1fcpwbHP3V9OeV2tFhanYygxGuC8k3\nHHrJ1G698UN2O6/uLHLPoiGXLzG11Et5L7gM1Ym/fKupNT8Q/dW3pej6uN3XLSfsRZlBz2Y5udJu\np+nB8DPhEToRkSfY0ImIPMGGTkTkCTZ0IiJPlH9QVPIMTQ0Px7P9CToA6tJ5vR3AXby2sGX3fskO\nygBAfbfNf87XnjG1gXcdNrUP7XrRuc77lt9pi0+Ps4MT3NJf/IWpzb7SXl4/9dUYdqYQ6fT4z8lj\n10NXOeutf7vP1FJddjqOHjsei+aS9yZa7V/8lqndcOfHTS2JQ4G207ytx9SimHSBR+hERJ5gQyci\n8gQbOhGRJ9jQiYg8UfZB0YRr3nO4B1sCcw3ATqD5oEfVtsx31utP5ru2cXwDc+0czwDw1x98ytR+\n+rUZBa3z+gb38UDvAv9uOtC8YZKpdb3JPm/q92PYmSyJKe4bJKd/81rJ63znYntVLQB0NjnuUez4\nnC76Ua+pDdzsnlSyfkN7cTsXQPrtdrD3ibN2FHv/HfaEjKUVMqcsj9CJiDzBhk5E5Ak2dCIiT7Ch\nExF5ouyDoul+e/PhqNQusIONw3v3x7b9Qp283j0oOv8r9gpOF9eUoIke9+Dvk0eXOaqFXRX31ROO\nSwIBNO6O728alxmOG0pPPl5fhj3JJTWO6Z+LsPsr9r1yYJ178H3h8cJu75lqsG3l0DvqnM9d/L+5\ng7pyNrpjzHTSZvVvn/gzU1v6ZPgDta+vnGpqrS+EvhkeoRMR+YINnYjIE2zoRESeYEMnIvIEGzoR\nkSfKfpZLFPJdDp06dCSebZ0Ots7jb3afZTDt4cKWn/uR3abW03mh87l6t+syf3uWy6HPvM3U/nOT\ne876y17Ydv4djFjiSnsz7fS20i+FB4CBZjudQcNP4rlB9/mkeuw828W46Go7z33v2jnubXWfKmid\nHR+xZ/9M3ud+T6f7cs+I0gDzuI+ndmP4p5UM33hNQdtp/eRzoW/bhUfoRESeYEMnIvIEGzoRkSfY\n0ImIPOHloGjqd5Y667L55dC3lT5zJvR1Ds12z11eqM8veMzUVrz2Cedz0y/9pqB1nrlywNQWPZRn\noOusvUw+TseuazK1mQHHaae27zW1mG5jHqkVLfYy9+/vvSXQOnf/4WpTu/aev3E/uYJu0u6y/077\nLli0sQw7ksEjdCIiT7ChExF5gg2diMgTbOhERJ7wclA06OBnYto0u875Fzmfm3p1Z6BtObffHezP\n8ulP32Vql/ww2JVqcx+z+1T3s/JfKelyZr4drJ0ZcJ3Dhwu8yjiRZ37yCTr4t2aXnQ+9+fEtBS/f\nu+I6U9vU/5Jd59aTzuWjuy60dD0ftq+p8aFnnc9tvdfeJLucg+U8Qici8gQbOhGRJ9jQiYg8wYZO\nROQJLwdFi3FypR0UmvG6vdIxiqtM82nsCPbf2ckBB0Bdpvwgnuk/w/CZFT80tUee+D37xGcLu0q2\nGJJwXz2rE3H0D0Dz+14PtPyf3ve4qd31TXtV8ryOFwNtJ079TfbzN+S48ToAzPyvzVHvTlF4hE5E\n5Ak2dCIiT7ChExF5YtyGLiIPiMhREdmWVWsWkZ+JyM7Mv3Z6O89t13b8Uh/DZn3yXG1IB/GibgKA\nK6sxF2ZibR9+Dr8Y+hEzybJtcDOe7vs+ft3/xqygo5mcxilUYyZhKeQIfQ2AsfNp3gtgo6q2AtiY\neVxV5mIhfhdvz6ntwWtoxoUAsA1VmAszseYmFuHq2nfl1Ko+k5rFuCb5npzaaCZTMR2owkzCMu5Z\nLqq6SUQuHlO+DcANmd8fBPALAPeEuF8FO/5XdvT5hfu+5Xzu/d0tpvbQffb0A3lu/Mmzm2QW+jR3\nLvRjOIRr8C50YBsQIJcL23uLXWRCiDKTYjxyuZ2mQersTaI14HZqGhtNbexNm2egeUJkEobDd9sb\nhX9jnU2x9cf2JuPD/W/cDHoGpmcyUSBzUtBoJkdwEChzJhfe/0w5NhuKUr9Dn62qnZnfDwOYHdL+\nVLRBDCApk0YfMhcwExdmYjGTcAQeFFVVxXkOdkRklYi0i0j7EOxdb3x1vlyYicVM3KoxF2ZSulIb\n+hERmQMAmX+P5nuiqq5W1TZVbatDssTNVYZ6JDGgfQDOnwszsZiJW7XkwkzCUWpDXwdgZeb3lQDW\nhrM7lW0W5qIT5+49yVzATFyYicVMwiEj/3dznieIfBcjA6AXADgC4D4APwbwCIAFAPYC+BNVtRMD\n23UdyzwfmfUdL3XHJ4BFAKZhZGB5GEAPgP0AlgCYjJFBnXFzYSaW55kcwsiB1AwUkQmQkwszyfDs\nvTJW9utZqKqzxltg3IYeFRFpV9W2smw8AmG8HmYSzTommqCviZlEt46JpJTXwytFiYg8wYZOROSJ\ncjb01WXcdhTCeD3MJJp1TDRBXxMziW4dE0nRr6ds36ETEVG4+JULEZEnYm/oInKLiOwQkQ4RqcgJ\neMKegZKZ5F1nRefCTNz4+bHCyiTWhi4iNQDuB/BeAMsA3CEiy+Lch5CsQUgzUDITN09yWQNm4rIG\n/PyMtQYhZBL3EfpyAB2quktVBwE8jJGZGyuKqm4CMPaih9swMkscMv++v8DVMRO3is+Fmbjx82OF\nlUncDX0eRq4cHHUgU/NBqTNQMhM3X3NhJm78/FhFZ8JB0QiMN1tcNWImFjNxYy5WoZnE3dAPAsi+\ny8T8TM0HBc9AOQYzcfM1F2bixs+PVXQmcTf0LQBaRWSRiNQDuB0jMzf6oNQZKJmJm6+5MBM3fn6s\n4jNR1Vh/ANwK4HUAvwXwubi3H9Jr+C6ATgBDGPnO7mMAZmJkJHongKcANDOT0jPxIRdmEk8uzOSN\nH14pSkTkCQ6KEhF5gg2diMgTbOhERJ5gQyci8gQbOhGRJ9jQiYg8wYZOROQJNnQiIk/8P2RfF2Zl\nfoE5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13bc90a2da0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 查看前10个特征, 由图中可以看到，每一个feature map所学习的特征大概都不相同，符合了局部特征的提取，最后进行空间层次的组合\n",
    "for i in range(10):\n",
    "    plt.subplot(2, 5, i+1) \n",
    "    plt.imshow(pred_features[0][:,:,i])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LeNet-5\n",
    "LeNet-5模型是Yann LeCun教授（卷积神经网络之父）在1998年提出的第一个成功应用在数字识别问题的卷积神经网络。<br>\n",
    "LeNet-5总共有7层网络，如下图：\n",
    "![title](images/LeNet-5.png)\n",
    "* conv1: kernel_size: 5*5, filters=6, strides=1\n",
    "* pool1: averagePooling, pool_size:2*2, strides=2\n",
    "* conv2: kernel_size: 5*5, filters=16, strides=1\n",
    "* pool2: averagePooling, pool_size=2*2, strides=2\n",
    "* dense1: units：120\n",
    "* dense2: units: 84\n",
    "* dense3: units: 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 下面我们使用keras构建一个LeNet模型\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, AveragePooling2D, Flatten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def LeNet():\n",
    "    '''\n",
    "    这个LeNet-5版本可能与LeCun教授的论文不那么一致，如activation等，但都是遵从了大概的结构模型\n",
    "    '''\n",
    "    LeNet_5 = Sequential()\n",
    "    LeNet_5.add(Conv2D(filters=6, kernel_size=(5, 5), activation='relu', strides=1, input_shape=(28, 28, 1)))\n",
    "    LeNet_5.add(AveragePooling2D(pool_size=(2, 2), strides=2))\n",
    "    LeNet_5.add(Conv2D(filters=16, kernel_size=(5, 5), activation='relu', strides=1))\n",
    "    LeNet_5.add(AveragePooling2D(pool_size=(2, 2), strides=2))\n",
    "    LeNet_5.add(Flatten())\n",
    "    LeNet_5.add(Dense(units=120, activation='relu'))\n",
    "    LeNet_5.add(Dense(units=84, activation='relu'))\n",
    "    LeNet_5.add(Dense(units=10, activation='softmax'))\n",
    "    \n",
    "    return LeNet_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LeNet = LeNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_3 (Conv2D)            (None, 24, 24, 6)         156       \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 12, 12, 6)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 8, 8, 16)          2416      \n",
      "_________________________________________________________________\n",
      "average_pooling2d_2 (Average (None, 4, 4, 16)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 120)               30840     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 84)                10164     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                850       \n",
      "=================================================================\n",
      "Total params: 44,426\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "LeNet.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LeNet.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 45000 samples, validate on 15000 samples\n",
      "Epoch 1/10\n",
      "45000/45000 [==============================] - 6s 137us/step - loss: 0.4272 - acc: 0.8707 - val_loss: 0.1510 - val_acc: 0.9546\n",
      "Epoch 2/10\n",
      "45000/45000 [==============================] - 4s 79us/step - loss: 0.1232 - acc: 0.9629 - val_loss: 0.1272 - val_acc: 0.9583\n",
      "Epoch 3/10\n",
      "45000/45000 [==============================] - 4s 89us/step - loss: 0.0847 - acc: 0.9742 - val_loss: 0.0728 - val_acc: 0.9777\n",
      "Epoch 4/10\n",
      "45000/45000 [==============================] - 4s 80us/step - loss: 0.0658 - acc: 0.9801 - val_loss: 0.0658 - val_acc: 0.9794\n",
      "Epoch 5/10\n",
      "45000/45000 [==============================] - 4s 80us/step - loss: 0.0533 - acc: 0.9836 - val_loss: 0.0774 - val_acc: 0.9755\n",
      "Epoch 6/10\n",
      "45000/45000 [==============================] - 4s 90us/step - loss: 0.0459 - acc: 0.9861 - val_loss: 0.0580 - val_acc: 0.9829\n",
      "Epoch 7/10\n",
      "45000/45000 [==============================] - 4s 88us/step - loss: 0.0398 - acc: 0.9873 - val_loss: 0.0543 - val_acc: 0.9843\n",
      "Epoch 8/10\n",
      "45000/45000 [==============================] - 3s 75us/step - loss: 0.0350 - acc: 0.9890 - val_loss: 0.0786 - val_acc: 0.9759\n",
      "Epoch 9/10\n",
      "45000/45000 [==============================] - 4s 80us/step - loss: 0.0315 - acc: 0.9905 - val_loss: 0.0473 - val_acc: 0.9865\n",
      "Epoch 10/10\n",
      "45000/45000 [==============================] - 3s 77us/step - loss: 0.0278 - acc: 0.9914 - val_loss: 0.0465 - val_acc: 0.9872\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x13bcfca13c8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 训练\n",
    "LeNet.fit(x=x_train, y=y_train, validation_data=(x_val, y_val), batch_size=128, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 117us/step\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = LeNet.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9878\n"
     ]
    }
   ],
   "source": [
    "# 同样地，LeNet在10个epoch后在测试集上达到了98.78%的准确率，并且参数型对于上面的模型更少，且更深。\n",
    "print(test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}