text-generation-lgqm-sample-ver-2-smaller-trn-CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ver 1 using char-cnn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from theano.sandbox import cuda\n",
    "%matplotlib inline\n",
    "from __future__ import division, print_function\n",
    "import operator\n",
    "\n",
    "import numpy as np\n",
    "from keras.layers import TimeDistributed, Activation\n",
    "from numpy.random import choice\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.utils import np_utils\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Input, Embedding, Reshape, merge, LSTM, Bidirectional\n",
    "from keras.layers import TimeDistributed, Activation, SimpleRNN, GRU\n",
    "from keras.layers.core import Flatten, Dense, Dropout, Lambda\n",
    "from keras.regularizers import l2, activity_l2, l1, activity_l1\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.optimizers import SGD, RMSprop, Adam\n",
    "from keras.utils.layer_utils import layer_from_config\n",
    "from keras.metrics import categorical_crossentropy, categorical_accuracy\n",
    "from keras.layers.convolutional import *\n",
    "from keras.preprocessing import image, sequence\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from numpy.random import choice\n",
    "import time\n",
    "from keras.layers import Merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of text:  9215340\n",
      "《临高启明》吹牛者\n",
      "\n",
      "严正声明：本书为丫丫小说网(www.shuyaya.com)的用户上传至其在本站的存储空间，本站只提供TXT全集电子书存储服务以及免费下载服务，以下作品内容之版权与本站无任何\n",
      "size of charset 5688\n",
      " \n",
      "，\n",
      "的\n",
      "\n",
      "\n",
      "\n",
      "。\n",
      "是\n",
      "了\n",
      "一\n",
      "不\n",
      "“\n",
      "”\n",
      "这\n",
      "人\n",
      "有\n",
      "在\n",
      "他\n",
      "来\n",
      "个\n",
      "大\n",
      "上\n",
      "就\n",
      "得\n",
      "到\n",
      "要\n",
      "们\n",
      "和\n",
      "说\n",
      "里\n",
      "―\n",
      "也\n",
      "着\n",
      "子\n",
      "下\n",
      "还\n",
      "出\n",
      "过\n",
      "地\n",
      "会\n",
      "道\n",
      "《临高启明》吹牛者\n",
      "\n",
      "严正声明：本书为丫丫小\n",
      "nb sequences: 9215292\n",
      "xxx done\n"
     ]
    }
   ],
   "source": [
    "# use entire text\n",
    "text = open('./data/lgqm.txt').read()\n",
    "text = unicode(text, \"utf-8\")\n",
    "print('length of text: ', len(text))\n",
    "print(text[:100])\n",
    "charset = set(text)\n",
    "char_freq_map = {c: 0 for c in charset}\n",
    "for c in text:\n",
    "    char_freq_map[c] += 1\n",
    "char_freq_list = sorted(char_freq_map.items(), key=operator.itemgetter(1))\n",
    "char_freq_list.reverse()\n",
    "charlist = [pair[0] for pair in char_freq_list]\n",
    "print('size of charset', len(charlist))\n",
    "for i in range(40):\n",
    "    print(charlist[i])\n",
    "char2idx = dict((c, i) for i, c in enumerate(charlist))\n",
    "text_in_idx = [char2idx[c] for c in text]\n",
    "#text_in_idx[:24]\n",
    "print(''.join(charlist[i] for i in text_in_idx[:24]))\n",
    "maxlen = 48\n",
    "sentences = []\n",
    "next_chars = []\n",
    "for i in range(0, len(text_in_idx) - maxlen):\n",
    "    sentences.append(text_in_idx[i: i + maxlen])\n",
    "    next_chars.append(text_in_idx[i+1: i+maxlen+1])\n",
    "print('nb sequences:', len(sentences))\n",
    "sentences = np.array(sentences)\n",
    "next_chars = np.array(next_chars)\n",
    "sentences.shape, next_chars.shape\n",
    "print('xxx done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# n_fac = 200\n",
    "# batch_size = 64\n",
    "# n_hidden = 512\n",
    "# vocab_size = len(charlist)\n",
    "# model = Sequential([\n",
    "#         Embedding(vocab_size, n_fac, input_length=maxlen, batch_input_shape=(batch_size, maxlen)),\n",
    "#         BatchNormalization(),\n",
    "#         LSTM(n_hidden, input_dim=n_fac,return_sequences=True, dropout_U=0.2, dropout_W=0.2,\n",
    "#              consume_less='gpu'),\n",
    "#         Dropout(0.2),\n",
    "#         LSTM(n_hidden, return_sequences=True, dropout_U=0.2, dropout_W=0.2,\n",
    "#              consume_less='gpu'),\n",
    "#         Dropout(0.2),\n",
    "#         TimeDistributed(Dense(n_hidden, activation='relu')),\n",
    "#         Dropout(0.2),\n",
    "#         TimeDistributed(Dense(vocab_size, activation='softmax')),\n",
    "# ])\n",
    "# model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())\n",
    "# model.summary()\n",
    "# model.optimizer.lr=0.01\n",
    "# # model.load_weights('./data/text-generation-lgqm-ver-1-char-cnn-1497597351.55.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-efed4ffe5a55>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mTimeDistributed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_hidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'relu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mDropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mTimeDistributed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'softmax'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m     ])\n\u001b[1;32m     23\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'binary_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, layers, name)\u001b[0m\n\u001b[1;32m    253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 255\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36madd\u001b[0;34m(self, layer)\u001b[0m\n\u001b[1;32m    306\u001b[0m                  output_shapes=[self.outputs[0]._keras_shape])\n\u001b[1;32m    307\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m             \u001b[0moutput_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    309\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_tensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m                 raise Exception('All layers in a Sequential model '\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/topology.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, x, mask)\u001b[0m\n\u001b[1;32m    485\u001b[0m                                     '`layer.build(batch_input_shape)`')\n\u001b[1;32m    486\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 487\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    488\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/layers/wrappers.pyc\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, input_shape)\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m         \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_spec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mInputSpec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mchild_input_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "n_fac = 200\n",
    "batch_size = 64\n",
    "vocab_size = len(charlist)\n",
    "graph_in = Input ((vocab_size, n_fac))\n",
    "convs = [ ] \n",
    "for fsz in range (3, 6): \n",
    "    x = Convolution1D(64, fsz, border_mode='same', activation=\"relu\")(graph_in)\n",
    "    x = MaxPooling1D()(x) \n",
    "    x = Flatten()(x) \n",
    "    convs.append(x)\n",
    "out = Merge(mode=\"concat\")(convs) \n",
    "graph = Model(graph_in, out)\n",
    "model = Sequential ([\n",
    "    Embedding(vocab_size, n_fac, input_length=maxlen, dropout=0.2, batch_input_shape=(batch_size, maxlen)),\n",
    "    BatchNormalization(),\n",
    "    graph,\n",
    "    Dropout (0.5),\n",
    "    Dense(n_hidden, activation='relu'),\n",
    "    Dropout(0.2),\n",
    "    TimeDistributed(Dense(vocab_size, activation='softmax')),\n",
    "    ])\n",
    "model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n",
    "model.summary()\n",
    "model.optimizer.lr=0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_epochs(n):\n",
    "    # input must be exact multiples of batch_size, for stateful\n",
    "    # multiples = len(sentences)//batch_size*batch_size\n",
    "    # sentences_truncated = sentences[:multiples]\n",
    "    # next_chars_truncated = next_chars[:multiples]\n",
    "    next_chars_adjusted = np.expand_dims(next_chars,-1)\n",
    "    trn_size = batch_size * 100\n",
    "    rand_trn_idx = np.random.randint(0, high=len(text)-1-trn_size)\n",
    "    \n",
    "    sentences_selected = sentences[rand_trn_idx:rand_trn_idx+trn_size]\n",
    "    next_chars_selected = next_chars_adjusted[rand_trn_idx:rand_trn_idx+trn_size]\n",
    "    \n",
    "    for i in range(n):\n",
    "        # model.reset_states()\n",
    "        h = model.fit(sentences_selected, next_chars_selected, batch_size=batch_size, nb_epoch=1, shuffle=True)\n",
    "        print(h.history, h.history['loss'])\n",
    "        weight_file_name = './data/text-generation-lgqm-ver-1-char-cnn-'+str(time.time())+'.h5'\n",
    "    model.save_weights(weight_file_name)\n",
    "    print('saved weights: ', weight_file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# NOTE: when cr is not together with lf, a line is biten! But, I doubt it ever happens in original text\n",
    "# Better learn more to let it avoid naturally\n",
    "\n",
    "def print_example_inline_seed_non_stateful():\n",
    "    pre_string = u\"第65535节 \"\n",
    "    \n",
    "    rand_idx = np.random.randint(0, high=len(text)-maxlen-1)\n",
    "    seed_string = text[rand_idx:rand_idx+maxlen]\n",
    "    \n",
    "    for i in range(maxlen+1):\n",
    "        x=np.array([char2idx[c] for c in seed_string[-maxlen:]])[np.newaxis,:]\n",
    "        preds = model.predict(x, verbose=0)[0][-1]\n",
    "        preds = preds/np.sum(preds)\n",
    "        next_char = choice(charlist, p=preds)\n",
    "        seed_string = seed_string + next_char\n",
    "    seed_string = pre_string+seed_string[maxlen:]\n",
    "    print(seed_string)\n",
    "    for i in range(800):\n",
    "        x=np.array([char2idx[c] for c in seed_string[-maxlen:]])[np.newaxis,:]\n",
    "        preds = model.predict(x, verbose=0)[0][-1]\n",
    "        preds = preds/np.sum(preds)\n",
    "        next_char = choice(charlist, p=preds)\n",
    "        seed_string = seed_string + next_char\n",
    "    print('final: ', seed_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "Error when checking model target: expected dense_4 to have 2 dimensions, but got array with shape (6400, 48, 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-9509fc0d8c05>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrun_epochs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-8-b208c11918b6>\u001b[0m in \u001b[0;36mrun_epochs\u001b[0;34m(n)\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0;31m# model.reset_states()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences_selected\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_chars_selected\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnb_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mweight_file_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'./data/text-generation-lgqm-ver-1-char-cnn-'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.h5'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, **kwargs)\u001b[0m\n\u001b[1;32m    618\u001b[0m                               \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    619\u001b[0m                               \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 620\u001b[0;31m                               sample_weight=sample_weight)\n\u001b[0m\u001b[1;32m    621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m     def evaluate(self, x, y, batch_size=32, verbose=1,\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight)\u001b[0m\n\u001b[1;32m   1032\u001b[0m                                                            \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m                                                            \u001b[0mcheck_batch_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m                                                            batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1035\u001b[0m         \u001b[0;31m# prepare validation data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_batch_dim, batch_size)\u001b[0m\n\u001b[1;32m    963\u001b[0m                                    \u001b[0moutput_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m                                    \u001b[0mcheck_batch_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 965\u001b[0;31m                                    exception_prefix='model target')\n\u001b[0m\u001b[1;32m    966\u001b[0m         sample_weights = standardize_sample_weights(sample_weight,\n\u001b[1;32m    967\u001b[0m                                                     self.output_names)\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_dim, exception_prefix)\u001b[0m\n\u001b[1;32m     95\u001b[0m                                 \u001b[0;34m' to have '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m                                 \u001b[0;34m' dimensions, but got array with shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m                                 str(array.shape))\n\u001b[0m\u001b[1;32m     98\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mref_dim\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcheck_batch_dim\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mException\u001b[0m: Error when checking model target: expected dense_4 to have 2 dimensions, but got array with shape (6400, 48, 1)"
     ]
    }
   ],
   "source": [
    "run_epochs(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "6400/6400 [==============================] - 41s - loss: 3.6252    \n",
      "{'loss': [3.6252318596839905]} [3.6252318596839905]\n",
      "saved weights:  ./data/text-generation-lgqm-ver-1-char-cnn-1497647977.22.h5\n",
      "Epoch 1/1\n",
      "6400/6400 [==============================] - 41s - loss: 1.7943    \n",
      "{'loss': [1.7942678320407868]} [1.7942678320407868]\n",
      "saved weights:  ./data/text-generation-lgqm-ver-1-char-cnn-1497648018.41.h5\n"
     ]
    }
   ],
   "source": [
    "model.optimizer.lr=0.1\n",
    "run_epochs(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "6400/6400 [==============================] - 41s - loss: 3.3394    \n",
      "{'loss': [3.3393771362304689]} [3.3393771362304689]\n",
      "saved weights:  ./data/text-generation-lgqm-ver-1-char-cnn-1497648086.64.h5\n"
     ]
    }
   ],
   "source": [
    "model.optimizer.lr=0.001\n",
    "run_epochs(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "6400/6400 [==============================] - 41s - loss: 3.5758    \n",
      "{'loss': [3.5757711648941042]} [3.5757711648941042]\n",
      "saved weights:  ./data/text-generation-lgqm-ver-1-char-cnn-1497648129.92.h5\n"
     ]
    }
   ],
   "source": [
    "model.optimizer.lr=0.01\n",
    "run_epochs(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "6400/6400 [==============================] - 41s - loss: 3.4361    \n",
      "{'loss': [3.436129505634308]} [3.436129505634308]\n",
      "saved weights:  ./data/text-generation-lgqm-ver-1-char-cnn-1497648183.13.h5\n"
     ]
    }
   ],
   "source": [
    "run_epochs(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第65535节 她立刻叫“师傅开始工作大腿的城门口的地方转了一个主要市镇里，未会穿迁上唯后，所以老际已经全体或者部\n",
      "final:  第65535节 她立刻叫“师傅开始工作大腿的城门口的地方转了一个主要市镇里，未会穿迁上唯后，所以老际已经全体或者部分书要人谈话已经堆到了办公室的几次关防在近的判决河、城内里却少了。\n",
      "\n",
      "    左亚美了车站要道，我想耽搁没有交代。二个月是个庞代来。”随着列时也失去了自己曾经不再是经济下摆却，每天们面堆满了。\n",
      "\n",
      "    “才员们……”左亚美意识到了。但是查询会，提出今天不是难公室，“晚上，虽然乘客们既然没证据不敢怠慢，面积控制小落――治安法庭的长度车。锦衣卫员出现了空心翼上的厢房。对于一个挂着来协理学院毕业”很少，交通便捷地。\n",
      "\n",
      "    好在这段路在城铁，“成了进口，却说，一旦受刑架上审理案件，要么种得上一个人。”\n",
      "\n",
      "    “她和你提出今天判决犯上参理员一面大量外迁。\n",
      "\n",
      "    如何证明呢？她虽然也整修过警衔乎为一样中在城内的时候在一部分腿脚企划院的城门口还没有煤气灯照英份。\n",
      "\n",
      "    夜间有着灯火的重要机构，她定很便在院子里还有个左亚美一些诸惊世骇俗，商馆。\n",
      "\n",
      "    左亚美出了灯笼就更好，十鞭子两间闪亮很容易事情吗？百仞城内就在女首长那么短寒舌，领周边有个时候已经不够，她定在城里的南宝的事情。她好像东门市――捆在东门市的扩展也不能接发了。百仞城吞免。\n",
      "\n",
      "    “我还没有独立人和做些辅助的地方正说。这里她很少来到了办公室。甚至紧挨，一路灯的李永薰微微鞠到近乎废墟。这衣服非常通亮，偌大的总务九小微弱的时候下，眼睛很得是什么在“和我是室友性命也没有警衔，就是按照英国的裙子堪称惊世骇俗的”李永薰微微鞠了县城里的还不需要这条临时的下力，刑务所看得控制的比较小。\n",
      "\n",
      "    夜间们一起来就睡觉，我们一般的话已经拿到所以乘客了。她想耽密了。她把自己带上相起了车。虽然年龄比自己第一次领到县城里自己带左亚美出示证件命也没有公共马车通亮，现在是晚上专门的手段瞒不是经济一样的事情吗？”\n",
      "\n",
      "    李永薰考虑再，毕竟地仆的裙子，后来看到这衣服，一裹圆住了涂乱院子里很少\n"
     ]
    }
   ],
   "source": [
    "print_example_inline_seed_non_stateful()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1152/6400 [====>.........................] - ETA: 34s - loss: 5.6278"
     ]
    }
   ],
   "source": [
    "run_epochs(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}