5.TEXT_SIM/Appendix/5.3.3_Quora_LSTM_Appendix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# from tensorflow.keras import backend as K\n",
    "\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initial global var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 미리 Global 변수를 지정하자. 파일 명, 파일 위치, 디렉토리 등이 있다.\n",
    "\n",
    "DATA_IN_PATH = '../data_in/'\n",
    "DATA_OUT_PATH = '../data_out/'\n",
    "\n",
    "TRAIN_Q1_DATA_FILE = 'train_q1.npy'\n",
    "TRAIN_Q2_DATA_FILE = 'train_q2.npy'\n",
    "TRAIN_LABEL_DATA_FILE = 'train_label.npy'\n",
    "NB_WORDS_DATA_FILE = 'data_configs.json'\n",
    "\n",
    "## 학습에 필요한 파라메터들에 대해서 지정하는 부분이다.\n",
    "\n",
    "BATCH_SIZE = 16\n",
    "EPOCH = 2\n",
    "HIDDEN = 64\n",
    "BUFFER_SIZE = 10000\n",
    "\n",
    "NUM_LAYERS = 3\n",
    "DROPOUT_RATIO = 0.3\n",
    "\n",
    "TEST_SPLIT = 0.1\n",
    "RNG_SEED = 13371447\n",
    "EMBEDDING_DIM = 128\n",
    "MAX_SEQ_LEN = 31"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 데이터를 불러오는 부분이다. 효과적인 데이터 불러오기를 위해, 미리 넘파이 형태로 저장시킨 데이터를 로드한다.\n",
    "\n",
    "q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb'))\n",
    "q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb'))\n",
    "labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))\n",
    "prepro_configs = None\n",
    "\n",
    "with open(DATA_IN_PATH + NB_WORDS_DATA_FILE, 'r') as f:\n",
    "    prepro_configs = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE = prepro_configs['vocab_size']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split train and test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q1_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q1_data])\n",
    "q2_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q2_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 데이터를 나누어 저장하자. sklearn의 train_test_split을 사용하면 유용하다. 하지만, 쿼라 데이터의 경우는\n",
    "## 입력이 1개가 아니라 2개이다. 따라서, np.stack을 사용하여 두개를 하나로 쌓은다음 활용하여 분류한다.\n",
    "\n",
    "X = np.stack((q1_data, q2_data), axis=1)\n",
    "y = labels\n",
    "train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)\n",
    "\n",
    "train_Q1 = train_X[:,0]\n",
    "train_Q2 = train_X[:,1]\n",
    "test_Q1 = test_X[:,0]\n",
    "test_Q2 = test_X[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rearrange(base, hypothesis, labels):\n",
    "    features = {\"base\": base, \"hypothesis\": hypothesis}\n",
    "    return features, labels\n",
    "\n",
    "def train_input_fn():\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((train_Q1, train_Q2, train_y))\n",
    "    dataset = dataset.shuffle(buffer_size=len(train_Q1))\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.map(rearrange)\n",
    "    dataset = dataset.repeat(EPOCH)\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    \n",
    "    return iterator.get_next()\n",
    "\n",
    "def eval_input_fn():\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((test_Q1, test_Q2, test_y))\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.map(rearrange)\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    \n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Malstm(features, labels, mode):\n",
    "        \n",
    "    TRAIN = mode == tf.estimator.ModeKeys.TRAIN\n",
    "    EVAL = mode == tf.estimator.ModeKeys.EVAL\n",
    "    PREDICT = mode == tf.estimator.ModeKeys.PREDICT\n",
    "\n",
    "    def basic_bilstm_network(inputs, name):\n",
    "        with tf.variable_scope(name, reuse=tf.AUTO_REUSE):\n",
    "            lstm_fw = [\n",
    "                    tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell(HIDDEN), output_keep_prob=DROPOUT_RATIO)\n",
    "                    for layer in range(NUM_LAYERS)\n",
    "                    ]\n",
    "            lstm_bw = [\n",
    "                    tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell(HIDDEN), output_keep_prob=DROPOUT_RATIO)\n",
    "                    for layer in range(NUM_LAYERS)\n",
    "                    ]\n",
    "            \n",
    "            multi_lstm_fw = tf.nn.rnn_cell.MultiRNNCell(lstm_fw)\n",
    "            multi_lstm_bw = tf.nn.rnn_cell.MultiRNNCell(lstm_bw)\n",
    "\n",
    "            (fw_outputs, bw_outputs), _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=multi_lstm_fw,\n",
    "                                                cell_bw=multi_lstm_bw,\n",
    "                                                inputs=inputs,\n",
    "                                                dtype=tf.float32)\n",
    "            \n",
    "            outputs = tf.concat([fw_outputs, bw_outputs], 2)\n",
    "            \n",
    "            return outputs[:,-1,:]\n",
    "            \n",
    "    embedding = tf.keras.layers.Embedding(VOCAB_SIZE, EMBEDDING_DIM)\n",
    "    \n",
    "    base_embedded_matrix = embedding(features['base'])\n",
    "    hypothesis_embedded_matrix = embedding(features['hypothesis'])\n",
    "    \n",
    "    base_sementic_matrix = basic_bilstm_network(base_embedded_matrix, 'base')\n",
    "    hypothesis_sementic_matrix = basic_bilstm_network(hypothesis_embedded_matrix, 'hypothesis')\n",
    "\n",
    "    base_sementic_matrix = tf.keras.layers.Dropout(DROPOUT_RATIO)(base_sementic_matrix)\n",
    "    hypothesis_sementic_matrix = tf.keras.layers.Dropout(DROPOUT_RATIO)(hypothesis_sementic_matrix)\n",
    "    \n",
    "#     merged_matrix = tf.concat([base_sementic_matrix, hypothesis_sementic_matrix], -1)\n",
    "#     logit_layer = tf.keras.layers.dot([base_sementic_matrix, hypothesis_sementic_matrix], axes=1, normalize=True)    \n",
    "#     logit_layer = K.exp(-K.sum(K.abs(base_sementic_matrix - hypothesis_sementic_matrix), axis=1, keepdims=True))\n",
    "\n",
    "    logit_layer = tf.exp(-tf.reduce_sum(tf.abs(base_sementic_matrix - hypothesis_sementic_matrix), axis=1, keepdims=True))\n",
    "    logit_layer = tf.squeeze(logit_layer, axis=-1)\n",
    "                \n",
    "    if PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "                  mode=mode,\n",
    "                  predictions={\n",
    "                      'is_duplicate':logit_layer\n",
    "                  })\n",
    "    \n",
    "    #prediction 진행 시, None\n",
    "    if labels is not None:\n",
    "        labels = tf.to_float(labels)\n",
    "    \n",
    "#     loss = tf.reduce_mean(tf.keras.metrics.binary_crossentropy(y_true=labels, y_pred=logit_layer))\n",
    "    loss = tf.losses.mean_squared_error(labels=labels, predictions=logit_layer)\n",
    "#     loss = tf.reduce_mean(tf.losses.sigmoid_cross_entropy(labels, logit_layer))\n",
    "    \n",
    "    if EVAL:\n",
    "        accuracy = tf.metrics.accuracy(labels, tf.round(logit_layer))\n",
    "        eval_metric_ops = {'acc': accuracy}\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "                  mode=mode,\n",
    "                  eval_metric_ops= eval_metric_ops,\n",
    "                  loss=loss)\n",
    "\n",
    "    elif TRAIN:\n",
    "\n",
    "        global_step = tf.train.get_global_step()\n",
    "        train_op = tf.train.AdamOptimizer(1e-3).minimize(loss, global_step)\n",
    "\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "                  mode=mode,\n",
    "                  train_op=train_op,\n",
    "                  loss=loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training & Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\" #For TEST\n",
    "\n",
    "model_dir = os.path.join(os.getcwd(), DATA_OUT_PATH + \"/checkpoint/rnn2/\")\n",
    "os.makedirs(model_dir, exist_ok=True)\n",
    "\n",
    "config_tf = tf.estimator.RunConfig()\n",
    "\n",
    "lstm_est = tf.estimator.Estimator(Malstm, model_dir=model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_est.train(train_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_est.evaluate(eval_input_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load test dataset |& create submit dataset to kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_Q1_DATA_FILE = 'test_q1.npy'\n",
    "TEST_Q2_DATA_FILE = 'test_q2.npy'\n",
    "TEST_ID_DATA_FILE = 'test_id.npy'\n",
    "\n",
    "test_q1_data = np.load(open(DATA_IN_PATH + TEST_Q1_DATA_FILE, 'rb'))\n",
    "test_q2_data = np.load(open(DATA_IN_PATH + TEST_Q2_DATA_FILE, 'rb'))\n",
    "test_id_data = np.load(open(DATA_IN_PATH + TEST_ID_DATA_FILE, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={\"base\":test_q1_data, \n",
    "                                                         \"hypothesis\":test_q2_data}, \n",
    "                                                      shuffle=False)\n",
    "\n",
    "predictions = np.array([p['is_duplicate'] for p in lstm_est.predict(input_fn=\n",
    "predict_input_fn)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(predictions)) #2345796\n",
    "\n",
    "output = pd.DataFrame( data={\"test_id\":test_id_data, \"is_duplicate\": list(predictions)} )\n",
    "output.to_csv( \"rnn_predict.csv\", index=False, quoting=3 )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}