learn_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "import pickle\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "import tqdm\n",
    "import random\n",
    "import numpy as np\n",
    "import more_itertools\n",
    "import os\n",
    "from prepare_dataset.config import SPARSE_DIR, TMP_DIR, TENSORBOARD_DIR\n",
    "\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def IP_score(y, p, ctr):\n",
    "    denom = len(p)*(1 + (1-ctr) / ctr *10)\n",
    "    return np.sum( y / p) / denom  * 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hX, hI, hC, hP, hCTR = pickle.load(open(TMP_DIR + '/holdout_pack.pickled', 'rb'))\n",
    "vX, vI, vC, vP, vCTR = pickle.load(open(TMP_DIR + '/valid_pack.pickled', 'rb'))\n",
    "batches = pickle.load(open(TMP_DIR + '/train_batches.pickled', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniform_policy_probs = np.ones(len(vP)) / 11\n",
    "print(IP_score(uniform_policy_probs, np.clip(vP, 1e-3, 1), vCTR))\n",
    "# > 41.9452974882"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearModel():\n",
    "    def __init__(self):\n",
    "        self.step = 0\n",
    "        self.graph = tf.Graph()\n",
    "        self.graph.seed = 42\n",
    "        self.build_model()\n",
    "        self.create_session()  \n",
    "        os.makedirs(TENSORBOARD_DIR, exist_ok=True)\n",
    "        \n",
    "    def create_session(self):\n",
    "        self.session = tf.Session(config=None, graph=self.graph)\n",
    "        self.session.run(self.init_all_vars)\n",
    "        \n",
    "    def close_session(self):\n",
    "        self.session.close()\n",
    "        self.graph = None\n",
    "    \n",
    "    def dump_summary(self, fd):\n",
    "        summary = self.session.run(self.summary_op, feed_dict=fd)\n",
    "        self.log_writer.add_summary(summary, self.step)\n",
    "\n",
    "    def build_model(self):\n",
    "        with self.graph.as_default():\n",
    "            self.place_X = tf.sparse_placeholder(dtype=tf.float32, shape=(None, 74000), name=\"input_X\")\n",
    "            self.place_I = tf.placeholder(dtype=tf.int32, shape=(None,2), name=\"input_Indices\")\n",
    "            self.place_C = tf.placeholder(dtype=tf.float32, shape=(None,), name=\"input_Cost\")\n",
    "            self.place_P = tf.placeholder(dtype=tf.float32, shape=(None,), name=\"input_Propensity\")\n",
    "            \n",
    "            self.W = tf.Variable(tf.random_normal(shape=(74000, 1)), name=\"weights\")\n",
    "\n",
    "            self.o_linear = tf.sparse_tensor_dense_matmul(self.place_X, self.W) \n",
    "            x = tf.reshape(self.o_linear, shape=[-1, 11], name='reshape') \n",
    "            self.probs = tf.nn.softmax(x)\n",
    "            \n",
    "            \n",
    "            self.sliced_probs = tf.gather_nd(self.probs, self.place_I)\n",
    "            self.ratio = self.sliced_probs / self.place_P\n",
    "            self.r = self.place_C * self.ratio\n",
    "            \n",
    "            r_mean, r_var = tf.nn.moments(self.r, axes=[0])\n",
    "            tf.summary.scalar('r_mean', r_mean)\n",
    "            tf.summary.scalar('r_std', tf.square(r_var))\n",
    "\n",
    "            self.loss = tf.reduce_mean(self.r) + 0.0001*tf.sqrt(tf.nn.l2_loss(self.o_linear))\n",
    "            tf.summary.scalar('loss', self.loss)\n",
    "            \n",
    "            self.adam3 = tf.train.AdamOptimizer(learning_rate=0.003).minimize(self.loss)\n",
    "            self.adam1 = tf.train.AdamOptimizer(learning_rate=0.001).minimize(self.loss)\n",
    "            \n",
    "            self.init_all_vars = tf.global_variables_initializer()\n",
    "            self.summary_op = tf.summary.merge_all()\n",
    "            self.saver = tf.train.Saver()\n",
    "            self.log_writer = tf.summary.FileWriter(TENSORBOARD_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! rm -rf {TENSORBOARD_DIR}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LinearModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('use `tensorboard --logdir={}` to see learning progress'.format(TENSORBOARD_DIR))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ep in range(40):\n",
    "    np.random.seed(ep)\n",
    "    np.random.shuffle(batches)\n",
    "    for i in range(3001):\n",
    "        tX, tI, tC, tP = batches[i]\n",
    "        click_mask = tC < 0.5\n",
    "        \n",
    "        # revert transformed by /10 negative probs and do propensity clipping\n",
    "        tP_ = tP.copy()\n",
    "        tP_[~click_mask] *= 10\n",
    "        tP_ = np.clip(tP_, 0.3, 1)\n",
    "\n",
    "        # set manual rewards (costs)\n",
    "        tC_ = tC.copy()\n",
    "        tC_[click_mask] = -1\n",
    "        tC_[~click_mask] = -0.1\n",
    "\n",
    "        # do optimization step\n",
    "        fd = {model.place_X: tX, model.place_I: tI, model.place_C: tC_, model.place_P: tP_}\n",
    "        _ = model.session.run(model.adam3, feed_dict=fd)\n",
    "        model.step += 1\n",
    "        \n",
    "        if i%50==0:\n",
    "            model.dump_summary(fd)\n",
    "\n",
    "        if i%1000==0:\n",
    "            fd = {model.place_X: vX, model.place_I: vI}\n",
    "            valid_probs = model.session.run(model.sliced_probs, feed_dict=fd)\n",
    "            valid_score = IP_score(valid_probs, np.clip(vP, 1e-3, 1), vCTR)\n",
    "            print(ep, i, valid_score)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fd = {model.place_X: vX, model.place_I: vI}\n",
    "valid_probs = model.session.run(model.sliced_probs, feed_dict=fd)\n",
    "valid_score = IP_score(valid_probs, np.clip(vP, 1e-7, 1), vCTR)\n",
    "print(valid_score)\n",
    "# > 56.0573730533"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fd = {model.place_X: hX, model.place_I: hI}\n",
    "holdout_probs = model.session.run(model.sliced_probs, feed_dict=fd)\n",
    "holdout_score = IP_score(holdout_probs, np.clip(hP, 1e-7, 1), hCTR)\n",
    "print(holdout_score)\n",
    "# > 55.062495803"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir ./models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.saver.save(model.session, './checkpoints/base.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict for test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import more_itertools\n",
    "\n",
    "def pack_samples_test(samples):\n",
    "    X_coo = sp.vstack([x['mat'] for x in samples])\n",
    "    indices = np.mat([X_coo.row, X_coo.col]).transpose()\n",
    "    return tf.SparseTensorValue(indices, np.ones_like(X_coo.data).astype(np.float32), X_coo.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = []\n",
    "for i in range(8):\n",
    "    ds = pickle.load(open(SPARSE_DIR + '/test_{}.pickled'.format(i), 'rb'))\n",
    "    for ds_chunk in more_itertools.chunked(tqdm.tqdm_notebook(ds), 50000):\n",
    "        tX = pack_samples_test(ds_chunk)\n",
    "        fd = {model.place_X: tX}\n",
    "        probs = model.session.run(model.o_linear, feed_dict=fd)\n",
    "        borders = np.cumsum(([0] + [x['n_candidates'] for x in ds_chunk]))\n",
    "        for i, sample in enumerate(ds_chunk):\n",
    "            result.append((sample['id'], probs[borders[i]:borders[i+1], 0][:]))\n",
    "        del tX\n",
    "    del ds\n",
    "pickle.dump(result, open(TMP_DIR + '/pre_submit.pickled', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fout = open(TMP_DIR + '/submit', 'wt')\n",
    "for pred in tqdm.tqdm_notebook(result):\n",
    "    id = pred[0]\n",
    "    logits = pred[1]\n",
    "    line = id + ';' + ','.join(['{}:{:.4f}'.format(i, v) for i, v in enumerate(logits)])\n",
    "    fout.write(line + '\\n')\n",
    "fout.close()\n",
    "! gzip {TMP_DIR}/submit\n",
    "# > IPS : 54.3729997773"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pickle.load(open(TMP_DIR + '/pre_submit.pickled', 'rb'))\n",
    "fout = open(TMP_DIR + '/submit_scaled', 'wt')\n",
    "for pred in tqdm.tqdm_notebook(result):\n",
    "    id = pred[0]\n",
    "    logits = pred[1] * 20\n",
    "    line = id + ';' + ','.join(['{}:{:.4f}'.format(i, v) for i, v in enumerate(logits)])\n",
    "    fout.write(line + '\\n')\n",
    "fout.close()\n",
    "! gzip {TMP_DIR}/submit_scaled\n",
    "# > IPS: 54.5564084314"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}