otto-common.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04fcd569",
   "metadata": {
    "papermill": {
     "duration": 0.006648,
     "end_time": "2023-03-23T12:10:52.370584",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.363936",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# OTTO common\n",
    "This notebook contains a number of functions and a class, that are used in most notebooks of the project. This notebook was made to avoid copying their definitions between notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8ebb76de",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.385233Z",
     "iopub.status.busy": "2023-03-23T12:10:52.384818Z",
     "iopub.status.idle": "2023-03-23T12:10:52.396368Z",
     "shell.execute_reply": "2023-03-23T12:10:52.394404Z"
    },
    "papermill": {
     "duration": 0.023981,
     "end_time": "2023-03-23T12:10:52.400153",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.376172",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "        \n",
    "import gc\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7cd9215",
   "metadata": {
    "papermill": {
     "duration": 0.004978,
     "end_time": "2023-03-23T12:10:52.411544",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.406566",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## General functions and classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e00bc126",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.424703Z",
     "iopub.status.busy": "2023-03-23T12:10:52.424217Z",
     "iopub.status.idle": "2023-03-23T12:10:52.432028Z",
     "shell.execute_reply": "2023-03-23T12:10:52.430419Z"
    },
    "papermill": {
     "duration": 0.017866,
     "end_time": "2023-03-23T12:10:52.434605",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.416739",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Split dataframe into chunks, while keeping all records with the same values of some column in a single chunk.\n",
    "def divide_df_by_column(df, n_splits, i, column_name):\n",
    "    min_col_value = df[column_name].min() + i*(df[column_name].max() - df[column_name].min())/n_splits\n",
    "    if i+1 == n_splits:\n",
    "        max_col_value = df[column_name].max() + 1\n",
    "    else:\n",
    "        max_col_value = df[column_name].min() + (i+1)*(df[column_name].max() - df[column_name].min())/n_splits\n",
    "    df_i = df.loc[(df[column_name] >= min_col_value) & (df[column_name] < max_col_value)]\n",
    "    return df_i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c8229692",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.448111Z",
     "iopub.status.busy": "2023-03-23T12:10:52.447697Z",
     "iopub.status.idle": "2023-03-23T12:10:52.454582Z",
     "shell.execute_reply": "2023-03-23T12:10:52.453117Z"
    },
    "papermill": {
     "duration": 0.016972,
     "end_time": "2023-03-23T12:10:52.457233",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.440261",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate local datetime from timestamp.\n",
    "def add_datetime(df, ts_col='ts'):\n",
    "    df['time'] = [datetime.utcfromtimestamp(x).strftime('%Y-%m-%d %H:%M:%S') for x in df[ts_col]]\n",
    "    df['time'] = pd.to_datetime(df['time'])\n",
    "    df['time'] = df['time'].dt.tz_localize('Etc/GMT', ambiguous=True).dt.tz_convert('Europe/Berlin')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bff7f106",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.470858Z",
     "iopub.status.busy": "2023-03-23T12:10:52.470432Z",
     "iopub.status.idle": "2023-03-23T12:10:52.478535Z",
     "shell.execute_reply": "2023-03-23T12:10:52.477092Z"
    },
    "papermill": {
     "duration": 0.01815,
     "end_time": "2023-03-23T12:10:52.481012",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.462862",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function is used in many notebooks. It selects up to n_max last events from each session if they are within time_frame from the last event in the session.\n",
    "def filter_by_time_and_n_max(df, time_frame, n_max):\n",
    "    df = df.sort_values(['session','ts'],ascending=[True,False])\n",
    "    df['n'] = df.groupby('session').cumcount().astype(np.int16)\n",
    "    df['time_delta'] = df.groupby('session')['ts'].transform(np.max).astype(np.int32)\n",
    "    df['time_delta'] = df['time_delta'] - df['ts']\n",
    "    df = df.loc[df['time_delta'] < time_frame]\n",
    "    gc.collect()\n",
    "    df = df[['session', 'aid', 'n']]\n",
    "    df = df.loc[df['n'] < n_max]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d34de66",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.494065Z",
     "iopub.status.busy": "2023-03-23T12:10:52.493648Z",
     "iopub.status.idle": "2023-03-23T12:10:52.499274Z",
     "shell.execute_reply": "2023-03-23T12:10:52.498140Z"
    },
    "papermill": {
     "duration": 0.015142,
     "end_time": "2023-03-23T12:10:52.501494",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.486352",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function is used in all notebooks that either train or use a w2vec model.\n",
    "def simple_hash_function(key):\n",
    "    return sum(\n",
    "        index * ord(character)\n",
    "        for index, character in enumerate(repr(key), start=1)\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1af824f",
   "metadata": {
    "papermill": {
     "duration": 0.005057,
     "end_time": "2023-03-23T12:10:52.511983",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.506926",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Functions and classes for co-visitation matrixes only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "50a85fc6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.524901Z",
     "iopub.status.busy": "2023-03-23T12:10:52.524252Z",
     "iopub.status.idle": "2023-03-23T12:10:52.530764Z",
     "shell.execute_reply": "2023-03-23T12:10:52.529852Z"
    },
    "papermill": {
     "duration": 0.015666,
     "end_time": "2023-03-23T12:10:52.533012",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.517346",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function is used to build a list of all aids that show up in cross-validation or test period.\n",
    "# Removing from a co-visitation matrix aid_x that do not show up in cross-validation/test data makes it possible to reduce matrix size without decrease in performance.\n",
    "def build_aid_list(trunked_sessions, trunked_sessions2=None):\n",
    "    df_cv = pd.read_parquet(trunked_sessions)\n",
    "    aid_list = list(set(df_cv['aid']))\n",
    "    if trunked_sessions2:\n",
    "        df_cv2 = pd.read_parquet(trunked_sessions2)\n",
    "        aid_list2 = list(set(df_cv2['aid']))\n",
    "        aid_list = list(set(aid_list + aid_list2))\n",
    "    return aid_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2f117cb7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.545986Z",
     "iopub.status.busy": "2023-03-23T12:10:52.545571Z",
     "iopub.status.idle": "2023-03-23T12:10:52.565818Z",
     "shell.execute_reply": "2023-03-23T12:10:52.564536Z"
    },
    "papermill": {
     "duration": 0.030112,
     "end_time": "2023-03-23T12:10:52.568594",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.538482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CalculateCovisitationMatrix:\n",
    "    '''\n",
    "        Class to calculate the co-visitation matrixes for OTTO project. This  parent class only has common\n",
    "        logic for all the matrixes, and only its child classes will be used for the calculation itself.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, n_splits, n_chunks_groupby, aid_list):\n",
    "        self.n_splits = n_splits\n",
    "        self.n_chunks_groupby = n_chunks_groupby\n",
    "        self.aid_list = aid_list\n",
    "        \n",
    "        self.aid_max = None\n",
    "        \n",
    "    @staticmethod\n",
    "    def groupby_reset_and_reduce(df):\n",
    "        df = df.groupby(['aid_x','aid_y']).wgt.sum()\n",
    "        df = df.reset_index()\n",
    "        df['aid_x'] = df['aid_x'].astype(np.int32)\n",
    "        df['aid_y'] = df['aid_y'].astype(np.int32)\n",
    "        return df\n",
    "    \n",
    "    # Perform groupby chunk by chunk to reduce RAM usage.\n",
    "    def groupby_in_chunks(self, df1, df2):\n",
    "        for j in range(self.n_chunks_groupby):\n",
    "            aid_x_min = j*(self.aid_max)/self.n_chunks_groupby\n",
    "            if j + 1 == self.n_chunks_groupby:\n",
    "                aid_x_max = self.aid_max + 1\n",
    "            else:\n",
    "                aid_x_max = (j+1)*(self.aid_max)/self.n_chunks_groupby\n",
    "            df2_j = df2.loc[(df2['aid_x'] >= aid_x_min) & (df2['aid_x'] < aid_x_max)].copy()\n",
    "            df1_j = df1.loc[(df1['aid_x'] >= aid_x_min) & (df1['aid_x'] < aid_x_max)].copy()\n",
    "            df2_j = pd.concat([df2_j, df1_j])\n",
    "            del df1_j\n",
    "            gc.collect()\n",
    "            df2_j = self.groupby_reset_and_reduce(df2_j)\n",
    "            if j == 0:\n",
    "                df_all = df2_j\n",
    "            else:\n",
    "                df_all = pd.concat([df_all, df2_j])\n",
    "        return df_all\n",
    "    \n",
    "    # Reduce the co-visitation matrix only to top_n rows for each aid_x.\n",
    "    def get_top_n(self, df_matrix, top_n):\n",
    "        for j in range(self.n_chunks_groupby):\n",
    "            df_matrix_chunk = divide_df_by_column(df_matrix, self.n_chunks_groupby, j, 'aid_x')\n",
    "            df_matrix_chunk = df_matrix_chunk.sort_values(['aid_x','wgt'],ascending=[True,False])\n",
    "            df_matrix_chunk = df_matrix_chunk.reset_index(drop=True)\n",
    "            df_matrix_chunk['n'] = df_matrix_chunk.groupby('aid_x').aid_y.cumcount()\n",
    "            df_matrix_chunk = df_matrix_chunk.loc[df_matrix_chunk.n<top_n].drop('n',axis=1)\n",
    "            gc.collect()\n",
    "            if j == 0:\n",
    "                df_matrix_top_n = df_matrix_chunk\n",
    "            else:\n",
    "                df_matrix_top_n = pd.concat([df_matrix_top_n, df_matrix_chunk])\n",
    "        return df_matrix_top_n\n",
    "        \n",
    "    # Main method of the class. Provides the framework for the calculation.\n",
    "    def generate_covisitation_matrix(self, data_path):\n",
    "        df_click_data = pd.read_parquet(data_path)\n",
    "        self.aid_max = df_click_data['aid'].max()\n",
    "    \n",
    "        for i in range(self.n_splits):\n",
    "            print(str(i))\n",
    "            df_i = divide_df_by_column(df_click_data, self.n_splits, i, 'session')\n",
    "            df_i_wgt = self.calculate_weights(df_i) \n",
    "            del df_i\n",
    "            gc.collect()\n",
    "            if str(i).endswith('0'): \n",
    "                df_wgt = df_i_wgt\n",
    "            else:\n",
    "                df_wgt = pd.concat([df_wgt, df_i_wgt], axis=0)\n",
    "            if str(i).endswith('9'):\n",
    "                df_wgt = self.groupby_reset_and_reduce(df_wgt)\n",
    "                if i == 9:\n",
    "                    df_wgt_all = df_wgt\n",
    "                else:\n",
    "                    df_wgt_all = self.groupby_in_chunks(df_wgt, df_wgt_all)\n",
    "                    del df_wgt\n",
    "                    gc.collect()\n",
    "        return df_wgt_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8090a4a8",
   "metadata": {
    "papermill": {
     "duration": 0.005113,
     "end_time": "2023-03-23T12:10:52.579247",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.574134",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Functions for count_clicks/count_buys notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "95bcf41c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.592520Z",
     "iopub.status.busy": "2023-03-23T12:10:52.591427Z",
     "iopub.status.idle": "2023-03-23T12:10:52.598356Z",
     "shell.execute_reply": "2023-03-23T12:10:52.597004Z"
    },
    "papermill": {
     "duration": 0.016533,
     "end_time": "2023-03-23T12:10:52.601118",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.584585",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate averaged aid counts after selecting data for required period of time.\n",
    "# Used by the create_average_daily_counts function.\n",
    "def count_aids(df):\n",
    "    df = df.groupby('aid')['session'].nunique()\n",
    "    total_clicks = df.sum()/10000\n",
    "    df = df.reset_index()\n",
    "    df['aid_count'] = df['session']/total_clicks\n",
    "    df = df[['aid', 'aid_count']]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "14955cc6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.614175Z",
     "iopub.status.busy": "2023-03-23T12:10:52.613766Z",
     "iopub.status.idle": "2023-03-23T12:10:52.623186Z",
     "shell.execute_reply": "2023-03-23T12:10:52.621964Z"
    },
    "papermill": {
     "duration": 0.018581,
     "end_time": "2023-03-23T12:10:52.625403",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.606822",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create averaged daily counts for 7 the last days of full data or the cross-validation data.\n",
    "def create_average_daily_counts(data_path, is_trunked, buy_type=None):\n",
    "    df_data = pd.read_parquet(data_path)\n",
    "    if buy_type:\n",
    "        df_data = df_data.loc[df_data['type'] == buy_type]\n",
    "    if is_trunked == False:\n",
    "        last_week_ts = df_data['ts'].max() - 7 * 24 * 60 * 60\n",
    "        df_data = df_data.loc[df_data['ts'] > last_week_ts]\n",
    "    df_data = add_datetime(df_data)\n",
    "    df_data['day_of_week'] = df_data['time'].dt.dayofweek.astype(np.int8)\n",
    "    for i in range(7):\n",
    "        df_i = df_data.loc[df_data['day_of_week'] == i]\n",
    "        df_i = df_i[['session', 'aid']]\n",
    "        df_i = count_aids(df_i)\n",
    "        df_i['day_of_week'] = i\n",
    "        if i == 0:\n",
    "            df_total = df_i\n",
    "        else:\n",
    "            df_total = pd.concat([df_total, df_i])\n",
    "    return df_total"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb838107",
   "metadata": {
    "papermill": {
     "duration": 0.005133,
     "end_time": "2023-03-23T12:10:52.636073",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.630940",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Functions for candidate generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "783bb11b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.649167Z",
     "iopub.status.busy": "2023-03-23T12:10:52.648702Z",
     "iopub.status.idle": "2023-03-23T12:10:52.657342Z",
     "shell.execute_reply": "2023-03-23T12:10:52.655877Z"
    },
    "papermill": {
     "duration": 0.018419,
     "end_time": "2023-03-23T12:10:52.659946",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.641527",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Builds a dictionary of items most commonly clicked/carted/ordered during a day.\n",
    "# If there are free slots left after generating candidates using all types of aids in history, aids from top_dict are used.\n",
    "def build_top_dict(df, n_candidates, event_type):   \n",
    "    top_df = df.loc[df['type']==event_type].groupby(['day_of_week', 'aid'])['session'].count()\n",
    "    top_df = top_df.reset_index()\n",
    "    top_df = top_df.sort_values(['day_of_week','session'],ascending=[True,False])\n",
    "    top_df['n'] = top_df.groupby('day_of_week').session.cumcount()\n",
    "    top_df = top_df.loc[top_df.n<n_candidates].drop('n',axis=1)\n",
    "    top_dict = top_df.groupby('day_of_week').aid.apply(list).to_dict()\n",
    "    return top_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "87253d72",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.673274Z",
     "iopub.status.busy": "2023-03-23T12:10:52.672821Z",
     "iopub.status.idle": "2023-03-23T12:10:52.681502Z",
     "shell.execute_reply": "2023-03-23T12:10:52.680169Z"
    },
    "papermill": {
     "duration": 0.01895,
     "end_time": "2023-03-23T12:10:52.684703",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.665753",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# When generating candidates for cross-validation dataset, it makes sense to leave only sessions with some positive target.\n",
    "# Most sessions do not have any aid carted or ordered, so keeping only sessions with some item carted/ordered will speed up the calculations\n",
    "# and will not harm the model performance.\n",
    "def reduce_df_prepare_answers(main_df, answers_path, col_name):\n",
    "    col_name_len = col_name + '_len'\n",
    "    df_answers = pd.read_parquet(answers_path)\n",
    "    df_answers = df_answers[['session', col_name]]\n",
    "    df_answers[col_name_len] = [len(x) for x in df_answers[col_name]]\n",
    "    df_answers = df_answers.loc[df_answers[col_name_len] > 0]\n",
    "    main_df = pd.merge(main_df, df_answers, on='session', how = 'inner')\n",
    "    main_df = main_df.drop(col_name,axis=1).drop(col_name_len,axis=1)\n",
    "    return main_df, df_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9c931a8c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.697933Z",
     "iopub.status.busy": "2023-03-23T12:10:52.697471Z",
     "iopub.status.idle": "2023-03-23T12:10:52.708660Z",
     "shell.execute_reply": "2023-03-23T12:10:52.706740Z"
    },
    "papermill": {
     "duration": 0.020959,
     "end_time": "2023-03-23T12:10:52.711363",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.690404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prints stats after candidates for a cross-validation dataset is ready.\n",
    "# Shows both absolute numbers and percentages of guessed aids.\n",
    "def print_stats(df_check, col_name):\n",
    "    col_name_len = col_name + '_len'\n",
    "    col_name_clipped = col_name_len + '_clipped'\n",
    "    \n",
    "    total_target = df_check[col_name_len].sum()\n",
    "    total_after_clip = df_check[col_name_clipped].sum()\n",
    "    total_guessed = df_check['pred_true'].sum()\n",
    "    print(f\"Total {col_name}:  {total_target}\")\n",
    "    print(f\"Total {col_name} clipped:  {total_after_clip}\")\n",
    "    print(f\"Total {col_name} guessed:  {total_guessed}\")\n",
    "\n",
    "    target_with_buys_in_history = df_check.loc[df_check['buys'] > 0, col_name_clipped].sum()\n",
    "    target_with_buys_guessed = df_check.loc[df_check['buys'] > 0, 'pred_true'].sum()\n",
    "    print(f\"Total {col_name} with buys in history:  {target_with_buys_in_history}\")\n",
    "    print(f\"{col_name.capitalize()} with buys in history guessed:  {target_with_buys_guessed}\")\n",
    "    \n",
    "    target_no_buys_in_history = df_check.loc[df_check['buys'] == 0, col_name_clipped].sum()\n",
    "    target_with_no_buys_in_history_guessed = df_check.loc[df_check['buys'] == 0, 'pred_true'].sum()\n",
    "    print(f\"Total {col_name} with no buys in history:  {target_no_buys_in_history}\")\n",
    "    print(f\"{col_name.capitalize()} with no buys in history guessed:  {target_with_no_buys_in_history_guessed}\")\n",
    "    \n",
    "    print(f\"Total:  {100*total_guessed/total_after_clip:.2f}%\")\n",
    "    print(f\"Buys:  {100*target_with_buys_guessed/target_with_buys_in_history:.2f}%\")\n",
    "    print(f\"No buys:  {100*target_with_no_buys_in_history_guessed/target_no_buys_in_history:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b3812a74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.727726Z",
     "iopub.status.busy": "2023-03-23T12:10:52.726883Z",
     "iopub.status.idle": "2023-03-23T12:10:52.739930Z",
     "shell.execute_reply": "2023-03-23T12:10:52.738045Z"
    },
    "papermill": {
     "duration": 0.025626,
     "end_time": "2023-03-23T12:10:52.742887",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.717261",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Join dataframe with generated candidates and dataframe with answers.\n",
    "# Prepares all the data needed to print the stats.\n",
    "def calculate_stats(prediction_df, answers_df, unique_session_aids, unique_session_buys, col_name, n_candidates):\n",
    "    col_name_len = col_name + '_len'\n",
    "    col_name_clipped = col_name_len + '_clipped'\n",
    "    col_name_prediction = col_name[:-1] + '_predictions'\n",
    "    \n",
    "    prediction_df = pd.merge(prediction_df, answers_df, on = 'session')\n",
    "    df_check_stats = prediction_df.explode(col_name).reset_index(drop=True)\n",
    "    df_check_stats['pred_true'] = df_check_stats.apply(lambda x: x[col_name] in x[col_name_prediction], axis=1)\n",
    "    df_check_stats['pred_true'] = df_check_stats['pred_true'].astype(np.int8)\n",
    "    df_check_stats = df_check_stats.groupby('session').agg({col_name_len: 'max', 'pred_true': 'sum'})\n",
    "    df_check_stats[col_name_clipped] = df_check_stats[col_name_len].clip(0,n_candidates)\n",
    "    df_check_stats = df_check_stats.reset_index()\n",
    "    df_check_stats = pd.merge(df_check_stats, unique_session_aids, how = 'left', on='session')\n",
    "    df_check_stats = pd.merge(df_check_stats, unique_session_buys, how = 'left', on='session')\n",
    "    df_check_stats['buys'] = df_check_stats['buys'].fillna(0)\n",
    "    print_stats(df_check_stats, col_name)\n",
    "        \n",
    "    del df_check_stats[col_name_len]\n",
    "    prediction_df = pd.merge(prediction_df, df_check_stats, left_on = 'session', right_on='session')\n",
    "        \n",
    "    prediction_df = prediction_df[['session', col_name_prediction, col_name, 'pred_true']]\n",
    "    return prediction_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ac4f3588",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.756522Z",
     "iopub.status.busy": "2023-03-23T12:10:52.756082Z",
     "iopub.status.idle": "2023-03-23T12:10:52.763260Z",
     "shell.execute_reply": "2023-03-23T12:10:52.761538Z"
    },
    "papermill": {
     "duration": 0.017,
     "end_time": "2023-03-23T12:10:52.765923",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.748923",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Reduce the co-visitation matrix to n_candidates and turn it from a dataframe to a dictionary.\n",
    "# Using dictionary speeds up the candidate generation process.\n",
    "def matrix_to_dict(click2buy_matrix_path, n_candidates):\n",
    "    df_matrix = pd.read_parquet(click2buy_matrix_path)\n",
    "    df_matrix['n'] = df_matrix.groupby('aid_x').aid_y.cumcount()\n",
    "    df_matrix = df_matrix.loc[df_matrix.n<n_candidates].drop('n',axis=1)\n",
    "    click2buy_dict = df_matrix.groupby('aid_x').aid_y.apply(list).to_dict()\n",
    "    return click2buy_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6581dfe3",
   "metadata": {
    "papermill": {
     "duration": 0.005787,
     "end_time": "2023-03-23T12:10:52.777880",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.772093",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Functions for the model notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "faee30d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.793307Z",
     "iopub.status.busy": "2023-03-23T12:10:52.792067Z",
     "iopub.status.idle": "2023-03-23T12:10:52.802131Z",
     "shell.execute_reply": "2023-03-23T12:10:52.801139Z"
    },
    "papermill": {
     "duration": 0.021876,
     "end_time": "2023-03-23T12:10:52.805655",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.783779",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prints some stats after running cross-validation.\n",
    "# It prints absolute number of aids guessed, percent of aids guessed, and average position of ground truth candidates after re-ranking.\n",
    "def calculate_recall(df_cv, result_col_name, const):\n",
    "    df_cv = df_cv[['session', result_col_name, 'target']]\n",
    "    gc.collect()\n",
    "    mean_prediction_true = df_cv.loc[df_cv['target'] == True, result_col_name].mean()\n",
    "    mean_prediction_total = df_cv[result_col_name].mean()\n",
    "    df_cv = df_cv.sort_values(['session', result_col_name],ascending=[True,False])\n",
    "    df_cv['n'] = df_cv.groupby('session').cumcount().astype(np.int8)\n",
    "    mean_n = df_cv.loc[df_cv['target'] == True, 'n'].mean()\n",
    "    df_cv = df_cv.loc[df_cv['n'] < 20].drop('n',axis=1)\n",
    "    total_guessed = len(df_cv.loc[df_cv['target'] == True])\n",
    "    print('Total_guessed: ' + str(total_guessed))\n",
    "    percent = total_guessed/const\n",
    "    print(f\"Percent {100*percent:.2f}%\")\n",
    "    print(f\"Av_n {mean_n:.2f}\")\n",
    "    del df_cv\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6f11a067",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.819766Z",
     "iopub.status.busy": "2023-03-23T12:10:52.819303Z",
     "iopub.status.idle": "2023-03-23T12:10:52.826445Z",
     "shell.execute_reply": "2023-03-23T12:10:52.825189Z"
    },
    "papermill": {
     "duration": 0.018009,
     "end_time": "2023-03-23T12:10:52.829730",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.811721",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Removes a fraction of \"negative\" (not clicked/carted/ordered) candidates from cross-validation dataset.\n",
    "# Having too much \"negative\" candidates actually harms the model's performance and leads to too high memory consumption.\n",
    "def remove_frac(train_index, df, frac):\n",
    "    df = df.iloc[train_index]\n",
    "    if frac > 0:\n",
    "        remove_index = df.loc[df['target'] == False].sample(frac=frac, random_state=25).index\n",
    "        df = df.drop(remove_index)\n",
    "    train_index = df.index\n",
    "    return train_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "41059bf1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-23T12:10:52.844194Z",
     "iopub.status.busy": "2023-03-23T12:10:52.843742Z",
     "iopub.status.idle": "2023-03-23T12:10:52.851999Z",
     "shell.execute_reply": "2023-03-23T12:10:52.850672Z"
    },
    "papermill": {
     "duration": 0.018658,
     "end_time": "2023-03-23T12:10:52.854548",
     "exception": false,
     "start_time": "2023-03-23T12:10:52.835890",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Function to select top_20 candidates and perform some formatting required to upload the results.\n",
    "def select_top_20_and_format(df, candidate_col, rating_col):\n",
    "    # Enumereate the results and select top 20 for each session.\n",
    "    df = df.sort_values(['session', rating_col],ascending=[True,False])\n",
    "    df['n'] = df.groupby('session').cumcount().astype(np.int8)\n",
    "    df = df.loc[df['n'] < 20].drop('n',axis=1)\n",
    "    \n",
    "    # Final formatting.\n",
    "    df[candidate_col] = df[candidate_col].apply(str)\n",
    "    df = (df.groupby('session').agg({candidate_col: lambda x: \" \".join(x)}))\n",
    "    return df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 13.275325,
   "end_time": "2023-03-23T12:10:53.585302",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-23T12:10:40.309977",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}