experiment_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports (make sure to install janus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install -e .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip show janus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-04T17:16:38.828666Z",
     "start_time": "2020-12-04T17:16:37.999997Z"
    }
   },
   "outputs": [],
   "source": [
    "import janus\n",
    "from janus.stats.experiment import Experiment, Variant\n",
    "import pandas as pd\n",
    "\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-27T17:29:01.054901Z",
     "start_time": "2020-11-27T17:29:01.049977Z"
    }
   },
   "source": [
    "## Load Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-04T17:16:38.970497Z",
     "start_time": "2020-12-04T17:16:38.949037Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>alternative</th>\n",
       "      <th>revenue</th>\n",
       "      <th>sum_revenue</th>\n",
       "      <th>sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>A</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>A</td>\n",
       "      <td>200</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>A</td>\n",
       "      <td>300</td>\n",
       "      <td>300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>B</td>\n",
       "      <td>230</td>\n",
       "      <td>230</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>B</td>\n",
       "      <td>170</td>\n",
       "      <td>170</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>B</td>\n",
       "      <td>320</td>\n",
       "      <td>320</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id alternative  revenue  sum_revenue  sales\n",
       "0   1           A        0            0      0\n",
       "1   2           A        0            0      0\n",
       "2   3           A        0            0      0\n",
       "3   4           A      100          100      1\n",
       "4   5           A      200          200      1\n",
       "5   6           A      300          300      1\n",
       "6   7           B        0            0      0\n",
       "7   8           B      230          230      1\n",
       "8   9           B      170          170      1\n",
       "9  10           B      320          320      1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"examples/results_per_user.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-27T23:08:00.040508Z",
     "start_time": "2020-11-27T23:08:00.031558Z"
    }
   },
   "source": [
    "We see that, for the data we have, alternative B looks better: 3 sales out of 4 participants, with average ticket of (230+170+320)/3 = 240 and average ARPU of 180. All metrics are higher that A's."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-04T17:16:38.974932Z",
     "start_time": "2020-12-04T17:16:38.972390Z"
    }
   },
   "outputs": [],
   "source": [
    "experiment = Experiment(name=\"my_experiment\",\n",
    "                       keymetrics=['conversion', 'revenue', 'arpu'],\n",
    "                       baseline_variant_name='A')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We run the experiment over our data to get results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-04T17:16:51.994576Z",
     "start_time": "2020-12-04T17:16:38.976479Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:INITIALIZING experiment 'my_experiment' evaluation...\n",
      "INFO:root:consolidating metrics over variants...\n",
      "INFO:root:calculating bayesian statistics over variants for metric conversion\n",
      "INFO:root:sampling data for variant A...\n",
      "INFO:root:sampling data for variant B...\n",
      "INFO:root:calculating probability to beat and expected loss for variant A...\n",
      "INFO:root:calculating probability to beat and expected loss for variant B...\n",
      "INFO:root:calculating bayesian statistics over variants for metric revenue\n",
      "INFO:root:sampling data for variant A...\n",
      "INFO:root:sampling data for variant B...\n",
      "INFO:root:calculating probability to beat and expected loss for variant A...\n",
      "INFO:root:calculating probability to beat and expected loss for variant B...\n",
      "INFO:root:calculating bayesian statistics over variants for metric arpu\n",
      "INFO:root:sampling data for variant A...\n",
      "INFO:root:sampling data for variant B...\n",
      "INFO:root:calculating probability to beat and expected loss for variant A...\n",
      "INFO:root:calculating probability to beat and expected loss for variant B...\n",
      "INFO:root:consolidating final results...\n",
      "INFO:root:consolidating final results for variant A...\n",
      "INFO:root:consolidating final results for variant B...\n",
      "INFO:root:FINISHED experiment evaluation.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'A': {'users': 6,\n",
       "  'sales': 3,\n",
       "  'paids': 3,\n",
       "  'revenue': 600,\n",
       "  'conversion': 0.5,\n",
       "  'ticket': 200.0,\n",
       "  'arpu': 100.0,\n",
       "  'statistics': {'conversion': {'chance_to_beat': 0.246727,\n",
       "    'expected_loss': 0.20485381083632845,\n",
       "    'lift': -0.33333333333333337,\n",
       "    'diff': -0.25},\n",
       "   'revenue': {'chance_to_beat': 0.401356,\n",
       "    'expected_loss': 90.26219646796406,\n",
       "    'lift': -0.16666666666666663,\n",
       "    'diff': -120},\n",
       "   'arpu': {'chance_to_beat': 0.289212,\n",
       "    'expected_loss': 81.58532656393245,\n",
       "    'lift': -0.4444444444444444,\n",
       "    'diff': -80.0}},\n",
       "  'ratio': 0.6},\n",
       " 'B': {'users': 4,\n",
       "  'sales': 3,\n",
       "  'paids': 3,\n",
       "  'revenue': 720,\n",
       "  'conversion': 0.75,\n",
       "  'ticket': 240.0,\n",
       "  'arpu': 180.0,\n",
       "  'statistics': {'conversion': {'chance_to_beat': 0.753273,\n",
       "    'expected_loss': 0.03789908703261173,\n",
       "    'lift': 0.5,\n",
       "    'diff': 0.25},\n",
       "   'revenue': {'chance_to_beat': 0.598644,\n",
       "    'expected_loss': 50.334123257073756,\n",
       "    'lift': 0.19999999999999996,\n",
       "    'diff': 120},\n",
       "   'arpu': {'chance_to_beat': 0.710788,\n",
       "    'expected_loss': 21.61296874689926,\n",
       "    'lift': 0.8,\n",
       "    'diff': 80.0}},\n",
       "  'ratio': 0.4}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment.run_experiment(df_results_per_user=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-04T17:16:52.010513Z",
     "start_time": "2020-12-04T17:16:51.996566Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversion</th>\n",
       "      <th>revenue</th>\n",
       "      <th>arpu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chance_to_beat</th>\n",
       "      <td>0.246727</td>\n",
       "      <td>0.401356</td>\n",
       "      <td>0.289212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expected_loss</th>\n",
       "      <td>0.204854</td>\n",
       "      <td>90.262196</td>\n",
       "      <td>81.585327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lift</th>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>-0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diff</th>\n",
       "      <td>-0.250000</td>\n",
       "      <td>-120.000000</td>\n",
       "      <td>-80.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                conversion     revenue       arpu\n",
       "chance_to_beat    0.246727    0.401356   0.289212\n",
       "expected_loss     0.204854   90.262196  81.585327\n",
       "lift             -0.333333   -0.166667  -0.444444\n",
       "diff             -0.250000 -120.000000 -80.000000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(experiment.results['A']['statistics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-04T17:16:52.031314Z",
     "start_time": "2020-12-04T17:16:52.012627Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>users</th>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sales</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>paids</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revenue</th>\n",
       "      <td>600</td>\n",
       "      <td>720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conversion</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ticket</th>\n",
       "      <td>200.0</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>arpu</th>\n",
       "      <td>100.0</td>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>statistics</th>\n",
       "      <td>{'conversion': {'chance_to_beat': 0.246727, 'e...</td>\n",
       "      <td>{'conversion': {'chance_to_beat': 0.753273, 'e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ratio</th>\n",
       "      <td>0.6</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                            A  \\\n",
       "users                                                       6   \n",
       "sales                                                       3   \n",
       "paids                                                       3   \n",
       "revenue                                                   600   \n",
       "conversion                                                0.5   \n",
       "ticket                                                  200.0   \n",
       "arpu                                                    100.0   \n",
       "statistics  {'conversion': {'chance_to_beat': 0.246727, 'e...   \n",
       "ratio                                                     0.6   \n",
       "\n",
       "                                                            B  \n",
       "users                                                       4  \n",
       "sales                                                       3  \n",
       "paids                                                       3  \n",
       "revenue                                                   720  \n",
       "conversion                                               0.75  \n",
       "ticket                                                  240.0  \n",
       "arpu                                                    180.0  \n",
       "statistics  {'conversion': {'chance_to_beat': 0.753273, 'e...  \n",
       "ratio                                                     0.4  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(experiment.results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>users</th>\n",
       "      <th>sales</th>\n",
       "      <th>paids</th>\n",
       "      <th>revenue</th>\n",
       "      <th>conversion</th>\n",
       "      <th>ticket</th>\n",
       "      <th>arpu</th>\n",
       "      <th>ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>600</td>\n",
       "      <td>0.5</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>720</td>\n",
       "      <td>0.75</td>\n",
       "      <td>240.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  users sales paids revenue conversion ticket   arpu ratio\n",
       "A     6     3     3     600        0.5  200.0  100.0   0.6\n",
       "B     4     3     3     720       0.75  240.0  180.0   0.4"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df = pd.DataFrame.from_dict(experiment.results).drop('statistics')\n",
    "\n",
    "_df.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conversion</th>\n",
       "      <th>revenue</th>\n",
       "      <th>arpu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chance_to_beat</th>\n",
       "      <td>0.246727</td>\n",
       "      <td>0.401356</td>\n",
       "      <td>0.289212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expected_loss</th>\n",
       "      <td>0.204854</td>\n",
       "      <td>90.262196</td>\n",
       "      <td>81.585327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lift</th>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>-0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diff</th>\n",
       "      <td>-0.250000</td>\n",
       "      <td>-120.000000</td>\n",
       "      <td>-80.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                conversion     revenue       arpu\n",
       "chance_to_beat    0.246727    0.401356   0.289212\n",
       "expected_loss     0.204854   90.262196  81.585327\n",
       "lift             -0.333333   -0.166667  -0.444444\n",
       "diff             -0.250000 -120.000000 -80.000000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statsA = pd.DataFrame.from_dict(experiment.results['A']['statistics'])\n",
    "statsB = pd.DataFrame.from_dict(experiment.results['B']['statistics'])\n",
    "\n",
    "statsA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References to compare\n",
    "\n",
    "https://vidogreg.shinyapps.io/bayes-arpu-test/\n",
    "\n",
    "https://abtestguide.com/bayesian/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}