eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pl.read_ndjson('./experiments/base_new/log-913.ndjson', infer_schema_length=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_schema = df.schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36000"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.scan_ndjson('./experiments/base_new2/log-9*.ndjson').collect()\n",
    "\n",
    "#df = df.filter(pl.col('symmetric') == True)\n",
    "#df = df.filter(pl.col('comparison_category') == 'riod_pooled_citest')\n",
    "#df = df.filter(pl.col('llf_neg_prob_fix') == 'clipping')\n",
    "df = df.filter(pl.col('num_samples') >= 100)\n",
    "#df = df.filter(pl.col('alpha') == 0.01)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "111eb37c264243f58e44d6af074a6f19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'13ea2600-4449-40a3-ac60-f5bbfd491828': {'version…"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df = df\n",
    "_df = _df.with_columns(accuracy=pl.col('tp') + pl.col('tn'))\n",
    "\n",
    "_df = _df.with_columns(comparison_type=pl.col('name').str.slice(0,3))\n",
    "_df = _df.with_columns(comparison_variables=pl.col('name').str.slice(0,3))\n",
    "_df = _df.with_columns(comparison_type=pl.col('name').str.slice(0,3))\n",
    "\n",
    "grouping_keys = ['comparison_type', 'num_samples', 'alpha', 'comparison_category']\n",
    "by_keys = ['name', 'num_clients']\n",
    "y_key = 'accuracy'\n",
    "\n",
    "_df.plot.box(y=y_key,\n",
    "             by=by_keys,\n",
    "             groupby=grouping_keys,\n",
    "             c=by_keys[0],\n",
    "             rot=60,\n",
    "             cmap='Pastel2',\n",
    "             ylim=(-0.1,1.1),\n",
    "             legend=False,\n",
    "             widget_location='bottom'\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd786e7d3b4547629f205e52feb85bde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'ce530e4a-1eb1-485d-844a-db75dd049854': {'version…"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "__df = df.select('name', 'num_clients', 'num_samples', 'symmetric', 'alpha', 'predicted_p_vals', 'true_p_vals') \\\n",
    "    .explode('predicted_p_vals', 'true_p_vals') \\\n",
    "    .group_by('name', 'symmetric', 'num_clients', 'num_samples', 'alpha') \\\n",
    "    .agg(pl.corr('predicted_p_vals', 'true_p_vals')) \\\n",
    "    .rename({'predicted_p_vals': 'p_value_correlation'}) \n",
    "    \n",
    "__df = __df.with_columns(comparison_type=pl.col('name').str.slice(0,3))\n",
    "    \n",
    "__df.sort('num_samples').plot.line(x='num_samples', y='p_value_correlation',alpha=0.6, by='name', groupby=['comparison_type','num_clients', 'alpha'], ylim=(0.8,1.001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48c358d5882f4c5ab115fdeeb05c51b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'c60f9dff-b493-423a-bc80-854fa7c6f5bb': {'version…"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "__df = df\n",
    "\n",
    "__df = __df.with_columns(comparison_type=pl.col('name').str.slice(0,3))\n",
    "\n",
    "#_df_decision_agreements.sort('num_samples').plot.line(x='num_samples', y='accuracy',alpha=0.6, by='name', groupby=['comparison_type','num_clients', 'symmetric', 'alpha', 'tikhonov_lambda', 'llf_neg_prob_fix'], ylim=(-0.1,1.1))\n",
    "\n",
    "\n",
    "__df = __df.explode('predicted_p_vals', 'true_p_vals')\n",
    "__df = __df.with_columns(decision_agreement=((pl.col('predicted_p_vals') > pl.col('alpha')) &\n",
    "                                                               (pl.col('true_p_vals') > pl.col('alpha'))) |\n",
    "                                                               ((pl.col('predicted_p_vals') < pl.col('alpha')) &\n",
    "                                                               (pl.col('true_p_vals') < pl.col('alpha')))\n",
    "                                                               )\n",
    "\n",
    "grouping_cols = ['num_samples', 'name', 'comparison_type','num_clients', 'symmetric', 'alpha']\n",
    "\n",
    "__df = __df.group_by(grouping_cols).agg(pl.col('decision_agreement').mean())\n",
    "\n",
    "__df.sort('num_samples').plot.line(x='num_samples', y='decision_agreement',alpha=0.6, by='name', groupby=['comparison_type','num_clients', 'symmetric', 'alpha'], ylim=(0.95,1.01))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8210d102eb9486db816a232a3b3fda0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'e4b1ab5c-8e62-4a9e-9249-127293d58f0f': {'version…"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df_decision_agreements = _df\n",
    "\n",
    "#_df_decision_agreements.sort('num_samples').plot.line(x='num_samples', y='accuracy',alpha=0.6, by='name', groupby=['comparison_type','num_clients', 'symmetric', 'alpha', 'tikhonov_lambda', 'llf_neg_prob_fix'], ylim=(-0.1,1.1))\n",
    "\n",
    "\n",
    "_df_decision_agreements = _df_decision_agreements.with_columns(accuracy=pl.col('tn')+pl.col('tp'))\n",
    "\n",
    "grouping_cols = ['num_samples', 'name', 'comparison_type','num_clients', 'symmetric', 'alpha']\n",
    "\n",
    "_df_decision_agreements = _df_decision_agreements.group_by(grouping_cols).agg(pl.col('accuracy').mean())\n",
    "\n",
    "_df_decision_agreements.sort('num_samples').plot.line(x='num_samples', y='accuracy',alpha=0.6, by='name', groupby=['comparison_type','num_clients', 'symmetric', 'alpha'], ylim=(0.95,1.01))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "887a60ea461d497ba3e2dbaa69981d98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'30199118-5ae5-415e-bba7-d3021fd4592e': {'version…"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df.explode('predicted_p_vals', 'true_p_vals').plot.scatter(x='predicted_p_vals',\n",
    "                                                             y='true_p_vals',\n",
    "                                                             groupby=['name', 'num_samples', 'num_clients'],\n",
    "                                                             ylim=(-0.1,1.1),\n",
    "                                                             xlim=(-0.1,1.1),\n",
    "                                                             width=400,\n",
    "                                                             height=400,\n",
    "                                                             alpha=0.5\n",
    "                                                             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape: (2, 2)\n",
      "┌────────────────────┬────────┐\n",
      "│ decision_agreement ┆ count  │\n",
      "│ ---                ┆ ---    │\n",
      "│ bool               ┆ u32    │\n",
      "╞════════════════════╪════════╡\n",
      "│ false              ┆ 2168   │\n",
      "│ true               ┆ 501832 │\n",
      "└────────────────────┴────────┘\n",
      "shape: (8, 2)\n",
      "┌─────────────────┬─────┐\n",
      "│ name            ┆ len │\n",
      "│ ---             ┆ --- │\n",
      "│ str             ┆ u32 │\n",
      "╞═════════════════╪═════╡\n",
      "│ L-M Con. Indep. ┆ 574 │\n",
      "│ L-M Con. Dep.   ┆ 552 │\n",
      "│ L-B Con. Dep.   ┆ 412 │\n",
      "│ L-B Con. Indep. ┆ 359 │\n",
      "│ L-M Unc. Dep.   ┆ 83  │\n",
      "│ L-B Unc. Dep.   ┆ 78  │\n",
      "│ L-M Unc. Indep. ┆ 70  │\n",
      "│ L-B Unc. Indep. ┆ 40  │\n",
      "└─────────────────┴─────┘\n",
      "shape: (3, 2)\n",
      "┌─────────────┬──────┐\n",
      "│ num_clients ┆ len  │\n",
      "│ ---         ┆ ---  │\n",
      "│ i64         ┆ u32  │\n",
      "╞═════════════╪══════╡\n",
      "│ 5           ┆ 1160 │\n",
      "│ 3           ┆ 764  │\n",
      "│ 1           ┆ 244  │\n",
      "└─────────────┴──────┘\n",
      "shape: (203, 4)\n",
      "┌─────────────────┬─────────────┬─────────────┬─────┐\n",
      "│ name            ┆ num_samples ┆ num_clients ┆ len │\n",
      "│ ---             ┆ ---         ┆ ---         ┆ --- │\n",
      "│ str             ┆ i64         ┆ i64         ┆ u32 │\n",
      "╞═════════════════╪═════════════╪═════════════╪═════╡\n",
      "│ L-M Con. Indep. ┆ 100         ┆ 5           ┆ 107 │\n",
      "│ L-M Con. Dep.   ┆ 100         ┆ 5           ┆ 99  │\n",
      "│ L-M Con. Dep.   ┆ 100         ┆ 3           ┆ 77  │\n",
      "│ L-B Con. Dep.   ┆ 100         ┆ 5           ┆ 74  │\n",
      "│ L-M Con. Indep. ┆ 100         ┆ 3           ┆ 67  │\n",
      "│ …               ┆ …           ┆ …           ┆ …   │\n",
      "│ L-M Unc. Indep. ┆ 200         ┆ 3           ┆ 1   │\n",
      "│ L-B Unc. Indep. ┆ 1000        ┆ 5           ┆ 1   │\n",
      "│ L-M Unc. Indep. ┆ 400         ┆ 3           ┆ 1   │\n",
      "│ L-B Unc. Indep. ┆ 900         ┆ 3           ┆ 1   │\n",
      "│ L-B Unc. Dep.   ┆ 900         ┆ 3           ┆ 1   │\n",
      "└─────────────────┴─────────────┴─────────────┴─────┘\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9640cf7a2e354e1480c0a7f5e99bbe6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'a9e07e77-ebda-4cf6-8ecf-cb985a9bcc52': {'version…"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "__df = _df\n",
    "__df = __df.explode('predicted_p_vals', 'true_p_vals')\n",
    "__df = __df.with_columns(decision_agreement=((pl.col('predicted_p_vals') > pl.col('alpha')) &\n",
    "                                            (pl.col('true_p_vals') > pl.col('alpha'))) |\n",
    "                                            ((pl.col('predicted_p_vals') < pl.col('alpha')) &\n",
    "                                            (pl.col('true_p_vals') < pl.col('alpha')))\n",
    "                                            )\n",
    "\n",
    "\n",
    "print(__df['decision_agreement'].value_counts())\n",
    "print(__df.filter(~pl.col('decision_agreement')).group_by('name').len().sort('len', descending=True))\n",
    "print(__df.filter(~pl.col('decision_agreement')).group_by('num_clients').len().sort('len', descending=True))\n",
    "print(__df.filter(~pl.col('decision_agreement')).group_by('name', 'num_samples', 'num_clients').len().sort('len', descending=True))\n",
    "\n",
    "__df.filter(~pl.col('decision_agreement')).plot.scatter(x='predicted_p_vals',\n",
    "                y='true_p_vals',\n",
    "                #by='decision_agreement',\n",
    "                #by=['name', 'num_samples', 'num_clients'],\n",
    "                groupby=['name', 'num_samples', 'num_clients'],#, 'llf_neg_prob_fix', 'alpha'],\n",
    "                ylim=(-0.1,1.1),\n",
    "                xlim=(-0.1,1.1),\n",
    "                width=400,\n",
    "                height=400,\n",
    "                alpha=0.5\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_schema' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df \u001b[38;5;241m=\u001b[39m pl\u001b[38;5;241m.\u001b[39mscan_ndjson(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./experiments/base_new/log-9*.ndjson\u001b[39m\u001b[38;5;124m'\u001b[39m, schema\u001b[38;5;241m=\u001b[39m\u001b[43mdf_schema\u001b[49m)\u001b[38;5;241m.\u001b[39mcollect()\n\u001b[1;32m      3\u001b[0m df \u001b[38;5;241m=\u001b[39m df\u001b[38;5;241m.\u001b[39mfilter(pl\u001b[38;5;241m.\u001b[39mcol(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnum_samples\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m100\u001b[39m)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m#df = df.filter(pl.col('alpha') == 0.01)\u001b[39;00m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df_schema' is not defined"
     ]
    }
   ],
   "source": [
    "df = pl.scan_ndjson('./experiments/base_new/log-9*.ndjson', schema=df_schema).collect()\n",
    "\n",
    "df = df.filter(pl.col('num_samples') >= 100)\n",
    "#df = df.filter(pl.col('alpha') == 0.01)\n",
    "\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb921a8086c34540a9cbff35940a00cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'a7fedc69-c50a-4e0d-b74b-f5121ade7a98': {'version…"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df = df\n",
    "_df = _df.explode('predicted_p_vals', 'true_p_vals')\n",
    "_df = _df.with_columns(decision_agreement=((pl.col('predicted_p_vals') > pl.col('alpha')) &\n",
    "                                            (pl.col('true_p_vals') > pl.col('alpha'))) |\n",
    "                                            ((pl.col('predicted_p_vals') < pl.col('alpha')) &\n",
    "                                            (pl.col('true_p_vals') < pl.col('alpha')))\n",
    "                                            )\n",
    "\n",
    "_df = _df.with_columns(comparison_variables=pl.col('name').str.slice(0,3))\n",
    "_df = _df.with_columns(comparison_type=pl.col('name').str.slice(4))\n",
    "\n",
    "_df = _df.group_by(['num_samples', 'comparison_variables', 'comparison_type', 'num_clients', 'alpha']).agg(pl.col('decision_agreement').mean())\n",
    "\n",
    "#cat_dtype = pl.Enum(('L-B', 'L-M', 'L-O', 'B-O', 'M-O'))\n",
    "#_df = _df.with_columns(pl.col('comparison_variables').cast(cat_dtype))\n",
    "\n",
    "_df = _df.sort('comparison_variables','num_samples')\n",
    "\n",
    "\n",
    "_df.plot.line(x='num_samples',\n",
    "                y='decision_agreement',\n",
    "                alpha=0.8,\n",
    "                #line_dash=dash_patterns,\n",
    "                #cmap=colrs,\n",
    "                #line_width=3,\n",
    "                by='comparison_variables',\n",
    "                groupby=['comparison_type','num_clients', 'alpha'],\n",
    "                ylim=(0.95,1.001)) #* _df.plot.errorbars(y='decision_agreement', yerr1='std')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd00e4623aaa425c88aa701f95bd8e4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "BokehModel(combine_events=True, render_bundle={'docs_json': {'54a69991-fc8f-4d3c-811d-c2df38f8662b': {'version…"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df = df.select('name', 'num_clients', 'num_samples', 'symmetric', 'alpha', 'predicted_p_vals', 'true_p_vals') \\\n",
    "    .explode('predicted_p_vals', 'true_p_vals') \\\n",
    "    .group_by('name', 'num_clients', 'num_samples', 'alpha') \\\n",
    "    .agg(pl.corr('predicted_p_vals', 'true_p_vals')) \\\n",
    "    .rename({'predicted_p_vals': 'p_value_correlation'}) \n",
    "    \n",
    "_df = _df.with_columns(comparison_variables=pl.col('name').str.slice(0,3))\n",
    "_df = _df.with_columns(comparison_type=pl.col('name').str.slice(4))\n",
    "    \n",
    "_df.sort('num_samples').plot.line(x='num_samples',\n",
    "                                  y='p_value_correlation',\n",
    "                                  alpha=0.6,\n",
    "                                  by='comparison_variables',\n",
    "                                  groupby=['comparison_type','num_clients', 'alpha'],\n",
    "                                  ylim=(0.95,1.001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "'decision_agreement' is not in list",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[39], line 11\u001b[0m\n\u001b[1;32m      6\u001b[0m _df \u001b[38;5;241m=\u001b[39m _df\u001b[38;5;241m.\u001b[39mwith_columns(comparison_type\u001b[38;5;241m=\u001b[39mpl\u001b[38;5;241m.\u001b[39mcol(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mslice(\u001b[38;5;241m4\u001b[39m))\n\u001b[1;32m      8\u001b[0m \u001b[38;5;66;03m#_df = _df.with_columns(pl.col(''))\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m \u001b[43m_df\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnum_samples\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mplot\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mline\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnum_samples\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdecision_agreement\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.8\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mby\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcomparison_variables\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     15\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mgroupby\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcomparison_type\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnum_clients\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43malpha\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     16\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mylim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m0.01\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m1.001\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/promotion/lib/python3.10/site-packages/hvplot/plotting/core.py:370\u001b[0m, in \u001b[0;36mhvPlotTabular.line\u001b[0;34m(self, x, y, **kwds)\u001b[0m\n\u001b[1;32m    275\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mline\u001b[39m(\u001b[38;5;28mself\u001b[39m, x\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, y\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds):\n\u001b[1;32m    276\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    277\u001b[0m \u001b[38;5;124;03m    The `line` plot connects the points with a continuous curve.\u001b[39;00m\n\u001b[1;32m    278\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    368\u001b[0m \u001b[38;5;124;03m    - Wiki: https://en.wikipedia.org/wiki/Line_chart\u001b[39;00m\n\u001b[1;32m    369\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 370\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mline\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/promotion/lib/python3.10/site-packages/hvplot/plotting/core.py:95\u001b[0m, in \u001b[0;36mhvPlotBase.__call__\u001b[0;34m(self, x, y, kind, **kwds)\u001b[0m\n\u001b[1;32m     92\u001b[0m         plot \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_converter(x, y, kind, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)(kind, x, y)\n\u001b[1;32m     93\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m pn\u001b[38;5;241m.\u001b[39mpanel(plot, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mpanel_dict)\n\u001b[0;32m---> 95\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_converter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m(kind, x, y)\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/promotion/lib/python3.10/site-packages/hvplot/plotting/core.py:1899\u001b[0m, in \u001b[0;36mhvPlotTabularPolars._get_converter\u001b[0;34m(self, x, y, kind, **kwds)\u001b[0m\n\u001b[1;32m   1897\u001b[0m         columns\u001b[38;5;241m.\u001b[39mdiscard(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m   1898\u001b[0m         \u001b[38;5;66;03m# Reorder the columns as in the data.\u001b[39;00m\n\u001b[0;32m-> 1899\u001b[0m         columns \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msorted\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mc\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_data\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[43m(\u001b[49m\u001b[43mc\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1901\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data, pl\u001b[38;5;241m.\u001b[39mDataFrame):\n\u001b[1;32m   1902\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data\u001b[38;5;241m.\u001b[39mselect(columns)\u001b[38;5;241m.\u001b[39mto_pandas()\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/promotion/lib/python3.10/site-packages/hvplot/plotting/core.py:1899\u001b[0m, in \u001b[0;36mhvPlotTabularPolars._get_converter.<locals>.<lambda>\u001b[0;34m(c)\u001b[0m\n\u001b[1;32m   1897\u001b[0m         columns\u001b[38;5;241m.\u001b[39mdiscard(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m   1898\u001b[0m         \u001b[38;5;66;03m# Reorder the columns as in the data.\u001b[39;00m\n\u001b[0;32m-> 1899\u001b[0m         columns \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msorted\u001b[39m(columns, key\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mlambda\u001b[39;00m c: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_data\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[43m(\u001b[49m\u001b[43mc\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   1901\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data, pl\u001b[38;5;241m.\u001b[39mDataFrame):\n\u001b[1;32m   1902\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data\u001b[38;5;241m.\u001b[39mselect(columns)\u001b[38;5;241m.\u001b[39mto_pandas()\n",
      "\u001b[0;31mValueError\u001b[0m: 'decision_agreement' is not in list"
     ]
    }
   ],
   "source": [
    "_df = df\n",
    "_df = _df.explode('predicted_p_vals', 'true_p_vals')\n",
    "\n",
    "\n",
    "_df = _df.with_columns(comparison_variables=pl.col('name').str.slice(0,3))\n",
    "_df = _df.with_columns(comparison_type=pl.col('name').str.slice(4))\n",
    "\n",
    "#_df = _df.with_columns(pl.col(''))\n",
    "\n",
    "\n",
    "_df.sort('num_samples').plot.line(x='num_samples',\n",
    "                                   y='decision_agreement',\n",
    "                                   alpha=0.8,\n",
    "                                   by='comparison_variables',\n",
    "                                   groupby=['comparison_type','num_clients', 'alpha'],\n",
    "                                   ylim=(-0.01,1.001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PLOTS FROM PAPER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "__df = df.select('name', 'num_clients', 'num_samples', 'symmetric', 'alpha', 'tikhonov_lambda', 'llf_neg_prob_fix', 'predicted_p_vals', 'true_p_vals') \\\n",
    "    .explode('predicted_p_vals', 'true_p_vals') \\\n",
    "    .group_by('name', 'symmetric', 'num_clients', 'num_samples', 'alpha', 'tikhonov_lambda', 'llf_neg_prob_fix') \\\n",
    "    .agg(pl.corr('predicted_p_vals', 'true_p_vals')) \\\n",
    "    .rename({'predicted_p_vals': 'p_value_correlation'}) \n",
    "    \n",
    "__df = __df.with_columns(comparison_type=pl.col('name').str.slice(0,3))\n",
    "    \n",
    "__df.sort('num_samples').plot.line(x='num_samples', y='p_value_correlation',alpha=0.6, by='name', groupby=['comparison_type','num_clients', 'alpha', 'tikhonov_lambda', 'llf_neg_prob_fix'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for file in glob.glob('./experiments/base/*.ndjson'):\n",
    "#     #print(file)\n",
    "#     df = pl.read_ndjson(file)\n",
    "#     #print(len(df))\n",
    "#     df = df.filter(pl.col('num_samples') >= 100)\n",
    "#     #print(len(df))\n",
    "#     #break\n",
    "#     df = df.write_ndjson(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "promotion",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}