tutorials/tutorial-corpus-graph.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IR Lab Tutorial: Document Expansion with DocT5Query\n",
    "\n",
    "TBD..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import All Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tira.third_party_integrations import ensure_pyterrier_is_loaded\n",
    "from tira.rest_api_client import Client\n",
    "ensure_pyterrier_is_loaded()\n",
    "import pyterrier as pt\n",
    "\n",
    "tira = Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pt.get_dataset(f'irds:antique/test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>label</th>\n",
       "      <th>iteration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1964316_5</td>\n",
       "      <td>4</td>\n",
       "      <td>U0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1674088_11</td>\n",
       "      <td>1</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1218838_13</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1519022_15</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1964316</td>\n",
       "      <td>3059341_5</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6584</th>\n",
       "      <td>1262692</td>\n",
       "      <td>247023_6</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6585</th>\n",
       "      <td>1262692</td>\n",
       "      <td>1499030_5</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6586</th>\n",
       "      <td>1262692</td>\n",
       "      <td>2916758_0</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6587</th>\n",
       "      <td>1262692</td>\n",
       "      <td>1105845_15</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6588</th>\n",
       "      <td>1262692</td>\n",
       "      <td>3699008_1</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6589 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          qid       docno  label iteration\n",
       "0     1964316   1964316_5      4        U0\n",
       "1     1964316  1674088_11      1        Q0\n",
       "2     1964316  1218838_13      2        Q0\n",
       "3     1964316  1519022_15      2        Q0\n",
       "4     1964316   3059341_5      2        Q0\n",
       "...       ...         ...    ...       ...\n",
       "6584  1262692    247023_6      3        Q0\n",
       "6585  1262692   1499030_5      3        Q0\n",
       "6586  1262692   2916758_0      3        Q0\n",
       "6587  1262692  1105845_15      3        Q0\n",
       "6588  1262692   3699008_1      2        Q0\n",
       "\n",
       "[6589 rows x 4 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qrels = dataset.get_qrels()\n",
    "\n",
    "qrels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "qrels_dict = {}\n",
    "\n",
    "for _, i in qrels.iterrows():\n",
    "    if i['qid'] not in qrels_dict:\n",
    "        qrels_dict[i['qid']] = {}\n",
    "    qrels_dict[i['qid']][i['docno']] = i['label']\n",
    "\n",
    "def count_relevant_neighbours(entry, relevance_level=1):\n",
    "    ret = 0\n",
    "    for neighbor in entry['neighbors']:\n",
    "        if qrels_dict[entry['qid']].get(neighbor, 0) >= relevance_level:\n",
    "            ret += 1\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download from the Incubator: https://files.webis.de/data-in-production/data-research/tira-zenodo-dump-preparation/corpus-graph/2024-03-21-15-00-49.zip\n",
      "\tThis is only used for last spot checks before archival to Zenodo.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 100%|██████████| 27.1M/27.1M [00:04<00:00, 6.83MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/antique-test-20230107-training/seanmacavaney\n"
     ]
    }
   ],
   "source": [
    "corpus_graph = tira.pt.transform_documents('ir-benchmarks/seanmacavaney/corpus-graph', dataset) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>label</th>\n",
       "      <th>iteration</th>\n",
       "      <th>neighbors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1964316_5</td>\n",
       "      <td>4</td>\n",
       "      <td>U0</td>\n",
       "      <td>[2245059_0, 1964316_2, 2273802_1, 3786452_1, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1674088_11</td>\n",
       "      <td>1</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[230048_7, 1136440_0, 3784253_16, 3786452_1, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1218838_13</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3786452_1, 3059341_5, 1248144_1, 1674088_11, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1519022_15</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3900143_6, 1519022_8, 3972356_5, 92480_12, 19...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1964316</td>\n",
       "      <td>3059341_5</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3786452_1, 1218838_13, 1248144_1, 1674088_11,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6584</th>\n",
       "      <td>1262692</td>\n",
       "      <td>247023_6</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3196461_3, 2338307_4, 4428688_2, 2910592_6, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6585</th>\n",
       "      <td>1262692</td>\n",
       "      <td>1499030_5</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[317469_9, 1134300_6, 4475151_0, 3982441_4, 39...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6586</th>\n",
       "      <td>1262692</td>\n",
       "      <td>2916758_0</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[1033541_0, 1134300_0, 2916758_11, 766628_0, 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6587</th>\n",
       "      <td>1262692</td>\n",
       "      <td>1105845_15</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[276463_4, 2917422_3, 247806_6, 317469_5, 9744...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6588</th>\n",
       "      <td>1262692</td>\n",
       "      <td>3699008_1</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[942039_0, 2744068_1, 1134300_5, 824738_5, 190...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6585 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          qid       docno  label iteration  \\\n",
       "0     1964316   1964316_5      4        U0   \n",
       "1     1964316  1674088_11      1        Q0   \n",
       "2     1964316  1218838_13      2        Q0   \n",
       "3     1964316  1519022_15      2        Q0   \n",
       "4     1964316   3059341_5      2        Q0   \n",
       "...       ...         ...    ...       ...   \n",
       "6584  1262692    247023_6      3        Q0   \n",
       "6585  1262692   1499030_5      3        Q0   \n",
       "6586  1262692   2916758_0      3        Q0   \n",
       "6587  1262692  1105845_15      3        Q0   \n",
       "6588  1262692   3699008_1      2        Q0   \n",
       "\n",
       "                                              neighbors  \n",
       "0     [2245059_0, 1964316_2, 2273802_1, 3786452_1, 2...  \n",
       "1     [230048_7, 1136440_0, 3784253_16, 3786452_1, 2...  \n",
       "2     [3786452_1, 3059341_5, 1248144_1, 1674088_11, ...  \n",
       "3     [3900143_6, 1519022_8, 3972356_5, 92480_12, 19...  \n",
       "4     [3786452_1, 1218838_13, 1248144_1, 1674088_11,...  \n",
       "...                                                 ...  \n",
       "6584  [3196461_3, 2338307_4, 4428688_2, 2910592_6, 1...  \n",
       "6585  [317469_9, 1134300_6, 4475151_0, 3982441_4, 39...  \n",
       "6586  [1033541_0, 1134300_0, 2916758_11, 766628_0, 3...  \n",
       "6587  [276463_4, 2917422_3, 247806_6, 317469_5, 9744...  \n",
       "6588  [942039_0, 2744068_1, 1134300_5, 824738_5, 190...  \n",
       "\n",
       "[6585 rows x 5 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if there is no neighbour, we have no entry in the corpus graph (maybe should allow default values in transform documents\n",
    "\n",
    "missing_neighbours = ('3862765_0', '4372730_24', '601023_0', '2615234_3')\n",
    "\n",
    "qrels_with_neighbors = corpus_graph(qrels.copy()[~qrels['docno'].isin(missing_neighbours)])\n",
    "qrels_with_neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>label</th>\n",
       "      <th>iteration</th>\n",
       "      <th>neighbors</th>\n",
       "      <th>relevant_neighbors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1964316_5</td>\n",
       "      <td>4</td>\n",
       "      <td>U0</td>\n",
       "      <td>[2245059_0, 1964316_2, 2273802_1, 3786452_1, 2...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1674088_11</td>\n",
       "      <td>1</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[230048_7, 1136440_0, 3784253_16, 3786452_1, 2...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1218838_13</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3786452_1, 3059341_5, 1248144_1, 1674088_11, ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1964316</td>\n",
       "      <td>1519022_15</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3900143_6, 1519022_8, 3972356_5, 92480_12, 19...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1964316</td>\n",
       "      <td>3059341_5</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3786452_1, 1218838_13, 1248144_1, 1674088_11,...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6584</th>\n",
       "      <td>1262692</td>\n",
       "      <td>247023_6</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[3196461_3, 2338307_4, 4428688_2, 2910592_6, 1...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6585</th>\n",
       "      <td>1262692</td>\n",
       "      <td>1499030_5</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[317469_9, 1134300_6, 4475151_0, 3982441_4, 39...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6586</th>\n",
       "      <td>1262692</td>\n",
       "      <td>2916758_0</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[1033541_0, 1134300_0, 2916758_11, 766628_0, 3...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6587</th>\n",
       "      <td>1262692</td>\n",
       "      <td>1105845_15</td>\n",
       "      <td>3</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[276463_4, 2917422_3, 247806_6, 317469_5, 9744...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6588</th>\n",
       "      <td>1262692</td>\n",
       "      <td>3699008_1</td>\n",
       "      <td>2</td>\n",
       "      <td>Q0</td>\n",
       "      <td>[942039_0, 2744068_1, 1134300_5, 824738_5, 190...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6585 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          qid       docno  label iteration  \\\n",
       "0     1964316   1964316_5      4        U0   \n",
       "1     1964316  1674088_11      1        Q0   \n",
       "2     1964316  1218838_13      2        Q0   \n",
       "3     1964316  1519022_15      2        Q0   \n",
       "4     1964316   3059341_5      2        Q0   \n",
       "...       ...         ...    ...       ...   \n",
       "6584  1262692    247023_6      3        Q0   \n",
       "6585  1262692   1499030_5      3        Q0   \n",
       "6586  1262692   2916758_0      3        Q0   \n",
       "6587  1262692  1105845_15      3        Q0   \n",
       "6588  1262692   3699008_1      2        Q0   \n",
       "\n",
       "                                              neighbors  relevant_neighbors  \n",
       "0     [2245059_0, 1964316_2, 2273802_1, 3786452_1, 2...                   3  \n",
       "1     [230048_7, 1136440_0, 3784253_16, 3786452_1, 2...                   7  \n",
       "2     [3786452_1, 3059341_5, 1248144_1, 1674088_11, ...                   7  \n",
       "3     [3900143_6, 1519022_8, 3972356_5, 92480_12, 19...                   1  \n",
       "4     [3786452_1, 1218838_13, 1248144_1, 1674088_11,...                   7  \n",
       "...                                                 ...                 ...  \n",
       "6584  [3196461_3, 2338307_4, 4428688_2, 2910592_6, 1...                   2  \n",
       "6585  [317469_9, 1134300_6, 4475151_0, 3982441_4, 39...                   2  \n",
       "6586  [1033541_0, 1134300_0, 2916758_11, 766628_0, 3...                   2  \n",
       "6587  [276463_4, 2917422_3, 247806_6, 317469_5, 9744...                   0  \n",
       "6588  [942039_0, 2744068_1, 1134300_5, 824738_5, 190...                   2  \n",
       "\n",
       "[6585 rows x 6 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qrels_with_neighbors['relevant_neighbors'] = qrels_with_neighbors.apply(count_relevant_neighbours, axis=1)\n",
    "qrels_with_neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6585.000000\n",
       "mean        1.991192\n",
       "std         2.747900\n",
       "min         0.000000\n",
       "25%         0.000000\n",
       "50%         1.000000\n",
       "75%         3.000000\n",
       "max        15.000000\n",
       "Name: relevant_neighbors, dtype: float64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# on average 2 relevant document retrieved via corpus graph for docs in the qrels\n",
    "qrels_with_neighbors['relevant_neighbors'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    4943.000000\n",
       "mean        2.257536\n",
       "std         2.854886\n",
       "min         0.000000\n",
       "25%         0.000000\n",
       "50%         1.000000\n",
       "75%         3.000000\n",
       "max        15.000000\n",
       "Name: relevant_neighbors, dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# on average 2.25 relevant document retrieved via corpus graph for docs with relevance > 1 in the qrels\n",
    "qrels_with_neighbors[qrels_with_neighbors['label'] > 1]['relevant_neighbors'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1334.000000\n",
       "mean        2.461769\n",
       "std         2.910976\n",
       "min         0.000000\n",
       "25%         0.000000\n",
       "50%         1.000000\n",
       "75%         4.000000\n",
       "max        15.000000\n",
       "Name: relevant_neighbors, dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# on average 2.46 relevant document retrieved via corpus graph for docs with relevance > 3 in the qrels\n",
    "qrels_with_neighbors[qrels_with_neighbors['label'] > 3]['relevant_neighbors'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis on LongEval\n",
    "\n",
    "As preparation for the IR Lab in Padua"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# not yet part of the official main branch of ir_datasets, so we load from tira\n",
    "\n",
    "dataset = pt.get_dataset(f'irds:ir-benchmarks/longeval-train-20230513-training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>label</th>\n",
       "      <th>iteration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200112743</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200205250</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200101983</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200204465</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200115614</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9651</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200205276</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9652</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200107121</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9653</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200204419</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9654</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200103774</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9655</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200110087</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9656 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             qid            docno  label iteration\n",
       "0      q06223196  doc062200112743      0         0\n",
       "1      q06223196  doc062200205250      0         0\n",
       "2      q06223196  doc062200101983      0         0\n",
       "3      q06223196  doc062200204465      1         0\n",
       "4      q06223196  doc062200115614      0         0\n",
       "...          ...              ...    ...       ...\n",
       "9651  q062225197  doc062200205276      0         0\n",
       "9652  q062225197  doc062200107121      1         0\n",
       "9653  q062225197  doc062200204419      0         0\n",
       "9654  q062225197  doc062200103774      0         0\n",
       "9655  q062225197  doc062200110087      0         0\n",
       "\n",
       "[9656 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qrels = dataset.get_qrels()\n",
    "\n",
    "qrels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download from the Incubator: https://files.webis.de/data-in-production/data-research/tira-zenodo-dump-preparation/corpus-graph/2024-03-21-12-46-50.zip\n",
      "\tThis is only used for last spot checks before archival to Zenodo.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 100%|██████████| 93.2M/93.2M [00:09<00:00, 10.8MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/longeval-train-20230513-training/seanmacavaney\n"
     ]
    }
   ],
   "source": [
    "corpus_graph = tira.pt.transform_documents('ir-benchmarks/seanmacavaney/corpus-graph', 'longeval-train-20230513-training') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "qrels_dict = {}\n",
    "\n",
    "for _, i in qrels.iterrows():\n",
    "    if i['qid'] not in qrels_dict:\n",
    "        qrels_dict[i['qid']] = {}\n",
    "    qrels_dict[i['qid']][i['docno']] = i['label']\n",
    "\n",
    "def count_relevant_neighbours(entry, relevance_level=1):\n",
    "    ret = 0\n",
    "    for neighbor in entry['neighbors']:\n",
    "        if qrels_dict[entry['qid']].get(neighbor, 0) >= relevance_level:\n",
    "            ret += 1\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>label</th>\n",
       "      <th>iteration</th>\n",
       "      <th>neighbors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200204465</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200401429, doc062200106171, doc06221110...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200205493</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062201708464, doc062200115614, doc06220500...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>q062228</td>\n",
       "      <td>doc062200116555</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062208807613, doc062208706096, doc06221040...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>q062228</td>\n",
       "      <td>doc062200116273</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200100875, doc062201300294, doc06220380...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>q062287</td>\n",
       "      <td>doc062200209981</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200116769, doc062215804698, doc06220010...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9636</th>\n",
       "      <td>q062225194</td>\n",
       "      <td>doc062200201379</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062208607784, doc062200116561, doc06220830...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9640</th>\n",
       "      <td>q062225194</td>\n",
       "      <td>doc062200205011</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062208406169, doc062208603880, doc06220860...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9641</th>\n",
       "      <td>q062225194</td>\n",
       "      <td>doc062200204433</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200112015, doc062200113353, doc06220841...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9647</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200207538</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062202104464, doc062201710012, doc06220200...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9652</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200107121</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062201902653, doc062200117385, doc06220880...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2626 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             qid            docno  label iteration  \\\n",
       "3      q06223196  doc062200204465      1         0   \n",
       "10     q06223196  doc062200205493      1         0   \n",
       "20       q062228  doc062200116555      1         0   \n",
       "21       q062228  doc062200116273      2         0   \n",
       "25       q062287  doc062200209981      1         0   \n",
       "...          ...              ...    ...       ...   \n",
       "9636  q062225194  doc062200201379      2         0   \n",
       "9640  q062225194  doc062200205011      1         0   \n",
       "9641  q062225194  doc062200204433      1         0   \n",
       "9647  q062225197  doc062200207538      1         0   \n",
       "9652  q062225197  doc062200107121      1         0   \n",
       "\n",
       "                                              neighbors  \n",
       "3     [doc062200401429, doc062200106171, doc06221110...  \n",
       "10    [doc062201708464, doc062200115614, doc06220500...  \n",
       "20    [doc062208807613, doc062208706096, doc06221040...  \n",
       "21    [doc062200100875, doc062201300294, doc06220380...  \n",
       "25    [doc062200116769, doc062215804698, doc06220010...  \n",
       "...                                                 ...  \n",
       "9636  [doc062208607784, doc062200116561, doc06220830...  \n",
       "9640  [doc062208406169, doc062208603880, doc06220860...  \n",
       "9641  [doc062200112015, doc062200113353, doc06220841...  \n",
       "9647  [doc062202104464, doc062201710012, doc06220200...  \n",
       "9652  [doc062201902653, doc062200117385, doc06220880...  \n",
       "\n",
       "[2626 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if there is no neighbour, we have no entry in the corpus graph (maybe should allow default values in transform documents\n",
    "qrels_with_neighbors = corpus_graph(qrels[qrels['label'] > 0].copy())\n",
    "qrels_with_neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>label</th>\n",
       "      <th>iteration</th>\n",
       "      <th>neighbors</th>\n",
       "      <th>relevant_neighbors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200204465</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200401429, doc062200106171, doc06221110...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>doc062200205493</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062201708464, doc062200115614, doc06220500...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>q062228</td>\n",
       "      <td>doc062200116555</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062208807613, doc062208706096, doc06221040...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>q062228</td>\n",
       "      <td>doc062200116273</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200100875, doc062201300294, doc06220380...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>q062287</td>\n",
       "      <td>doc062200209981</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200116769, doc062215804698, doc06220010...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9636</th>\n",
       "      <td>q062225194</td>\n",
       "      <td>doc062200201379</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062208607784, doc062200116561, doc06220830...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9640</th>\n",
       "      <td>q062225194</td>\n",
       "      <td>doc062200205011</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062208406169, doc062208603880, doc06220860...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9641</th>\n",
       "      <td>q062225194</td>\n",
       "      <td>doc062200204433</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062200112015, doc062200113353, doc06220841...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9647</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200207538</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062202104464, doc062201710012, doc06220200...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9652</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>doc062200107121</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[doc062201902653, doc062200117385, doc06220880...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2626 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             qid            docno  label iteration  \\\n",
       "3      q06223196  doc062200204465      1         0   \n",
       "10     q06223196  doc062200205493      1         0   \n",
       "20       q062228  doc062200116555      1         0   \n",
       "21       q062228  doc062200116273      2         0   \n",
       "25       q062287  doc062200209981      1         0   \n",
       "...          ...              ...    ...       ...   \n",
       "9636  q062225194  doc062200201379      2         0   \n",
       "9640  q062225194  doc062200205011      1         0   \n",
       "9641  q062225194  doc062200204433      1         0   \n",
       "9647  q062225197  doc062200207538      1         0   \n",
       "9652  q062225197  doc062200107121      1         0   \n",
       "\n",
       "                                              neighbors  relevant_neighbors  \n",
       "3     [doc062200401429, doc062200106171, doc06221110...                   0  \n",
       "10    [doc062201708464, doc062200115614, doc06220500...                   0  \n",
       "20    [doc062208807613, doc062208706096, doc06221040...                   0  \n",
       "21    [doc062200100875, doc062201300294, doc06220380...                   0  \n",
       "25    [doc062200116769, doc062215804698, doc06220010...                   0  \n",
       "...                                                 ...                 ...  \n",
       "9636  [doc062208607784, doc062200116561, doc06220830...                   0  \n",
       "9640  [doc062208406169, doc062208603880, doc06220860...                   0  \n",
       "9641  [doc062200112015, doc062200113353, doc06220841...                   1  \n",
       "9647  [doc062202104464, doc062201710012, doc06220200...                   0  \n",
       "9652  [doc062201902653, doc062200117385, doc06220880...                   0  \n",
       "\n",
       "[2626 rows x 6 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qrels_with_neighbors['relevant_neighbors'] = qrels_with_neighbors.apply(count_relevant_neighbours, axis=1)\n",
    "qrels_with_neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "656"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "len(qrels_with_neighbors['qid'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.009160305343512"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# On average 4 relevant documents per query\n",
    "2626/655"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2626.000000\n",
       "mean        0.610815\n",
       "std         0.941377\n",
       "min         0.000000\n",
       "25%         0.000000\n",
       "50%         0.000000\n",
       "75%         1.000000\n",
       "max         7.000000\n",
       "Name: relevant_neighbors, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# on average 0.6 relevant document retrieved via corpus graph for docs in the qrels\n",
    "# Reasonable, as there are only 4 relevant documents per query on average, but if you find one, you are likely to find \"0.6 more\" via the corpus graph, which is especially reasonable as you can aggregate this over multiple top results per query.\n",
    "qrels_with_neighbors['relevant_neighbors'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 1.11MiB [00:00, 7.31MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/longeval-train-20230513-training/tira-ir-starter\n"
     ]
    }
   ],
   "source": [
    "#Usually, you apply it to some tranformer-based model...\n",
    "monot5 = tira.pt.from_submission(\"ir-benchmarks/tira-ir-starter/MonoT5 Base (tira-ir-starter-gygaggle)\", \"longeval-train-20230513-training\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>query</th>\n",
       "      <th>q0</th>\n",
       "      <th>rank</th>\n",
       "      <th>score</th>\n",
       "      <th>system</th>\n",
       "      <th>docno</th>\n",
       "      <th>tira_task</th>\n",
       "      <th>tira_dataset</th>\n",
       "      <th>tira_first_stage_run_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>car shelter</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.003520</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062201708464</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>car shelter</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.005353</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062200108613</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>car shelter</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.006328</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062200206319</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>car shelter</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.006333</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062200112743</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>q06223196</td>\n",
       "      <td>car shelter</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.006599</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062201708471</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66562</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>cheapest car</td>\n",
       "      <td>0</td>\n",
       "      <td>96</td>\n",
       "      <td>-10.073474</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062202202627</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66563</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>cheapest car</td>\n",
       "      <td>0</td>\n",
       "      <td>97</td>\n",
       "      <td>-10.074920</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062202005382</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66564</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>cheapest car</td>\n",
       "      <td>0</td>\n",
       "      <td>98</td>\n",
       "      <td>-10.288811</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062202002893</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66565</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>cheapest car</td>\n",
       "      <td>0</td>\n",
       "      <td>99</td>\n",
       "      <td>-10.347551</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062214906085</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66566</th>\n",
       "      <td>q062225197</td>\n",
       "      <td>cheapest car</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>-10.347551</td>\n",
       "      <td>castorini/monot5-base-msmarco-10k</td>\n",
       "      <td>doc062214903805</td>\n",
       "      <td>ir-benchmarks</td>\n",
       "      <td>longeval-train-20230513-training</td>\n",
       "      <td>2024-03-18-12-56-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>66567 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              qid         query  q0  rank      score  \\\n",
       "0       q06223196   car shelter   0     1  -0.003520   \n",
       "1       q06223196   car shelter   0     2  -0.005353   \n",
       "2       q06223196   car shelter   0     3  -0.006328   \n",
       "3       q06223196   car shelter   0     4  -0.006333   \n",
       "4       q06223196   car shelter   0     5  -0.006599   \n",
       "...           ...           ...  ..   ...        ...   \n",
       "66562  q062225197  cheapest car   0    96 -10.073474   \n",
       "66563  q062225197  cheapest car   0    97 -10.074920   \n",
       "66564  q062225197  cheapest car   0    98 -10.288811   \n",
       "66565  q062225197  cheapest car   0    99 -10.347551   \n",
       "66566  q062225197  cheapest car   0   100 -10.347551   \n",
       "\n",
       "                                  system            docno      tira_task  \\\n",
       "0      castorini/monot5-base-msmarco-10k  doc062201708464  ir-benchmarks   \n",
       "1      castorini/monot5-base-msmarco-10k  doc062200108613  ir-benchmarks   \n",
       "2      castorini/monot5-base-msmarco-10k  doc062200206319  ir-benchmarks   \n",
       "3      castorini/monot5-base-msmarco-10k  doc062200112743  ir-benchmarks   \n",
       "4      castorini/monot5-base-msmarco-10k  doc062201708471  ir-benchmarks   \n",
       "...                                  ...              ...            ...   \n",
       "66562  castorini/monot5-base-msmarco-10k  doc062202202627  ir-benchmarks   \n",
       "66563  castorini/monot5-base-msmarco-10k  doc062202005382  ir-benchmarks   \n",
       "66564  castorini/monot5-base-msmarco-10k  doc062202002893  ir-benchmarks   \n",
       "66565  castorini/monot5-base-msmarco-10k  doc062214906085  ir-benchmarks   \n",
       "66566  castorini/monot5-base-msmarco-10k  doc062214903805  ir-benchmarks   \n",
       "\n",
       "                           tira_dataset tira_first_stage_run_id  \n",
       "0      longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "1      longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "2      longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "3      longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "4      longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "...                                 ...                     ...  \n",
       "66562  longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "66563  longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "66564  longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "66565  longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "66566  longeval-train-20230513-training     2024-03-18-12-56-01  \n",
       "\n",
       "[66567 rows x 10 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "monot5(dataset.get_topics('title'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToDo: now integrate the adaptive re-ranking..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}