examples/trec_29_deep_documents_preferences_depth_20.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "from sys import modules\n",
    "\n",
    "IN_COLAB = 'google.colab' in modules\n",
    "if IN_COLAB:\n",
    "    !pip install -q ir_axioms[examples] python-terrier"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "# Start/initialize PyTerrier.\n",
    "from pyterrier import started, init\n",
    "\n",
    "if not started():\n",
    "    init(tqdm=\"auto\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "edition = 29\n",
    "track = \"deep.documents\"\n",
    "dataset_name = \"msmarco-document/trec-dl-2020/judged\"\n",
    "contents_field = \"body\"\n",
    "depth = 20"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "from pyterrier.datasets import get_dataset\n",
    "from ir_datasets import load\n",
    "\n",
    "dataset = get_dataset(f\"irds:{dataset_name}\")\n",
    "ir_dataset = load(dataset_name)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "cache_dir = Path(\"cache/\")\n",
    "index_dir = cache_dir / \"indices\" / dataset_name.split(\"/\")[0]\n",
    "result_dir = Path(\n",
    "    \"/mnt/ceph/storage/data-in-progress/data-research/\"\n",
    "    \"web-search/web-search-trec/trec-system-runs\"\n",
    ") / f\"trec{edition}\" / track\n",
    "result_files = list(result_dir.iterdir())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "from pyterrier.index import IterDictIndexer\n",
    "\n",
    "if not index_dir.exists():\n",
    "    indexer = IterDictIndexer(str(index_dir.absolute()))\n",
    "    indexer.index(\n",
    "        dataset.get_corpus_iter(),\n",
    "        fields=[contents_field]\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "from pyterrier.io import read_results\n",
    "from pyterrier import Transformer\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "results = [\n",
    "    Transformer.from_df(read_results(result_file))\n",
    "    for result_file in tqdm(result_files, desc=\"Load results\")\n",
    "]\n",
    "results_names = [result_file.stem.replace(\"input.\", \"\") for result_file in result_files]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "from ir_axioms.axiom import (\n",
    "    ArgUC, QTArg, QTPArg, aSL, PROX1, PROX2, PROX3, PROX4, PROX5, TFC1, TFC3, RS_TF, RS_TF_IDF, RS_BM25, RS_PL2, RS_QL,\n",
    "    AND, LEN_AND, M_AND, LEN_M_AND, DIV, LEN_DIV, M_TDC, LEN_M_TDC, STMC1, STMC1_f, STMC2, STMC2_f, LNC1, TF_LNC, LB1,\n",
    "    REG, ANTI_REG, ASPECT_REG, REG_f, ANTI_REG_f, ASPECT_REG_f\n",
    ")\n",
    "\n",
    "axioms = [\n",
    "    ~ArgUC(),  # Very slow due to network access.\n",
    "    ~QTArg(),  # Very slow due to network access.\n",
    "    ~QTPArg(),  # Very slow due to network access.\n",
    "    ~aSL(),\n",
    "    ~LNC1(),\n",
    "    ~TF_LNC(),\n",
    "    ~LB1(),\n",
    "    ~PROX1(),\n",
    "    ~PROX2(),\n",
    "    ~PROX3(),\n",
    "    ~PROX4(),\n",
    "    ~PROX5(),\n",
    "    ~REG(),\n",
    "    ~REG_f(),\n",
    "    ~ANTI_REG(),\n",
    "    ~ANTI_REG_f(),\n",
    "    ~ASPECT_REG(),\n",
    "    ~ASPECT_REG_f(),\n",
    "    ~AND(),\n",
    "    ~LEN_AND(),\n",
    "    ~M_AND(),\n",
    "    ~LEN_M_AND(),\n",
    "    ~DIV(),\n",
    "    ~LEN_DIV(),\n",
    "    ~RS_TF(),\n",
    "    ~RS_TF_IDF(),\n",
    "    ~RS_BM25(),\n",
    "    ~RS_PL2(),\n",
    "    ~RS_QL(),\n",
    "    ~TFC1(),\n",
    "    ~TFC3(),\n",
    "    ~M_TDC(),\n",
    "    ~LEN_M_TDC(),\n",
    "    ~STMC1(),  # Rather slow due many similarity calculations.\n",
    "    ~STMC1_f(),  # Rather slow due many similarity calculations.\n",
    "    ~STMC2(),\n",
    "    ~STMC2_f(),\n",
    "]\n",
    "axiom_names = [axiom.axiom.name for axiom in axioms]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "from ir_axioms.backend.pyterrier.experiment import AxiomaticExperiment\n",
    "\n",
    "experiment = AxiomaticExperiment(\n",
    "    retrieval_systems=results,\n",
    "    topics=dataset.get_topics(),\n",
    "    qrels=dataset.get_qrels(),\n",
    "    index=index_dir,\n",
    "    dataset=ir_dataset,\n",
    "    contents_accessor=contents_field,\n",
    "    axioms=axioms,\n",
    "    axiom_names=axiom_names,\n",
    "    depth=depth,\n",
    "    filter_by_qrels=False,\n",
    "    filter_by_topics=False,\n",
    "    verbose=True,\n",
    "    cache_dir=cache_dir,\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "Computing system axiomatic preferences:   0%|          | 0/37 [00:00<?, ?system/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "8389cac5b2504aba9a213714560e9cf5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Computing query axiom preferences:   0%|          | 0/43 [00:00<?, ?query/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d9bfda8f62ba49f9a7d0a18fec74032d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Input \u001B[0;32mIn [22]\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0m preferences \u001B[38;5;241m=\u001B[39m \u001B[43maxiomatic_experiment\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreferences\u001B[49m\n\u001B[1;32m      2\u001B[0m preferences\n",
      "File \u001B[0;32m/usr/lib/python3.9/functools.py:993\u001B[0m, in \u001B[0;36mcached_property.__get__\u001B[0;34m(self, instance, owner)\u001B[0m\n\u001B[1;32m    991\u001B[0m val \u001B[38;5;241m=\u001B[39m cache\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mattrname, _NOT_FOUND)\n\u001B[1;32m    992\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m val \u001B[38;5;129;01mis\u001B[39;00m _NOT_FOUND:\n\u001B[0;32m--> 993\u001B[0m     val \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstance\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    994\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m    995\u001B[0m         cache[\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mattrname] \u001B[38;5;241m=\u001B[39m val\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/experiment.py:117\u001B[0m, in \u001B[0;36mAxiomaticExperiment.preferences\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    111\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose:\n\u001B[1;32m    112\u001B[0m     pipelines \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    113\u001B[0m         pipelines,\n\u001B[1;32m    114\u001B[0m         desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing system axiomatic preferences\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    115\u001B[0m         unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msystem\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    116\u001B[0m     )\n\u001B[0;32m--> 117\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m concat([\n\u001B[1;32m    118\u001B[0m     pipeline\u001B[38;5;241m.\u001B[39mtransform(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtopics)\n\u001B[1;32m    119\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m pipeline \u001B[38;5;129;01min\u001B[39;00m pipelines\n\u001B[1;32m    120\u001B[0m ])\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/experiment.py:118\u001B[0m, in \u001B[0;36m<listcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m    111\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose:\n\u001B[1;32m    112\u001B[0m     pipelines \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    113\u001B[0m         pipelines,\n\u001B[1;32m    114\u001B[0m         desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing system axiomatic preferences\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    115\u001B[0m         unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msystem\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    116\u001B[0m     )\n\u001B[1;32m    117\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m concat([\n\u001B[0;32m--> 118\u001B[0m     \u001B[43mpipeline\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtopics\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    119\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m pipeline \u001B[38;5;129;01min\u001B[39;00m pipelines\n\u001B[1;32m    120\u001B[0m ])\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pyterrier/transformer.py:879\u001B[0m, in \u001B[0;36mComposedPipeline.transform\u001B[0;34m(self, topics)\u001B[0m\n\u001B[1;32m    877\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mtransform\u001B[39m(\u001B[38;5;28mself\u001B[39m, topics):\n\u001B[1;32m    878\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodels:\n\u001B[0;32m--> 879\u001B[0m         topics \u001B[38;5;241m=\u001B[39m \u001B[43mm\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtopics\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    880\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m topics\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:59\u001B[0m, in \u001B[0;36mPerGroupTransformer.transform\u001B[0;34m(self, topics_or_res)\u001B[0m\n\u001B[1;32m     53\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose:\n\u001B[1;32m     54\u001B[0m     \u001B[38;5;66;03m# Show progress during reranking queries.\u001B[39;00m\n\u001B[1;32m     55\u001B[0m     tqdm\u001B[38;5;241m.\u001B[39mpandas(\n\u001B[1;32m     56\u001B[0m         desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdescription,\n\u001B[1;32m     57\u001B[0m         unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39munit,\n\u001B[1;32m     58\u001B[0m     )\n\u001B[0;32m---> 59\u001B[0m     query_rankings \u001B[38;5;241m=\u001B[39m \u001B[43mquery_rankings\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mprogress_apply\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     60\u001B[0m \u001B[43m        \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform_group\u001B[49m\n\u001B[1;32m     61\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     62\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m     63\u001B[0m     query_rankings \u001B[38;5;241m=\u001B[39m query_rankings\u001B[38;5;241m.\u001B[39mapply(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtransform_group)\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/tqdm/std.py:814\u001B[0m, in \u001B[0;36mtqdm.pandas.<locals>.inner_generator.<locals>.inner\u001B[0;34m(df, func, *args, **kwargs)\u001B[0m\n\u001B[1;32m    811\u001B[0m \u001B[38;5;66;03m# Apply the provided function (in **kwargs)\u001B[39;00m\n\u001B[1;32m    812\u001B[0m \u001B[38;5;66;03m# on the df using our wrapper (which provides bar updating)\u001B[39;00m\n\u001B[1;32m    813\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 814\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mgetattr\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mdf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdf_function\u001B[49m\u001B[43m)\u001B[49m\u001B[43m(\u001B[49m\u001B[43mwrapper\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    815\u001B[0m \u001B[38;5;28;01mfinally\u001B[39;00m:\n\u001B[1;32m    816\u001B[0m     t\u001B[38;5;241m.\u001B[39mclose()\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pandas/core/groupby/groupby.py:1414\u001B[0m, in \u001B[0;36mGroupBy.apply\u001B[0;34m(self, func, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1412\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m option_context(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmode.chained_assignment\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m   1413\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1414\u001B[0m         result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_python_apply_general\u001B[49m\u001B[43m(\u001B[49m\u001B[43mf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_selected_obj\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1415\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m   1416\u001B[0m         \u001B[38;5;66;03m# gh-20949\u001B[39;00m\n\u001B[1;32m   1417\u001B[0m         \u001B[38;5;66;03m# try again, with .apply acting as a filtering\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1421\u001B[0m         \u001B[38;5;66;03m# fails on *some* columns, e.g. a numeric operation\u001B[39;00m\n\u001B[1;32m   1422\u001B[0m         \u001B[38;5;66;03m# on a string grouper column\u001B[39;00m\n\u001B[1;32m   1424\u001B[0m         \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_group_selection_context():\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pandas/core/groupby/groupby.py:1455\u001B[0m, in \u001B[0;36mGroupBy._python_apply_general\u001B[0;34m(self, f, data, not_indexed_same)\u001B[0m\n\u001B[1;32m   1429\u001B[0m \u001B[38;5;129m@final\u001B[39m\n\u001B[1;32m   1430\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_python_apply_general\u001B[39m(\n\u001B[1;32m   1431\u001B[0m     \u001B[38;5;28mself\u001B[39m,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1434\u001B[0m     not_indexed_same: \u001B[38;5;28mbool\u001B[39m \u001B[38;5;241m|\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m   1435\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m DataFrame \u001B[38;5;241m|\u001B[39m Series:\n\u001B[1;32m   1436\u001B[0m     \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m   1437\u001B[0m \u001B[38;5;124;03m    Apply function f in python space\u001B[39;00m\n\u001B[1;32m   1438\u001B[0m \n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1453\u001B[0m \u001B[38;5;124;03m        data after applying f\u001B[39;00m\n\u001B[1;32m   1454\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m-> 1455\u001B[0m     values, mutated \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgrouper\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mapply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1457\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m not_indexed_same \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m   1458\u001B[0m         not_indexed_same \u001B[38;5;241m=\u001B[39m mutated \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmutated\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pandas/core/groupby/ops.py:761\u001B[0m, in \u001B[0;36mBaseGrouper.apply\u001B[0;34m(self, f, data, axis)\u001B[0m\n\u001B[1;32m    759\u001B[0m \u001B[38;5;66;03m# group might be modified\u001B[39;00m\n\u001B[1;32m    760\u001B[0m group_axes \u001B[38;5;241m=\u001B[39m group\u001B[38;5;241m.\u001B[39maxes\n\u001B[0;32m--> 761\u001B[0m res \u001B[38;5;241m=\u001B[39m \u001B[43mf\u001B[49m\u001B[43m(\u001B[49m\u001B[43mgroup\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    762\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m mutated \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m _is_indexed_like(res, group_axes, axis):\n\u001B[1;32m    763\u001B[0m     mutated \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/tqdm/std.py:809\u001B[0m, in \u001B[0;36mtqdm.pandas.<locals>.inner_generator.<locals>.inner.<locals>.wrapper\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    803\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrapper\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m    804\u001B[0m     \u001B[38;5;66;03m# update tbar correctly\u001B[39;00m\n\u001B[1;32m    805\u001B[0m     \u001B[38;5;66;03m# it seems `pandas apply` calls `func` twice\u001B[39;00m\n\u001B[1;32m    806\u001B[0m     \u001B[38;5;66;03m# on the first column/row to decide whether it can\u001B[39;00m\n\u001B[1;32m    807\u001B[0m     \u001B[38;5;66;03m# take a fast or slow code path; so stop when t.total==t.n\u001B[39;00m\n\u001B[1;32m    808\u001B[0m     t\u001B[38;5;241m.\u001B[39mupdate(n\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m t\u001B[38;5;241m.\u001B[39mtotal \u001B[38;5;129;01mor\u001B[39;00m t\u001B[38;5;241m.\u001B[39mn \u001B[38;5;241m<\u001B[39m t\u001B[38;5;241m.\u001B[39mtotal \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;241m0\u001B[39m)\n\u001B[0;32m--> 809\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:134\u001B[0m, in \u001B[0;36mAxiomTransformer.transform_group\u001B[0;34m(self, topics_or_res)\u001B[0m\n\u001B[1;32m    124\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    125\u001B[0m     documents \u001B[38;5;241m=\u001B[39m [\n\u001B[1;32m    126\u001B[0m         RankedDocument(\n\u001B[1;32m    127\u001B[0m             \u001B[38;5;28mid\u001B[39m\u001B[38;5;241m=\u001B[39mrow[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdocno\u001B[39m\u001B[38;5;124m\"\u001B[39m],\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    131\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m index, row \u001B[38;5;129;01min\u001B[39;00m topics_or_res\u001B[38;5;241m.\u001B[39miterrows()\n\u001B[1;32m    132\u001B[0m     ]\n\u001B[0;32m--> 134\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform_query_ranking\u001B[49m\u001B[43m(\u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocuments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtopics_or_res\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:313\u001B[0m, in \u001B[0;36mAxiomaticPreferences.transform_query_ranking\u001B[0;34m(self, query, documents, topics_or_res)\u001B[0m\n\u001B[1;32m    306\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;241m0\u001B[39m \u001B[38;5;241m<\u001B[39m logger\u001B[38;5;241m.\u001B[39mlevel \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m DEBUG:\n\u001B[1;32m    307\u001B[0m         \u001B[38;5;66;03m# Very verbose progress bars.\u001B[39;00m\n\u001B[1;32m    308\u001B[0m         document_pairs \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    309\u001B[0m             document_pairs,\n\u001B[1;32m    310\u001B[0m             desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing axiom preference\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    311\u001B[0m             unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpair\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    312\u001B[0m         )\n\u001B[0;32m--> 313\u001B[0m     pairs[\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mname\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m_preference\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m [\n\u001B[1;32m    314\u001B[0m         axiom\u001B[38;5;241m.\u001B[39mpreference(context, query, document1, document2)\n\u001B[1;32m    315\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m document1, document2 \u001B[38;5;129;01min\u001B[39;00m document_pairs\n\u001B[1;32m    316\u001B[0m     ]\n\u001B[1;32m    318\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m pairs\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:314\u001B[0m, in \u001B[0;36m<listcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m    306\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;241m0\u001B[39m \u001B[38;5;241m<\u001B[39m logger\u001B[38;5;241m.\u001B[39mlevel \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m DEBUG:\n\u001B[1;32m    307\u001B[0m         \u001B[38;5;66;03m# Very verbose progress bars.\u001B[39;00m\n\u001B[1;32m    308\u001B[0m         document_pairs \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    309\u001B[0m             document_pairs,\n\u001B[1;32m    310\u001B[0m             desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing axiom preference\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    311\u001B[0m             unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpair\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    312\u001B[0m         )\n\u001B[1;32m    313\u001B[0m     pairs[\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mname\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m_preference\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m [\n\u001B[0;32m--> 314\u001B[0m         \u001B[43maxiom\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreference\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocument1\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocument2\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    315\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m document1, document2 \u001B[38;5;129;01min\u001B[39;00m document_pairs\n\u001B[1;32m    316\u001B[0m     ]\n\u001B[1;32m    318\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m pairs\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/cache.py:60\u001B[0m, in \u001B[0;36mCachedAxiom.preference\u001B[0;34m(self, context, query, document1, document2)\u001B[0m\n\u001B[1;32m     57\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m preference\n\u001B[1;32m     59\u001B[0m \u001B[38;5;66;03m# Cache miss.\u001B[39;00m\n\u001B[0;32m---> 60\u001B[0m preference \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43maxiom\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreference\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     61\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     62\u001B[0m \u001B[43m    \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     63\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdocument1\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     64\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdocument2\u001B[49m\n\u001B[1;32m     65\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     66\u001B[0m cache[key] \u001B[38;5;241m=\u001B[39m preference\n\u001B[1;32m     67\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m preference\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/preconditions.py:66\u001B[0m, in \u001B[0;36mLEN_Mixin.preference\u001B[0;34m(self, context, query, document1, document2)\u001B[0m\n\u001B[1;32m     58\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m approximately_same_length(\n\u001B[1;32m     59\u001B[0m         context,\n\u001B[1;32m     60\u001B[0m         document1,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m     63\u001B[0m ):\n\u001B[1;32m     64\u001B[0m     \u001B[38;5;66;03m# Documents have different lengths.\u001B[39;00m\n\u001B[1;32m     65\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;241m0\u001B[39m\n\u001B[0;32m---> 66\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreference\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocument1\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocument2\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/argumentative.py:231\u001B[0m, in \u001B[0;36mQueryTermOccurrenceInArgumentativeUnitsAxiom.preference\u001B[0;34m(self, context, query, document1, document2)\u001B[0m\n\u001B[1;32m    228\u001B[0m arguments1 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfetch_arguments(context, document1)\n\u001B[1;32m    229\u001B[0m arguments2 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfetch_arguments(context, document2)\n\u001B[0;32m--> 231\u001B[0m count1 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43msum\u001B[39;49m\u001B[43m(\u001B[49m\n\u001B[1;32m    232\u001B[0m \u001B[43m    \u001B[49m\u001B[43m_count_query_terms\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msentences\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    233\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43m_\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msentences\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43marguments1\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mitems\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    234\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    235\u001B[0m count2 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28msum\u001B[39m(\n\u001B[1;32m    236\u001B[0m     _count_query_terms(context, sentences, query)\n\u001B[1;32m    237\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m _, sentences \u001B[38;5;129;01min\u001B[39;00m arguments2\u001B[38;5;241m.\u001B[39mitems()\n\u001B[1;32m    238\u001B[0m )\n\u001B[1;32m    240\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m count1 \u001B[38;5;241m>\u001B[39m count2:\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/argumentative.py:232\u001B[0m, in \u001B[0;36m<genexpr>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m    228\u001B[0m arguments1 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfetch_arguments(context, document1)\n\u001B[1;32m    229\u001B[0m arguments2 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfetch_arguments(context, document2)\n\u001B[1;32m    231\u001B[0m count1 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28msum\u001B[39m(\n\u001B[0;32m--> 232\u001B[0m     \u001B[43m_count_query_terms\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msentences\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    233\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m _, sentences \u001B[38;5;129;01min\u001B[39;00m arguments1\u001B[38;5;241m.\u001B[39mitems()\n\u001B[1;32m    234\u001B[0m )\n\u001B[1;32m    235\u001B[0m count2 \u001B[38;5;241m=\u001B[39m \u001B[38;5;28msum\u001B[39m(\n\u001B[1;32m    236\u001B[0m     _count_query_terms(context, sentences, query)\n\u001B[1;32m    237\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m _, sentences \u001B[38;5;129;01min\u001B[39;00m arguments2\u001B[38;5;241m.\u001B[39mitems()\n\u001B[1;32m    238\u001B[0m )\n\u001B[1;32m    240\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m count1 \u001B[38;5;241m>\u001B[39m count2:\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/argumentative.py:91\u001B[0m, in \u001B[0;36m_count_query_terms\u001B[0;34m(context, sentences, query, normalize)\u001B[0m\n\u001B[1;32m     89\u001B[0m term_count \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m     90\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m term \u001B[38;5;129;01min\u001B[39;00m context\u001B[38;5;241m.\u001B[39mterms(query):\n\u001B[0;32m---> 91\u001B[0m     normalized_term \u001B[38;5;241m=\u001B[39m \u001B[43m_normalize\u001B[49m\u001B[43m(\u001B[49m\u001B[43mterm\u001B[49m\u001B[43m)\u001B[49m \u001B[38;5;28;01mif\u001B[39;00m normalize \u001B[38;5;28;01melse\u001B[39;00m term\n\u001B[1;32m     92\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m sentence \u001B[38;5;129;01min\u001B[39;00m sentences:\n\u001B[1;32m     93\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m tag \u001B[38;5;129;01min\u001B[39;00m sentence:\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/argumentative.py:22\u001B[0m, in \u001B[0;36m_normalize\u001B[0;34m(word)\u001B[0m\n\u001B[1;32m     20\u001B[0m download_nltk_dependencies(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwordnet\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124momw-1.4\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m     21\u001B[0m _word_net_lemmatizer \u001B[38;5;241m=\u001B[39m WordNetLemmatizer()\n\u001B[0;32m---> 22\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43m_word_net_lemmatizer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlemmatize\u001B[49m\u001B[43m(\u001B[49m\u001B[43mword\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241m.\u001B[39mlower()\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/stem/wordnet.py:45\u001B[0m, in \u001B[0;36mWordNetLemmatizer.lemmatize\u001B[0;34m(self, word, pos)\u001B[0m\n\u001B[1;32m     33\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mlemmatize\u001B[39m(\u001B[38;5;28mself\u001B[39m, word: \u001B[38;5;28mstr\u001B[39m, pos: \u001B[38;5;28mstr\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mn\u001B[39m\u001B[38;5;124m\"\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28mstr\u001B[39m:\n\u001B[1;32m     34\u001B[0m     \u001B[38;5;124;03m\"\"\"Lemmatize `word` using WordNet's built-in morphy function.\u001B[39;00m\n\u001B[1;32m     35\u001B[0m \u001B[38;5;124;03m    Returns the input word unchanged if it cannot be found in WordNet.\u001B[39;00m\n\u001B[1;32m     36\u001B[0m \n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m     43\u001B[0m \u001B[38;5;124;03m    :return: The lemma of `word`, for the given `pos`.\u001B[39;00m\n\u001B[1;32m     44\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m---> 45\u001B[0m     lemmas \u001B[38;5;241m=\u001B[39m \u001B[43mwn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_morphy\u001B[49m(word, pos)\n\u001B[1;32m     46\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mmin\u001B[39m(lemmas, key\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mlen\u001B[39m) \u001B[38;5;28;01mif\u001B[39;00m lemmas \u001B[38;5;28;01melse\u001B[39;00m word\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/corpus/util.py:121\u001B[0m, in \u001B[0;36mLazyCorpusLoader.__getattr__\u001B[0;34m(self, attr)\u001B[0m\n\u001B[1;32m    118\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m attr \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m__bases__\u001B[39m\u001B[38;5;124m\"\u001B[39m:\n\u001B[1;32m    119\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mAttributeError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mLazyCorpusLoader object has no attribute \u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m__bases__\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m--> 121\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m__load\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    122\u001B[0m \u001B[38;5;66;03m# This looks circular, but its not, since __load() changes our\u001B[39;00m\n\u001B[1;32m    123\u001B[0m \u001B[38;5;66;03m# __class__ to something new:\u001B[39;00m\n\u001B[1;32m    124\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mgetattr\u001B[39m(\u001B[38;5;28mself\u001B[39m, attr)\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/corpus/util.py:89\u001B[0m, in \u001B[0;36mLazyCorpusLoader.__load\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m     86\u001B[0m             \u001B[38;5;28;01mraise\u001B[39;00m e\n\u001B[1;32m     88\u001B[0m \u001B[38;5;66;03m# Load the corpus.\u001B[39;00m\n\u001B[0;32m---> 89\u001B[0m corpus \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m__reader_cls\u001B[49m\u001B[43m(\u001B[49m\u001B[43mroot\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m__args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m__kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     91\u001B[0m \u001B[38;5;66;03m# This is where the magic happens!  Transform ourselves into\u001B[39;00m\n\u001B[1;32m     92\u001B[0m \u001B[38;5;66;03m# the corpus by modifying our own __dict__ and __class__ to\u001B[39;00m\n\u001B[1;32m     93\u001B[0m \u001B[38;5;66;03m# match that of the corpus.\u001B[39;00m\n\u001B[1;32m     95\u001B[0m args, kwargs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m__args, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m__kwargs\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/corpus/reader/wordnet.py:1195\u001B[0m, in \u001B[0;36mWordNetCorpusReader.__init__\u001B[0;34m(self, root, omw_reader)\u001B[0m\n\u001B[1;32m   1192\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_lexnames\u001B[38;5;241m.\u001B[39mappend(lexname)\n\u001B[1;32m   1194\u001B[0m \u001B[38;5;66;03m# Load the indices for lemmas and synset offsets\u001B[39;00m\n\u001B[0;32m-> 1195\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_load_lemma_pos_offset_map\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1197\u001B[0m \u001B[38;5;66;03m# load the exception file data into memory\u001B[39;00m\n\u001B[1;32m   1198\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_load_exception_map()\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/corpus/reader/wordnet.py:1307\u001B[0m, in \u001B[0;36mWordNetCorpusReader._load_lemma_pos_offset_map\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m   1303\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m suffix \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_FILEMAP\u001B[38;5;241m.\u001B[39mvalues():\n\u001B[1;32m   1304\u001B[0m \n\u001B[1;32m   1305\u001B[0m     \u001B[38;5;66;03m# parse each line of the file (ignoring comment lines)\u001B[39;00m\n\u001B[1;32m   1306\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mopen(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mindex.\u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m suffix) \u001B[38;5;28;01mas\u001B[39;00m fp:\n\u001B[0;32m-> 1307\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m i, line \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28menumerate\u001B[39m(fp):\n\u001B[1;32m   1308\u001B[0m             \u001B[38;5;28;01mif\u001B[39;00m line\u001B[38;5;241m.\u001B[39mstartswith(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m \u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[1;32m   1309\u001B[0m                 \u001B[38;5;28;01mcontinue\u001B[39;00m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/data.py:1152\u001B[0m, in \u001B[0;36mSeekableUnicodeStreamReader.__next__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m   1151\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__next__\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m-> 1152\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mnext\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/data.py:1145\u001B[0m, in \u001B[0;36mSeekableUnicodeStreamReader.next\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m   1143\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mnext\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m   1144\u001B[0m     \u001B[38;5;124;03m\"\"\"Return the next decoded line from the underlying stream.\"\"\"\u001B[39;00m\n\u001B[0;32m-> 1145\u001B[0m     line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadline\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1146\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m line:\n\u001B[1;32m   1147\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m line\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/data.py:1100\u001B[0m, in \u001B[0;36mSeekableUnicodeStreamReader.readline\u001B[0;34m(self, size)\u001B[0m\n\u001B[1;32m   1098\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m   1099\u001B[0m     startpos \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstream\u001B[38;5;241m.\u001B[39mtell() \u001B[38;5;241m-\u001B[39m \u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbytebuffer)\n\u001B[0;32m-> 1100\u001B[0m     new_chars \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read\u001B[49m\u001B[43m(\u001B[49m\u001B[43mreadsize\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1102\u001B[0m     \u001B[38;5;66;03m# If we're at a '\\r', then read one extra character, since\u001B[39;00m\n\u001B[1;32m   1103\u001B[0m     \u001B[38;5;66;03m# it might be a '\\n', to get the proper line ending.\u001B[39;00m\n\u001B[1;32m   1104\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m new_chars \u001B[38;5;129;01mand\u001B[39;00m new_chars\u001B[38;5;241m.\u001B[39mendswith(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;130;01m\\r\u001B[39;00m\u001B[38;5;124m\"\u001B[39m):\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/nltk/data.py:1344\u001B[0m, in \u001B[0;36mSeekableUnicodeStreamReader._read\u001B[0;34m(self, size)\u001B[0m\n\u001B[1;32m   1341\u001B[0m \u001B[38;5;28mbytes\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbytebuffer \u001B[38;5;241m+\u001B[39m new_bytes\n\u001B[1;32m   1343\u001B[0m \u001B[38;5;66;03m# Decode the bytes into unicode characters\u001B[39;00m\n\u001B[0;32m-> 1344\u001B[0m chars, bytes_decoded \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_incr_decode\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mbytes\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1346\u001B[0m \u001B[38;5;66;03m# If we got bytes but couldn't decode any, then read further.\u001B[39;00m\n\u001B[1;32m   1347\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (size \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m) \u001B[38;5;129;01mand\u001B[39;00m (\u001B[38;5;129;01mnot\u001B[39;00m chars) \u001B[38;5;129;01mand\u001B[39;00m (\u001B[38;5;28mlen\u001B[39m(new_bytes) \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m0\u001B[39m):\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "preferences = experiment.preferences"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "preferences.to_csv(f\"trec-{edition}-{track}-preferences-all-axioms-depth-{depth}.csv\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}