playground-spacy-entity-linking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:12:49 DEBUG:test\n",
      "11:12:49 INFO:this is an info message\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "logger = logging.getLogger()\n",
    "logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%I:%M:%S')\n",
    "logger.debug(\"test\")\n",
    "logging.info('this is an info message')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-02 11:24:24,467 - root - INFO - Test info\n",
      "2019-12-02 11:24:24,469 - root - DEBUG - Test debug\n",
      "2019-12-02 11:24:24,470 - root - ERROR - Test error\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "logger = logging.getLogger()\n",
    "\n",
    "formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')\n",
    "\n",
    "# Setup file handler\n",
    "fhandler  = logging.FileHandler('retrain-wd-entity-linking.log', mode='a')\n",
    "fhandler.setLevel(logging.DEBUG)\n",
    "fhandler.setFormatter(formatter)\n",
    "\n",
    "# Configure stream handler for the cells\n",
    "chandler = logging.StreamHandler()\n",
    "chandler.setLevel(logging.DEBUG)\n",
    "chandler.setFormatter(formatter)\n",
    "\n",
    "# Add both handlers\n",
    "logger.addHandler(fhandler)\n",
    "logger.addHandler(chandler)\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "# Show the handlers\n",
    "logger.handlers\n",
    "\n",
    "# Log Something\n",
    "logger.info(\"Test info\")\n",
    "logger.debug(\"Test debug\")\n",
    "logger.error(\"Test error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_kb = Path(\"/data/users/romain.claret/tm/data2/\")\n",
    "KB_FILE = \"kb\"\n",
    "KB_MODEL_DIR = \"nlp_kb\"\n",
    "OUTPUT_MODEL_DIR = \"nlp_custom\"\n",
    "\n",
    "nlp_dir = dir_kb / OUTPUT_MODEL_DIR\n",
    "kb_path = dir_kb / KB_FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(1, '/data/users/romain.claret/tm/spaCy/bin/wiki_entity_linking')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=10\n",
    "dropout=0.5\n",
    "lr=0.005\n",
    "l2=1e-6\n",
    "train_inst=1000000\n",
    "dev_inst=159000\n",
    "labels_discard=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_path = dir_kb / \"pieces/x01.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(nlp_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import kb_creator as kbc\n",
    "kb = kbc.read_kb(nlp, kb_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tagger', <spacy.pipeline.pipes.Tagger at 0x7fe5774bddd0>),\n",
       " ('parser', <spacy.pipeline.pipes.DependencyParser at 0x7fe577387280>),\n",
       " ('ner', <spacy.pipeline.pipes.EntityRecognizer at 0x7fe5773872f0>),\n",
       " ('entity_linker', <spacy.pipeline.pipes.EntityLinker at 0x7fe576667c90>)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"entity_linker\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tagger', <spacy.pipeline.pipes.Tagger at 0x7fe5774bddd0>),\n",
       " ('parser', <spacy.pipeline.pipes.DependencyParser at 0x7fe577387280>),\n",
       " ('ner', <spacy.pipeline.pipes.EntityRecognizer at 0x7fe5773872f0>),\n",
       " ('entity_linker', <spacy.pipeline.pipes.EntityLinker at 0x7fe576667c90>)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "el_pipe = nlp.get_pipe(\"entity_linker\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with nlp.disable_pipes(*other_pipes):  # only train Entity Linking\n",
    "    optimizer = nlp.begin_training()\n",
    "    optimizer.learn_rate = lr\n",
    "    optimizer.L2 = l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "01:44:59 INFO:Reading train data with limit 1000000\n",
      " 27%|██▋       | 274523/1000000 [1:37:46<3:48:43, 52.86it/s] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 82%|████████▏ | 824119/1000000 [4:58:14<1:02:15, 47.08it/s] "
     ]
    }
   ],
   "source": [
    "import wikipedia_processor as wp\n",
    "train_data = wp.read_training(\n",
    "        nlp=nlp,\n",
    "        entity_file_path=training_path,\n",
    "        dev=False,\n",
    "        limit=train_inst,\n",
    "        kb=kb,\n",
    "        labels_discard=labels_discard\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 99690/159000 [29:11<28:03, 35.24it/s]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import wikipedia_processor as wp\n",
    "dev_data = wp.read_training(\n",
    "        nlp=nlp,\n",
    "        entity_file_path=training_path,\n",
    "        dev=True,\n",
    "        limit=dev_inst,\n",
    "        kb=None,\n",
    "        labels_discard=labels_discard\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:22:06 INFO:Counts: {'CARDINAL': 254, 'DATE': 1693, 'EVENT': 1635, 'FAC': 4534, 'GPE': 38917, 'LANGUAGE': 536, 'LAW': 203, 'LOC': 4080, 'MONEY': 32, 'NORP': 5402, 'ORDINAL': 72, 'ORG': 32536, 'PERCENT': 2, 'PERSON': 61579, 'PRODUCT': 1765, 'QUANTITY': 43, 'TIME': 51, 'WORK_OF_ART': 5674}\n",
      "12:22:06 INFO:Random: F-score = 0.55 | Recall = 0.507 | Precision = 0.602 | F-score by label = {'CARDINAL': 0.20824295010845983, 'DATE': 0.11950257810130421, 'EVENT': 0.39567128846804195, 'FAC': 0.633625619519634, 'GPE': 0.29359390822888853, 'LANGUAGE': 0.23076923076923075, 'LAW': 0.5714285714285714, 'LOC': 0.5399048625792812, 'MONEY': 0.5384615384615384, 'NORP': 0.277179050860276, 'ORDINAL': 0.16901408450704225, 'ORG': 0.5946940389608157, 'PERCENT': 0.0, 'PERSON': 0.7387328955309931, 'PRODUCT': 0.6228695382708398, 'QUANTITY': 0.5599999999999999, 'TIME': 0.46153846153846156, 'WORK_OF_ART': 0.6014630724521495}\n",
      "12:22:06 INFO:Prior: F-score = 0.796 | Recall = 0.734 | Precision = 0.871 | F-score by label = {'CARDINAL': 0.2993492407809111, 'DATE': 0.14619350925083407, 'EVENT': 0.8075752451809266, 'FAC': 0.7317321133562079, 'GPE': 0.8337782921472441, 'LANGUAGE': 0.7504690431519699, 'LAW': 0.6938775510204082, 'LOC': 0.8266384778012685, 'MONEY': 0.5769230769230769, 'NORP': 0.6931366988088487, 'ORDINAL': 0.295774647887324, 'ORG': 0.7911830952012516, 'PERCENT': 0.6666666666666666, 'PERSON': 0.8156996279580935, 'PRODUCT': 0.7784319801673381, 'QUANTITY': 0.72, 'TIME': 0.6593406593406592, 'WORK_OF_ART': 0.7265257039783544}\n",
      "12:22:06 INFO:Oracle: F-score = 0.888 | Recall = 0.798 | Precision = 1.0 | F-score by label = {'CARDINAL': 0.57703081232493, 'DATE': 0.45599635202918376, 'EVENT': 0.8653712699514226, 'FAC': 0.8211128445137806, 'GPE': 0.9543079010653877, 'LANGUAGE': 0.979047619047619, 'LAW': 0.7771084337349398, 'LOC': 0.9041095890410958, 'MONEY': 0.72, 'NORP': 0.9510679611650484, 'ORDINAL': 0.5882352941176471, 'ORG': 0.8585261459119056, 'PERCENT': 0.6666666666666666, 'PERSON': 0.8710526074561604, 'PRODUCT': 0.8772264631043257, 'QUANTITY': 0.8219178082191781, 'TIME': 0.7710843373493976, 'WORK_OF_ART': 0.831307929969104}\n"
     ]
    }
   ],
   "source": [
    "import entity_linker_evaluation as ele\n",
    "ele.measure_performance(dev_data, kb, el_pipe, baseline=True, context=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:23:36 INFO:Counts: {'CARDINAL': 254, 'DATE': 1693, 'EVENT': 1635, 'FAC': 4534, 'GPE': 38917, 'LANGUAGE': 536, 'LAW': 203, 'LOC': 4080, 'MONEY': 32, 'NORP': 5402, 'ORDINAL': 72, 'ORG': 32536, 'PERCENT': 2, 'PERSON': 61579, 'PRODUCT': 1765, 'QUANTITY': 43, 'TIME': 51, 'WORK_OF_ART': 5674}\n",
      "12:23:36 INFO:Random: F-score = 0.55 | Recall = 0.506 | Precision = 0.601 | F-score by label = {'CARDINAL': 0.20390455531453364, 'DATE': 0.11950257810130421, 'EVENT': 0.40108217788298955, 'FAC': 0.6249841148811793, 'GPE': 0.29601378276365453, 'LANGUAGE': 0.2270168855534709, 'LAW': 0.6064139941690962, 'LOC': 0.5354122621564483, 'MONEY': 0.42307692307692313, 'NORP': 0.2826621289468708, 'ORDINAL': 0.14084507042253522, 'ORG': 0.5931809975756951, 'PERCENT': 0.0, 'PERSON': 0.73676909079444, 'PRODUCT': 0.6241090796405329, 'QUANTITY': 0.5866666666666667, 'TIME': 0.5054945054945055, 'WORK_OF_ART': 0.6010622306844373}\n",
      "12:23:36 INFO:Prior: F-score = 0.796 | Recall = 0.734 | Precision = 0.871 | F-score by label = {'CARDINAL': 0.2993492407809111, 'DATE': 0.14619350925083407, 'EVENT': 0.8075752451809266, 'FAC': 0.7317321133562079, 'GPE': 0.8337782921472441, 'LANGUAGE': 0.7504690431519699, 'LAW': 0.6938775510204082, 'LOC': 0.8266384778012685, 'MONEY': 0.5769230769230769, 'NORP': 0.6931366988088487, 'ORDINAL': 0.295774647887324, 'ORG': 0.7911830952012516, 'PERCENT': 0.6666666666666666, 'PERSON': 0.8156996279580935, 'PRODUCT': 0.7784319801673381, 'QUANTITY': 0.72, 'TIME': 0.6593406593406592, 'WORK_OF_ART': 0.7265257039783544}\n",
      "12:23:36 INFO:Oracle: F-score = 0.888 | Recall = 0.798 | Precision = 1.0 | F-score by label = {'CARDINAL': 0.57703081232493, 'DATE': 0.45599635202918376, 'EVENT': 0.8653712699514226, 'FAC': 0.8211128445137806, 'GPE': 0.9543079010653877, 'LANGUAGE': 0.979047619047619, 'LAW': 0.7771084337349398, 'LOC': 0.9041095890410958, 'MONEY': 0.72, 'NORP': 0.9510679611650484, 'ORDINAL': 0.5882352941176471, 'ORG': 0.8585261459119056, 'PERCENT': 0.6666666666666666, 'PERSON': 0.8710526074561604, 'PRODUCT': 0.8772264631043257, 'QUANTITY': 0.8219178082191781, 'TIME': 0.7710843373493976, 'WORK_OF_ART': 0.831307929969104}\n",
      "12:53:56 INFO:Context Only: F-score = 0.688 | Recall = 0.634 | Precision = 0.753 | F-score by label = {'CARDINAL': 0.1908893709327549, 'DATE': 0.12556869881710647, 'EVENT': 0.7291173486641866, 'FAC': 0.6605667810395223, 'GPE': 0.661230716624801, 'LANGUAGE': 0.4878048780487805, 'LAW': 0.6064139941690962, 'LOC': 0.6582980972515857, 'MONEY': 0.42307692307692313, 'NORP': 0.40707128001512577, 'ORDINAL': 0.14084507042253522, 'ORG': 0.6755729784563539, 'PERCENT': 0.6666666666666666, 'PERSON': 0.7714689799926131, 'PRODUCT': 0.6705918810040284, 'QUANTITY': 0.5599999999999999, 'TIME': 0.5934065934065933, 'WORK_OF_ART': 0.6193005311153421}\n",
      "01:24:24 INFO:Context And Prior: F-score = 0.778 | Recall = 0.717 | Precision = 0.851 | F-score by label = {'CARDINAL': 0.22993492407809113, 'DATE': 0.14740673339399454, 'EVENT': 0.7886371322286101, 'FAC': 0.7177532087940018, 'GPE': 0.8101319094651288, 'LANGUAGE': 0.7148217636022514, 'LAW': 0.6822157434402333, 'LOC': 0.808139534883721, 'MONEY': 0.5384615384615384, 'NORP': 0.6089998109283419, 'ORDINAL': 0.18309859154929575, 'ORG': 0.7749866749196197, 'PERCENT': 0.6666666666666666, 'PERSON': 0.807430028195912, 'PRODUCT': 0.7592190889370933, 'QUANTITY': 0.6399999999999999, 'TIME': 0.6373626373626374, 'WORK_OF_ART': 0.7002705681932058}\n"
     ]
    }
   ],
   "source": [
    "ele.measure_performance(dev_data, kb, el_pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for itn in range(epochs):\n",
    "    random.shuffle(train_data)\n",
    "    losses = {}\n",
    "    batches = minibatch(train_data, size=compounding(4.0, 128.0, 1.001))\n",
    "    batchnr = 0\n",
    "\n",
    "    with nlp.disable_pipes(*other_pipes):\n",
    "        for batch in batches:\n",
    "            try:\n",
    "                docs, golds = zip(*batch)\n",
    "                nlp.update(\n",
    "                    docs=docs,\n",
    "                    golds=golds,\n",
    "                    sgd=optimizer,\n",
    "                    drop=dropout,\n",
    "                    losses=losses,\n",
    "                )\n",
    "                batchnr += 1\n",
    "            except Exception as e:\n",
    "                logger.error(\"Error updating batch:\" + str(e))\n",
    "    if batchnr > 0:\n",
    "        logging.info(\"Epoch {}, train loss {}\".format(itn, round(losses[\"entity_linker\"] / batchnr, 2)))\n",
    "        ele.measure_performance(dev_data, kb, el_pipe, baseline=False, context=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02:22:58 INFO:Counts: {'CARDINAL': 254, 'DATE': 1693, 'EVENT': 1635, 'FAC': 4534, 'GPE': 38917, 'LANGUAGE': 536, 'LAW': 203, 'LOC': 4080, 'MONEY': 32, 'NORP': 5402, 'ORDINAL': 72, 'ORG': 32536, 'PERCENT': 2, 'PERSON': 61579, 'PRODUCT': 1765, 'QUANTITY': 43, 'TIME': 51, 'WORK_OF_ART': 5674}\n",
      "02:22:58 INFO:Random: F-score = 0.55 | Recall = 0.507 | Precision = 0.602 | F-score by label = {'CARDINAL': 0.19956616052060738, 'DATE': 0.12132241431604489, 'EVENT': 0.3963476496449104, 'FAC': 0.6229508196721311, 'GPE': 0.2945145126714625, 'LANGUAGE': 0.20075046904315197, 'LAW': 0.553935860058309, 'LOC': 0.5422832980972515, 'MONEY': 0.4615384615384615, 'NORP': 0.28379655889582156, 'ORDINAL': 0.18309859154929575, 'ORG': 0.5938687436598408, 'PERCENT': 0.0, 'PERSON': 0.7388590114315055, 'PRODUCT': 0.6253486210102263, 'QUANTITY': 0.5066666666666667, 'TIME': 0.5714285714285715, 'WORK_OF_ART': 0.6028660186391422}\n",
      "02:22:58 INFO:Prior: F-score = 0.796 | Recall = 0.734 | Precision = 0.871 | F-score by label = {'CARDINAL': 0.2993492407809111, 'DATE': 0.14619350925083407, 'EVENT': 0.8075752451809266, 'FAC': 0.7317321133562079, 'GPE': 0.8337782921472441, 'LANGUAGE': 0.7504690431519699, 'LAW': 0.6938775510204082, 'LOC': 0.8266384778012685, 'MONEY': 0.5769230769230769, 'NORP': 0.6931366988088487, 'ORDINAL': 0.295774647887324, 'ORG': 0.7911830952012516, 'PERCENT': 0.6666666666666666, 'PERSON': 0.8156996279580935, 'PRODUCT': 0.7784319801673381, 'QUANTITY': 0.72, 'TIME': 0.6593406593406592, 'WORK_OF_ART': 0.7265257039783544}\n",
      "02:22:58 INFO:Oracle: F-score = 0.888 | Recall = 0.798 | Precision = 1.0 | F-score by label = {'CARDINAL': 0.57703081232493, 'DATE': 0.45599635202918376, 'EVENT': 0.8653712699514226, 'FAC': 0.8211128445137806, 'GPE': 0.9543079010653877, 'LANGUAGE': 0.979047619047619, 'LAW': 0.7771084337349398, 'LOC': 0.9041095890410958, 'MONEY': 0.72, 'NORP': 0.9510679611650484, 'ORDINAL': 0.5882352941176471, 'ORG': 0.8585261459119056, 'PERCENT': 0.6666666666666666, 'PERSON': 0.8710526074561604, 'PRODUCT': 0.8772264631043257, 'QUANTITY': 0.8219178082191781, 'TIME': 0.7710843373493976, 'WORK_OF_ART': 0.831307929969104}\n",
      "02:53:09 INFO:Context Only: F-score = 0.688 | Recall = 0.634 | Precision = 0.753 | F-score by label = {'CARDINAL': 0.1908893709327549, 'DATE': 0.12496208674552624, 'EVENT': 0.7291173486641866, 'FAC': 0.6608209429406532, 'GPE': 0.661230716624801, 'LANGUAGE': 0.4878048780487805, 'LAW': 0.6064139941690962, 'LOC': 0.6582980972515857, 'MONEY': 0.42307692307692313, 'NORP': 0.4066931366988089, 'ORDINAL': 0.14084507042253522, 'ORG': 0.6757105276731831, 'PERCENT': 0.6666666666666666, 'PERSON': 0.7714509634353971, 'PRODUCT': 0.6705918810040284, 'QUANTITY': 0.5599999999999999, 'TIME': 0.5934065934065933, 'WORK_OF_ART': 0.6195009519991984}\n",
      "03:23:01 INFO:Context And Prior: F-score = 0.778 | Recall = 0.717 | Precision = 0.851 | F-score by label = {'CARDINAL': 0.22993492407809113, 'DATE': 0.14740673339399454, 'EVENT': 0.7886371322286101, 'FAC': 0.7180073706951329, 'GPE': 0.8101319094651288, 'LANGUAGE': 0.7148217636022514, 'LAW': 0.6822157434402333, 'LOC': 0.808139534883721, 'MONEY': 0.5384615384615384, 'NORP': 0.6089998109283419, 'ORDINAL': 0.18309859154929575, 'ORG': 0.7749866749196197, 'PERCENT': 0.6666666666666666, 'PERSON': 0.807430028195912, 'PRODUCT': 0.7592190889370933, 'QUANTITY': 0.6399999999999999, 'TIME': 0.6373626373626374, 'WORK_OF_ART': 0.7002705681932058}\n"
     ]
    }
   ],
   "source": [
    "ele.measure_performance(dev_data, kb, el_pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02:42:39 INFO:In The Hitchhiker's Guide to the Galaxy, written by Douglas Adams, Douglas reminds us to always bring our towel, even in China or Brazil. The main character in Doug's novel is the man Arthur Dent, but Dougledydoug doesn't write about George Washington or Homer Simpson.\n",
      "02:42:39 INFO:ent Douglas Adams PERSON Q42\n",
      "02:42:39 INFO:ent Douglas PERSON Q112220\n",
      "02:42:39 INFO:ent China GPE Q148\n",
      "02:42:39 INFO:ent Brazil GPE Q155\n",
      "02:42:39 INFO:ent Doug PERSON Q1251705\n",
      "02:42:39 INFO:ent Arthur Dent PERSON Q613901\n",
      "02:42:39 INFO:ent Dougledydoug ORG NIL\n",
      "02:42:39 INFO:ent George Washington PERSON Q23\n",
      "02:42:39 INFO:ent Homer Simpson PERSON Q7810\n"
     ]
    }
   ],
   "source": [
    "def run_el_toy_example(nlp):\n",
    "    text = (\n",
    "        \"In The Hitchhiker's Guide to the Galaxy, written by Douglas Adams, \"\n",
    "        \"Douglas reminds us to always bring our towel, even in China or Brazil. \"\n",
    "        \"The main character in Doug's novel is the man Arthur Dent, \"\n",
    "        \"but Dougledydoug doesn't write about George Washington or Homer Simpson.\"\n",
    "    )\n",
    "    doc = nlp(text)\n",
    "    logger.info(text)\n",
    "    for ent in doc.ents:\n",
    "        logger.info(\" \".join([\"ent\", ent.text, ent.label_, ent.kb_id_]))\n",
    "run_el_toy_example(nlp=nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp_output_dir = dir_kb / \"nlp_custom\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.to_disk(nlp_output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-02 18:08:24,740 - root - INFO - PRE 0: loading nlp model: /data/users/romain.claret/tm/data2/nlp_custom_1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-294e5068b4bc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"PRE 0: loading nlp model: \"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m \u001b[0mnlp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"PRE 1: loading kb model: \"\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkb_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0mkb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkbc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_kb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkb_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmarkdown\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msilent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mcli_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmarkdown\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msilent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m    217\u001b[0m     \u001b[0;31m# indirect, but it's otherwise very difficult to find the package.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m     \u001b[0mpkg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 219\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpkg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__file__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    220\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model_from_path\u001b[0;34m(model_path, meta, **overrides)\u001b[0m\n\u001b[1;32m    260\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    263\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_PRINT_ENV\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"=\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"via\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'$'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/language.py\u001b[0m in \u001b[0;36mfrom_disk\u001b[0;34m(self, path, exclude, disable)\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mfrom_disk\u001b[0;34m(path, readers, exclude)\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/language.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(p)\u001b[0m\n",
      "\u001b[0;32mvocab.pyx\u001b[0m in \u001b[0;36mspacy.vocab.Vocab.from_disk\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mvectors.pyx\u001b[0m in \u001b[0;36mspacy.vectors.Vectors.from_disk\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/tm/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mfrom_disk\u001b[0;34m(path, readers, exclude)\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from pathlib import Path\n",
    "import sys\n",
    "sys.path.insert(1, '/data/users/romain.claret/tm/spaCy/bin/wiki_entity_linking')\n",
    "import kb_creator as kbc\n",
    "import wikipedia_processor as wp\n",
    "import entity_linker_evaluation as ele\n",
    "\n",
    "dir_kb = Path(\"/data/users/romain.claret/tm/data2/\")\n",
    "KB_FILE = \"kb\"\n",
    "KB_MODEL_DIR = \"nlp_kb\"\n",
    "OUTPUT_MODEL_DIR = \"nlp_custom_1\"\n",
    "nlp_dir = dir_kb / OUTPUT_MODEL_DIR\n",
    "kb_path = dir_kb / KB_FILE\n",
    "\n",
    "paths = [\"pieces/x00.jsonl\",\n",
    "        \"pieces/x01.jsonl\",\n",
    "        \"pieces/x02.jsonl\",\n",
    "        \"pieces/x03.jsonl\",\n",
    "        \"pieces/x04.jsonl\",\n",
    "        \"pieces/x05.jsonl\",\n",
    "        \"pieces/x06.jsonl\"]\n",
    "\n",
    "epochs=10\n",
    "dropout=0.5\n",
    "lr=0.005\n",
    "l2=1e-6\n",
    "train_inst=1000000\n",
    "dev_inst=159000\n",
    "labels_discard=None\n",
    "\n",
    "logger.info(\"PRE 0: loading nlp model: \"+str(nlp_dir))\n",
    "nlp = spacy.load(nlp_dir)\n",
    "logger.info(\"PRE 1: loading kb model: \"+str(kb_path))\n",
    "kb = kbc.read_kb(nlp, kb_path)\n",
    "logger.info(\"PRE 2: setup entity linker\")\n",
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"entity_linker\"]\n",
    "el_pipe = nlp.get_pipe(\"entity_linker\")\n",
    "\n",
    "with nlp.disable_pipes(*other_pipes):  # only train Entity Linking\n",
    "    optimizer = nlp.begin_training()\n",
    "    optimizer.learn_rate = lr\n",
    "    optimizer.L2 = l2\n",
    "\n",
    "start_from = 2\n",
    "\n",
    "#for i, path in enumerate(paths):\n",
    "#    if i >= start_from:\n",
    "training_path = dir_kb / paths[start_from]\n",
    "logger.info(\"STEP 0: starting with: \"+str(training_path))\n",
    "\n",
    "logger.info(\"STEP 1: loading training data\")\n",
    "train_data = wp.read_training(\n",
    "        nlp=nlp,\n",
    "        entity_file_path=training_path,\n",
    "        dev=False,\n",
    "        limit=train_inst,\n",
    "        kb=kb,\n",
    "        labels_discard=labels_discard\n",
    "    )\n",
    "\n",
    "logger.info(\"STEP 2: loading dev data\")\n",
    "dev_data = wp.read_training(\n",
    "    nlp=nlp,\n",
    "    entity_file_path=training_path,\n",
    "    dev=True,\n",
    "    limit=dev_inst,\n",
    "    kb=None,\n",
    "    labels_discard=labels_discard\n",
    ")\n",
    "\n",
    "logger.info(\"STEP 3: evaluating the baseline\")\n",
    "ele.measure_performance(dev_data, kb, el_pipe, baseline=True, context=False)\n",
    "\n",
    "logger.info(\"STEP 4: starting training\")\n",
    "for itn in range(epochs):\n",
    "    random.shuffle(train_data)\n",
    "    losses = {}\n",
    "    batches = minibatch(train_data, size=compounding(4.0, 128.0, 1.001))\n",
    "    batchnr = 0\n",
    "\n",
    "    with nlp.disable_pipes(*other_pipes):\n",
    "        for batch in batches:\n",
    "            try:\n",
    "                docs, golds = zip(*batch)\n",
    "                nlp.update(\n",
    "                    docs=docs,\n",
    "                    golds=golds,\n",
    "                    sgd=optimizer,\n",
    "                    drop=dropout,\n",
    "                    losses=losses,\n",
    "                )\n",
    "                batchnr += 1\n",
    "            except Exception as e:\n",
    "                logger.error(\"Error updating batch:\" + str(e))\n",
    "    if batchnr > 0:\n",
    "        logging.info(\"Epoch {}, train loss {}\".format(itn, round(losses[\"entity_linker\"] / batchnr, 2)))\n",
    "        ele.measure_performance(dev_data, kb, el_pipe, baseline=False, context=True)\n",
    "\n",
    "logger.info(\"STEP 5: evaluating training\")\n",
    "ele.measure_performance(dev_data, kb, el_pipe)\n",
    "\n",
    "logger.info(\"STEP 6: evaluating with example\")\n",
    "run_el_toy_example(nlp)\n",
    "\n",
    "logger.info(\"STEP 7: save current state of nlp model\")\n",
    "nlp_output_dir = dir_kb / str(\"nlp_custom_\"+str(i))\n",
    "nlp.to_disk(nlp_output_dir)\n",
    "    \n",
    "logger.info(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/users/romain.claret/tm/data2/nlp_custom_2\n",
      "/data/users/romain.claret/tm/data2/nlp_custom_3\n",
      "/data/users/romain.claret/tm/data2/nlp_custom_4\n",
      "/data/users/romain.claret/tm/data2/nlp_custom_5\n",
      "/data/users/romain.claret/tm/data2/nlp_custom_6\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "for i, path in enumerate(paths):\n",
    "    start_from = 2\n",
    "    if i >= start_from:\n",
    "        nlp_output_dir = dir_kb / str(\"nlp_custom_\"+str(i))\n",
    "        print(nlp_output_dir)\n",
    "    #continue_link_entities(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pieces/x02.jsonl'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03:20:21 INFO:STEP 4: Final performance measurement of Entity Linking pipelol\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"STEP 4: Final performance measurement of Entity Linking pipe\"+training_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:11:02 INFO:STEP 0: starting with: /data/users/romain.claret/tm/data2/pieces/x02.jsonl\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"STEP 0: starting with: \"+str(training_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n"
     ]
    }
   ],
   "source": [
    "print(\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'nlp' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-1de6044c37c7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mdel\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mkb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'nlp' is not defined"
     ]
    }
   ],
   "source": [
    "del nlp\n",
    "del kb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tm] *",
   "language": "python",
   "name": "conda-env-tm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}