playground-benchmarking-conversational-multihop-qa-systems-Copy3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(180000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 180 seconds\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:25: The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions instead.\n",
      "\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:27: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:27: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading the params file\n",
      "Input encoding {'o': 2, '{': 3, '.': 4, 'J': 5, '0': 6, '1': 7, '<': 8, 'B': 9, 'd': 10, '£': 11, 'e': 12, '6': 13, '!': 14, 'O': 15, 'M': 16, 'X': 17, 'f': 18, 't': 19, 'C': 20, 'V': 21, 'z': 22, 'K': 23, '\\\\': 24, '9': 25, 'P': 26, 'S': 27, '/': 28, '₹': 29, 'F': 30, 'G': 31, '=': 32, '8': 33, ')': 34, '+': 35, ']': 36, 'U': 37, \"'\": 38, '\"': 39, 'g': 40, 'N': 41, 'r': 42, 'u': 43, '&': 44, '$': 45, 'x': 46, '%': 47, ':': 48, '@': 49, '^': 50, 'I': 51, 'L': 52, 'Z': 53, 'h': 54, 'W': 55, 'A': 56, 'v': 57, '?': 58, '2': 59, '~': 60, 's': 61, 'T': 62, 'R': 63, ',': 64, '|': 65, '4': 66, '>': 67, 'y': 68, '(': 69, '[': 70, 'k': 71, 'H': 72, 'l': 73, 'j': 74, '7': 75, 'n': 76, 'i': 77, 'D': 78, 'Q': 79, ' ': 80, 'm': 81, 'Y': 82, '*': 83, '}': 84, '#': 85, 'p': 86, 'q': 87, '5': 88, 'c': 89, '`': 90, 'a': 91, 'b': 92, 'w': 93, '3': 94, 'E': 95, ';': 96, '-': 97}\n",
      "Input decoding {2: 'o', 3: '{', 4: '.', 5: 'J', 6: '0', 7: '1', 8: '<', 9: 'B', 10: 'd', 11: '£', 12: 'e', 13: '6', 14: '!', 15: 'O', 16: 'M', 17: 'X', 18: 'f', 19: 't', 20: 'C', 21: 'V', 22: 'z', 23: 'K', 24: '\\\\', 25: '9', 26: 'P', 27: 'S', 28: '/', 29: '₹', 30: 'F', 31: 'G', 32: '=', 33: '8', 34: ')', 35: '+', 36: ']', 37: 'U', 38: \"'\", 39: '\"', 40: 'g', 41: 'N', 42: 'r', 43: 'u', 44: '&', 45: '$', 46: 'x', 47: '%', 48: ':', 49: '@', 50: '^', 51: 'I', 52: 'L', 53: 'Z', 54: 'h', 55: 'W', 56: 'A', 57: 'v', 58: '?', 59: '2', 60: '~', 61: 's', 62: 'T', 63: 'R', 64: ',', 65: '|', 66: '4', 67: '>', 68: 'y', 69: '(', 70: '[', 71: 'k', 72: 'H', 73: 'l', 74: 'j', 75: '7', 76: 'n', 77: 'i', 78: 'D', 79: 'Q', 80: ' ', 81: 'm', 82: 'Y', 83: '*', 84: '}', 85: '#', 86: 'p', 87: 'q', 88: '5', 89: 'c', 90: '`', 91: 'a', 92: 'b', 93: 'w', 94: '3', 95: 'E', 96: ';', 97: '-'}\n",
      "Output encoding {'o': 2, '{': 3, '.': 4, 'J': 5, '0': 6, '1': 7, '<': 8, 'B': 9, 'd': 10, '£': 11, 'e': 12, '6': 13, '!': 14, 'O': 15, 'M': 16, 'X': 17, 'f': 18, 't': 19, 'C': 20, 'V': 21, 'z': 22, 'K': 23, '\\\\': 24, '9': 25, 'P': 26, 'S': 27, '/': 28, '₹': 29, 'F': 30, 'G': 31, '=': 32, '8': 33, ')': 34, '+': 35, ']': 36, 'U': 37, \"'\": 38, '\"': 39, 'g': 40, 'N': 41, 'r': 42, 'u': 43, '&': 44, '$': 45, 'x': 46, '%': 47, ':': 48, '@': 49, '^': 50, 'I': 51, 'L': 52, 'Z': 53, 'h': 54, 'W': 55, 'A': 56, 'v': 57, '?': 58, '2': 59, '~': 60, 's': 61, 'T': 62, 'R': 63, ',': 64, '|': 65, '4': 66, '>': 67, 'y': 68, '(': 69, '[': 70, 'k': 71, 'H': 72, 'l': 73, 'j': 74, '7': 75, 'n': 76, 'i': 77, 'D': 78, 'Q': 79, ' ': 80, 'm': 81, 'Y': 82, '*': 83, '}': 84, '#': 85, 'p': 86, 'q': 87, '5': 88, 'c': 89, '`': 90, 'a': 91, 'b': 92, 'w': 93, '3': 94, 'E': 95, ';': 96, '-': 97}\n",
      "Output decoding {2: 'o', 3: '{', 4: '.', 5: 'J', 6: '0', 7: '1', 8: '<', 9: 'B', 10: 'd', 11: '£', 12: 'e', 13: '6', 14: '!', 15: 'O', 16: 'M', 17: 'X', 18: 'f', 19: 't', 20: 'C', 21: 'V', 22: 'z', 23: 'K', 24: '\\\\', 25: '9', 26: 'P', 27: 'S', 28: '/', 29: '₹', 30: 'F', 31: 'G', 32: '=', 33: '8', 34: ')', 35: '+', 36: ']', 37: 'U', 38: \"'\", 39: '\"', 40: 'g', 41: 'N', 42: 'r', 43: 'u', 44: '&', 45: '$', 46: 'x', 47: '%', 48: ':', 49: '@', 50: '^', 51: 'I', 52: 'L', 53: 'Z', 54: 'h', 55: 'W', 56: 'A', 57: 'v', 58: '?', 59: '2', 60: '~', 61: 's', 62: 'T', 63: 'R', 64: ',', 65: '|', 66: '4', 67: '>', 68: 'y', 69: '(', 70: '[', 71: 'k', 72: 'H', 73: 'l', 74: 'j', 75: '7', 76: 'n', 77: 'i', 78: 'D', 79: 'Q', 80: ' ', 81: 'm', 82: 'Y', 83: '*', 84: '}', 85: '#', 86: 'p', 87: 'q', 88: '5', 89: 'c', 90: '`', 91: 'a', 92: 'b', 93: 'w', 94: '3', 95: 'E', 96: ';', 97: '-'}\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:3673: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 202)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 202)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 202, 256)     25088       input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 202, 128)     12544       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, 202, 256)     525312      embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 202, 256)     263168      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 202, 202)     0           lstm_2[0][0]                     \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "attention (Activation)          (None, 202, 202)     0           dot_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, 202, 256)     0           attention[0][0]                  \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 202, 512)     0           dot_2[0][0]                      \n",
      "                                                                 lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 202, 128)     65664       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, 202, 98)      12642       time_distributed_1[0][0]         \n",
      "==================================================================================================\n",
      "Total params: 904,418\n",
      "Trainable params: 904,418\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Hi! My PID is 31571\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import time\n",
    "from datetime import datetime\n",
    "import networkx as nx\n",
    "import convex as cx\n",
    "import tmqa35 as graphqa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import importlib\n",
    "#import tmqa35 as graphqa\n",
    "#importlib.reload(graphqa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "conversations_path = \"/data/users/romain.claret/tm/mse.tm.chatbot.base/data/convex/test_set/test_set_ALL.json\"\n",
    "with open(conversations_path, \"r\") as data:\n",
    "    conversations = json.load(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pickle_data(df, filename):\n",
    "    pd.set_option('display.max_columns', None)\n",
    "    filename = \"/data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/\"+filename+'.pickle.bz2'\n",
    "    #df.summary = df.summary.map(sanitize_str)\n",
    "    print(\"Saving Dataframe Done!\",filename)\n",
    "    return df.to_pickle(filename, compression='bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardize_graph(graph):\n",
    "    this_graph = graph.copy()\n",
    "    for n in this_graph.nodes():\n",
    "        n_pos = n.find(\"-\")\n",
    "        n_name = n\n",
    "        if n_pos != -1: n_name = n[:n_pos]\n",
    "        this_graph.nodes[n][\"name\"] = graphqa.get_wd_label(n_name)\n",
    "        this_graph.nodes[n][\"weight\"] = 1\n",
    "        \n",
    "    return this_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lang: en, fr, de, it, es, zh\n",
    "#kb: dbpedia, wikidata, dblp, freebase\n",
    "def ask_qanswer(question):\n",
    "    data = {'query': question,'lang': 'en','kb': 'wikidata'}\n",
    "    headers = {\"Authorization\":\"Bearer eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIzNDIiLCJpYXQiOjE1NzkyNTYxNDQsImV4cCI6MTU3OTg2MDk0NH0.YPFBZ-Xc8OI7eeTTkQaVT5a-CA5VONiCr_VIViG3t8tjVv7eRKgz_X_1KWDnly_F08rLXwpPcDUMBt8_M8-S8w\"}\n",
    "    query = requests.post('http://qanswer-core1.univ-st-etienne.fr/api/gerbil', data=data, headers=headers)\n",
    "    \n",
    "#    var settings = {\n",
    "#  \"async\": true,\n",
    "#  \"crossDomain\": true,\n",
    "#  \"url\": \"http://qanswer-core1.univ-st-etienne.fr/api/qa/full?question=what%20is%20a%20margerita&lang=en&kb=cocktails\",\n",
    "#  \"method\": \"GET\",\n",
    "#  \"headers\": {\n",
    "#    \"Authorization\": \"Bearer eyJhbGciOiJIUzUxMi.....\",\n",
    "#  }\n",
    "#}\n",
    "    \n",
    "    if not query:\n",
    "        return False,False\n",
    "    if (query.json()['questions'][0]['question']['answers']) == None:\n",
    "        return False,False\n",
    "    #if (query.json()['questions'][0]['question']['answers'].replace('\\n', '')) == None:\n",
    "    #    return False\n",
    "    #print(query.json()['questions'][0]['question']['answers'].replace('\\n', '').get(\"results\"))\n",
    "    try:\n",
    "        response = (json.loads(query.json()\n",
    "                .get(\"questions\")[0]\n",
    "                .get(\"question\")\n",
    "                .get(\"answers\")\n",
    "                .replace('\\n', ''))\n",
    "         .get(\"results\").get(\"bindings\"))\n",
    "    except:\n",
    "        return False,False\n",
    "    \n",
    "    if response:\n",
    "        result = response[0].get(\"o1\").get(\"value\")[len(\"http://www.wikidata.org/entity/\"):] if response[0].get(\"o1\") is not None else False\n",
    "        return result,False\n",
    "    else:\n",
    "        return False,False\n",
    "\n",
    "#ask_qanswer(\"Who is the wife of Barack Obama\")\n",
    "#ask_qanwser(\"Which equestrian was born in dublin?\")\n",
    "#ask_qanswer(\"what is the main language spoken in a ghentar si muore facile\")\n",
    "#ask_qanswer(\"was the film helpmates in color or black-and-white?\")\n",
    "#ask_qanswer(\"how does engelbert zaschka identify\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_platypus(question):\n",
    "    headers = {'Accept': 'application/json','Accept-Language': 'en',}\n",
    "    params = (('q', question),('lang', 'en'))\n",
    "\n",
    "    response = requests.get('https://qa.askplatyp.us/v0/ask', headers=headers, params=params)\n",
    "    if response:\n",
    "        if type(response.json()['member']) is list:\n",
    "            #print(response.json()['member'][0]['result'])\n",
    "            if response.json()['member'] != []:\n",
    "                if '@id' in (json.dumps(response.json()['member'][0]['result'])):\n",
    "                    try:\n",
    "                        ps_result = (json.dumps(response.json()['member'][0]['result']['@id']))\n",
    "                    except:\n",
    "                        return False, False\n",
    "                else: return False, False\n",
    "            else: return False, False\n",
    "        else:\n",
    "            try:\n",
    "                if '@id' in (json.dumps(response.json()['member']['result'])):\n",
    "                    ps_result = (json.dumps(response.json()[\"member\"]['result']['@id']))\n",
    "                else: return False, False\n",
    "            except:\n",
    "                return False, False\n",
    "    else: return False, False\n",
    "    ps_result = ps_result[4:-1]\n",
    "    #print(result[:1])\n",
    "    #if ps_result[:1] != 'P' and ps_result[:1] != 'Q':\n",
    "    #    return False, False\n",
    "    return ps_result,False\n",
    "\n",
    "#ask_platypus(\"Which genre of album is harder.....faster?\")\n",
    "#ask_platypus(\"how does engelbert zaschka identify\")\n",
    "#ask_platypus(\"Which Swiss conductor's cause of death is myoc...\")\n",
    "#ask_platypus(\"where was padraic mcguinness's place of death\")\n",
    "#ask_platypus(\"was the film helpmates in color or black-and-white?\")\n",
    "#ask_platypus(\"Who created the show life on earth\")\n",
    "#ask_platypus(\"Who is the wife of Barack Obama\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_convex(question):\n",
    "    cx_result = cx.answer_complete_question(question, cx.tagmeToken)\n",
    "    graph = cx.gp.expand_context_with_statements(None, [cx_result['context']], qa=True) \n",
    "    graph = standardize_graph(graph)\n",
    "    #print(cx_result)\n",
    "    #answer = str(cx.wd.wikidata_id_to_label(result['answers'][0]['answer']))\n",
    "    try:\n",
    "        if not cx_result:\n",
    "            return False, False\n",
    "        return [[r[\"answer\"] for r in cx_result['answers']],\n",
    "                [cx_result['context'][\"entity\"][\"id\"],cx_result['context'][\"predicate\"][\"id\"],cx_result['context'][\"object\"][\"id\"]]], graph\n",
    "    except:\n",
    "        return False, False\n",
    "\n",
    "#ask_convex(\"Which actor voiced the Unicorn in The Last Unicorn?\")\n",
    "#ask_convex(\"Which genre of album is harder.....faster?\")\n",
    "#ask_convex(\"Which label is somevelvetsidewalk signed to ttle of fort fisher \")\n",
    "#ask_convex(\"Who is the wife of Barack Obama\")\n",
    "#ask_convex(\"100% senorita is a television show in what language?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_graphqa(question, verbose=True, timer=True, show_graph=False, cores=graphqa.mp.cpu_count(), banning_str=False,\n",
    "            answer_context=False, context_graph=False, use_convex=False, turn=1):\n",
    "    \n",
    "    frontier_detection=[0.9, 0.6, 0.3] #random_access\n",
    "    answer_detection=[0.9, 0.1] #total_distance_qa_nodes, total_distance_frontiers\n",
    "    frontiers=3\n",
    "    \n",
    "    if use_convex:\n",
    "        if not context_graph:\n",
    "            context_graph=nx.Graph()\n",
    "            if answer_context:\n",
    "                if answer_context[0]:\n",
    "                    context_graph.add_node(answer_context[0][0], name=graphqa.get_wd_label(answer_context[0][0]), type='entity', turn=i_q+1, weight=1, qa=True)\n",
    "                \n",
    "        answer_context_convex,context_graph = cx.answer_follow_up_question(question, turn, context_graph, frontier_detection+answer_detection, frontiers)\n",
    "        if context_graph: context_graph = standardize_graph(context_graph)\n",
    "        #answer_context = False\n",
    "        #print(\"answer_context\",answer_context[0]['rank'])\n",
    "        \n",
    "        answer_context=[]\n",
    "        for ac in answer_context_convex:\n",
    "            answer_context.append(ac[\"answer\"])\n",
    "        answer_context = [answer_context,[]]\n",
    "        \n",
    "        if show_graph: graphqa.plot_graph(context_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "        #if verbose: print(\"Answer:\",graphqa.convert_to_literal(graphqa.get_wd_label(answer_context[0][0])), \"(\"+str(answer_context[0][0])+\")\\n\")\n",
    "        result = answer_context,context_graph\n",
    "    \n",
    "    else:\n",
    "        result = graphqa.answer_question(\n",
    "            question, verbose=verbose, timer=timer, show_graph=show_graph, cores=cores, banning_str=banning_str,\n",
    "            previous_answer=answer_context, previous_graph=context_graph)\n",
    "                    \n",
    "    if not result:\n",
    "        return (False,False)\n",
    "    #if result == (False,False):\n",
    "    #    return (False,False)\n",
    "\n",
    "    return result\n",
    "\n",
    "conversation_questions = [\n",
    "    \"Which actor voiced the Unicorn in The Last Unicorn?\",\n",
    "    \"And Alan Arkin was behind..\",\n",
    "    \"Who did the score?\",\n",
    "    \"So who performed the songs?\",\n",
    "    \"Genre of this band's music?\",\n",
    "    \"By the way, who was the director?\",\n",
    "    \"Is Alan Arkin in the cast ?\",\n",
    "]\n",
    "\n",
    "\n",
    "#for i_q,question in enumerate(conversation_questions):\n",
    "#    if i_q >= 0:\n",
    "#        if i_q == 0:\n",
    "#            answer_context_1,context_graph_1 = ask_graphqa(question ,answer_context=False, context_graph=False, verbose=True, timer=True, show_graph=True)\n",
    "#            answer_context = answer_context_1.copy()\n",
    "#            context_graph = context_graph_1.copy()\n",
    "#        elif context_graph:\n",
    "#            print(\"Context Question:\",question)\n",
    "#            answer_context,context_graph = ask_graphqa(question,answer_context=answer_context, context_graph=context_graph, verbose=True, timer=True, show_graph=True)\n",
    "#        else:\n",
    "#            print(\"NO CONTEXT ERROR\")\n",
    "#            break\n",
    "#\n",
    "#    if answer_context: print(\"Answer:\",graphqa.convert_to_literal(graphqa.get_wd_label(answer_context[0][0])), \"(\"+str(answer_context[0][0])+\")\\n\")\n",
    "#    #break\n",
    "\n",
    "#for i_q,question in enumerate(conversation_questions):\n",
    "#    if i_q >= 0:\n",
    "#        if i_q == 0:\n",
    "#            #answer_context_1,context_graph_1 = ask_graphqa(question ,previous_answer=False, previous_graph=False, verbose=True, timer=True, show_graph=True)\n",
    "#            answer_context = answer_context_1.copy()\n",
    "#            context_graph = context_graph_1.copy()\n",
    "#            continue\n",
    "#        elif context_graph:\n",
    "#            print(\"Context Question:\",question)\n",
    "#            answer_context,context_graph = ask_graphqa(question,answer_context=answer_context, context_graph=context_graph, verbose=True, timer=True, show_graph=True,\n",
    "#                                                       use_convex=True, turn=i_q+1)\n",
    "#        else:\n",
    "#            print(\"NO CONTEXT ERROR\")\n",
    "#            break\n",
    "#\n",
    "#    if answer_context: print(\"Answer:\",graphqa.convert_to_literal(graphqa.get_wd_label(answer_context[0][0])), \"(\"+str(answer_context[0][0])+\")\\n\")\n",
    "#    #break\n",
    "\n",
    "#answer_convex,context_graph = answer_conversation(conversation_questions,answer_convex=False,context_graph=False)\n",
    "\n",
    "#answer = ask_graphqa(\"Which actor voiced the Unicorn in The Last Unicorn?\", verbose=True)\n",
    "#answer = ask_graphqa(\"what's akbar tandjung's ethnicity\", verbose=True)\n",
    "#ask_graphqa(\"Which genre of album is harder.....faster?\")\n",
    "#ask_graphqa(\"Which label is somevelvetsidewalk signed to ttle of fort fisher \")\n",
    "#ask_graphqa(\"Who is the wife of Barack Obama\")\n",
    "#print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#answer_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "HEADERS = ['conversation_id','turn',\"plus_convex\",\n",
    "           'question', 'answer', 'domain',\n",
    "           'qanswer','qanswer_time', 'qanswer_rr',\n",
    "           'platypus','platypus_time', 'platypus_rr',\n",
    "           'convex','convex_time', 'convex_rr',\n",
    "           'graphqa', \"graphqa_time\", \"graphqa_top2\", \"graphqa_top3\", \"graphqa_top4\", \"graphqa_top5\", \"graphqa_topall\",\"graphqa_rr\"]\n",
    "#df = pd.DataFrame(columns=HEADERS)\n",
    "#LOADING\n",
    "df = pd.read_pickle(\"/data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-135-ic1513-iq4-pcTrue.pickle.bz2\")\n",
    "#len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for conversation in conversations:\n",
    "#    questions = [turn['question'] for turn in conversation['questions']]\n",
    "#    print(questions)\n",
    "#    golden_answers = [graphqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "#    print(golden_answers)\n",
    "#    \n",
    "#    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#banning_str = False #[[\"ř\",\"r\"]]\n",
    "#conversations_len = len(conversations)\n",
    "#\n",
    "#last_i = 0\n",
    "#for i, conversation in enumerate(conversations):\n",
    "#    if i >= last_i:\n",
    "#        print(\"\\n-->\",str(i)+\"/\"+str(conversations_len), \"New conversation\")\n",
    "#        questions = [turn['question'] for turn in conversation['questions']]\n",
    "#        questions_len = len(questions)\n",
    "#        for i_q, question in enumerate(questions):\n",
    "#            print(str(i_q)+\"/\"+str(questions_len),question)\n",
    "#            print(tmqa.get_nlp(question,autocorrect=True,banning_str=banning_str))\n",
    "#    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rr(results, answer):\n",
    "    if answer in results:\n",
    "        ans_position = results.index(answer)+1\n",
    "        if ans_position == 1:\n",
    "            return 1.0\n",
    "        return float(ans_position/len(results))\n",
    "    else: return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It is  2020-02-06 11:52:45.743261\n",
      "\t>>> Processing 1515/2240 -> 1/5 -> Convex=False: (Q34660) Who wrote Harry Potter?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer Q34660\n",
      "df_qanswer_rr 1.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Asking Convex\n",
      "df_convex Q34660\n",
      "df_convex_rr 1.0\n",
      "\n",
      "CORRECT 1515 - 1 -> qAnswer Q34660\n",
      "\n",
      "CORRECT 1515 - 1 -> Convex Q34660\n",
      "\n",
      "Asking GraphQA\n",
      "User input: Who wrote Harry Potter?\n",
      "--> Auto correcting question in progress...\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "-> Auto corrected q_nlp: Who wrote Harry Potter \n",
      "-> q_themes: ([(Harry Potter, ['Q8337', 'Q3244512']), (Potter, ['Q3400050', 'Q15299467'])], [wrote Harry])\n",
      "-> q_themes_enhanced: [('write', ['Q29465908']), ('Write', ['Q1215628'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(wrote, ['P1412'])]\n",
      "-> q_predicates \tRunning time is 15.81s\n",
      "--> Potential meaningful keywords for the sentence: ['Harry Potter', 'Potter', 'write', 'Write']\n",
      "q_focused_parts: [(Harry Potter, ['Q216930', 'Q1809924', 'Q17146193'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 71.7s\n",
      "-->  90 nodes and 88 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 77 nodes and 76 edges\n",
      "---> Rebuilding the graph with k_deep 4 ... Previously: 77 nodes or 76 edges was below the limit of 100\n",
      "->New graph \tRunning time is 63.09s\n",
      "-->  102 nodes and 100 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 87 nodes and 86 edges\n",
      "---> Rebuilding the graph with k_deep 5 ... Previously: 87 nodes or 86 edges was below the limit of 100\n",
      "->New graph \tRunning time is 51.58s\n",
      "-->  129 nodes and 130 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 108 nodes and 110 edges\n",
      "-> predicates_dict: {'P1412': 2, 'P3744': 1, 'P407': 3, 'P3984': 1, 'P1559': 1, 'P1013': 1, 'P805': 2, 'P1343': 2, 'P2868': 4, 'P582': 1, 'P20': 1, 'P570': 1, 'P641': 1, 'P1881': 2, 'P3989': 1, 'P136': 3, 'P2521': 3, 'P31': 5, 'P50': 1, 'P3831': 2, 'P674': 2, 'P585': 1, 'P108': 1, 'P1039': 4, 'P1038': 4, 'P361': 1, 'P463': 1, 'P138': 3, 'P642': 1, 'P376': 1, 'P1445': 1, 'P800': 2, 'P1545': 1, 'P735': 1, 'P1113': 1, 'P157': 1, 'P625': 1}\n",
      "-> paths_keywords: (['harry potter', 'henry potter'], {'languages spoken, written or signed': [languages spoken written or signed, ['P1412']]}, [Who])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 186\n",
      "->Computing possible paths \tRunning time is 18.49s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 186\n",
      "->\tRunning time is 3.04s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q1860', 1.9398311273961375], ['Q215972', 1.08146078630692], ['Q677191', 0.6729063979977694]]\n",
      "->Computing hypothesises \tRunning time is 16.46s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 18\n",
      "->\tRunning time is 12.48s\n",
      "--> len(cleared_golden_paths): 17\n",
      "---> First path: ['Q677191', 'P735', 'Q3244512', 'P138', 'Q717594']\n",
      "->\tTotal Running time is 256.18s\n",
      "\n",
      "df_graphqa Q677191\n",
      "df_graphqa_rr 0.0\n",
      "    conversation_id turn plus_convex                 question  answer domain  \\\n",
      "135            1514    0       False  Who wrote Harry Potter?  Q34660  books   \n",
      "\n",
      "    qanswer  qanswer_time  qanswer_rr platypus  ...  convex_time  convex_rr  \\\n",
      "135  Q34660          0.69         1.0    False  ...         3.94        1.0   \n",
      "\n",
      "     graphqa  graphqa_time  graphqa_top2 graphqa_top3  graphqa_top4  \\\n",
      "135  Q677191        256.43         False        False         False   \n",
      "\n",
      "    graphqa_top5 graphqa_topall graphqa_rr  \n",
      "135        False          False        0.0  \n",
      "\n",
      "[1 rows x 23 columns]\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-136-ic1514-iq0-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 11:57:07.193122\n",
      "\t>>> Processing 1515/2240 -> 2/5 -> Convex=False: (1997-01-01T00:00:00Z) What was the year of publication for the first book?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: What was the year of publication for the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What was the date of publication for the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What was the date of publication for the first book\n",
      "> Time related question detected\n",
      "-> q_themes: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book date])\n",
      "-> q_themes_enhanced: [('Date', ['Q36603893'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (date, ['P837']), (publication, ['P577']), (first, ['P577']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 11.61s\n",
      "--> Predicates enhanced by previous context: [(be, ['P31']), (date, ['P837']), (publication, ['P577']), (book, ['P50'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [first, publication, date, book, Book, Publication, first book, Date, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (J. K. Rowling, ['Q34660']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 40.55s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: What was the year of publication for the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What was the date of publication for the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: What the date of publication for the first book\n",
      "> Time related question detected\n",
      "-> q_themes: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book date])\n",
      "-> q_themes_enhanced: [('Date', ['Q36603893'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (date, ['P837']), (publication, ['P577']), (first, ['P577']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 10.19s\n",
      "--> Predicates enhanced by previous context: [(be, ['P31']), (date, ['P837']), (publication, ['P577']), (book, ['P50'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date', 'J. K. Rowling']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meaningful_names_no_previous_answer [first, publication, date, book, Book, Publication, first book, Date, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (J. K. Rowling, ['Q34660']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 40.2s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: What was the year of publication for the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What was the date of publication for the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What was the date of publication for the first book\n",
      "> Time related question detected\n",
      "-> q_themes: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book date])\n",
      "-> q_themes_enhanced: [('Date', ['Q36603893'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (date, ['P837']), (publication, ['P577']), (first, ['P577']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 9.22s\n",
      "--> Predicates enhanced by previous context: [(author, ['P50']), (be, ['P31']), (date, ['P837']), (publication, ['P577'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date', 'Harry Potter', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [first, publication, date, book, Book, Publication, first book, Date, Harry Potter, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 63.37s\n",
      "-->  19 nodes and 24 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 19 nodes and 24 edges\n",
      "-> predicates_dict: {'P50': 27, 'P106': 1, 'P582': 3, 'P157': 1, 'P585': 5, 'P2868': 3, 'P20': 1, 'P1686': 3, 'P1411': 3, 'P1932': 3, 'P569': 3, 'P570': 2, 'P577': 2, 'P580': 3, 'P2031': 1, 'P1672': 1, 'P180': 1, 'P186': 1, 'P31': 16, 'P428': 1, 'P291': 1, 'P69': 2, 'P812': 1, 'P279': 7, 'P2408': 1, 'P1013': 5, 'P19': 1, 'P1476': 1, 'P805': 1, 'P1343': 1, 'P1104': 1, 'P3245': 1, 'P3250': 2, 'P1441': 9, 'P642': 4, 'P407': 5, 'P443': 1, 'P3744': 2, 'P3984': 1, 'P123': 1, 'P1113': 1, 'P1557': 2, 'P735': 1, 'P3831': 1, 'P674': 1, 'P1477': 1, 'P747': 1, 'P921': 1, 'P958': 1, 'P92': 1, 'P170': 1, 'P1881': 1, 'P856': 1, 'P136': 1, 'P58': 1, 'P1709': 1, 'P344': 1, 'P1114': 1, 'P1552': 1, 'P641': 1, 'P282': 1, 'P453': 3, 'P161': 3, 'P2002': 1, 'P1344': 1, 'P793': 1, 'P734': 1, 'P2096': 1, 'P373': 1, 'P527': 2, 'P1545': 1}\n",
      "-> paths_keywords: (['first', 'publication', 'date', 'book', 'harry potter', 'j. k. rowling', 'first book'], {'author': [author, ['P50']], 'instance of': [instance of, ['P31']], 'day in year for periodic occurrence': [day in date for periodic occurrence, ['P837']], 'date of publication': [date of publication, ['P577']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 168\n",
      "->Computing possible paths \tRunning time is 18.37s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 156\n",
      "->\tRunning time is 3.78s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q5', 0.6633366750279106], ['Q215972', 0.4929079908427412], ['1997-01-01T00:00:00Z', 0.4121800461827992]]\n",
      "->Computing hypothesises \tRunning time is 106.11s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 10\n",
      "->\tRunning time is 4.55s\n",
      "--> len(cleared_golden_paths): 5\n",
      "---> First path: ['1997-01-01T00:00:00Z', 'P577', 'Q8337', 'P50', 'Q34660', 'P31', 'Q5']\n",
      "->\tTotal Running time is 209.71s\n",
      "\n",
      "df_convex 1997-01-01T00:00:00Z\n",
      "df_convex_rr 1.0\n",
      "\n",
      "CORRECT 1515 - 2 -> Convex 1997-01-01T00:00:00Z\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: What was the year of publication for the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What was the date of publication for the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What was the date of publication for the first book\n",
      "> Time related question detected\n",
      "-> q_themes: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book date])\n",
      "-> q_themes_enhanced: [('Date', ['Q36603893'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (date, ['P837']), (publication, ['P577']), (first, ['P577']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 10.93s\n",
      "--> Predicates enhanced by previous context: [(characters, ['P674']), (be, ['P31']), (date, ['P837']), (publication, ['P577']), (book, ['P50']), (given name, ['P735']), (named after, ['P138'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (publication, ['Q732577', 'Q15852766']), (date, ['Q3016931', 'Q1652093']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (Publication, ['Q51523527', 'Q15728967']), (first book, ['Q1419297'])], [Book])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'publication', 'date', 'book', 'Book', 'Publication', 'first book', 'Date', 'Harry Potter', 'Harry Potter', 'James Potter', 'James']\n",
      "meaningful_names_no_previous_answer [first, publication, date, book, Book, Publication, first book, Date, Harry Potter, Harry Potter, James Potter, James]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (James, ['Q1277864', 'Q16654695', 'Q12188082'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (publication, ['Q15852766']), (date, ['Q1652093', 'Q3016931']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Publication, ['Q51523527', 'Q15728967']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (James, ['Q1277864', 'Q16654695', 'Q12188082']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "->New graph \tRunning time is 111.71s\n",
      "-->  35 nodes and 48 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 35 nodes and 48 edges\n",
      "-> predicates_dict: {'P674': 26, 'P642': 5, 'P138': 19, 'P735': 5, 'P1013': 6, 'P582': 2, 'P157': 2, 'P585': 1, 'P2868': 3, 'P20': 2, 'P569': 4, 'P570': 4, 'P577': 1, 'P580': 2, 'P571': 2, 'P1672': 1, 'P180': 1, 'P186': 3, 'P1932': 3, 'P50': 1, 'P1545': 3, 'P31': 22, 'P428': 1, 'P291': 1, 'P407': 3, 'P443': 1, 'P734': 6, 'P1881': 1, 'P279': 8, 'P2354': 1, 'P1104': 1, 'P3245': 1, 'P3250': 2, 'P453': 3, 'P161': 3, 'P1441': 10, 'P1877': 1, 'P509': 1, 'P518': 1, 'P361': 1, 'P527': 5, 'P1557': 2, 'P19': 1, 'P195': 1, 'P217': 1, 'P1039': 1, 'P1038': 1, 'P373': 1, 'P747': 1, 'P958': 1, 'P92': 1, 'P1709': 1, 'P170': 3, 'P856': 2, 'P364': 1, 'P58': 1, 'P282': 2, 'P344': 1, 'P136': 1, 'P910': 1, 'P1114': 1, 'P1552': 1, 'P1412': 2, 'P264': 1, 'P641': 1, 'P4675': 1, 'P1344': 1, 'P17': 1, 'P21': 1, 'P793': 1, 'P495': 1, 'P973': 1, 'P175': 1}\n",
      "-> paths_keywords: (['first', 'publication', 'date', 'book', 'harry potter', 'james', 'first book'], {'characters': [characters, ['P674']], 'instance of': [instance of, ['P31']], 'day in year for periodic occurrence': [day in date for periodic occurrence, ['P837']], 'date of publication': [date of publication, ['P577']], 'author': [author, ['P50']], 'given name': [given name, ['P735']], 'named after': [named after, ['P138']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 1476\n",
      "->Computing possible paths \tRunning time is 18.44s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 1476\n",
      "->\tRunning time is 4.32s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q34660', 11.429856764654263], ['Q20711488', 1.960234416665501], ['Q46758', 1.1707410001059717], ['Q1860', 1.1428834096775282]]\n",
      "->Computing hypothesises \tRunning time is 226.7s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 2\n",
      "->\tRunning time is 8.97s\n",
      "--> len(cleared_golden_paths): 1\n",
      "---> First path: ['Q34660', 'P170', 'Q717594', 'P674', 'Q8337', 'P138', 'Q3244512']\n",
      "->\tTotal Running time is 387.57s\n",
      "\n",
      "df_graphqa Q34660\n",
      "df_graphqa_rr 0.0\n",
      "    conversation_id turn plus_convex  \\\n",
      "136            1514    1       False   \n",
      "\n",
      "                                              question                answer  \\\n",
      "136  What was the year of publication for the first...  1997-01-01T00:00:00Z   \n",
      "\n",
      "    domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "136  books   False        102.89         0.0    False           2.12   \n",
      "\n",
      "     platypus_rr                convex  convex_time  convex_rr graphqa  \\\n",
      "136          0.0  1997-01-01T00:00:00Z       209.92        1.0  Q34660   \n",
      "\n",
      "     graphqa_time graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5  \\\n",
      "136        387.84        False        False        False        False   \n",
      "\n",
      "    graphqa_topall  graphqa_rr  \n",
      "136          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-137-ic1514-iq1-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1515/2240 -> 2/5 -> Convex=True: (1997-01-01T00:00:00Z) What was the year of publication for the first book?                                  \n",
      "qAnswer extended by Convex\n",
      "df_qanswer Q1148668\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex 1997-01-01T00:00:00Z\n",
      "df_convex_rr 1.0\n",
      "\n",
      "CORRECT 1515 - 2 -> Convex 1997-01-01T00:00:00Z\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa 1997-01-01T00:00:00Z\n",
      "df_graphqa_rr 1.0\n",
      "\n",
      "CORRECT 1515 - 2 -> graphqa 1997-01-01T00:00:00Z\n",
      "\n",
      "PARTIAL_CORRECT 1515 - 2 -> graphqa in answers ['1997-01-01T00:00:00Z', 'Q34660']\n",
      "    conversation_id turn plus_convex  \\\n",
      "137            1514    1        True   \n",
      "\n",
      "                                              question                answer  \\\n",
      "137  What was the year of publication for the first...  1997-01-01T00:00:00Z   \n",
      "\n",
      "    domain   qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "137  books  Q1148668          1.21         0.0    False           2.03   \n",
      "\n",
      "     platypus_rr                convex  convex_time  convex_rr  \\\n",
      "137          0.0  1997-01-01T00:00:00Z         1.87        1.0   \n",
      "\n",
      "                  graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "137  1997-01-01T00:00:00Z          2.24         True         True   \n",
      "\n",
      "    graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "137         True         True           True         1.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-138-ic1514-iq1-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 12:08:57.347886\n",
      "\t>>> Processing 1515/2240 -> 3/5 -> Convex=False: (Q43361) Title of the first book?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: Title of the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Title of the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Title of the first book\n",
      "-> q_themes: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(title, [])]\n",
      "-> q_predicates \tRunning time is 4.76s\n",
      "--> Predicates enhanced by previous context: [(title, [])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'Title', 'book', 'Book', 'title', 'first book']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'Title', 'book', 'Book', 'title', 'first book', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [first, Title, book, Book, title, first book, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (J. K. Rowling, ['Q34660']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 41.56s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Title of the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Title of the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Title of the first book\n",
      "-> q_themes: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(title, []), (Title, ['P1476']), (first, ['P577']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 5.31s\n",
      "--> Predicates enhanced by previous context: [(title, []), (Title, ['P1476']), (first, ['P577']), (book, ['P50'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'Title', 'book', 'Book', 'title', 'first book']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'Title', 'book', 'Book', 'title', 'first book', 'J. K. Rowling']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meaningful_names_no_previous_answer [first, Title, book, Book, title, first book, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (J. K. Rowling, ['Q34660']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 45.68s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: Title of the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Title of the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Title of the first book\n",
      "-> q_themes: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(title, [])]\n",
      "-> q_predicates \tRunning time is 4.85s\n",
      "--> Predicates enhanced by previous context: [(author, ['P50']), (title, []), (date of publication, ['P577']), (instance of, ['P31'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'Title', 'book', 'Book', 'title', 'first book']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'Title', 'book', 'Book', 'title', 'first book', 'Harry Potter', 'J. K. Rowling', 'Harry Potter', '1997-01-01T00:00:00Z', 'human']\n",
      "meaningful_names_no_previous_answer [first, Title, book, Book, title, first book, Harry Potter, J. K. Rowling, Harry Potter, 1997 - 01 01T00:00:00Z, human]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (human, ['Q5', 'Q20094897', 'Q26190966'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (human, ['Q5', 'Q20094897', 'Q26190966']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 80.01s\n",
      "-->  27 nodes and 32 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 27 nodes and 32 edges\n",
      "-> predicates_dict: {'P50': 26, 'P585': 9, 'P2031': 1, 'P577': 4, 'P1441': 3, 'P674': 2, 'P170': 2, 'P642': 6, 'P31': 533, 'P106': 3, 'P1113': 1, 'P1476': 1, 'P805': 1, 'P1343': 1, 'P279': 6, 'P569': 3, 'P2868': 4, 'P582': 3, 'P570': 2, 'P166': 2, 'P157': 1, 'P20': 1, 'P580': 2, 'P69': 2, 'P1013': 2, 'P3744': 1, 'P407': 3, 'P3984': 1, 'P463': 1, 'P1971': 1, 'P812': 1, 'P495': 1, 'P1039': 3, 'P1038': 3, 'P3831': 1, 'P19': 1, 'P1686': 5, 'P1411': 6, 'P360': 3, 'P527': 3, 'P108': 1, 'P123': 1, 'P1881': 1, 'P136': 3, 'P1557': 2, 'P1424': 1, 'P138': 4, 'P1545': 3, 'P735': 4, 'P1423': 1, 'P453': 3, 'P161': 3, 'P58': 1, 'P344': 1, 'P1114': 1, 'P1552': 1, 'P910': 2, 'P641': 1, 'P282': 1, 'P373': 1, 'P747': 1, 'P734': 1, 'P2096': 1, 'P21': 1}\n",
      "-> paths_keywords: (['first', 'book', 'title', 'harry potter', 'j. k. rowling', 'human', 'first book'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 286\n",
      "->Computing possible paths \tRunning time is 41.28s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 284\n",
      "->\tRunning time is 2.76s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['1997-01-01T00:00:00Z', 0.19625126011352115], ['Q3246821', 0.16989719433423453], ['Q215972', 0.12216983225469241]]\n",
      "->Computing hypothesises \tRunning time is 50.63s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 3\n",
      "->\tRunning time is 9.9s\n",
      "--> len(cleared_golden_paths): 3\n",
      "---> First path: ['1997-01-01T00:00:00Z', 'P585', 'Q34660', 'P50', 'Q8337', 'P1441', 'Q3244512', 'P2868', 'Q3246821']\n",
      "->\tTotal Running time is 192.67s\n",
      "\n",
      "df_convex 1997-01-01T00:00:00Z\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: Title of the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Title of the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Title of the first book\n",
      "-> q_themes: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(title, [])]\n",
      "-> q_predicates \tRunning time is 4.87s\n",
      "--> Predicates enhanced by previous context: [(characters, ['P674']), (title, []), (creator, ['P170']), (named after, ['P138'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'Title', 'book', 'Book', 'title', 'first book']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'Title', 'book', 'Book', 'title', 'first book', 'Harry Potter', 'Harry Potter', 'James Potter', 'Harry Potter', 'J. K. Rowling', 'James']\n",
      "meaningful_names_no_previous_answer [first, Title, book, Book, title, first book, Harry Potter, Harry Potter, James Potter, Harry Potter, J. K. Rowling, James]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (James, ['Q1277864', 'Q16654695', 'Q12188082'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (James, ['Q1277864', 'Q16654695', 'Q12188082']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 114.04s\n",
      "-->  126 nodes and 170 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 126 nodes and 170 edges\n",
      "---> Rebuilding the graph with k_deep 2 ... Previously: 126 nodes or 170 edges was above the limit of 100\n",
      "-> predicates_dict: {'P674': 58, 'P642': 3, 'P138': 36, 'P453': 4, 'P1441': 8, 'P1877': 4, 'P170': 62, 'P735': 6, 'P1476': 1, 'P805': 1, 'P1343': 1, 'P279': 5, 'P1545': 3, 'P3831': 2, 'P2868': 2, 'P31': 11, 'P1881': 2, 'P161': 2, 'P1557': 2, 'P1039': 2, 'P1038': 2, 'P2354': 1, 'P734': 4, 'P1013': 3, 'P166': 1, 'P3744': 1, 'P407': 3, 'P3984': 1, 'P509': 1, 'P157': 2, 'P585': 7, 'P1686': 5, 'P1411': 5, 'P527': 2, 'P106': 2, 'P110': 1, 'P1113': 1, 'P921': 1, 'P162': 1, 'P463': 1, 'P577': 2, 'P136': 3, 'P910': 2, 'P195': 1, 'P217': 1, 'P276': 1, 'P518': 1, 'P186': 1, 'P582': 2, 'P20': 2, 'P69': 1, 'P1114': 1, 'P1552': 1, 'P373': 2, 'P570': 2, 'P571': 1, 'P264': 1, 'P2096': 1, 'P175': 1, 'P360': 2, 'P27': 1, 'P495': 1}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> paths_keywords: (['first', 'book', 'title', 'harry potter', 'j. k. rowling', 'james', 'first book'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 176.0s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 2.73s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.08s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: Title of the first book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Title of the first book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Title creator of the first book\n",
      "-> q_themes: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(title, []), (Title, ['P1476']), (first, ['P577']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 4.78s\n",
      "--> Predicates enhanced by previous context: [(characters, ['P674']), (title, []), (Title, ['P1476']), (first, ['P577']), (book, ['P50']), (creator, ['P170']), (named after, ['P138'])]\n",
      "----> q_themes in context: ([(first, ['Q19269277']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (Book, ['Q16860229', 'Q11515178']), (title, ['P1476', 'Q13629195']), (first book, ['Q1419297'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['first', 'Title', 'book', 'Book', 'title', 'first book']\n",
      "---> Meaningful keywords enhanced by previous context: ['first', 'Title', 'book', 'Book', 'title', 'first book', 'Harry Potter', 'Harry Potter', 'James Potter', 'Harry Potter', 'J. K. Rowling', 'James']\n",
      "meaningful_names_no_previous_answer [first, Title, book, Book, title, first book, Harry Potter, Harry Potter, James Potter, Harry Potter, J. K. Rowling, James]\n",
      "----> Meaningful keywords casted as theme ([(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (James, ['Q1277864', 'Q16654695', 'Q12188082'])], [])\n",
      "q_focused_parts: [(first, ['Q19269277']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (title, ['P1476', 'Q13629195', 'Q216353']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (James, ['Q1277864', 'Q16654695', 'Q12188082']), (Title, ['Q216353', 'Q783521']), (book, ['Q571', 'Q421300']), (first book, ['Q1419297'])]\n",
      "-> Building the graph with k_deep 2 ... (could be long)\n",
      "->New graph \tRunning time is 158.47s\n",
      "-->  129 nodes and 178 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 129 nodes and 178 edges\n",
      "---> Rebuilding the graph with k_deep 1 ... Previously: 129 nodes or 178 edges was above the limit of 100\n",
      "-> predicates_dict: {'P674': 58, 'P642': 3, 'P138': 36, 'P453': 4, 'P1441': 9, 'P1877': 3, 'P170': 62, 'P735': 5, 'P1476': 1, 'P805': 1, 'P1343': 1, 'P279': 5, 'P1013': 3, 'P106': 3, 'P577': 2, 'P3831': 2, 'P50': 26, 'P1557': 2, 'P1545': 2, 'P569': 5, 'P2868': 2, 'P571': 2, 'P1881': 2, 'P582': 2, 'P157': 2, 'P69': 1, 'P31': 12, 'P1686': 4, 'P585': 6, 'P1411': 4, 'P161': 2, 'P360': 1, 'P1039': 2, 'P1038': 2, 'P166': 2, 'P361': 1, 'P463': 3, 'P3744': 1, 'P407': 3, 'P3984': 1, 'P527': 5, 'P2354': 1, 'P1113': 1, 'P19': 1, 'P734': 3, 'P518': 1, 'P186': 1, 'P509': 1, 'P195': 1, 'P217': 1, 'P747': 1, 'P110': 1, 'P921': 1, 'P162': 1, 'P27': 1, 'P910': 2, 'P58': 1, 'P282': 2, 'P136': 3, 'P1412': 1, 'P20': 2, 'P1114': 1, 'P373': 2, 'P264': 1, 'P570': 2, 'P2096': 1, 'P1552': 1, 'P175': 1, 'P495': 1}\n",
      "-> paths_keywords: (['first', 'book', 'title', 'harry potter', 'j. k. rowling', 'james', 'first book', 'creator'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 8000\n",
      "->Computing possible paths \tRunning time is 33.53s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 15960\n",
      "->\tRunning time is 37.16s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q102438', 0.8783527544968897], ['Q102235', 0.8737629244011381], ['Q161687', 0.8642800277216793], ['Q102244', 0.8559292362116625], ['Q102448', 0.8443791015426357], ['Q102225', 0.8380413508965852], ['Q43361', 0.8366916581494263], ['Q80817', 0.832236823056491], ['Q20711488', 0.8263516356293122], ['Q46887', 0.8240105458548372], ['Q47209', 0.8149276551078822], ['Q46758', 0.8052537405780129], ['Q47598', 0.8037172302820623], ['Q46751', 0.7975658840667776], ['Q176132', 0.5929233429992015], ['Q173998', 0.5735544128657779]]\n",
      "->Computing hypothesises \tRunning time is 945.81s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 29\n",
      "->\tRunning time is 93.63s\n",
      "--> len(cleared_golden_paths): 13\n",
      "---> First path: ['Q102438', 'P674', 'Q717594', 'P1441', 'Q216930', 'P453', 'Q173998', 'P170', 'Q34660', 'P50', 'Q43361']\n",
      "->\tTotal Running time is 1409.59s\n",
      "\n",
      "df_graphqa Q102438\n",
      "df_graphqa_rr 0.4375\n",
      "\n",
      "PARTIAL_CORRECT 1515 - 3 -> graphqa in answers ['Q102438', 'Q102235', 'Q161687', 'Q102244', 'Q102448', 'Q102225', 'Q43361', 'Q80817', 'Q20711488', 'Q46887', 'Q47209', 'Q46758', 'Q47598', 'Q46751', 'Q176132', 'Q173998']\n",
      "    conversation_id turn plus_convex                  question  answer domain  \\\n",
      "138            1514    2       False  Title of the first book?  Q43361  books   \n",
      "\n",
      "    qanswer  qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "138   False         97.73         0.0    False           0.34          0.0   \n",
      "\n",
      "                   convex  convex_time  convex_rr  graphqa  graphqa_time  \\\n",
      "138  1997-01-01T00:00:00Z       192.86        0.0  Q102438       1815.41   \n",
      "\n",
      "    graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "138        False        False        False        False           True   \n",
      "\n",
      "     graphqa_rr  \n",
      "138      0.4375  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-139-ic1514-iq2-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1515/2240 -> 3/5 -> Convex=True: (Q43361) Title of the first book?                                  \n",
      "qAnswer extended by Convex\n",
      "df_qanswer Q17290934\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q17290934\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q17290934\n",
      "df_graphqa_rr 0.0\n",
      "    conversation_id turn plus_convex                  question  answer domain  \\\n",
      "139            1514    2        True  Title of the first book?  Q43361  books   \n",
      "\n",
      "       qanswer  qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "139  Q17290934          0.53         0.0    False           0.34          0.0   \n",
      "\n",
      "        convex  convex_time  convex_rr    graphqa  graphqa_time graphqa_top2  \\\n",
      "139  Q17290934         0.96        0.0  Q17290934          1.33        False   \n",
      "\n",
      "    graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "139        False        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-140-ic1514-iq2-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 12:44:06.892061\n",
      "\t>>> Processing 1515/2240 -> 4/5 -> Convex=False: (Q21) What country was the book set in?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: What country was the book set in?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What country was the book set in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What country was the book set in\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> q_themes: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: set\n",
      "-> q_predicates: [(be, ['P31']), (set, ['P4809']), (country, [])]\n",
      "-> q_predicates \tRunning time is 5.18s\n",
      "--> Predicates enhanced by previous context: [(be, ['P31']), (set, ['P4809']), (country, [])]\n",
      "----> q_themes in context: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['country', 'the book', 'Country']\n",
      "---> Meaningful keywords enhanced by previous context: ['country', 'the book', 'Country', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [country, the book, Country, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (J. K. Rowling, ['Q34660']), (the book, ['Q3794440'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 53.77s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: What country was the book set in?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What country was the book set in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: What country the book set in\n",
      "-> q_themes: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "behold: get_most_similar started with: set\n",
      "-> q_predicates: [(be, ['P31']), (set, ['P4809']), (country, ['P17', 'P3005']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 6.77s\n",
      "--> Predicates enhanced by previous context: [(be, ['P31']), (set, ['P4809']), (country, ['P17', 'P3005']), (book, ['P50'])]\n",
      "----> q_themes in context: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['country', 'the book', 'Country']\n",
      "---> Meaningful keywords enhanced by previous context: ['country', 'the book', 'Country', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [country, the book, Country, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (J. K. Rowling, ['Q34660']), (the book, ['Q3794440'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 63.72s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: What country was the book set in?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What country was the book set in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What country was the book set in\n",
      "-> q_themes: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: set\n",
      "-> q_predicates: [(be, ['P31']), (set, ['P4809']), (country, [])]\n",
      "-> q_predicates \tRunning time is 5.35s\n",
      "--> Predicates enhanced by previous context: [(author, ['P50']), (be, ['P31']), (set, ['P4809']), (country, []), (point in time, ['P585']), (present in work, ['P1441']), (subject has role, ['P2868'])]\n",
      "----> q_themes in context: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['country', 'the book', 'Country']\n",
      "---> Meaningful keywords enhanced by previous context: ['country', 'the book', 'Country', 'Harry Potter', 'J. K. Rowling', 'Harry Potter', '1997-01-01T00:00:00Z', 'Harry Potter', 'title character', 'human']\n",
      "meaningful_names_no_previous_answer [country, the book, Country, Harry Potter, J. K. Rowling, Harry Potter, 1997 - 01 01T00:00:00Z, Harry Potter, title character, human]\n",
      "----> Meaningful keywords casted as theme ([(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (title character, ['Q3246821']), (human, ['Q5', 'Q20094897', 'Q26190966'])], [])\n",
      "q_focused_parts: [(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (title character, ['Q3246821']), (human, ['Q5', 'Q20094897', 'Q26190966']), (the book, ['Q3794440'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 112.12s\n",
      "-->  134 nodes and 172 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 134 nodes and 172 edges\n",
      "---> Rebuilding the graph with k_deep 2 ... Previously: 134 nodes or 172 edges was above the limit of 100\n",
      "-> predicates_dict: {'P50': 26, 'P585': 16, 'P2031': 2, 'P577': 3, 'P1441': 601, 'P674': 3, 'P170': 2, 'P642': 5, 'P2868': 172, 'P453': 4, 'P31': 234, 'P106': 3, 'P17': 1, 'P1282': 1, 'P1963': 4, 'P1686': 11, 'P1411': 9, 'P582': 2, 'P20': 1, 'P570': 1, 'P3831': 2, 'P2453': 1, 'P805': 2, 'P1476': 1, 'P1343': 1, 'P495': 2, 'P27': 3, 'P527': 6, 'P364': 1, 'P800': 2, 'P3744': 1, 'P407': 3, 'P3984': 1, 'P161': 2, 'P921': 4, 'P166': 14, 'P2408': 1, 'P291': 1, 'P463': 1, 'P157': 1, 'P136': 1, 'P735': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P1113': 1, 'P1423': 1, 'P2452': 2, 'P1039': 2, 'P1038': 2, 'P1877': 2, 'P734': 1, 'P58': 1, 'P279': 1, 'P5800': 1, 'P1881': 1, 'P360': 2, 'P5008': 1, 'P1424': 1, 'P641': 1, 'P580': 1, 'P2096': 1}\n",
      "-> paths_keywords: (['country', 'harry potter', 'j. k. rowling', 'title character', 'human', 'the book', 'set'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 8000\n",
      "->Computing possible paths \tRunning time is 32.81s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 14642\n",
      "->\tRunning time is 25.85s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q176132', 33.02994876251193], ['Q173998', 32.96411795619901], ['Q145', 4.286935756799589], ['Q215972', 2.6785526338097543], ['Q102235', 2.5659655988037255], ['Q15299049', 2.0379724598495974], ['Q10298203', 0.6139685973728591], ['1997-01-01T00:00:00Z', 0.40428827733590994]]\n",
      "->Computing hypothesises \tRunning time is 333.96s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 2\n",
      "->\tRunning time is 303.67s\n",
      "--> len(cleared_golden_paths): 2\n",
      "---> First path: ['Q176132', 'P1441', 'Q8337', 'P577', '1997-01-01T00:00:00Z', 'P585', 'Q34660', 'P170', 'Q3244512', 'P2868', 'Q3246821']\n",
      "->\tTotal Running time is 923.63s\n",
      "\n",
      "df_convex Q176132\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: What country was the book set in?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What country was the book set in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What country was the book set in\n",
      "-> q_themes: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "behold: get_most_similar started with: set\n",
      "-> q_predicates: [(be, ['P31']), (set, ['P4809']), (country, [])]\n",
      "-> q_predicates \tRunning time is 6.35s\n",
      "--> Predicates enhanced by previous context: [(characters, ['P674']), (be, ['P31']), (set, ['P4809']), (country, []), (present in work, ['P1441']), (character role, ['P453']), (creator, ['P170']), (author, ['P50'])]\n",
      "----> q_themes in context: ([(country, ['Q6256', 'P17']), (the book, ['Q3794440']), (Country, ['Q11070708', 'Q1754454'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['country', 'the book', 'Country']\n",
      "---> Meaningful keywords enhanced by previous context: ['country', 'the book', 'Country', 'Harry Potter', 'Harry Potter', 'James Potter', 'Harry Potter', 'J. K. Rowling', \"Harry Potter and the Philosopher's Stone\", 'Ron Weasley', \"Harry Potter and the Philosopher's Stone\", 'James']\n",
      "meaningful_names_no_previous_answer [country, the book, Country, Harry Potter, Harry Potter, James Potter, Harry Potter, J. K. Rowling, Harry Potter and the Philosopher Stone, Ron Weasley, Harry Potter and the Philosopher Stone, James]\n",
      "----> Meaningful keywords casted as theme ([(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Ron Weasley, ['Q173998']), (James, ['Q1277864', 'Q16654695', 'Q12188082'])], [])\n",
      "q_focused_parts: [(country, ['Q6256', 'P17']), (Country, ['Q1754454', 'Q11070708']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Ron Weasley, ['Q173998']), (James, ['Q1277864', 'Q16654695', 'Q12188082']), (the book, ['Q3794440'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 189.95s\n",
      "-->  399 nodes and 576 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 399 nodes and 576 edges\n",
      "---> Rebuilding the graph with k_deep 2 ... Previously: 399 nodes or 576 edges was above the limit of 100\n",
      "-> predicates_dict: {'P674': 157, 'P642': 2, 'P138': 31, 'P453': 44, 'P1441': 63, 'P1877': 3, 'P170': 82, 'P50': 44, 'P1686': 8, 'P527': 8, 'P735': 4, 'P106': 5, 'P17': 3, 'P1282': 1, 'P407': 5, 'P1104': 3, 'P585': 16, 'P2664': 3, 'P1963': 4, 'P161': 42, 'P1411': 11, 'P2096': 2, 'P1346': 1, 'P166': 3, 'P495': 5, 'P27': 6, 'P31': 231, 'P1881': 1, 'P364': 2, 'P800': 2, 'P2868': 3, 'P582': 2, 'P20': 2, 'P570': 1, 'P1922': 1, 'P3132': 1, 'P1013': 1, 'P2408': 2, 'P518': 1, 'P186': 1, 'P291': 3, 'P577': 3, 'P2354': 1, 'P2453': 3, 'P805': 3, 'P1476': 1, 'P1343': 1, 'P571': 2, 'P4241': 1, 'P463': 2, 'P195': 1, 'P217': 1, 'P276': 1, 'P509': 1, 'P157': 2, 'P813': 1, 'P973': 1, 'P1709': 1, 'P123': 1, 'P110': 1, 'P437': 3, 'P1545': 3, 'P156': 2, 'P179': 2, 'P2452': 2, 'P1040': 1, 'P136': 9, 'P4675': 1, 'P1039': 3, 'P1038': 3, 'P1074': 1, 'P162': 4, 'P734': 3, 'P58': 1, 'P264': 1, 'P282': 1, 'P1434': 1, 'P360': 2, 'P5008': 1, 'P175': 1, 'P910': 1, 'P580': 1}\n",
      "-> paths_keywords: (['country', 'harry potter', 'j. k. rowling', 'ron weasley', 'james', 'the book', 'set'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 8000\n",
      "->Computing possible paths \tRunning time is 37.88s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 15960\n",
      "->\tRunning time is 33.44s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q102235', 16.135465563103413], ['Q102244', 12.61712359399394], ['Q102448', 10.263929790160848], ['Q102225', 7.305271188167599], ['Q176132', 6.984690591816961], ['Q20711488', 6.735776061753956], ['Q102438', 6.1077101771951625], ['Q43361', 2.8380300114401376], ['Q1860', 0.7975245278901837]]\n",
      "->Computing hypothesises \tRunning time is 476.42s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 45\n",
      "->\tRunning time is 714.33s\n",
      "--> len(cleared_golden_paths): 24\n",
      "---> First path: ['Q102235', 'P674', 'Q173998', 'P1441', 'Q8337', 'P642', 'Q3244512', 'P453', 'Q216930', 'P1877', 'Q34660', 'P50', 'Q43361', 'P17', 'Q30']\n",
      "->\tTotal Running time is 1618.42s\n",
      "\n",
      "df_graphqa Q102235\n",
      "df_graphqa_rr 0.0\n",
      "    conversation_id turn plus_convex                           question  \\\n",
      "140            1514    3       False  What country was the book set in?   \n",
      "\n",
      "    answer domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "140    Q21  books   False        129.81         0.0    False           0.23   \n",
      "\n",
      "     platypus_rr   convex  convex_time  convex_rr  graphqa  graphqa_time  \\\n",
      "140          0.0  Q176132       923.84        0.0  Q102235       1618.85   \n",
      "\n",
      "    graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "140        False        False        False        False          False   \n",
      "\n",
      "     graphqa_rr  \n",
      "140         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-141-ic1514-iq3-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1515/2240 -> 4/5 -> Convex=True: (Q21) What country was the book set in?                                  \n",
      "qAnswer extended by Convex\n",
      "df_qanswer Q34660\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q145\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q145\n",
      "df_graphqa_rr 0.0\n",
      "    conversation_id turn plus_convex                           question  \\\n",
      "141            1514    3        True  What country was the book set in?   \n",
      "\n",
      "    answer domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "141    Q21  books  Q34660          0.58         0.0    False           0.22   \n",
      "\n",
      "     platypus_rr convex  convex_time  convex_rr graphqa  graphqa_time  \\\n",
      "141          0.0   Q145         1.31        0.0    Q145          1.53   \n",
      "\n",
      "    graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "141        False        False        False        False          False   \n",
      "\n",
      "     graphqa_rr  \n",
      "141         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-3/benchmarking-qanswer-platypus-convex-qagraph-142-ic1514-iq3-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 13:28:43.294380\n",
      "\t>>> Processing 1515/2240 -> 5/5 -> Convex=False: (Q80817) What's the longest book?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: What's the longest book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What is the longest book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What is the longest book\n",
      "-> q_themes: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the longest book, The Longest Book, the long book, longest book, the longest Book])\n",
      "-> q_themes_enhanced: [('long', ['P2043']), ('The Long', ['Q22928385']), ('The Book', ['Q10695431']), ('Longest', ['Q37176116']), ('Long', ['Q3794484'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (longest, ['P2787']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 7.27s\n",
      "--> Predicates enhanced by previous context: [(be, ['P31']), (longest, ['P2787']), (book, ['P50'])]\n",
      "----> q_themes in context: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the, The, longest])\n",
      "--> Potential meaningful keywords for the sentence: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long']\n",
      "---> Meaningful keywords enhanced by previous context: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [what, book, What, Book, long, The Long, The Book, Longest, Long, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (J. K. Rowling, ['Q34660']), (what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "->New graph \tRunning time is 31.63s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: What's the longest book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What is the longest book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: What the longest book\n",
      "-> q_themes: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the longest book, The Longest Book, the long book, longest book, the longest Book])\n",
      "-> q_themes_enhanced: [('long', ['P2043']), ('The Long', ['Q22928385']), ('The Book', ['Q10695431']), ('Longest', ['Q37176116']), ('Long', ['Q3794484'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (longest, ['P2787']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 7.47s\n",
      "--> Predicates enhanced by previous context: [(be, ['P31']), (longest, ['P2787']), (book, ['P50'])]\n",
      "----> q_themes in context: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the, The, longest])\n",
      "--> Potential meaningful keywords for the sentence: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long']\n",
      "---> Meaningful keywords enhanced by previous context: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long', 'J. K. Rowling']\n",
      "meaningful_names_no_previous_answer [what, book, What, Book, long, The Long, The Book, Longest, Long, J. K. Rowling]\n",
      "----> Meaningful keywords casted as theme ([(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (J. K. Rowling, ['Q34660'])], [])\n",
      "q_focused_parts: [(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (J. K. Rowling, ['Q34660']), (what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 31.76s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: What's the longest book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What is the longest book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What is the longest book\n",
      "-> q_themes: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the longest book, The Longest Book, the long book, longest book, the longest Book])\n",
      "-> q_themes_enhanced: [('long', ['P2043']), ('The Long', ['Q22928385']), ('The Book', ['Q10695431']), ('Longest', ['Q37176116']), ('Long', ['Q3794484'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (longest, ['P2787']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 8.11s\n",
      "--> Predicates enhanced by previous context: [(author, ['P50']), (be, ['P31']), (longest, ['P2787']), (present in work, ['P1441']), (date of publication, ['P577']), (point in time, ['P585']), (creator, ['P170']), (subject has role, ['P2868'])]\n",
      "----> q_themes in context: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the, The, longest])\n",
      "--> Potential meaningful keywords for the sentence: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long']\n",
      "---> Meaningful keywords enhanced by previous context: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long', 'Harry Potter', 'J. K. Rowling', 'Harry Potter', '1997-01-01T00:00:00Z', 'Harry Potter', 'title character', 'Lord Voldemort', 'human']\n",
      "meaningful_names_no_previous_answer [what, book, What, Book, long, The Long, The Book, Longest, Long, Harry Potter, J. K. Rowling, Harry Potter, 1997 - 01 01T00:00:00Z, Harry Potter, title character, Lord Voldemort, human]\n",
      "----> Meaningful keywords casted as theme ([(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (title character, ['Q3246821']), (Lord Voldemort, ['Q176132']), (human, ['Q5', 'Q20094897', 'Q26190966'])], [])\n",
      "q_focused_parts: [(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (title character, ['Q3246821']), (Lord Voldemort, ['Q176132']), (human, ['Q5', 'Q20094897', 'Q26190966']), (what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 97.22s\n",
      "-->  177 nodes and 244 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 177 nodes and 244 edges\n",
      "---> Rebuilding the graph with k_deep 2 ... Previously: 177 nodes or 244 edges was above the limit of 100\n",
      "-> predicates_dict: {'P50': 26, 'P585': 16, 'P2031': 2, 'P577': 5, 'P1441': 854, 'P674': 5, 'P170': 60, 'P642': 8, 'P2868': 174, 'P453': 4, 'P157': 3, 'P31': 34, 'P106': 4, 'P582': 3, 'P20': 2, 'P570': 2, 'P1686': 11, 'P1411': 8, 'P3831': 3, 'P175': 3, 'P569': 4, 'P1476': 1, 'P805': 1, 'P1343': 1, 'P527': 6, 'P571': 1, 'P364': 1, 'P69': 2, 'P580': 3, 'P108': 1, 'P800': 2, 'P3744': 1, 'P407': 2, 'P3984': 1, 'P161': 2, 'P921': 4, 'P166': 14, 'P1013': 2, 'P812': 1, 'P518': 1, 'P186': 2, 'P463': 3, 'P509': 1, 'P195': 1, 'P217': 1, 'P276': 1, 'P136': 2, 'P735': 2, 'P1113': 1, 'P607': 1, 'P1423': 1, 'P1039': 4, 'P1038': 4, 'P110': 1, 'P1881': 1, 'P162': 1, 'P58': 3, 'P279': 2, 'P2408': 1, 'P5800': 1, 'P86': 1, 'P282': 2, 'P2096': 1, 'P2061': 1, 'P495': 1, 'P19': 2, 'P54': 1, 'P1424': 1, 'P734': 3, 'P138': 1, 'P1545': 1, 'P361': 1, 'P21': 1, 'P3879': 1, 'P27': 1, 'P641': 1}\n",
      "-> paths_keywords: (['what', 'book', 'harry potter', 'j. k. rowling', 'title character', 'lord voldemort', 'human'], {'author': [author, ['P50']], 'instance of': [instance of, ['P31']], 'longest span': [longest span, ['P2787']], 'present in work': [present in work, ['P1441']], 'date of publication': [date of publication, ['P577']], 'point in time': [point in time, ['P585']], 'creator': [creator, ['P170']], 'subject has role': [subject has role, ['P2868']], 'long': [length, ['P2043']], 'length': [length, ['P2043']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 8000\n",
      "->Computing possible paths \tRunning time is 36.13s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 15992\n",
      "->\tRunning time is 46.82s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q173998', 37.333289426802004], ['Q174009', 37.3198972766287], ['Q176772', 37.29220566025061], ['Q187923', 37.286302902301045], ['Q179641', 37.28520271396789], ['Q712548', 37.259380817787935], ['Q190282', 37.24594847230377], ['Q177439', 37.22886469592356], ['Q192179', 37.20385428569538], ['Q1159850', 6.7437106747947055], ['Q20711488', 4.2410198824893985], ['Q15298259', 3.2464972608637783], ['Q215972', 2.5652875070908676], ['Q15299049', 1.9134875694587214], ['Q3758946', 1.6016127994738762], ['Q15553735', 1.4307568524288508], ['1997-01-01T00:00:00Z', 1.1073428066498807], ['Q10298203', 0.5667487869929559], ['Q1254953', -0.4909175931942589], ['Q1057918', -1.3766738718396057], ['Q6012217', -1.5985639244821255]]\n",
      "->Computing hypothesises \tRunning time is 2212.89s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 9\n",
      "->\tRunning time is 187.1s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> len(cleared_golden_paths): 9\n",
      "---> First path: ['Q173998', 'P1441', 'Q8337', 'P577', '1997-01-01T00:00:00Z', 'P585', 'Q34660', 'P170', 'Q176132', 'P31', 'Q15298259', 'P31', 'Q3244512', 'P2868', 'Q3246821']\n",
      "->\tTotal Running time is 2683.15s\n",
      "\n",
      "df_convex Q173998\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: What's the longest book?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What is the longest book \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What is the longest book\n",
      "-> q_themes: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [the longest book, The Longest Book, the long book, longest book, the longest Book])\n",
      "-> q_themes_enhanced: [('long', ['P2043']), ('The Long', ['Q22928385']), ('The Book', ['Q10695431']), ('Longest', ['Q37176116']), ('Long', ['Q3794484'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: book\n",
      "-> q_predicates: [(be, ['P31']), (longest, ['P2787']), (book, ['P50'])]\n",
      "-> q_predicates \tRunning time is 8.11s\n",
      "--> Predicates enhanced by previous context: [(characters, ['P674']), (be, ['P31']), (longest, ['P2787']), (book, ['P50']), (present in work, ['P1441']), (of, ['P642']), (character role, ['P453']), (after a work by, ['P1877']), (country, ['P17'])]\n",
      "----> q_themes in context: ([(what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300']), (What, ['Q22073920']), (Book, ['Q16860229', 'Q11515178'])], [The, longest])\n",
      "--> Potential meaningful keywords for the sentence: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long']\n",
      "---> Meaningful keywords enhanced by previous context: ['what', 'book', 'What', 'Book', 'long', 'The Long', 'The Book', 'Longest', 'Long', 'Harry Potter', 'Harry Potter', 'Harry Potter', 'J. K. Rowling', \"Harry Potter and the Philosopher's Stone\", 'Ron Weasley', \"Harry Potter and the Philosopher's Stone\", 'United States of America', 'Harry Potter and the Order of the Phoenix', 'Harry Potter and the Order of the Phoenix', 'James Potter', 'James']\n",
      "meaningful_names_no_previous_answer [what, book, What, Book, long, The Long, The Book, Longest, Long, Harry Potter, Harry Potter, Harry Potter, J. K. Rowling, Harry Potter and the Philosopher Stone, Ron Weasley, Harry Potter and the Philosopher Stone, United States of America, Harry Potter and the Order of the Phoenix, Harry Potter and the Order of the Phoenix, James Potter, James]\n",
      "----> Meaningful keywords casted as theme ([(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Ron Weasley, ['Q173998']), (United States of America, ['Q30', 'Q19971019']), (Harry Potter and the Order of the Phoenix, ['Q102235', 'Q2102791', 'Q1148993']), (Harry Potter and the Order of the Phoenix, ['Q102235', 'Q2102791', 'Q1148993']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (James, ['Q1277864', 'Q16654695', 'Q12188082'])], [])\n",
      "q_focused_parts: [(What, ['Q22073920', 'Q28036789']), (Book, ['Q421300', 'Q11515178', 'Q16860229']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (Harry Potter, ['Q3244512', 'Q216930', 'Q17146193']), (J. K. Rowling, ['Q34660']), (Ron Weasley, ['Q173998']), (United States of America, ['Q30', 'Q19971019']), (Harry Potter and the Order of the Phoenix, ['Q102235', 'Q2102791', 'Q1148993']), (Harry Potter and the Order of the Phoenix, ['Q102235', 'Q2102791', 'Q1148993']), (James Potter, ['Q19826368', 'Q25208700', 'Q20863042']), (James, ['Q1277864', 'Q16654695', 'Q12188082']), (what, ['Q20656446', 'Q28036789']), (book, ['Q571', 'Q421300'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 178.1s\n",
      "-->  517 nodes and 732 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 517 nodes and 732 edges\n",
      "---> Rebuilding the graph with k_deep 2 ... Previously: 517 nodes or 732 edges was above the limit of 100\n",
      "-> predicates_dict: {'P674': 161, 'P642': 2, 'P453': 80, 'P1441': 613, 'P1877': 15, 'P170': 27, 'P138': 31, 'P50': 69, 'P1686': 13, 'P527': 16, 'P17': 2, 'P1346': 1, 'P179': 2, 'P735': 4, 'P1013': 3, 'P31': 27, 'P106': 3, 'P161': 51, 'P3831': 2, 'P2868': 3, 'P361': 2, 'P585': 11, 'P1411': 9, 'P2096': 1, 'P1113': 1, 'P166': 3, 'P495': 5, 'P27': 5, 'P1545': 2, 'P155': 2, 'P156': 3, 'P291': 1, 'P577': 5, 'P1881': 2, 'P364': 2, 'P800': 2, 'P3744': 2, 'P407': 4, 'P3984': 1, 'P582': 2, 'P20': 2, 'P570': 2, 'P463': 2, 'P518': 2, 'P186': 3, 'P2354': 1, 'P1476': 2, 'P805': 1, 'P1343': 1, 'P571': 3, 'P2002': 1, 'P195': 2, 'P217': 2, 'P276': 2, 'P406': 1, 'P136': 7, 'P1039': 3, 'P1038': 3, 'P2047': 1, 'P1040': 1, 'P910': 2, 'P58': 3, 'P282': 3, 'P2031': 1, 'P2408': 1, 'P264': 1, 'P1412': 2, 'P19': 3, 'P157': 1, 'P973': 1, 'P734': 1, 'P2061': 1, 'P1884': 2, 'P921': 2, 'P54': 2, 'P21': 2, 'P3879': 1, 'P580': 1}\n",
      "-> paths_keywords: (['what', 'book', 'harry potter', 'j. k. rowling', 'ron weasley', 'united states of america', 'james'], {'characters': [characters, ['P674']], 'instance of': [instance of, ['P31']], 'longest span': [longest span, ['P2787']], 'author': [author, ['P50']], 'present in work': [present in work, ['P1441']], 'of': [of, ['P642']], 'character role': [character role, ['P453']], 'after a work by': [after a work by, ['P1877']], 'country': [country, ['P17']], 'long': [length, ['P2043']], 'length': [length, ['P2043']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 8000\n",
      "->Computing possible paths \tRunning time is 42.82s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 16000\n",
      "->\tRunning time is 63.21s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q176132', 59.654816913638996], ['Q174009', 59.59658642665917], ['Q176772', 59.543995923121315], ['Q712548', 59.50913998348424], ['Q192179', 59.45347695398624], ['Q713701', 55.27179049748187], ['Q187923', 55.154267656245885], ['Q179641', 55.12084329729074], ['Q208657', 55.11534410875722], ['Q190282', 55.08693103402364], ['Q190366', 55.07358955038244], ['Q177439', 55.05421763475874], ['Q102235', 13.772851363012997], ['Q145', 8.261633696362262], ['Q102438', 4.953273091916802], ['Q15298259', 4.907685120966128], ['Q161687', 4.847528957190867], ['Q754837', 4.502768888535013], ['Q1263791', 4.2772007518570225], ['Q13359613', 3.3111282643408733], ['Q33179355', 3.2045404476796104], ['Q43361', 3.1233276375809353], ['Q1250951', 3.057263291688786], ['Q840305', 2.7014017407325546], ['Q13359612', 2.637523913659968], ['Q102225', 2.5843367782477498], ['Q6042850', 2.57025503912947], ['Q7881262', 2.546809854318271], ['Q4833247', 2.5347671827936233], ['Q3255012', 2.3584980124219315], ['Q835558', 2.1475591154953872], ['Q716534', 1.9319955143402519], ['Q10394844', 1.9119433823173524], ['Q3284911', 1.6912993778205678], ['Q1056425', 1.584217589168467], ['Q1860', 1.5303399071865487], ['Q715312', 1.2733696612049752], ['Q3656721', 1.127744355499986], ['Q3224418', 1.035951440541525], ['Q3223764', 0.8676596848244746], ['Q3432612', 0.8330975847759889], ['Q11910388', 0.7784704064477002], ['Q842178', 0.7352252694967969], ['Q15242205', 0.6425685898310263], ['Q5700415', 0.5708858514119243], ['Q3221958', 0.5625621835149414], ['Q716941', 0.5454604312317798], ['Q3350144', 0.3270859703090983], ['Q1963397', 0.21566128099446089], ['Q841669', 0.11352224418519069], ['Q3221667', 0.03429383864332128], ['Q1057918', -0.16251979676532186], ['Q11958446', -0.18729361001136516], ['Q1256952', -0.8267996278041495], ['Q774644', -1.010740413743266]]\n",
      "->Computing hypothesises \tRunning time is 10555.26s\n",
      "-> Computing golden paths...\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "[Errno 12] Cannot allocate memory",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-6d9e6ff355d6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    221\u001b[0m                 graphqa_answer, graphqa_graph = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n\u001b[1;32m    222\u001b[0m                                          \u001b[0manswer_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgraphqa_answer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgraphqa_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m                                          use_convex=use_convex)\n\u001b[0m\u001b[1;32m    224\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgraphqa_answer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mgraphqa_answer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-b91dccfc5f02>\u001b[0m in \u001b[0;36mask_graphqa\u001b[0;34m(question, verbose, timer, show_graph, cores, banning_str, answer_context, context_graph, use_convex, turn)\u001b[0m\n\u001b[1;32m     30\u001b[0m         result = graphqa.answer_question(\n\u001b[1;32m     31\u001b[0m             \u001b[0mquestion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshow_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbanning_str\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbanning_str\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m             previous_answer=answer_context, previous_graph=context_graph)\n\u001b[0m\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tm/mse.tm.chatbot.base/tmqa35.py\u001b[0m in \u001b[0;36manswer_question\u001b[0;34m(question, verbose, aggressive, looped, deep_k, deep_k_step, deep_k_max, graph_size_min, graph_size_target, graph_size_max, paths_filter_max, paths_max, timer, g_paths, show_graph, cores, banning_str, reload_cache, answer_sentence, previous_answer, previous_graph, graph_size_target_context, deep_match, k_context, in_context, k_deep_followup, k_deep_context_graph, context_themes, previous_answers, max_deepness, g_autocorrect)\u001b[0m\n\u001b[1;32m   5611\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhypothesises\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5612\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"-> Computing golden paths...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5613\u001b[0;31m             \u001b[0mgolden_paths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmatch_hypothesises\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_nlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_themes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_themes_enhanced\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_focused_parts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_predicates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhypothesises\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpaths_nodes_filtered\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_reward\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwinner_threshold_diff\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtime_sensitive\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime_sensitive\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdeep_match\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeep_match\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5614\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"--> len(golden_paths):\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgolden_paths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5615\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tm/mse.tm.chatbot.base/tmqa35.py\u001b[0m in \u001b[0;36mmatch_hypothesises\u001b[0;34m(graph, question, themes, themes_enchanced, focused_parts, predicates, hypothesises, paths, threshold, max_reward, winner_threshold_diff, time_sensitive, cores, deep_match)\u001b[0m\n\u001b[1;32m   4548\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mproc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprocs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4549\u001b[0m         \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdaemon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4550\u001b[0;31m         \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4551\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mproc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprocs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4552\u001b[0m         \u001b[0min_mp_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/process.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    110\u001b[0m                \u001b[0;34m'daemonic processes are not allowed to have children'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0m_cleanup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sentinel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0;31m# Avoid a refcycle if the target function holds an indirect\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/context.py\u001b[0m in \u001b[0;36m_Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    221\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_default_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mProcess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mDefaultContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBaseContext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/context.py\u001b[0m in \u001b[0;36m_Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    275\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m             \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mpopen_fork\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32mclass\u001b[0m \u001b[0mSpawnProcess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBaseProcess\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinalizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_launch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mduplicate_for_child\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36m_launch\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0mcode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0mparent_r\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [Errno 12] Cannot allocate memory"
     ]
    }
   ],
   "source": [
    "### Evaluate\n",
    "banning_str = False\n",
    "show_graph = False\n",
    "\n",
    "start_time = time.time()\n",
    "conversations_len = len(conversations)\n",
    "\n",
    "for i_c, conversation in enumerate(conversations):\n",
    "    questions = [turn['question'] for turn in conversation['questions']]\n",
    "    answers = [graphqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "    domain = conversation['domain']\n",
    "    questions_len = len(questions)\n",
    "    \n",
    "    qanswer_answer, qanswer_graph = False,False\n",
    "    platypus_answer, platypus_graph = False,False\n",
    "    convex_answer, convex_graph = False,False\n",
    "    graphqa_answer, graphqa_graph = False,False\n",
    "\n",
    "    qanswer_answer_convex, qanswer_graph_convex = False,False\n",
    "    platypus_answer_convex, platypus_graph_convex = False,False\n",
    "    convex_answer_convex, convex_graph_convex = False,False\n",
    "    graphqa_answer_convex, graphqa_graph_convex = False,False\n",
    "    \n",
    "    #break\n",
    "    if i_c+1 < 1515:\n",
    "        continue\n",
    "        \n",
    "    for i_q,question in enumerate(questions):\n",
    "        if i_q+1 <0:\n",
    "            continue\n",
    "            \n",
    "        print(\"It is \", datetime.now())\n",
    "                \n",
    "        for use_convex in [False,True]:\n",
    "\n",
    "            answer = answers[i_q]\n",
    "            print(\"\\r\\t>>> Processing {}/{} -> {}/{} -> Convex={}: ({}) {}\".format(i_c+1,conversations_len,i_q+1,questions_len,use_convex,answer,question), \n",
    "                  end='                                  ')\n",
    "            #time.sleep(1)\n",
    "\n",
    "            #ASK QANSWER\n",
    "            start_time = time.time()            \n",
    "            if qanswer_graph and not use_convex:\n",
    "                print(\"\\nqAnswer extended by GraphQA\")\n",
    "                qanswer_answer, qanswer_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=qanswer_answer, context_graph=qanswer_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if qanswer_answer: \n",
    "                    df_qanswer_rr = get_rr(qanswer_answer[0], answer)\n",
    "                    if qanswer_answer[0]: df_qanswer = qanswer_answer[0][0]\n",
    "                    else: df_qanswer = False\n",
    "                else: \n",
    "                    df_qanswer = False\n",
    "                    df_qanswer_rr = 0\n",
    "\n",
    "            elif qanswer_graph_convex and use_convex:\n",
    "                print(\"\\nqAnswer extended by Convex\")\n",
    "                qanswer_answer_convex, qanswer_graph_convex = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=qanswer_answer_convex, context_graph=qanswer_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if qanswer_answer_convex:\n",
    "                    df_qanswer_rr = get_rr(qanswer_answer_convex[0], answer)\n",
    "                    if qanswer_answer_convex[0]: df_qanswer = qanswer_answer_convex[0][0]\n",
    "                    else: df_qanswer = False\n",
    "                else: \n",
    "                    df_qanswer = False\n",
    "                    df_qanswer_rr = 0\n",
    "\n",
    "            else:\n",
    "                print(\"\\nAsking qAnswer\")\n",
    "                qanswer_answer, qanswer_graph = ask_qanswer(question)\n",
    "                if qanswer_answer: \n",
    "                    qanswer_answer=[[qanswer_answer],[]]\n",
    "                    qanswer_graph=nx.Graph()\n",
    "                    qanswer_graph.add_node(qanswer_answer[0][0], name=graphqa.get_wd_label(qanswer_answer[0][0]), type='entity', turn=i_q+1, weight=1, qa=True)\n",
    "                else: \n",
    "                    qanswer_answer=[[],[]]\n",
    "                    qanswer_graph=nx.Graph()\n",
    "                if qanswer_answer:\n",
    "                    df_qanswer_rr = get_rr(qanswer_answer[0], answer)\n",
    "                    if qanswer_answer[0]: df_qanswer = qanswer_answer[0][0]\n",
    "                    else: df_qanswer = False\n",
    "                else: \n",
    "                    df_qanswer = False\n",
    "                    df_qanswer_rr = 0\n",
    "                if show_graph: graphqa.plot_graph(qanswer_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "                \n",
    "                qanswer_answer_convex = qanswer_answer.copy()\n",
    "                qanswer_graph_convex = qanswer_graph.copy()\n",
    "            \n",
    "            print(\"df_qanswer\",df_qanswer) \n",
    "            print(\"df_qanswer_rr\",df_qanswer_rr)\n",
    "            \n",
    "            df_qanswer_time = round(time.time()-start_time,2)\n",
    "            \n",
    "            ## ASK PLATYPUS\n",
    "            start_time = time.time()\n",
    "            if platypus_graph and not use_convex:\n",
    "                print(\"\\nPlatypus extended by GraphQA\")\n",
    "                platypus_answer, platypus_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=platypus_answer, context_graph=platypus_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if platypus_answer: \n",
    "                    df_platypus_rr = get_rr(platypus_answer[0], answer)\n",
    "                    if platypus_answer[0]: platypus_answer[0][0]: df_platypus = platypus_answer[0][0]\n",
    "                    else: df_platypus = False\n",
    "                else: \n",
    "                    df_platypus_rr = 0\n",
    "                    df_platypus = False\n",
    "\n",
    "            elif platypus_graph_convex and use_convex:\n",
    "                print(\"\\nPlatypus extended by Convex\")\n",
    "                platypus_answer_convex, platypus_graph_convex = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=platypus_answer_convex, context_graph=platypus_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if platypus_answer_convex: \n",
    "                    df_platypus_rr = get_rr(platypus_answer_convex[0], answer)\n",
    "                    if platypus_answer_convex[0]: df_platypus = platypus_answer_convex[0][0]\n",
    "                    else: df_platypus = False\n",
    "                else: \n",
    "                    df_platypus_rr = 0\n",
    "                    df_platypus = False\n",
    "\n",
    "            else:\n",
    "                print(\"\\nAsking Platypus\")\n",
    "                platypus_answer, platypus_graph = ask_platypus(question)\n",
    "                if platypus_answer: \n",
    "                    platypus_answer=[[platypus_answer],[]]\n",
    "                    platypus_graph=nx.Graph()\n",
    "                    platypus_graph.add_node(platypus_answer[0][0], name=graphqa.get_wd_label(platypus_answer[0][0]), type='entity', turn=i_q+1, weight=1, qa=True)\n",
    "                else: \n",
    "                    platypus_answer=[[],[]]\n",
    "                    platypus_graph=nx.Graph()\n",
    "                if platypus_answer:\n",
    "                    df_platypus_rr = get_rr(platypus_answer[0], answer)\n",
    "                    if platypus_answer[0]: df_platypus = platypus_answer[0][0]\n",
    "                    else: df_platypus = False\n",
    "                else: \n",
    "                    df_platypus_rr = 0\n",
    "                    df_platypus = False\n",
    "                if show_graph: graphqa.plot_graph(platypus_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "                \n",
    "                platypus_answer_convex = platypus_answer.copy()\n",
    "                platypus_graph_convex = platypus_graph.copy()\n",
    "            \n",
    "            print(\"df_platypus\",df_platypus) \n",
    "            print(\"df_platypus_rr\",df_platypus_rr)\n",
    "            \n",
    "            df_platypus_time = round(time.time()-start_time,2)\n",
    "            \n",
    "            \n",
    "            ## ASK CONVEX\n",
    "            start_time = time.time()\n",
    "            if convex_graph and not use_convex:\n",
    "                print(\"\\nConvex extended by GraphQA\")\n",
    "                convex_answer, convex_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=convex_answer, context_graph=convex_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if convex_answer:\n",
    "                    df_convex_rr = get_rr(convex_answer[0], answer)\n",
    "                    if convex_answer[0]: df_convex = convex_answer[0][0]\n",
    "                    else: df_convex = False\n",
    "                else: \n",
    "                    df_convex_rr = 0\n",
    "                    df_convex = False\n",
    "                    \n",
    "            elif convex_graph_convex and use_convex:\n",
    "                print(\"\\nConvex extended by Convex\")\n",
    "                convex_answer_convex, convex_graph_convex = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=convex_answer_convex, context_graph=convex_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if convex_answer_convex:\n",
    "                    df_convex_rr = get_rr(convex_answer_convex[0], answer)\n",
    "                    if convex_answer_convex[0]: df_convex = convex_answer_convex[0][0]\n",
    "                    else: df_convex = False\n",
    "                else: \n",
    "                    df_convex = False\n",
    "                    df_convex_rr = 0\n",
    "                    \n",
    "            else:\n",
    "                print(\"\\nAsking Convex\")\n",
    "                convex_answer, convex_graph = ask_convex(question)\n",
    "                #([['Q766106'], ['Q76', 'P25', 'Q766106']],<networkx.classes.graph.Graph at 0x7f94423d1f90>)\n",
    "                if not convex_answer: \n",
    "                    convex_answer=[[],[]]\n",
    "                    convex_graph=nx.Graph()\n",
    "                if convex_answer:\n",
    "                    df_convex_rr = get_rr(convex_answer[0], answer)\n",
    "                    if convex_answer[0]: df_convex = convex_answer[0][0]\n",
    "                    else: ddf_convex = False\n",
    "                else: \n",
    "                    df_convex = False\n",
    "                    df_convex_rr = 0\n",
    "                if show_graph: graphqa.plot_graph(convex_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "                convex_answer_convex = convex_answer.copy()\n",
    "                convex_graph_convex = convex_graph.copy()\n",
    "                \n",
    "            print(\"df_convex\",df_convex) \n",
    "            print(\"df_convex_rr\",df_convex_rr)\n",
    "            \n",
    "            df_convex_time = round(time.time()-start_time,2)\n",
    "                \n",
    "                \n",
    "            \n",
    "\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> qAnswer\", df_qanswer) if df_qanswer == answer else False\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> Platypus\", df_platypus) if df_platypus == answer else False\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> Convex\", df_convex) if df_convex == answer else False\n",
    "\n",
    "            df_graphqa = False\n",
    "            df_graphqa_top2 = False\n",
    "            df_graphqa_top3 = False\n",
    "            df_graphqa_top4 = False\n",
    "            df_graphqa_top5 = False\n",
    "            df_graphqa_topall = False\n",
    "            df_graphqa_rr = 0\n",
    "            \n",
    "            start_time = time.time()\n",
    "            if graphqa_graph and not use_convex:\n",
    "                print(\"\\nGraphQA extended by GraphQA\")\n",
    "                graphqa_answer, graphqa_graph = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n",
    "                                         answer_context=graphqa_answer, context_graph=graphqa_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if graphqa_answer:\n",
    "                    if graphqa_answer[0]:\n",
    "                        df_graphqa_rr = get_rr(graphqa_answer[0], answer)\n",
    "                        if graphqa_answer[0][0]: df_graphqa = graphqa_answer[0][0]\n",
    "                        if answer in graphqa_answer[0][:2]: df_graphqa_top2 = True\n",
    "                        if answer in graphqa_answer[0][:3]: df_graphqa_top3 = True\n",
    "                        if answer in graphqa_answer[0][:4]: df_graphqa_top4 = True\n",
    "                        if answer in graphqa_answer[0][:5]: df_graphqa_top5 = True\n",
    "                        if answer in graphqa_answer[0]: df_graphqa_topall = True\n",
    "                \n",
    "            elif graphqa_graph and use_convex:\n",
    "                print(\"\\nGraphQA extended by Convex\")\n",
    "                graphqa_answer_convex, graphqa_graph_convex = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n",
    "                                         answer_context=graphqa_answer_convex, context_graph=graphqa_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if graphqa_answer_convex:\n",
    "                    if graphqa_answer_convex[0]:\n",
    "                        df_graphqa_rr = get_rr(graphqa_answer_convex[0], answer)\n",
    "                        if graphqa_answer_convex[0][0]: df_graphqa = graphqa_answer_convex[0][0]\n",
    "                        if answer in graphqa_answer_convex[0][:2]: df_graphqa_top2 = True\n",
    "                        if answer in graphqa_answer_convex[0][:3]: df_graphqa_top3 = True\n",
    "                        if answer in graphqa_answer_convex[0][:4]: df_graphqa_top4 = True\n",
    "                        if answer in graphqa_answer_convex[0][:5]: df_graphqa_top5 = True\n",
    "                        if answer in graphqa_answer_convex[0]: df_graphqa_topall = True\n",
    "                \n",
    "            else:\n",
    "                print(\"\\nAsking GraphQA\")\n",
    "                graphqa_answer, graphqa_graph = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n",
    "                                         answer_context=graphqa_answer, context_graph=graphqa_graph,\n",
    "                                         use_convex=False)\n",
    "                if not graphqa_answer: \n",
    "                    graphqa_answer=[[],[]]\n",
    "                    graphqa_graph=nx.Graph()\n",
    "                else:\n",
    "                    graphqa_answer_convex = graphqa_answer.copy()\n",
    "                    graphqa_graph_convex = graphqa_graph.copy()\n",
    "                \n",
    "                if graphqa_answer:\n",
    "                    if graphqa_answer[0]:\n",
    "                        df_graphqa_rr = get_rr(graphqa_answer[0], answer)\n",
    "                        if graphqa_answer[0][0]: df_graphqa = graphqa_answer[0][0]\n",
    "                        if answer in graphqa_answer[0][:2]: df_graphqa_top2 = True\n",
    "                        if answer in graphqa_answer[0][:3]: df_graphqa_top3 = True\n",
    "                        if answer in graphqa_answer[0][:4]: df_graphqa_top4 = True\n",
    "                        if answer in graphqa_answer[0][:5]: df_graphqa_top5 = True\n",
    "                        if answer in graphqa_answer[0]: df_graphqa_topall = True\n",
    "                            \n",
    "            print(\"df_graphqa\",df_graphqa) \n",
    "            print(\"df_graphqa_rr\",df_graphqa_rr)\n",
    "                \n",
    "\n",
    "            df_graphqa_time = round(time.time()-start_time,2)\n",
    "\n",
    "            df = df.append({\n",
    "                'conversation_id':i_c,'turn':i_q,\"plus_convex\":use_convex,\n",
    "                'question':question, 'answer':answer,'domain':domain,\n",
    "                'qanswer':df_qanswer,'qanswer_time':df_qanswer_time, 'qanswer_rr':df_qanswer_rr,\n",
    "                'platypus':df_platypus,'platypus_time':df_platypus_time, 'platypus_rr':df_platypus_rr,\n",
    "                'convex':df_convex,'convex_time':df_convex_time, 'convex_rr':df_convex_rr,\n",
    "                'graphqa':df_graphqa, 'graphqa_time':df_graphqa_time, 'graphqa_top2':df_graphqa_top2,\n",
    "                \"graphqa_top3\":df_graphqa_top3,\"graphqa_top4\":df_graphqa_top4, \"graphqa_top5\":df_graphqa_top5,\n",
    "                \"graphqa_topall\":df_graphqa_topall, \"graphqa_rr\":df_graphqa_rr},\n",
    "               ignore_index=True)\n",
    "\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> graphqa\", df_graphqa) if str(df_graphqa) == str(answer) else False\n",
    "            if use_convex: print(\"\\nPARTIAL_CORRECT\",i_c+1,\"-\",i_q+1, \"-> graphqa in answers\", graphqa_answer_convex[0]) if df_graphqa_topall == True else False\n",
    "            else: print(\"\\nPARTIAL_CORRECT\",i_c+1,\"-\",i_q+1, \"-> graphqa in answers\", graphqa_answer[0]) if df_graphqa_topall == True else False\n",
    "\n",
    "            print(df.tail(1))\n",
    "\n",
    "            pickle_data(df, \"benchmarking-qanswer-platypus-convex-qagraph-\"+str(len(df))+\"-ic\"+str(i_c)+\"-iq\"+str(i_q)+\"-pc\"+str(use_convex))\n",
    "\n",
    "            print(\"\\n\")\n",
    "            \n",
    "            if i_q == 0: \n",
    "                break\n",
    "\n",
    "        #if i_q >= 1:      \n",
    "            #break\n",
    "    \n",
    "    #break\n",
    "\n",
    "print(\"->\\tRunning time is {}s\".format(round(time.time()-start_time,2)))\n",
    "df.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result_convex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.drop(df.tail(1).index,inplace=True) # drop last n rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "\n",
    "print(df_copy[df_copy.graphqa == df_copy.answer][\"graphqa_rr\"]) \n",
    "df_copy.loc[df_copy[\"graphqa\"] == df_copy[\"answer\"], 'graphqa_rr'] = 1\n",
    "print(df_copy[df_copy.graphqa == df_copy.answer][\"graphqa_rr\"]) \n",
    "\n",
    "print(df_copy[df_copy.convex == df_copy.answer][\"convex_rr\"]) \n",
    "df_copy.loc[df_copy[\"convex\"] == df_copy[\"answer\"], 'convex_rr'] = 1  \n",
    "print(df_copy[df_copy.convex == df_copy.answer][\"convex_rr\"]) \n",
    "\n",
    "print(df_copy)  \n",
    "df = df_copy # made at from 0 to 278, len of 279"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SAVING\n",
    "#pickle_data(df_loaded, \"benchmarking-qanswer-platypus-convex-tm1-from-0-to-\"+str(len(df_loaded)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOADING\n",
    "#df_loaded = pd.read_pickle(\"/data/users/romain.claret/tm/wikidata-simplequestions/benchmark_pickles/benchmarking-qanswer-platypus-convex-tm1-from-0-to-9961.pickle.bz2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded = df_loaded.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded['qanswer'][34] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del df_loaded['tm2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded.rename({'mine':'tm1'}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded['tm1_top4'] = \"False\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded = df_loaded[['question','source','qanswer','platypus','convex','tm1','tm1_time','tm1_top2','tm1_top3','tm1_top4','tm1_top5','tm1_topall']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded_len = len(df_loaded)\n",
    "#for i, question in enumerate(df_loaded['question']):\n",
    "#    if i >= 0:\n",
    "#    #if i >= 497:\n",
    "#        source = str(df_loaded['source'][i])\n",
    "#        print(str(i)+\"/\"+str(df_loaded_len),question,\"-> source:\",source)\n",
    "#        \n",
    "#        start_time = time.time()\n",
    "#        result_tmqa_1 = ask_tmqa_1(question, verbose=True)\n",
    "#        \n",
    "#        if result_tmqa_1:\n",
    "#            df_loaded['tm1'][i] = result_tmqa_1[0]\n",
    "#            if source in result_tmqa_1[:2]:\n",
    "#                df_loaded['tm1_top2'][i] = True\n",
    "#            if source in result_tmqa_1[:3]:\n",
    "#                df_loaded['tm1_top3'][i] = True\n",
    "#            if source in result_tmqa_1[:4]:\n",
    "#                df_loaded['tm1_top4'][i] = True\n",
    "#            if source in result_tmqa_1[:5]:\n",
    "#                df_loaded['tm1_top5'][i] = True\n",
    "#            if source in result_tmqa_1:\n",
    "#                df_loaded['tm1_topall'][i] = True\n",
    "#        else:\n",
    "#            df_loaded['tm1'][i] = False\n",
    "#        end_time = time.time()\n",
    "#        df_loaded['tm1_time'][i] = round(end_time-start_time,2)\n",
    "#        print(\"->\\tRunning time is {}s\".format(round(end_time-start_time,2)))\n",
    "#        print(str(str(df_loaded['tm1'][i])==str(source)),\"---> result_tmqa_1:\",str(result_tmqa_1)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df_loaded.copy()\n",
    "#df = df.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_backup = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_row = 496\n",
    "#df_len = len(df)\n",
    "#df_qanswer_max = df[(df.index<=max_row) & (df.qanswer == df.source)]\n",
    "#df_qanswer_max_len = len(df_qanswer_max)\n",
    "#\n",
    "#df_platypus_max = df[(df.index<=max_row) & (df.platypus == df.source)]\n",
    "#df_platypus_max_len = len(df_platypus_max)\n",
    "#\n",
    "#df_convex_max = df[(df.index<=max_row) & (df.convex == df.source)]\n",
    "#df_convex_max_len = len(df_convex_max)\n",
    "#\n",
    "#df_tm1_max = df[(df.index<=max_row) & (df.tm1 == df.source)]\n",
    "#df_tm1_max_len = len(df_tm1_max)\n",
    "#\n",
    "#df_tm1_max_top2 = df[(df.index<=max_row) & (df.tm1_top2 == True)]\n",
    "#df_tm1_max_top2_len = len(df_tm1_max_top2)\n",
    "#\n",
    "#df_tm1_max_top3 = df[(df.index<=max_row) & (df.tm1_top3 == True)]\n",
    "#df_tm1_max_top3_len = len(df_tm1_max_top3)\n",
    "#\n",
    "#df_tm1_max_top4 = df[(df.index<=max_row) & (df.tm1_top4 == True)]\n",
    "#df_tm1_max_top4_len = len(df_tm1_max_top4)\n",
    "#\n",
    "#df_tm1_max_top5 = df[(df.index<=max_row) & (df.tm1_top5 == True)]\n",
    "#df_tm1_max_top5_len = len(df_tm1_max_top5)\n",
    "#\n",
    "#df_tm1_max_topall = df[(df.index<=max_row) & (df.tm1_topall == True)]\n",
    "#df_tm1_max_topall_len = len(df_tm1_max_topall)\n",
    "#\n",
    "#print(\"qanswer:\", df_qanswer_max_len,df_qanswer_max_len/max_row)\n",
    "#print(\"platypus:\", df_platypus_max_len, df_platypus_max_len/max_row)\n",
    "#print(\"convex:\", df_convex_max_len, df_convex_max_len/max_row)\n",
    "#print(\"tm1:\", df_tm1_max_len, df_tm1_max_len/max_row)\n",
    "#print(\"tm1_top2:\", df_tm1_max_top2_len, df_tm1_max_top2_len/max_row)\n",
    "#print(\"tm1_top3:\", df_tm1_max_top3_len, df_tm1_max_top3_len/max_row)\n",
    "#print(\"tm1_top4:\", df_tm1_max_top4_len, df_tm1_max_top4_len/max_row)\n",
    "#print(\"tm1_top5:\", df_tm1_max_top5_len, df_tm1_max_top5_len/max_row)\n",
    "#print(\"tm1_topall:\", df_tm1_max_topall_len, df_tm1_max_topall_len/max_row)\n",
    "#\n",
    "#df[ & (df.qanswer == df.source)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"qanswer:\",len(df[df.qanswer == df.source]),len(df[df.qanswer == df.source])/len(df))\n",
    "#print(\"platypus:\",len(df[df.platypus == df.source]),len(df[df.platypus == df.source])/len(df))\n",
    "#print(\"convex:\",len(df[df.convex == df.source]),len(df[df.convex == df.source])/len(df))\n",
    "#print(\"tm1:\",len(df[df.tm1 == df.source]),len(df[df.tm1 == df.source])/len(df))\n",
    "#print(\"tm1_top2:\",len(df[df.tm1_top2 == True]),len(df[df.tm1_top2 == True])/len(df))\n",
    "#print(\"tm1_top3:\",len(df[df.tm1_top3 == True]),len(df[df.tm1_top3 == True])/len(df))\n",
    "#print(\"tm1_top4:\",len(df[df.tm1_top4 == True]),len(df[df.tm1_top4 == True])/len(df))\n",
    "#print(\"tm1_top5:\",len(df[df.tm1_top5 == True]),len(df[df.tm1_top5 == True])/len(df))\n",
    "#print(\"tm1_topall:\",len(df[df.tm1_topall == True]),len(df[df.tm1_topall == True])/len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.tm1_top2 = df.tm1_top3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:qa]",
   "language": "python",
   "name": "conda-env-qa-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}