Parsing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding pages which pages which contains infoboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    Author : Shubham Agrawal\\n    Roll no: 2019201085\\n    Institute: IIIT-Hyderabad\\n'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "    Author : Shubham Agrawal\n",
    "    Roll no: 2019201085\n",
    "    Institute: IIIT-Hyderabad\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.sax\n",
    "import re\n",
    "import string\n",
    "import timeit\n",
    "import os\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing hindi_wiki_dump of around 1.2 gb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ContentHandler\n",
    "infoboxes=[]\n",
    "infoboxes1=[]\n",
    "infoboxes2=[]\n",
    "infoboxes3=[]\n",
    "titles = []\n",
    "counter = 0\n",
    "\n",
    "def find_infobox(text):\n",
    "  flag1 = text.find('{{Infobox')\n",
    "  flag2 = text.find('{{infobox')\n",
    "  flag3 = text.find('{{ज्ञानसंदूक')   \n",
    "  if (flag1==-1 and flag2==-1 and flag3==-1):\n",
    "    return 0\n",
    "  return 1\n",
    "\n",
    "def find_infobox1(text):\n",
    "  flag1 = text.find('{{Infobox') \n",
    "  if (flag1==-1):\n",
    "    return 0\n",
    "  return 1\n",
    "\n",
    "def find_infobox2(text):\n",
    "  flag1 = text.find('{{infobox') \n",
    "  if (flag1==-1):\n",
    "    return 0\n",
    "  return 1\n",
    "\n",
    "def find_infobox3(text):\n",
    "  flag1 = text.find('{{ज्ञानसंदूक') \n",
    "  if (flag1==-1):\n",
    "    return 0\n",
    "  return 1\n",
    "\n",
    "class WikiContentHandler( xml.sax.ContentHandler ):\n",
    "    def __init__(self):\n",
    "        self.current_data_buffer = \"\"\n",
    "        self.doc_id = []\n",
    "        self.text = \"\"\n",
    "        self.title = \"\"\n",
    "\n",
    "    def startElement(self, tag, attributes):\n",
    "        global counter\n",
    "        self.current_data_buffer = tag\n",
    "        if (tag == \"page\"):\n",
    "            counter+=1\n",
    "\n",
    "    def endElement(self, tag):\n",
    "        global infoboxes, infoboxes1, infoboxes2, infoboxes3, titles\n",
    "        if (self.current_data_buffer == \"title\"):\n",
    "            titles.append(self.title)\n",
    "            if(counter%20000==0):\n",
    "              print(counter)\n",
    "            self.title = \"\"\n",
    "\n",
    "        if (self.current_data_buffer == \"text\"):\n",
    "\n",
    "            if(find_infobox(self.text)):\n",
    "              infoboxes.append(self.doc_id[0])\n",
    "\n",
    "            if(find_infobox1(self.text)):\n",
    "              infoboxes1.append(self.doc_id[0])\n",
    "\n",
    "            if(find_infobox2(self.text)):\n",
    "              infoboxes2.append(self.doc_id[0])\n",
    "\n",
    "            if(find_infobox3(self.text)):\n",
    "              infoboxes3.append(self.doc_id[0])\n",
    "\n",
    "            self.text = \"\"\n",
    "            self.doc_id = []\n",
    "\n",
    "        self.current_data_buffer=\"\"\n",
    "        \n",
    "    \n",
    "    def characters(self, content):\n",
    "        if (self.current_data_buffer == \"title\"):\n",
    "            self.title = content\n",
    "\n",
    "        elif (self.current_data_buffer == \"id\"):\n",
    "            self.doc_id.append(content)\n",
    "            \n",
    "        elif (self.current_data_buffer == \"text\"):\n",
    "            self.text += content\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_filename = 'hindi_wiki_dump.xml'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing Started\n",
      "hindi_wiki_dump.xml\n",
      "20000\n",
      "40000\n",
      "60000\n",
      "80000\n",
      "100000\n",
      "120000\n",
      "140000\n",
      "160000\n",
      "180000\n",
      "200000\n",
      "220000\n",
      "240000\n",
      "260000\n",
      "Parsing Completed\n",
      "Time Taken(seconds) :  168.74127571999998\n",
      "271489\n",
      "271489\n"
     ]
    }
   ],
   "source": [
    "print(\"Parsing Started\")\n",
    "parser = xml.sax.make_parser()\n",
    "parser.setFeature(xml.sax.handler.feature_namespaces, 0)\n",
    "Handler = WikiContentHandler()\n",
    "parser.setContentHandler( Handler )\n",
    "start_time = timeit.default_timer()\n",
    "print(data_filename)\n",
    "parser.parse(data_filename)\n",
    "stop_time = timeit.default_timer()\n",
    "print(\"Parsing Completed\")\n",
    "print(\"Time Taken(seconds) : \", stop_time - start_time)\n",
    "print(counter)\n",
    "print(len(titles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35477\n",
      "33467\n",
      "706\n",
      "1383\n"
     ]
    }
   ],
   "source": [
    "print(len(infoboxes)) # previously 33467\n",
    "print(len(infoboxes1)) #Infobox\n",
    "print(len(infoboxes2)) #infobox\n",
    "print(len(infoboxes3)) #ज्ञानसंदूक\n",
    "# infoboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25426\n",
      "25426\n",
      "Present\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "fp = open('parsed_data.pkl', 'rb')\n",
    "parsed_data = pickle.load(fp)\n",
    "\n",
    "for k, v in parsed_data.items():\n",
    "    if(v['wikibase_item']=='Q7291'):\n",
    "        print(k)\n",
    "        print(v['id'])\n",
    "        \n",
    "for doc_id in infoboxes:\n",
    "    if(doc_id == '25426'):\n",
    "        print(\"Present\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parsing Cirrus dump to get wikibase_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_filename =  \"cirrus_out/65.065/wiki_00\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# s=\"shubham\"\n",
    "# s[1:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cirrus_out/65.065/wiki_23\n",
      "cirrus_out/65.065/wiki_13\n",
      "cirrus_out/65.065/wiki_74\n",
      "cirrus_out/65.065/wiki_66\n",
      "cirrus_out/65.065/wiki_73\n",
      "cirrus_out/65.065/wiki_51\n",
      "cirrus_out/65.065/wiki_63\n",
      "cirrus_out/65.065/wiki_89\n",
      "cirrus_out/65.065/wiki_21\n",
      "cirrus_out/65.065/wiki_95\n",
      "cirrus_out/65.065/wiki_88\n",
      "cirrus_out/65.065/wiki_28\n",
      "cirrus_out/65.065/wiki_07\n",
      "cirrus_out/65.065/wiki_02\n",
      "cirrus_out/65.065/wiki_25\n",
      "cirrus_out/65.065/wiki_82\n",
      "cirrus_out/65.065/wiki_04\n",
      "cirrus_out/65.065/wiki_39\n",
      "cirrus_out/65.065/wiki_43\n",
      "cirrus_out/65.065/wiki_86\n",
      "cirrus_out/65.065/wiki_75\n",
      "cirrus_out/65.065/wiki_45\n",
      "cirrus_out/65.065/wiki_09\n",
      "cirrus_out/65.065/wiki_77\n",
      "cirrus_out/65.065/wiki_15\n",
      "cirrus_out/65.065/wiki_84\n",
      "cirrus_out/65.065/wiki_57\n",
      "cirrus_out/65.065/wiki_16\n",
      "cirrus_out/65.065/wiki_69\n",
      "cirrus_out/65.065/wiki_68\n",
      "cirrus_out/65.065/wiki_40\n",
      "cirrus_out/65.065/wiki_34\n",
      "cirrus_out/65.065/wiki_12\n",
      "cirrus_out/65.065/wiki_99\n",
      "cirrus_out/65.065/wiki_10\n",
      "cirrus_out/65.065/wiki_97\n",
      "cirrus_out/65.065/wiki_37\n",
      "cirrus_out/65.065/wiki_70\n",
      "cirrus_out/65.065/wiki_53\n",
      "cirrus_out/65.065/wiki_79\n",
      "cirrus_out/65.065/wiki_55\n",
      "cirrus_out/65.065/wiki_08\n",
      "cirrus_out/65.065/wiki_65\n",
      "cirrus_out/65.065/wiki_36\n",
      "cirrus_out/65.065/wiki_92\n",
      "cirrus_out/65.065/wiki_42\n",
      "cirrus_out/65.065/wiki_00\n",
      "cirrus_out/65.065/wiki_60\n",
      "cirrus_out/65.065/wiki_32\n",
      "cirrus_out/65.065/wiki_29\n",
      "cirrus_out/65.065/wiki_87\n",
      "cirrus_out/65.065/wiki_56\n",
      "cirrus_out/65.065/wiki_80\n",
      "cirrus_out/65.065/wiki_76\n",
      "cirrus_out/65.065/wiki_30\n",
      "cirrus_out/65.065/wiki_31\n",
      "cirrus_out/65.065/wiki_19\n",
      "cirrus_out/65.065/wiki_54\n",
      "cirrus_out/65.065/wiki_85\n",
      "cirrus_out/65.065/wiki_64\n",
      "cirrus_out/65.065/wiki_81\n",
      "cirrus_out/65.065/wiki_05\n",
      "cirrus_out/65.065/wiki_26\n",
      "cirrus_out/65.065/wiki_03\n",
      "cirrus_out/65.065/wiki_72\n",
      "cirrus_out/65.065/wiki_78\n",
      "cirrus_out/65.065/wiki_41\n",
      "cirrus_out/65.065/wiki_49\n",
      "cirrus_out/65.065/wiki_33\n",
      "cirrus_out/65.065/wiki_11\n",
      "cirrus_out/65.065/wiki_90\n",
      "cirrus_out/65.065/wiki_24\n",
      "cirrus_out/65.065/wiki_20\n",
      "cirrus_out/65.065/wiki_47\n",
      "cirrus_out/65.065/wiki_18\n",
      "cirrus_out/65.065/wiki_59\n",
      "cirrus_out/65.065/wiki_14\n",
      "cirrus_out/65.065/wiki_98\n",
      "cirrus_out/65.065/wiki_22\n",
      "cirrus_out/65.065/wiki_35\n",
      "cirrus_out/65.065/wiki_94\n",
      "cirrus_out/65.065/wiki_27\n",
      "cirrus_out/65.065/wiki_61\n",
      "cirrus_out/65.065/wiki_48\n",
      "cirrus_out/65.065/wiki_50\n",
      "cirrus_out/65.065/wiki_96\n",
      "cirrus_out/65.065/wiki_01\n",
      "cirrus_out/65.065/wiki_44\n",
      "cirrus_out/65.065/wiki_52\n",
      "cirrus_out/65.065/wiki_93\n",
      "cirrus_out/65.065/wiki_83\n",
      "cirrus_out/65.065/wiki_06\n",
      "cirrus_out/65.065/wiki_46\n",
      "cirrus_out/65.065/wiki_58\n",
      "cirrus_out/65.065/wiki_17\n",
      "cirrus_out/65.065/wiki_71\n",
      "cirrus_out/65.065/wiki_62\n",
      "cirrus_out/65.065/wiki_67\n",
      "cirrus_out/65.065/wiki_38\n",
      "cirrus_out/65.065/wiki_91\n",
      "cirrus_out/65.1538461538461669/wiki_23\n",
      "cirrus_out/65.1538461538461669/wiki_13\n",
      "cirrus_out/65.1538461538461669/wiki_74\n",
      "cirrus_out/65.1538461538461669/wiki_66\n",
      "cirrus_out/65.1538461538461669/wiki_73\n",
      "cirrus_out/65.1538461538461669/wiki_51\n",
      "cirrus_out/65.1538461538461669/wiki_63\n",
      "cirrus_out/65.1538461538461669/wiki_89\n",
      "cirrus_out/65.1538461538461669/wiki_21\n",
      "cirrus_out/65.1538461538461669/wiki_95\n",
      "cirrus_out/65.1538461538461669/wiki_88\n",
      "cirrus_out/65.1538461538461669/wiki_28\n",
      "cirrus_out/65.1538461538461669/wiki_07\n",
      "cirrus_out/65.1538461538461669/wiki_02\n",
      "cirrus_out/65.1538461538461669/wiki_25\n",
      "cirrus_out/65.1538461538461669/wiki_82\n",
      "cirrus_out/65.1538461538461669/wiki_04\n",
      "cirrus_out/65.1538461538461669/wiki_39\n",
      "cirrus_out/65.1538461538461669/wiki_43\n",
      "cirrus_out/65.1538461538461669/wiki_86\n",
      "cirrus_out/65.1538461538461669/wiki_75\n",
      "cirrus_out/65.1538461538461669/wiki_45\n",
      "cirrus_out/65.1538461538461669/wiki_09\n",
      "cirrus_out/65.1538461538461669/wiki_77\n",
      "cirrus_out/65.1538461538461669/wiki_15\n",
      "cirrus_out/65.1538461538461669/wiki_84\n",
      "cirrus_out/65.1538461538461669/wiki_57\n",
      "cirrus_out/65.1538461538461669/wiki_16\n",
      "cirrus_out/65.1538461538461669/wiki_69\n",
      "cirrus_out/65.1538461538461669/wiki_68\n",
      "cirrus_out/65.1538461538461669/wiki_40\n",
      "cirrus_out/65.1538461538461669/wiki_34\n",
      "cirrus_out/65.1538461538461669/wiki_12\n",
      "cirrus_out/65.1538461538461669/wiki_99\n",
      "cirrus_out/65.1538461538461669/wiki_10\n",
      "cirrus_out/65.1538461538461669/wiki_97\n",
      "cirrus_out/65.1538461538461669/wiki_37\n",
      "cirrus_out/65.1538461538461669/wiki_70\n",
      "cirrus_out/65.1538461538461669/wiki_53\n",
      "cirrus_out/65.1538461538461669/wiki_79\n",
      "cirrus_out/65.1538461538461669/wiki_55\n",
      "cirrus_out/65.1538461538461669/wiki_08\n",
      "cirrus_out/65.1538461538461669/wiki_65\n",
      "cirrus_out/65.1538461538461669/wiki_36\n",
      "cirrus_out/65.1538461538461669/wiki_92\n",
      "cirrus_out/65.1538461538461669/wiki_42\n",
      "cirrus_out/65.1538461538461669/wiki_00\n",
      "cirrus_out/65.1538461538461669/wiki_60\n",
      "cirrus_out/65.1538461538461669/wiki_32\n",
      "cirrus_out/65.1538461538461669/wiki_29\n",
      "cirrus_out/65.1538461538461669/wiki_87\n",
      "cirrus_out/65.1538461538461669/wiki_56\n",
      "cirrus_out/65.1538461538461669/wiki_80\n",
      "cirrus_out/65.1538461538461669/wiki_76\n",
      "cirrus_out/65.1538461538461669/wiki_30\n",
      "cirrus_out/65.1538461538461669/wiki_31\n",
      "cirrus_out/65.1538461538461669/wiki_19\n",
      "cirrus_out/65.1538461538461669/wiki_54\n",
      "cirrus_out/65.1538461538461669/wiki_85\n",
      "cirrus_out/65.1538461538461669/wiki_64\n",
      "cirrus_out/65.1538461538461669/wiki_81\n",
      "cirrus_out/65.1538461538461669/wiki_05\n",
      "cirrus_out/65.1538461538461669/wiki_26\n",
      "cirrus_out/65.1538461538461669/wiki_03\n",
      "cirrus_out/65.1538461538461669/wiki_72\n",
      "cirrus_out/65.1538461538461669/wiki_78\n",
      "cirrus_out/65.1538461538461669/wiki_41\n",
      "cirrus_out/65.1538461538461669/wiki_49\n",
      "cirrus_out/65.1538461538461669/wiki_33\n",
      "cirrus_out/65.1538461538461669/wiki_11\n",
      "cirrus_out/65.1538461538461669/wiki_90\n",
      "cirrus_out/65.1538461538461669/wiki_24\n",
      "cirrus_out/65.1538461538461669/wiki_20\n",
      "cirrus_out/65.1538461538461669/wiki_47\n",
      "cirrus_out/65.1538461538461669/wiki_18\n",
      "cirrus_out/65.1538461538461669/wiki_59\n",
      "cirrus_out/65.1538461538461669/wiki_14\n",
      "cirrus_out/65.1538461538461669/wiki_98\n",
      "cirrus_out/65.1538461538461669/wiki_22\n",
      "cirrus_out/65.1538461538461669/wiki_35\n",
      "cirrus_out/65.1538461538461669/wiki_94\n",
      "cirrus_out/65.1538461538461669/wiki_27\n",
      "cirrus_out/65.1538461538461669/wiki_61\n",
      "cirrus_out/65.1538461538461669/wiki_48\n",
      "cirrus_out/65.1538461538461669/wiki_50\n",
      "cirrus_out/65.1538461538461669/wiki_96\n",
      "cirrus_out/65.1538461538461669/wiki_01\n",
      "cirrus_out/65.1538461538461669/wiki_44\n",
      "cirrus_out/65.1538461538461669/wiki_52\n",
      "cirrus_out/65.1538461538461669/wiki_93\n",
      "cirrus_out/65.1538461538461669/wiki_83\n",
      "cirrus_out/65.1538461538461669/wiki_06\n",
      "cirrus_out/65.1538461538461669/wiki_46\n",
      "cirrus_out/65.1538461538461669/wiki_58\n",
      "cirrus_out/65.1538461538461669/wiki_17\n",
      "cirrus_out/65.1538461538461669/wiki_71\n",
      "cirrus_out/65.1538461538461669/wiki_62\n",
      "cirrus_out/65.1538461538461669/wiki_67\n",
      "cirrus_out/65.1538461538461669/wiki_38\n",
      "cirrus_out/65.1538461538461669/wiki_91\n",
      "cirrus_out/65.2307692307692371/wiki_23\n",
      "cirrus_out/65.2307692307692371/wiki_13\n",
      "cirrus_out/65.2307692307692371/wiki_21\n",
      "cirrus_out/65.2307692307692371/wiki_28\n",
      "cirrus_out/65.2307692307692371/wiki_07\n",
      "cirrus_out/65.2307692307692371/wiki_02\n",
      "cirrus_out/65.2307692307692371/wiki_25\n",
      "cirrus_out/65.2307692307692371/wiki_04\n",
      "cirrus_out/65.2307692307692371/wiki_39\n",
      "cirrus_out/65.2307692307692371/wiki_09\n",
      "cirrus_out/65.2307692307692371/wiki_15\n",
      "cirrus_out/65.2307692307692371/wiki_16\n",
      "cirrus_out/65.2307692307692371/wiki_40\n",
      "cirrus_out/65.2307692307692371/wiki_34\n",
      "cirrus_out/65.2307692307692371/wiki_12\n",
      "cirrus_out/65.2307692307692371/wiki_10\n",
      "cirrus_out/65.2307692307692371/wiki_37\n",
      "cirrus_out/65.2307692307692371/wiki_08\n",
      "cirrus_out/65.2307692307692371/wiki_36\n",
      "cirrus_out/65.2307692307692371/wiki_42\n",
      "cirrus_out/65.2307692307692371/wiki_00\n",
      "cirrus_out/65.2307692307692371/wiki_32\n",
      "cirrus_out/65.2307692307692371/wiki_29\n",
      "cirrus_out/65.2307692307692371/wiki_30\n",
      "cirrus_out/65.2307692307692371/wiki_31\n",
      "cirrus_out/65.2307692307692371/wiki_19\n",
      "cirrus_out/65.2307692307692371/wiki_05\n",
      "cirrus_out/65.2307692307692371/wiki_26\n",
      "cirrus_out/65.2307692307692371/wiki_03\n",
      "cirrus_out/65.2307692307692371/wiki_41\n",
      "cirrus_out/65.2307692307692371/wiki_33\n",
      "cirrus_out/65.2307692307692371/wiki_11\n",
      "cirrus_out/65.2307692307692371/wiki_24\n",
      "cirrus_out/65.2307692307692371/wiki_20\n",
      "cirrus_out/65.2307692307692371/wiki_18\n",
      "cirrus_out/65.2307692307692371/wiki_14\n",
      "cirrus_out/65.2307692307692371/wiki_22\n",
      "cirrus_out/65.2307692307692371/wiki_35\n",
      "cirrus_out/65.2307692307692371/wiki_27\n",
      "cirrus_out/65.2307692307692371/wiki_01\n",
      "cirrus_out/65.2307692307692371/wiki_06\n",
      "cirrus_out/65.2307692307692371/wiki_17\n",
      "cirrus_out/65.2307692307692371/wiki_38\n",
      "cirrus_out/65.1153846153846168/wiki_23\n",
      "cirrus_out/65.1153846153846168/wiki_13\n",
      "cirrus_out/65.1153846153846168/wiki_74\n",
      "cirrus_out/65.1153846153846168/wiki_66\n",
      "cirrus_out/65.1153846153846168/wiki_73\n",
      "cirrus_out/65.1153846153846168/wiki_51\n",
      "cirrus_out/65.1153846153846168/wiki_63\n",
      "cirrus_out/65.1153846153846168/wiki_89\n",
      "cirrus_out/65.1153846153846168/wiki_21\n",
      "cirrus_out/65.1153846153846168/wiki_95\n",
      "cirrus_out/65.1153846153846168/wiki_88\n",
      "cirrus_out/65.1153846153846168/wiki_28\n",
      "cirrus_out/65.1153846153846168/wiki_07\n",
      "cirrus_out/65.1153846153846168/wiki_02\n",
      "cirrus_out/65.1153846153846168/wiki_25\n",
      "cirrus_out/65.1153846153846168/wiki_82\n",
      "cirrus_out/65.1153846153846168/wiki_04\n",
      "cirrus_out/65.1153846153846168/wiki_39\n",
      "cirrus_out/65.1153846153846168/wiki_43\n",
      "cirrus_out/65.1153846153846168/wiki_86\n",
      "cirrus_out/65.1153846153846168/wiki_75\n",
      "cirrus_out/65.1153846153846168/wiki_45\n",
      "cirrus_out/65.1153846153846168/wiki_09\n",
      "cirrus_out/65.1153846153846168/wiki_77\n",
      "cirrus_out/65.1153846153846168/wiki_15\n",
      "cirrus_out/65.1153846153846168/wiki_84\n",
      "cirrus_out/65.1153846153846168/wiki_57\n",
      "cirrus_out/65.1153846153846168/wiki_16\n",
      "cirrus_out/65.1153846153846168/wiki_69\n",
      "cirrus_out/65.1153846153846168/wiki_68\n",
      "cirrus_out/65.1153846153846168/wiki_40\n",
      "cirrus_out/65.1153846153846168/wiki_34\n",
      "cirrus_out/65.1153846153846168/wiki_12\n",
      "cirrus_out/65.1153846153846168/wiki_99\n",
      "cirrus_out/65.1153846153846168/wiki_10\n",
      "cirrus_out/65.1153846153846168/wiki_97\n",
      "cirrus_out/65.1153846153846168/wiki_37\n",
      "cirrus_out/65.1153846153846168/wiki_70\n",
      "cirrus_out/65.1153846153846168/wiki_53\n",
      "cirrus_out/65.1153846153846168/wiki_79\n",
      "cirrus_out/65.1153846153846168/wiki_55\n",
      "cirrus_out/65.1153846153846168/wiki_08\n",
      "cirrus_out/65.1153846153846168/wiki_65\n",
      "cirrus_out/65.1153846153846168/wiki_36\n",
      "cirrus_out/65.1153846153846168/wiki_92\n",
      "cirrus_out/65.1153846153846168/wiki_42\n",
      "cirrus_out/65.1153846153846168/wiki_00\n",
      "cirrus_out/65.1153846153846168/wiki_60\n",
      "cirrus_out/65.1153846153846168/wiki_32\n",
      "cirrus_out/65.1153846153846168/wiki_29\n",
      "cirrus_out/65.1153846153846168/wiki_87\n",
      "cirrus_out/65.1153846153846168/wiki_56\n",
      "cirrus_out/65.1153846153846168/wiki_80\n",
      "cirrus_out/65.1153846153846168/wiki_76\n",
      "cirrus_out/65.1153846153846168/wiki_30\n",
      "cirrus_out/65.1153846153846168/wiki_31\n",
      "cirrus_out/65.1153846153846168/wiki_19\n",
      "cirrus_out/65.1153846153846168/wiki_54\n",
      "cirrus_out/65.1153846153846168/wiki_85\n",
      "cirrus_out/65.1153846153846168/wiki_64\n",
      "cirrus_out/65.1153846153846168/wiki_81\n",
      "cirrus_out/65.1153846153846168/wiki_05\n",
      "cirrus_out/65.1153846153846168/wiki_26\n",
      "cirrus_out/65.1153846153846168/wiki_03\n",
      "cirrus_out/65.1153846153846168/wiki_72\n",
      "cirrus_out/65.1153846153846168/wiki_78\n",
      "cirrus_out/65.1153846153846168/wiki_41\n",
      "cirrus_out/65.1153846153846168/wiki_49\n",
      "cirrus_out/65.1153846153846168/wiki_33\n",
      "cirrus_out/65.1153846153846168/wiki_11\n",
      "cirrus_out/65.1153846153846168/wiki_90\n",
      "cirrus_out/65.1153846153846168/wiki_24\n",
      "cirrus_out/65.1153846153846168/wiki_20\n",
      "cirrus_out/65.1153846153846168/wiki_47\n",
      "cirrus_out/65.1153846153846168/wiki_18\n",
      "cirrus_out/65.1153846153846168/wiki_59\n",
      "cirrus_out/65.1153846153846168/wiki_14\n",
      "cirrus_out/65.1153846153846168/wiki_98\n",
      "cirrus_out/65.1153846153846168/wiki_22\n",
      "cirrus_out/65.1153846153846168/wiki_35\n",
      "cirrus_out/65.1153846153846168/wiki_94\n",
      "cirrus_out/65.1153846153846168/wiki_27\n",
      "cirrus_out/65.1153846153846168/wiki_61\n",
      "cirrus_out/65.1153846153846168/wiki_48\n",
      "cirrus_out/65.1153846153846168/wiki_50\n",
      "cirrus_out/65.1153846153846168/wiki_96\n",
      "cirrus_out/65.1153846153846168/wiki_01\n",
      "cirrus_out/65.1153846153846168/wiki_44\n",
      "cirrus_out/65.1153846153846168/wiki_52\n",
      "cirrus_out/65.1153846153846168/wiki_93\n",
      "cirrus_out/65.1153846153846168/wiki_83\n",
      "cirrus_out/65.1153846153846168/wiki_06\n",
      "cirrus_out/65.1153846153846168/wiki_46\n",
      "cirrus_out/65.1153846153846168/wiki_58\n",
      "cirrus_out/65.1153846153846168/wiki_17\n",
      "cirrus_out/65.1153846153846168/wiki_71\n",
      "cirrus_out/65.1153846153846168/wiki_62\n",
      "cirrus_out/65.1153846153846168/wiki_67\n",
      "cirrus_out/65.1153846153846168/wiki_38\n",
      "cirrus_out/65.1153846153846168/wiki_91\n",
      "cirrus_out/65.0769230769230867/wiki_23\n",
      "cirrus_out/65.0769230769230867/wiki_13\n",
      "cirrus_out/65.0769230769230867/wiki_74\n",
      "cirrus_out/65.0769230769230867/wiki_66\n",
      "cirrus_out/65.0769230769230867/wiki_73\n",
      "cirrus_out/65.0769230769230867/wiki_51\n",
      "cirrus_out/65.0769230769230867/wiki_63\n",
      "cirrus_out/65.0769230769230867/wiki_89\n",
      "cirrus_out/65.0769230769230867/wiki_21\n",
      "cirrus_out/65.0769230769230867/wiki_95\n",
      "cirrus_out/65.0769230769230867/wiki_88\n",
      "cirrus_out/65.0769230769230867/wiki_28\n",
      "cirrus_out/65.0769230769230867/wiki_07\n",
      "cirrus_out/65.0769230769230867/wiki_02\n",
      "cirrus_out/65.0769230769230867/wiki_25\n",
      "cirrus_out/65.0769230769230867/wiki_82\n",
      "cirrus_out/65.0769230769230867/wiki_04\n",
      "cirrus_out/65.0769230769230867/wiki_39\n",
      "cirrus_out/65.0769230769230867/wiki_43\n",
      "cirrus_out/65.0769230769230867/wiki_86\n",
      "cirrus_out/65.0769230769230867/wiki_75\n",
      "cirrus_out/65.0769230769230867/wiki_45\n",
      "cirrus_out/65.0769230769230867/wiki_09\n",
      "cirrus_out/65.0769230769230867/wiki_77\n",
      "cirrus_out/65.0769230769230867/wiki_15\n",
      "cirrus_out/65.0769230769230867/wiki_84\n",
      "cirrus_out/65.0769230769230867/wiki_57\n",
      "cirrus_out/65.0769230769230867/wiki_16\n",
      "cirrus_out/65.0769230769230867/wiki_69\n",
      "cirrus_out/65.0769230769230867/wiki_68\n",
      "cirrus_out/65.0769230769230867/wiki_40\n",
      "cirrus_out/65.0769230769230867/wiki_34\n",
      "cirrus_out/65.0769230769230867/wiki_12\n",
      "cirrus_out/65.0769230769230867/wiki_99\n",
      "cirrus_out/65.0769230769230867/wiki_10\n",
      "cirrus_out/65.0769230769230867/wiki_97\n",
      "cirrus_out/65.0769230769230867/wiki_37\n",
      "cirrus_out/65.0769230769230867/wiki_70\n",
      "cirrus_out/65.0769230769230867/wiki_53\n",
      "cirrus_out/65.0769230769230867/wiki_79\n",
      "cirrus_out/65.0769230769230867/wiki_55\n",
      "cirrus_out/65.0769230769230867/wiki_08\n",
      "cirrus_out/65.0769230769230867/wiki_65\n",
      "cirrus_out/65.0769230769230867/wiki_36\n",
      "cirrus_out/65.0769230769230867/wiki_92\n",
      "cirrus_out/65.0769230769230867/wiki_42\n",
      "cirrus_out/65.0769230769230867/wiki_00\n",
      "cirrus_out/65.0769230769230867/wiki_60\n",
      "cirrus_out/65.0769230769230867/wiki_32\n",
      "cirrus_out/65.0769230769230867/wiki_29\n",
      "cirrus_out/65.0769230769230867/wiki_87\n",
      "cirrus_out/65.0769230769230867/wiki_56\n",
      "cirrus_out/65.0769230769230867/wiki_80\n",
      "cirrus_out/65.0769230769230867/wiki_76\n",
      "cirrus_out/65.0769230769230867/wiki_30\n",
      "cirrus_out/65.0769230769230867/wiki_31\n",
      "cirrus_out/65.0769230769230867/wiki_19\n",
      "cirrus_out/65.0769230769230867/wiki_54\n",
      "cirrus_out/65.0769230769230867/wiki_85\n",
      "cirrus_out/65.0769230769230867/wiki_64\n",
      "cirrus_out/65.0769230769230867/wiki_81\n",
      "cirrus_out/65.0769230769230867/wiki_05\n",
      "cirrus_out/65.0769230769230867/wiki_26\n",
      "cirrus_out/65.0769230769230867/wiki_03\n",
      "cirrus_out/65.0769230769230867/wiki_72\n",
      "cirrus_out/65.0769230769230867/wiki_78\n",
      "cirrus_out/65.0769230769230867/wiki_41\n",
      "cirrus_out/65.0769230769230867/wiki_49\n",
      "cirrus_out/65.0769230769230867/wiki_33\n",
      "cirrus_out/65.0769230769230867/wiki_11\n",
      "cirrus_out/65.0769230769230867/wiki_90\n",
      "cirrus_out/65.0769230769230867/wiki_24\n",
      "cirrus_out/65.0769230769230867/wiki_20\n",
      "cirrus_out/65.0769230769230867/wiki_47\n",
      "cirrus_out/65.0769230769230867/wiki_18\n",
      "cirrus_out/65.0769230769230867/wiki_59\n",
      "cirrus_out/65.0769230769230867/wiki_14\n",
      "cirrus_out/65.0769230769230867/wiki_98\n",
      "cirrus_out/65.0769230769230867/wiki_22\n",
      "cirrus_out/65.0769230769230867/wiki_35\n",
      "cirrus_out/65.0769230769230867/wiki_94\n",
      "cirrus_out/65.0769230769230867/wiki_27\n",
      "cirrus_out/65.0769230769230867/wiki_61\n",
      "cirrus_out/65.0769230769230867/wiki_48\n",
      "cirrus_out/65.0769230769230867/wiki_50\n",
      "cirrus_out/65.0769230769230867/wiki_96\n",
      "cirrus_out/65.0769230769230867/wiki_01\n",
      "cirrus_out/65.0769230769230867/wiki_44\n",
      "cirrus_out/65.0769230769230867/wiki_52\n",
      "cirrus_out/65.0769230769230867/wiki_93\n",
      "cirrus_out/65.0769230769230867/wiki_83\n",
      "cirrus_out/65.0769230769230867/wiki_06\n",
      "cirrus_out/65.0769230769230867/wiki_46\n",
      "cirrus_out/65.0769230769230867/wiki_58\n",
      "cirrus_out/65.0769230769230867/wiki_17\n",
      "cirrus_out/65.0769230769230867/wiki_71\n",
      "cirrus_out/65.0769230769230867/wiki_62\n",
      "cirrus_out/65.0769230769230867/wiki_67\n",
      "cirrus_out/65.0769230769230867/wiki_38\n",
      "cirrus_out/65.0769230769230867/wiki_91\n",
      "cirrus_out/65.0384615384615366/wiki_23\n",
      "cirrus_out/65.0384615384615366/wiki_13\n",
      "cirrus_out/65.0384615384615366/wiki_74\n",
      "cirrus_out/65.0384615384615366/wiki_66\n",
      "cirrus_out/65.0384615384615366/wiki_73\n",
      "cirrus_out/65.0384615384615366/wiki_51\n",
      "cirrus_out/65.0384615384615366/wiki_63\n",
      "cirrus_out/65.0384615384615366/wiki_89\n",
      "cirrus_out/65.0384615384615366/wiki_21\n",
      "cirrus_out/65.0384615384615366/wiki_95\n",
      "cirrus_out/65.0384615384615366/wiki_88\n",
      "cirrus_out/65.0384615384615366/wiki_28\n",
      "cirrus_out/65.0384615384615366/wiki_07\n",
      "cirrus_out/65.0384615384615366/wiki_02\n",
      "cirrus_out/65.0384615384615366/wiki_25\n",
      "cirrus_out/65.0384615384615366/wiki_82\n",
      "cirrus_out/65.0384615384615366/wiki_04\n",
      "cirrus_out/65.0384615384615366/wiki_39\n",
      "cirrus_out/65.0384615384615366/wiki_43\n",
      "cirrus_out/65.0384615384615366/wiki_86\n",
      "cirrus_out/65.0384615384615366/wiki_75\n",
      "cirrus_out/65.0384615384615366/wiki_45\n",
      "cirrus_out/65.0384615384615366/wiki_09\n",
      "cirrus_out/65.0384615384615366/wiki_77\n",
      "cirrus_out/65.0384615384615366/wiki_15\n",
      "cirrus_out/65.0384615384615366/wiki_84\n",
      "cirrus_out/65.0384615384615366/wiki_57\n",
      "cirrus_out/65.0384615384615366/wiki_16\n",
      "cirrus_out/65.0384615384615366/wiki_69\n",
      "cirrus_out/65.0384615384615366/wiki_68\n",
      "cirrus_out/65.0384615384615366/wiki_40\n",
      "cirrus_out/65.0384615384615366/wiki_34\n",
      "cirrus_out/65.0384615384615366/wiki_12\n",
      "cirrus_out/65.0384615384615366/wiki_99\n",
      "cirrus_out/65.0384615384615366/wiki_10\n",
      "cirrus_out/65.0384615384615366/wiki_97\n",
      "cirrus_out/65.0384615384615366/wiki_37\n",
      "cirrus_out/65.0384615384615366/wiki_70\n",
      "cirrus_out/65.0384615384615366/wiki_53\n",
      "cirrus_out/65.0384615384615366/wiki_79\n",
      "cirrus_out/65.0384615384615366/wiki_55\n",
      "cirrus_out/65.0384615384615366/wiki_08\n",
      "cirrus_out/65.0384615384615366/wiki_65\n",
      "cirrus_out/65.0384615384615366/wiki_36\n",
      "cirrus_out/65.0384615384615366/wiki_92\n",
      "cirrus_out/65.0384615384615366/wiki_42\n",
      "cirrus_out/65.0384615384615366/wiki_00\n",
      "cirrus_out/65.0384615384615366/wiki_60\n",
      "cirrus_out/65.0384615384615366/wiki_32\n",
      "cirrus_out/65.0384615384615366/wiki_29\n",
      "cirrus_out/65.0384615384615366/wiki_87\n",
      "cirrus_out/65.0384615384615366/wiki_56\n",
      "cirrus_out/65.0384615384615366/wiki_80\n",
      "cirrus_out/65.0384615384615366/wiki_76\n",
      "cirrus_out/65.0384615384615366/wiki_30\n",
      "cirrus_out/65.0384615384615366/wiki_31\n",
      "cirrus_out/65.0384615384615366/wiki_19\n",
      "cirrus_out/65.0384615384615366/wiki_54\n",
      "cirrus_out/65.0384615384615366/wiki_85\n",
      "cirrus_out/65.0384615384615366/wiki_64\n",
      "cirrus_out/65.0384615384615366/wiki_81\n",
      "cirrus_out/65.0384615384615366/wiki_05\n",
      "cirrus_out/65.0384615384615366/wiki_26\n",
      "cirrus_out/65.0384615384615366/wiki_03\n",
      "cirrus_out/65.0384615384615366/wiki_72\n",
      "cirrus_out/65.0384615384615366/wiki_78\n",
      "cirrus_out/65.0384615384615366/wiki_41\n",
      "cirrus_out/65.0384615384615366/wiki_49\n",
      "cirrus_out/65.0384615384615366/wiki_33\n",
      "cirrus_out/65.0384615384615366/wiki_11\n",
      "cirrus_out/65.0384615384615366/wiki_90\n",
      "cirrus_out/65.0384615384615366/wiki_24\n",
      "cirrus_out/65.0384615384615366/wiki_20\n",
      "cirrus_out/65.0384615384615366/wiki_47\n",
      "cirrus_out/65.0384615384615366/wiki_18\n",
      "cirrus_out/65.0384615384615366/wiki_59\n",
      "cirrus_out/65.0384615384615366/wiki_14\n",
      "cirrus_out/65.0384615384615366/wiki_98\n",
      "cirrus_out/65.0384615384615366/wiki_22\n",
      "cirrus_out/65.0384615384615366/wiki_35\n",
      "cirrus_out/65.0384615384615366/wiki_94\n",
      "cirrus_out/65.0384615384615366/wiki_27\n",
      "cirrus_out/65.0384615384615366/wiki_61\n",
      "cirrus_out/65.0384615384615366/wiki_48\n",
      "cirrus_out/65.0384615384615366/wiki_50\n",
      "cirrus_out/65.0384615384615366/wiki_96\n",
      "cirrus_out/65.0384615384615366/wiki_01\n",
      "cirrus_out/65.0384615384615366/wiki_44\n",
      "cirrus_out/65.0384615384615366/wiki_52\n",
      "cirrus_out/65.0384615384615366/wiki_93\n",
      "cirrus_out/65.0384615384615366/wiki_83\n",
      "cirrus_out/65.0384615384615366/wiki_06\n",
      "cirrus_out/65.0384615384615366/wiki_46\n",
      "cirrus_out/65.0384615384615366/wiki_58\n",
      "cirrus_out/65.0384615384615366/wiki_17\n",
      "cirrus_out/65.0384615384615366/wiki_71\n",
      "cirrus_out/65.0384615384615366/wiki_62\n",
      "cirrus_out/65.0384615384615366/wiki_67\n",
      "cirrus_out/65.0384615384615366/wiki_38\n",
      "cirrus_out/65.0384615384615366/wiki_91\n",
      "cirrus_out/65.192307692307770/wiki_23\n",
      "cirrus_out/65.192307692307770/wiki_13\n",
      "cirrus_out/65.192307692307770/wiki_74\n",
      "cirrus_out/65.192307692307770/wiki_66\n",
      "cirrus_out/65.192307692307770/wiki_73\n",
      "cirrus_out/65.192307692307770/wiki_51\n",
      "cirrus_out/65.192307692307770/wiki_63\n",
      "cirrus_out/65.192307692307770/wiki_89\n",
      "cirrus_out/65.192307692307770/wiki_21\n",
      "cirrus_out/65.192307692307770/wiki_95\n",
      "cirrus_out/65.192307692307770/wiki_88\n",
      "cirrus_out/65.192307692307770/wiki_28\n",
      "cirrus_out/65.192307692307770/wiki_07\n",
      "cirrus_out/65.192307692307770/wiki_02\n",
      "cirrus_out/65.192307692307770/wiki_25\n",
      "cirrus_out/65.192307692307770/wiki_82\n",
      "cirrus_out/65.192307692307770/wiki_04\n",
      "cirrus_out/65.192307692307770/wiki_39\n",
      "cirrus_out/65.192307692307770/wiki_43\n",
      "cirrus_out/65.192307692307770/wiki_86\n",
      "cirrus_out/65.192307692307770/wiki_75\n",
      "cirrus_out/65.192307692307770/wiki_45\n",
      "cirrus_out/65.192307692307770/wiki_09\n",
      "cirrus_out/65.192307692307770/wiki_77\n",
      "cirrus_out/65.192307692307770/wiki_15\n",
      "cirrus_out/65.192307692307770/wiki_84\n",
      "cirrus_out/65.192307692307770/wiki_57\n",
      "cirrus_out/65.192307692307770/wiki_16\n",
      "cirrus_out/65.192307692307770/wiki_69\n",
      "cirrus_out/65.192307692307770/wiki_68\n",
      "cirrus_out/65.192307692307770/wiki_40\n",
      "cirrus_out/65.192307692307770/wiki_34\n",
      "cirrus_out/65.192307692307770/wiki_12\n",
      "cirrus_out/65.192307692307770/wiki_99\n",
      "cirrus_out/65.192307692307770/wiki_10\n",
      "cirrus_out/65.192307692307770/wiki_97\n",
      "cirrus_out/65.192307692307770/wiki_37\n",
      "cirrus_out/65.192307692307770/wiki_70\n",
      "cirrus_out/65.192307692307770/wiki_53\n",
      "cirrus_out/65.192307692307770/wiki_79\n",
      "cirrus_out/65.192307692307770/wiki_55\n",
      "cirrus_out/65.192307692307770/wiki_08\n",
      "cirrus_out/65.192307692307770/wiki_65\n",
      "cirrus_out/65.192307692307770/wiki_36\n",
      "cirrus_out/65.192307692307770/wiki_92\n",
      "cirrus_out/65.192307692307770/wiki_42\n",
      "cirrus_out/65.192307692307770/wiki_00\n",
      "cirrus_out/65.192307692307770/wiki_60\n",
      "cirrus_out/65.192307692307770/wiki_32\n",
      "cirrus_out/65.192307692307770/wiki_29\n",
      "cirrus_out/65.192307692307770/wiki_87\n",
      "cirrus_out/65.192307692307770/wiki_56\n",
      "cirrus_out/65.192307692307770/wiki_80\n",
      "cirrus_out/65.192307692307770/wiki_76\n",
      "cirrus_out/65.192307692307770/wiki_30\n",
      "cirrus_out/65.192307692307770/wiki_31\n",
      "cirrus_out/65.192307692307770/wiki_19\n",
      "cirrus_out/65.192307692307770/wiki_54\n",
      "cirrus_out/65.192307692307770/wiki_85\n",
      "cirrus_out/65.192307692307770/wiki_64\n",
      "cirrus_out/65.192307692307770/wiki_81\n",
      "cirrus_out/65.192307692307770/wiki_05\n",
      "cirrus_out/65.192307692307770/wiki_26\n",
      "cirrus_out/65.192307692307770/wiki_03\n",
      "cirrus_out/65.192307692307770/wiki_72\n",
      "cirrus_out/65.192307692307770/wiki_78\n",
      "cirrus_out/65.192307692307770/wiki_41\n",
      "cirrus_out/65.192307692307770/wiki_49\n",
      "cirrus_out/65.192307692307770/wiki_33\n",
      "cirrus_out/65.192307692307770/wiki_11\n",
      "cirrus_out/65.192307692307770/wiki_90\n",
      "cirrus_out/65.192307692307770/wiki_24\n",
      "cirrus_out/65.192307692307770/wiki_20\n",
      "cirrus_out/65.192307692307770/wiki_47\n",
      "cirrus_out/65.192307692307770/wiki_18\n",
      "cirrus_out/65.192307692307770/wiki_59\n",
      "cirrus_out/65.192307692307770/wiki_14\n",
      "cirrus_out/65.192307692307770/wiki_98\n",
      "cirrus_out/65.192307692307770/wiki_22\n",
      "cirrus_out/65.192307692307770/wiki_35\n",
      "cirrus_out/65.192307692307770/wiki_94\n",
      "cirrus_out/65.192307692307770/wiki_27\n",
      "cirrus_out/65.192307692307770/wiki_61\n",
      "cirrus_out/65.192307692307770/wiki_48\n",
      "cirrus_out/65.192307692307770/wiki_50\n",
      "cirrus_out/65.192307692307770/wiki_96\n",
      "cirrus_out/65.192307692307770/wiki_01\n",
      "cirrus_out/65.192307692307770/wiki_44\n",
      "cirrus_out/65.192307692307770/wiki_52\n",
      "cirrus_out/65.192307692307770/wiki_93\n",
      "cirrus_out/65.192307692307770/wiki_83\n",
      "cirrus_out/65.192307692307770/wiki_06\n",
      "cirrus_out/65.192307692307770/wiki_46\n",
      "cirrus_out/65.192307692307770/wiki_58\n",
      "cirrus_out/65.192307692307770/wiki_17\n",
      "cirrus_out/65.192307692307770/wiki_71\n",
      "cirrus_out/65.192307692307770/wiki_62\n",
      "cirrus_out/65.192307692307770/wiki_67\n",
      "cirrus_out/65.192307692307770/wiki_38\n",
      "cirrus_out/65.192307692307770/wiki_91\n"
     ]
    }
   ],
   "source": [
    "files = []\n",
    "# r=root, d=directories, f = files\n",
    "path = \"cirrus_out\"\n",
    "for r, d, f in os.walk(path):\n",
    "    for file in f:\n",
    "        files.append(os.path.join(r, file))\n",
    "\n",
    "for f in files:\n",
    "    print(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "145805"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data = {}\n",
    "\n",
    "for file in files:\n",
    "    temp_data = {}\n",
    "    doc_id=\"\"\n",
    "    fp = open(file)\n",
    "    #print(file)\n",
    "    for line in fp:\n",
    "        if line.startswith('<doc'):\n",
    "            elements = line.strip()[1:-1]\n",
    "            \n",
    "            #docid\n",
    "            indurl=line.find(\"url\")\n",
    "            docid=line[9:indurl-2]\n",
    "            doc_id = docid\n",
    "            \n",
    "            temp_data = {}\n",
    "            temp_data['id'] = docid\n",
    "            \n",
    "            #url\n",
    "            inttile=line.find(\"title\")\n",
    "            url=line[indurl+5:inttile-2]\n",
    "            temp_data['url'] = url\n",
    "                        \n",
    "            #language\n",
    "            lan=line.find(\"language\")\n",
    "            language=line[lan+10:lan+12]\n",
    "            temp_data['language'] = language\n",
    "            \n",
    "            #title\n",
    "            title=line[inttile+7:lan-2]\n",
    "            temp_data['title'] = title\n",
    "\n",
    "            #revision\n",
    "            rs=line.find(\"revision\")\n",
    "            wb=line.find(\"wikibase\")\n",
    "            revision=line[rs+10:wb-2]\n",
    "            temp_data['revision'] = revision\n",
    "\n",
    "            #wikibase\n",
    "            lan=line.find(\"wikibase\")\n",
    "            wikibase=line[lan+15:-3]\n",
    "            temp_data['wikibase_item'] = wikibase\n",
    "            \n",
    "\n",
    "        if line.startswith('['):\n",
    "            line2 = line.strip()[1:-1]\n",
    "            line2 = line2.split(\",\")\n",
    "            for i in range(len(line2)):\n",
    "                line2[i] = line2[i].strip()[1:-1]\n",
    "\n",
    "            temp_data['tags'] = line2\n",
    "            main_data[doc_id]=temp_data\n",
    "#         print(line)\n",
    "#         print(temp_data)\n",
    "#         break\n",
    "\n",
    "# print(temp_data)\n",
    "# for k, v in main_data.items():\n",
    "#     print(\"Key = \", k)\n",
    "#     for k1, v1 in v.items():\n",
    "#         print(k1, \" = \", v1)\n",
    "# print(main_data)\n",
    "len(main_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# af = open(\"metadata.pkl\", \"wb\")\n",
    "# pickle.dump(main_data, af)\n",
    "# af.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '10138',\n",
       " 'language': 'hi',\n",
       " 'revision': '4982868',\n",
       " 'tags': ['वेबग्रंथागार साँचा वेबैक कड़ियाँ',\n",
       "  'सीएस1 अंग्रेज़ी-भाषा स्रोत (en)',\n",
       "  'सीएस1 त्रुटियाँ: तिथियाँ',\n",
       "  'पृष्ठ जिनमें उद्धरणों में शीर्षक का अभाव है',\n",
       "  'पृष्ठ जिनमें केवल यूआरएल मात्र उद्धृत है',\n",
       "  'टेम्पलेट कॉल में डुप्लिकेट तर्क का उपयोग करते हुए पन्ने',\n",
       "  'वी॰आई॰एफ॰ए॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'एल॰सी॰सी॰एन॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'आई॰एस॰एन॰आई॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'जी॰एन॰डी॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'एस॰ई॰एल॰आई॰बी॰आर॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'बी॰एन॰एफ॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'बी॰आई॰बी॰एस॰वाई॰एस॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'एन॰एल॰ए॰ अभिज्ञापक वाले विकिपीडिया पृष्ठ',\n",
       "  'भारतीय स्वतंत्रता सेनानी',\n",
       "  'पश्चिम बंगाल',\n",
       "  'बंगाल के लोग',\n",
       "  'पश्चिम बंगाल के लोग',\n",
       "  'व्यक्तिगत जीवन',\n",
       "  'भारत का इतिहास',\n",
       "  'मृत लोग',\n",
       "  '1897 में जन्मे लोग',\n",
       "  '१९४५ में निधन',\n",
       "  'सुभाष चन्द्र बोस',\n",
       "  'भारतीय राष्ट्रीय कांग्रेस के अध्यक्ष'],\n",
       " 'title': 'सुभाष चन्द्र बोस',\n",
       " 'url': 'http://hi.wikipedia.org/wiki?curid=10138',\n",
       " 'wikibase_item': 'Q2153'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data['10138']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '14',\n",
       " 'language': 'hi',\n",
       " 'revision': '4955335',\n",
       " 'tags': ['हिन्दू धर्म'],\n",
       " 'title': 'दैनिक पूजा',\n",
       " 'url': 'http://hi.wikipedia.org/wiki?curid=14',\n",
       " 'wikibase_item': 'Q15649343'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data['14']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# af = open(\"infobox_present.pkl\", \"rb\")\n",
    "# info_present = pickle.load(af)\n",
    "# af.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35477\n"
     ]
    }
   ],
   "source": [
    "info_present = {}\n",
    "for doc_id in infoboxes:\n",
    "    info_present[doc_id]=1\n",
    "print(len(info_present))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in main_data.items():\n",
    "    if k in info_present:\n",
    "        main_data[k]['infobox']=1\n",
    "    else:\n",
    "        main_data[k]['infobox']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for k, v in main_data.items():\n",
    "    print(\"Key : \", k)\n",
    "    for k1, v1 in v.items():\n",
    "        print(k1, \" : \", v1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '25426',\n",
       " 'infobox': 1,\n",
       " 'language': 'hi',\n",
       " 'revision': '4978218',\n",
       " 'tags': ['Pages using citations with accessdate and no URL',\n",
       "  'सीएस1 रखरखाव: एक से अधिक नाम: authors list',\n",
       "  'सीएस1 त्रुटियाँ: लेखक अथवा संपादक प्राचल अनुपस्थित',\n",
       "  'लेख जो September 2009 से स्रोतहीन हैं',\n",
       "  'सभी स्रोतहीन लेख',\n",
       "  'टूटी हुई फ़ाइल कड़ियों वाले पृष्ठ',\n",
       "  'सुधार योग्य सभी लेख',\n",
       "  'लेख जिनमें जून 2020 से मृत कड़ियाँ हैं',\n",
       "  'मृत कड़ियों वाले लेख',\n",
       "  'सीएस1 रखरखाव: फालतू पाठ: authors list',\n",
       "  'सीएस1 अंग्रेज़ी-भाषा स्रोत (en)',\n",
       "  'खेल',\n",
       "  'बैडमिंटन'],\n",
       " 'title': 'बैडमिंटन',\n",
       " 'url': 'http://hi.wikipedia.org/wiki?curid=25426',\n",
       " 'wikibase_item': 'Q7291'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data['25426']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "af = open(\"updated_parsed_data.pkl\", \"wb\")\n",
    "pickle.dump(main_data, af)\n",
    "af.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "145805\n"
     ]
    }
   ],
   "source": [
    "af = open(\"updated_parsed_data.pkl\", \"rb\")\n",
    "data = pickle.load(af)\n",
    "af.close()\n",
    "print(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}