Summarizing_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk import pos_tag, ne_chunk\n",
    "from nltk.chunk import conlltags2tree, tree2conlltags\n",
    "from nltk.tag.stanford import StanfordNERTagger\n",
    "import nltk\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from  textacy.vsm import Vectorizer\n",
    "\n",
    "from tqdm import *\n",
    "from pprint import pprint\n",
    "\n",
    "from pymprog import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting all the tweets from the twitter api and then saving it here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_csv('./tweet.csv',encoding='ISO-8859-1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5120, 6)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.dropna(inplace=True)\n",
    "tweets = tweets[tweets.retweets != 0]\n",
    "tweets.to_csv('./preprocessed_tweets.csv')\n",
    "tweets.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting text from the tweets dataframe\n",
    "\n",
    "Removing URLs, Removing @..., and the hashtags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building the corpus\n",
    "tweet_text = []\n",
    "tweets.text = tweets.text.apply(lambda x: re.sub(u'https:\\S+', u'', x))\n",
    "tweets.text = tweets.text.apply(lambda x: re.sub(u'http:\\S+', u'', x))\n",
    "tweets.text = tweets.text.apply(lambda x: re.sub(u'@\\w+', u'', x))\n",
    "tweets.text = tweets.text.apply(lambda x: re.sub(u'#', u'', x))\n",
    "tweets.text = tweets.text.apply(lambda x: re.sub(u'â|¦|:|;|-|\\x80|Â|µ|à|&|\\/', u'', x))\n",
    "tweets.text = tweets.text.apply(lambda x: x.replace(u'RT', u''))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizing with nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['While',\n",
       " 'good',\n",
       " 'for',\n",
       " 'nothing',\n",
       " 'spits',\n",
       " 'her',\n",
       " 'venom',\n",
       " 'on',\n",
       " 'them',\n",
       " ',',\n",
       " 'these',\n",
       " 'heros',\n",
       " 'are',\n",
       " 'elsewhere',\n",
       " 'doing',\n",
       " 'the',\n",
       " 'job',\n",
       " 'asusual',\n",
       " '.',\n",
       " 'To',\n",
       " 'the',\n",
       " 'Army',\n",
       " '.',\n",
       " 'KeralaFloods',\n",
       " 'IndianArmy',\n",
       " 'Keralarains',\n",
       " 'twitter',\n",
       " '.',\n",
       " 'comfirstpoststat',\n",
       " 'us1027831034149261312']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tknzr = TweetTokenizer()\n",
    "\n",
    "nltk_tweets = []\n",
    "for text in tweets.text:\n",
    "    nltk_tweets.append(tknzr.tokenize(text))\n",
    "nltk_tweets[-68]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nltk.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using POS tagger to get the array of various part of speech in the tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('While', 'IN'),\n",
      " ('good', 'JJ'),\n",
      " ('for', 'IN'),\n",
      " ('nothing', 'NN'),\n",
      " ('spits', 'VBZ'),\n",
      " ('her', 'PRP$'),\n",
      " ('venom', 'NN'),\n",
      " ('on', 'IN'),\n",
      " ('them', 'PRP'),\n",
      " (',', ','),\n",
      " ('these', 'DT'),\n",
      " ('heros', 'NNS'),\n",
      " ('are', 'VBP'),\n",
      " ('elsewhere', 'RB'),\n",
      " ('doing', 'VBG'),\n",
      " ('the', 'DT'),\n",
      " ('job', 'NN'),\n",
      " ('asusual', 'JJ'),\n",
      " ('.', '.'),\n",
      " ('To', 'TO'),\n",
      " ('the', 'DT'),\n",
      " ('Army', 'NNP'),\n",
      " ('.', '.'),\n",
      " ('KeralaFloods', 'NNP'),\n",
      " ('IndianArmy', 'NNP'),\n",
      " ('Keralarains', 'NNP'),\n",
      " ('twitter', 'NN'),\n",
      " ('.', '.'),\n",
      " ('comfirstpoststat', 'NN'),\n",
      " ('us1027831034149261312', 'NN')]\n"
     ]
    }
   ],
   "source": [
    "nltk_pos = []\n",
    "\n",
    "for text in nltk_tweets:\n",
    "    nltk_pos.append(pos_tag(text))\n",
    "pprint(nltk_pos[-68])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tried Named entity recognition using NLTK but not accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pattern = 'NP: {<DT>?<JJ>*<NN>}'\n",
    "#cp = nltk.RegexpParser(pattern)\n",
    "#cs = cp.parse(nltk_pos[-68])\n",
    "#print(cs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#iob_tagged= tree2conlltags(cs)\n",
    "#pprint(iob_tagged)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now using Stanford Natural Processing!!\n",
    "First, we will set the config_java file for nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.internals.config_java(\"/usr/lib/jvm/java-11-openjdk-amd64/bin/java\")\n",
    "st = StanfordNERTagger('/home/pranav/Desktop/zine/Twitter-Mining/stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz',\n",
    "           '/home/pranav/Desktop/zine/Twitter-Mining/stanford-ner-2018-10-16/stanford-ner.jar', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5120/5120 [3:11:15<00:00,  1.85s/it]     \n"
     ]
    }
   ],
   "source": [
    "nltk_ents = []\n",
    "for tweet in tqdm(nltk_tweets):\n",
    "    entity_tagged_tweet = st.tag(tweet)\n",
    "    nltk_ents.append([tag for tag in entity_tagged_tweet if tag[1] != 'O'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Standford Named Entity Recognition library labels the text in the tweets, particularly into 3 classes (PERSON, ORGANIZATION, LOCATION).<br>\n",
    "As, numerals will also be significant in the tweets we will concatenate it to the entity text. Hence, from the text we will take care about the entities and numbers.<br>\n",
    "I will name these array content_tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, entities that are labelled as PERSON tend to be related more to feelings of the person, hence I will remove them as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "content_tweets = []\n",
    "for pos_tweet, tweet_entity in zip(nltk_pos, nltk_ents):\n",
    "    # starting by appending all of the entities\n",
    "    tweet_content = [word[0] for word in tweet_entity if word[1] != 'PERSON']\n",
    "    \n",
    "    # next by appending all of the numerals\n",
    "    for token in pos_tweet:\n",
    "         if token[1] == u'CD':\n",
    "            tweet_content.append(token[0])\n",
    "    content_tweets.append(tweet_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the tf-idf score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we will take out tl-idf score for the tweet that will determine how much the word present in the tweet is importants.<br>\n",
    "So, I will take out the tl-idf score of all of the nltk_tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I care about the tf-idf scores of the entire tweet, so will tf-idf score across the entire corpus of original tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5120, 18883)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = Vectorizer(tf_type='linear', apply_idf=True, idf_type='smooth')\n",
    "term_matrix = vectorizer.fit_transform(nltk_tweets)\n",
    "term_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The term_matrix is a document-term matrix of shape (#document, #unique terms).<br>\n",
    "Each row is a document and each column is a  unique word.<br>\n",
    "The values of the matrix is the tf-idf score of the particular unique word in the column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.847957830902605"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_matrix = term_matrix.todense()\n",
    "np_matrix.shape\n",
    "\n",
    "np.max(np_matrix[:,527])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The goal is to make a dictionary, which map from tokens in the content_tweets to some tf-idf score.<br>\n",
    "Each word has a unique tf-idf value<br>\n",
    "In simple words, we will find the column number of each word from the content_tweets in the document-term matrix, as now the vectorizer model is trained, for this we will use vectorizer.vocablury_terms[word]<br>\n",
    "Each column is a distinct word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10300995006 527\n",
      "10301000518 528\n",
      "10301010413 529\n",
      "10301015371 530\n",
      "10301053520 531\n",
      "10301069730 532\n",
      "10301074070 533\n",
      "10301081111 534\n",
      "10301083441 535\n",
      "10301097646 536\n",
      "10301143512 537\n",
      "10301146224 538\n",
      "10301167993 539\n",
      "10301267691 540\n",
      "10301323682 541\n",
      "10301388428 542\n",
      "10301395399 543\n",
      "10301404464 544\n",
      "10301411160 545\n",
      "10301447925 546\n",
      "10301454128 547\n",
      "10301556459 548\n",
      "10301616328 549\n",
      "10301629472 550\n",
      "10301637024 551\n",
      "10301667916 552\n",
      "10301708726 553\n",
      "10301723228 554\n",
      "10301740195 555\n",
      "10301784633 556\n",
      "10301866916 557\n",
      "10301899468 558\n",
      "10301911889 559\n",
      "10302070006 560\n",
      "10302088181 561\n",
      "10302151702 562\n",
      "10302167079 563\n",
      "10302327271 564\n",
      "10302570793 565\n",
      "10302612194 566\n",
      "10302811384 567\n",
      "10302835155 568\n",
      "10302862659 569\n",
      "10302920407 570\n",
      "10302972395 571\n",
      "10303002310 572\n",
      "10303052258 573\n",
      "10303057788 574\n",
      "10303074424 575\n",
      "10303140485 576\n",
      "10303169204 577\n",
      "10303184414 578\n",
      "10303185098 579\n",
      "10303190337 580\n",
      "10303234040 581\n",
      "10303260009 582\n",
      "10303294381 583\n",
      "10303314058 584\n",
      "10303341425 585\n",
      "10303367359 586\n",
      "10303368144 587\n",
      "10303406319 588\n",
      "10303420293 589\n",
      "10303430246 590\n",
      "10303432039 591\n",
      "10303451059 592\n",
      "10303462205 593\n",
      "10303475219 594\n",
      "10303480107 595\n",
      "10303588287 596\n",
      "10303590876 597\n",
      "10303638753 598\n",
      "10303651284 599\n",
      "10303662769 600\n",
      "10303688203 601\n",
      "1030370 8327 602\n",
      "10303713870 603\n",
      "10303733718 604\n",
      "10303771793 605\n",
      "10303834092 606\n",
      "10303855410 607\n",
      "10303858000 608\n",
      "10303858036 609\n",
      "10303872184 610\n",
      "10303886935 611\n",
      "10303901820 612\n",
      "10303903806 613\n",
      "10303923911 614\n",
      "10303977029 615\n",
      "10303985858 616\n",
      "10304091413 617\n",
      "10304116644 618\n",
      "10304137180 619\n",
      "10304153007 620\n",
      "10304220206 621\n",
      "10304228825 622\n",
      "10304305256 623\n",
      "10304352443 624\n",
      "10304359545 625\n",
      "10304372267 626\n",
      "10304420833 627\n",
      "10304449770 628\n",
      "10304454480 629\n",
      "10304502050 630\n",
      "10304505960 631\n",
      "10304510656 632\n",
      "10304525893 633\n",
      "10304538993 634\n",
      "10304556127 635\n",
      "10304559782 636\n",
      "10304561839 637\n",
      "10304574798 638\n",
      "10304575913 639\n",
      "10304596966 640\n",
      "10304613424 641\n",
      "10304683047 642\n",
      "10304683926 643\n",
      "10304699099 644\n",
      "10304723838 645\n",
      "10304792548 646\n",
      "10304814017 647\n",
      "10304822386 648\n",
      "10304856335 649\n",
      "10304878119 650\n",
      "10304932858 651\n",
      "10304945104 652\n",
      "10304955036 653\n",
      "10304988288 654\n",
      "10305026865 655\n",
      "10305028333 656\n",
      "10305038950 657\n",
      "10305053767 658\n",
      "10305119105 659\n",
      "10305121513 660\n",
      "10305137373 661\n",
      "10305139137 662\n",
      "10305150470 663\n",
      "10305165307 664\n",
      "10305181738 665\n",
      "10305196605 666\n",
      "10305206910 667\n",
      "10305215342 668\n",
      "10305231410 669\n",
      "10305280556 670\n",
      "10305288516 671\n",
      "10305297456 672\n",
      "10305305828 673\n",
      "10305316285 674\n",
      "10305319142 675\n",
      "10305325071 676\n",
      "104OTi 677\n",
      "1056 678\n",
      "106 679\n",
      "1077 680\n",
      "108 681\n",
      "1093status 682\n",
      "10AM 683\n",
      "10CroresForKerala 684\n",
      "10JPdoormat 685\n",
      "10K 686\n",
      "10Lakh 687\n",
      "10Lakhs 688\n",
      "10Tons 689\n",
      "10cinema 690\n",
      "10cr 691\n",
      "10k 692\n",
      "10k7 693\n",
      "10km 694\n",
      "10lakhcompensation 695\n",
      "10lakhs 696\n",
      "10th 697\n",
      "10updates1899255 698\n",
      "10Ã 699\n",
      "11 700\n",
      "11.15 701\n",
      "11.3 702\n",
      "11.30 703\n",
      "11.35 704\n",
      "1100 705\n",
      "1100am 706\n",
      "11035016 707\n",
      "11114496 708\n",
      "1130pm 709\n",
      "1133pm 710\n",
      "113A1 711\n",
      "114 712\n",
      "11447040 713\n",
      "115 714\n"
     ]
    }
   ],
   "source": [
    "for key in sorted(vectorizer.vocabulary_terms)[527:715]:\n",
    "    print(key, vectorizer.vocabulary_terms[key])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now to get the tf-idf score of all entities related to a specific content_tweet, we will use np.max function <br>\n",
    "We will taken all the words from a specific content tweet and then use the vectorizer.vocabulary_terms to get the column-number of the word<br>\n",
    "Then, we use np.max to take out the maximum value in the entire column for the word and that will be its tf-idf score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for token in content_tweets[1190]:\n",
    "    print(token, vectorizer.vocabulary_terms[token], np.max(np_matrix[:,vectorizer.vocabulary_terms[token]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We will now go through all of the content_tweets to get the dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5283\n"
     ]
    }
   ],
   "source": [
    "tfidf_dict = {}\n",
    "content_vocab = []\n",
    "for tweet in content_tweets: \n",
    "    for token in tweet: \n",
    "        if token not in tfidf_dict: \n",
    "            if token in vectorizer.vocabulary_terms:\n",
    "                content_vocab.append(token)\n",
    "                tfidf_dict[token] = np.max(np_matrix[:,vectorizer.vocabulary_terms[token]])\n",
    "print(len(tfidf_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WORD:18on -- tf-idf SCORE:8.847957830902605\n",
      "WORD:18piSlWC7o -- tf-idf SCORE:8.847957830902605\n",
      "WORD:18qnhvlZkF -- tf-idf SCORE:8.847957830902605\n",
      "WORD:18stat -- tf-idf SCORE:8.154810650342661\n",
      "WORD:18th -- tf-idf SCORE:7.238519918468505\n",
      "WORD:18zdata -- tf-idf SCORE:8.847957830902605\n",
      "WORD:19 -- tf-idf SCORE:11.971513899946274\n",
      "WORD:19.1 -- tf-idf SCORE:8.847957830902605\n",
      "WORD:1900136 -- tf-idf SCORE:8.847957830902605\n",
      "WORD:1900751 -- tf-idf SCORE:8.847957830902605\n",
      "WORD:1902159 -- tf-idf SCORE:8.847957830902605\n"
     ]
    }
   ],
   "source": [
    "for key in sorted(tfidf_dict)[900:911]:\n",
    "    print (\"WORD:\" + str(key) + \" -- tf-idf SCORE:\" +  str(tfidf_dict[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Content Word-based Tweet Summarization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To generate something that woul be more useful to people and other volunteers, It has to be something with content-words with high tf-idf scores.<br>\n",
    "This can be done by using Integer Linear Programming(ILP) where we will maximize an equation given some constraints.<br>\n",
    "Equation: Maximize the total score of content words in my summary<br>\n",
    "\\begin{equation}\n",
    "\\sum_{i=1}^n x_{i} + \\sum_{j = 1}^{m} Score(j) \\cdot y_{j}\n",
    "\\end{equation}\n",
    "where, $x_{i}$ is 1 if the tweet is selected or 0 if the tweet is not selected, where $y_{j}$ is 1 or 0 if each content word is included (and Score(j) is that word's tf-idf score).<br>\n",
    "\\begin{equation}\n",
    "\\sum_{i=1}^n x_{i}.Length(i) <= L\n",
    "\\end{equation}\n",
    "Constraint 1: The total length of all the selected tweets to be less than some value L, which will be the length of my summary, L. I can vary L depending on how long I want my summary to be. \n",
    "<br>\n",
    "\\begin{equation}\n",
    "\\sum_{i \\in T_{j}} x_{i} \\geq y_{j}, j = [1,...,m]\n",
    "\\end{equation}\n",
    "Contraint 2:If I pick some content word $y_{j}$ (out of my $m$ possible content words) , then I want to have at least one tweet from the set of tweets which contain that content word, $T_{j}$.<br>\n",
    "\\begin{equation}\n",
    "\\sum_{j \\in C_{i}} y_{j} \\leq |C_{i}| \\times x_{i}, i = [1,...,n]\n",
    "\\end{equation}\n",
    "Constraint 3: If I pick a tweet in my summary, then all the content words in that tweet should be present in the summary<br>\n",
    "\n",
    "Variables that the equation depend on are integers, 1 if the word is included and 0, if it is not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model('COWABS') is the default model."
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "begin('COWABS')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "var function is used to create variables,  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0 <= x[1000] <= 1 binary"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Defining the first variable x,\n",
    "# This definies whether or not a tweet is selected\n",
    "x = var('x', len(nltk_tweets), bool)\n",
    "x[1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the second variable y,\n",
    "# This defines whether or not a content word is selected\n",
    "y = var('y', len(content_vocab), bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5283, 0 <= y[0] <= 1 binary)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y), y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the equation that needs to be maximized\n",
    "maximize(sum(x) + sum([tfidf_dict[content_vocab[j]]*y[j] for j in range(len(y))]));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Constraint 1: Maximum length of entire tweet summary\n",
    "# should be less than or equal to 150\n",
    "\n",
    "L = 150\n",
    "sum([x[i] * len(nltk_tweets[i]) for i in range(len(x))]) <= L;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constraint 2: If I pick a content word then I have to pick a tweet that contains the content word\n",
    "def tweet_with_content_words(j):\n",
    "    content_word = content_vocab[j]\n",
    "    index_term_matrix = vectorizer.vocabulary_terms[content_word]\n",
    "    matrix = np_matrix[:, index_term_matrix]\n",
    "    \n",
    "    return np.nonzero(matrix)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(len(y)):\n",
    "    sum([x[i] for i in tweet_with_content_words(j)]) >= y[j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constraint 3: If i pick a tweet, then all of the content words from the tweet must be selected\n",
    "def content_words(i):\n",
    "    tweet = nltk_tweets[i]\n",
    "    content_indices = []\n",
    "    \n",
    "    for token in tweet:\n",
    "        if token in content_vocab:\n",
    "            content_indices.append(content_vocab.index(token))\n",
    "    \n",
    "    return content_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(x)):\n",
    "    sum(y[j] for j in content_words(i)) >= len(content_words(i)) * x[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0,\n",
       " 'The MIP problem instance has been successfully solved. (This code\\ndoes {\\\\it not} necessarily mean that the solver has found optimal\\nsolution. It only means that the solution process was successful.)')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_x =  [value.primal for value in x]\n",
    "result_y = [value.primal for value in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model('COWABS') is not the default model."
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 856, 2002, 3221, 3437, 3461, 4668, 4859, 4997]),)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_tweets = np.nonzero(result_x)\n",
    "chosen_words = np.nonzero(result_y)\n",
    "chosen_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 83)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(chosen_tweets[0]), len(chosen_words[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------\n",
      "Please support KeralaFloods KeralaFloodRelief Medicine Food IndiaForKerala Contact and account details Abhayaloka Trust AC NO 4504220016 1601 Syndicate Bank Myngappally Branch IFSC SYNB 0004504 Email abhayalokakerala.com Contact + 9185477991 20 + 9194468670 74 pic.twitter . comQIHKP 639UN\n",
      "--------------\n",
      "* Declare KeralaFloods as nation calamity Rescuekerala RebuildKerala Morefundsforkerala Keralaflood 2018\n",
      "--------------\n",
      "Help KeralaFloods People 5 People ( 2 Old people above 70yrs ) Description Evacuation Number 9496950810 8547054877 Location Thiruvalla Nallad , North Side near Subramanian Swamy Temple District pathanamthitta Time of SoS call 500pm\n",
      "--------------\n",
      "Respect fans keralafloods twitter . comSuriyaFansClub status 10300204513 8743296 0 ? s = 19\n",
      "--------------\n",
      "KeralaFloods Prime Minister Asks Army to Intensify Rescue Operations twitter . comeOrganisersta tus 10300238020 8687513 8\n",
      "--------------\n",
      "Thank U Donated  ¹5L To KeralaFloods Releif Fund twitter . comTheDeverakonda status 10285463585 9088588 9\n",
      "--------------\n",
      "donated 25lakhs to Kerala Government for Flood relief .. KeralaFloods KeralaFloodRelief\n",
      "--------------\n",
      "LIVE FaceOff with | KeralaFloods pscp . tvwbj 2Y3jUzMjE2 NjJ 8MW1uR2VvVmVhV1B4WG2DaDZeY1p1aDT93btfUz4WmXXA4VFjzNSzImiQOuOa\n"
     ]
    }
   ],
   "source": [
    "for i in chosen_tweets[0]:\n",
    "    print ('--------------')\n",
    "    print (\" \".join(nltk_tweets[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}