Assignment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "syCjHgxXsEku",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 108
    },
    "colab_type": "code",
    "id": "egQUO6ITGOhw",
    "outputId": "3cfa1e32-23d3-4fa8-f2d5-ee7b3d1c5703",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/shashank/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/shashank/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download(\"stopwords\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 285
    },
    "colab_type": "code",
    "id": "Aw61jL6wqmWg",
    "outputId": "04bdd3cf-e1ff-4272-b9db-b2c3dfd565b8",
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>CONAME</th>\n",
       "      <th>FYRMO</th>\n",
       "      <th>FDATE</th>\n",
       "      <th>FORM</th>\n",
       "      <th>SECFNAME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199803</td>\n",
       "      <td>1998-03-06</td>\n",
       "      <td>10-K405</td>\n",
       "      <td>edgar/data/3662/0000950170-98-000413.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199805</td>\n",
       "      <td>1998-05-15</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/3662/0000950170-98-001001.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199808</td>\n",
       "      <td>1998-08-13</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-000783.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-12</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/3662/0000950170-98-002145.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-001203.txt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    CIK            CONAME   FYRMO      FDATE     FORM  \\\n",
       "0  3662  SUNBEAM CORP/FL/  199803 1998-03-06  10-K405   \n",
       "1  3662  SUNBEAM CORP/FL/  199805 1998-05-15     10-Q   \n",
       "2  3662  SUNBEAM CORP/FL/  199808 1998-08-13  NT 10-Q   \n",
       "3  3662  SUNBEAM CORP/FL/  199811 1998-11-12   10-K/A   \n",
       "4  3662  SUNBEAM CORP/FL/  199811 1998-11-16  NT 10-Q   \n",
       "\n",
       "                                   SECFNAME  \n",
       "0  edgar/data/3662/0000950170-98-000413.txt  \n",
       "1  edgar/data/3662/0000950170-98-001001.txt  \n",
       "2  edgar/data/3662/0000950172-98-000783.txt  \n",
       "3  edgar/data/3662/0000950170-98-002145.txt  \n",
       "4  edgar/data/3662/0000950172-98-001203.txt  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel('cik_list.xlsx')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 55
    },
    "colab_type": "code",
    "id": "sMSsORmgrYXo",
    "outputId": "2c819feb-c4bb-436e-a1cc-c3974dba423c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y = 'https://www.sec.gov/Archives/'\n",
    "links = [y+x for x in df['SECFNAME']]\n",
    "#print(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "Gelsgid4sOzf",
    "outputId": "8eb9615b-9bce-45f1-bfdc-1f3750b8ce20"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 152 reports saved\n"
     ]
    }
   ],
   "source": [
    "reports = []\n",
    "for url in links:\n",
    "    r = requests.get(url)\n",
    "    data = r.text\n",
    "    soup = BeautifulSoup(data, \"html.parser\")\n",
    "    reports.append(soup.get_text())\n",
    "\n",
    "print(f'Total {len(reports)} reports saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iHSvuPPuqZKh"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of Stop Words are 121\n"
     ]
    }
   ],
   "source": [
    "with open('StopWords_Generic.txt','r') as f:\n",
    "    stop_words = f.read()\n",
    "\n",
    "stop_words = stop_words.split('\\n')\n",
    "print(f'Total number of Stop Words are {len(stop_words)}')\n",
    "\n",
    "#print(stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uXMQRv4DKBas",
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Sequence Number</th>\n",
       "      <th>Word Count</th>\n",
       "      <th>Word Proportion</th>\n",
       "      <th>Average Proportion</th>\n",
       "      <th>Std Dev</th>\n",
       "      <th>Doc Count</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Uncertainty</th>\n",
       "      <th>Litigious</th>\n",
       "      <th>Constraining</th>\n",
       "      <th>Superfluous</th>\n",
       "      <th>Interesting</th>\n",
       "      <th>Modal</th>\n",
       "      <th>Irr_Verb</th>\n",
       "      <th>Harvard_IV</th>\n",
       "      <th>Syllables</th>\n",
       "      <th>Source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AARDVARK</td>\n",
       "      <td>1</td>\n",
       "      <td>277</td>\n",
       "      <td>1.480368e-08</td>\n",
       "      <td>1.239377e-08</td>\n",
       "      <td>3.564730e-06</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AARDVARKS</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1.603287e-10</td>\n",
       "      <td>9.725110e-12</td>\n",
       "      <td>9.863549e-09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABACI</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4.275431e-10</td>\n",
       "      <td>1.386497e-10</td>\n",
       "      <td>6.225591e-08</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABACK</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>6.413147e-10</td>\n",
       "      <td>3.159061e-10</td>\n",
       "      <td>9.383557e-08</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABACUS</td>\n",
       "      <td>5</td>\n",
       "      <td>7250</td>\n",
       "      <td>3.874610e-07</td>\n",
       "      <td>3.681624e-07</td>\n",
       "      <td>3.366553e-05</td>\n",
       "      <td>914</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Word  Sequence Number  Word Count  Word Proportion  \\\n",
       "0   AARDVARK                1         277     1.480368e-08   \n",
       "1  AARDVARKS                2           3     1.603287e-10   \n",
       "2      ABACI                3           8     4.275431e-10   \n",
       "3      ABACK                4          12     6.413147e-10   \n",
       "4     ABACUS                5        7250     3.874610e-07   \n",
       "\n",
       "   Average Proportion       Std Dev  Doc Count  Negative  Positive  \\\n",
       "0        1.239377e-08  3.564730e-06         84         0         0   \n",
       "1        9.725110e-12  9.863549e-09          1         0         0   \n",
       "2        1.386497e-10  6.225591e-08          7         0         0   \n",
       "3        3.159061e-10  9.383557e-08         12         0         0   \n",
       "4        3.681624e-07  3.366553e-05        914         0         0   \n",
       "\n",
       "   Uncertainty  Litigious  Constraining  Superfluous  Interesting  Modal  \\\n",
       "0            0          0             0            0            0      0   \n",
       "1            0          0             0            0            0      0   \n",
       "2            0          0             0            0            0      0   \n",
       "3            0          0             0            0            0      0   \n",
       "4            0          0             0            0            0      0   \n",
       "\n",
       "   Irr_Verb  Harvard_IV  Syllables     Source  \n",
       "0         0           0          2  12of12inf  \n",
       "1         0           0          2  12of12inf  \n",
       "2         0           0          3  12of12inf  \n",
       "3         0           0          2  12of12inf  \n",
       "4         0           0          3  12of12inf  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_dic = pd.read_excel('LoughranMcDonald_MasterDictionary_2018.xlsx')\n",
    "master_dic.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total positve words in dictionary are 354\n",
      "Total negative words in dictionary are 2355\n"
     ]
    }
   ],
   "source": [
    "positive_dictionary = [x for x in master_dic[master_dic['Positive'] != 0]['Word']]\n",
    "\n",
    "negative_dictionary = [x for x in master_dic[master_dic['Negative'] != 0]['Word']]\n",
    "\n",
    "print(f\"Total positve words in dictionary are {len(positive_dictionary)}\")\n",
    "print(f\"Total negative words in dictionary are {len(negative_dictionary)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncertainity = pd.read_excel('uncertainty_dictionary.xlsx')\n",
    "uncertainity_words = list(uncertainity['Word'])\n",
    "\n",
    "constraining = pd.read_excel('constraining_dictionary.xlsx')\n",
    "constraining_words = list(constraining['Word'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    text = re.sub(r'[^A-Za-z]',' ',text.upper())\n",
    "    tokenized_words = word_tokenize(text)\n",
    "    return tokenized_words\n",
    "\n",
    "def remove_stopwords(words, stop_words):\n",
    "    return [x for x in words if x not in stop_words]\n",
    "    \n",
    "def countfunc(store, words):\n",
    "    score = 0\n",
    "    for x in words:\n",
    "        if(x in store):\n",
    "            score = score+1\n",
    "    return score\n",
    "\n",
    "def sentiment(score):\n",
    "    if(score < -0.5):\n",
    "        return 'Most Negative'\n",
    "    elif(score >= -0.5 and score < 0):\n",
    "        return 'Negative'\n",
    "    elif(score == 0):\n",
    "        return 'Neutral'\n",
    "    elif(score > 0 and score < 0.5):\n",
    "        return 'Positive'\n",
    "    else:\n",
    "        return 'Very Positive'\n",
    "    \n",
    "\n",
    "def polarity(positive_score, negative_score):\n",
    "    return (positive_score - negative_score)/((positive_score + negative_score)+ 0.000001)\n",
    "     \n",
    "\n",
    "def subjectivity(positive_score, negative_score, num_words):\n",
    "    return (positive_score+negative_score)/(num_words+ 0.000001)\n",
    "\n",
    "def syllable_morethan2(word):\n",
    "    if(len(word) > 2 and (word[-2:] == 'es' or word[-2:] == 'ed')):\n",
    "        return False\n",
    "    \n",
    "    count =0\n",
    "    vowels = ['a','e','i','o','u']\n",
    "    for i in word:\n",
    "        if(i.lower() in vowels):\n",
    "            count = count +1\n",
    "        \n",
    "    if(count > 2):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "    \n",
    "def fog_index_cal(average_sentence_length, percentage_complexwords):\n",
    "    return 0.4*(average_sentence_length + percentage_complexwords)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sections = [\"Management's Discussion and Analysis\",\n",
    "            \"Quantitative and Qualitative Disclosures about Market Risk\\n\",\n",
    "            \"Risk Factors\\n\"]\n",
    "caps = [x.upper() for x in sections]\n",
    "\n",
    "caps.extend(sections)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = ['mda','qqdmr','rf']\n",
    "var = ['positive_score',\n",
    "      'negative_score',\n",
    "      'polarity_score',\n",
    "      'average_sentence_length',\n",
    "      'percentage_of_complex_words',\n",
    "      'fog_index',\n",
    "      'complex_word_count',\n",
    "      'word_count',\n",
    "      'uncertainity_score',\n",
    "      'constraining_score',\n",
    "      'positive_word_proportion',\n",
    "      'negative_word_proportion',\n",
    "      'uncertainity_word_proportion',\n",
    "      'constraining_word_proportion',\n",
    "      'constraining_words_whole_report']\n",
    "\n",
    "\n",
    "for c in col:\n",
    "    for v in var[:-1]:\n",
    "        df[c+'_'+v] = 0.0\n",
    "\n",
    "df[var[-1]] = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>CONAME</th>\n",
       "      <th>FYRMO</th>\n",
       "      <th>FDATE</th>\n",
       "      <th>FORM</th>\n",
       "      <th>SECFNAME</th>\n",
       "      <th>mda_positive_score</th>\n",
       "      <th>mda_negative_score</th>\n",
       "      <th>mda_polarity_score</th>\n",
       "      <th>mda_average_sentence_length</th>\n",
       "      <th>...</th>\n",
       "      <th>rf_fog_index</th>\n",
       "      <th>rf_complex_word_count</th>\n",
       "      <th>rf_word_count</th>\n",
       "      <th>rf_uncertainity_score</th>\n",
       "      <th>rf_constraining_score</th>\n",
       "      <th>rf_positive_word_proportion</th>\n",
       "      <th>rf_negative_word_proportion</th>\n",
       "      <th>rf_uncertainity_word_proportion</th>\n",
       "      <th>rf_constraining_word_proportion</th>\n",
       "      <th>constraining_words_whole_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199803</td>\n",
       "      <td>1998-03-06</td>\n",
       "      <td>10-K405</td>\n",
       "      <td>edgar/data/3662/0000950170-98-000413.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199805</td>\n",
       "      <td>1998-05-15</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/3662/0000950170-98-001001.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199808</td>\n",
       "      <td>1998-08-13</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-000783.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-12</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/3662/0000950170-98-002145.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-001203.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    CIK            CONAME   FYRMO      FDATE     FORM  \\\n",
       "0  3662  SUNBEAM CORP/FL/  199803 1998-03-06  10-K405   \n",
       "1  3662  SUNBEAM CORP/FL/  199805 1998-05-15     10-Q   \n",
       "2  3662  SUNBEAM CORP/FL/  199808 1998-08-13  NT 10-Q   \n",
       "3  3662  SUNBEAM CORP/FL/  199811 1998-11-12   10-K/A   \n",
       "4  3662  SUNBEAM CORP/FL/  199811 1998-11-16  NT 10-Q   \n",
       "\n",
       "                                   SECFNAME  mda_positive_score  \\\n",
       "0  edgar/data/3662/0000950170-98-000413.txt                 0.0   \n",
       "1  edgar/data/3662/0000950170-98-001001.txt                 0.0   \n",
       "2  edgar/data/3662/0000950172-98-000783.txt                 0.0   \n",
       "3  edgar/data/3662/0000950170-98-002145.txt                 0.0   \n",
       "4  edgar/data/3662/0000950172-98-001203.txt                 0.0   \n",
       "\n",
       "   mda_negative_score  mda_polarity_score  mda_average_sentence_length  ...  \\\n",
       "0                 0.0                 0.0                          0.0  ...   \n",
       "1                 0.0                 0.0                          0.0  ...   \n",
       "2                 0.0                 0.0                          0.0  ...   \n",
       "3                 0.0                 0.0                          0.0  ...   \n",
       "4                 0.0                 0.0                          0.0  ...   \n",
       "\n",
       "   rf_fog_index  rf_complex_word_count  rf_word_count  rf_uncertainity_score  \\\n",
       "0           0.0                    0.0            0.0                    0.0   \n",
       "1           0.0                    0.0            0.0                    0.0   \n",
       "2           0.0                    0.0            0.0                    0.0   \n",
       "3           0.0                    0.0            0.0                    0.0   \n",
       "4           0.0                    0.0            0.0                    0.0   \n",
       "\n",
       "   rf_constraining_score  rf_positive_word_proportion  \\\n",
       "0                    0.0                          0.0   \n",
       "1                    0.0                          0.0   \n",
       "2                    0.0                          0.0   \n",
       "3                    0.0                          0.0   \n",
       "4                    0.0                          0.0   \n",
       "\n",
       "   rf_negative_word_proportion  rf_uncertainity_word_proportion  \\\n",
       "0                          0.0                              0.0   \n",
       "1                          0.0                              0.0   \n",
       "2                          0.0                              0.0   \n",
       "3                          0.0                              0.0   \n",
       "4                          0.0                              0.0   \n",
       "\n",
       "   rf_constraining_word_proportion  constraining_words_whole_report  \n",
       "0                              0.0                              0.0  \n",
       "1                              0.0                              0.0  \n",
       "2                              0.0                              0.0  \n",
       "3                              0.0                              0.0  \n",
       "4                              0.0                              0.0  \n",
       "\n",
       "[5 rows x 49 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "section_map = {i:j for i,j in zip(sections, col)}\n",
    "s_map = {i.upper():j for i,j in zip(sections, col)}\n",
    "\n",
    "section_map.update(s_map)\n",
    "#print(section_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(reports)):\n",
    "    text = re.sub('Item','ITEM',reports[i])\n",
    "    for j in caps:\n",
    "        x = re.search('ITEM\\s+[\\d]\\(*[A-Za-z]*\\)*.*\\s+\\-*\\s*'+j, text)\n",
    "        \n",
    "        if x:\n",
    "            start,end = x.span()\n",
    "            content = (text[start:]).split('ITEM')[1]\n",
    "            if ('...' not in content) and ('. . .' not in content) and len(content) > 200:\n",
    "                tokenized_words = tokenize(content)\n",
    "                #print(f'Total tokenized words are {len(tokenized_words)}')\n",
    "                words = remove_stopwords(tokenized_words, stop_words)\n",
    "                num_words = len(words)\n",
    "                #print(f'Total words after removing stop words are {len(words)}')\n",
    "                positive_score = countfunc(positive_dictionary, words)\n",
    "                negative_score = countfunc(negative_dictionary, words)\n",
    "                #print(f'Total positive score is {positive_score}')\n",
    "                #print(f'Total negative score is {negative_score}')\n",
    "                polarity_score = polarity(positive_score, negative_score)\n",
    "                #print(polarity_score)\n",
    "                subjectivity_score = subjectivity(positive_score, negative_score, num_words)\n",
    "                #print(subjectivity_score)\n",
    "                #print(sentiment(polarity_score))\n",
    "                \n",
    "                sentences = sent_tokenize(content)\n",
    "                num_sentences = len(sentences)\n",
    "                average_sentence_length = num_words/num_sentences\n",
    "                #print(average_sentence_length)\n",
    "                \n",
    "                \n",
    "                num_complexword =0\n",
    "                uncertainity_score = 0\n",
    "                constraining_score = 0\n",
    "                \n",
    "                for word in words:\n",
    "                    if(syllable_morethan2(word)):\n",
    "                        num_complexword = num_complexword+1\n",
    "                        \n",
    "                    if(word in uncertainity_words):\n",
    "                        uncertainity_score = uncertainity_score+1\n",
    "                        \n",
    "                    if(word in constraining_words):\n",
    "                        constraining_score = constraining_score+1\n",
    "                        \n",
    "                #print(num_complexword)\n",
    "                #print(uncertainity_score)\n",
    "                #print(constraining_score)\n",
    "                \n",
    "                \n",
    "                percentage_complexwords = num_complexword/num_words\n",
    "                #print(percentage_complexwords)\n",
    "                fog_index = fog_index_cal(average_sentence_length, percentage_complexwords)\n",
    "                #print(fog_index)\n",
    "                \n",
    "                positive_word_proportion = positive_score/num_words\n",
    "                negative_word_proportion = negative_score/num_words\n",
    "                uncertainity_word_proportion = uncertainity_score/num_words\n",
    "                constraining_word_proportion = constraining_score/num_words\n",
    "                \n",
    "                #print(positive_word_proportion)\n",
    "                #print(negative_word_proportion)\n",
    "                #print(uncertainity_word_proportion)\n",
    "                #print(constraining_word_proportion)\n",
    "                \n",
    "                \n",
    "                df.at[i,section_map[j]+'_positive_score'] = positive_score\n",
    "                df.at[i,section_map[j]+'_negative_score'] = negative_score\n",
    "                df.at[i,section_map[j]+'_polarity_score'] = polarity_score\n",
    "                df.at[i,section_map[j]+'_average_sentence_length'] = average_sentence_length\n",
    "                df.at[i,section_map[j]+'_percentage_of_complex_words'] = percentage_complexwords\n",
    "                df.at[i,section_map[j]+'_fog_index'] = fog_index\n",
    "                df.at[i,section_map[j]+'_complex_word_count'] = num_complexword\n",
    "                df.at[i,section_map[j]+'_word_count'] = num_words\n",
    "                df.at[i,section_map[j]+'_uncertainity_score'] = uncertainity_score\n",
    "                df.at[i,section_map[j]+'_constraining_score'] = constraining_score\n",
    "                df.at[i,section_map[j]+'_positive_word_proportion'] = positive_word_proportion\n",
    "                df.at[i,section_map[j]+'_negative_word_proportion'] = negative_word_proportion\n",
    "                df.at[i,section_map[j]+'_uncertainity_word_proportion'] = uncertainity_word_proportion\n",
    "                df.at[i,section_map[j]+'_constraining_word_proportion'] = constraining_word_proportion\n",
    "                \n",
    "                \n",
    "                \n",
    "                \n",
    "                \n",
    "    constraining_words_whole_report = 0\n",
    "    tokenized_report_words = tokenize(reports[i])\n",
    "    report_words = remove_stopwords(tokenized_report_words, stop_words)\n",
    "    for word in report_words:\n",
    "        if word in constraining_words:\n",
    "            constraining_words_whole_report = 1+ constraining_words_whole_report\n",
    "    #print(constraining_words_whole_report)\n",
    "    df.at[i,'constraining_words_whole_report'] = constraining_words_whole_report\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>CONAME</th>\n",
       "      <th>FYRMO</th>\n",
       "      <th>FDATE</th>\n",
       "      <th>FORM</th>\n",
       "      <th>SECFNAME</th>\n",
       "      <th>mda_positive_score</th>\n",
       "      <th>mda_negative_score</th>\n",
       "      <th>mda_polarity_score</th>\n",
       "      <th>mda_average_sentence_length</th>\n",
       "      <th>...</th>\n",
       "      <th>rf_fog_index</th>\n",
       "      <th>rf_complex_word_count</th>\n",
       "      <th>rf_word_count</th>\n",
       "      <th>rf_uncertainity_score</th>\n",
       "      <th>rf_constraining_score</th>\n",
       "      <th>rf_positive_word_proportion</th>\n",
       "      <th>rf_negative_word_proportion</th>\n",
       "      <th>rf_uncertainity_word_proportion</th>\n",
       "      <th>rf_constraining_word_proportion</th>\n",
       "      <th>constraining_words_whole_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199803</td>\n",
       "      <td>1998-03-06</td>\n",
       "      <td>10-K405</td>\n",
       "      <td>edgar/data/3662/0000950170-98-000413.txt</td>\n",
       "      <td>34.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>18.305263</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1487.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199805</td>\n",
       "      <td>1998-05-15</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/3662/0000950170-98-001001.txt</td>\n",
       "      <td>11.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>-0.671642</td>\n",
       "      <td>23.753425</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1046.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199808</td>\n",
       "      <td>1998-08-13</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-000783.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-12</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/3662/0000950170-98-002145.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-001203.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    CIK            CONAME   FYRMO      FDATE     FORM  \\\n",
       "0  3662  SUNBEAM CORP/FL/  199803 1998-03-06  10-K405   \n",
       "1  3662  SUNBEAM CORP/FL/  199805 1998-05-15     10-Q   \n",
       "2  3662  SUNBEAM CORP/FL/  199808 1998-08-13  NT 10-Q   \n",
       "3  3662  SUNBEAM CORP/FL/  199811 1998-11-12   10-K/A   \n",
       "4  3662  SUNBEAM CORP/FL/  199811 1998-11-16  NT 10-Q   \n",
       "\n",
       "                                   SECFNAME  mda_positive_score  \\\n",
       "0  edgar/data/3662/0000950170-98-000413.txt                34.0   \n",
       "1  edgar/data/3662/0000950170-98-001001.txt                11.0   \n",
       "2  edgar/data/3662/0000950172-98-000783.txt                 0.0   \n",
       "3  edgar/data/3662/0000950170-98-002145.txt                 0.0   \n",
       "4  edgar/data/3662/0000950172-98-001203.txt                 0.0   \n",
       "\n",
       "   mda_negative_score  mda_polarity_score  mda_average_sentence_length  ...  \\\n",
       "0                14.0            0.416667                    18.305263  ...   \n",
       "1                56.0           -0.671642                    23.753425  ...   \n",
       "2                 0.0            0.000000                     0.000000  ...   \n",
       "3                 0.0            0.000000                     0.000000  ...   \n",
       "4                 0.0            0.000000                     0.000000  ...   \n",
       "\n",
       "   rf_fog_index  rf_complex_word_count  rf_word_count  rf_uncertainity_score  \\\n",
       "0           0.0                    0.0            0.0                    0.0   \n",
       "1           0.0                    0.0            0.0                    0.0   \n",
       "2           0.0                    0.0            0.0                    0.0   \n",
       "3           0.0                    0.0            0.0                    0.0   \n",
       "4           0.0                    0.0            0.0                    0.0   \n",
       "\n",
       "   rf_constraining_score  rf_positive_word_proportion  \\\n",
       "0                    0.0                          0.0   \n",
       "1                    0.0                          0.0   \n",
       "2                    0.0                          0.0   \n",
       "3                    0.0                          0.0   \n",
       "4                    0.0                          0.0   \n",
       "\n",
       "   rf_negative_word_proportion  rf_uncertainity_word_proportion  \\\n",
       "0                          0.0                              0.0   \n",
       "1                          0.0                              0.0   \n",
       "2                          0.0                              0.0   \n",
       "3                          0.0                              0.0   \n",
       "4                          0.0                              0.0   \n",
       "\n",
       "   rf_constraining_word_proportion  constraining_words_whole_report  \n",
       "0                              0.0                           1487.0  \n",
       "1                              0.0                           1046.0  \n",
       "2                              0.0                              5.0  \n",
       "3                              0.0                            716.0  \n",
       "4                              0.0                              4.0  \n",
       "\n",
       "[5 rows x 49 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel('output.xlsx')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Assignment.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}