movie_sentiment_analysis using linear regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "movie_sentiment_analysis.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kmSwfTD1xfu1",
        "colab_type": "text"
      },
      "source": [
        "# NLP - movie review sentiment analysis in python"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f5vKzXTsxfvN",
        "colab_type": "text"
      },
      "source": [
        "## Data Facts and Import \n",
        "First download movie dataset from kaggle and uplod the file."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n-_DF4emCdC9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd \n",
        "\n",
        "Reviewdata = pd.read_csv('/train.tsv',sep='\\t')\n",
        "\n",
        "#Data Credit - https://www.kaggle.com/anu0012/hotel-review/data"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WKJfZX4JxfwI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "13733e23-aa57-4306-eadd-51f726502b45"
      },
      "source": [
        "#Reviewdata.shape\n",
        "#Reviewdata.head()\n",
        "Reviewdata.info()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 156060 entries, 0 to 156059\n",
            "Data columns (total 4 columns):\n",
            " #   Column      Non-Null Count   Dtype \n",
            "---  ------      --------------   ----- \n",
            " 0   PhraseId    156060 non-null  int64 \n",
            " 1   SentenceId  156060 non-null  int64 \n",
            " 2   Phrase      156060 non-null  object\n",
            " 3   Sentiment   156060 non-null  int64 \n",
            "dtypes: int64(3), object(1)\n",
            "memory usage: 4.8+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ddPn3gi-xfwW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "7e07c64f-3852-47dc-b405-a5430af29d62"
      },
      "source": [
        "Reviewdata.describe().transpose()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>25%</th>\n",
              "      <th>50%</th>\n",
              "      <th>75%</th>\n",
              "      <th>max</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>PhraseId</th>\n",
              "      <td>156060.0</td>\n",
              "      <td>78030.500000</td>\n",
              "      <td>45050.785842</td>\n",
              "      <td>1.0</td>\n",
              "      <td>39015.75</td>\n",
              "      <td>78030.5</td>\n",
              "      <td>117045.25</td>\n",
              "      <td>156060.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SentenceId</th>\n",
              "      <td>156060.0</td>\n",
              "      <td>4079.732744</td>\n",
              "      <td>2502.764394</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1861.75</td>\n",
              "      <td>4017.0</td>\n",
              "      <td>6244.00</td>\n",
              "      <td>8544.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Sentiment</th>\n",
              "      <td>156060.0</td>\n",
              "      <td>2.063578</td>\n",
              "      <td>0.893832</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.00</td>\n",
              "      <td>2.0</td>\n",
              "      <td>3.00</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               count          mean           std  ...      50%        75%       max\n",
              "PhraseId    156060.0  78030.500000  45050.785842  ...  78030.5  117045.25  156060.0\n",
              "SentenceId  156060.0   4079.732744   2502.764394  ...   4017.0    6244.00    8544.0\n",
              "Sentiment   156060.0      2.063578      0.893832  ...      2.0       3.00       4.0\n",
              "\n",
              "[3 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "klOogvqXxfwj",
        "colab_type": "text"
      },
      "source": [
        "## Data visualization using matplotlib\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HJSFECKvxfw1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 414
        },
        "outputId": "34f8e605-d8e9-4455-a5ad-02dd642f9c6e"
      },
      "source": [
        "### Checking for the Distribution of Default ###\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "print('Percentage for default\\n')\n",
        "print(round(Reviewdata.Sentiment.value_counts(normalize=True)*100,2))\n",
        "round(Reviewdata.Sentiment.value_counts(normalize=True)*100,2).plot(kind='bar')\n",
        "plt.title('Percentage Distributions by review type')\n",
        "plt.show()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Percentage for default\n",
            "\n",
            "2    50.99\n",
            "3    21.10\n",
            "1    17.48\n",
            "4     5.90\n",
            "0     4.53\n",
            "Name: Sentiment, dtype: float64\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEFCAYAAAD69rxNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAWFklEQVR4nO3de7hddX3n8fc3hHBVLkOaCQQSFUTADqgR9PFSBClBVNAiI6MSFQdtZcRHp4K0zmBbn+IzThk6U2tTLwQvXIplgtDKUDCoo6AJ4AWjFTCQYG5AGEAQDXznj9/vmOXOOTk7yTlnn194v57nPGet9Vt7re9ea+3PXvu31j4nMhNJUnumDLoASdLWMcAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgGvCRMQBEfFoROwwRsv7VER8pA4fHRErx2K5dXmviIifjNXy+lzn4oh410Susx8RcUdEHD3oOrQpA3wLRMTyiHi8htCaiLg4InYfdF1DIuL8iPjCgNb99oh4sm6bRyPiZxHxuYh47tA8mXlvZu6emU/2saxvjrbOzHxPZv75GNWfEXFgZ9nfyMyDx2LZrcvMwzJz8Xgtv9/9rU0Z4FvudZm5O/BCYC7wp1vy4Ci21+3+7bpt9gBeDTwOLI2I54/1isbqLP7pICKmDroGjZPM9KfPH2A58OrO+H8DrqnDLwG+BTwEfA84ujPfYuBjwP+lhNqBwGHA9cCDwBrgvDrvFOBc4C7gAeAKYO/aNgdIYD5wL3A/8Ce1bR7wK+DXwKPA9+r0dwDLgEeAu4F39zynDwGrgJ8D76rLP7C27QR8oq5rDfApYJcRts3bgW8OM/0a4Mqe+qd2HnN3re1nwFuAQ4BfAk/W5/FQnfdi4G+BfwJ+QXmDuBj4i9p+NLASOK9ul+XAW3r2wbuGqxf4eq3rF3Wd/35oeZ35D6nLeAi4A3h9p+1i4G+Aa+tzuQV4Tm0L4EJgLfAw8APg+SNsw8XAXwLfqfMu6uz7a4H/1DP/94E3DLOcoe18Rt13X6/T31mPhfXAdcDsOv1vgU/0LGMR8IHe457NH58LgQ/W4f1qDe+t48+hHOtTetazyf4GXkw53nbozPdGNh7T5wNXApfX7X0rcHhn3n2BLwPrKMfV+wadHeOWSYMuoKWfngN5//pC/vN6sD4AvKYe4MfV8el13sX1hXQYMBV4BiU0PwjsXMePqvOeDdwMzKIE6N8Bl9a2oRfm3wO7AIcDTwCH1PbzgS/01HxiffEE8HvAY8ALa9s8YHWta1fgC/x2gF8IXA3sXWv8CvCXI2ybtzN8gL8TWNNT/1RgN0pIHVzbZgKHjbQsSkj+P+BldRvvzKYBvgH4q7rdfo8SyAd39sGwAV7Hf/O8O8tbWYd3BO6kvDlMA46hBMfBndoeAI6sz+2LwGW17XhgKbBn3QeHADNH2IaLgfuA59ft8+Wh/QmcCtzSmffwus5pwyxnaDtfUpezC3BSfQ6H1Br/FPhWnf+VwAog6vhelBONfYc57jd3fL4T+Eod/g+UkL+807ao32MH+BFwQmf8Kja+OZxPOVE5pe6b/0wJ6h3rsbEU+C91Xz2bcpJw/KDzY1wyadAFtPRTD+Shs4R7gE/WF8c5wOd75r0OmF+HFwN/1mk7DbhthHUsA47tjM+sB+vUzgtzVqf9O8Cb6/D59AT4MMv/38DZdfizdAKZ8skg6++gBOBzOu0vBX42wnI3eRHW6fOAX9fhofqHAvwh4A/oOasf4QV9MXDJMNN6A3y3TvsVwEc6+2BrA/wVlDe6KZ32S4HzO3V8utP2GuDHdfgY4F8pn9Cm9G6fnuezGLigM34o5VPVDpQ3rPXAQbXtE8AnR1jO0HZ+dmfaPwNndManUN7MZ9d9fS/wytr2H4Ebe477oQDf3PH5nFrjFMqntXd3tuFC6hl9P8cO5TX1xTq8d611Zuc4v7nnuayq++ko4N6eZX0Y+NxY58Fk+Nle+2LH08mZuWdmzs7MP8rMxykvgjdFxENDP8DLKQf3kBWd4f0pZyfDmQ1c1VnOMsrHyxmdeVZ3hh8DRryQGhEnRMTNEfFgXd5rgH1q8749dXWHp1POypd2avlqnb4l9qN8dP4tmfkLSlfFe4BVEXFtRDxvlGWtGKV9fV3ukHsoz3Fb7QusyMynepa9X2d82H2SmTcC/4vSxbI2IhZExDM3s67uc7yHcla5T2b+ktJl8NZ6DeU04POj1N1d1mzgos6+fJAS3PtlSbnL6jKhnD1/cYRljnh8ZuZdlDf9Iyhheg3w84g4mPKJ6KZR6u36AvC6iNiN8unjG5m5arjnVvfLSsp+mg3s2/NaPI/ffv1sNwzwsbGCcga+Z+dnt8y8oDNP9sz/7M0s64SeZe2cmff1UUd3HUTETpSP4Z+gvMD2pPQhR51lFeWj8JD9O8P3Uz5GH9apY48sFym3xBuAbwxbbOZ1mXkc5Y3ux5SuoU2eR/cho6xrr/qCH3IApW8fSrDs2mn7t6Msq+vnwP49F58PoHR3jCoz/zozX0Q5o34u8Mebmb27Dw6gnN3eX8cXUq4THAs8lpnfHm3VneEVlOsf3eNql8z8Vm2/FDglImZTzmK/PMIyRzs+b6J0bUyr026iXLPZC7i9jzrLhPLYb1P6vt/Gpm9Wv9lOdb/MouynFZRPid36npGZrxlh3U0zwMfG0NnC8RGxQ0TsXO9LnjXC/NcAMyPi/RGxU0Q8IyKOqm2fAj5WX0hExPSIOKnPOtYAczpBM43ST7kO2BARJwC/35n/CuAdEXFIROwKfGSooZ7V/D1wYUT8Tq1lv4g4frQi6jZ4VkT8T0pXxEeHmWdGRJxUA/cJStfU0BnuGmBWREzr83l3fTQipkXEK4DXAv9Qp98OvDEidq23C57R87g1jPymegvlrPpDEbFjvSf6dZSz1s2KiBdHxFERsSPlTeSXbHyew3lrRBxa98efUS4APwlQA/sp4L8z+tl3r08BH46Iw2pde0TEm4YaM/M2yhvFp4HrMvOhzSxnc8fnTcBZlAvDULqFzqJ0kYx0++hI+/sSykX23wX+saftRRHxxnqHzfspx9DNlC7FRyLinIjYpR6Lz4+IF4+w7qYZ4GMgM1dQLhKdRwnLFZSzrGG3b2Y+QrnQ+TrKR++fAq+qzRdRLhz+n4h4hHJQHjXccoYxFFYPRMStdT3vowT1espH46s7dfwz8NfA1ygXuG6uTU/U3+cMTY+Ih4F/ATZ3b/RLI+JRysXJxcAzgRdn5g+GmXcK8AHKWdODlI/Yf1jbbqRcIF4dEfcP89iRrK7P8+eULoD3ZOaPa9uFlP7kNZQz2d4ugvOBhfVj96ndhsz8FWVfnUAJuU8Cp3eWvTnPpLwRrqd0iTxAuXtpJJ+n9KmvpvR7v6+n/RJKoG3R/f6ZeRXwceCyui9/SHk+XV+i3N3zpc0sarTj8ybKBe+hAP8m5ZPP1xnZSPv7KmqXTWY+1vOYRZQuuPWUM/Q3Zuav65vEayndOD9j45vSHptZf7OGrjpLRMQhlBf2Tpm5YdD1aFMRcTpwZma+fNC1TISIuIvS9fMvnWnnUy44v3VghU0SnoE/zUXEG2o3zl6UM7SvGN6TU+1W+SNgwaBrmQgR8QeU/vEbB13LZGWA692UL5ncRbmb4A83P7sGoV57WEfpAtpcF8d2ISIWU75g9N6eu3/UYReKJDXKM3BJapQBLkmNmtC/UrbPPvvknDlzJnKVktS8pUuX3p+Zm3wLekIDfM6cOSxZsmQiVylJzYuIe4abbheKJDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVET+kWesTDn3GsHXQLLLzhx0CVIUn8BHhHLgUcof250Q2bOjYi9Kf9kdQ7lv1afmpnrx6dMSVKvLelCeVVmHpGZc+v4ucANmXkQcEMdlyRNkG3pAz+J8r8Fqb9P3vZyJEn96jfAk/JPTJdGxJl12ozMXFWHVwMzhntgRJwZEUsiYsm6deu2sVxJ0pB+L2K+PDPvi4jfAa6PiN/6b9yZmREx7L/2ycwF1P/hN3fuXP/9jySNkb7OwDPzvvp7LXAVcCSwJiJmAtTfa8erSEnSpkYN8IjYLSKeMTQM/D7wQ+BqYH6dbT6waLyKlCRtqp8ulBnAVRExNP+XMvOrEfFd4IqIOAO4Bzh1/MqUJPUaNcAz827g8GGmPwAcOx5FSZJG51fpJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqVN8BHhE7RMRtEXFNHX9WRNwSEXdGxOURMW38ypQk9dqSM/CzgWWd8Y8DF2bmgcB64IyxLEyStHl9BXhEzAJOBD5dxwM4BriyzrIQOHk8CpQkDa/fM/D/AXwIeKqO/xvgoczcUMdXAvuNcW2SpM0YNcAj4rXA2sxcujUriIgzI2JJRCxZt27d1ixCkjSMfs7AXwa8PiKWA5dRuk4uAvaMiKl1nlnAfcM9ODMXZObczJw7ffr0MShZkgR9BHhmfjgzZ2XmHODNwI2Z+Rbga8Apdbb5wKJxq1KStIltuQ/8HOADEXEnpU/8M2NTkiSpH1NHn2WjzFwMLK7DdwNHjn1JkqR++E1MSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqNGDfCI2DkivhMR34uIOyLio3X6syLiloi4MyIuj4hp41+uJGlIP2fgTwDHZObhwBHAvIh4CfBx4MLMPBBYD5wxfmVKknqNGuBZPFpHd6w/CRwDXFmnLwROHpcKJUnD6qsPPCJ2iIjbgbXA9cBdwEOZuaHOshLYb3xKlCQNp68Az8wnM/MIYBZwJPC8flcQEWdGxJKIWLJu3bqtLFOS1GuL7kLJzIeArwEvBfaMiKm1aRZw3wiPWZCZczNz7vTp07epWEnSRv3chTI9Ivasw7sAxwHLKEF+Sp1tPrBovIqUJG1q6uizMBNYGBE7UAL/isy8JiJ+BFwWEX8B3AZ8ZhzrlCT1GDXAM/P7wAuGmX43pT9ckjQAfhNTkhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGTR1thojYH7gEmAEksCAzL4qIvYHLgTnAcuDUzFw/fqWq15xzrx10CSy/4MRBlyA9bfVzBr4B+GBmHgq8BHhvRBwKnAvckJkHATfUcUnSBBk1wDNzVWbeWocfAZYB+wEnAQvrbAuBk8erSEnSpraoDzwi5gAvAG4BZmTmqtq0mtLFIkmaIH0HeETsDnwZeH9mPtxty8yk9I8P97gzI2JJRCxZt27dNhUrSdqorwCPiB0p4f3FzPzHOnlNRMys7TOBtcM9NjMXZObczJw7ffr0sahZkkQfAR4RAXwGWJaZf9VpuhqYX4fnA4vGvjxJ0khGvY0QeBnwNuAHEXF7nXYecAFwRUScAdwDnDo+JUqShjNqgGfmN4EYofnYsS1HktQvv4kpSY0ywCWpUQa4JDXKAJekRhngktSofm4jlCa1yfBXGcG/zKiJ5xm4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRo0a4BHx2YhYGxE/7EzbOyKuj4if1t97jW+ZkqRe/ZyBXwzM65l2LnBDZh4E3FDHJUkTaNQAz8yvAw/2TD4JWFiHFwInj3FdkqRRbG0f+IzMXFWHVwMzxqgeSVKftvkiZmYmkCO1R8SZEbEkIpasW7duW1cnSaq2NsDXRMRMgPp77UgzZuaCzJybmXOnT5++lauTJPXa2gC/Gphfh+cDi8amHElSv/q5jfBS4NvAwRGxMiLOAC4AjouInwKvruOSpAk0dbQZMvO0EZqOHeNaJElbwG9iSlKjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjRr1b6FIasecc68ddAkALL/gxEGX8LTgGbgkNcoAl6RGGeCS1CgDXJIaZYBLUqO8C0XSdunpcEeOZ+CS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVHbFOARMS8ifhIRd0bEuWNVlCRpdFsd4BGxA/A3wAnAocBpEXHoWBUmSdq8bTkDPxK4MzPvzsxfAZcBJ41NWZKk0URmbt0DI04B5mXmu+r424CjMvOsnvnOBM6sowcDP9n6csfEPsD9A65hsnBbbOS22MhtsdFk2RazM3N678Rx/5dqmbkAWDDe6+lXRCzJzLmDrmMycFts5LbYyG2x0WTfFtvShXIfsH9nfFadJkmaANsS4N8FDoqIZ0XENODNwNVjU5YkaTRb3YWSmRsi4izgOmAH4LOZeceYVTZ+Jk13ziTgttjIbbGR22KjSb0ttvoipiRpsPwmpiQ1ygCXpEYZ4JLUqHG/D3zQIuJ5wH7ALZn5aGf6vMz86uAqm3gRcSSQmfnd+mcP5gE/zsx/GnBpmkQi4pLMPH3QdQxCzYuTKJkB5dboqzNz2eCqGtl2fREzIt4HvBdYBhwBnJ2Zi2rbrZn5wkHWN5Ei4r9S/m7NVOB64Cjga8BxwHWZ+bEBljdpRMQ7MvNzg65jokRE762/AbwKuBEgM18/4UUNSEScA5xG+bMgK+vkWZRbpC/LzAsGVdtItvcA/wHw0sx8NCLmAFcCn8/MiyLitsx8wUALnEB1WxwB7ASsBmZl5sMRsQvl08m/G2iBk0RE3JuZBwy6jokSEbcCPwI+DSQlwC+lhBaZedPgqptYEfGvwGGZ+eue6dOAOzLzoMFUNrLtvQtlylC3SWYuj4ijgSsjYjblQH062ZCZTwKPRcRdmfkwQGY+HhFPDbi2CRUR3x+pCZgxkbVMAnOBs4E/Af44M2+PiMefTsHd8RSwL3BPz/SZtW3S2d4DfE1EHJGZtwPUM/HXAp8FfnewpU24X0XErpn5GPCioYkRsQeT9OAcRzOA44H1PdMD+NbElzM4mfkUcGFE/EP9vYbtPxdG8n7ghoj4KbCiTjsAOBA4a8RHDdD2vqNOBzZ0J2TmBuD0iPi7wZQ0MK/MzCfgNy/aITsC8wdT0sBcA+w+9MbeFRGLJ76cwcvMlcCbIuJE4OFB1zMImfnViHgu5U9ldy9ifrd+ep10tus+cEnannkfuCQ1ygCXpEYZ4JLUKANckhplgEtSo/4/COpOixIs2jEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vmoVBEGrxfxB",
        "colab_type": "text"
      },
      "source": [
        "#Removing columns\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aH8z4ifb4v19",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Reviewdata.drop(columns = ['PhraseId', 'SentenceId'], inplace = True)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h4s3WCDsuM1z",
        "colab_type": "text"
      },
      "source": [
        "## DATA CLEANING USING REGEX "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bsQGVulBxfxN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Apply first level cleaning\n",
        "import re\n",
        "import string\n",
        "\n",
        "#This function converts to lower-case, removes square bracket, removes numbers and punctuation\n",
        "def text_clean_1(text):\n",
        "    text = text.lower()\n",
        "    text = re.sub('\\[.*?\\]', '', text)\n",
        "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
        "    text = re.sub('\\w*\\d\\w*', '', text)\n",
        "    return text\n",
        "\n",
        "cleaned1 = lambda x: text_clean_1(x)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qJ2iJEFtxfxZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "7bd57715-5af6-4936-8f12-e5b4dc999d76"
      },
      "source": [
        "# Let's take a look at the updated text\n",
        "Reviewdata['cleaned_description'] = pd.DataFrame(Reviewdata.Phrase.apply(cleaned1))\n",
        "Reviewdata.head(5)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Phrase</th>\n",
              "      <th>Sentiment</th>\n",
              "      <th>cleaned_description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A series of escapades demonstrating the adage ...</td>\n",
              "      <td>1</td>\n",
              "      <td>a series of escapades demonstrating the adage ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A series of escapades demonstrating the adage ...</td>\n",
              "      <td>2</td>\n",
              "      <td>a series of escapades demonstrating the adage ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>A series</td>\n",
              "      <td>2</td>\n",
              "      <td>a series</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>A</td>\n",
              "      <td>2</td>\n",
              "      <td>a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>series</td>\n",
              "      <td>2</td>\n",
              "      <td>series</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              Phrase  ...                                cleaned_description\n",
              "0  A series of escapades demonstrating the adage ...  ...  a series of escapades demonstrating the adage ...\n",
              "1  A series of escapades demonstrating the adage ...  ...  a series of escapades demonstrating the adage ...\n",
              "2                                           A series  ...                                           a series\n",
              "3                                                  A  ...                                                  a\n",
              "4                                             series  ...                                             series\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A2cNFZoSxfxq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Apply a second round of cleaning\n",
        "def text_clean_2(text):\n",
        "    text = re.sub('[‘’“”…]', '', text)\n",
        "    text = re.sub('\\n', '', text)\n",
        "    return text\n",
        "\n",
        "cleaned2 = lambda x: text_clean_2(x)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UEoTPeUZxfx3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "971abb9f-a1cd-4238-8e58-26594d9e1c2a"
      },
      "source": [
        "# Let's take a look at the updated text\n",
        "Reviewdata['cleaned_description_new'] = pd.DataFrame(Reviewdata['cleaned_description'].apply(cleaned2))\n",
        "Reviewdata.head(5)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Phrase</th>\n",
              "      <th>Sentiment</th>\n",
              "      <th>cleaned_description</th>\n",
              "      <th>cleaned_description_new</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A series of escapades demonstrating the adage ...</td>\n",
              "      <td>1</td>\n",
              "      <td>a series of escapades demonstrating the adage ...</td>\n",
              "      <td>a series of escapades demonstrating the adage ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A series of escapades demonstrating the adage ...</td>\n",
              "      <td>2</td>\n",
              "      <td>a series of escapades demonstrating the adage ...</td>\n",
              "      <td>a series of escapades demonstrating the adage ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>A series</td>\n",
              "      <td>2</td>\n",
              "      <td>a series</td>\n",
              "      <td>a series</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>A</td>\n",
              "      <td>2</td>\n",
              "      <td>a</td>\n",
              "      <td>a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>series</td>\n",
              "      <td>2</td>\n",
              "      <td>series</td>\n",
              "      <td>series</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              Phrase  ...                            cleaned_description_new\n",
              "0  A series of escapades demonstrating the adage ...  ...  a series of escapades demonstrating the adage ...\n",
              "1  A series of escapades demonstrating the adage ...  ...  a series of escapades demonstrating the adage ...\n",
              "2                                           A series  ...                                           a series\n",
              "3                                                  A  ...                                                  a\n",
              "4                                             series  ...                                             series\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LUDQPXnL0Ixb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Reviewdata.drop(columns = ['Phrase', 'cleaned_description'], inplace = True)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fpPFvN_2xfyD",
        "colab_type": "text"
      },
      "source": [
        "## Model training "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JNGy8jJnxfyF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "c247c63a-4a70-40b3-9a3f-cb7da8b36e15"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "Independent_var = Reviewdata.cleaned_description_new\n",
        "Dependent_var = Reviewdata.Sentiment\n",
        "\n",
        "IV_train, IV_test, DV_train, DV_test = train_test_split(Independent_var, Dependent_var, test_size = 0.1, random_state = 225)\n",
        "\n",
        "print('IV_train :', len(IV_train))\n",
        "print('IV_test  :', len(IV_test))\n",
        "print('DV_train :', len(DV_train))\n",
        "print('DV_test  :', len(DV_test))\n"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "IV_train : 140454\n",
            "IV_test  : 15606\n",
            "DV_train : 140454\n",
            "DV_test  : 15606\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6nMYUCbNxfyR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "tvec = TfidfVectorizer()\n",
        "clf2 = LogisticRegression(solver = \"lbfgs\")\n",
        "\n",
        "\n",
        "from sklearn.pipeline import Pipeline"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ouJpmWFxfyc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "ff9b8e67-6f5b-417a-8d0f-1709b6886a74"
      },
      "source": [
        "model = Pipeline([('vectorizer',tvec),('classifier',clf2)])\n",
        "\n",
        "model.fit(IV_train, DV_train)\n",
        "\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "predictions = model.predict(IV_test)\n",
        "\n",
        "confusion_matrix(predictions, DV_test)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 119,   99,   11,    2,    1],\n",
              "       [ 368,  993,  397,   87,   11],\n",
              "       [ 181, 1597, 6998, 1510,  149],\n",
              "       [  15,   87,  512, 1502,  528],\n",
              "       [   0,    7,   16,  151,  265]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FPWdgaFtxfyo",
        "colab_type": "text"
      },
      "source": [
        "## Model prediciton "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DcwBYp06xfyq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "c093cc06-78f5-4363-b5ce-684b67f70753"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
        "\n",
        "print(\"Accuracy : \", accuracy_score(predictions, DV_test))\n",
        "print(\"Precision : \", precision_score(predictions, DV_test, average = 'weighted'))\n",
        "print(\"Recall : \", recall_score(predictions, DV_test, average = 'weighted'))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy :  0.6328976034858388\n",
            "Precision :  0.720859674869459\n",
            "Recall :  0.6328976034858388\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zWgjYMx8xfy4",
        "colab_type": "text"
      },
      "source": [
        "## Trying on new reviews "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjck9_0_xfzF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "7f5d5bd9-75ff-4195-f6d4-a1c830a3437a"
      },
      "source": [
        "ex=[input((\"enter a string: \"))]\n",
        "re2=model.predict(ex)\n",
        "if(re2==0):\n",
        "  print(\"very negative\")\n",
        "elif (re2==1):\n",
        "  print(\"negative\")\n",
        "elif (re2==2):\n",
        "  print(\"moderate\")\n",
        "elif (re==3):\n",
        "  print(\"happy\")\n",
        "else:\n",
        "  print(\"very happy\")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "enter a string: very bad movie\n",
            "very negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k5BLyjaTvyAk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "84e405a5-f801-4f5f-efaa-87949dd54ed7"
      },
      "source": [
        "ex=[input((\"enter a string: \"))]\n",
        "re2=model.predict(ex)\n",
        "if(re2==0):\n",
        "  print(\"very negative\")\n",
        "elif (re2==1):\n",
        "  print(\"negative\")\n",
        "elif (re2==2):\n",
        "  print(\"moderate\")\n",
        "elif (re==3):\n",
        "  print(\"happy\")\n",
        "else:\n",
        "  print(\"very happy\")"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "enter a string: its a very good movie\n",
            "very happy\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}