fake_news.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOpE2biwBMAqJYivQlxe8e3",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/redom13/ML/blob/main/fake_news.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Importing Dependencies"
      ],
      "metadata": {
        "id": "txfSXhUhrljH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score"
      ],
      "metadata": {
        "id": "ideobRVPrrGd"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1o2ZxCw-o11y",
        "outputId": "b7f8be6d-0e9e-41a4-c337-78a41543e1b5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(stopwords.words('english')) #The words that are mainly used to make up a sentence\n",
        "# These words don't hold much importance"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V5dYg8gss7r8",
        "outputId": "a0878c4f-58c4-4d3b-a582-659ef45daef0"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Preprocessing"
      ],
      "metadata": {
        "id": "REP_IIPBt6y9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset=pd.read_csv('/content/train.csv',index_col='id')"
      ],
      "metadata": {
        "id": "48oKodRztBKb"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        },
        "id": "W5YUCQe4ua2y",
        "outputId": "5d994cec-4a6f-4ce2-a895-c9e1e88069fa"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                title              author  \\\n",
              "id                                                                          \n",
              "0   House Dem Aide: We Didn’t Even See Comey’s Let...       Darrell Lucus   \n",
              "1   FLYNN: Hillary Clinton, Big Woman on Campus - ...     Daniel J. Flynn   \n",
              "2                   Why the Truth Might Get You Fired  Consortiumnews.com   \n",
              "3   15 Civilians Killed In Single US Airstrike Hav...     Jessica Purkiss   \n",
              "4   Iranian woman jailed for fictional unpublished...      Howard Portnoy   \n",
              "\n",
              "                                                 text  label  \n",
              "id                                                            \n",
              "0   House Dem Aide: We Didn’t Even See Comey’s Let...      1  \n",
              "1   Ever get the feeling your life circles the rou...      0  \n",
              "2   Why the Truth Might Get You Fired October 29, ...      1  \n",
              "3   Videos 15 Civilians Killed In Single US Airstr...      1  \n",
              "4   Print \\nAn Iranian woman has been sentenced to...      1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5ef865a4-807a-4773-bf01-5826994da422\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>author</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>House Dem Aide: We Didn’t Even See Comey’s Let...</td>\n",
              "      <td>Darrell Lucus</td>\n",
              "      <td>House Dem Aide: We Didn’t Even See Comey’s Let...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>FLYNN: Hillary Clinton, Big Woman on Campus - ...</td>\n",
              "      <td>Daniel J. Flynn</td>\n",
              "      <td>Ever get the feeling your life circles the rou...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Why the Truth Might Get You Fired</td>\n",
              "      <td>Consortiumnews.com</td>\n",
              "      <td>Why the Truth Might Get You Fired October 29, ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>15 Civilians Killed In Single US Airstrike Hav...</td>\n",
              "      <td>Jessica Purkiss</td>\n",
              "      <td>Videos 15 Civilians Killed In Single US Airstr...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Iranian woman jailed for fictional unpublished...</td>\n",
              "      <td>Howard Portnoy</td>\n",
              "      <td>Print \\nAn Iranian woman has been sentenced to...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5ef865a4-807a-4773-bf01-5826994da422')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5ef865a4-807a-4773-bf01-5826994da422 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5ef865a4-807a-4773-bf01-5826994da422');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-331918bb-1b24-4b01-8cf2-39f9c759ddaf\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-331918bb-1b24-4b01-8cf2-39f9c759ddaf')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-331918bb-1b24-4b01-8cf2-39f9c759ddaf button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"news_dataset[:5]\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1,\n        \"min\": 0,\n        \"max\": 4,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          1,\n          4,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"title\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"FLYNN: Hillary Clinton, Big Woman on Campus - Breitbart\",\n          \"Iranian woman jailed for fictional unpublished story about woman stoned to death for adultery\",\n          \"Why the Truth Might Get You Fired\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"author\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"Daniel J. Flynn\",\n          \"Howard Portnoy\",\n          \"Consortiumnews.com\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"Ever get the feeling your life circles the roundabout rather than heads in a straight line toward the intended destination? [Hillary Clinton remains the big woman on campus in leafy, liberal Wellesley, Massachusetts. Everywhere else votes her most likely to don her inauguration dress for the remainder of her days the way Miss Havisham forever wore that wedding dress.  Speaking of Great Expectations, Hillary Rodham overflowed with them 48 years ago when she first addressed a Wellesley graduating class. The president of the college informed those gathered in 1969 that the students needed \\u201cno debate so far as I could ascertain as to who their spokesman was to be\\u201d (kind of the like the Democratic primaries in 2016 minus the   terms unknown then even at a Seven Sisters school). \\u201cI am very glad that Miss Adams made it clear that what I am speaking for today is all of us \\u2014  the 400 of us,\\u201d Miss Rodham told her classmates. After appointing herself Edger Bergen to the Charlie McCarthys and Mortimer Snerds in attendance, the    bespectacled in granny glasses (awarding her matronly wisdom \\u2014  or at least John Lennon wisdom) took issue with the previous speaker. Despite becoming the first   to win election to a seat in the U. S. Senate since Reconstruction, Edward Brooke came in for criticism for calling for \\u201cempathy\\u201d for the goals of protestors as he criticized tactics. Though Clinton in her senior thesis on Saul Alinsky lamented \\u201cBlack Power demagogues\\u201d and \\u201celitist arrogance and repressive intolerance\\u201d within the New Left, similar words coming out of a Republican necessitated a brief rebuttal. \\u201cTrust,\\u201d Rodham ironically observed in 1969, \\u201cthis is one word that when I asked the class at our rehearsal what it was they wanted me to say for them, everyone came up to me and said \\u2018Talk about trust, talk about the lack of trust both for us and the way we feel about others. Talk about the trust bust.\\u2019 What can you say about it? What can you say about a feeling that permeates a generation and that perhaps is not even understood by those who are distrusted?\\u201d The \\u201ctrust bust\\u201d certainly busted Clinton\\u2019s 2016 plans. She certainly did not even understand that people distrusted her. After Whitewater, Travelgate, the vast   conspiracy, Benghazi, and the missing emails, Clinton found herself the distrusted voice on Friday. There was a load of compromising on the road to the broadening of her political horizons. And distrust from the American people \\u2014  Trump edged her 48 percent to 38 percent on the question immediately prior to November\\u2019s election \\u2014  stood as a major reason for the closing of those horizons. Clinton described her vanquisher and his supporters as embracing a \\u201clie,\\u201d a \\u201ccon,\\u201d \\u201calternative facts,\\u201d and \\u201ca   assault on truth and reason. \\u201d She failed to explain why the American people chose his lies over her truth. \\u201cAs the history majors among you here today know all too well, when people in power invent their own facts and attack those who question them, it can mark the beginning of the end of a free society,\\u201d she offered. \\u201cThat is not hyperbole. \\u201d Like so many people to emerge from the 1960s, Hillary Clinton embarked upon a long, strange trip. From high school Goldwater Girl and Wellesley College Republican president to Democratic politician, Clinton drank in the times and the place that gave her a degree. More significantly, she went from idealist to cynic, as a comparison of her two Wellesley commencement addresses show. Way back when, she lamented that \\u201cfor too long our leaders have viewed politics as the art of the possible, and the challenge now is to practice politics as the art of making what appears to be impossible possible. \\u201d Now, as the big woman on campus but the odd woman out of the White House, she wonders how her current station is even possible. \\u201cWhy aren\\u2019t I 50 points ahead?\\u201d she asked in September. In May she asks why she isn\\u2019t president. The woman famously dubbed a \\u201ccongenital liar\\u201d by Bill Safire concludes that lies did her in \\u2014  theirs, mind you, not hers. Getting stood up on Election Day, like finding yourself the jilted bride on your wedding day, inspires dangerous delusions.\",\n          \"Print \\nAn Iranian woman has been sentenced to six years in prison after Iran\\u2019s Revolutionary Guard searched her home and found a notebook that contained a fictional story she\\u2019d written about a woman who was stoned to death, according to the Eurasia Review . \\nGolrokh Ebrahimi Iraee, 35, is the wife of political prisoner Arash Sadeghi, 36, who is serving a 19-year prison sentence for being a human rights activist, the publication reported. \\n\\u201cWhen the intelligence unit of the Revolutionary Guards came to arrest her husband, they raided their apartment \\u2013 without a warrant \\u2013 and found drafts of stories that Ebrahimi Iraee had written,\\u201d the article stated. \\n\\u201cOne of the confiscated drafts was a story about stoning women to death for adultery \\u2013 never published, never presented to anyone,\\u201d the article stated. \\u201cThe narrative followed the story of a protagonist that watched a movie about stoning of women under Islamic law for adultery.\",\n          \"Why the Truth Might Get You Fired October 29, 2016 \\nThe tension between intelligence analysts and political policymakers has always been between honest assessments and desired results, with the latter often overwhelming the former, as in the Iraq War, writes Lawrence Davidson. \\nBy Lawrence Davidson \\nFor those who might wonder why foreign policy makers repeatedly make bad choices, some insight might be drawn from the following analysis. The action here plays out in the United States, but the lessons are probably universal. \\nBack in the early spring of 2003, George W. Bush initiated the invasion of Iraq. One of his key public reasons for doing so was the claim that the country\\u2019s dictator, Saddam Hussein, was on the verge of developing nuclear weapons and was hiding other weapons of mass destruction. The real reason went beyond that charge and included a long-range plan for \\u201cregime change\\u201d in the Middle East. President George W. Bush and Vice President Dick Cheney receive an Oval Office briefing from CIA Director George Tenet. Also present is Chief of Staff Andy Card (on right). (White House photo) \\nFor our purposes, we will concentrate on the belief that Iraq was about to become a hostile nuclear power. Why did President Bush and his close associates accept this scenario so readily? \\nThe short answer is Bush wanted, indeed needed, to believe it as a rationale for invading Iraq. At first he had tried to connect Saddam Hussein to the 9/11 attacks on the U.S. Though he never gave up on that stratagem, the lack of evidence made it difficult to rally an American people, already fixated on Afghanistan, to support a war against Baghdad. \\nBut the nuclear weapons gambit proved more fruitful, not because there was any hard evidence for the charge, but because supposedly reliable witnesses, in the persons of exiled anti-Saddam Iraqis (many on the U.S. government\\u2019s payroll ), kept telling Bush and his advisers that the nuclear story was true. \\nWhat we had was a U.S. leadership cadre whose worldview literally demanded a mortally dangerous Iraq, and informants who, in order to precipitate the overthrow of Saddam, were willing to tell the tale of pending atomic weapons. The strong desire to believe the tale of a nuclear Iraq lowered the threshold for proof . Likewise, the repeated assertions by assumed dependable Iraqi sources underpinned a nationwide U.S. campaign generating both fear and war fever. \\nSo the U.S. and its allies insisted that the United Nations send in weapons inspectors to scour Iraq for evidence of a nuclear weapons program (as well as chemical and biological weapons). That the inspectors could find no convincing evidence only frustrated the Bush administration and soon forced its hand. \\nOn March 19, 2003, Bush launched the invasion of Iraq with the expectation was that, once in occupation of the country, U.S. inspectors would surely find evidence of those nukes (or at least stockpiles of chemical and biological weapons). They did not. Their Iraqi informants had systematically lied to them. \\nSocial and Behavioral Sciences to the Rescue? \\nThe various U.S. intelligence agencies were thoroughly shaken by this affair, and today, 13 years later, their directors and managers are still trying to sort it out \\u2013 specifically, how to tell when they are getting \\u201ctrue\\u201d intelligence and when they are being lied to. Or, as one intelligence worker has put it, we need \\u201c help to protect us against armies of snake oil salesmen. \\u201d To that end the CIA et al. are in the market for academic assistance. Ahmed Chalabi, head of the Iraqi National Congress, a key supplier of Iraqi defectors with bogus stories of hidden WMD. \\nA \\u201cpartnership\\u201d is being forged between the Office of the Director of National Intelligence (ODNI), which serves as the coordinating center for the sixteen independent U.S. intelligence agencies, and the National Academies of Sciences, Engineering and Medicine . The result of this collaboration will be a \\u201c permanent Intelligence Community Studies Board\\u201d to coordinate programs in \\u201csocial and behavioral science research [that] might strengthen national security .\\u201d \\nDespite this effort, it is almost certain that the \\u201csocial and behavioral sciences\\u201d cannot give the spy agencies what they want \\u2013 a way of detecting lies that is better than their present standard procedures of polygraph tests and interrogations. But even if they could, it might well make no difference, because the real problem is not to be found with the liars. It is to be found with the believers. \\nThe Believers \\nIt is simply not true, as the ODNI leaders seem to assert, that U.S. intelligence agency personnel cannot tell, more often than not, that they are being lied to. This is the case because there are thousands of middle-echelon intelligence workers, desk officers, and specialists who know something closely approaching the truth \\u2013 that is, they know pretty well what is going on in places like Afghanistan, Iraq, Syria, Libya, Israel, Palestine and elsewhere. Director of National Intelligence James Clapper (right) talks with President Barack Obama in the Oval Office, with John Brennan and other national security aides present. (Photo credit: Office of Director of National Intelligence) \\nTherefore, if someone feeds them \\u201csnake oil,\\u201d they usually know it. However, having an accurate grasp of things is often to no avail because their superiors \\u2013 those who got their appointments by accepting a pre-structured worldview \\u2013 have different criterion for what is \\u201ctrue\\u201d than do the analysts. \\nListen to Charles Gaukel, of the National Intelligence Council \\u2013 yet another organization that acts as a meeting ground for the 16 intelligence agencies. Referring to the search for a way to avoid getting taken in by lies, Gaukel has declared, \\u201c We\\u2019re looking for truth. But we\\u2019re particularly looking for truth that works. \\u201d Now what might that mean? \\nI can certainly tell you what it means historically. It means that for the power brokers, \\u201ctruth\\u201d must match up, fit with, their worldview \\u2013 their political and ideological precepts. If it does not fit, it does not \\u201cwork.\\u201d So the intelligence specialists who send their usually accurate assessments up the line to the policy makers often hit a roadblock caused by \\u201cgroup think,\\u201d ideological blinkers, and a \\u201cwe know better\\u201d attitude. \\nOn the other hand, as long as what you\\u2019re selling the leadership matches up with what they want to believe, you can peddle them anything: imaginary Iraqi nukes, Israel as a Western-style democracy, Saudi Arabia as an indispensable ally, Libya as a liberated country, Bashar al-Assad as the real roadblock to peace in Syria, the Strategic Defense Initiative (SDI) aka Star Wars, a world that is getting colder and not warmer, American exceptionalism in all its glory \\u2013 the list is almost endless. \\nWhat does this sad tale tell us? If you want to spend millions of dollars on social and behavioral science research to improve the assessment and use of intelligence, forget about the liars. What you want to look for is an antidote to the narrow-mindedness of the believers \\u2013 the policymakers who seem not to be able to rise above the ideological presumptions of their class \\u2013 presumptions that underpin their self-confidence as they lead us all down slippery slopes. \\nIt has happened this way so often, and in so many places, that it is the source of Shakespeare\\u2019s determination that \\u201cwhat is past, is prelude.\\u201d Our elites play out our destinies as if they have no free will \\u2013 no capacity to break with structured ways of seeing. Yet the middle-echelon specialists keep sending their relatively accurate assessments up the ladder of power. Hope springs eternal.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nFC0HbwZueMi",
        "outputId": "4b441823-2f58-4961-c345-6c44c58935e7"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20800, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset.isnull().sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209
        },
        "id": "p8U5W7Izuxe_",
        "outputId": "99d40d19-00cc-48cc-83a1-713b3ea6f72a"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "title      558\n",
              "author    1957\n",
              "text        39\n",
              "label        0\n",
              "dtype: int64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>title</th>\n",
              "      <td>558</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>author</th>\n",
              "      <td>1957</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>text</th>\n",
              "      <td>39</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>label</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> int64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset=news_dataset.fillna('')"
      ],
      "metadata": {
        "id": "G0uAY1mpwKHO"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset['content']=news_dataset['author']+' '+news_dataset['title']"
      ],
      "metadata": {
        "id": "kt70u6kFxMVF"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "p0YheKEexx6m",
        "outputId": "04464dc1-99ba-4ae9-8222-bed364280348"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                title              author  \\\n",
              "id                                                                          \n",
              "0   House Dem Aide: We Didn’t Even See Comey’s Let...       Darrell Lucus   \n",
              "1   FLYNN: Hillary Clinton, Big Woman on Campus - ...     Daniel J. Flynn   \n",
              "2                   Why the Truth Might Get You Fired  Consortiumnews.com   \n",
              "3   15 Civilians Killed In Single US Airstrike Hav...     Jessica Purkiss   \n",
              "4   Iranian woman jailed for fictional unpublished...      Howard Portnoy   \n",
              "\n",
              "                                                 text  label  \\\n",
              "id                                                             \n",
              "0   House Dem Aide: We Didn’t Even See Comey’s Let...      1   \n",
              "1   Ever get the feeling your life circles the rou...      0   \n",
              "2   Why the Truth Might Get You Fired October 29, ...      1   \n",
              "3   Videos 15 Civilians Killed In Single US Airstr...      1   \n",
              "4   Print \\nAn Iranian woman has been sentenced to...      1   \n",
              "\n",
              "                                              content  \n",
              "id                                                     \n",
              "0   Darrell Lucus House Dem Aide: We Didn’t Even S...  \n",
              "1   Daniel J. Flynn FLYNN: Hillary Clinton, Big Wo...  \n",
              "2   Consortiumnews.com Why the Truth Might Get You...  \n",
              "3   Jessica Purkiss 15 Civilians Killed In Single ...  \n",
              "4   Howard Portnoy Iranian woman jailed for fictio...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-12fa9c9c-3bf8-4bc2-a545-c06b029d0cda\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>author</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>content</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>House Dem Aide: We Didn’t Even See Comey’s Let...</td>\n",
              "      <td>Darrell Lucus</td>\n",
              "      <td>House Dem Aide: We Didn’t Even See Comey’s Let...</td>\n",
              "      <td>1</td>\n",
              "      <td>Darrell Lucus House Dem Aide: We Didn’t Even S...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>FLYNN: Hillary Clinton, Big Woman on Campus - ...</td>\n",
              "      <td>Daniel J. Flynn</td>\n",
              "      <td>Ever get the feeling your life circles the rou...</td>\n",
              "      <td>0</td>\n",
              "      <td>Daniel J. Flynn FLYNN: Hillary Clinton, Big Wo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Why the Truth Might Get You Fired</td>\n",
              "      <td>Consortiumnews.com</td>\n",
              "      <td>Why the Truth Might Get You Fired October 29, ...</td>\n",
              "      <td>1</td>\n",
              "      <td>Consortiumnews.com Why the Truth Might Get You...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>15 Civilians Killed In Single US Airstrike Hav...</td>\n",
              "      <td>Jessica Purkiss</td>\n",
              "      <td>Videos 15 Civilians Killed In Single US Airstr...</td>\n",
              "      <td>1</td>\n",
              "      <td>Jessica Purkiss 15 Civilians Killed In Single ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Iranian woman jailed for fictional unpublished...</td>\n",
              "      <td>Howard Portnoy</td>\n",
              "      <td>Print \\nAn Iranian woman has been sentenced to...</td>\n",
              "      <td>1</td>\n",
              "      <td>Howard Portnoy Iranian woman jailed for fictio...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-12fa9c9c-3bf8-4bc2-a545-c06b029d0cda')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-12fa9c9c-3bf8-4bc2-a545-c06b029d0cda button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-12fa9c9c-3bf8-4bc2-a545-c06b029d0cda');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fcb344dc-23a8-485f-b9af-04a884f24227\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fcb344dc-23a8-485f-b9af-04a884f24227')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fcb344dc-23a8-485f-b9af-04a884f24227 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "news_dataset",
              "summary": "{\n  \"name\": \"news_dataset\",\n  \"rows\": 20800,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6004,\n        \"min\": 0,\n        \"max\": 20799,\n        \"num_unique_values\": 20800,\n        \"samples\": [\n          14649,\n          9231,\n          6473\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"title\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 19804,\n        \"samples\": [\n          \"Florida Republicans Voting for Hillary Clinton \\u00b7 Guardian Liberty Voice\",\n          \"A Suit Over Schiele Drawings Invokes New Law on Nazi-Looted Art - The New York Times\",\n          \"MMR Vaccines Cause 340% Increased Risk of Autism in African American Infants\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"author\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4202,\n        \"samples\": [\n          \"OZbanislam\",\n          \"Julia Hahn\",\n          \"Cassandra Fairbanks\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 20387,\n        \"samples\": [\n          \"Multan a un ultrasur por lanzar a un jugador del Bar\\u00e7a al campo durante el partido Madrid-Legia EL PARTIDO SE CONSIDERABA DE ALTO RIESGO PARA LOS JUGADORES DEL BARCELONA Real Madrid \\nUn jugador barcelonista golpe\\u00f3 a varios jugadores durante el encuentro de la Champions que tuvo lugar ayer entre el Real Madrid y el Legia Varsovia despu\\u00e9s de ser arrojado desde la grada por un hincha madridista. \\nEl portero del Legia, muy molesto, intent\\u00f3 devolver el jugador a las gradas de una patada y se encar\\u00f3 con los Ultrasur, que ya preparaban a otro jugador para echarlo al campo y entorpecer el encuentro. En esta ocasi\\u00f3n pretend\\u00edan prenderlo con un mechero antes de lanzarlo al c\\u00e9sped. \\nLas autoridades han vuelto a pedir a los aficionados que dejen de lanzar cosas al campo como botellas, monedas y jugadores del Barcelona. El dispositivo de seguridad no puedo hacer nada por evitar que algunos hinchas introdujeran a jugadores barcelonistas a la grada, pues los llevaban camuflados dentro de bocadillos de gran tama\\u00f1o. \\nEl partido pudo ser reanudado cuando el \\u00e1rbitro apart\\u00f3 a un rinc\\u00f3n del c\\u00e9sped al jugador barcelonista.\",\n          \"Written by Eric Margolis   As a former soldier and war correspondent who has covered 14 conflicts, I look at all the media hoopla over tightening siege of Mosul, Iraq and shake my head. This western-organized \\u201cliberation\\u201d of Mosul is one of the bigger pieces of political-military theater that I\\u2019ve seen. Islamic State (IS), the defender of Mosul, is a paper tiger, blown out of all proportion by western media. IS is, as this writer has been saying for years, an armed mob made up of 20-something malcontents, religious fanatics, and modern-day anarchists. At its top is a cadre of former Iraqi Army officers with military experience. These former officers of Saddam Hussain are bent on revenge for the US destruction of their nation and the lynching of its late leader. But IS rank and file has no military training, little discipline, degraded communications, and ragged logistics. In fact, today\\u2019s Islamic State is what the Ottoman Empire used to term, \\u2018bashi-bazouks,\\u201d a collection of irregular cut-throats and scum of the gutter sent to punish and terrorize enemies by means of torture, rapine, looting, and arson. What has amazed me about the faux western war against ISIS is its leisurely nature, lack of \\u00e9lan , and hesitancy. In my view, ISIS was mostly created by the US and its allies as a weapon to be used against Syria\\u2019s government \\u2013 just as the Afghan mujahadin were used by the US and the Saudis to overthrow the Soviet-backed Afghan government. Israel tried the same tactics by helping create Hamas in Palestine and Hezbollah in Lebanon. Both were cultivated to split the PLO. ISIS is an ad hoc movement that wants to punish the West and the Saudis for the gross carnage they have inflicted on the Arab world. Western and Kurdish auxiliary forces have been sitting 1.5 hours drive from Mosul and the IS town of Raqqa for over a year. Instead, western \\u2013 mainly US \\u2013 warplanes have been gingerly bombing around these targets in what may be an effort to convince breakaway ISIS to rejoin US-led forces fight the Damascus regime. Note that ISIS does not appear to have ever attacked Israel though it is playing an important role in the destruction of Syria. Some reports say Israel is providing logistic and medical support for IS. The siege of Mosul is being played up by western media as a heroic second Stalingrad. Don\\u2019t be fooled. IS has only 3-5,000 lightly armed fighters in Mosul and Raqqa, maybe even less. The leaders of IS are likely long gone. IS has few heavy weapons, no air cover at all, and poor communications. Its rag-tag fighters will run out of ammunitions and explosives very quickly. Encircling Mosul are at least 50,000 western-led soldiers, backed by heavy artillery, rocket batteries, tanks, armored vehicles and awesome air power The western imperial forces are composed of tough Kurdish peshmerga fighters, Iraqi army and special forces, some Syrian Kurds, Iranian \\u2018volunteers\\u2019 irregular forces and at least 5,000 US combat troops called \\u201cadvisors\\u201d, plus small numbers of French, Canadian and British special forces. Hovering in the background are some thousands of Turkish troops, supported by armor and artillery ready to \\\"liberate\\\" Iraq \\u2013 which was once part of the Ottoman Empire. For the US, current military operations in Syria and Iraq are the realization of an imperialist\\u2019s fondest dream: native troops led by white officers, the model of the old British Indian Raj. Washington arms, trained, equips and financed all its native auxiliaries. The IS is caught in a dangerous dilemma. To be a political movement, it was delighted to control Iraq\\u2019s second largest city. But as a guerilla force, it should not have holed up in an urban area where it was highly vulnerable to concentrated air attack and being surrounded. This is what\\u2019s happening right now. In the mostly flat Fertile Crescent with too few trees, ground forces are totally vulnerable to air power, as the recent 1967, 1973 Israel-Arab wars and 2003 Iraq wars have shown. Dispersion and guerilla tactics are the only hope for those that lack air cover. IS forces would best advise to disperse across the region and continue their hit-and-run attacks. Otherwise, they risk being destroyed. But being mostly bloody-minded young fanatics, IS may not heed military logic and precedent in favor of making a last stand in the ruins of Mosul and Raqqa. When this happens, western leaders will compete to claim authorship of the faux crusade against the paper tiger of ISIS.   LewRockwell.com . Related\",\n          \"The   at the Democratic National Committee after an embarrassing breach of its email system continued on Tuesday with the departure of three senior officials. Amy Dacey, the committee\\u2019s chief executive Luis Miranda, its communications director and Brad Marshall, its chief financial officer, will leave amid a reshuffling of leadership positions, said Donna Brazile, the interim chairwoman. The departures came more than a week after WikiLeaks posted almost 20, 000 of the committee\\u2019s emails, a number of which revealed officials showing favoritism toward Hillary Clinton in her primary campaign against Senator Bernie Sanders of Vermont. The messages confirmed the concerns expressed by Mr. Sanders throughout the campaign, cast a cloud over the start of the Democratic National Convention in Philadelphia last week and led to the resignation of Representative Debbie Wasserman Schultz of Florida as the committee\\u2019s chairwoman. Ms. Brazile praised the outgoing staff members and made no mention of the controversy. \\u201cThanks in part to the hard work of Amy, Luis, and Brad, the Democratic Party has adopted the most progressive platform in history, has put itself in financial position to win in November, and has begun the important work of investing in state party partnerships,\\u201d she said in a statement. \\u201cI\\u2019m so grateful for their commitment to this cause, and I wish them continued success in the next chapter of their career. \\u201d American intelligence officials believe that the Russian government was behind the breach of the committee\\u2019s emails and documents, possibly as part of an effort to damage Mrs. Clinton and sow discord in the Democratic Party. An email from Mr. Marshall to Mark Paustenbach, a communications official, and Ms. Dacey suggesting that the committee promote questions about Mr. Sanders\\u2019s faith drew particular scorn from the senator and his supporters. \\u201cIt might may no difference, but for KY and WVA can we get someone to ask his belief. Does he believe in a God,\\u201d Mr. Marshall wrote, referring to Kentucky and West Virginia. \\u201cHe had skated on saying he has a Jewish heritage. I think I read he is an atheist. This could make several points difference with my peeps. \\u201d In May, Mr. Paustenbach wrote to Mr. Miranda about encouraging reporters to write that Mr. Sanders\\u2019s campaign was \\u201ca mess\\u201d after a glitch on the committee\\u2019s servers gave the Sanders campaign access to the Clinton campaign\\u2019s voter database. \\u201cWondering if there\\u2019s a good Bernie narrative for a story, which is that Bernie never ever had his act together, that his campaign was a mess,\\u201d Mr. Paustenbach wrote. Mr. Miranda wrote back: \\u201cTrue, but the Chair has been advised to not engage. So we\\u2019ll have to leave it alone. \\u201d The committee has since apologized to the Sanders campaign. To help fill the void, the committee announced, it is bringing on Tom McMahon, a former executive director of the committee, to lead a transition team.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"content\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 20412,\n        \"samples\": [\n          \"Wrenfoe Fellated by a dead pig? You may have been a victim of Tory bullying\",\n          \"Dan Riehl Stephen Miller: Trump Has \\u2019Better Sense of the Pulse of the People\\u2019 Than Any President Since Andrew Jackson\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y=news_dataset['label']\n",
        "X=news_dataset.drop(columns='label',axis=1)"
      ],
      "metadata": {
        "id": "ADulcdxdx09H"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X),print(y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bcFM5Juey2_S",
        "outputId": "f80f0586-3049-496f-d742-ce49aa00e675"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                                   title  \\\n",
            "id                                                         \n",
            "0      House Dem Aide: We Didn’t Even See Comey’s Let...   \n",
            "1      FLYNN: Hillary Clinton, Big Woman on Campus - ...   \n",
            "2                      Why the Truth Might Get You Fired   \n",
            "3      15 Civilians Killed In Single US Airstrike Hav...   \n",
            "4      Iranian woman jailed for fictional unpublished...   \n",
            "...                                                  ...   \n",
            "20795  Rapper T.I.: Trump a ’Poster Child For White S...   \n",
            "20796  N.F.L. Playoffs: Schedule, Matchups and Odds -...   \n",
            "20797  Macy’s Is Said to Receive Takeover Approach by...   \n",
            "20798  NATO, Russia To Hold Parallel Exercises In Bal...   \n",
            "20799                          What Keeps the F-35 Alive   \n",
            "\n",
            "                                          author  \\\n",
            "id                                                 \n",
            "0                                  Darrell Lucus   \n",
            "1                                Daniel J. Flynn   \n",
            "2                             Consortiumnews.com   \n",
            "3                                Jessica Purkiss   \n",
            "4                                 Howard Portnoy   \n",
            "...                                          ...   \n",
            "20795                              Jerome Hudson   \n",
            "20796                           Benjamin Hoffman   \n",
            "20797  Michael J. de la Merced and Rachel Abrams   \n",
            "20798                                Alex Ansary   \n",
            "20799                              David Swanson   \n",
            "\n",
            "                                                    text  \\\n",
            "id                                                         \n",
            "0      House Dem Aide: We Didn’t Even See Comey’s Let...   \n",
            "1      Ever get the feeling your life circles the rou...   \n",
            "2      Why the Truth Might Get You Fired October 29, ...   \n",
            "3      Videos 15 Civilians Killed In Single US Airstr...   \n",
            "4      Print \\nAn Iranian woman has been sentenced to...   \n",
            "...                                                  ...   \n",
            "20795  Rapper T. I. unloaded on black celebrities who...   \n",
            "20796  When the Green Bay Packers lost to the Washing...   \n",
            "20797  The Macy’s of today grew from the union of sev...   \n",
            "20798  NATO, Russia To Hold Parallel Exercises In Bal...   \n",
            "20799    David Swanson is an author, activist, journa...   \n",
            "\n",
            "                                                 content  \n",
            "id                                                        \n",
            "0      Darrell Lucus House Dem Aide: We Didn’t Even S...  \n",
            "1      Daniel J. Flynn FLYNN: Hillary Clinton, Big Wo...  \n",
            "2      Consortiumnews.com Why the Truth Might Get You...  \n",
            "3      Jessica Purkiss 15 Civilians Killed In Single ...  \n",
            "4      Howard Portnoy Iranian woman jailed for fictio...  \n",
            "...                                                  ...  \n",
            "20795  Jerome Hudson Rapper T.I.: Trump a ’Poster Chi...  \n",
            "20796  Benjamin Hoffman N.F.L. Playoffs: Schedule, Ma...  \n",
            "20797  Michael J. de la Merced and Rachel Abrams Macy...  \n",
            "20798  Alex Ansary NATO, Russia To Hold Parallel Exer...  \n",
            "20799            David Swanson What Keeps the F-35 Alive  \n",
            "\n",
            "[20800 rows x 4 columns]\n",
            "id\n",
            "0        1\n",
            "1        0\n",
            "2        1\n",
            "3        1\n",
            "4        1\n",
            "        ..\n",
            "20795    0\n",
            "20796    0\n",
            "20797    0\n",
            "20798    1\n",
            "20799    1\n",
            "Name: label, Length: 20800, dtype: int64\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(None, None)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Stemming**: The process of reduceing a word to its root word"
      ],
      "metadata": {
        "id": "dk-HdUStLpJI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "port_stem=PorterStemmer()"
      ],
      "metadata": {
        "id": "dM1xSbCVy_tA"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def stemming(content):\n",
        "  stemmed_content=re.sub('[^a-zA-Z]',' ',content) # Substitute everyting except alphabets with space in the string-->content\n",
        "  stemmed_content=stemmed_content.lower()\n",
        "  stemmed_content=stemmed_content.split() # Splitted the whole sentences & Converted to a list of words\n",
        "  stemmed_content=[port_stem.stem(word) for word in stemmed_content if not word in stopwords.words('english')]\n",
        "  # Stem all the words that r not stopword\n",
        "  stemmed_content=' '.join(stemmed_content) # Make the sentences again(No list again)\n",
        "  return stemmed_content\n"
      ],
      "metadata": {
        "id": "JtGd3mx2MB1t"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset['content']=news_dataset['content'].apply(stemming)"
      ],
      "metadata": {
        "id": "Q3AXoc-_XrG8"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "news_dataset['content']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "id": "d1dNmXTvYd4H",
        "outputId": "3507bb69-7c05-4750-b0a8-64a863e23138"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "id\n",
              "0        darrel lucu hous dem aid even see comey letter...\n",
              "1        daniel j flynn flynn hillari clinton big woman...\n",
              "2                   consortiumnew com truth might get fire\n",
              "3        jessica purkiss civilian kill singl us airstri...\n",
              "4        howard portnoy iranian woman jail fiction unpu...\n",
              "                               ...                        \n",
              "20795    jerom hudson rapper trump poster child white s...\n",
              "20796    benjamin hoffman n f l playoff schedul matchup...\n",
              "20797    michael j de la merc rachel abram maci said re...\n",
              "20798    alex ansari nato russia hold parallel exercis ...\n",
              "20799                            david swanson keep f aliv\n",
              "Name: content, Length: 20800, dtype: object"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>content</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>darrel lucu hous dem aid even see comey letter...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>daniel j flynn flynn hillari clinton big woman...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>consortiumnew com truth might get fire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>jessica purkiss civilian kill singl us airstri...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>howard portnoy iranian woman jail fiction unpu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20795</th>\n",
              "      <td>jerom hudson rapper trump poster child white s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20796</th>\n",
              "      <td>benjamin hoffman n f l playoff schedul matchup...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20797</th>\n",
              "      <td>michael j de la merc rachel abram maci said re...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20798</th>\n",
              "      <td>alex ansari nato russia hold parallel exercis ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20799</th>\n",
              "      <td>david swanson keep f aliv</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>20800 rows × 1 columns</p>\n",
              "</div><br><label><b>dtype:</b> object</label>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X=news_dataset['content'].values\n",
        "y=news_dataset['label'].values"
      ],
      "metadata": {
        "id": "pHBsxt4TZAwt"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X,y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8I3w02L6agsf",
        "outputId": "665991fd-7917-43f6-a017-841afce045ff"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['darrel lucu hous dem aid even see comey letter jason chaffetz tweet',\n",
              "        'daniel j flynn flynn hillari clinton big woman campu breitbart',\n",
              "        'consortiumnew com truth might get fire', ...,\n",
              "        'michael j de la merc rachel abram maci said receiv takeov approach hudson bay new york time',\n",
              "        'alex ansari nato russia hold parallel exercis balkan',\n",
              "        'david swanson keep f aliv'], dtype=object),\n",
              " array([1, 0, 1, ..., 0, 1, 1]))"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer=TfidfVectorizer(max_features=9000)\n",
        "vectorizer.fit(X)\n",
        "X=vectorizer.transform(X)"
      ],
      "metadata": {
        "id": "sbhm1sCoaiIw"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zEo-wcnScQ7R",
        "outputId": "f699fe1d-c3c2-4246-c0d0-b54f73860153"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  (0, 8330)\t0.28485063562728646\n",
            "  (0, 7172)\t0.2565896679337957\n",
            "  (0, 4655)\t0.3635963806326075\n",
            "  (0, 4481)\t0.29212514087043684\n",
            "  (0, 4080)\t0.24785219520671603\n",
            "  (0, 3706)\t0.21874169089359144\n",
            "  (0, 2726)\t0.233316966909351\n",
            "  (0, 2047)\t0.2705332480845492\n",
            "  (0, 1928)\t0.3598939188262559\n",
            "  (0, 1580)\t0.2468450128533713\n",
            "  (0, 1294)\t0.3676519686797209\n",
            "  (0, 145)\t0.27010124977708766\n",
            "  (1, 8846)\t0.30071745655510157\n",
            "  (1, 3605)\t0.1904660198296849\n",
            "  (1, 3014)\t0.7143299355715573\n",
            "  (1, 1910)\t0.26373768806048464\n",
            "  (1, 1494)\t0.19094574062359204\n",
            "  (1, 1165)\t0.3827320386859759\n",
            "  (1, 1006)\t0.15521974226349364\n",
            "  (1, 797)\t0.2939891562094648\n",
            "  (2, 8286)\t0.41544962664721613\n",
            "  (2, 5052)\t0.49351492943649944\n",
            "  (2, 3252)\t0.3474613386728292\n",
            "  (2, 2950)\t0.3866530551182615\n",
            "  (2, 1662)\t0.46097489583229645\n",
            "  :\t:\n",
            "  (20797, 6982)\t0.2482526352197606\n",
            "  (20797, 6540)\t0.27263457663336677\n",
            "  (20797, 6440)\t0.24778257724396507\n",
            "  (20797, 5437)\t0.08038079000566466\n",
            "  (20797, 5033)\t0.174553480255222\n",
            "  (20797, 4996)\t0.2954204003420313\n",
            "  (20797, 4692)\t0.36160868928090795\n",
            "  (20797, 4360)\t0.22322585870464118\n",
            "  (20797, 3731)\t0.21799048897828688\n",
            "  (20797, 1952)\t0.21155500613623743\n",
            "  (20797, 696)\t0.33538056804139865\n",
            "  (20797, 386)\t0.30685846079762347\n",
            "  (20797, 18)\t0.29710241860700626\n",
            "  (20798, 6939)\t0.22363267488270608\n",
            "  (20798, 5871)\t0.4460515589182236\n",
            "  (20798, 5361)\t0.3192496370187028\n",
            "  (20798, 3636)\t0.32496285694299426\n",
            "  (20798, 2757)\t0.4083701450239529\n",
            "  (20798, 634)\t0.4460515589182236\n",
            "  (20798, 327)\t0.3112141524638974\n",
            "  (20798, 187)\t0.28446937819072576\n",
            "  (20799, 7859)\t0.5677577267055112\n",
            "  (20799, 4241)\t0.45983893273780013\n",
            "  (20799, 1940)\t0.37927626273066584\n",
            "  (20799, 206)\t0.5677577267055112\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Splitting the training set"
      ],
      "metadata": {
        "id": "pBqCt8Di4mMO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,stratify=y,random_state=2)"
      ],
      "metadata": {
        "id": "3GM9rYVkcukX"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training the Model"
      ],
      "metadata": {
        "id": "BzrILfRs72oT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model=LogisticRegression()"
      ],
      "metadata": {
        "id": "Tkhxkxp95lw0"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(X_train,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "id": "qo1ojvXS70H6",
        "outputId": "0577df45-0fd9-4f4f-e9b3-7189c98a7cbf"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Predicting"
      ],
      "metadata": {
        "id": "xAPLDM9z8BcI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_score=model.score(X_test,y_test)"
      ],
      "metadata": {
        "id": "t-e_QoT67_9n"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_score"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RcJYe4Y_797U",
        "outputId": "77dcb9e7-5ca2-4d83-cb65-a0d2e3083138"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9790865384615385"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Transform and prepare raw data for **validation dataset**"
      ],
      "metadata": {
        "id": "FjqYH4yaB2Lv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "validation_df=pd.read_csv('/content/test.csv',index_col='id')\n",
        "validation_df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJVGAVvt8SZD",
        "outputId": "ad50e135-3213-4db3-bca3-a3f861142806"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5200, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "validation_df.isnull().sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "id": "8qOPl0kFCabH",
        "outputId": "086ba61c-ebb4-4ede-c1d5-0a203aed3a4d"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "title     122\n",
              "author    503\n",
              "text        7\n",
              "dtype: int64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>title</th>\n",
              "      <td>122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>author</th>\n",
              "      <td>503</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>text</th>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> int64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "validation_df=validation_df.fillna('')"
      ],
      "metadata": {
        "id": "BprR3IuiClVT"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "validation_df['content']=validation_df['author']+\" \"+validation_df['title']"
      ],
      "metadata": {
        "id": "5MQZUdtdCs2l"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "validation_df['content']=validation_df['content'].apply(stemming)"
      ],
      "metadata": {
        "id": "STVZwMneC_JC"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_valid=validation_df['content'].values"
      ],
      "metadata": {
        "id": "r7k3tS8rEVEd"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer=TfidfVectorizer(max_features=9000)\n",
        "vectorizer.fit(X_valid)\n",
        "X_valid=vectorizer.transform(X_valid)"
      ],
      "metadata": {
        "id": "_YT1-_TSEdhb"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_valid=model.predict(X_valid)"
      ],
      "metadata": {
        "id": "f6nS_LILE4C1"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_valid.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "05LpYvLOE9fb",
        "outputId": "d04503db-9626-45ef-9361-3dd93aabd44c"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5200,)"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "submit_df=pd.read_csv('/content/submit.csv')\n",
        "final_submit=pd.DataFrame()\n",
        "final_submit['id']=submit_df['id']\n",
        "final_submit['label']=y_valid\n",
        "final_submit.to_csv('final_submit.csv',index=False)\n",
        "final_submit.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "MUwTAdjFaO3W",
        "outputId": "311b2311-63ed-401d-d4c9-5ec6650f0ca6"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      id  label\n",
              "0  20800      1\n",
              "1  20801      1\n",
              "2  20802      1\n",
              "3  20803      1\n",
              "4  20804      1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fe6ff850-d936-42d1-8027-3b23b1e43e67\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>20800</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>20801</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>20802</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>20803</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>20804</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fe6ff850-d936-42d1-8027-3b23b1e43e67')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fe6ff850-d936-42d1-8027-3b23b1e43e67 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fe6ff850-d936-42d1-8027-3b23b1e43e67');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-cd7d0b38-90bf-409e-9587-82d5a6dbd7a0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-cd7d0b38-90bf-409e-9587-82d5a6dbd7a0')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-cd7d0b38-90bf-409e-9587-82d5a6dbd7a0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "final_submit",
              "summary": "{\n  \"name\": \"final_submit\",\n  \"rows\": 5200,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1501,\n        \"min\": 20800,\n        \"max\": 25999,\n        \"num_unique_values\": 5200,\n        \"samples\": [\n          21926,\n          20879,\n          24060\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "final_submit"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "4Ja3CSJ7e0QE",
        "outputId": "1c4cdcfe-0482-49d2-bbef-84c2089af696"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         id  label\n",
              "0     20800      1\n",
              "1     20801      1\n",
              "2     20802      1\n",
              "3     20803      1\n",
              "4     20804      1\n",
              "...     ...    ...\n",
              "5195  25995      1\n",
              "5196  25996      1\n",
              "5197  25997      1\n",
              "5198  25998      1\n",
              "5199  25999      1\n",
              "\n",
              "[5200 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d55d52d9-8349-49e6-baa3-56979ecc60c5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>20800</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>20801</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>20802</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>20803</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>20804</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5195</th>\n",
              "      <td>25995</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5196</th>\n",
              "      <td>25996</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5197</th>\n",
              "      <td>25997</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5198</th>\n",
              "      <td>25998</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5199</th>\n",
              "      <td>25999</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5200 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d55d52d9-8349-49e6-baa3-56979ecc60c5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d55d52d9-8349-49e6-baa3-56979ecc60c5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d55d52d9-8349-49e6-baa3-56979ecc60c5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-eb32a9a9-f71e-481a-a5ce-26da09cb74a6\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-eb32a9a9-f71e-481a-a5ce-26da09cb74a6')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-eb32a9a9-f71e-481a-a5ce-26da09cb74a6 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_c1cc351e-eab1-4d73-902c-62617dc0b37b\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('final_submit')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_c1cc351e-eab1-4d73-902c-62617dc0b37b button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('final_submit');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "final_submit",
              "summary": "{\n  \"name\": \"final_submit\",\n  \"rows\": 5200,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1501,\n        \"min\": 20800,\n        \"max\": 25999,\n        \"num_unique_values\": 5200,\n        \"samples\": [\n          21926,\n          20879,\n          24060\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "FjLmEouPhgP8"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}