Text_Summarization_using_BERT,_GPT2,XLNET.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text Summarization using BERT, GPT2,XLNET.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P06w20oVdaHn",
        "outputId": "3a39e4eb-d54d-40e8-cfea-849bb525d59b"
      },
      "source": [
        "!pip install bert-extractive-summarizer"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: bert-extractive-summarizer in /usr/local/lib/python3.7/dist-packages (0.7.1)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (from bert-extractive-summarizer) (2.2.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from bert-extractive-summarizer) (0.22.2.post1)\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (from bert-extractive-summarizer) (2.2.4)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (1.17.50)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (4.41.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (0.0.44)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (0.1.95)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (2.23.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (2017.4.5)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers->bert-extractive-summarizer) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->bert-extractive-summarizer) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->bert-extractive-summarizer) (1.0.1)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (1.0.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (0.8.2)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (7.4.0)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (0.4.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (54.2.0)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (0.28.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (2.0.5)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (0.9.6)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy->bert-extractive-summarizer) (3.0.5)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers->bert-extractive-summarizer) (0.10.0)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers->bert-extractive-summarizer) (0.3.6)\n",
            "Requirement already satisfied: botocore<1.21.0,>=1.20.50 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers->bert-extractive-summarizer) (1.20.50)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers->bert-extractive-summarizer) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers->bert-extractive-summarizer) (7.1.2)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->bert-extractive-summarizer) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->bert-extractive-summarizer) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->bert-extractive-summarizer) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->bert-extractive-summarizer) (2020.12.5)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy->bert-extractive-summarizer) (3.10.0)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/dist-packages (from botocore<1.21.0,>=1.20.50->boto3->transformers->bert-extractive-summarizer) (2.8.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->bert-extractive-summarizer) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->bert-extractive-summarizer) (3.7.4.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pnJ1Zos9dlja",
        "outputId": "29f2ecd8-9191-48a5-af00-5884e369bbbc"
      },
      "source": [
        "!pip install transformers==2.2.0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers==2.2.0 in /usr/local/lib/python3.7/dist-packages (2.2.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (2017.4.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (2.23.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (0.1.95)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (1.17.50)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (0.0.44)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (1.19.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from transformers==2.2.0) (4.41.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.2.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.2.0) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.2.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.2.0) (3.0.4)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers==2.2.0) (0.3.6)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers==2.2.0) (0.10.0)\n",
            "Requirement already satisfied: botocore<1.21.0,>=1.20.50 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers==2.2.0) (1.20.50)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==2.2.0) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==2.2.0) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==2.2.0) (1.0.1)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/dist-packages (from botocore<1.21.0,>=1.20.50->boto3->transformers==2.2.0) (2.8.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ikwMl426d6Ul",
        "outputId": "b561bf96-c37b-4e3a-e2e2-1827155329c4"
      },
      "source": [
        "!pip install spacy"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (2.2.4)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.19.5)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.9.6)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (7.4.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy) (54.2.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.28.0)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.5)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.1)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.0)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.23.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.41.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.8.2)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.5)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy) (3.10.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.12.5)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy) (3.4.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tkYxe1FpeHIk"
      },
      "source": [
        "from summarizer import Summarizer,TransformerSummarizer"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_S-l8K6kffiV"
      },
      "source": [
        "body = '''\n",
        "       Scientists say they have discovered a new species of orangutans on Indonesia’s island of Sumatra.\n",
        "The population differs in several ways from the two existing orangutan species found in Sumatra and the neighboring island of Borneo.\n",
        "The orangutans were found inside North Sumatra’s Batang Toru forest, the science publication Current Biology reported.\n",
        "Researchers named the new species the Tapanuli orangutan. They say the animals are considered a new species because of genetic, skeletal and tooth differences.\n",
        "Michael Kruetzen is a geneticist with the University of Zurich who has studied the orangutans for several years. He said he was excited to be part of the unusual discovery of a new great ape in the present day. He noted that most great apes are currently considered endangered or severely endangered.\n",
        "Gorillas, chimpanzees and bonobos also belong to the great ape species.\n",
        "Orangutan – which means person of the forest in the Indonesian and Malay languages - is the world’s biggest tree-living mammal. The orange-haired animals can move easily among the trees because their arms are longer than their legs. They live more lonely lives than other great apes, spending a lot of time sleeping and eating fruit in the forest.\n",
        "The new study said fewer than 800 of the newly-described orangutans exist. Their low numbers make the group the most endangered of all the great ape species.\n",
        "They live within an area covering about 1,000 square kilometers. The population is considered highly vulnerable. That is because the environment which they depend on is greatly threatened by development.\n",
        "Researchers say if steps are not taken quickly to reduce the current and future threats, the new species could become extinct “within our lifetime.”\n",
        "Research into the new species began in 2013, when an orangutan protection group in Sumatra found an injured orangutan in an area far away from the other species. The adult male orangutan had been beaten by local villagers and died of his injuries. The complete skull was examined by researchers.\n",
        "Among the physical differences of the new species are a notably smaller head and frizzier hair. The Tapanuli orangutans also have a different diet and are found only in higher forest areas.\n",
        "There is no unified international system for recognizing new species. But to be considered, discovery claims at least require publication in a major scientific publication.\n",
        "Russell Mittermeier is head of the primate specialist group at the International Union for the Conservation of Nature. He called the finding a “remarkable discovery.” He said it puts responsibility on the Indonesian government to help the species survive.\n",
        "Matthew Nowak is one of the writers of the study. He told the Associated Press that there are three groups of the Tapanuli orangutans that are separated by non-protected land.He said forest land needs to connect the separated groups.\n",
        "In addition, the writers of the study are recommending that plans for a hydropower center in the area be stopped by the government.\n",
        "It also recommended that remaining forest in the Sumatran area where the orangutans live be protected.\n",
        "I’m Bryan Lynn.\n",
        "\n",
        "        '''"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y7qZlFLqg2a7",
        "outputId": "70db2745-9fb8-40c0-a3db-ba41eee27c81"
      },
      "source": [
        "bert_model = Summarizer()\n",
        "bert_summary = ''.join(bert_model(body, min_length=60))\n",
        "print(bert_summary)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 434/434 [00:00<00:00, 108604.97B/s]\n",
            "100%|██████████| 1344997306/1344997306 [00:22<00:00, 59156793.99B/s]\n",
            "100%|██████████| 231508/231508 [00:00<00:00, 17530509.67B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Scientists say they have discovered a new species of orangutans on Indonesia’s island of Sumatra. They say the animals are considered a new species because of genetic, skeletal and tooth differences. He said he was excited to be part of the unusual discovery of a new great ape in the present day. He noted that most great apes are currently considered endangered or severely endangered. Orangutan – which means person of the forest in the Indonesian and Malay languages - is the world’s biggest tree-living mammal. It also recommended that remaining forest in the Sumatran area where the orangutans live be protected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BRlVvp4Ug-PF",
        "outputId": "476c47a6-b61d-44b7-9cca-e3c231eb2127"
      },
      "source": [
        "GPT2_model = TransformerSummarizer(transformer_type=\"GPT2\",transformer_model_key=\"gpt2-medium\")\n",
        "full = ''.join(GPT2_model(body, min_length=60))\n",
        "print(full)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 718/718 [00:00<00:00, 175250.83B/s]\n",
            "100%|██████████| 1520013706/1520013706 [00:26<00:00, 56809199.94B/s]\n",
            "100%|██████████| 1042301/1042301 [00:00<00:00, 31741055.05B/s]\n",
            "100%|██████████| 456318/456318 [00:00<00:00, 19174069.19B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Scientists say they have discovered a new species of orangutans on Indonesia’s island of Sumatra. The orangutans were found inside North Sumatra’s Batang Toru forest, the science publication Current Biology reported. The new study said fewer than 800 of the newly-described orangutans exist. They live within an area covering about 1,000 square kilometers. That is because the environment which they depend on is greatly threatened by development. He told the Associated Press that there are three groups of the Tapanuli orangutans that are separated by non-protected land.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OE0HzUQ_jtOU",
        "outputId": "371a5a63-900f-4158-b5a5-763e8ae2b01c"
      },
      "source": [
        "model = TransformerSummarizer(transformer_type=\"XLNet\",transformer_model_key=\"xlnet-base-cased\")\n",
        "full = ''.join(model(body, min_length=60))\n",
        "print(full)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 760/760 [00:00<00:00, 255565.71B/s]\n",
            "100%|██████████| 467042463/467042463 [00:08<00:00, 55288530.16B/s]\n",
            "100%|██████████| 798011/798011 [00:00<00:00, 25629820.12B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Scientists say they have discovered a new species of orangutans on Indonesia’s island of Sumatra. They say the animals are considered a new species because of genetic, skeletal and tooth differences. Their low numbers make the group the most endangered of all the great ape species. Researchers say if steps are not taken quickly to reduce the current and future threats, the new species could become extinct “within our lifetime.” The Tapanuli orangutans also have a different diet and are found only in higher forest areas. It also recommended that remaining forest in the Sumatran area where the orangutans live be protected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HY0UM4W8l53k"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}