ASSESSMENT2__vicio_di_cara.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "3oKqY2KcV6hG",
        "gyFaidWaWQUS"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/VicDc/Uruz01/blob/main/ASSESSMENT2__vicio_di_cara.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Natural Language Processing (NLP)"
      ],
      "metadata": {
        "id": "T5iGp_WgVvIA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GyZgAKiFkNAp",
        "outputId": "a36413f0-8bef-4ff2-d845-90c2833bdff3",
        "collapsed": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (3.1.0)\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (0.4.3)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.10/dist-packages (0.1.1)\n",
            "Requirement already satisfied: transformers[sentencepiece] in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3070, in _dep_map\n",
            "    return self.__dep_map\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2863, in __getattr__\n",
            "    raise AttributeError(attr)\n",
            "AttributeError: _DistInfoDistribution__dep_map\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 179, in exc_logging_wrapper\n",
            "    status = run_func(*args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/req_command.py\", line 67, in wrapper\n",
            "    return func(self, options, args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/commands/install.py\", line 377, in run\n",
            "    requirement_set = resolver.resolve(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/resolution/resolvelib/resolver.py\", line 95, in resolve\n",
            "    result = self._result = resolver.resolve(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/resolvelib/resolvers.py\", line 546, in resolve\n",
            "    state = resolution.resolve(requirements, max_rounds=max_rounds)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/resolvelib/resolvers.py\", line 427, in resolve\n",
            "    failure_causes = self._attempt_to_pin_criterion(name)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/resolvelib/resolvers.py\", line 239, in _attempt_to_pin_criterion\n",
            "    criteria = self._get_updated_criteria(candidate)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/resolvelib/resolvers.py\", line 229, in _get_updated_criteria\n",
            "    for requirement in self._p.get_dependencies(candidate=candidate):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/resolution/resolvelib/provider.py\", line 247, in get_dependencies\n",
            "    return [r for r in candidate.iter_dependencies(with_requires) if r is not None]\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/resolution/resolvelib/provider.py\", line 247, in <listcomp>\n",
            "    return [r for r in candidate.iter_dependencies(with_requires) if r is not None]\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/resolution/resolvelib/candidates.py\", line 401, in iter_dependencies\n",
            "    for r in self.dist.iter_dependencies():\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/metadata/pkg_resources.py\", line 247, in iter_dependencies\n",
            "    return self._dist.requires(extras)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2786, in requires\n",
            "    dm = self._dep_map\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3072, in _dep_map\n",
            "    self.__dep_map = self._compute_dependencies()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3094, in _compute_dependencies\n",
            "    dm[s_extra] = [r for r in reqs_for_extra(extra) if r not in common]\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3094, in <listcomp>\n",
            "    dm[s_extra] = [r for r in reqs_for_extra(extra) if r not in common]\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3086, in reqs_for_extra\n",
            "    if not req.marker or req.marker.evaluate({'extra': extra}):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/packaging/markers.py\", line 325, in evaluate\n",
            "    return _evaluate_markers(self._markers, current_environment)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/packaging/markers.py\", line 208, in _evaluate_markers\n",
            "    assert isinstance(marker, (list, tuple, str))\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/pip3\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/main.py\", line 80, in main\n",
            "    return command.main(cmd_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 100, in main\n",
            "    return self._main(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 232, in _main\n",
            "    return run(options, args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 215, in exc_logging_wrapper\n",
            "    logger.critical(\"Operation cancelled by user\")\n",
            "  File \"/usr/lib/python3.10/logging/__init__.py\", line 1524, in critical\n",
            "    self._log(CRITICAL, msg, args, **kwargs)\n",
            "  File \"/usr/lib/python3.10/logging/__init__.py\", line 1600, in _log\n",
            "    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False,\n",
            "KeyboardInterrupt\n",
            "^C\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (2024.9.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets evaluate sacremoses transformers[sentencepiece]\n",
        "#I have to update fsspec because return an error\n",
        "!pip install fsspec"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "damBM287qgAI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Summarization"
      ],
      "metadata": {
        "id": "ERFaRmz-V0wI"
      }
    },
    {
      "source": [
        "!pip install datasets evaluate sacremoses transformers[sentencepiece]\n",
        "!pip install fsspec\n",
        "!pip install pypdf2\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import PyPDF2\n",
        "from transformers import pipeline\n",
        "\n",
        "pipe = pipeline(\"summarization\", model=\"sshleifer/distilbart-cnn-12-6\")\n",
        "\n",
        "#summarize the PDF file. extract the txt using using PyPDF2.\n",
        "#doesn't work with GEMMA2\n",
        "\n",
        "#select the \"article 6 BloombergGPT\" after upload it in the sample data and copy the path\n",
        "with open(\"/content/sample_data/Article 6 BloombergGPT_ A Large Language Model for Finance.pdf\", \"rb\") as f:\n",
        "    pdf_reader = PyPDF2.PdfReader(f)\n",
        "    text = \"\"\n",
        "    for page in pdf_reader.pages:\n",
        "        text += page.extract_text()"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "OouJHopUolS8",
        "outputId": "d7f14f8e-2b96-4f20-af66-6d208013f221",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (3.1.0)\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (0.4.3)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.10/dist-packages (0.1.1)\n",
            "Requirement already satisfied: transformers[sentencepiece] in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.16.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.26.4)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n",
            "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.6)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.5.0)\n",
            "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.10.10)\n",
            "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.24.7)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.2)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacremoses) (2024.9.11)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from sacremoses) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from sacremoses) (1.4.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.19.1)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (3.20.3)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.2.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.17.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.23.0->datasets) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (2024.8.30)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (2024.9.0)\n",
            "Requirement already satisfied: pypdf2 in /usr/local/lib/python3.10/dist-packages (3.0.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#I tried to use a model GEMMA-2 but it is too heavy for the colab account (base) so I make a summerization with LMStudio using Llama-3-2 with this query\n",
        "#\"you are an associate professor for a PhD classroom. Generate 10 chapters about the content of the article\"\n",
        "\n",
        "max_input_length = 1024\n",
        "truncated_text = text[:max_input_length]\n",
        "\n",
        "summary = pipe(truncated_text, max_length=300, min_length=30, do_sample=False, truncation=True)\n",
        "# Added truncation=True to handle long inputs\n",
        "\n",
        "print(summary[0]['summary_text']) # Access the summary text"
      ],
      "metadata": {
        "id": "02K7KyfMpC9J",
        "outputId": "fdc9cf2e-aa72-47f0-8940-5b62f57162fc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Your max_length is set to 300, but your input_length is only 267. Since this is a summarization task, where outputs shorter than the input are typically wanted, you might consider decreasing max_length manually, e.g. summarizer('...', max_length=133)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " The use of NLP in the realm of financial technology is broad and complex, with applications ranging from sentiment analysis and named entity recognition to question answering . In this work, we purposefully present BloombergGPT, a 50 billion parameter language model that is trained on a wide range of  nancial data .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Types of Question Answering"
      ],
      "metadata": {
        "id": "gyFaidWaWQUS"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "QA models come in different forms based on their inputs and outputs.\n",
        "\n",
        "* **Extractive QA** is one variant where the model extracts the answer from a given context, such as text, a table, or HTML, typically using BERT-like models;\n",
        "* **Open Generative QA** is another form where the model freely generates text based on the provided context;\n",
        "* **Closed Generative QA** does not rely on any provided context; instead, the model completely generates the answer.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "lv5qXiWDWTP6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extractive Question Answering\n",
        "#I tried to use a model GEMMA-2 but it is too heavy for the colab account (base) so I make a summerization with LMStudio using Llama-3-2 with this query\n",
        "\n",
        "###\"Generate 10 chapters about the content of the article\"\n"
      ],
      "metadata": {
        "id": "uq_k45aQZ7Kz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#############\n",
        "I made a summary of the pdf with another AI and create 10 chapters, then I create different questions from every chapter and test the correct answer.\n",
        "#############"
      ],
      "metadata": {
        "id": "JZ-ARZKiC3Sq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#These are the 10 questions created and tested individually. TINYROBERTA answers succinctly. FLAN-T5 does not seem to respond correctly.\n",
        "\n",
        "question = \"Question: How many parameters does BloombergGPT contain?\"\n",
        "#\"What contribution does BloombergGPT make to the field of NLP?\"\n",
        "#\"What ethical considerations are discussed in the article regarding BloombergGPT?\"\n",
        "#\"How does BloombergGPT's performance compare to existing financial models?\"\n",
        "#\"Which financial NLP tasks were used to evaluate BloombergGPT's performance?\"\n",
        "#\"What techniques were implemented to enhance the efficiency of BloombergGPT's training?\"\n",
        "#What type of architecture does BloombergGPT utilize?\"\n",
        "#How many tokens does the FinPile dataset comprise?\"\n",
        "#What is the name of the training dataset used for BloombergGPT?\"\n",
        "#How many parameters does BloombergGPT contain?\"\n",
        "#What is the primary purpose of BloombergGPT?\"\n",
        "\n",
        "model = \"deepset/tinyroberta-squad2\"\n",
        "\n",
        "context = \"\"\"This article presents BloombergGPT, a specialized large language model (LLM) designed to cater to the financial domain. Developed by Bloomberg, it consists of 50 billion parameters and is trained on a vast dataset comprising financial data and general knowledge, amounting to over 700 billion tokens.\n",
        "\n",
        "Introduction\n",
        "The emergence of large-scale LLMs has transformed the natural language processing (NLP) landscape. While existing models like GPT-3 have demonstrated remarkable capabilities across various domains, there is a lack of specialized models for finance. BloombergGPT aims to fill this gap, leveraging Bloomberg's extensive financial data resources.\n",
        "\n",
        "Dataset\n",
        "The training dataset, referred to as FinPile, is comprised of 363 billion tokens from diverse financial documents, news articles, and web sources. Additionally, 345 billion tokens from public datasets were incorporated to enhance the model's generalization capabilities. This unique blend allows BloombergGPT to excel in financial tasks while maintaining strong performance on general NLP benchmarks.\n",
        "\n",
        "Model Architecture\n",
        "BloombergGPT employs a decoder-only transformer architecture with 70 layers, designed to effectively process and generate sequential data. The model utilizes multi-head self-attention mechanisms to capture contextual relationships within the text, which is critical for understanding complex financial documents.\n",
        "\n",
        "Training Methodology\n",
        "The training process involved optimizing the model using a mixed approach, combining domain-specific and general data. Various techniques, such as mixed precision training and activation checkpointing, were implemented to enhance efficiency and manage the large memory requirements associated with training such a substantial model.\n",
        "\n",
        "Evaluation\n",
        "The performance of BloombergGPT was assessed using a range of financial and general NLP tasks, including sentiment analysis, named entity recognition, and question answering. The model outperformed existing financial models, showcasing its effectiveness in handling nuanced financial language and reasoning tasks.\n",
        "\n",
        "Results\n",
        "BloombergGPT demonstrated superior accuracy and better contextual understanding in financial tasks compared to its peers. The evaluations highlighted its capability to adapt to various scenarios and provide valuable insights into financial data, reinforcing the need for specialized models in the financial sector.\n",
        "\n",
        "Ethical Considerations\n",
        "The article also addresses the ethical implications of deploying large language models like BloombergGPT in finance. It emphasizes the importance of responsible use, mitigating biases present in training data, and ensuring transparency in decision-making processes influenced by these models.\n",
        "\n",
        "Conclusion\n",
        "In conclusion, BloombergGPT represents a significant advancement in the development of LLMs for finance, providing a robust tool for analyzing and understanding financial texts. Its unique training methodology and architecture highlight the potential of specialized models to outperform general-purpose LLMs in domain-specific tasks.\n",
        "\n",
        "The findings from this research contribute to the ongoing discourse in the NLP community about the benefits and challenges of developing tailored language models for specialized domains.\"\"\""
      ],
      "metadata": {
        "id": "UHuBLSgPlQtF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "question = \"Question: How many parameters does BloombergGPT contain?\"\n",
        "#\"What contribution does BloombergGPT make to the field of NLP?\"\n",
        "#\"What ethical considerations are discussed in the article regarding BloombergGPT?\"\n",
        "#\"How does BloombergGPT's performance compare to existing financial models?\"\n",
        "#\"Which financial NLP tasks were used to evaluate BloombergGPT's performance?\"\n",
        "#\"What techniques were implemented to enhance the efficiency of BloombergGPT's training?\"\n",
        "#What type of architecture does BloombergGPT utilize?\"\n",
        "#How many tokens does the FinPile dataset comprise?\"\n",
        "#What is the name of the training dataset used for BloombergGPT?\"\n",
        "#How many parameters does BloombergGPT contain?\"\n",
        "#What is the primary purpose of BloombergGPT?\"\n",
        "\n",
        "model = \"google/flan-t5-base\"\n",
        "\n",
        "context = \"\"\"This article presents BloombergGPT, a specialized large language model (LLM) designed to cater to the financial domain. Developed by Bloomberg, it consists of 50 billion parameters and is trained on a vast dataset comprising financial data and general knowledge, amounting to over 700 billion tokens.\n",
        "\n",
        "Introduction\n",
        "The emergence of large-scale LLMs has transformed the natural language processing (NLP) landscape. While existing models like GPT-3 have demonstrated remarkable capabilities across various domains, there is a lack of specialized models for finance. BloombergGPT aims to fill this gap, leveraging Bloomberg's extensive financial data resources.\n",
        "\n",
        "Dataset\n",
        "The training dataset, referred to as FinPile, is comprised of 363 billion tokens from diverse financial documents, news articles, and web sources. Additionally, 345 billion tokens from public datasets were incorporated to enhance the model's generalization capabilities. This unique blend allows BloombergGPT to excel in financial tasks while maintaining strong performance on general NLP benchmarks.\n",
        "\n",
        "Model Architecture\n",
        "BloombergGPT employs a decoder-only transformer architecture with 70 layers, designed to effectively process and generate sequential data. The model utilizes multi-head self-attention mechanisms to capture contextual relationships within the text, which is critical for understanding complex financial documents.\n",
        "\n",
        "Training Methodology\n",
        "The training process involved optimizing the model using a mixed approach, combining domain-specific and general data. Various techniques, such as mixed precision training and activation checkpointing, were implemented to enhance efficiency and manage the large memory requirements associated with training such a substantial model.\n",
        "\n",
        "Evaluation\n",
        "The performance of BloombergGPT was assessed using a range of financial and general NLP tasks, including sentiment analysis, named entity recognition, and question answering. The model outperformed existing financial models, showcasing its effectiveness in handling nuanced financial language and reasoning tasks.\n",
        "\n",
        "Results\n",
        "BloombergGPT demonstrated superior accuracy and better contextual understanding in financial tasks compared to its peers. The evaluations highlighted its capability to adapt to various scenarios and provide valuable insights into financial data, reinforcing the need for specialized models in the financial sector.\n",
        "\n",
        "Ethical Considerations\n",
        "The article also addresses the ethical implications of deploying large language models like BloombergGPT in finance. It emphasizes the importance of responsible use, mitigating biases present in training data, and ensuring transparency in decision-making processes influenced by these models.\n",
        "\n",
        "Conclusion\n",
        "In conclusion, BloombergGPT represents a significant advancement in the development of LLMs for finance, providing a robust tool for analyzing and understanding financial texts. Its unique training methodology and architecture highlight the potential of specialized models to outperform general-purpose LLMs in domain-specific tasks.\n",
        "\n",
        "The findings from this research contribute to the ongoing discourse in the NLP community about the benefits and challenges of developing tailored language models for specialized domains.\"\"\""
      ],
      "metadata": {
        "id": "QOh1cgSMrL-c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### ANSWER PIPELINES"
      ],
      "metadata": {
        "id": "3nyUdLjOmvg1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers"
      ],
      "metadata": {
        "id": "wA6Q2G3-lRfP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "qa_pipeline = pipeline(\"question-answering\")#, model)\n",
        "answer = qa_pipeline(question = question, context = context)"
      ],
      "metadata": {
        "id": "b1z6iEzvkVkq",
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd84e495-5d88-4e78-b8de-bcc1cef18d87"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to distilbert/distilbert-base-cased-distilled-squad and revision 626af31 (https://huggingface.co/distilbert/distilbert-base-cased-distilled-squad).\n",
            "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**#### ANSWERING to the Question**"
      ],
      "metadata": {
        "id": "3HpaHx1bs4th"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "answer"
      ],
      "metadata": {
        "id": "p1DVBzb7isRr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a86976cf-0d08-4665-9220-cbb6bd7600d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'score': 0.7990682721138, 'start': 159, 'end': 169, 'answer': '50 billion'}"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### ANSWERING to the Question\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "fM-hfwojnfnA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
        "tokenizer = AutoTokenizer.from_pretrained(model)"
      ],
      "metadata": {
        "id": "4ShKAC6un063"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_input_question = tokenizer(question)\n",
        "tokens_question = tokenizer.convert_ids_to_tokens(encoded_input_question.input_ids)\n",
        "print(\"Word Tokens Question:\", tokens_question)\n",
        "\n",
        "encoded_input_context = tokenizer(context)\n",
        "tokens_context = tokenizer.convert_ids_to_tokens(encoded_input_context.input_ids)\n",
        "print(\"Word Tokens Context:\", tokens_context)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P9Nu58s-poiR",
        "outputId": "505bbe46-2ba8-496e-9105-87e2154d6230"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (570 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Word Tokens Question: ['<s>', 'Question', ':', 'ĠHow', 'Ġmany', 'Ġparameters', 'Ġdoes', 'ĠBloomberg', 'G', 'PT', 'Ġcontain', '?', '</s>']\n",
            "Word Tokens Context: ['<s>', 'This', 'Ġarticle', 'Ġpresents', 'ĠBloomberg', 'G', 'PT', ',', 'Ġa', 'Ġspecialized', 'Ġlarge', 'Ġlanguage', 'Ġmodel', 'Ġ(', 'LL', 'M', ')', 'Ġdesigned', 'Ġto', 'Ġcater', 'Ġto', 'Ġthe', 'Ġfinancial', 'Ġdomain', '.', 'ĠDevelop', 'ed', 'Ġby', 'ĠBloomberg', ',', 'Ġit', 'Ġconsists', 'Ġof', 'Ġ50', 'Ġbillion', 'Ġparameters', 'Ġand', 'Ġis', 'Ġtrained', 'Ġon', 'Ġa', 'Ġvast', 'Ġdataset', 'Ġcomprising', 'Ġfinancial', 'Ġdata', 'Ġand', 'Ġgeneral', 'Ġknowledge', ',', 'Ġamount', 'ing', 'Ġto', 'Ġover', 'Ġ700', 'Ġbillion', 'Ġtokens', '.', 'Ċ', 'Ċ', 'Introduction', 'Ċ', 'The', 'Ġemergence', 'Ġof', 'Ġlarge', '-', 'scale', 'ĠLL', 'Ms', 'Ġhas', 'Ġtransformed', 'Ġthe', 'Ġnatural', 'Ġlanguage', 'Ġprocessing', 'Ġ(', 'N', 'LP', ')', 'Ġlandscape', '.', 'ĠWhile', 'Ġexisting', 'Ġmodels', 'Ġlike', 'ĠG', 'PT', '-', '3', 'Ġhave', 'Ġdemonstrated', 'Ġremarkable', 'Ġcapabilities', 'Ġacross', 'Ġvarious', 'Ġdomains', ',', 'Ġthere', 'Ġis', 'Ġa', 'Ġlack', 'Ġof', 'Ġspecialized', 'Ġmodels', 'Ġfor', 'Ġfinance', '.', 'ĠBloomberg', 'G', 'PT', 'Ġaims', 'Ġto', 'Ġfill', 'Ġthis', 'Ġgap', ',', 'Ġleveraging', 'ĠBloomberg', \"'s\", 'Ġextensive', 'Ġfinancial', 'Ġdata', 'Ġresources', '.', 'Ċ', 'Ċ', 'Dat', 'as', 'et', 'Ċ', 'The', 'Ġtraining', 'Ġdataset', ',', 'Ġreferred', 'Ġto', 'Ġas', 'ĠFin', 'P', 'ile', ',', 'Ġis', 'Ġcomprised', 'Ġof', 'Ġ363', 'Ġbillion', 'Ġtokens', 'Ġfrom', 'Ġdiverse', 'Ġfinancial', 'Ġdocuments', ',', 'Ġnews', 'Ġarticles', ',', 'Ġand', 'Ġweb', 'Ġsources', '.', 'ĠAdditionally', ',', 'Ġ345', 'Ġbillion', 'Ġtokens', 'Ġfrom', 'Ġpublic', 'Ġdatasets', 'Ġwere', 'Ġincorporated', 'Ġto', 'Ġenhance', 'Ġthe', 'Ġmodel', \"'s\", 'Ġgeneral', 'ization', 'Ġcapabilities', '.', 'ĠThis', 'Ġunique', 'Ġblend', 'Ġallows', 'ĠBloomberg', 'G', 'PT', 'Ġto', 'Ġexcel', 'Ġin', 'Ġfinancial', 'Ġtasks', 'Ġwhile', 'Ġmaintaining', 'Ġstrong', 'Ġperformance', 'Ġon', 'Ġgeneral', 'ĠN', 'LP', 'Ġbenchmarks', '.', 'Ċ', 'Ċ', 'Model', 'ĠArchitecture', 'Ċ', 'Bloomberg', 'G', 'PT', 'Ġemploys', 'Ġa', 'Ġdec', 'oder', '-', 'only', 'Ġtransformer', 'Ġarchitecture', 'Ġwith', 'Ġ70', 'Ġlayers', ',', 'Ġdesigned', 'Ġto', 'Ġeffectively', 'Ġprocess', 'Ġand', 'Ġgenerate', 'Ġsequential', 'Ġdata', '.', 'ĠThe', 'Ġmodel', 'Ġutilizes', 'Ġmulti', '-', 'head', 'Ġself', '-', 'att', 'ention', 'Ġmechanisms', 'Ġto', 'Ġcapture', 'Ġcontextual', 'Ġrelationships', 'Ġwithin', 'Ġthe', 'Ġtext', ',', 'Ġwhich', 'Ġis', 'Ġcritical', 'Ġfor', 'Ġunderstanding', 'Ġcomplex', 'Ġfinancial', 'Ġdocuments', '.', 'Ċ', 'Ċ', 'Training', 'ĠMethod', 'ology', 'Ċ', 'The', 'Ġtraining', 'Ġprocess', 'Ġinvolved', 'Ġoptimizing', 'Ġthe', 'Ġmodel', 'Ġusing', 'Ġa', 'Ġmixed', 'Ġapproach', ',', 'Ġcombining', 'Ġdomain', '-', 'specific', 'Ġand', 'Ġgeneral', 'Ġdata', '.', 'ĠVarious', 'Ġtechniques', ',', 'Ġsuch', 'Ġas', 'Ġmixed', 'Ġprecision', 'Ġtraining', 'Ġand', 'Ġactivation', 'Ġcheckpoint', 'ing', ',', 'Ġwere', 'Ġimplemented', 'Ġto', 'Ġenhance', 'Ġefficiency', 'Ġand', 'Ġmanage', 'Ġthe', 'Ġlarge', 'Ġmemory', 'Ġrequirements', 'Ġassociated', 'Ġwith', 'Ġtraining', 'Ġsuch', 'Ġa', 'Ġsubstantial', 'Ġmodel', '.', 'Ċ', 'Ċ', 'E', 'val', 'uation', 'Ċ', 'The', 'Ġperformance', 'Ġof', 'ĠBloomberg', 'G', 'PT', 'Ġwas', 'Ġassessed', 'Ġusing', 'Ġa', 'Ġrange', 'Ġof', 'Ġfinancial', 'Ġand', 'Ġgeneral', 'ĠN', 'LP', 'Ġtasks', ',', 'Ġincluding', 'Ġsentiment', 'Ġanalysis', ',', 'Ġnamed', 'Ġentity', 'Ġrecognition', ',', 'Ġand', 'Ġquestion', 'Ġanswering', '.', 'ĠThe', 'Ġmodel', 'Ġoutper', 'formed', 'Ġexisting', 'Ġfinancial', 'Ġmodels', ',', 'Ġshowcasing', 'Ġits', 'Ġeffectiveness', 'Ġin', 'Ġhandling', 'Ġnuanced', 'Ġfinancial', 'Ġlanguage', 'Ġand', 'Ġreasoning', 'Ġtasks', '.', 'Ċ', 'Ċ', 'Results', 'Ċ', 'Bloomberg', 'G', 'PT', 'Ġdemonstrated', 'Ġsuperior', 'Ġaccuracy', 'Ġand', 'Ġbetter', 'Ġcontextual', 'Ġunderstanding', 'Ġin', 'Ġfinancial', 'Ġtasks', 'Ġcompared', 'Ġto', 'Ġits', 'Ġpeers', '.', 'ĠThe', 'Ġevaluations', 'Ġhighlighted', 'Ġits', 'Ġcapability', 'Ġto', 'Ġadapt', 'Ġto', 'Ġvarious', 'Ġscenarios', 'Ġand', 'Ġprovide', 'Ġvaluable', 'Ġinsights', 'Ġinto', 'Ġfinancial', 'Ġdata', ',', 'Ġreinforcing', 'Ġthe', 'Ġneed', 'Ġfor', 'Ġspecialized', 'Ġmodels', 'Ġin', 'Ġthe', 'Ġfinancial', 'Ġsector', '.', 'Ċ', 'Ċ', 'Eth', 'ical', 'ĠConsider', 'ations', 'Ċ', 'The', 'Ġarticle', 'Ġalso', 'Ġaddresses', 'Ġthe', 'Ġethical', 'Ġimplications', 'Ġof', 'Ġdeploying', 'Ġlarge', 'Ġlanguage', 'Ġmodels', 'Ġlike', 'ĠBloomberg', 'G', 'PT', 'Ġin', 'Ġfinance', '.', 'ĠIt', 'Ġemphasizes', 'Ġthe', 'Ġimportance', 'Ġof', 'Ġresponsible', 'Ġuse', ',', 'Ġmitigating', 'Ġbiases', 'Ġpresent', 'Ġin', 'Ġtraining', 'Ġdata', ',', 'Ġand', 'Ġensuring', 'Ġtransparency', 'Ġin', 'Ġdecision', '-', 'making', 'Ġprocesses', 'Ġinfluenced', 'Ġby', 'Ġthese', 'Ġmodels', '.', 'Ċ', 'Ċ', 'Conclusion', 'Ċ', 'In', 'Ġconclusion', ',', 'ĠBloomberg', 'G', 'PT', 'Ġrepresents', 'Ġa', 'Ġsignificant', 'Ġadvancement', 'Ġin', 'Ġthe', 'Ġdevelopment', 'Ġof', 'ĠLL', 'Ms', 'Ġfor', 'Ġfinance', ',', 'Ġproviding', 'Ġa', 'Ġrobust', 'Ġtool', 'Ġfor', 'Ġanalyzing', 'Ġand', 'Ġunderstanding', 'Ġfinancial', 'Ġtexts', '.', 'ĠIts', 'Ġunique', 'Ġtraining', 'Ġmethodology', 'Ġand', 'Ġarchitecture', 'Ġhighlight', 'Ġthe', 'Ġpotential', 'Ġof', 'Ġspecialized', 'Ġmodels', 'Ġto', 'Ġoutper', 'form', 'Ġgeneral', '-', 'purpose', 'ĠLL', 'Ms', 'Ġin', 'Ġdomain', '-', 'specific', 'Ġtasks', '.', 'Ċ', 'Ċ', 'The', 'Ġfindings', 'Ġfrom', 'Ġthis', 'Ġresearch', 'Ġcontribute', 'Ġto', 'Ġthe', 'Ġongoing', 'Ġdiscourse', 'Ġin', 'Ġthe', 'ĠN', 'LP', 'Ġcommunity', 'Ġabout', 'Ġthe', 'Ġbenefits', 'Ġand', 'Ġchallenges', 'Ġof', 'Ġdeveloping', 'Ġtailored', 'Ġlanguage', 'Ġmodels', 'Ġfor', 'Ġspecialized', 'Ġdomains', '.', '</s>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the model name or path\n",
        "model = \"deepset/roberta-base-squad2\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model, model_max_length=512, truncation=True)\n",
        "qa_model = AutoModelForQuestionAnswering.from_pretrained(model)"
      ],
      "metadata": {
        "id": "ezDigYSkpD29"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The second model is GOOGLE/FLAN-T5-BASE**"
      ],
      "metadata": {
        "id": "e2LOLikRseCZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the model name or path\n",
        "model = \"google/flan-t5-base\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model, model_max_length=512, truncation=True)\n",
        "qa_model = AutoModelForQuestionAnswering.from_pretrained(model)"
      ],
      "metadata": {
        "id": "BiBAaSPxrenB",
        "outputId": "8777d307-8fc7-48c1-83be-b66222bd3044",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of T5ForQuestionAnswering were not initialized from the model checkpoint at google/flan-t5-base and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_input = tokenizer(question, context, return_tensors='pt', truncation=True, max_length=512)\n",
        "outputs = qa_model(**encoded_input)"
      ],
      "metadata": {
        "id": "WntYWWU7paoT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answer_start_index = outputs.start_logits.argmax()\n",
        "answer_end_index = outputs.end_logits.argmax()\n",
        "\n",
        "predict_answer_tokens = encoded_input.input_ids[0, answer_start_index : answer_end_index + 1]\n",
        "\n",
        "answer = tokenizer.decode(predict_answer_tokens, skip_special_tokens=True)"
      ],
      "metadata": {
        "id": "dzbCjeHvpxIu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(question)\n",
        "print(\"---\")\n",
        "print('The answer is: '+answer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C-dUNIYdZLsP",
        "outputId": "2ab5c267-e9dd-4823-b32b-9e78fce03e72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: How many parameters does BloombergGPT contain?\n",
            "---\n",
            "The answer is:  50 billion\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "uaing \"Flan-t5-base\" the answer is more complex but"
      ],
      "metadata": {
        "id": "03YUyxVNtWog"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generative Question Answering\n"
      ],
      "metadata": {
        "id": "9LQtOSPE3zRt"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Encoder-only models like BERT and RoBERTa are good at extracting answers from a given context. Nevertheless, they are not able to generate answers.  For Generative QA, encoder-decoder models such as T5 are often used, as they can synthesize information similar to text summarization.\n",
        "\n",
        "Since the task of Generative QA is very complex, we illustrate the example of generating questions given a certain answer."
      ],
      "metadata": {
        "id": "Pj62de0uZuME"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Question Generation"
      ],
      "metadata": {
        "id": "51YfClQmaJas"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "question_generation_pipeline = pipeline(\"text2text-generation\", model=\"deepset/tinyroberta-squad2\" )"
      ],
      "metadata": {
        "id": "flanO2YSqk2e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "outputId": "790b45a3-ab7a-408f-ac56-f9cea9d8cd89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The model 'RobertaForQuestionAnswering' is not supported for text2text-generation. Supported models are ['BartForConditionalGeneration', 'BigBirdPegasusForConditionalGeneration', 'BlenderbotForConditionalGeneration', 'BlenderbotSmallForConditionalGeneration', 'EncoderDecoderModel', 'FSMTForConditionalGeneration', 'GPTSanJapaneseForConditionalGeneration', 'LEDForConditionalGeneration', 'LongT5ForConditionalGeneration', 'M2M100ForConditionalGeneration', 'MarianMTModel', 'MBartForConditionalGeneration', 'MT5ForConditionalGeneration', 'MvpForConditionalGeneration', 'NllbMoeForConditionalGeneration', 'PegasusForConditionalGeneration', 'PegasusXForConditionalGeneration', 'PLBartForConditionalGeneration', 'ProphetNetForConditionalGeneration', 'SeamlessM4TForTextToText', 'SeamlessM4Tv2ForTextToText', 'SwitchTransformersForConditionalGeneration', 'T5ForConditionalGeneration', 'UMT5ForConditionalGeneration', 'XLMProphetNetForConditionalGeneration'].\n"
          ]
        }
      ]
    },
    {
      "source": [
        "!pip install transformers\n",
        "\n",
        "from transformers import pipeline\n",
        "\n",
        "# Use a model designed for text generation, such as 'deepset/roberta-base-squad2'\n",
        "question_generation_pipeline = pipeline(\"text2text-generation\", model=\"deepset/roberta-base-squad2\")\n",
        "\n",
        "# Provide a clear instruction for the model and define placeholders for the context\n",
        "generated_question = question_generation_pipeline(\n",
        "    \"generate question: context: {context} answer: <hl> {answer} <hl>\",\n",
        "    context=context,\n",
        "    answer=\"the answer you want to generate a question for\"\n",
        ")\n",
        "\n",
        "print(generated_question)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "F2jgOvybr4Pu",
        "outputId": "41e8e656-1b03-47ca-f265-0058c3c9b967",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 688
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.7)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.6)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.9.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The model 'RobertaForQuestionAnswering' is not supported for text2text-generation. Supported models are ['BartForConditionalGeneration', 'BigBirdPegasusForConditionalGeneration', 'BlenderbotForConditionalGeneration', 'BlenderbotSmallForConditionalGeneration', 'EncoderDecoderModel', 'FSMTForConditionalGeneration', 'GPTSanJapaneseForConditionalGeneration', 'LEDForConditionalGeneration', 'LongT5ForConditionalGeneration', 'M2M100ForConditionalGeneration', 'MarianMTModel', 'MBartForConditionalGeneration', 'MT5ForConditionalGeneration', 'MvpForConditionalGeneration', 'NllbMoeForConditionalGeneration', 'PegasusForConditionalGeneration', 'PegasusXForConditionalGeneration', 'PLBartForConditionalGeneration', 'ProphetNetForConditionalGeneration', 'SeamlessM4TForTextToText', 'SeamlessM4Tv2ForTextToText', 'SwitchTransformersForConditionalGeneration', 'T5ForConditionalGeneration', 'UMT5ForConditionalGeneration', 'XLMProphetNetForConditionalGeneration'].\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "The current model class (RobertaForQuestionAnswering) is not compatible with `.generate()`, as it doesn't have a language model head. Please use one of the following classes instead: {'RobertaForCausalLM'}",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-99-7a51bf16dcd2>\u001b[0m in \u001b[0;36m<cell line: 9>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m# Provide a clear instruction for the model and define placeholders for the context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m generated_question = question_generation_pipeline(\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0;34m\"generate question: context: {context} answer: <hl> {answer} <hl>\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mcontext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text2text_generation.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    165\u001b[0m         \"\"\"\n\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m         if (\n\u001b[1;32m    169\u001b[0m             \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, num_workers, batch_size, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1255\u001b[0m             )\n\u001b[1;32m   1256\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1257\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_single\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocess_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpostprocess_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1259\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrun_multi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocess_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpostprocess_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\u001b[0m in \u001b[0;36mrun_single\u001b[0;34m(self, inputs, preprocess_params, forward_params, postprocess_params)\u001b[0m\n\u001b[1;32m   1262\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrun_single\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocess_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpostprocess_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1263\u001b[0m         \u001b[0mmodel_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpreprocess_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1264\u001b[0;31m         \u001b[0mmodel_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mforward_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1265\u001b[0m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpostprocess_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1266\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, model_inputs, **forward_params)\u001b[0m\n\u001b[1;32m   1162\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0minference_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1163\u001b[0m                     \u001b[0mmodel_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_tensor_on_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1164\u001b[0;31m                     \u001b[0mmodel_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mforward_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1165\u001b[0m                     \u001b[0mmodel_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_tensor_on_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1166\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text2text_generation.py\u001b[0m in \u001b[0;36m_forward\u001b[0;34m(self, model_inputs, **generate_kwargs)\u001b[0m\n\u001b[1;32m    189\u001b[0m             \u001b[0mgenerate_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"max_length\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         )\n\u001b[0;32m--> 191\u001b[0;31m         \u001b[0moutput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mgenerate_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m         \u001b[0mout_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput_ids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"pt\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    114\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mctx_factory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m   1684\u001b[0m         \"\"\"\n\u001b[1;32m   1685\u001b[0m         \u001b[0;31m# 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1686\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_model_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1687\u001b[0m         \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"tokenizer\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Pull this out first, we only use it for stopping criteria\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1688\u001b[0m         \u001b[0mgeneration_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prepare_generation_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgeneration_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\u001b[0m in \u001b[0;36m_validate_model_class\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1165\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mgenerate_compatible_classes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1166\u001b[0m                 \u001b[0mexception_message\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34mf\" Please use one of the following classes instead: {generate_compatible_classes}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1167\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexception_message\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1169\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_validate_assistant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0massistant_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: The current model class (RobertaForQuestionAnswering) is not compatible with `.generate()`, as it doesn't have a language model head. Please use one of the following classes instead: {'RobertaForCausalLM'}"
          ]
        }
      ]
    },
    {
      "source": [
        "!pip install transformers\n",
        "\n",
        "from transformers import pipeline\n",
        "\n",
        "# Use a model that is capable of text generation, such as 'text2text-generation'.\n",
        "# The previous model 'deepset/tinyroberta-squad2' is for question answering, not generation.\n",
        "question_generation_pipeline = pipeline(\"text2text-generation\", model=\"google/flan-t5-base\")\n",
        "\n",
        "# Provide the context and desired answer within the input text.\n",
        "generated_question = question_generation_pipeline(\n",
        "    f\"generate question: context: {context} <hl> _ <hl>\"\n",
        ")\n",
        "\n",
        "print(generated_question)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "_ZiJdSNrfenV",
        "outputId": "24089acb-fa70-42fe-bc09-8415a965e510",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.7)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.6)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.9.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (620 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'generated_text': 'What is BloombergGPT?'}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generated_question"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6nidIbBD5Pwt",
        "outputId": "f1cdb5d5-4706-4310-d34c-5bd0da556dc9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': 'What is BloombergGPT?'}]"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    }
  ]
}