Hosting_LLM_as_an_API.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Project: Hosting LLAMA model as an api using Ngrok and Fast API"
      ],
      "metadata": {
        "id": "q4VWRCpOakw7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Step 1 : Installing dependencies for the project as follows\n",
        "\n",
        "1. Dependencies for FastAPI\n",
        "2. Installing Ngrok to make a local server be publicly accessible\n",
        "3. Dependencies required to download any model from hugging face that we wanted to create the api for ; so api requests can be used to make predictions or classifications or chat completions etc."
      ],
      "metadata": {
        "id": "cpzDlO3da7Li"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lBvVwwkbaciV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c5dfd110-6e97-4428-a675-de623a8d0976"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting llama-cpp-python\n",
            "  Downloading llama_cpp_python-0.2.90.tar.gz (63.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.8/63.8 MB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (4.12.2)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (1.26.4)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: jinja2>=2.11.3 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (3.1.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.3->llama-cpp-python) (2.1.5)\n",
            "Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: llama-cpp-python\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.90-cp310-cp310-linux_x86_64.whl size=3414591 sha256=2651b865ee7b3a4ee8116141d0b46c79ebe3edda93b10cf5b99a90243fdf78be\n",
            "  Stored in directory: /root/.cache/pip/wheels/3d/67/02/f950031435db4a5a02e6269f6adb6703bf1631c3616380f3c6\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: diskcache, llama-cpp-python\n",
            "Successfully installed diskcache-5.6.3 llama-cpp-python-0.2.90\n",
            "Collecting pyngrok\n",
            "  Downloading pyngrok-7.2.0-py3-none-any.whl.metadata (7.4 kB)\n",
            "Collecting uvicorn\n",
            "  Downloading uvicorn-0.30.6-py3-none-any.whl.metadata (6.6 kB)\n",
            "Collecting python-multipart\n",
            "  Downloading python_multipart-0.0.9-py3-none-any.whl.metadata (2.5 kB)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (2.9.1)\n",
            "Requirement already satisfied: tensorflow in /usr/local/lib/python3.10/dist-packages (2.17.0)\n",
            "Collecting fastapi[all]\n",
            "  Downloading fastapi-0.114.1-py3-none-any.whl.metadata (27 kB)\n",
            "Requirement already satisfied: PyYAML>=5.1 in /usr/local/lib/python3.10/dist-packages (from pyngrok) (6.0.2)\n",
            "Collecting starlette<0.39.0,>=0.37.2 (from fastapi[all])\n",
            "  Downloading starlette-0.38.5-py3-none-any.whl.metadata (6.0 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from fastapi[all]) (4.12.2)\n",
            "Collecting fastapi-cli>=0.0.5 (from fastapi-cli[standard]>=0.0.5; extra == \"all\"->fastapi[all])\n",
            "  Downloading fastapi_cli-0.0.5-py3-none-any.whl.metadata (7.0 kB)\n",
            "Collecting httpx>=0.23.0 (from fastapi[all])\n",
            "  Downloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Requirement already satisfied: jinja2>=2.11.2 in /usr/local/lib/python3.10/dist-packages (from fastapi[all]) (3.1.4)\n",
            "Requirement already satisfied: itsdangerous>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from fastapi[all]) (2.2.0)\n",
            "Collecting ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 (from fastapi[all])\n",
            "  Downloading ujson-5.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (9.3 kB)\n",
            "Collecting orjson>=3.2.1 (from fastapi[all])\n",
            "  Downloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting email-validator>=2.0.0 (from fastapi[all])\n",
            "  Downloading email_validator-2.2.0-py3-none-any.whl.metadata (25 kB)\n",
            "Collecting pydantic-settings>=2.0.0 (from fastapi[all])\n",
            "  Downloading pydantic_settings-2.5.2-py3-none-any.whl.metadata (3.5 kB)\n",
            "Collecting pydantic-extra-types>=2.0.0 (from fastapi[all])\n",
            "  Downloading pydantic_extra_types-2.9.0-py3-none-any.whl.metadata (3.4 kB)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn) (8.1.7)\n",
            "Collecting h11>=0.8 (from uvicorn)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.6)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.5.15)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.5)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.3 in /usr/local/lib/python3.10/dist-packages (from pydantic) (2.23.3)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=24.3.25 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (24.3.25)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.6.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: h5py>=3.10.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.11.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (18.1.1)\n",
            "Requirement already satisfied: ml-dtypes<0.5.0,>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.4.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.3.0)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow) (71.0.4)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.16.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.4.0)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.16.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.64.1)\n",
            "Requirement already satisfied: tensorboard<2.18,>=2.17 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.17.0)\n",
            "Requirement already satisfied: keras>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.4.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.37.1)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow) (0.44.0)\n",
            "Collecting dnspython>=2.0.0 (from email-validator>=2.0.0->fastapi[all])\n",
            "  Downloading dnspython-2.6.1-py3-none-any.whl.metadata (5.8 kB)\n",
            "Requirement already satisfied: idna>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from email-validator>=2.0.0->fastapi[all]) (3.8)\n",
            "Requirement already satisfied: typer>=0.12.3 in /usr/local/lib/python3.10/dist-packages (from fastapi-cli>=0.0.5->fastapi-cli[standard]>=0.0.5; extra == \"all\"->fastapi[all]) (0.12.5)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi[all]) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi[all]) (2024.8.30)\n",
            "Collecting httpcore==1.* (from httpx>=0.23.0->fastapi[all])\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi[all]) (1.3.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.6.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.2->fastapi[all]) (2.1.5)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow) (13.8.1)\n",
            "Requirement already satisfied: namex in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow) (0.0.8)\n",
            "Requirement already satisfied: optree in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow) (0.12.1)\n",
            "Collecting python-dotenv>=0.21.0 (from pydantic-settings>=2.0.0->fastapi[all])\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow) (3.7)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow) (3.0.4)\n",
            "Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.12.0; extra == \"all\"->fastapi[all])\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.12.0; extra == \"all\"->fastapi[all])\n",
            "  Downloading uvloop-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Collecting watchfiles>=0.13 (from uvicorn[standard]>=0.12.0; extra == \"all\"->fastapi[all])\n",
            "  Downloading watchfiles-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Collecting websockets>=10.4 (from uvicorn[standard]>=0.12.0; extra == \"all\"->fastapi[all])\n",
            "  Downloading websockets-13.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.23.0->fastapi[all]) (1.2.2)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.12.3->fastapi-cli>=0.0.5->fastapi-cli[standard]>=0.0.5; extra == \"all\"->fastapi[all]) (1.5.4)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow) (2.16.1)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->keras>=3.2.0->tensorflow) (0.1.2)\n",
            "Downloading pyngrok-7.2.0-py3-none-any.whl (22 kB)\n",
            "Downloading uvicorn-0.30.6-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n",
            "Downloading email_validator-2.2.0-py3-none-any.whl (33 kB)\n",
            "Downloading fastapi_cli-0.0.5-py3-none-any.whl (9.5 kB)\n",
            "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pydantic_extra_types-2.9.0-py3-none-any.whl (30 kB)\n",
            "Downloading pydantic_settings-2.5.2-py3-none-any.whl (26 kB)\n",
            "Downloading starlette-0.38.5-py3-none-any.whl (71 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.4/71.4 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ujson-5.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (53 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastapi-0.114.1-py3-none-any.whl (94 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.0/94.0 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m28.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Downloading uvloop-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m89.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading watchfiles-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (425 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m425.7/425.7 kB\u001b[0m \u001b[31m32.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading websockets-13.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (157 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m157.3/157.3 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: websockets, uvloop, ujson, python-multipart, python-dotenv, pyngrok, orjson, httptools, h11, dnspython, watchfiles, uvicorn, starlette, httpcore, email-validator, pydantic-settings, pydantic-extra-types, httpx, fastapi, fastapi-cli\n",
            "Successfully installed dnspython-2.6.1 email-validator-2.2.0 fastapi-0.114.1 fastapi-cli-0.0.5 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.2 orjson-3.10.7 pydantic-extra-types-2.9.0 pydantic-settings-2.5.2 pyngrok-7.2.0 python-dotenv-1.0.1 python-multipart-0.0.9 starlette-0.38.5 ujson-5.10.0 uvicorn-0.30.6 uvloop-0.20.0 watchfiles-0.24.0 websockets-13.0.1\n"
          ]
        }
      ],
      "source": [
        "!pip install llama-cpp-python\n",
        "!pip install pyngrok fastapi[all] uvicorn python-multipart transformers pydantic tensorflow requests"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# I have used\n",
        "# !wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip -q\n",
        "# !unzip -q ngrok-stable-linux-amd64.zip\n",
        "import getpass\n",
        "from pyngrok import ngrok, conf\n",
        "\n",
        "print(\"Enter your authtoken, which can be copied from https://dashboard.ngrok.com/auth\")\n",
        "conf.get_default().auth_token = getpass.getpass()\n",
        "\n"
      ],
      "metadata": {
        "id": "0_FMFO-DfHuG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff0a266a-8fbf-474f-a2b9-929294a5c12c"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your authtoken, which can be copied from https://dashboard.ngrok.com/auth\n",
            "··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "auth_token = 'get your oath key from ngrok dashboard by signing in'"
      ],
      "metadata": {
        "id": "9XANGnWAbwfs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## Using Fast API to create endpoints for calling the model\n",
        "\n",
        "%%writefile app.py\n",
        "\n",
        "from fastapi import FastAPI\n",
        "from pydantic import BaseModel\n",
        "import uvicorn\n",
        "import threading\n",
        "from pyngrok import ngrok\n",
        "from huggingface_hub import hf_hub_download\n",
        "from llama_cpp import Llama\n",
        "from fastapi import HTTPException\n",
        "from typing import Any\n",
        "\n",
        "class request(BaseModel):\n",
        "  inputs: str\n",
        "  parameters: dict[str, Any] | None\n",
        "\n",
        "GENERATIVE_AI_MODEL_REPO = \"TheBloke/Llama-2-7B-GGUF\"\n",
        "GENERATIVE_AI_MODEL_FILE = \"llama-2-7b.Q4_K_M.gguf\"\n",
        "\n",
        "model_path  = hf_hub_download(filename = GENERATIVE_AI_MODEL_FILE , repo_id = GENERATIVE_AI_MODEL_REPO )\n",
        "llama2_model = Llama(model_path=model_path,n_gpu_layers=64,n_ctx=2000)\n",
        "\n",
        "app = FastAPI()\n",
        "\n",
        "@app.get('/')\n",
        "def index():\n",
        "  return \"Welcome to Llama2 assistant Page\"\n",
        "\n",
        "\n",
       
        "@app.post('/complete_text')\n",
        "def complete_text(input: request):\n",
        "  params = input.parameters or {}\n",
        "  response = llama2_model(prompt= input.inputs, **params)\n",
        "  output = response['choices'][0]['text']\n",
        "  return {\"generated_text\": output}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "zTTWnciqhchf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d3e1f87c-9628-419c-b495-7c7dfbf880f0"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting app.py\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Using uvicorn to host our Fast API app. Since we are in Colab, we use get_ipython to run the command in our interactive python session.\n",
        "## Now our Fast API is hosted on colabs environment which is not publicly accessible\n",
        "get_ipython().system_raw('uvicorn app:app --host 0.0.0.0 --port 8000 --reload > server.log 2>&1 &')"
      ],
      "metadata": {
        "id": "6_tS9nuYlDZb"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## ngrok is used to connect the colabs environemnt port 8000 to a public URL\n",
        "\n",
        "from pyngrok import ngrok, conf\n",
        "public_url = ngrok.connect(8000)\n",
        "print(f\"Public URL: {public_url}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TvwzngQEqmDc",
        "outputId": "c0928a00-88d3-4ed7-97f5-9d67c5144d92"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Public URL: NgrokTunnel: \"https://bbbe-34-125-210-28.ngrok-free.app\" -> \"http://localhost:8000\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use the `<Public URL>/docs` to access the end points defined in our Fast API\n",
        "app which includes\n",
        "\n",
        "1. `/` : index page that welcomes to Fast API app\n",
        "2. `/generate_text`:  to call our LLAMA model to finish the sentence for us\n",
        "\n"
      ],
      "metadata": {
        "id": "AjfN0wc_oOR2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## We can also check using the request module\n",
        "import requests\n",
        "url = public_url.public_url\n",
        "\n",
        "print(requests.get(f'{url}/').json())\n",
        "data = {\n",
        "  \"inputs\": '''\n",
        "Tell me how to make a chocolate cake?\n",
        "''',\n",
        "  #paramaters can be found here https://abetlen.github.io/llama-cpp-python/#llama_cpp.llama.Llama.create_completion\n",
        "  \"parameters\": {\"temperature\":0.1,\n",
        "                 \"max_tokens\":200}\n",
        "  #higher temperature, more creative response is, lower more precise\n",
        "  #max_token is the max amount of (simplified) \"words\" allowed to be generated\n",
        "}\n",
        "\n",
        "response = requests.post(f'{url}/complete_text',json = data)\n",
        "print(\"Generated text\",response.json())"
      ],
      "metadata": {
        "id": "doigU0Ogk4m3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d98e8f23-4745-4c30-f474-688da7ba29c3"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Welcome to Llama2 assistant Page\n",
            "Generated text {'generated_text': \"\\nI'm not sure if this is the right place to ask this question, but I'm not sure where else to go.\\n\\nI'm trying to make a chocolate cake, but I'm not sure how to do it. I've tried looking online, but I can't find anything that's really helpful.\\n\\nI'm not sure if I need to use a mixer or not, and I'm not sure what kind of chocolate to use.\\n\\nI'm not sure if I need to use a mixer or not, and I'm not sure what kind of chocolate to use.\\n\\nI'm not sure if I need to use a mixer or not, and I'm not sure what kind of chocolate to use.\\n\\nI'm not sure if I need to use a mixer or not, and I'm not sure what kind of\"}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oShN0Shlmz-X"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}