Ml_demo1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ylPsjdbRRfZg"
      },
      "outputs": [],
      "source": [
        "# загружаем модель с HaggingFace"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/ai-forever/sbert_large_mt_nlu_ru"
      ],
      "metadata": {
        "id": "ybYdV_XnUIpe"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install optimum[exporters]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n97068txUcv7",
        "outputId": "91bc4dae-705e-4531-b8ef-9bb8514b8695"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: onnxscript in /usr/local/lib/python3.10/dist-packages (0.1.0.dev20231213)\n",
            "Collecting optimum[exporters]\n",
            "  Downloading optimum-1.16.1-py3-none-any.whl (403 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m403.3/403.3 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from onnxscript) (1.23.5)\n",
            "Requirement already satisfied: onnx>=1.14 in /usr/local/lib/python3.10/dist-packages (from onnxscript) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from onnxscript) (4.5.0)\n",
            "Collecting coloredlogs (from optimum[exporters])\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (1.12)\n",
            "Requirement already satisfied: transformers[sentencepiece]>=4.26.0 in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (4.35.2)\n",
            "Requirement already satisfied: torch>=1.9 in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (2.1.0+cu121)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (23.2)\n",
            "Requirement already satisfied: huggingface-hub>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from optimum[exporters]) (0.19.4)\n",
            "Collecting datasets (from optimum[exporters])\n",
            "  Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime (from optimum[exporters])\n",
            "  Downloading onnxruntime-1.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.4/6.4 MB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting timm (from optimum[exporters])\n",
            "  Downloading timm-0.9.12-py3-none-any.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m30.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (4.66.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.8.0->optimum[exporters]) (6.0.1)\n",
            "Requirement already satisfied: protobuf>=3.20.2 in /usr/local/lib/python3.10/dist-packages (from onnx>=1.14->onnxscript) (3.20.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[exporters]) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[exporters]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9->optimum[exporters]) (2.1.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[exporters]) (2023.6.3)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[exporters]) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]>=4.26.0->optimum[exporters]) (0.4.1)\n",
            "Collecting sentencepiece!=0.1.92,>=0.1.91 (from transformers[sentencepiece]>=4.26.0->optimum[exporters])\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m34.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting humanfriendly>=9.1 (from coloredlogs->optimum[exporters])\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (10.0.1)\n",
            "Collecting pyarrow-hotfix (from datasets->optimum[exporters])\n",
            "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets->optimum[exporters])\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (1.5.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (3.4.1)\n",
            "Collecting multiprocess (from datasets->optimum[exporters])\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->optimum[exporters]) (3.9.1)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime->optimum[exporters]) (23.5.26)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum[exporters]) (1.3.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from timm->optimum[exporters]) (0.16.0+cu121)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->optimum[exporters]) (4.0.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.8.0->optimum[exporters]) (2023.11.17)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.9->optimum[exporters]) (2.1.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum[exporters]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->optimum[exporters]) (2023.3.post1)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->timm->optimum[exporters]) (9.4.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets->optimum[exporters]) (1.16.0)\n",
            "Installing collected packages: sentencepiece, pyarrow-hotfix, humanfriendly, dill, multiprocess, coloredlogs, onnxruntime, timm, datasets, optimum\n",
            "Successfully installed coloredlogs-15.0.1 datasets-2.15.0 dill-0.3.7 humanfriendly-10.0 multiprocess-0.70.15 onnxruntime-1.16.3 optimum-1.16.1 pyarrow-hotfix-0.6 sentencepiece-0.1.99 timm-0.9.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModel\n",
        "import torch\n",
        "\n",
        "#Mean Pooling - Take attention mask into account for correct averaging\n",
        "def mean_pooling(model_output, attention_mask):\n",
        "    token_embeddings = model_output[0] #First element of model_output contains all token embeddings\n",
        "    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
        "    sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)\n",
        "    sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
        "    emb =  sum_embeddings / sum_mask\n",
        "    return emb / torch.linalg.norm(emb, dim=1).reshape(-1,1)\n",
        "\n",
        "\n",
        "#Sentences we want sentence embeddings for\n",
        "sentences = ['Привет! Как твои дела?',\n",
        "             'А правда, что 42 твое любимое число?',\n",
        "             'Здорово, рад видеть!']\n",
        "\n",
        "#Load AutoModel from huggingface model repository\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"ai-forever/sbert_large_mt_nlu_ru\")\n",
        "model = AutoModel.from_pretrained(\"ai-forever/sbert_large_mt_nlu_ru\")\n",
        "\n",
        "#Tokenize sentences\n",
        "encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=24, return_tensors='pt')\n",
        "\n",
        "#Compute token embeddings\n",
        "with torch.no_grad():\n",
        "    model_output = model(**encoded_input)\n",
        "\n",
        "#Perform pooling. In this case, mean pooling\n",
        "sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])\n"
      ],
      "metadata": {
        "id": "RryScuCQRjVW"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence_embeddings"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gslYu3jBcWYy",
        "outputId": "7abcf204-c435-4f3a-ac9d-8101241f02fd"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.0060, -0.0177, -0.0595,  ...,  0.0345, -0.0672, -0.0134],\n",
              "        [ 0.0083, -0.0186, -0.0018,  ...,  0.0101, -0.0319,  0.0486],\n",
              "        [ 0.0093, -0.0058, -0.0027,  ...,  0.0117, -0.0385, -0.0273]])"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.norm(sentence_embeddings[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wjhe6V1pfEdl",
        "outputId": "cd5fd01e-fa2c-4fec-acfc-a815652b3789"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(1.0000)"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sum(sentence_embeddings[0] * sentence_embeddings[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "izpltW5je9BU",
        "outputId": "91ea3edf-2306-4c5b-fd5a-d02bb4f448e6"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0.2704)"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! optimum-cli export onnx --model ai-forever/sbert_large_mt_nlu_ru model_onnx/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EXuZolN_cWbn",
        "outputId": "f2111785-af02-415c-afc3-3c7eb200a5ad"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-21 13:04:48.832019: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-21 13:04:48.832186: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-21 13:04:48.995254: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-21 13:04:52.210973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Framework not specified. Using pt to export to ONNX.\n",
            "Automatic task detection to feature-extraction (possible synonyms are: default, mask-generation, sentence-similarity).\n",
            "Using the export variant default. Available variants are:\n",
            "    - default: The default ONNX variant.\n",
            "Using framework PyTorch: 2.1.0+cu121\n",
            "Overriding 1 configuration item(s)\n",
            "\t- use_cache -> False\n",
            "^C\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "tOU5AcEZcWea"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "358LAVQQcWhE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Model NER"
      ],
      "metadata": {
        "id": "sg1yjKjfffBs"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/models?pipeline_tag=token-classification&language=ru&sort=trending"
      ],
      "metadata": {
        "id": "tzv8MfXEffE8"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/surdan/LaBSE_ner_nerel"
      ],
      "metadata": {
        "id": "BUr5p82NffHr"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/surdan/LaBSE_ner_nerel/blob/main/Train_model.ipynb\n",
        "# ноутбук с обучением модели"
      ],
      "metadata": {
        "id": "lHwdNvWLffKP"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/surdan/LaBSE_ner_nerel/blob/main/Inference.ipynb\n",
        "# ноутбук с инференсом модели"
      ],
      "metadata": {
        "id": "6Wsbxj0wffMr"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/datasets/surdan/nerel_short\n",
        "# датасет для обучения"
      ],
      "metadata": {
        "id": "e5ayRP2qffPI"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zpbrMkN8gK0V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "PTGYpfJ9gK2_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "wgJsKqdUgK5X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "dbReNQhdZ8MW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "H_7UwsfpZ8PD"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}