build_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "VEANywYAVyRi"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "import torch\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "import faiss\n",
    "import numpy as np\n",
    "import pickle\n",
    "from pathlib import Path\n",
    "from vector_engine.utils import vector_search, id2details\n",
    "\n",
    "#df = pd.read_csv('/content/Kittlerorakel/paragraphs.csv')\n",
    "df = pd.read_csv('sentences.csv')\n",
    "\n",
    "#df.head(3)\n",
    "#print(f\"Kittler paragraphs: {df.Id.unique().shape[0]}\")\n",
    "\n",
    "model = SentenceTransformer('T-Systems-onsite/cross-en-de-roberta-sentence-transformer')\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    model = model.to(torch.device(\"cuda\"))\n",
    "print(model.device)\n",
    "\n",
    "embeddings = model.encode(df.Text.to_list(), show_progress_bar=True)\n",
    "#print(f'Shape of the vectorised paragraphs: {embeddings[0].shape}')\n",
    "embeddings = np.array([embedding for embedding in embeddings]).astype(\"float32\")\n",
    "index = faiss.IndexFlatL2(embeddings.shape[1])\n",
    "index = faiss.IndexIDMap(index)\n",
    "index.add_with_ids(embeddings, df.Id.values)\n",
    "\n",
    "#print(f\"Number of vectors in the Faiss index: {index.ntotal}\")\n",
    "\n",
    "# Paragraph\n",
    "#df.iloc[5415, 2]\n",
    "\n",
    "# Retrieve the 10 nearest neighbours\n",
    "#D, I = index.search(np.array([embeddings[5415]]), k=10)\n",
    "#print(f'L2 distance: {D.flatten().tolist()}\\n\\nParagraph Ids: {I.flatten().tolist()}')\n",
    "\n",
    "# Fetch the original titles based on their index\n",
    "#id2details(df, I, 'Title')\n",
    "\n",
    "# Fetch the similar texts based on their index\n",
    "#id2details(df, I, 'Text')\n",
    "\n",
    "with open(\"models/faiss_index.pickle\", \"wb\") as h:\n",
    "    pickle.dump(faiss.serialize_index(index), h)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "001-vector-search.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "interpreter": {
   "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
  },
  "kernelspec": {
   "display_name": "Python [conda env:myenv]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "15c5bbc768db41e4bc06c9405539091c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1e1408cddd284bc4a4b5484be0561991": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3e9c24c8488b431fb88a0d045d85b700": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Batches: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_15c5bbc768db41e4bc06c9405539091c",
      "max": 264,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f9c22552944b4e2dafe1f98170665293",
      "value": 264
     }
    },
    "7a7e927567024c578b33b15000d5e531": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3e9c24c8488b431fb88a0d045d85b700",
       "IPY_MODEL_7aaf8a423e7f48bbac24d6970ed4dfe9"
      ],
      "layout": "IPY_MODEL_da5011a6565e45e2b5d0b37634498648"
     }
    },
    "7aaf8a423e7f48bbac24d6970ed4dfe9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7f1cd9dbb4b742ab8d78e073fb9b0f90",
      "placeholder": "​",
      "style": "IPY_MODEL_1e1408cddd284bc4a4b5484be0561991",
      "value": " 264/264 [00:31&lt;00:00,  8.30it/s]"
     }
    },
    "7f1cd9dbb4b742ab8d78e073fb9b0f90": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "da5011a6565e45e2b5d0b37634498648": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f9c22552944b4e2dafe1f98170665293": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}