iter

notebooks/Untitled.ipynb

@@ -0,0 +1,169 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "0f28aac8-48f9-44b4-bbd2-d56f4745a4d7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "sys.path.append(\"/Users/glemaitre/Documents/scratch/rag_based_llm\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "36377022-3f0c-474a-8620-aff7a3beee78",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "API_DOC = Path(\n",
+    "    \"/Users/glemaitre/Documents/packages/scikit-learn/doc/_build/html/stable/\"\n",
+    "    \"modules/generated\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e0686ed4-c1a1-4644-aa0e-0a677b6be63b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sphinx_rag_search_engine.scraping import APIDocExtractor\n",
+    "\n",
+    "extractor = APIDocExtractor(output_type=\"list\", n_jobs=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "fd978b63-d3f8-4dde-b400-3bb7cf53bf45",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "6.42 s ± 85.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit\n",
+    "output = extractor.fit_transform(API_DOC)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a55920b4-974e-4514-99a7-1a2d60e262b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "extractor = APIDocExtractor(output_type=\"list\", n_jobs=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "e86ec478-35a1-4e80-8746-f7b5370e158e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2.03 s ± 38 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit\n",
+    "output = extractor.fit_transform(API_DOC)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "91d0c4f7-82a9-4ac6-b710-52ea3d54cd04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "extractor = APIDocExtractor(output_type=\"generator\", n_jobs=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4230f054-d663-4714-83a2-8a7e6a725265",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "6.32 s ± 15.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit\n",
+    "output = list(extractor.fit_transform(API_DOC))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ae4b0f76-082c-4f71-9f26-673a0aee343f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "extractor = APIDocExtractor(output_type=\"generator_unordered\", n_jobs=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "343e4e58-378e-42c2-bffb-fafa86e07d3c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%timeit\n",
+    "output = list(extractor.fit_transform(API_DOC))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d8fa9e0-a6a3-4d8b-b1c6-f802a4cf3a1a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}