From 8b8d1fe7c74caf2e648f00a4f48f3466e03cacc5 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Thu, 19 Oct 2023 17:22:25 +0530 Subject: [PATCH 01/11] add langchain loaders to docs --- docs/concepts/testset_generation.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/concepts/testset_generation.md b/docs/concepts/testset_generation.md index 99ca0328a..abe95067a 100644 --- a/docs/concepts/testset_generation.md +++ b/docs/concepts/testset_generation.md @@ -35,6 +35,27 @@ Moving forward, we are will be expanding the range of evolution techniques to of ## Example +```{code-block} python +:caption: loading documents using langchain +from langchain.document_loaders import PubMedLoader + +loader = PubMedLoader("liver", load_max_docs=10) +documents = loader.load() +``` +Checkout [langchain](https://python.langchain.com/docs/modules/data_connection/document_loaders/) document loaders to see more examples + +```{code-block} python +:caption: loading documents using llama-index +from llama_index import download_loader + +SemanticScholarReader = download_loader("SemanticScholarReader") +loader = SemanticScholarReader() +query_space = "large language models" +documents = loader.load_data(query=query_space, limit=10) +``` +Checkout [llama-index](https://gpt-index.readthedocs.io/en/stable/core_modules/data_modules/connector/root.html) document loaders to see more examples + + ```{code-block} python :caption: Customising test set generation from ragas.testset import TestsetGenerator From 110cc023401844239eba4e15245a7955ae9df121 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Mon, 20 Nov 2023 21:38:32 +0530 Subject: [PATCH 02/11] reformat to json format --- src/ragas/metrics/_context_recall.py | 84 +++++++++++++++++++++------- 1 file changed, 64 insertions(+), 20 deletions(-) diff --git a/src/ragas/metrics/_context_recall.py b/src/ragas/metrics/_context_recall.py index d0165fe2a..417cd6ac4 100644 --- a/src/ragas/metrics/_context_recall.py +++ b/src/ragas/metrics/_context_recall.py @@ -1,8 +1,10 @@ from __future__ import annotations +import re import typing as t from dataclasses import dataclass +import numpy as np from datasets import Dataset from langchain.callbacks.manager import CallbackManager, trace_as_chain_group from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate @@ -11,19 +13,51 @@ CONTEXT_RECALL_RA = HumanMessagePromptTemplate.from_template( """ -Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. -Think in steps and reason before coming to conclusion. +Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Output json with reason. + +question: What can you tell me about albert Albert Einstein? context: Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called "the world's most famous equation". He received the 1921 Nobel Prize in Physics "for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect", a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius. answer: Albert Einstein born in 14 March 1879 was German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895 -classification -1. Albert Einstein born in 14 March 1879 was German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. The date of birth of Einstein is mentioned clearly in the context. So [Attributed] -2. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. The exact sentence is present in the given context. So [Attributed] -3. He published 4 papers in 1905. There is no mention about papers he wrote in given the context. So [Not Attributed] -4. Einstein moved to Switzerland in 1895. There is not supporting evidence for this in the given the context. So [Not Attributed] - +classification: +[ + {{ "statement_1":"Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.", + "reason": "The date of birth of Einstein is mentioned clearly in the context.", + "Attributed": "Yes" + }}, + {{ + "statement_2":"He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics.", + "reason": "The exact sentence is present in the given context.", + "Attributed": "Yes" + }}, + {{ + "statement_3": "He published 4 papers in 1905.", + "reason": "There is no mention about papers he wrote in the given context.", + "Attributed": "No" + }}, + {{ + "statement_4":"Einstein moved to Switzerland in 1895.", + "reason": "There is no supporting evidence for this in the given context.", + "Attributed": "No" + }} +] + +question: who won 2020 icc world cup? +context: Who won the 2022 ICC Men's T20 World Cup? +The 2022 ICC Men's T20 World Cup, held from October 16 to November 13, 2022, in Australia, was the eighth edition of the tournament. Originally scheduled for 2020, it was postponed due to the COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets in the final to clinch their second ICC Men's T20 World Cup title. +answer: England +classification: +[ + {{ + "statement_1":"England won the 2022 ICC Men's T20 World Cup.", + "reason": "From context it is clear that England defeated Pakistan to win the World Cup.", + "Attributed": "Yes" + }} +] + +question:{question} context:{context} -answer:{ground_truth} +answer:{answer} classification: """ # noqa: E501 ) @@ -44,7 +78,7 @@ class ContextRecall(MetricWithLLM): """ name: str = "context_recall" - evaluation_mode: EvaluationMode = EvaluationMode.gc + evaluation_mode: EvaluationMode = EvaluationMode.qcg batch_size: int = 15 def _score_batch( @@ -53,17 +87,22 @@ def _score_batch( callbacks: t.Optional[CallbackManager] = None, callback_group_name: str = "batch", ) -> list: - verdict_token = "[Attributed]" prompts = [] - ground_truths, contexts = dataset["ground_truths"], dataset["contexts"] + question, ground_truths, contexts = ( + dataset["question"], + dataset["ground_truths"], + dataset["contexts"], + ) with trace_as_chain_group( callback_group_name, callback_manager=callbacks ) as batch_group: - for gt, ctx in zip(ground_truths, contexts): + for qstn, gt, ctx in zip(question, ground_truths, contexts): gt = "\n".join(gt) if isinstance(gt, list) else gt ctx = "\n".join(ctx) if isinstance(ctx, list) else ctx - human_prompt = CONTEXT_RECALL_RA.format(context=ctx, ground_truth=gt) + human_prompt = CONTEXT_RECALL_RA.format( + question=qstn, context=ctx, answer=gt + ) prompts.append(ChatPromptTemplate.from_messages([human_prompt])) responses: list[list[str]] = [] @@ -75,12 +114,17 @@ def _score_batch( responses = [[i.text for i in r] for r in results.generations] scores = [] for response in responses: - sentences = response[0].split("\n") - denom = len(sentences) - numerator = sum( - bool(sentence.find(verdict_token) != -1) for sentence in sentences - ) - scores.append(numerator / denom) + pattern = "\[\s*\{.*?\}(\s*,\s*\{.*?\})*\s*\]" + match = re.search(pattern, response[0].replace("\n", "")) + if match: + response = eval(response[0]) + denom = len(response) + numerator = sum( + item.get("Attributed").lower() == "yes" for item in response + ) + scores.append(numerator / denom) + else: + scores.append(np.nan) return scores From c4036f239a9ee51036865a43b84395266f9025bd Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Mon, 20 Nov 2023 21:40:01 +0530 Subject: [PATCH 03/11] add qcg to validate --- src/ragas/metrics/base.py | 2 +- src/ragas/validation.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py index 94c1612ee..8afb43cde 100644 --- a/src/ragas/metrics/base.py +++ b/src/ragas/metrics/base.py @@ -40,7 +40,7 @@ def make_batches(total_size: int, batch_size: int) -> list[range]: return batches -EvaluationMode = Enum("EvaluationMode", "qac qa qc gc ga qga") +EvaluationMode = Enum("EvaluationMode", "qac qa qc gc ga qga qcg") @dataclass diff --git a/src/ragas/validation.py b/src/ragas/validation.py index 18c4d853c..f5aa784c2 100644 --- a/src/ragas/validation.py +++ b/src/ragas/validation.py @@ -43,6 +43,7 @@ def validate_column_dtypes(ds: Dataset): EvaluationMode.gc: ["ground_truths", "contexts"], EvaluationMode.ga: ["ground_truths", "answer"], EvaluationMode.qga: ["question", "ground_truths", "answer"], + EvaluationMode.qcg: ["question", "contexts", "ground_truths"], } From cf14e39b05f03c6a91617261f68d5c9c60e5e83f Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Mon, 20 Nov 2023 21:40:34 +0530 Subject: [PATCH 04/11] determinism experiments --- .../assesments/Deterministic output.ipynb | 583 ++++++++++++++++++ 1 file changed, 583 insertions(+) create mode 100644 experiments/assesments/Deterministic output.ipynb diff --git a/experiments/assesments/Deterministic output.ipynb b/experiments/assesments/Deterministic output.ipynb new file mode 100644 index 000000000..b3653d472 --- /dev/null +++ b/experiments/assesments/Deterministic output.ipynb @@ -0,0 +1,583 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ed913729", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/ragas/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import os\n", + "import openai\n", + "\n", + "from openai import OpenAI\n", + "from datasets import load_dataset\n", + "import json\n", + "import numpy as np\n", + "from scipy.stats import entropy\n", + "import matplotlib.pyplot as plt\n", + "client = OpenAI()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b2de3222", + "metadata": {}, + "outputs": [], + "source": [ + "def llm2(prompt, **kwargs):\n", + " response = client.chat.completions.create(\n", + " model=kwargs.get(\"model\", \"gpt-3.5-turbo\"),\n", + " messages=[{\"role\": \"system\", \"content\": prompt}],\n", + " temperature=kwargs.get(\"temperature\", 0),\n", + " top_p=kwargs.get(\"top_p\", 1),\n", + " frequency_penalty=kwargs.get(\"frequency_penalty\", 0.0),\n", + " presence_penalty=kwargs.get(\"presence_penalty\", 0.0),\n", + " max_tokens=kwargs.get(\"max_tokens\", 500),\n", + " n=kwargs.get(\"n\", 1),\n", + " )\n", + " return response" + ] + }, + { + "cell_type": "markdown", + "id": "c46a9958", + "metadata": {}, + "source": [ + "## Experiment 1\n", + "AIM: To quantify the consistency of ragas recall score \n", + "\n", + "- before prompt change\n", + "- After prompt change\n", + "- Prompt change + self consistency method\n", + "\n", + "Quantify \n", + "- KL divergance\n", + "- Mean absolute change (x_i - y_i)\n", + "- Mean of scores\n", + "\n", + "Dataset used:\n", + "- explodinggradients/WikiEval" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ecf10292", + "metadata": {}, + "outputs": [], + "source": [ + "# with open(\"consistency_recall.json\",'w') as file:\n", + "# json.dump({}, file, indent=4)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "34acc81f", + "metadata": {}, + "outputs": [], + "source": [ + "def read_score(name):\n", + " filename = \"consistency_recall.json\"\n", + " data = json.load(open(filename))\n", + " return data.get(name)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ba69e301", + "metadata": {}, + "outputs": [], + "source": [ + "def record_score(score,name):\n", + " \n", + " filename = \"consistency_recall.json\"\n", + " data = json.load(open(filename))\n", + " data[name] = score\n", + " with open(filename,'w') as file:\n", + " json.dump(data, file, indent=4)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cf059b7b", + "metadata": {}, + "outputs": [], + "source": [ + "def KL(dist1,dist2):\n", + " \n", + " if isinstance(dist1, list):\n", + " dist1 = np.array(dist1) + 1e-8\n", + " \n", + " if isinstance(dist2, list):\n", + " dist2 = np.array(dist2) + 1e-8\n", + " \n", + " dist1_normalized = dist1 / dist1.sum()\n", + " dist2_normalized = dist2 / dist2.sum()\n", + " kl_divergence = entropy(dist1_normalized, dist2_normalized)\n", + " return kl_divergence" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "id": "ae5d00fc", + "metadata": {}, + "outputs": [], + "source": [ + "recall_prompt = \"\"\"\n", + "Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Output json with reason.\n", + "\n", + "\n", + "question: What can you tell me about albert Albert Einstein?\n", + "context: Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called \"the world's most famous equation\". He received the 1921 Nobel Prize in Physics \"for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect\", a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius.\n", + "answer: Albert Einstein born in 14 March 1879 was German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics \"for his services to theoretical physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895 \n", + "classification:\n", + "[\n", + " {{ \"statement_1\":\"Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.\",\n", + " \"reason\": \"The date of birth of Einstein is mentioned clearly in the context.\",\n", + " \"Attributed\": \"Yes\"\n", + " }},\n", + " {{\n", + " \"statement_2\":\"He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics.\",\n", + " \"reason\": \"The exact sentence is present in the given context.\",\n", + " \"Attributed\": \"Yes\"\n", + " }},\n", + " {{\n", + " \"statement_3\": \"He published 4 papers in 1905.\",\n", + " \"reason\": \"There is no mention about papers he wrote in the given context.\",\n", + " \"Attributed\": \"No\"\n", + " }},\n", + " {{\n", + " \"statement_4\":\"Einstein moved to Switzerland in 1895.\",\n", + " \"reason\": \"There is no supporting evidence for this in the given context.\",\n", + " \"Attributed\": \"No\"\n", + " }}\n", + "]\n", + "\n", + "question: who won 2020 icc world cup?\n", + "context: Who won the 2022 ICC Men's T20 World Cup?\n", + "The 2022 ICC Men's T20 World Cup, held from October 16 to November 13, 2022, in Australia, was the eighth edition of the tournament. Originally scheduled for 2020, it was postponed due to the COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets in the final to clinch their second ICC Men's T20 World Cup title.\n", + "answer: England \n", + "classification: \n", + "[\n", + " {{\n", + " \"statement_1\":\"England won the 2022 ICC Men's T20 World Cup.\",\n", + " \"reason\": \"From context it is clear that England defeated Pakistan to win the World Cup.\",\n", + " \"Attributed\": \"Yes\"\n", + " }}\n", + "]\n", + "\n", + "question:{question}\n", + "context:{context}\n", + "answer:{answer}\n", + "classification:\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "757d9b75", + "metadata": {}, + "outputs": [], + "source": [ + "c = \"\"\"\n", + "Black holes, one of the most enigmatic and fascinating phenomena in the universe, are regions of spacetime where gravity is so intense that nothing, not even light, can escape their grasp. Formed from the remnants of massive stars that have undergone gravitational collapse, black holes are characterized by their event horizon, a boundary beyond which all matter and radiation are irretrievably pulled in. Intriguingly, while they are invisible, their presence can be inferred through their interaction with nearby matter and the emission of high-energy radiation. Black holes challenge our understanding of physics, particularly at the intersection of quantum mechanics and general relativity, presenting intriguing questions about the nature of space, time, and the fabric of the universe itself.\n", + "\"\"\"\n", + "q = \"What is the term used to describe the boundary around a black hole where nothing, not even light, can escape?\"\n", + "a = \"Event Horizon.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "6fc3ebd7", + "metadata": {}, + "outputs": [], + "source": [ + "output = llm2(recall_prompt.format(context=c,question=q,answer=a))" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "46b0fca3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'statement_1': 'The term used to describe the boundary around a black hole where nothing, not even light, can escape is the event horizon.',\n", + " 'reason': \"The exact term 'event horizon' is mentioned in the context.\",\n", + " 'Attributed': 'Yes'}]" + ] + }, + "execution_count": 103, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval(output.choices[0].message.content)" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "72bdeac3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'[\\n {\\n \"statement_1\":\"The term used to describe the boundary around a black hole where nothing, not even light, can escape is the event horizon.\",\\n \"reason\": \"The exact term \\'event horizon\\' is mentioned in the context.\",\\n \"Attributed\": \"Yes\"\\n }\\n]'" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output.choices[0].message.content" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "id": "3485a0fb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'statement_1': 'The term used to describe the boundary around a black hole where nothing, not even light, can escape is the event horizon.',\n", + " 'reason': \"The exact term 'event horizon' is mentioned in the context.\",\n", + " 'Attributed': 'Yes'}]" + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval(match.group())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7eaa3091", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset parquet (/Users/shahules/.cache/huggingface/datasets/Pakulski___parquet/Pakulski--ELI5-test-ed159b4d22db0b30/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)\n", + "100%|█████████████████████████████████████████████████████| 1/1 [00:00<00:00, 7.06it/s]\n" + ] + } + ], + "source": [ + "# dataset = load_dataset(\"explodinggradients/WikiEval\")\n", + "dataset = load_dataset(\"Pakulski/ELI5-test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "bfd13691", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.metrics import context_recall\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b1b1fc2a", + "metadata": {}, + "outputs": [], + "source": [ + "# from ragas.metrics._context_recall import ContextRecallImproved\n", + "# context_recall = ContextRecallImproved()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "dc8c4899", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas import evaluate" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "83f0a0de", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "dataset = dataset['train'].select(range(0,10))\n", + "dataset = dataset.rename_columns({\"document\":\"contexts\",\"goldenAnswer\":\"ground_truths\"})\n", + "dataset = dataset.map(lambda x : {'ground_truths':[x['ground_truths']],'contexts':[x['contexts']] })" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "bc5daf8e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading cached processed dataset at /Users/shahules/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--WikiEval-33bd2cbc490cc57b/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7/cache-5fe04aab11621603.arrow\n" + ] + } + ], + "source": [ + "# dataset = dataset['train'].map(lambda x : {'ground_truths':[x['ungrounded_answer']]})" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "33ff7b32", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['id', 'question', 'contexts', 'ground_truths'],\n", + " num_rows: 100\n", + "})" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "80917afa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "evaluating with [context_recall]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████| 4/4 [01:13<00:00, 18.34s/it]\n" + ] + } + ], + "source": [ + "# ragas_score = evaluate(dataset=dataset,metrics=[context_recall],column_map={\"question\":\"question\",\"contexts\":\"context_v1\",\"ground_truths\":\"ground_truths\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b29ec575", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "evaluating with [context_recall]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████| 1/1 [00:23<00:00, 23.22s/it]\n" + ] + } + ], + "source": [ + "ragas_score = evaluate(dataset=dataset,metrics=[context_recall])" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "aaa176e8", + "metadata": {}, + "outputs": [], + "source": [ + "record_score(ragas_score.to_pandas()['context_recall'].values.tolist(),'prompt_to_json_eli5_1')" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "5642abde", + "metadata": {}, + "outputs": [], + "source": [ + "score = KL(read_score(\"prompt_to_json_eli5\"),read_score(\"prompt_to_json_eli5_1\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "25333eb5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "score" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "982131d0", + "metadata": {}, + "outputs": [], + "source": [ + "record_score(score,\"prompt_to_json_prompt_to_json_2\")" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "id": "de059ca8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAFRCAYAAADaTrE/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzdUlEQVR4nO3deXhTdaLG8TdJW9pSKDQUapUBLSCLXBSrCDgsQ3XuwFWQEVFBRWQcRRRwFBFZFOEBF0QEHBGQRXmkbizX3cJlEZRhq+LKUjdsC11YS2uXc+4flUhogQSac5L2+3keHsjvnCRvkvLj5WxxmKZpCgAAALZx2h0AAACgpqOQAQAA2IxCBgAAYDMKGQAAgM0oZAAAADajkAEAANiMQgYAgA0GDRqklJQUu2MgSFDIYIsTJyLDMPTss8/qkksuUe3atVWvXj21a9dOY8eO9bpPSUmJnn76af3Xf/2XoqKiVLduXXXp0kXvvPOO13o//vijHA6H6tSpo3379nktGzJkiLp16xbQ1wagehg0aJAcDodGjRrlNb537145HA6tWbPGp8d57bXX5HA4KozPmDFDb775ZlVERTVAIYPtJk6cqMmTJ2v06NH68ssvtWHDBo0ZM0YFBQWedUpKSvS3v/1N06ZN04gRI/TNN9/o888/V48ePdS/f389/vjjFR63tLRUEyZMsPCVAKhuIiMj9cILL+inn36q8seOjY1V/fr1q/xxEZooZLDd8uXLddddd2ngwIFKSkpSmzZt1L9/f02fPt2zzsyZM7Vq1SqtXLlSgwcP1oUXXqjWrVtrwoQJmjRpkp544glt3brV63FHjBihefPm6dtvv7X6JQGoJjp16qR27dppzJgxp1znscceU6tWrRQdHa3GjRvrnnvu0aFDhyRJa9as0W233SZJcjgccjgcGjRokCTvPQWffPKJXC6X9u7d6/XYqampio6O1uHDhyVJ+/bt06BBgxQfH686deqoc+fOWrduXVW/bNiAQgbbnXfeeVq7dq1+/fXXU67z6quvqkePHurQoUOFZcOHD1d0dLSWLFniNd6rVy917dq1wu4GAPCVw+HQs88+q9dff11btmypdJ2oqCi9/PLL+uabb7Rw4UKtWbNGDzzwgKTyQjdr1ixJUlZWlrKysjRjxowKj9GjRw+dd955FeaxRYsWqU+fPqpbt64KCwvVvXt3HTlyRB988IG2b9+unj176pprruE/ntUAhQy2mz59ugoLC9W4cWNdfPHFuuOOO7RkyRKVlpZ61vn+++/Vpk2bSu8fGRmppKQkff/99xWWPfvss3rvvff0f//3fwHLD6B6+/Of/6zevXvroYceqnT52LFj9ec//1lNmzZVjx49NGXKFC1dulSGYSgiIkKxsbGSpISEBCUkJHhun8jpdGrgwIF69dVXPWP79u3Txx9/rDvuuENS+dayw4cPKzU1VcnJyWrWrJkee+wxde7cWXPmzAnAK4eVKGSwXcuWLbVjxw5t3bpVw4YNU3FxsYYMGaKrrrpKhYWF5/TYl112mQYOHKiHH35YpmlWUWIANc1TTz2lDRs2aOXKlRWWvfPOO+rSpYsSExMVExOjAQMGqLi4WNnZ2X49xx133KGvv/5a27ZtkyQtWbJEDRs29OzW3Lx5s7Kzs1WvXj3FxMR4fq1fv167du069xcJW1HIEBQcDocuu+wy3X///Xr99df1ySefaOvWrXrjjTckSS1atNBXX31V6X2Lioq0Z88eXXzxxZUunzx5sr755psKuwIAwFctWrTQP//5Tz3yyCNeW+83bdqkfv36qUuXLlq2bJm2bduml156SZJUXFzs13O0atVKycnJWrx4sSRp8eLFGjhwoFwul6TyM9JbtWql9PR0r1/ffvut5s6dW0WvFHahkCEotWrVSpK0f/9+SdLAgQO1evVqbdq0qcK6M2bM0LFjxzRgwIBKH6tx48YaMWKEHnvsMRUVFQUuNIBqbcKECcrMzNTLL7/sGfv000/VoEEDTZo0SR06dFCLFi0qHJgfEREhSSorKzvjc9xxxx16/fXXtW3bNn3xxRe6/fbbPcuSk5OVkZGhunXrqlmzZl6/EhMTq+hVwi4UMtju73//u6ZNm6bPPvtMP/30kzZu3KjbbrtN4eHh6tWrl6TyA/e7deum66+/XgsWLNAPP/ygb7/9Vk888YTGjh2r8ePH6/LLLz/lc4wePVqFhYUVrlkGAL6Kj4/X6NGj9fzzz3vGLr74YuXk5Gj+/PnKyMjQ4sWL9eKLL3rd78ILL5QkrVy5Ujk5OTp69Ogpn+OWW27RgQMHdNddd6l9+/a65JJLPMsGDBigCy+8UL169dLHH3+sH3/8UZs2bdKUKVO0fPnyKn2tsB6FDLYwDENhYWGSpP/+7//Whx9+qL59+6pFixbq16+fIiIitHbtWrVu3VqSFB4ero8++kgjR47UtGnT1KpVK1155ZVKS0vT0qVL9cQTT5z2+erWrasJEyac8zFpAGq2kSNHqkGDBp7b//M//6PHHntMY8aMUdu2bbV06VI988wzXve54oorNHz4cP3zn/9Uw4YNNWzYsFM+vtvtVq9evZSenu61dUwqP4Fp7dq1Sk5O1p133qkWLVqob9+++s9//qMmTZpU7QuF5RwmRzrDBtdee63OP/98LViwwO4oAADYji1ksFRubq5WrFihtWvX6pprrrE7DgAAQSHM7gCoWfr166ddu3bpwQcfVP/+/e2OAwBAUGCXJQAAgM3YZQkAAGAzChkAAIDNKGQAAAA2C/mD+jMzM31eNzEx0a/1gwW5rUVua/mbuzpdkZz5K3iR21qhmlvyL/vp5i+2kAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgMwoZAACAzSy5DtmLL76obdu2KTY2VtOmTauw3DRNLViwQNu3b1etWrU0dOhQXXTRRVZEA2CTsn9cL0n65YQx19yV9oQ5AzvnsLLvdkgLZ2hv0TGZkdHSoOFytWxbJY8N4OyVzZosfbHpjzmsXQe5hj121o9nyRaybt26acyYMadcvn37dmVnZ+uFF17Q3XffrXnz5lkRC4BNjpcxX8ftZtccVvbdDmn6eClvv8yCo1Lefmn6+PJxALY5Xsa8fLGpfPwsWVLIWrdurZiYmFMu37Jli7p06SKHw6EWLVqooKBABw4csCIaAJyRbXPYwhmSUeY9ZpSVjwOwz8ll7EzjPgiKr07Kz89XgwYNPLfdbrfy8/NVv379CuumpaUpLS1NkjR16lS/v0YlVL92hdzWIndg/XKaZaHyGk7k6xzm7/y1t+iYzErGHUXHQup9CqWsJyK3tUIpdyDmsKAoZP5ISUlRSkqK5zbfBRe8yG2tUM19sjO9hlCatE/m7/xlRkZLBUcrHQ+VzzpUfy7Jba1QzV2Z072OoP8uy7i4OOXm5npu5+XlKS4uzsZEAOC7gM1hg4ZLTpf3mNNVPg7APu06+Dfug6AoZMnJyVq3bp1M09TOnTsVHR1d6e5KANXDqc6mDNazLM8kUHOYq2VbaeREyd1QjtoxkruhNHIiZ1kCNnMNe6xi+TrHsywt2WX5/PPP65tvvtGRI0d0zz336KabblJpaakk6dprr9Vll12mbdu26YEHHlBERISGDh1qRSwANjpevkJhV4Wdc5irZVtp6ryQeJ+AmuR4+aqqv5uWFLIRI0acdrnD4dCQIUOsiAIAfmMOAxBoQbHLEgAAoCajkAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgMwoZAACAzShkAAAANqOQAQAA2IxCBgAAYDMKGQAAgM0oZAAAADajkAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgszC7AwAIPkZOtrRiicyD+XLUi5N6D5AzPsHuWABQbVHIAHgxcrJlTh8v5WRLkkxJyvhexsiJlDIACBB2WQLwtmKJp4x5/L7FDAAQGBQyAF7Mg/l+jQMAzh2FDIAXR704v8YBAOeOQgbAW+8B0snHisUnlI8DAAKCg/oBeHHGJ8gYOZGzLAHAQhQyABU44xOkIf+yOwYA1BgUMgAAAD8dv17jvsKjMqJiznlPAoUMAADADyder7H4+OA5Xq+Rg/oBAAD8EYDrNVLIAAAA/BCI6zVSyAAAAPwQiOs1UsgAAAD8EYDrNXJQPwAAgB9OvF5jROFRFXOWJQAAgPWOX6+xYWKiMjMzz/3xqiATAAAAzgGFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALCZZWdZpqena8GCBTIMQz169FCfPn28lufm5mr27NkqKCiQYRi69dZb1b59e6viAcApMX8BCDRLCplhGJo/f77Gjh0rt9utRx99VMnJybrgggs867z99tvq2LGjrr32Wu3du1dTpkxhQgN8UPbdDmnhDO0tOiYzMloaNFyulm3tjlVtMH8BsIIluyx3796thIQENWrUSGFhYerUqZM2b97stY7D4dCxY8ckSceOHVP9+vWtiAaEtLLvdkjTx0t5+2UWHJXy9kvTx5ePo0owfwGwgiVbyPLz8+V2uz233W63du3a5bVOv379NGnSJH344Yf67bffNG7cOCuiAaFt4QzJKPMeM8rKx6fOsydTNcP8BcAKQXOl/g0bNqhbt2667rrrtHPnTs2cOVPTpk2T0+m9ES8tLU1paWmSpKlTpyoxMdGv5/F3/WBBbmuFSu69RcdkVjLuKDoWMq9BCp33+1SYv06P3NYit/WqIrslhSwuLk55eXme23l5eYqL8/5G9NWrV2vMmDGSpBYtWqikpERHjhxRbGys13opKSlKSUnx3Pbn6woSq+jrDaxGbmuFUm4zMloqOFrpeKi8Bn/fb6snbeavc0Nua5Hbev5kP938ZckxZElJScrKytL+/ftVWlqqjRs3Kjk52WudBg0a6KuvvpIk7d27VyUlJapbt64V8YDQNWi45HR5jzld5eOoEsxfAKxgyRYyl8ulwYMHa/LkyTIMQ927d1fjxo2VmpqqpKQkJScn6/bbb9ecOXP03nvvSZKGDh0qh8NhRTwgZLlatlXZyInSwhlycJZlQDB/AbCCwzTNyg5BCRls8g9e5LZWTckdyseZnIz5K3iR21qhmlsKsV2WAAAAODUKGQAAgM0oZAAAADajkAEAANgsaC4MC+APZUtekta8/8dAt55yDbjHvkBAgBk52dKKJTIP5stRL07qPUDO+AS7Y+Es8FmeHQoZEGQqlDFJWvO+yiRKGaolIydb5vTxUk62JJV/+0TG9zJGTuQf8hDDZ3n22GUJBJuTy9iZxoFQt2KJ5x9wj9+3siDE8FmeNQoZAMBW5sF8v8YRvPgszx6FDABgK0e9OL/GEbz4LM8ehQwINt16+jcOhLreA6STjy+KTygfR2jhszxrHNQPBBnXgHtUJnGWJWoMZ3yCjJETOTOvGuCzPHsUMiAIuQbcI1HAUIM44xOkIf+yOwaqAJ/l2WGXJQAAgM0oZAAAADajkAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgMwoZAACAzShkAAAANqOQAQAA2IxCBgAAYDMKGQAAgM3C7A4AVAdl3+2QFs6QjhVI0bWlQcPlatnW7lgA4JOyTWulxbOkkhIpPFy6fZhcHbraHatGYQsZcI7KvtshTR8v5e2XCgvKf58+vnwcAIJc2aa10rxpUvFvkmmU/z5vWvk4LEMhA87VwhmSUeY9ZpSVjwNAsFs8y79xBASFDDhXxwr8GweAYFJS4t84AoJCBpyr6Nr+jQNAMAkP928cAXHGQpabm6v//Oc/yszMrLDs008/DUgoIKQMGi45Xd5jTlf5OAAEu9uH+TeOgDjtWZbp6emaPn26GjZsqKysLHXr1k2DBw+W01ne4+bOnaurr77akqBAsHK1bKuykRM5yxJASHJ16KoyibMsbXbaQvb6669r+PDhat++vQ4ePKiZM2fq6aef1kMPPaSwsDCZpmlVTiCouVq2labOszsGAJwVV4euEgXMVqfdZZmdna327dtLkurVq6cxY8YoMjJSU6ZM0W+//WZJQAAAgOrutIUsJiZGubm5ntsul0vDhw+X2+3Wk08+KcMwAh4QAACgujttIWvbtq3WrFnjNeZwODR06FD96U9/UgmnxAIAAJyz0x5DNmTIEJWVlVW67O6771bfvn0DEgoAAKAmOW0hCwsLU1jYqVdp0KBBlQcCAACoabgwLAAAgM0oZAAAADY77S7LqpSenq4FCxbIMAz16NFDffr0qbDOxo0b9eabb8rhcKhJkyYaPpwrnQOwH/MXgEDzqZCNGjVKTz/9dIXx0aNHa+rUqWe8v2EYmj9/vsaOHSu3261HH31UycnJuuCCCzzrZGVlafny5XryyScVExOjQ4cO+fEyACAwmL8QCoycbGnFEu0rPCojKkbqPUDO+AS7Y8EPPhWy7OzsCmOmaWrfvn0+Pcnu3buVkJCgRo0aSZI6deqkzZs3e01oq1at0l//+lfFxMRIkmJjY316bAAIJOYvBDsjJ1vm9PFSTraKjw9mfC9j5ERKWQg5bSGbNWuWJKm0tNTz5+NycnLUuHFjn54kPz9fbrfbc9vtdmvXrl1e6xz/8vJx48bJMAz169dPl156qU+PDwCBwvyFoLdiiZRz0oaT37eYaci/7MkEv522kB3/H+HJf3Y4HLr44ovVsWPHKgtiGIaysrI0YcIE5efna8KECXr22WdVu3Ztr/XS0tKUlpYmSZo6daoSExP9eh5/1w8W5LYWua0VqrmPY/46PXIH1r7Co39sGTtBROFRNQyR1yCFzvtdmarIftpC1q9fP0lS8+bNz+l/e3FxccrLy/PczsvLU1xcXIV1mjdvrrCwMDVs2FDnnXeesrKy1KxZM6/1UlJSlJKS4rl9/H+mvkhMTPRr/WBBbmuR21r+5rZ60mb+OjfkDjwjKqbS8eKomJB5DaH0fp/Mn+ynm798uuzFpZdeqszMTG3cuFGrV6/2+uWLpKQkZWVlaf/+/SotLdXGjRuVnJzstc6VV16pr7/+WpJ0+PBhZWVleW2VAwA7MH8h6PUeIJ18rFh8Qvk4QoZPB/W/8847evvtt9WkSRPVqlXLa9lf/vKXM97f5XJp8ODBmjx5sgzDUPfu3dW4cWOlpqYqKSlJycnJateunb744guNHDlSTqdTAwcOVJ06dc7uVQFAFWH+QrBzxifIGDlRWrFEEYVHVcxZliHJYZqmeaaVhgwZonHjxqlJkyZWZPILm/yDF7mtVVNyh/JxJidj/gpe5LZWqOaWLN5lGRERofPPP9+3ZAAAAPCLT4Wsf//+euWVV3TgwAEZhuH1CwAAAOfGp2PIXnzxRUnlFz88WWpqatUmAgAAqGF8KmQnXxQWAAAAVcenQhYfHy+p/OKHhw4dUv369QMaCgAAoCbxqZAVFBRo3rx5+vzzzxUWFqZXX31VW7Zs0e7du3XzzTcHOiMAAEC15tNB/XPnzlV0dLRefPFFhYWVd7gWLVpo48aNAQ0HAABQE/i0hWzHjh2aM2eOp4xJUt26dXXo0KGABQMAAKgpfNpCFh0drSNHjniN5ebmciwZAABAFfCpkPXo0UPTpk3TV199JdM0tXPnTs2ePVvXXHNNoPMBAABUez7tsuzdu7ciIiI0f/58lZWV6d///rdSUlLUs2fPQOcDAACo9nwqZA6HQz179qSAAQAABIBPhUyScnJy9NNPP6moqMhr/Oqrr67yUAAAADWJT4Vs2bJlevvtt3XBBRcoIiLCM+5wOChkAAAA58inQvbuu+9q6tSpuuCCCwKdBwAAoMbx6SzLmJgYz9cnAQAAoGr5tIVs0KBBmjNnjnr16qXY2FivZQ0aNAhIMAAAgJrCp0JWWlqqL7/8Uhs2bKiwLDU1tcpDAadS9slK6a1XJMOQnE7pxsFyXXO93bEA4IyMnGxpxRKZB/PlqBcn9R4gZ3yC3bFwlo5/nvsKj8qIijnnz9OnQjZv3jzdcsst6ty5s9dB/YCVyj5ZKb0x748Bw5DemKcyiVIGIKgZOdkyp4+XcrIlSaYkZXwvY+RESlkIOvHzLD4+eI6fp0/HkBmGoe7duysyMlJOp9PrF2CZt17xbxwAgsWKJZ4y5vH7FhaEoAB8nj41quuuu07Lly+XaZpn/UTAOTMM/8YBIEiYB/P9GkdwC8Tn6dMuyw8++EAHDx7UsmXLFBMT47Xs3//+91k/OeAXp7Py8sWWWgBBzlEvTpVt0nDUi7M8C85dID5PnwrZ/ffff9ZPAFSZGwd7H0N24jgABLPeA6SM7713c8UnlI8j9ATg8/SpkLVu3fqsnwCoKq5rrleZxFmWAEKOMz5BxsiJnGVZTZz4eUYUHlVxIM+yfOedd9S3b19Jp7+0Rf/+/c/6yQF/ua65XqKAAQhBzvgEaci/7I6BKnL882yYmKjMzMxzfrxTFrK8vLxK/wwAAICqdcpC9o9//ENS+SUvunTpoosvvljh4eGWBQMAAKgpznh6mtPp1NNPP00ZAwAACBCfrhfQqlUr7dy5M9BZAAAAaiSfzrKMj4/XlClTlJycLLfbLYfD4VnGQf0AAADnxqdCVlxcrCuuuEKSlJ/PVYUBAACqkk+FbOjQoYHOAQAAUGP5VMgk6ddff9Vnn32mQ4cO6a677lJmZqZKSkrUpEmTQOYDAACo9nw6qP+zzz7T+PHjlZ+fr3Xr1kmSCgsLtXjx4oCGAwAAqAl82kL2xhtvaNy4cWratKk+++wzSVKTJk30448/BjIbAABAjeDTFrJDhw5V2DXpcDi8zrYEAADA2fGpkF100UWeXZXHbdiwQc2aNQtIKAAAgJrEp12Wd955pyZNmqTVq1frt99+0+TJk5WZmamxY8cGOh8AAEC151MhO//88/X8889r69atuvzyy+V2u3X55ZcrMjIy0PkAAACqPZ8ve1GrVi21bNlS+fn5iouLo4wBAABUEZ8KWW5url544QXt2rVLtWvXVkFBgZo3b677779f8fHxgc4IAABQrfl0UP/s2bN10UUXacGCBZo3b54WLFigiy66SLNnz/b5idLT0zV8+HDdf//9Wr58+SnX+/zzz3XTTTdpz549Pj82AAQS8xeAQPOpkGVkZGjgwIGe3ZSRkZEaOHCgMjIyfHoSwzA0f/58jRkzRtOnT9eGDRu0d+/eCusVFhbqgw8+UPPmzf14CQg2Rk62jHnTtO/Rf8qYN01GTrbdkYCzxvwFwAo+FbLmzZtr9+7dXmN79uxRixYtfHqS3bt3KyEhQY0aNVJYWJg6deqkzZs3V1gvNTVVvXv3Vnh4uE+Pi+Bj5GTLnD5e5qa1Kv5yq8xNa2VOH08pQ8hi/gJgBZ+OIWvUqJGmTJmi9u3by+12Ky8vT9u3b9fVV1+t1NRUz3r9+/ev9P75+flyu92e2263W7t27fJaJyMjQ7m5uWrfvr1Wrlx5Nq8FwWDFEunk8pWTXT4+5F/2ZALOAfMXACv4VMhKSkrUoUMHSdLhw4cVHh6uK6+8UsXFxcrLyzvnEIZhaPHixRo6dOgZ101LS1NaWpokaerUqUpMTPTrufxdP1iESu59hUdVXMl4ROFRNQyR1yCFzvt9MnJbj/nrzMhtLXJbryqy+1TIfJloTicuLs6ruOXl5SkuLs5zu6ioSL/88oueeOIJSdLBgwf19NNPa9SoUUpKSvJ6rJSUFKWkpHhuZ2Zm+pwjMTHRr/WDRSjlNqJiKh0vjooJmdcQSu/3iWpKbqsnbeavc0Nua5Hbev5kP9385dMxZCd/bZIkmaapZcuW+RQgKSlJWVlZ2r9/v0pLS7Vx40YlJyd7lkdHR2v+/PmaPXu2Zs+erebNm1c6mSEE9B4gxSd4j8UnlI8DIYj5C4AVfNpC9tZbb2nr1q36xz/+oZiYGO3bt0+zZs2Sw+HQDTfccMb7u1wuDR48WJMnT5ZhGOrevbsaN26s1NRUJSUleU1uCG3O+AQZIydKK5YoovCoiqNipN4D5Dy5pAEhgvkLgBUcpmmaZ1qpqKhICxcu1BdffKGuXbvq448/1nXXXafevXvL6fRpI1vAsMk/eJHbWjUldygfZ3Iy5q/gRW5rhWpuyeJdlpGRkbr11ltVu3ZtLVu2TMnJyerTp4/tZQwAAKA68KlRbdu2TQ8//LDatGmjZ555RpmZmRo/frz2798f6HwAAADVnk+FbO7cubrvvvt055136k9/+pMmTpyodu3aafTo0YHOBwAAUO35dFD/M888o5iYPy5n4HQ6deONN6p9+/YBCwYAAFBT+FTIYmJi9OWXX+rTTz/V4cOHNXr0aO3Zs0eFhYWBzgcAAFDt+bTL8oMPPtDcuXOVmJiob7/9VpIUERGhpUuXBjQcAABATeBTIXv//fc1btw4rzMrzz///JA9RRUAACCY+FTICgsL1aBBA6+x0tJShYX5tMcTAAAAp+FTo2rVqpWWL1+uvn37esY++OADtWnTJmDBYJ+yJS9Ja97/Y6BbT7kG3GNfIADwQ9mmtdLiWVJJiRQeLt0+TK4OXe2OBZyWT4Vs8ODBeuqpp7Rq1SoVFRVp+PDhioqK4rIX1VCFMiZJa95XmUQpAxD0yjatleZN+2Og+Ddp3rTyOYxShiDmUyGrX7++pkyZoj179ignJ0dut1vNmjXjSv3V0cll7MRxChmAYLd41qnHKWQIYj4fBOZwONSsWTM1a9YskHkAADh7JSX+jQNBgk1cAIDqIzzcv3EgSFDI4K1bT//GASCY3D7Mv3EgSHDdCnhxDbhHZRJnWQIISa4OXcvnMM6yRIihkKEC14B7OIAfQMhydejKAfwIOeyyBAAAsBmFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZhQyAAAAm1HIAAAAbEYhAwAAsFmY3QFQNcq+2yEtnCEdK5Cia0uDhsvVsq3dsQDAJ8xhqOnYQlYNlH23Q5o+XsrbLxUWlP8+fXz5OAAEOeYwgEJWPSycIRll3mNGWfk4AAQ75jCAQlYtHCvwbxwAgglzGEAhqxaia/s3DgDBhDkMoJBVC4OGS06X95jTVT4OAMGOOQygkFUHrpZtpZETJXdDKap2+e8jJ3KGEoCQwBwGcNmLasPVsq00dZ7dMQDgrDjc8VKzVjIP5stRL05yx9sdCbAUhQwAYCsjJ1vm9PFSTrYkyZSkjO9ljJwoZ3yCrdkAq7DLEgBgrxVLPGXMIye7fByoIShkAABbmQfz/RoHqiPLdlmmp6drwYIFMgxDPXr0UJ8+fbyWv/vuu1q1apVcLpfq1q2re++9V/HxHEMAwH7MX4HlqBdXvpuyknGgprCkkBmGofnz52vs2LFyu9169NFHlZycrAsuuMCzTtOmTTV16lTVqlVLH3/8sV577TWNHDnSinhBy/h9k/2+wqMyomKk3gM4ngKwGPPX2fFr/uo9QMr43nu3ZXxC+ThQQ1hSyHbv3q2EhAQ1atRIktSpUydt3rzZa0K75JJLPH9u3ry51q9fb0W0oHXiQa7Fxwc5yBWwHPOX//ydv5zxCTJGTpRWLPnjLEv+A4oaxpJjyPLz8+V2uz233W638vNPfWzA6tWrdemll1qQLIhxkCsQFJi/zsJZzF/O+AQ5h/xLrocmyznkX5Qx1DhBd9mLdevWKSMjQ48//nily9PS0pSWliZJmjp1qhITE/16fH/Xt8u+wqN//M/yBBGFR9UwRF6DFDrv98nIba1QzX0y5q9yzF/2Irf1qiK7JYUsLi5OeXl5ntt5eXmKi6t4sOaXX36pZcuW6fHHH1d4eHilj5WSkqKUlBTP7czMTJ9zJCYm+rW+nYyomErHi6NiQuY1hNL7fSJyW8vf3FZP2sxf/mP+sg+5redP9tPNX5bsskxKSlJWVpb279+v0tJSbdy4UcnJyV7r/PDDD5o7d65GjRql2NhYK2IFt94Dyg9qPREHuQKWY/46C8xfgN8s2ULmcrk0ePBgTZ48WYZhqHv37mrcuLFSU1OVlJSk5ORkvfbaayoqKtJzzz0nSWrQoIEeeeQRK+IFpRMPco0oPKpizrIEbMH85T/mL8B/DtM0K7v8S8iorpv8T0Rua5HbWsG+yzKQmL+CF7mtFaq5parbZRl0B/XXJGXf7ZAWzpCOFUjRtaVBw8u/JBwAgtzx64xxmQqgalDIbFL23Q5p+njJKCsfKCyQpo9X2ciJlDIAQY0vAweqHt9laZeFM/4oY8cZZeXjABDMuE4iUOUoZHY5VuDfOAAECb4MHKh6FDK7RNf2bxwAgsSpvvSbLwMHzh6FzC6DhktOl/eY01U+DgDBjOuMAVWOg/pt4mrZVmUjJ3KWJYCQw5eBA1WPQmYjV8u20tR5dscAAL854xOkIf+yOwZQbbDLEgAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZhQyAAAAm1HIAAAAbEYhAwAAsBmFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZhQyAAAAm1HIAAAAbEYhAwAAsBmFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZmFWPVF6eroWLFggwzDUo0cP9enTx2t5SUmJZs2apYyMDNWpU0cjRoxQw4YNq+S5y/5xvSTplxPGXHNXVsljA6j+mL9Ci5GTLa1YIvNgvhz14qTeA+SMT7A7FnBalmwhMwxD8+fP15gxYzR9+nRt2LBBe/fu9Vpn9erVql27tmbOnKlevXppyZIlVfLcxyczX8cB4ETMX6HFyMmWOX28zE1rpe93yNy0Vub08eUlDQhilhSy3bt3KyEhQY0aNVJYWJg6deqkzZs3e62zZcsWdevWTZJ01VVX6auvvpJpmlbEA4BTYv4KMSuWSCeXr9+3mAHBzJJdlvn5+XK73Z7bbrdbu3btOuU6LpdL0dHROnLkiOrWreu1XlpamtLS0iRJU6dOVWJi4mmf+5fTLDvTfYNJKGU9EbmtRe6qx/x17qzMuq/wqIorGY8oPKqGfuYIpff4ROS2hmmaysnJ0a+//lphWXh4uOLj4+VwOHx+PMuOIasqKSkpSklJ8dzOzMw868c6l/taKTExMWSynojc1qopuUNt0j4R81fgGVExlY4XR8X4/XMWKu/xichtncLCQoWHhysqKkolJSVey4qKipSRkaGoqCiv8dPNX5bssoyLi1NeXp7ndl5enuLi4k65TllZmY4dO6Y6depYEQ8ATon5K8T0HiCdfAB/fEL5OFCFDMNQWFjl27XCwsJkGIZfj2dJIUtKSlJWVpb279+v0tJSbdy4UcnJyV7rXH755VqzZo0k6fPPP1ebNm382tR3Kqc6G4mzlAD4gvkrtDjjE+QYOVGODl2li9vK0aGrHCMncpYlqtyZ/o77OwdYssvS5XJp8ODBmjx5sgzDUPfu3dW4cWOlpqYqKSlJycnJ+stf/qJZs2bp/vvvV0xMjEaMGFF1z//75BWKm0QB2Iv5K/Q44xOkIf+yOwbgF8uOIWvfvr3at2/vNda/f3/PnyMiIvTggw9aFQcAfMb8BSDQuFI/AACAn850aRt/L31DIQMAAPCT0+lUaWlppctKS0vldPpXsULushcAAAB2i4yMVFFRkVwulwoLCz3jpmnK6XQqMjLSr8ejkAEAAPjJ4XAoKiqqyk64YZclAACAzShkAAAANqOQAQAA2Mxh+nteJgAAAKpUjdpCNnr0aLsjnBVyW4vc1grV3FYL1feJ3NYit/WqKnuNKmQAAADBiEIGAABgsxpVyFJSUuyOcFbIbS1yWytUc1stVN8ncluL3Narquwc1A8AAGCzGrWFDAAAIBhVu69OSk9P14IFC2QYhnr06KE+ffp4LS8pKdGsWbOUkZGhOnXqaMSIEWrYsKE9YU9wptzvvvuuVq1aJZfLpbp16+ree+9VfHy8PWFPcqbsx33++ed67rnnNGXKFCUlJVkbshK+5N64caPefPNNORwONWnSRMOHD7c+6EnOlDs3N1ezZ89WQUGBDMPQrbfeqvbt29sT9ncvvviitm3bptjYWE2bNq3CctM0tWDBAm3fvl21atXS0KFDddFFF9mQ1H7MYdZi/rJWKM5fkkVzmFmNlJWVmcOGDTOzs7PNkpIS86GHHjJ/+eUXr3U+/PBDc86cOaZpmuann35qPvfcc3ZE9eJL7h07dphFRUWmaZrmRx99FBS5TdO37KZpmseOHTPHjx9vjhkzxty9e7cNSb35kjszM9N8+OGHzSNHjpimaZoHDx60I6oXX3K/9NJL5kcffWSapmn+8ssv5tChQ+2I6uXrr7829+zZYz744IOVLt+6das5efJk0zAM8/vvvzcfffRRixMGB+YwazF/WStU5y/TtGYOq1a7LHfv3q2EhAQ1atRIYWFh6tSpkzZv3uy1zpYtW9StWzdJ0lVXXaWvvvpKps2H0fmS+5JLLlGtWrUkSc2bN1d+fr4dUSvwJbskpaamqnfv3goPD7chZUW+5F61apX++te/KiYmRpIUGxtrR1QvvuR2OBw6duyYJOnYsWOqX7++HVG9tG7d2vM+VmbLli3q0qWLHA6HWrRooYKCAh04cMDChMGBOcxazF/WCtX5S7JmDqtWhSw/P19ut9tz2+12V/hLf+I6LpdL0dHROnLkiKU5T+ZL7hOtXr1al156qQXJzsyX7BkZGcrNzQ2Kzc7H+ZI7MzNTWVlZGjdunB577DGlp6dbnLIiX3L369dP69ev1z333KMpU6Zo8ODBVsf0W35+vho0aOC5faa/A9UVc5i1mL+sVV3nL6lq5rBqVchqgnXr1ikjI0PXX3+93VF8YhiGFi9erNtvv93uKH4zDENZWVmaMGGChg8frjlz5qigoMDuWGe0YcMGdevWTS+99JIeffRRzZw5U4Zh2B0LkBRacxjzl/Vq8vxVrQpZXFyc8vLyPLfz8vIUFxd3ynXKysp07Ngx1alTx9KcJ/MltyR9+eWXWrZsmUaNGhU0m87PlL2oqEi//PKLnnjiCd13333atWuXnn76ae3Zs8eOuB6+/qwkJycrLCxMDRs21HnnnaesrCyro1bIdKbcq1evVseOHSVJLVq0UElJie1bUM4kLi5Oubm5ntun+jtQ3TGHWYv5y1rVdf6SqmYOq1aFLCkpSVlZWdq/f79KS0u1ceNGJScne61z+eWXa82aNZLKz5pp06aNHA6HDWn/4EvuH374QXPnztWoUaOC4liA486UPTo6WvPnz9fs2bM1e/ZsNW/eXKNGjbL9LCVf3vMrr7xSX3/9tSTp8OHDysrKUqNGjeyI6+FL7gYNGuirr76SJO3du1clJSWqW7euHXF9lpycrHXr1sk0Te3cuVPR0dFBc+yIlZjDrMX8Za3qOn9JVTOHVbsLw27btk2LFi2SYRjq3r27+vbtq9TUVCUlJSk5OVnFxcWaNWuWfvjhB8XExGjEiBG2/5D6kvvJJ5/Uzz//rHr16kkq/6F95JFH7A39uzNlP9Hjjz+u2267zfYJTTpzbtM0tXjxYqWnp8vpdKpv377q3Lmz3bHPmHvv3r2aM2eOioqKJEkDBw5Uu3btbM38/PPP65tvvtGRI0cUGxurm266SaWlpZKka6+9VqZpav78+friiy8UERGhoUOHBsXPiB2Yw6zF/BVcuYNx/pKsmcOqXSEDAAAINdVqlyUAAEAoopABAADYjEIGAABgMwoZAACAzShkAAAANqOQIei9/PLLeuutt+yOAQBAwHDZC+AU3njjDWVnZ+uBBx445Toffvih1qxZo59//lmdO3fWfffdZ2FCAEB1EWZ3AOB0DMOQ0xm8G3Lr16+vvn376osvvlBxcbHdcQAAIYpChlPKz8/XK6+8om+//VaRkZHq1auXevbsqaNHj+qhhx7SkCFDlJycrKKiIj388MO68cYb1bVrV82ePVvh4eHat2+fdu3apQsvvFDDhg1TfHy8JOnXX3/VK6+8ooyMDNWtW1f9+/dXp06dJEmzZ89WRESEcnNz9c033+jhhx/W+vXr5Xa7dfPNN+vrr7/WzJkz9be//U3/+7//K6fTqSFDhigsLEyLFi3S4cOHdd1116lv376SygvdypUrtWrVKhUUFOiSSy7R3XffrZiYGO3fv1/Dhg3T0KFDlZqaquLiYvXq1Ut9+/ZVenq6li1bJknavHmzEhIS9Mwzz1R4jzp06CBJysjI8PqONgAA/BG8mx5gK8Mw9NRTT6lp06aaM2eOxo8fr/fff1/p6emKiYnRvffeqzlz5ujQoUNauHChmjZtqq5du3ru/+mnn+rvf/+75s+fr6ZNm+qFF16QVP5lvZMmTdLVV1+tefPmacSIEZo/f7727t3rdd8bbrhBixYtUsuWLStkO3jwoEpKSvTSSy/ppptu0pw5c7R+/XpNnTpVEydO1Ntvv639+/dLKt+luHnzZj3++OOaM2eOYmJiNG/ePK/H++677zRjxgyNGzdOb731lvbu3atLL71UN9xwgzp27KhXX3210jIGAEBVoZChUnv27NHhw4d14403KiwsTI0aNVKPHj20ceNGSVK7du3UsWNHTZw4Udu3b9fdd9/tdf/27durdevWCg8P1y233KKdO3cqNzdX27ZtU3x8vLp37y6Xy6ULL7xQHTp00Geffea57xVXXKGWLVvK6XQqIiKiQjaXy6W+ffsqLCxMnTt31pEjR9SzZ09FRUWpcePGuuCCC/Tjjz9Kkj755BPdfPPNcrvdCg8PV79+/bRp0yaVlZV5Hq9fv36KiIhQ06ZN1aRJE/30008BeEcBADg1dlmiUjk5OTpw4IAGDRrkGTMMQ61atfLcTklJ0YcffqgbbrhBderU8bq/2+32/DkyMlIxMTE6cOCAcnJytGvXLq/HLSsrU5cuXSq9b2Xq1KnjOa7seGGLjY31LI+IiPB8MW1OTo6effZZORwOz3Kn06lDhw55bh//smNJqlWrlue+AABYhUKGSjVo0EANGzb07Go8mWEYmjNnjrp06aKPP/5Y3bt3V0JCgmf5icdTFRUV6ejRo6pfv77cbrdat26tcePGnfK5TyxP58rtduvee++tdNfn8d2aVuQAAOB02GWJSjVr1kxRUVFavny5iouLZRiGfv75Z+3evVuS9M4778jhcGjo0KG67rrrNGvWLBmG4bn/9u3b9d1336m0tFRLly5VixYt1KBBA11++eXKysrSunXrVFpaqtLSUu3evdvrGLKqdM0112jp0qXKycmRJB0+fFibN2/26b6xsbHKycnxel0nKysr87w/hmGouLjYa3coAAC+YAsZKuV0OvXII49o8eLFuu+++1RaWqrExET1799fGRkZeu+99zRlyhQ5nU716dNH27dv1/Llyz1nN3bu3Flvvvmmdu7cqYsuukj333+/JCkqKkpjx47VokWLtGjRIpmmqSZNmuiOO+4IyOvo2bOnJGnSpEk6cOCAYmNj1bFjR11xxRVnvG/Hjh21fv163XXXXWrYsKGeeuqpCuu8/fbbXhetXb9+vW688UbddNNNVfciAADVHheGRZWbPXu25zIVAADgzNhlCQAAYDMKGQAAgM3YZQkAAGAztpABAADYjEIGAABgMwoZAACAzShkAAAANqOQAQAA2IxCBgAAYLP/B5nB+ofFwG+kAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.style.use('ggplot') \n", + "fig,ax = plt.subplots(1,2,figsize=(10, 5))\n", + "ax[0].scatter(read_score(\"prompt_to_json_eli5\"),read_score(\"prompt_to_json_eli5_1\"),marker=\"o\")\n", + "ax[0].set_title(\"JSON\")\n", + "ax[1].scatter(read_score(\"native_eli5\"),read_score(\"native_eli5_1\"),marker=\"o\")\n", + "ax[1].set_title(\"Native\")\n", + "ax[0].set_xlabel(\"experiment 1\")\n", + "ax[0].set_ylabel(\"experiment 2\")\n", + "\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "b7e6a507", + "metadata": {}, + "source": [ + "## Self consistency \n", + "The idea is to select the sample with highest similarity to other samples, among k generated samples\n", + "1) n-gram + cosine similarity\n", + "\n", + "2) embeddings + cosine similarity\n", + "\n", + "### Use case\n", + "- building this to select the best from k generations that are all json\n", + "- For each json in output\n", + " - for each key in json\n", + " - select sample with that maximises sim score " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77aff1ff", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "ragas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 7ba5f46b4736e7f3a73477f6da439a2c012ffa1b Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Tue, 21 Nov 2023 23:18:55 +0530 Subject: [PATCH 05/11] json loader --- src/ragas/metrics/_context_precision.py | 30 ++++++++++++++++++++----- src/ragas/utils.py | 16 +++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/ragas/metrics/_context_precision.py b/src/ragas/metrics/_context_precision.py index 322ccb0c9..143728d67 100644 --- a/src/ragas/metrics/_context_precision.py +++ b/src/ragas/metrics/_context_precision.py @@ -9,14 +9,28 @@ from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate from ragas.metrics.base import EvaluationMode, MetricWithLLM +from ragas.utils import load_as_json CONTEXT_PRECISION = HumanMessagePromptTemplate.from_template( """\ -Given a question and a context, verify if the information in the given context is useful in answering the question. Return a Yes/No answer. +Verify if the information in the given context is useful in answering the question. + +question: What are the health benefits of green tea? +context: +This article explores the rich history of tea cultivation in China, tracing its roots back to the ancient dynasties. It discusses how different regions have developed their unique tea varieties and brewing techniques. The article also delves into the cultural significance of tea in Chinese society and how it has become a symbol of hospitality and relaxation. +verification: +{{"reason":"The context, while informative about the history and cultural significance of tea in China, does not provide specific information about the health benefits of green tea. Thus, it is not useful for answering the question about health benefits.", "verdict":"No"}} + +question: How does photosynthesis work in plants? +context: +Photosynthesis in plants is a complex process involving multiple steps. This paper details how chlorophyll within the chloroplasts absorbs sunlight, which then drives the chemical reaction converting carbon dioxide and water into glucose and oxygen. It explains the role of light and dark reactions and how ATP and NADPH are produced during these processes. +verification: +{{"reason":"This context is extremely relevant and useful for answering the question. It directly addresses the mechanisms of photosynthesis, explaining the key components and processes involved.", "verdict":"Yes"}} + question:{question} -context:\n{context} -answer: -""" # noqa: E501 +context: +{context} +verification:""" # noqa: E501 ) @@ -75,7 +89,13 @@ def _score_batch( scores = [] for response in grouped_responses: - response = [int(any("yes" in r.lower() for r in resp)) for resp in response] + response = [load_as_json(item) for item in sum(response, [])] + response = [ + int("yes" in resp.get("verdict").lower()) + if resp.get("verdict") + else np.nan + for resp in response + ] denominator = sum(response) + 1e-10 numerator = sum( [ diff --git a/src/ragas/utils.py b/src/ragas/utils.py index c5db04ba9..e1890df08 100644 --- a/src/ragas/utils.py +++ b/src/ragas/utils.py @@ -1,7 +1,9 @@ from __future__ import annotations +import json import logging import os +import warnings from functools import lru_cache DEBUG_ENV_VAR = "RAGAS_DEBUG" @@ -16,3 +18,17 @@ def get_debug_mode() -> bool: return True else: return False + + +def load_as_json(text): + """ + validate and return given text as json + """ + + try: + return json.loads(text) + except ValueError: + print(text) + warnings.warn("Invalid json") + + return {} From 1df47bf6a14b2b6c82a6153a252235c8e63c9d55 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Tue, 21 Nov 2023 23:19:16 +0530 Subject: [PATCH 06/11] replace with nanmean --- src/ragas/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ragas/evaluation.py b/src/ragas/evaluation.py index e7e270b03..05be351cf 100644 --- a/src/ragas/evaluation.py +++ b/src/ragas/evaluation.py @@ -131,7 +131,7 @@ class Result(dict): def __post_init__(self): values = [] for cn in self.scores.column_names: - value = np.mean(self.scores[cn]) + value = np.nanmean(self.scores[cn]) self[cn] = value if cn not in self.binary_columns: value = t.cast(float, value) From d54ef720fc963fa83e559364cabe52507b5ab6e5 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Tue, 21 Nov 2023 23:19:40 +0530 Subject: [PATCH 07/11] move json loader --- src/ragas/testset/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/ragas/testset/utils.py b/src/ragas/testset/utils.py index bb97e5ac4..4582a89ec 100644 --- a/src/ragas/testset/utils.py +++ b/src/ragas/testset/utils.py @@ -1,23 +1,9 @@ from __future__ import annotations -import json import re import warnings -def load_as_json(text): - """ - validate and return given text as json - """ - - try: - return json.loads(text) - except ValueError: - warnings.warn("Invalid json") - - return {} - - def load_as_score(text): """ validate and returns given text as score From 3b1878d7ed05983319c7cc273e1f67de714fc7a3 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Tue, 21 Nov 2023 23:19:49 +0530 Subject: [PATCH 08/11] move json loader --- src/ragas/testset/testset_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ragas/testset/testset_generator.py b/src/ragas/testset/testset_generator.py index 67cbcc3c9..4726f5182 100644 --- a/src/ragas/testset/testset_generator.py +++ b/src/ragas/testset/testset_generator.py @@ -32,7 +32,8 @@ SCORE_CONTEXT, SEED_QUESTION, ) -from ragas.testset.utils import load_as_json, load_as_score +from ragas.testset.utils import load_as_score +from ragas.utils import load_as_json if t.TYPE_CHECKING: from ragas.llms.base import RagasLLM From cc128c942145f5f33e535458df2982d353ff5753 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 22 Nov 2023 11:32:54 +0530 Subject: [PATCH 09/11] fix type error --- src/ragas/metrics/_context_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ragas/metrics/_context_precision.py b/src/ragas/metrics/_context_precision.py index 143728d67..26e27082e 100644 --- a/src/ragas/metrics/_context_precision.py +++ b/src/ragas/metrics/_context_precision.py @@ -91,7 +91,7 @@ def _score_batch( for response in grouped_responses: response = [load_as_json(item) for item in sum(response, [])] response = [ - int("yes" in resp.get("verdict").lower()) + int("yes" in resp.get("verdict", " ").lower()) if resp.get("verdict") else np.nan for resp in response From 16821c44093e502a51a82a9985073e394e739ec2 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 22 Nov 2023 23:43:21 +0530 Subject: [PATCH 10/11] add error string --- src/ragas/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ragas/utils.py b/src/ragas/utils.py index 128076b64..cbdb73d40 100644 --- a/src/ragas/utils.py +++ b/src/ragas/utils.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import logging import os import warnings from functools import lru_cache @@ -26,8 +25,7 @@ def load_as_json(text): try: return json.loads(text) - except ValueError: - print(text) - warnings.warn("Invalid json") + except ValueError as e: + warnings.warn(f"Invalid json: {e}") return {} From 35fb0e68c00cb1541e182d7f5eb21266e92c16f6 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Fri, 24 Nov 2023 17:54:24 +0530 Subject: [PATCH 11/11] structured output --- src/ragas/metrics/_faithfulness.py | 145 ++++++++++++++++++----------- 1 file changed, 93 insertions(+), 52 deletions(-) diff --git a/src/ragas/metrics/_faithfulness.py b/src/ragas/metrics/_faithfulness.py index 1da97b955..95460180a 100644 --- a/src/ragas/metrics/_faithfulness.py +++ b/src/ragas/metrics/_faithfulness.py @@ -3,56 +3,107 @@ import typing as t from dataclasses import dataclass +import numpy as np from langchain.callbacks.manager import CallbackManager, trace_as_chain_group from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate from ragas.metrics.base import EvaluationMode, MetricWithLLM +from ragas.utils import load_as_json if t.TYPE_CHECKING: from datasets import Dataset -################# -# NLI Score -################# + LONG_FORM_ANSWER_PROMPT = HumanMessagePromptTemplate.from_template( """\ -Given a question and answer, create one or more statements from each sentence in the given answer. +Create one or more statements from each sentence in the given answer. + question: Who was Albert Einstein and what is he best known for? answer: He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics. -statements:\nAlbert Einstein was born in Germany.\nAlbert Einstein was best known for his theory of relativity. +statements in json: +{{ + "statements": [ + "Albert Einstein was born in Germany.", + "Albert Einstein was best known for his theory of relativity." + ] +}} + question: Cadmium Chloride is slightly soluble in this chemical, it is also called what? answer: alcohol -statements:\nCadmium Chloride is slightly soluble in alcohol. +statements in json: +{{ + "statements": [ + "Cadmium Chloride is slightly soluble in alcohol." + ] +}} + question: Were Shahul and Jithin of the same nationality? answer: They were from different countries. -statements:\nShahul and Jithin were from different countries. +statements in json: +{{ + "statements": [ + "Shahul and Jithin were from different countries." + ] +}} + question:{question} answer: {answer} -statements:\n""" # noqa: E501 +statements in json:""" # noqa: E501 ) NLI_STATEMENTS_MESSAGE = HumanMessagePromptTemplate.from_template( """ -Prompt: Natural language inference -Consider the given context and following statements, then determine whether they are supported by the information present in the context.Provide a brief explanation for each statement before arriving at the verdict (Yes/No). Provide a final verdict for each statement in order at the end in the given format. Do not deviate from the specified format. - -Context:\nJohn is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects. -statements:\n1. John is majoring in Biology.\n2. John is taking a course on Artificial Intelligence.\n3. John is a dedicated student.\n4. John has a part-time job.\n5. John is interested in computer programming.\n + Natural language inference + +Context: +John is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects. +statement_1: John is majoring in Biology. +statement_2: John is taking a course on Artificial Intelligence. +statement_3: John is a dedicated student. +statement_4: John has a part-time job. +Answer: +[ + {{ + "statement_1": "John is majoring in Biology.", + "reason": "John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.", + "verdict": "No" + }}, + {{ + "statement_2": "John is taking a course on Artificial Intelligence.", + "reason": "The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI.", + "verdict": "No" + }}, + {{ + "statement_3": "John is a dedicated student.", + "reason": "The context states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication.", + "verdict": "Yes" + }}, + {{ + "statement_4": "John has a part-time job.", + "reason": "There is no information given in the context about John having a part-time job.", + "verdict": "No" + }} +] + +Context: +Photosynthesis is a process used by plants, algae, and certain bacteria to convert light energy into chemical energy. +statement_1: Answer not found in given context Answer: -1. John is majoring in Biology. -Explanation: John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology. Verdict: No. -2. John is taking a course on Artificial Intelligence. -Explanation: The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI. Verdict: No. -3. John is a dedicated student. -Explanation: The prompt states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication. Verdict: Yes. -4. John has a part-time job. -Explanation: There is no information given in the context about John having a part-time job. Therefore, it cannot be deduced that John has a part-time job. Verdict: No. -5. John is interested in computer programming. -Explanation: The context states that John is pursuing a degree in Computer Science, which implies an interest in computer programming. Verdict: Yes. -Final verdict for each statement in order: No. No. Yes. No. Yes. -context:\n{context} -statements:\n{statements} +[ + {{ + "statement_4": "Answer not found in given context", + "reason": "The context does not provide enough information to determine the validity of the statement." + "verdict": "NULL" + }} +] + + + +context: +{context} +statements: +{statements} Answer: """ # noqa: E501 ) @@ -84,18 +135,13 @@ def _score_batch( human_prompt = LONG_FORM_ANSWER_PROMPT.format(question=q, answer=a) prompts.append(ChatPromptTemplate.from_messages([human_prompt])) - result = self.llm.generate(prompts, callbacks=batch_group) - list_statements: list[list[str]] = [] - for output in result.generations: - # use only the first generation for each prompt - statements = output[0].text.split("\n") - list_statements.append(statements) prompts = [] - for context, statements in zip(contexts, list_statements): + for context, output in zip(contexts, result.generations): + statements = load_as_json(output[0].text).get("statements", []) statements_str: str = "\n".join( - [f"{i+1}.{st}" for i, st in enumerate(statements)] + [f"statement_{i+1}: {st}" for i, st in enumerate(statements)] ) contexts_str: str = "\n".join(context) human_prompt = NLI_STATEMENTS_MESSAGE.format( @@ -105,26 +151,21 @@ def _score_batch( result = self.llm.generate(prompts, callbacks=batch_group) outputs = result.generations - + verdict_score_map = {"yes": 1, "no": 0, "null": np.nan} scores = [] - final_answer = "Final verdict for each statement in order:" - final_answer = final_answer.lower() - for i, output in enumerate(outputs): - output = output[0].text.lower().strip() - if final_answer in output: - output = output[output.find(final_answer) + len(final_answer) :] - score = sum( - 0 if "yes" in answer else 1 - for answer in output.strip().split(".") - if answer != "" - ) - score = score / len(list_statements[i]) + for output in outputs: + output = load_as_json(output[0].text) + output = output if output else [] + faithful_statements = sum( + verdict_score_map.get(dict.get("verdict", "").lower(), np.nan) + for dict in output + ) + num_statements = len(output) + if num_statements: + score = faithful_statements / num_statements else: - score = max(0, output.count("verdict: no")) / len( - list_statements[i] - ) - - scores.append(1 - score) + score = np.nan + scores.append(score) return scores