Fixed metric adaptation issue with the prompt factory

src/ragas/llms/prompt.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import ast
+import copy
 import json
 import logging
 import os
@@ -54,6 +55,12 @@ class Prompt(BaseModel):
     output_type: t.Literal["json", "str"] = "json"
     language: str = "english"
 
+    def factory(self):
+        """
+        Returns a new Prompt instance for this prompt.
+        """
+        return copy.deepcopy(self)
+
     @root_validator
     def validate_prompt(cls, values: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
         """
@@ -306,6 +313,7 @@ def _load(cls, language: str, name: str, cache_dir: str) -> Prompt:
     output_type="str",
 )
 
+
 json_translatation = Prompt(
     name="json_translation",
     instruction="Translate values in given json to target language and output the translated json",

src/ragas/metrics/_answer_correctness.py

@@ -158,9 +158,9 @@ class AnswerCorrectness(MetricWithLLM, MetricWithEmbeddings):
 
     name: str = "answer_correctness"  # type: ignore[reportIncompatibleMethodOverride]
     evaluation_mode: EvaluationMode = EvaluationMode.qga  # type: ignore[reportIncompatibleMethodOverride]
-    correctness_prompt: Prompt = field(default_factory=lambda: CORRECTNESS_PROMPT)
+    correctness_prompt: Prompt = field(default_factory=CORRECTNESS_PROMPT.factory)
     long_form_answer_prompt: Prompt = field(
-        default_factory=lambda: LONG_FORM_ANSWER_PROMPT
+        default_factory=LONG_FORM_ANSWER_PROMPT.factory
     )
     weights: list[float] = field(default_factory=lambda: [0.75, 0.25])
     answer_similarity: t.Optional[AnswerSimilarity] = None

src/ragas/metrics/_answer_relevance.py

@@ -103,7 +103,7 @@ class AnswerRelevancy(MetricWithLLM, MetricWithEmbeddings):
 
     name: str = "answer_relevancy"  # type: ignore
     evaluation_mode: EvaluationMode = EvaluationMode.qac  # type: ignore
-    question_generation: Prompt = field(default_factory=lambda: QUESTION_GEN)
+    question_generation: Prompt = field(default_factory=QUESTION_GEN.factory)
     strictness: int = 3
 
     def calculate_similarity(

src/ragas/metrics/_context_entities_recall.py

@@ -132,7 +132,7 @@ class ContextEntityRecall(MetricWithLLM):
     name: str = "context_entity_recall"  # type: ignore
     evaluation_mode: EvaluationMode = EvaluationMode.gc  # type: ignore
     context_entity_recall_prompt: Prompt = field(
-        default_factory=lambda: TEXT_ENTITY_EXTRACTION
+        default_factory=TEXT_ENTITY_EXTRACTION.factory
     )
     batch_size: int = 15
     max_retries: int = 1

src/ragas/metrics/_context_precision.py

@@ -87,7 +87,7 @@ class ContextPrecision(MetricWithLLM):
 
     name: str = "context_precision"  # type: ignore
     evaluation_mode: EvaluationMode = EvaluationMode.qcg  # type: ignore
-    context_precision_prompt: Prompt = field(default_factory=lambda: CONTEXT_PRECISION)
+    context_precision_prompt: Prompt = field(default_factory=CONTEXT_PRECISION.factory)
     max_retries: int = 1
     _reproducibility: int = 1
 

src/ragas/metrics/_context_recall.py

@@ -120,7 +120,7 @@ class ContextRecall(MetricWithLLM):
 
     name: str = "context_recall"  # type: ignore
     evaluation_mode: EvaluationMode = EvaluationMode.qcg  # type: ignore
-    context_recall_prompt: Prompt = field(default_factory=lambda: CONTEXT_RECALL_RA)
+    context_recall_prompt: Prompt = field(default_factory=CONTEXT_RECALL_RA.factory)
     max_retries: int = 1
     _reproducibility: int = 1
 

src/ragas/metrics/_faithfulness.py

@@ -168,9 +168,9 @@ class Faithfulness(MetricWithLLM):
     name: str = "faithfulness"  # type: ignore
     evaluation_mode: EvaluationMode = EvaluationMode.qac  # type: ignore
     nli_statements_message: Prompt = field(
-        default_factory=lambda: NLI_STATEMENTS_MESSAGE
+        default_factory=NLI_STATEMENTS_MESSAGE.factory
     )
-    statement_prompt: Prompt = field(default_factory=lambda: LONG_FORM_ANSWER_PROMPT)
+    statement_prompt: Prompt = field(default_factory=LONG_FORM_ANSWER_PROMPT.factory)
     sentence_segmenter: t.Optional[HasSegmentMethod] = None
     max_retries: int = 1
     _reproducibility: int = 1

src/ragas/metrics/_rubrics_based.py

@@ -112,7 +112,7 @@ class LabelledRubricsScore(MetricWithLLM):
         default_factory=lambda: DEFAULT_WITH_REFERENCE_RUBRICS
     )
     scoring_prompt: Prompt = field(
-        default_factory=lambda: WITH_REFERENCE_SCORING_PROMPT
+        default_factory=WITH_REFERENCE_SCORING_PROMPT.factory
     )
     max_retries: int = 1
 
@@ -165,7 +165,7 @@ class ReferenceFreeRubricsScore(LabelledRubricsScore):
         default_factory=lambda: DEFAULT_REFERENCE_FREE_RUBRICS
     )
     scoring_prompt: Prompt = field(
-        default_factory=lambda: WITHOUT_REFERENCE_SCORING_PROMPT
+        default_factory=WITHOUT_REFERENCE_SCORING_PROMPT.factory
     )
     max_retries: int = 1
 

src/ragas/metrics/_summarization.py

@@ -147,10 +147,10 @@ class SummarizationScore(MetricWithLLM):
     length_penalty: bool = True
     evaluation_mode: EvaluationMode = EvaluationMode.ca  # type: ignore[reportIncompatibleMethodOverride]
     question_generation_prompt: Prompt = field(
-        default_factory=lambda: TEXT_GENERATE_QUESTIONS
+        default_factory=TEXT_GENERATE_QUESTIONS.factory
     )
     answer_generation_prompt: Prompt = field(
-        default_factory=lambda: TEXT_GENERATE_ANSWERS
+        default_factory=TEXT_GENERATE_ANSWERS.factory
     )
 
     def _get_extract_keyphrases_prompt(self, text) -> PromptValue:

src/ragas/metrics/critique.py

@@ -73,7 +73,7 @@ class AspectCritique(MetricWithLLM):
 
     name: str = field(default="", repr=True)  # type: ignore
     evaluation_mode: EvaluationMode = EvaluationMode.qac  # type: ignore
-    critic_prompt: Prompt = field(default_factory=lambda: CRITIQUE_PROMPT)
+    critic_prompt: Prompt = field(default_factory=CRITIQUE_PROMPT.factory)
     definition: str = field(default="", repr=True)
     strictness: int = field(default=1, repr=False)
     llm: BaseRagasLLM | None = field(