prompt_templates.py

# Query engine setup
TEXT_QA_TEMPLATE = (
    "<|system|>\n"
    "You are an AI language model developed by IBM Research. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless, and you follow ethical guidelines and promote positive behavior.\n"
    "<|user|>\n"
    "{context_str}\n"
    "Given the above context information, answer the query.\n"
    "{query_str}\n"
    "<|assistant|>\n"
)
TEXT_QA_TEMPLATE_DEFAULT = (
    "Context information is below.\n"
    "---------------------\n"
    "{context_str}\n"
    "---------------------\n"
    "Given the context information, answer the query.\n"
    "Query: {query_str}\n"
    "Answer:\n"
)
REFINE_TEMPLATE = (
    "<|system|>\n"
    "You are an AI language model developed by IBM Research. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless, and you follow ethical guidelines and promote positive behavior.\n"
    "<|user|>\n"
    "Document 0:"
    "{context_msg}\n"
    "Document 1:\n"
    "{existing_answer}\n"
    "Given the context information, answer the query.\n"
    "{query_str}\n"
    "<|assistant|>\n"
)

SCORING_PROMPT_A = (
    "Document: {document}\n"
    "\n"
    "Question: {question}\n"
    "\n"
    "Ground Truth: {Ground Truth}\n"
    "\n"
    "RAG Answer from System A: {agent_rag_model_0}\n"
    "\n"
    "RAG Answer from System B: {agent_rag_model_1}\n"
    "\n"
    "Please evaluate the two RAG answers based on the following criteria:\n"
    "\n"
    "1. Accuracy: Which answer more closely aligns with the ground truth?\n"
    "2. Completeness: Which answer provides more relevant details from the document?\n"
    "3. Conciseness: Which answer conveys the necessary information more efficiently?\n"
    "4. Clarity: Which answer is easier to understand and more clearly expressed?\n"
    "5. Relevance: Which answer better addresses the specific question asked?\n"
    "6. Source Utilization: Which answer demonstrates better use of the information provided in the document?\n"
    "7. Objectivity: Are both answers equally objective, or does one show bias?\n"
    "8. Consistency: Is one answer more internally consistent than the other?\n"
    "9. Precision: Which answer provides more precise information when applicable?\n"
    "10. Overall Quality: Considering all factors, which answer would be more helpful to a user seeking information?\n"
    "\n"
    "Based on these criteria, which RAG answer is better: System A, System B, or are they equally good? Please provide a brief explanation for your choice. If both answers are equally good, please write 'EQUAL' in the answer tag.\n"
    "\n"
    "Example response when one system is better:\n"
    "```\n"
    "<reasoning>System A's answer is more accurate and aligns closely with the ground truth. It provides more relevant details from the document and addresses the question more directly. While System B's answer is concise, it lacks some key information. System A's response is clearer and more precise, demonstrating better use of the source material. Both answers appear objective, but System A's answer is more internally consistent and provides a more comprehensive explanation.</reasoning>\n"
    "<answer>System A</answer>\n"
    "```\n"
    "Example response when both systems are equally good:\n"
    "```\n"
    "<reasoning>Both System A and System B's answers are accurate and align closely with the ground truth. They provide relevant details from the document and address the question directly. Both answers are clear, precise, and demonstrate good use of the source material. They are objective and internally consistent. Therefore, it's hard to determine which one is better.</reasoning>\n"
    "<answer>EQUAL</answer>\n"
    "```\n"
    "Please format your response similarly, with your reasoning followed by your final answer enclosed in <answer> tags."
)

SCORING_PROMPT_B = (
    "Document: {document}\n"
    "\n"
    "Question: {question}\n"
    "\n"
    "Ground Truth: {Ground Truth}\n"
    "\n"
    "RAG Answer from System A: {agent_rag_model_1}\n"
    "\n"
    "RAG Answer from System B: {agent_rag_model_0}\n"
    "\n"
    "Please evaluate the two RAG answers based on the following criteria:\n"
    "\n"
    "1. Accuracy: Which answer more closely aligns with the ground truth?\n"
    "2. Completeness: Which answer provides more relevant details from the document?\n"
    "3. Conciseness: Which answer conveys the necessary information more efficiently?\n"
    "4. Clarity: Which answer is easier to understand and more clearly expressed?\n"
    "5. Relevance: Which answer better addresses the specific question asked?\n"
    "6. Source Utilization: Which answer demonstrates better use of the information provided in the document?\n"
    "7. Objectivity: Are both answers equally objective, or does one show bias?\n"
    "8. Consistency: Is one answer more internally consistent than the other?\n"
    "9. Precision: Which answer provides more precise information when applicable?\n"
    "10. Overall Quality: Considering all factors, which answer would be more helpful to a user seeking information?\n"
    "\n"
    "Based on these criteria, which RAG answer is better: System A, System B, or are they equally good? Please provide a brief explanation for your choice. If both answers are equally good, please write 'EQUAL' in the answer tag.\n"
    "\n"
    "Example response when one system is better:\n"
    "```\n"
    "<reasoning>System A's answer is more accurate and aligns closely with the ground truth. It provides more relevant details from the document and addresses the question more directly. While System B's answer is concise, it lacks some key information. System A's response is clearer and more precise, demonstrating better use of the source material. Both answers appear objective, but System A's answer is more internally consistent and provides a more comprehensive explanation.</reasoning>\n"
    "<answer>System A</answer>\n"
    "```\n"
    "Example response when both systems are equally good:\n"
    "```\n"
    "<reasoning>Both System A and System B's answers are accurate and align closely with the ground truth. They provide relevant details from the document and address the question directly. Both answers are clear, precise, and demonstrate good use of the source material. They are objective and internally consistent. Therefore, it's hard to determine which one is better.</reasoning>\n"
    "<answer>EQUAL</answer>\n"
    "```\n"
    "Please format your response similarly, with your reasoning followed by your final answer enclosed in <answer> tags."
)