From 5cdd83c51ed95ae7f87de6ee0fbab6bad035c343 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 13:10:39 -0700 Subject: [PATCH 1/7] Updating doc strings for evaluate API --- .../promptflow/evals/evaluate/_evaluate.py | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index 9437541d082..a5b1fbb7424 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -269,12 +269,13 @@ def evaluate( target: Optional[Callable] = None, data: Optional[str] = None, evaluators: Optional[Dict[str, Callable]] = None, - evaluator_config: Optional[Dict[str, Dict[str, str]]] = {}, + evaluator_config: Optional[Dict[str, Dict[str, str]]] = None, azure_ai_project: Optional[Dict] = None, output_path: Optional[str] = None, **kwargs, ): - """Evaluates target or data with built-in evaluation metrics + """Evaluates target or data with built-in or custom evaluators. If both target and data are provided, + data will be run through target function and then results will be evaluated. :keyword evaluation_name: Display name of the evaluation. :paramtype evaluation_name: Optional[str] @@ -283,14 +284,26 @@ def evaluate( :keyword data: Path to the data to be evaluated or passed to target if target is set. Only .jsonl format files are supported. `target` and `data` both cannot be None :paramtype data: Optional[str] - :keyword evaluator_config: Configuration for evaluators. + :keyword evaluators: Evaluators to be used for evaluation. It should be a dictionary with key as alias for evaluator + and value as the evaluator function. + :paramtype evaluators: Optional[Dict[str, Callable] + :keyword evaluator_config: Configuration for evaluators. The configuration should be a dictionary with evaluator + names as keys and a dictionary of column mappings as values. The column mappings should be a dictionary with + keys as the column names in the evaluator input and values as the column names in the input data or data + generated by target. :paramtype evaluator_config: Optional[Dict[str, Dict[str, str]] - :keyword output_path: The local folder path to save evaluation artifacts to if set + :keyword output_path: The local folder or file path to save evaluation results to if set. If folder path is provided, + the results will be saved to a file named `evaluation_results.json` in the folder. :paramtype output_path: Optional[str] :keyword azure_ai_project: Logs evaluation results to AI Studio + Example: { + "subscription_id": "", + "resource_group_name": "", + "project_name": "" + } :paramtype azure_ai_project: Optional[Dict] - :return: A EvaluationResult object. - :rtype: ~azure.ai.generative.evaluate.EvaluationResult + :return: Evaluation results. + :rtype: dict """ trace_destination = _trace_destination_from_project_scope(azure_ai_project) if azure_ai_project else None @@ -298,6 +311,8 @@ def evaluate( input_data_df = _validate_and_load_data(target, data, evaluators, output_path, azure_ai_project, evaluation_name) # Process evaluator config to replace ${target.} with ${data.} + if evaluator_config is None: + evaluator_config = {} evaluator_config = _process_evaluator_config(evaluator_config) _validate_columns(input_data_df, evaluators, target, evaluator_config) From b1f8e9db7fde2b40e91a0a8cb3e5e6e4a001b110 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 13:14:40 -0700 Subject: [PATCH 2/7] Updating doc strings for evaluate API --- src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index a5b1fbb7424..5865431f669 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -292,7 +292,7 @@ def evaluate( keys as the column names in the evaluator input and values as the column names in the input data or data generated by target. :paramtype evaluator_config: Optional[Dict[str, Dict[str, str]] - :keyword output_path: The local folder or file path to save evaluation results to if set. If folder path is provided, + :keyword output_path: The local folder or file path to save evaluation results to if set. If folder path is provided the results will be saved to a file named `evaluation_results.json` in the folder. :paramtype output_path: Optional[str] :keyword azure_ai_project: Logs evaluation results to AI Studio From ab37789d935f2dea8813ae68bda9e2c2fb9dce8a Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 14:17:11 -0700 Subject: [PATCH 3/7] Fixing doc string --- .../promptflow/evals/evaluate/_evaluate.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index 5865431f669..41e8c5d02c7 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -295,12 +295,7 @@ def evaluate( :keyword output_path: The local folder or file path to save evaluation results to if set. If folder path is provided the results will be saved to a file named `evaluation_results.json` in the folder. :paramtype output_path: Optional[str] - :keyword azure_ai_project: Logs evaluation results to AI Studio - Example: { - "subscription_id": "", - "resource_group_name": "", - "project_name": "" - } + :keyword azure_ai_project: Logs evaluation results to AI Studio if set. :paramtype azure_ai_project: Optional[Dict] :return: Evaluation results. :rtype: dict From 50819cc3801e94d4cc6125335692189135ab84a6 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 16:53:11 -0700 Subject: [PATCH 4/7] Adding example to doc string --- .../promptflow/evals/evaluate/_evaluate.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index 41e8c5d02c7..bc3eea45935 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -299,6 +299,40 @@ def evaluate( :paramtype azure_ai_project: Optional[Dict] :return: Evaluation results. :rtype: dict + + :Example: + + Evaluate API can be used as follows: + + .. code-block:: python + + from promptflow.evals.evaluate import evaluate + from promptflow.evals.evaluators import RelevanceEvaluator, CohereEvaluator + + coherence_eval = CohereEvaluator() + relevance_eval = RelevanceEvaluator() + + path = "evaluate_test_data.jsonl" + result = evaluate( + data=path, + evaluators={ + "coherence": coherence_eval, + "relevance": relevance_eval, + }, + evaluator_config={ + "coherence": { + "answer": "data.answer", + "context": "data.context", + "question": "data.question" + }, + "relevance": { + "answer": "data.answer", + "context": "data.context", + "question": "data.question" + } + } + ) + """ trace_destination = _trace_destination_from_project_scope(azure_ai_project) if azure_ai_project else None From 4c1c4835a1613a9ca75acfb6525a5c13af94daef Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 18:54:39 -0700 Subject: [PATCH 5/7] Adding example to doc string --- .../promptflow/evals/evaluate/_evaluate.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index bc3eea45935..93c3110df0f 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -306,11 +306,19 @@ def evaluate( .. code-block:: python + from promptflow.core import AzureOpenAIModelConfiguration from promptflow.evals.evaluate import evaluate from promptflow.evals.evaluators import RelevanceEvaluator, CohereEvaluator - coherence_eval = CohereEvaluator() - relevance_eval = RelevanceEvaluator() + + model_config = AzureOpenAIModelConfiguration( + azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=os.environ.get("AZURE_OPENAI_KEY"), + azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT") + ) + + coherence_eval = CohereEvaluator(model_config=model_config) + relevance_eval = RelevanceEvaluator(model_config=model_config) path = "evaluate_test_data.jsonl" result = evaluate( @@ -321,14 +329,14 @@ def evaluate( }, evaluator_config={ "coherence": { - "answer": "data.answer", - "context": "data.context", - "question": "data.question" + "answer": "${data.answer}", + "context": "${data.context}", + "question": "${data.question}" }, "relevance": { - "answer": "data.answer", - "context": "data.context", - "question": "data.question" + "answer": "${data.answer}", + "context": "${data.context}", + "question": "${data.question}" } } ) From 5abcd770e189274f79c7b3e2820c20dc5e7ff1d2 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 18:56:09 -0700 Subject: [PATCH 6/7] Adding example to doc string --- src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index 93c3110df0f..3e2516d8c97 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -308,7 +308,7 @@ def evaluate( from promptflow.core import AzureOpenAIModelConfiguration from promptflow.evals.evaluate import evaluate - from promptflow.evals.evaluators import RelevanceEvaluator, CohereEvaluator + from promptflow.evals.evaluators import RelevanceEvaluator, CoherenceEvaluator model_config = AzureOpenAIModelConfiguration( @@ -317,7 +317,7 @@ def evaluate( azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT") ) - coherence_eval = CohereEvaluator(model_config=model_config) + coherence_eval = CoherenceEvaluator(model_config=model_config) relevance_eval = RelevanceEvaluator(model_config=model_config) path = "evaluate_test_data.jsonl" From 7f9b786e4e634324b0f12e0e081fa72644a1ab86 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 15 May 2024 19:48:18 -0700 Subject: [PATCH 7/7] Adding example to doc string --- src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index 3e2516d8c97..4fd030eaa47 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -330,7 +330,6 @@ def evaluate( evaluator_config={ "coherence": { "answer": "${data.answer}", - "context": "${data.context}", "question": "${data.question}" }, "relevance": {