Merge branch 'main' into users/ninhu/eval_perf

microsoft · May 15, 2024 · 2aa8ea8 · 2aa8ea8
2 parents 46b3d4b + eddbebb
commit 2aa8ea8
Show file tree

Hide file tree

Showing 12 changed files with 349 additions and 332 deletions.
diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml
@@ -16,6 +16,7 @@ on:
       - 'src/promptflow-devkit/promptflow/**'
       - 'src/promptflow-azure/promptflow/**'
       - 'src/promptflow-rag/promptflow/**'
+      - 'src/promptflow-evals/promptflow/**'
 
 env:
   packageSetupType: promptflow_with_extra
@@ -38,12 +39,13 @@ jobs:
         shell: pwsh
         # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip install -e ${{ github.workspace }}/src/promptflow-core
           pip install -e ${{ github.workspace }}/src/promptflow-devkit
           pip install -e ${{ github.workspace }}/src/promptflow-azure
           pip install -e ${{ github.workspace }}/src/promptflow-rag
+          pip install -e ${{ github.workspace }}/src/promptflow-evals
           pip freeze
 
       - name: Build doc with reference doc
@@ -70,12 +72,13 @@ jobs:
         shell: pwsh
         # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip install -e ${{ github.workspace }}/src/promptflow-core
           pip install -e ${{ github.workspace }}/src/promptflow-devkit
           pip install -e ${{ github.workspace }}/src/promptflow-azure
           pip install -e ${{ github.workspace }}/src/promptflow-rag
+          pip install -e ${{ github.workspace }}/src/promptflow-evals
           pip freeze
 
       - name: Build LinkCheck

diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml
@@ -17,6 +17,7 @@ on:
       - 'src/promptflow-devkit/promptflow/**'
       - 'src/promptflow-azure/promptflow/**'
       - 'src/promptflow-rag/promptflow/**'
+      - 'src/promptflow-evals/promptflow/**'
 
 # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
 permissions:
@@ -48,12 +49,13 @@ jobs:
         shell: pwsh
         # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip install -e ${{ github.workspace }}/src/promptflow-core
           pip install -e ${{ github.workspace }}/src/promptflow-devkit
           pip install -e ${{ github.workspace }}/src/promptflow-azure
           pip install -e ${{ github.workspace }}/src/promptflow-rag
+          pip install -e ${{ github.workspace }}/src/promptflow-evals
           pip freeze
 
       - name: Build Doc

diff --git a/docs/reference/index.md b/docs/reference/index.md
@@ -47,6 +47,7 @@ python-library-reference/promptflow-core/promptflow
 python-library-reference/promptflow-devkit/promptflow
 python-library-reference/promptflow-azure/promptflow
 python-library-reference/promptflow-rag/promptflow
+python-library-reference/promptflow-evals/promptflow
 ```
 
 ```{toctree}

diff --git a/scripts/docs/doc_generation.ps1 b/scripts/docs/doc_generation.ps1
@@ -24,7 +24,7 @@ param(
 [string] $SphinxApiDoc = [System.IO.Path]::Combine($DocPath, "sphinx_apidoc.log")
 [string] $SphinxBuildDoc = [System.IO.Path]::Combine($DocPath, "sphinx_build.log")
 [string] $WarningErrorPattern = "WARNING:|ERROR:|CRITICAL:| broken "
-[System.Collections.ArrayList]$IncludeList = @("promptflow-tracing", "promptflow-core", "promptflow-devkit", "promptflow-azure", "promptflow-rag")
+[System.Collections.ArrayList]$IncludeList = @("promptflow-tracing", "promptflow-core", "promptflow-devkit", "promptflow-azure", "promptflow-rag", "promptflow-evals")
 $apidocWarningsAndErrors = $null
 $buildWarningsAndErrors = $null
 

diff --git a/src/promptflow-evals/CHANGELOG.md b/src/promptflow-evals/CHANGELOG.md
@@ -0,0 +1,8 @@
+# promptflow-evals package
+
+Please insert change log into "Next Release" ONLY.
+
+## Next release
+
+## 0.0.1
+- Introduced package
diff --git a/...tflow-evals/promptflow/evals/evaluators/_content_safety/flow/evaluate_with_rai_service.py b/...tflow-evals/promptflow/evals/evaluators/_content_safety/flow/evaluate_with_rai_service.py
@@ -1,17 +1,24 @@
+import importlib.metadata
 import re
 import time
 from typing import List
+from urllib.parse import urlparse
 
 import numpy as np
 import requests
 from azure.core.credentials import TokenCredential
 from azure.identity import DefaultAzureCredential
 from constants import RAIService, Tasks
 from utils import get_harm_severity_level
-from urllib.parse import urlparse
 
 from promptflow.core import tool
 
+try:
+    version = importlib.metadata.version("promptflow-evals")
+except importlib.metadata.PackageNotFoundError:
+    version = "unknown"
+USER_AGENT = "{}/{}".format("promptflow-evals", version)
+
 
 def ensure_service_availability(rai_svc_url: str):
     svc_liveness_url = rai_svc_url.split("/subscriptions")[0] + "/meta/version"
@@ -27,7 +34,11 @@ def submit_request(question: str, answer: str, metric: str, rai_svc_url: str, cr
 
     url = rai_svc_url + "/submitannotation"
     bearer_token = credential.get_token("https://management.azure.com/.default").token
-    headers = {"Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json"}
+    headers = {
+        "Authorization": f"Bearer {bearer_token}",
+        "Content-Type": "application/json",
+        "User-Agent": USER_AGENT,
+    }
 
     response = requests.post(url, json=payload, headers=headers)
     if response.status_code != 202:

diff --git a/src/promptflow-evals/promptflow/evals/synthetic/_model_tools/_proxy_completion_model.py b/src/promptflow-evals/promptflow/evals/synthetic/_model_tools/_proxy_completion_model.py
@@ -12,6 +12,8 @@
 from aiohttp.web import HTTPException
 from aiohttp_retry import JitterRetry, RetryClient
 
+from promptflow.evals._user_agent import USER_AGENT
+
 from .models import AsyncHTTPClientWithRetry, OpenAIChatCompletionsModel
 
 
@@ -99,6 +101,7 @@ async def request_api(
         proxy_headers = {
             "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
+            "User-Agent": USER_AGENT,
         }
 
         headers = {

diff --git a/src/promptflow-evals/promptflow/evals/synthetic/_model_tools/_rai_client.py b/src/promptflow-evals/promptflow/evals/synthetic/_model_tools/_rai_client.py
@@ -8,6 +8,8 @@
 
 import requests
 
+from promptflow.evals._user_agent import USER_AGENT
+
 from ._async_http_client import AsyncHTTPClientWithRetry
 
 api_url = None
@@ -82,6 +84,7 @@ async def get(self, url: str) -> Any:
         headers = {
             "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
+            "User-Agent": USER_AGENT,
         }
 
         async with self._create_async_client().client as session: