Merge remote-tracking branch 'origin/main' into release/promptflow/1.9.0

microsoft · Apr 18, 2024 · f0e9579 · f0e9579
2 parents c030d91 + 17c2bd3
commit f0e9579
Show file tree

Hide file tree

Showing 408 changed files with 13,943 additions and 3,746 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -24,6 +24,9 @@
     "**/*.xml",
     "**/*.txt",
     ".gitignore",
+    "examples/README.md",
+    "examples/flex-flows/README.md",
+    "examples/prompty/README.md",
     "scripts/docs/_build/**",
     "src/promptflow-azure/promptflow/azure/_restclient/flow/**",
     "src/promptflow-azure/promptflow/azure/_restclient/swagger.json",
@@ -188,6 +191,8 @@
     "otel",
     "OTLP",
     "spawnv",
+    "arxiv",
+    "autogen",
     "spawnve",
     "addrs",
     "pywin",
@@ -201,6 +206,8 @@
     "wscript",
     "raisvc",
     "evals",
+    "redoc",
+    "starlette",
     "mlindex",
     "redef",
     "rcts",
@@ -213,4 +220,4 @@
     "Prompt Flow"
   ],
   "allowCompoundWords": true
-}
+}
diff --git a/.github/workflows/promptflow-core-test.yml b/.github/workflows/promptflow-core-test.yml
@@ -5,6 +5,7 @@ on:
     - cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
   pull_request:
     paths:
+      - src/promptflow-tracing/**
       - src/promptflow-core/**
       - .github/workflows/promptflow-core-test.yml
   workflow_dispatch:
@@ -33,18 +34,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - uses: snok/install-poetry@v1
-      - name: install promptflow-tracing
-        run: poetry install
-        working-directory: ${{ env.TRACING_DIRECTORY }}
-      - name: install promptflow-core
-        run: poetry install
-        working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install test dependency group
-        run: poetry install --only test
+        run: |
+          poetry install -E executor-service --with ci,test
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run core tests
         run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -73,18 +68,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - uses: snok/install-poetry@v1
-      - name: install promptflow-tracing
-        run: poetry install
-        working-directory: ${{ env.TRACING_DIRECTORY }}
-      - name: install promptflow-core[azureml-serving]
-        run: poetry install -E azureml-serving
-        working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install test dependency group
-        run: poetry install --only test
+        run: |
+          poetry install -E azureml-serving --with ci,test
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run azureml-serving tests
         run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}

diff --git a/.github/workflows/promptflow-evals-e2e-test.yml b/.github/workflows/promptflow-evals-e2e-test.yml
@@ -33,9 +33,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        # TODO: Following up with PF team for the attribute error from 3.8 and 3.9.
-        python-version: ['3.10', '3.11']
-        #python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:
@@ -58,8 +56,10 @@ jobs:
           path: ${{ env.WORKING_DIRECTORY }}
       - name: install promptflow packages in editable mode
         run: |
+          poetry run pip install -e ../promptflow
           poetry run pip install -e ../promptflow-core
           poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
           poetry run pip install -e ../promptflow-tools
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install promptflow-evals from wheel
@@ -73,8 +73,7 @@ jobs:
         run: poetry install
         working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: generate end-to-end test config from secret
-        # TODO: replace with evals secret
-        run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
+        run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: run e2e tests
         run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml

diff --git a/.github/workflows/promptflow-evals-unit-test.yml b/.github/workflows/promptflow-evals-unit-test.yml
@@ -52,8 +52,10 @@ jobs:
           path: ${{ env.WORKING_DIRECTORY }}
       - name: install promptflow packages in editable mode
         run: |
+          poetry run pip install -e ../promptflow
           poetry run pip install -e ../promptflow-core
           poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
           poetry run pip install -e ../promptflow-tools
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install promptflow-evals from wheel

diff --git a/.github/workflows/promptflow-release-testing-matrix.yml b/.github/workflows/promptflow-release-testing-matrix.yml
@@ -323,7 +323,7 @@ jobs:
         uses: "./.github/actions/step_generate_configs"
         with:
           targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
-      - name: install promptflow-devkit from wheel
+      - name: install promptflow-azure from wheel
         # wildcard expansion (*) does not work in Windows, so leverage python to find and install
         run: |
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")

diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml
@@ -12,6 +12,12 @@ on:
       - .github/workflows/promptflow-sdk-cli-test.yml
       - src/promptflow-recording/**
   workflow_dispatch:
+    inputs:
+      filepath:
+        description: file or paths you want to trigger a test
+        required: true
+        default: "./tests/sdk_cli_test ./tests/sdk_pfs_test"
+        type: string
 env:
   IS_IN_CI_PIPELINE: "true"
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
@@ -30,7 +36,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - name: set test mode
-      run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      run: |
+        echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+        echo "FILE_PATHS=$(if [[ "${{ inputs.filepath }}" == "" ]]; then echo "./tests/sdk_cli_test ./tests/sdk_pfs_test"; else echo ${{ inputs.filepath }}; fi)" >> $GITHUB_ENV
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
@@ -39,15 +47,7 @@ jobs:
     - name: install test dependency group
       run: |
         set -xe
-        poetry install --only test
-        poetry run pip install ${{ env.TRACING_DIRECTORY }}
-        poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
-        poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
-
-        echo "Need to install promptflow to avoid tool dependency issue"
-        poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
-        poetry run pip install ${{ env.TOOL_DIRECTORY }}
-        poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+        poetry install -E pyarrow --with ci,test
 
         poetry run pip show promptflow-tracing
         poetry run pip show promptflow-core
@@ -71,7 +71,7 @@ jobs:
         cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
     - name: run devkit tests
       run: |
-        poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \
+        poetry run pytest ${{ env.FILE_PATHS }} -p promptflow --cov=promptflow --cov-config=pyproject.toml \
         --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \
         --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
       working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -84,7 +84,7 @@ jobs:
           ${{ env.WORKING_DIRECTORY }}/*.xml
           ${{ env.WORKING_DIRECTORY }}/htmlcov/
           ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
-    - run: poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[executable]
+    - run: poetry install -E executable
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: run devkit executable tests
       run: |

diff --git a/.github/workflows/promptflow-tracing-e2e-test.yml b/.github/workflows/promptflow-tracing-e2e-test.yml
@@ -28,7 +28,7 @@ jobs:
         name: promptflow-tracing
         path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
 
-  test:
+  tracing-e2e-test:
     needs: build
     strategy:
       matrix:
@@ -77,7 +77,7 @@ jobs:
             ${{ env.WORKING_DIRECTORY }}/htmlcov/
 
   report:
-    needs: test
+    needs: tracing-e2e-test
     runs-on: ubuntu-latest
     permissions:
       checks: write

diff --git a/.github/workflows/promptflow-tracing-unit-test.yml b/.github/workflows/promptflow-tracing-unit-test.yml
@@ -28,7 +28,7 @@ jobs:
         name: promptflow-tracing
         path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
 
-  test:
+  tracing-unit-test:
     needs: build
     strategy:
       matrix:
@@ -72,7 +72,7 @@ jobs:
             ${{ env.WORKING_DIRECTORY }}/htmlcov/
 
   report:
-    needs: test
+    needs: tracing-unit-test
     runs-on: ubuntu-latest
     permissions:
       checks: write

diff --git a/.github/workflows/samples_connections.yml b/.github/workflows/samples_connections.yml
@@ -48,6 +48,11 @@ jobs:
             sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
             mv .env.example .env
           fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
       - name: Create run.yml
         working-directory: examples/connections
         run: |
@@ -74,6 +79,8 @@ jobs:
         run: |
           export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
           export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
           export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
           export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
           export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
@@ -84,6 +91,8 @@ jobs:
         run: |
           export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
           export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
           export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
           export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
           export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}

diff --git a/.github/workflows/samples_connections_connection.yml b/.github/workflows/samples_connections_connection.yml
@@ -34,6 +34,22 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r ${{ github.workspace }}/examples/requirements.txt
           pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/connections
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
       - name: Create Aoai Connection
         run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
       - name: Test Notebook