[Fundamental] Move devkit and azure tests

.cspell.json

@@ -31,6 +31,8 @@
     "src/promptflow-azure/tests/**",
     "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
+    "src/promptflow-devkit/tests/**",
+    "src/promptflow-azure/tests/**",
     "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
     "src/promptflow-devkit/promptflow/_sdk/_service/static/index.html",

.github/actions/step_generate_configs/action.yml

@@ -12,6 +12,8 @@ runs:
     shell: pwsh
     run: |
       pip list
+      pip install azure-identity
+      pip install azure-keyvault
       echo "Generating connection config file..."
       python ./scripts/building/generate_connection_config.py `
           --target_folder ${{ inputs.targetFolder }}

.github/workflows/promptflow-global-config-test.yml

@@ -4,16 +4,23 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request_target:
     paths:
-      - src/promptflow-core/*
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
+      - src/promptflow-tracing/**
+      - src/promptflow-azure/**
       - src/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-global-config-test.yml
   workflow_dispatch:
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+  AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
 jobs:
   authorize:
     environment:
@@ -41,70 +48,60 @@ jobs:
       uses: "./.github/actions/step_merge_main"
     - name: Display and Set Environment Variables
       run: |
-        if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then
-          export pyVersion="3.9";
-        elif [ "macos-latest" == "${{ matrix.os }}" ]; then
-          export pyVersion="3.10";
-        else
-          echo "Unsupported OS: ${{ matrix.os }}";
-          exit 1;
-        fi
+        export pyVersion="3.9"
         env | sort >> $GITHUB_OUTPUT
       id: display_env
       shell: bash -el {0}
-    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
-      uses: "./.github/actions/step_create_python_environment"
+    - uses: actions/setup-python@v5
       with:
-        pythonVersion: ${{ steps.display_env.outputs.pyVersion }}
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: ${{ env.packageSetupType }}
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Install dependency
-      shell: pwsh
+        python-version: ${{ steps.display_env.outputs.pyVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
+      working-directory: ${{ env.WORKING_DIRECTORY }}
       run: |
-        pip uninstall -y promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-tracing
-        echo "Installed promptflow-tracing"
-        pip uninstall -y promptflow-core
-        pip install ${{ github.workspace }}/src/promptflow-core
-        pip uninstall -y promptflow-devkit
-        pip install ${{ github.workspace }}/src/promptflow-devkit
-        pip uninstall -y promptflow-azure
-        pip install ${{ github.workspace }}/src/promptflow-azure
-        pip freeze
+        set -xe
+        poetry install --only test
+        poetry run pip install ${{ env.TRACING_DIRECTORY }}
+        poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
+        poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
+        poetry run pip install -e ${{ env.AZURE_DIRECTORY }}
+
+        echo "Need to install promptflow to avoid tool dependency issue"
+        poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
+        poetry run pip install ${{ env.TOOL_DIRECTORY }}
+        poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-azure
+        poetry run pip show promptflow-tools
     - name: Azure Login
       uses: azure/login@v1
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: Install Azure Login items
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+      run: |
+        pip install azure-identity
+        pip install azure-keyvault
     - name: Generate Configs
       uses: "./.github/actions/step_generate_configs"
       with:
-        targetFolder: ${{ env.testWorkingDirectory }}
-    - name: Get number of CPU cores
-      uses: SimenB/github-actions-cpu-cores@v1
-      id: cpu-cores
-    - name: Run Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+    - name: run devkit tests
       run: |
-        gci env:* | sort-object name
-        az account show
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_global_config_test `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -n ${{ steps.cpu-cores.outputs.count }} `
+        poetry run pytest ./tests/sdk_cli_global_config_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest"
+      working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results
       if: always()
       uses: actions/upload-artifact@v3
       with:
         name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }})
         path: |
-          ${{ env.testWorkingDirectory }}/*.xml
-          ${{ env.testWorkingDirectory }}/htmlcov/
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
   publish-test-results-global-config-test:
     needs: sdk_cli_global_config_tests
     runs-on: ubuntu-latest
@@ -117,7 +114,7 @@ jobs:
 
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha || github.ref }}
     - name: Publish Test Results

.github/workflows/promptflow-sdk-cli-test.yml

@@ -13,43 +13,15 @@ on:
       - src/promptflow-recording/**
   workflow_dispatch:
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
 jobs:
-  build:
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Display and Set Environment Variables
-      run: |
-        env | sort >> $GITHUB_OUTPUT
-      id: display_env
-      shell: bash -el {0}
-    - name: Python Setup - ubuntu-latest - Python Version 3.9
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: 3.9
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: promptflow_with_extra
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Upload Wheel
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: wheel
-        path: |
-          ${{ github.workspace }}/src/promptflow/dist/*.whl
-          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
   sdk_cli_tests:
-    needs: build
     strategy:
       fail-fast: false
       matrix:
@@ -59,39 +31,29 @@ jobs:
     steps:
     - name: set test mode
       run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Display and Set Environment Variables
-      run: |
-        env | sort >> $GITHUB_OUTPUT
-      id: display_env
-      shell: bash -el {0}
-    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: ${{ matrix.pythonVersion }}
-    - name: Download Artifacts
-      uses: actions/download-artifact@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        name: wheel
-        path: artifacts
-    - name: Install wheel
-      shell: pwsh
-      working-directory: artifacts
-      run: |
-        Set-PSDebug -Trace 1
-        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        pip install ${{ github.workspace }}/src/promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-core
-        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
-        pip install ${{ github.workspace }}/src/promptflow
-        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip freeze
-    - name: install recording
+        python-version: ${{ matrix.pythonVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
       run: |
-        pip install vcrpy
-        pip install .
-      working-directory: ${{ env.RECORD_DIRECTORY }}
+        set -xe
+        poetry install --only test
+        poetry run pip install ${{ env.TRACING_DIRECTORY }}
+        poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
+        poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
+
+        echo "Need to install promptflow to avoid tool dependency issue"
+        poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
+        poetry run pip install ${{ env.TOOL_DIRECTORY }}
+        poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-tools
+      working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Azure login (non pull_request workflow)
       if: github.event_name != 'pull_request'
       uses: azure/login@v1
@@ -101,62 +63,33 @@ jobs:
       if: github.event_name != 'pull_request'
       uses: "./.github/actions/step_generate_configs"
       with:
-        targetFolder: ${{ env.testWorkingDirectory }}
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
     - name: generate live test resources (pull_request workflow)
       if: github.event_name == 'pull_request'
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
+      working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
       run: |
         cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
-    - name: Run SDK CLI Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
-          -l eastus `
-          -m "unittest or e2etest" `
-          --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc `
-          -o "${{ env.testWorkingDirectory }}/test-results-sdk-cli.xml" `
-          --ignore-glob ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
-    - name: Install pf executable
-      shell: pwsh
-      working-directory: artifacts
+    - name: run devkit tests
       run: |
-        Set-PSDebug -Trace 1
-        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable]
-        pip freeze
-    - name: Run SDK CLI Executable Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -o "${{ env.testWorkingDirectory }}/test-results-sdk-cli-executable.xml"
-    - name: Run PFS Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_pfs_test `
-          -l eastus `
-          -m "e2etest" `
-          --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_pfs_test/.coveragerc `
-          -o "${{ env.testWorkingDirectory }}/test-results-pfs.xml"
+        poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \
+        --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
+      working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results
       if: always()
       uses: actions/upload-artifact@v3
       with:
         name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
         path: |
-          ${{ env.testWorkingDirectory }}/*.xml
-          ${{ env.testWorkingDirectory }}/htmlcov/
-          ${{ env.testWorkingDirectory }}/tests/sdk_cli_test/count.json
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
+          ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
+    - run: poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[executable]
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: run devkit executable tests
+      run: |
+        poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py
+      working-directory: ${{ env.WORKING_DIRECTORY }}
 
 
   publish-test-results-sdk-cli-test: