diff --git a/.cspell.json b/.cspell.json index 1e86c090af7..35a190a2805 100644 --- a/.cspell.json +++ b/.cspell.json @@ -31,6 +31,8 @@ "src/promptflow-azure/tests/**", "src/promptflow-core/promptflow/core/_connection_provider/_models/**", "src/promptflow/tests/**", + "src/promptflow-devkit/tests/**", + "src/promptflow-azure/tests/**", "src/promptflow-recording/**", "src/promptflow-tools/tests/**", "src/promptflow-devkit/promptflow/_sdk/_service/static/index.html", diff --git a/.github/actions/step_generate_configs/action.yml b/.github/actions/step_generate_configs/action.yml index b0fc838cbe4..9e2a3480f36 100644 --- a/.github/actions/step_generate_configs/action.yml +++ b/.github/actions/step_generate_configs/action.yml @@ -12,6 +12,8 @@ runs: shell: pwsh run: | pip list + pip install azure-identity + pip install azure-keyvault echo "Generating connection config file..." python ./scripts/building/generate_connection_config.py ` --target_folder ${{ inputs.targetFolder }} diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml index fe00424371c..3831e1d3186 100644 --- a/.github/workflows/promptflow-global-config-test.yml +++ b/.github/workflows/promptflow-global-config-test.yml @@ -4,16 +4,23 @@ on: - cron: "40 18 * * *" # Every day starting at 2:40 BJT pull_request_target: paths: - - src/promptflow-core/* + - src/promptflow-core/** + - src/promptflow-devkit/** + - src/promptflow-tracing/** + - src/promptflow-azure/** - src/promptflow/** - scripts/building/** - .github/workflows/promptflow-global-config-test.yml workflow_dispatch: env: - packageSetupType: promptflow_with_extra - testWorkingDirectory: ${{ github.workspace }}/src/promptflow - PYTHONPATH: ${{ github.workspace }}/src/promptflow IS_IN_CI_PIPELINE: "true" + RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording + TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing + CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core + WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit + PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow + TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools + AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure jobs: authorize: environment: @@ -41,70 +48,60 @@ jobs: uses: "./.github/actions/step_merge_main" - name: Display and Set Environment Variables run: | - if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then - export pyVersion="3.9"; - elif [ "macos-latest" == "${{ matrix.os }}" ]; then - export pyVersion="3.10"; - else - echo "Unsupported OS: ${{ matrix.os }}"; - exit 1; - fi + export pyVersion="3.9" env | sort >> $GITHUB_OUTPUT id: display_env shell: bash -el {0} - - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }} - uses: "./.github/actions/step_create_python_environment" + - uses: actions/setup-python@v5 with: - pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - - name: Build wheel - uses: "./.github/actions/step_sdk_setup" - with: - setupType: ${{ env.packageSetupType }} - scriptPath: ${{ env.testWorkingDirectory }} - - name: Install dependency - shell: pwsh + python-version: ${{ steps.display_env.outputs.pyVersion }} + - uses: snok/install-poetry@v1 + - name: install test dependency group + working-directory: ${{ env.WORKING_DIRECTORY }} run: | - pip uninstall -y promptflow-tracing - pip install ${{ github.workspace }}/src/promptflow-tracing - echo "Installed promptflow-tracing" - pip uninstall -y promptflow-core - pip install ${{ github.workspace }}/src/promptflow-core - pip uninstall -y promptflow-devkit - pip install ${{ github.workspace }}/src/promptflow-devkit - pip uninstall -y promptflow-azure - pip install ${{ github.workspace }}/src/promptflow-azure - pip freeze + set -xe + poetry install --only test + poetry run pip install ${{ env.TRACING_DIRECTORY }} + poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving] + poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow] + poetry run pip install -e ${{ env.AZURE_DIRECTORY }} + + echo "Need to install promptflow to avoid tool dependency issue" + poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }} + poetry run pip install ${{ env.TOOL_DIRECTORY }} + poetry run pip install -e ${{ env.RECORD_DIRECTORY }} + + poetry run pip show promptflow-tracing + poetry run pip show promptflow-core + poetry run pip show promptflow-devkit + poetry run pip show promptflow-azure + poetry run pip show promptflow-tools - name: Azure Login uses: azure/login@v1 with: creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Install Azure Login items + working-directory: ${{ env.WORKING_DIRECTORY }} + run: | + pip install azure-identity + pip install azure-keyvault - name: Generate Configs uses: "./.github/actions/step_generate_configs" with: - targetFolder: ${{ env.testWorkingDirectory }} - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 - id: cpu-cores - - name: Run Test - shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} + targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }} + - name: run devkit tests run: | - gci env:* | sort-object name - az account show - python "../../scripts/building/run_coverage_tests.py" ` - -p promptflow ` - -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_global_config_test ` - -l eastus ` - -m "unittest or e2etest" ` - -n ${{ steps.cpu-cores.outputs.count }} ` + poetry run pytest ./tests/sdk_cli_global_config_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \ + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" + working-directory: ${{ env.WORKING_DIRECTORY }} - name: Upload Test Results if: always() uses: actions/upload-artifact@v3 with: name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }}) path: | - ${{ env.testWorkingDirectory }}/*.xml - ${{ env.testWorkingDirectory }}/htmlcov/ + ${{ env.WORKING_DIRECTORY }}/*.xml + ${{ env.WORKING_DIRECTORY }}/htmlcov/ publish-test-results-global-config-test: needs: sdk_cli_global_config_tests runs-on: ubuntu-latest @@ -117,7 +114,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Publish Test Results diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index 9215cbc760b..5d85fec622f 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -13,43 +13,15 @@ on: - src/promptflow-recording/** workflow_dispatch: env: - packageSetupType: promptflow_with_extra - testWorkingDirectory: ${{ github.workspace }}/src/promptflow - PYTHONPATH: ${{ github.workspace }}/src/promptflow IS_IN_CI_PIPELINE: "true" RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording + TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing + CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core + WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit + PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow + TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools jobs: - build: - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v4 - - name: Display and Set Environment Variables - run: | - env | sort >> $GITHUB_OUTPUT - id: display_env - shell: bash -el {0} - - name: Python Setup - ubuntu-latest - Python Version 3.9 - uses: "./.github/actions/step_create_python_environment" - with: - pythonVersion: 3.9 - - name: Build wheel - uses: "./.github/actions/step_sdk_setup" - with: - setupType: promptflow_with_extra - scriptPath: ${{ env.testWorkingDirectory }} - - name: Upload Wheel - if: always() - uses: actions/upload-artifact@v3 - with: - name: wheel - path: | - ${{ github.workspace }}/src/promptflow/dist/*.whl - ${{ github.workspace }}/src/promptflow-tools/dist/*.whl sdk_cli_tests: - needs: build strategy: fail-fast: false matrix: @@ -59,39 +31,29 @@ jobs: steps: - name: set test mode run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV - - name: checkout - uses: actions/checkout@v4 - - name: Display and Set Environment Variables - run: | - env | sort >> $GITHUB_OUTPUT - id: display_env - shell: bash -el {0} - - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }} - uses: "./.github/actions/step_create_python_environment" - with: - pythonVersion: ${{ matrix.pythonVersion }} - - name: Download Artifacts - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - name: wheel - path: artifacts - - name: Install wheel - shell: pwsh - working-directory: artifacts - run: | - Set-PSDebug -Trace 1 - pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt - pip install ${{ github.workspace }}/src/promptflow-tracing - pip install ${{ github.workspace }}/src/promptflow-core - pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow] - pip install ${{ github.workspace }}/src/promptflow - gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip freeze - - name: install recording + python-version: ${{ matrix.pythonVersion }} + - uses: snok/install-poetry@v1 + - name: install test dependency group run: | - pip install vcrpy - pip install . - working-directory: ${{ env.RECORD_DIRECTORY }} + set -xe + poetry install --only test + poetry run pip install ${{ env.TRACING_DIRECTORY }} + poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving] + poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow] + + echo "Need to install promptflow to avoid tool dependency issue" + poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }} + poetry run pip install ${{ env.TOOL_DIRECTORY }} + poetry run pip install -e ${{ env.RECORD_DIRECTORY }} + + poetry run pip show promptflow-tracing + poetry run pip show promptflow-core + poetry run pip show promptflow-devkit + poetry run pip show promptflow-tools + working-directory: ${{ env.WORKING_DIRECTORY }} - name: Azure login (non pull_request workflow) if: github.event_name != 'pull_request' uses: azure/login@v1 @@ -101,62 +63,33 @@ jobs: if: github.event_name != 'pull_request' uses: "./.github/actions/step_generate_configs" with: - targetFolder: ${{ env.testWorkingDirectory }} + targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }} - name: generate live test resources (pull_request workflow) if: github.event_name == 'pull_request' - shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} + working-directory: ${{ env.PROMPTFLOW_DIRECTORY }} run: | cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json - - name: Run SDK CLI Test - shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} - run: | - python "../../scripts/building/run_coverage_tests.py" ` - -p promptflow ` - -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test ` - -l eastus ` - -m "unittest or e2etest" ` - --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc ` - -o "${{ env.testWorkingDirectory }}/test-results-sdk-cli.xml" ` - --ignore-glob ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py - - name: Install pf executable - shell: pwsh - working-directory: artifacts + - name: run devkit tests run: | - Set-PSDebug -Trace 1 - pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable] - pip freeze - - name: Run SDK CLI Executable Test - shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} - run: | - python "../../scripts/building/run_coverage_tests.py" ` - -p promptflow ` - -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py ` - -l eastus ` - -m "unittest or e2etest" ` - -o "${{ env.testWorkingDirectory }}/test-results-sdk-cli-executable.xml" - - name: Run PFS Test - shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} - run: | - python "../../scripts/building/run_coverage_tests.py" ` - -p promptflow ` - -t ${{ github.workspace }}/src/promptflow/tests/sdk_pfs_test ` - -l eastus ` - -m "e2etest" ` - --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_pfs_test/.coveragerc ` - -o "${{ env.testWorkingDirectory }}/test-results-pfs.xml" + poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \ + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \ + --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py + working-directory: ${{ env.WORKING_DIRECTORY }} - name: Upload Test Results if: always() uses: actions/upload-artifact@v3 with: name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) path: | - ${{ env.testWorkingDirectory }}/*.xml - ${{ env.testWorkingDirectory }}/htmlcov/ - ${{ env.testWorkingDirectory }}/tests/sdk_cli_test/count.json + ${{ env.WORKING_DIRECTORY }}/*.xml + ${{ env.WORKING_DIRECTORY }}/htmlcov/ + ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json + - run: poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[executable] + working-directory: ${{ env.WORKING_DIRECTORY }} + - name: run devkit executable tests + run: | + poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py + working-directory: ${{ env.WORKING_DIRECTORY }} publish-test-results-sdk-cli-test: diff --git a/.github/workflows/sdk-cli-azure-test-pull-request.yml b/.github/workflows/sdk-cli-azure-test-pull-request.yml index c509eceb3d5..9477d537416 100644 --- a/.github/workflows/sdk-cli-azure-test-pull-request.yml +++ b/.github/workflows/sdk-cli-azure-test-pull-request.yml @@ -6,56 +6,29 @@ name: sdk-cli-azure-test-pull-request on: pull_request: paths: - - src/promptflow-core/** - - src/promptflow-devkit/** - src/promptflow/** - scripts/building/** - - src/promptflow-tracing/** - .github/workflows/sdk-cli-azure-test-pull-request.yml + - src/promptflow-tracing/** + - src/promptflow-core/** + - src/promptflow-devkit/** + - src/promptflow-azure/** env: - packageSetupType: promptflow_with_extra - testWorkingDirectory: ${{ github.workspace }}/src/promptflow - PYTHONPATH: ${{ github.workspace }}/src/promptflow IS_IN_CI_PIPELINE: "true" PROMPT_FLOW_TEST_MODE: "replay" + TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing + WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure + CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core + DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit + PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow + TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording jobs: - build: - strategy: - fail-fast: false - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v4 - - name: Display and Set Environment Variables - run: | - env | sort >> $GITHUB_OUTPUT - id: display_env - shell: bash -el {0} - - name: Python Setup - ubuntu-latest - Python Version 3.9 - uses: "./.github/actions/step_create_python_environment" - with: - pythonVersion: 3.9 - - name: Build wheel - uses: "./.github/actions/step_sdk_setup" - with: - setupType: promptflow_with_extra - scriptPath: ${{ env.testWorkingDirectory }} - - name: Upload Wheel - if: always() - uses: actions/upload-artifact@v3 - with: - name: wheel - path: | - ${{ github.workspace }}/src/promptflow/dist/*.whl - ${{ github.workspace }}/src/promptflow-tools/dist/*.whl - sdk_cli_azure_test_replay: - needs: build strategy: fail-fast: false matrix: @@ -71,47 +44,42 @@ jobs: - name: Display and Set Environment Variables run: env | sort >> $GITHUB_OUTPUT - - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }} - uses: "./.github/actions/step_create_python_environment" - with: - pythonVersion: ${{ matrix.pythonVersion }} - - - name: Download Artifacts - uses: actions/download-artifact@v3 + - uses: actions/setup-python@v5 with: - name: wheel - path: artifacts - - - name: Install wheel - shell: pwsh - working-directory: artifacts + python-version: ${{ matrix.pythonVersion }} + - uses: snok/install-poetry@v1 + - name: install test dependency group run: | - Set-PSDebug -Trace 1 - pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt - pip install ${{ github.workspace }}/src/promptflow-tracing - pip install ${{ github.workspace }}/src/promptflow-core - pip install ${{ github.workspace }}/src/promptflow-devkit - pip install ${{ github.workspace }}/src/promptflow-azure - gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}} - gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip freeze - - - name: install recording + set -xe + poetry install --only test + poetry run pip install ${{ env.TRACING_DIRECTORY }} + poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving] + poetry run pip install ${{ env.DEVKIT_DIRECTORY }}[pyarrow] + poetry run pip install -e ${{ env.WORKING_DIRECTORY }} + + echo "Need to install promptflow to avoid tool dependency issue" + poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }} + poetry run pip install ${{ env.TOOL_DIRECTORY }} + poetry run pip install -e ${{ env.RECORD_DIRECTORY }} + + poetry run pip show promptflow-tracing + poetry run pip show promptflow-core + poetry run pip show promptflow-devkit + poetry run pip show promptflow-azure + poetry run pip show promptflow-tools + working-directory: ${{ env.WORKING_DIRECTORY }} + + - name: generate live test resources + working-directory: ${{ env.PROMPTFLOW_DIRECTORY }} run: | - pip install vcrpy - pip install -e . - working-directory: ${{ env.RECORD_DIRECTORY }} + cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json - name: Run SDK CLI Azure Test (replay mode) shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} + working-directory: ${{ env.WORKING_DIRECTORY }} run: | - python "../../scripts/building/run_coverage_tests.py" ` - -p promptflow ` - -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ` - -l eastus ` - -m "unittest or e2etest" ` - --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc + poetry run pytest ./tests/sdk_cli_azure_test -p promptflow --cov=promptflow --cov-config=pyproject.toml ` + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" - name: Upload Test Results if: always() @@ -119,8 +87,8 @@ jobs: with: name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) path: | - ${{ env.testWorkingDirectory }}/*.xml - ${{ env.testWorkingDirectory }}/htmlcov/ + ${{ env.WORKING_DIRECTORY }}/*.xml + ${{ env.WORKING_DIRECTORY }}/htmlcov/ publish-test-results-sdk-cli-azure-test: needs: sdk_cli_azure_test_replay diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index f3aa9f2f821..a66a02b35a3 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -9,6 +9,7 @@ on: - src/promptflow/** - src/promptflow-core/** - src/promptflow-devkit/** + - src/promptflow-azure/** - scripts/building/** - .github/workflows/sdk-cli-perf-monitor-test.yml @@ -19,20 +20,25 @@ on: env: - packageSetupType: promptflow_with_extra - testWorkingDirectory: ${{ github.workspace }}/src/promptflow - PYTHONPATH: ${{ github.workspace }}/src/promptflow IS_IN_CI_PIPELINE: "true" PROMPT_FLOW_TEST_MODE: "replay" + TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing + WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure + CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core + DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit + PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow + TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording - jobs: sdk_cli_perf_monitor_test: strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + defaults: + run: + shell: bash runs-on: ${{ matrix.os }} steps: @@ -50,63 +56,54 @@ jobs: export pyVersion="3.9"; env | sort >> $GITHUB_OUTPUT id: display_env - shell: bash -el {0} - - - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }} - uses: "./.github/actions/step_create_python_environment" - with: - pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - - name: Build wheel - uses: "./.github/actions/step_sdk_setup" - with: - setupType: promptflow_with_extra - scriptPath: ${{ env.testWorkingDirectory }} - - name: Upload Wheel - if: always() - uses: actions/upload-artifact@v3 + - uses: actions/setup-python@v5 with: - name: wheel - path: | - ${{ github.workspace }}/src/promptflow/dist/*.whl - ${{ github.workspace }}/src/promptflow-tools/dist/*.whl - - name: Download Artifacts - uses: actions/download-artifact@v3 + python-version: ${{ steps.display_env.outputs.pyVersion }} + - uses: snok/install-poetry@v1 with: - name: wheel - path: artifacts - - name: Install wheel - shell: pwsh - working-directory: artifacts + virtualenvs-create: true + virtualenvs-in-project: true + - name: install test dependency group + working-directory: ${{ env.WORKING_DIRECTORY }} run: | - Set-PSDebug -Trace 1 - pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt - pip install ${{ github.workspace }}/src/promptflow-tracing - pip install ${{ github.workspace }}/src/promptflow-core - pip install ${{ github.workspace }}/src/promptflow-devkit - pip install ${{ github.workspace }}/src/promptflow-azure - gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[all]"}} - gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip freeze - - - name: install recording - run: - pip install vcrpy - pip install -e . - working-directory: ${{ env.RECORD_DIRECTORY }} + poetry install --with test + - run: | + source .venv/scripts/activate + pytest --version + if: runner.os == 'Windows' + working-directory: ${{ env.WORKING_DIRECTORY }} + - run: | + source .venv/bin/activate + pytest --version + if: runner.os != 'Windows' + working-directory: ${{ env.WORKING_DIRECTORY }} + - run: | + set -xe + poetry run pip install ../promptflow-tracing + poetry run pip install ../promptflow-core[azureml-serving] + poetry run pip install ../promptflow-devkit[pyarrow] + poetry run pip install ../promptflow-azure + + echo "Need to install promptflow to avoid tool dependency issue" + poetry run pip install ../promptflow + poetry run pip install ../promptflow-tools + poetry run pip install ../promptflow-recording + + poetry run pip show promptflow-tracing + poetry run pip show promptflow-core + poetry run pip show promptflow-devkit + poetry run pip show promptflow-azure + poetry run pip show promptflow-tools + working-directory: ${{ env.WORKING_DIRECTORY }} - name: Generate (mock) connections.json shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} + working-directory: ${{ env.PROMPTFLOW_DIRECTORY }} run: cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json - name: Run Test - shell: pwsh - working-directory: ${{ env.testWorkingDirectory }} + working-directory: ${{ env.WORKING_DIRECTORY }} run: | - gci env:* | sort-object name - python "../../scripts/building/run_coverage_tests.py" ` - -p promptflow ` - -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test ` - -l eastus ` - -m "perf_monitor_test" + poetry run pytest ./tests/sdk_cli_azure_test ../promptflow-azure/tests/sdk_cli_azure_test -n auto -m "perf_monitor_test" + diff --git a/docs/reference/pf-command-reference.md b/docs/reference/pf-command-reference.md index 7eacf367ee1..539b2737e8a 100644 --- a/docs/reference/pf-command-reference.md +++ b/docs/reference/pf-command-reference.md @@ -903,6 +903,6 @@ pf upgrade --yes To activate autocomplete features for the pf CLI you need to add the following snippet to your ~/.bashrc or ~/.zshrc: -``` - source /pf.completion.sh +```bash +source /pf.completion.sh ``` diff --git a/scripts/check_enforcer/check_enforcer.py b/scripts/check_enforcer/check_enforcer.py index 6755b95f77d..ab3974943d7 100644 --- a/scripts/check_enforcer/check_enforcer.py +++ b/scripts/check_enforcer/check_enforcer.py @@ -43,19 +43,27 @@ # Copy from original yaml pipelines checks = { "sdk_cli_tests": [ + "src/promptflow-core/**", + "src/promptflow-devkit/**", "src/promptflow/**", + "src/promptflow-tracing/**", "scripts/building/**", ".github/workflows/promptflow-sdk-cli-test.yml", + "src/promptflow-recording/**", ], - "sdk_cli_global_config_tests": [ - "src/promptflow/**", - "scripts/building/**", - ".github/workflows/promptflow-global-config-test.yml", - ], + # "sdk_cli_global_config_tests": [ + # "src/promptflow/**", + # "scripts/building/**", + # ".github/workflows/promptflow-global-config-test.yml", + # ], "sdk_cli_azure_test_replay": [ "src/promptflow/**", "scripts/building/**", - ".github/workflows/promptflow-sdk-cli-azure-test-pull-request.yml", + ".github/workflows/sdk-cli-azure-test-pull-request.yml", + "src/promptflow-tracing/**", + "src/promptflow-core/**", + "src/promptflow-devkit/**", + "src/promptflow-azure/**", ], } diff --git a/scripts/json_schema/Flow.schema.json b/scripts/json_schema/Flow.schema.json index 3a8a81b8a5d..ba5d69ba374 100644 --- a/scripts/json_schema/Flow.schema.json +++ b/scripts/json_schema/Flow.schema.json @@ -24,6 +24,11 @@ "type": "object", "additionalProperties": {} }, + "environment_variables": { + "title": "environment_variables", + "type": "object", + "additionalProperties": {} + }, "inputs": { "title": "inputs", "type": "object", @@ -34,7 +39,13 @@ }, "language": { "title": "language", - "type": "string" + "type": "string", + "default": "python", + "enum": [ + "python", + "csharp" + ], + "enumNames": [] }, "node_variants": { "title": "node_variants", @@ -116,8 +127,47 @@ "type": "object", "additionalProperties": {} }, + "environment_variables": { + "title": "environment_variables", + "type": "object", + "additionalProperties": {} + }, + "init": { + "title": "init", + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#/definitions/FlexFlowInitSchema" + } + }, + "inputs": { + "title": "inputs", + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#/definitions/FlexFlowInputSchema" + } + }, "language": { "title": "language", + "type": "string", + "default": "python", + "enum": [ + "python", + "csharp" + ], + "enumNames": [] + }, + "outputs": { + "title": "outputs", + "type": "object", + "additionalProperties": { + "type": "object", + "$ref": "#/definitions/FlexFlowOutputSchema" + } + }, + "sample": { + "title": "sample", "type": "string" }, "$schema": { diff --git a/src/promptflow-azure/promptflow/azure/_storage/cosmosdb/client.py b/src/promptflow-azure/promptflow/azure/_storage/cosmosdb/client.py index 22e7f173c9c..5f175b090ed 100644 --- a/src/promptflow-azure/promptflow/azure/_storage/cosmosdb/client.py +++ b/src/promptflow-azure/promptflow/azure/_storage/cosmosdb/client.py @@ -78,11 +78,15 @@ def _get_resource_token( ) -> object: from promptflow.azure import PFClient + # The default connection_time and read_timeout are both 300s. + # The get token operation should be fast, so we set a short timeout. pf_client = PFClient( credential=credential, subscription_id=subscription_id, resource_group_name=resource_group_name, workspace_name=workspace_name, + connection_timeout=15.0, + read_timeout=30.0, ) token_resp = pf_client._traces._get_cosmos_db_token(container_name=container_name, acquire_write=True) diff --git a/src/promptflow-azure/promptflow/azure/operations/_run_operations.py b/src/promptflow-azure/promptflow/azure/operations/_run_operations.py index f50d2a98449..a163b86af89 100644 --- a/src/promptflow-azure/promptflow/azure/operations/_run_operations.py +++ b/src/promptflow-azure/promptflow/azure/operations/_run_operations.py @@ -125,7 +125,7 @@ def _workspace_default_datastore(self): if kind not in [AzureWorkspaceKind.DEFAULT, AzureWorkspaceKind.PROJECT]: raise RunOperationParameterError( "Failed to get default workspace datastore. Please make sure you are using the right workspace which " - f"is either an azure machine learning studio workspace or an azure ai project. Got {kind!r} instead." + f"is either an azure machine learning workspace or an azure ai project. Got {kind!r} instead." ) return self._datastore_operations.get_default() diff --git a/src/promptflow-azure/pyproject.toml b/src/promptflow-azure/pyproject.toml index 9f7b310c009..a950a1483ee 100644 --- a/src/promptflow-azure/pyproject.toml +++ b/src/promptflow-azure/pyproject.toml @@ -55,6 +55,11 @@ import-linter = "*" pytest = "*" pytest-cov = "*" pytest-xdist = "*" +pytest-mock = "*" +pytest-asyncio = "*" +mock = "*" +keyrings-alt = "*" +bs4 = "*" [build-system] requires = ["poetry-core>=1.5.0"] @@ -66,16 +71,12 @@ pfazure = "promptflow.azure._cli.entry:main" [tool.pytest.ini_options] markers = [ "unittest", + "e2etest" ] # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action) # durations - list the slowest test durations addopts = """ --junit-xml=test-results.xml \ ---cov=promptflow \ ---cov-config=pyproject.toml \ ---cov-report=term \ ---cov-report=html \ ---cov-report=xml \ --dist loadfile \ --log-level=info \ --log-format="%(asctime)s %(levelname)s %(message)s" \ @@ -87,8 +88,22 @@ addopts = """ testpaths = ["tests"] [tool.coverage.run] +source = [ + "*/promptflow/azure/*" +] omit = [ - "__init__.py", + "*/__init__.py", + "*/promptflow/azure/_restclient/*", + "*/promptflow/azure/_models/*", + "*/promptflow/azure/_cli/*", + "*/promptflow/recording/*", + "*/promptflow/tracing/*", + "*/promptflow/tools/*", + "*/promptflow/executor/*", + "*/promptflow/core/*", + "*/promptflow/_sdk/_service/*", + "*/promptflow/_orchestrator/*", + "*/promptflow/_cli/*", ] [tool.black] diff --git a/src/promptflow-azure/tests/_constants.py b/src/promptflow-azure/tests/_constants.py new file mode 100644 index 00000000000..9929a01ff68 --- /dev/null +++ b/src/promptflow-azure/tests/_constants.py @@ -0,0 +1,14 @@ +from pathlib import Path + +PROMPTFLOW_ROOT = Path(__file__).parent.parent.parent / "promptflow" +RUNTIME_TEST_CONFIGS_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/runtime") +CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() +ENV_FILE = (PROMPTFLOW_ROOT / ".env").resolve().absolute().as_posix() + +# below constants are used for pfazure and global config tests +DEFAULT_SUBSCRIPTION_ID = "96aede12-2f73-41cb-b983-6d11a904839b" +DEFAULT_RESOURCE_GROUP_NAME = "promptflow" +DEFAULT_WORKSPACE_NAME = "promptflow-eastus2euap" +DEFAULT_COMPUTE_INSTANCE_NAME = "ci-lin-cpu-sp" +DEFAULT_RUNTIME_NAME = "test-runtime-ci" +DEFAULT_REGISTRY_NAME = "promptflow-preview" diff --git a/src/promptflow-azure/tests/conftest.py b/src/promptflow-azure/tests/conftest.py new file mode 100644 index 00000000000..290bb420cb2 --- /dev/null +++ b/src/promptflow-azure/tests/conftest.py @@ -0,0 +1,74 @@ +import json +import os +import tempfile +from pathlib import Path + +import pytest +from _constants import CONNECTION_FILE, PROMPTFLOW_ROOT +from _pytest.monkeypatch import MonkeyPatch +from dotenv import load_dotenv +from pytest_mock import MockerFixture + +from promptflow._constants import PROMPTFLOW_CONNECTIONS +from promptflow._core.connection_manager import ConnectionManager +from promptflow._sdk.entities._connection import AzureOpenAIConnection +from promptflow._utils.context_utils import _change_working_dir + +load_dotenv() + + +@pytest.fixture(autouse=True, scope="session") +def mock_build_info(): + """Mock BUILD_INFO environment variable in pytest. + + BUILD_INFO is set as environment variable in docker image, but not in local test. + So we need to mock it in test senario. Rule - build_number is set as + ci- in CI pipeline, and set as local in local dev test.""" + if "BUILD_INFO" not in os.environ: + m = MonkeyPatch() + build_number = os.environ.get("BUILD_BUILDNUMBER", "") + buid_info = {"build_number": f"ci-{build_number}" if build_number else "local-pytest"} + m.setenv("BUILD_INFO", json.dumps(buid_info)) + yield m + + +@pytest.fixture +def use_secrets_config_file(mocker: MockerFixture): + mocker.patch.dict(os.environ, {PROMPTFLOW_CONNECTIONS: CONNECTION_FILE}) + + +@pytest.fixture +def azure_open_ai_connection() -> AzureOpenAIConnection: + return ConnectionManager().get("azure_open_ai_connection") + + +@pytest.fixture +def temp_output_dir() -> str: + with tempfile.TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def prepare_symbolic_flow() -> str: + flows_dir = PROMPTFLOW_ROOT / "tests" / "test_configs" / "flows" + target_folder = flows_dir / "web_classification_with_symbolic" + source_folder = flows_dir / "web_classification" + + with _change_working_dir(target_folder): + + for file_name in os.listdir(source_folder): + if not Path(file_name).exists(): + os.symlink(source_folder / file_name, file_name) + return target_folder + + +@pytest.fixture +def enable_logger_propagate(): + """This is for test cases that need to check the log output.""" + from promptflow._utils.logger_utils import get_cli_sdk_logger + + logger = get_cli_sdk_logger() + original_value = logger.propagate + logger.propagate = True + yield + logger.propagate = original_value diff --git a/src/promptflow/tests/sdk_cli_azure_test/__init__.py b/src/promptflow-azure/tests/sdk_cli_azure_test/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/__init__.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/__init__.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/_azure_utils.py b/src/promptflow-azure/tests/sdk_cli_azure_test/_azure_utils.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/_azure_utils.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/_azure_utils.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/conftest.py b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py similarity index 98% rename from src/promptflow/tests/sdk_cli_azure_test/conftest.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py index 40bc9231e54..0dde0a79cca 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/conftest.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py @@ -44,13 +44,16 @@ def is_replay(): return False +from _constants import PROMPTFLOW_ROOT + from ._azure_utils import get_cred -FLOWS_DIR = "./tests/test_configs/flows" -EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows" -DATAS_DIR = "./tests/test_configs/datas" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" AZUREML_RESOURCE_PROVIDER = "Microsoft.MachineLearningServices" RESOURCE_ID_FORMAT = "/subscriptions/{}/resourceGroups/{}/providers/{}/workspaces/{}" +MODEL_ROOT = FLOWS_DIR def pytest_configure(): @@ -217,10 +220,6 @@ def runtime(runtime_name: str) -> str: return runtime_name -PROMPTFLOW_ROOT = Path(__file__) / "../../.." -MODEL_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/flows") - - @pytest.fixture def flow_serving_client_remote_connection(mocker: MockerFixture, remote_workspace_resource_id): from promptflow.core._serving.app import create_app as create_serving_app @@ -445,7 +444,7 @@ def mock_get_user_identity_info(user_object_id: str, tenant_id: str) -> None: def created_flow(pf: PFClient, randstr: Callable[[str], str], variable_recorder) -> Flow: """Create a flow for test.""" flow_display_name = randstr("flow_display_name") - flow_source = FLOWS_DIR + "/simple_hello_world/" + flow_source = FLOWS_DIR / "simple_hello_world" description = "test flow description" tags = {"owner": "sdk-test"} result = pf.flows.create_or_update( @@ -610,5 +609,5 @@ def mock_check_latest_version() -> None: As CI uses docker, it will always trigger this check behavior, and we don't have recording for this; and this will hit many unknown issue with vcrpy. """ - with patch("promptflow._utils.version_hint_utils.check_latest_version", new=lambda: None): + with patch("promptflow._sdk._version_hint_utils.check_latest_version", new=lambda: None): yield diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/__init__.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/__init__.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/__init__.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py similarity index 97% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py index 2d247dec949..64393c271fc 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py @@ -5,14 +5,12 @@ from unittest import mock import pytest +from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR from promptflow._cli._user_agent import USER_AGENT as CLI_USER_AGENT # noqa: E402 from promptflow._sdk._telemetry import log_activity from promptflow._utils.user_agent_utils import ClientUserAgentUtil -FLOWS_DIR = "./tests/test_configs/flows" -DATAS_DIR = "./tests/test_configs/datas" - def mock_log_activity(*args, **kwargs): custom_message = "github run: https://github.com/microsoft/promptflow/actions/runs/{0}".format( diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli.py similarity index 92% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli.py index f586e8a87c3..298880ff3dc 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli.py @@ -4,13 +4,13 @@ from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT +from sdk_cli_azure_test.conftest import FLOWS_DIR from promptflow._cli._pf.entry import main -FLOWS_DIR = "./tests/test_configs/flows" -RUNS_DIR = "./tests/test_configs/runs" -CONNECTIONS_DIR = "./tests/test_configs/connections" -DATAS_DIR = "./tests/test_configs/datas" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" +CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections" # TODO: move this to a shared utility module diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py similarity index 97% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py index 3c313d74ce9..4975ea04498 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py @@ -8,7 +8,9 @@ from typing import Callable import pytest +from _constants import PROMPTFLOW_ROOT from mock.mock import patch +from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR from promptflow._constants import PF_USER_AGENT from promptflow._sdk.entities import Run @@ -19,9 +21,7 @@ from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD -FLOWS_DIR = "./tests/test_configs/flows" -DATAS_DIR = "./tests/test_configs/datas" -RUNS_DIR = "./tests/test_configs/runs" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" # TODO: move this to a shared utility module diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py similarity index 90% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py index 1267edae5a9..7f11dcb2c8e 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py @@ -3,18 +3,13 @@ import pydash import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._utils.yaml_utils import dump_yaml, load_yaml_string from promptflow.connections import AzureOpenAIConnection from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() - def assert_dict_equals_with_skip_fields(item1, item2, skip_fields): for fot_key in skip_fields: @@ -106,7 +101,7 @@ def test_flow_as_component( # keep the simplest test here, more tests are in azure-ai-ml from azure.ai.ml import load_component - flows_dir = "./tests/test_configs/flows" + flows_dir = PROMPTFLOW_ROOT / "tests/test_configs/flows" flow_func: Component = load_component( f"{flows_dir}/web_classification/flow.dag.yaml", params_override=[load_params] @@ -114,10 +109,9 @@ def test_flow_as_component( # TODO: snapshot of flow component changed every time? created_component = ml_client.components.create_or_update(flow_func, is_anonymous=True) + spec_path = flows_dir / "saved_component_spec" / f"{request.node.callspec.id}.yaml" - update_saved_spec( - created_component, f"./tests/test_configs/flows/saved_component_spec/{request.node.callspec.id}.yaml" - ) + update_saved_spec(created_component, spec_path.resolve().absolute().as_posix()) component_dict = created_component._to_dict() slimmed_created_component_attrs = {key: pydash.get(component_dict, key) for key in expected_spec_attrs.keys()} diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py similarity index 91% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py index 103c74b318d..c2d33b1481a 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py @@ -2,19 +2,15 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- import json -from pathlib import Path import pytest +from sdk_cli_azure_test.conftest import FLOWS_DIR from promptflow.azure._entities._flow import Flow from promptflow.exceptions import UserErrorException from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD -tests_root_dir = Path(__file__).parent.parent.parent -flow_test_dir = tests_root_dir / "test_configs/flows" -data_dir = tests_root_dir / "test_configs/datas" - @pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD) @pytest.mark.e2etest @@ -64,7 +60,7 @@ def test_flow_test_with_config(self, remote_workspace_resource_id): from promptflow import PFClient client = PFClient(config={"connection.provider": remote_workspace_resource_id}) - output = client.test(flow=flow_test_dir / "web_classification") + output = client.test(flow=FLOWS_DIR / "web_classification") assert output.keys() == {"category", "evidence"} @pytest.mark.usefixtures("mock_get_user_identity_info") diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_run_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py similarity index 99% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_run_operations.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py index 33be3624166..e8874b2073b 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_run_operations.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py @@ -16,8 +16,10 @@ import pandas as pd import pydash import pytest +from _constants import PROMPTFLOW_ROOT from azure.ai.ml import ManagedIdentityConfiguration from azure.ai.ml.entities import IdentityConfiguration +from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR from promptflow._constants import FLOW_FLEX_YAML from promptflow._sdk._configuration import Configuration @@ -40,15 +42,8 @@ from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() -FLOWS_DIR = "./tests/test_configs/flows" -EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows" -RUNS_DIR = "./tests/test_configs/runs" -DATAS_DIR = "./tests/test_configs/datas" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" def create_registry_run(name: str, registry_name: str, runtime: str, pf: PFClient): diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_telemetry.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_telemetry.py similarity index 88% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_telemetry.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_telemetry.py index 08954802339..04d0f06cdb9 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_telemetry.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_telemetry.py @@ -15,6 +15,8 @@ import pydash import pytest +from _constants import PROMPTFLOW_ROOT +from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR from promptflow import load_run from promptflow._constants import PF_USER_AGENT @@ -36,6 +38,9 @@ from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" + @contextlib.contextmanager def cli_consent_config_overwrite(val): @@ -65,10 +70,6 @@ def extension_consent_config_overwrite(val): config.set_config(key=Configuration.EXTENSION_COLLECT_TELEMETRY, value=True) -RUNS_DIR = "./tests/test_configs/runs" -FLOWS_DIR = "./tests/test_configs/flows" - - @pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD) @pytest.mark.usefixtures("mock_set_headers_with_user_aml_token", "single_worker_thread_pool", "vcr_recording") @pytest.mark.e2etest @@ -125,47 +126,51 @@ def check_evelope(): assert isinstance(custom_dimensions, dict) # Note: need privacy review if we add new fields. if "start" in envelope.data.base_data.name: - assert custom_dimensions.keys() == { - "request_id", - "activity_name", - "activity_type", - "subscription_id", - "resource_group_name", - "workspace_name", - "level", - "python_version", - "user_agent", - "installation_id", - "first_call", - "from_ci", - "error_category", - "error_type", - "error_target", - "error_message", - "error_detail", - } + assert sorted(custom_dimensions.keys()) == sorted( + { + "request_id", + "activity_name", + "activity_type", + "subscription_id", + "resource_group_name", + "workspace_name", + "level", + "python_version", + "user_agent", + "installation_id", + "first_call", + "from_ci", + "error_category", + "error_type", + "error_target", + "error_message", + "error_detail", + } + ) elif "complete" in envelope.data.base_data.name: - assert custom_dimensions.keys() == { - "request_id", - "activity_name", - "activity_type", - "subscription_id", - "resource_group_name", - "workspace_name", - "completion_status", - "duration_ms", - "level", - "python_version", - "user_agent", - "installation_id", - "first_call", - "from_ci", - "error_category", - "error_type", - "error_target", - "error_message", - "error_detail", - } + assert sorted(custom_dimensions.keys()) == sorted( + { + "request_id", + "activity_name", + "activity_type", + "subscription_id", + "resource_group_name", + "workspace_name", + "completion_status", + "duration_ms", + "level", + "python_version", + "user_agent", + "installation_id", + "first_call", + "from_ci", + "error_category", + "error_type", + "error_target", + "error_message", + "error_detail", + } + ) else: raise ValueError("Invalid message: {}".format(envelope.data.base_data.name)) assert envelope.data.base_data.name.startswith("pfazure.runs.get") @@ -445,8 +450,8 @@ def check_evelope(): ): flow_type = FlowType.DAG_FLOW pf.run( - flow="./tests/test_configs/flows/print_input_flow", - data="./tests/test_configs/datas/print_input_flow.jsonl", + flow=FLOWS_DIR / "print_input_flow", + data=DATAS_DIR / "print_input_flow.jsonl", name=randstr("name"), ) logger = get_telemetry_logger() @@ -455,8 +460,8 @@ def check_evelope(): flow_type = FlowType.FLEX_FLOW pf.run( - flow="./tests/test_configs/eager_flows/simple_with_req", - data="./tests/test_configs/datas/simple_eager_flow_data.jsonl", + flow=EAGER_FLOWS_DIR / "simple_with_req", + data=DATAS_DIR / "simple_eager_flow_data.jsonl", name=randstr("name"), ) logger.handlers[0].flush() diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/__init__.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/__init__.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/__init__.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_blob_client.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_blob_client.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_blob_client.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_blob_client.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cli.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cli.py similarity index 97% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_cli.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cli.py index 25ec32e7b06..2cc95f4d4d9 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cli.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cli.py @@ -8,13 +8,10 @@ import pandas as pd import pytest from pytest_mock import MockFixture +from sdk_cli_azure_test.conftest import FLOWS_DIR from promptflow._sdk._constants import VIS_PORTAL_URL_TMPL -tests_root_dir = Path(__file__).parent.parent.parent -flow_test_dir = tests_root_dir / "test_configs/flows" -data_dir = tests_root_dir / "test_configs/datas" - def run_pf_command(*args, cwd=None): from promptflow.azure._cli.entry import main @@ -243,7 +240,7 @@ def test_flow_create( mocked = mocker.patch.object(FlowOperations, "create_or_update") mocked.return_value._to_dict.return_value = {"name": "test_run"} - flow_dir = Path(flow_test_dir, "web_classification").resolve().as_posix() + flow_dir = Path(FLOWS_DIR, "web_classification").resolve().as_posix() run_pf_command( "flow", "create", @@ -269,7 +266,7 @@ def test_flow_create_with_unknown_field(self, mocker: MockFixture, operation_sco mocked = mocker.patch.object(FlowOperations, "create_or_update") mocked.return_value._to_dict.return_value = {"name": "test_run"} - flow_dir = Path(flow_test_dir, "web_classification").resolve().as_posix() + flow_dir = Path(FLOWS_DIR, "web_classification").resolve().as_posix() run_pf_command( "flow", "create", diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_collection.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_collection.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_collection.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_collection.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_config.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_config.py similarity index 94% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_config.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_config.py index 4d32e2cc6ab..caa19eed957 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_config.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_config.py @@ -1,9 +1,9 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._configuration import ConfigFileNotFound, Configuration, InvalidConfigFile from promptflow._utils.context_utils import _change_working_dir @@ -11,7 +11,8 @@ AZUREML_RESOURCE_PROVIDER = "Microsoft.MachineLearningServices" RESOURCE_ID_FORMAT = "/subscriptions/{}/resourceGroups/{}/providers/{}/workspaces/{}" -CONFIG_DATA_ROOT = Path(__file__).parent.parent.parent / "test_configs" / "configs" + +CONFIG_DATA_ROOT = PROMPTFLOW_ROOT / "tests/test_configs/configs" @pytest.fixture diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_exceptions.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_exceptions.py similarity index 99% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_exceptions.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_exceptions.py index 1ce7c1206e7..607027349dd 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_exceptions.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_exceptions.py @@ -12,8 +12,6 @@ from promptflow.executor import FlowValidator from promptflow.executor._errors import InvalidNodeReference -FLOWS_DIR = "./tests/test_configs/flows/print_input_flow" - def is_match_error_detail(expected_info, actual_info): expected_info = re.sub(r"line \d+", r"", expected_info).replace("\n", "").replace(" ", "") diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_entity.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py similarity index 95% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_entity.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py index a1945ba230a..9ba0ce5629d 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_entity.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py @@ -8,7 +8,9 @@ from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from mock.mock import Mock +from sdk_cli_azure_test.conftest import FLOWS_DIR from promptflow import load_run from promptflow._sdk._vendor import get_upload_files_from_folder @@ -17,9 +19,7 @@ from promptflow.azure._entities._flow import Flow from promptflow.exceptions import ValidationException -tests_root_dir = Path(__file__).parent.parent.parent -FLOWS_DIR = (tests_root_dir / "test_configs/flows").resolve() -RUNS_DIR = (tests_root_dir / "test_configs/runs").resolve() +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" def load_flow(source): @@ -33,7 +33,7 @@ class TestFlow: @pytest.mark.skip(reason="TODO: add back when we bring back meta.yaml") def test_load_flow(self): - local_file = tests_root_dir / "test_configs/flows/meta_files/flow.meta.yaml" + local_file = FLOWS_DIR / "meta_files/flow.meta.yaml" flow = load_flow(source=local_file) @@ -58,7 +58,7 @@ def test_load_flow(self): def test_load_flow_from_remote_storage(self): from promptflow.azure.operations._flow_operations import FlowOperations - local_file = tests_root_dir / "test_configs/flows/meta_files/remote_fs.meta.yaml" + local_file = FLOWS_DIR / "meta_files/remote_fs.meta.yaml" flow = load_flow(source=local_file) @@ -79,7 +79,7 @@ def test_load_flow_from_remote_storage(self): } def test_ignore_files_in_flow(self): - local_file = tests_root_dir / "test_configs/flows/web_classification" + local_file = FLOWS_DIR / "web_classification" with tempfile.TemporaryDirectory() as temp: flow_path = Path(temp) / "flow" shutil.copytree(local_file, flow_path) diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_operations.py similarity index 89% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_operations.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_operations.py index 15831f80ea4..e1e55a27184 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_operations.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_operations.py @@ -1,20 +1,18 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -from pathlib import Path from unittest.mock import patch import pytest +from _constants import PROMPTFLOW_ROOT +from sdk_cli_azure_test.conftest import FLOWS_DIR from promptflow._sdk._constants import AzureFlowSource from promptflow._sdk._errors import FlowOperationError from promptflow.azure._entities._flow import Flow from promptflow.exceptions import UserErrorException -tests_root_dir = Path(__file__).parent.parent.parent -eager_flow_test_dir = tests_root_dir / "test_configs/eager_flows" -flow_test_dir = tests_root_dir / "test_configs/flows" -data_dir = tests_root_dir / "test_configs/datas" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" @pytest.mark.unittest @@ -23,7 +21,7 @@ def test_create_flow_with_invalid_parameters(self, pf): with pytest.raises(UserErrorException, match=r"fake_source does not exist."): pf.flows.create_or_update(flow="fake_source") - flow_source = flow_test_dir / "web_classification/" + flow_source = FLOWS_DIR / "web_classification/" with pytest.raises(UserErrorException, match="Not a valid string"): pf.flows.create_or_update(flow=flow_source, display_name=False) @@ -42,7 +40,7 @@ def test_update_flow_with_invalid_parameters(self, pf): @pytest.mark.usefixtures("enable_logger_propagate") def test_create_flow_with_warnings(self, pf, caplog): - flow_source = flow_test_dir / "web_classification/" + flow_source = FLOWS_DIR / "web_classification/" pf.flows._validate_flow_creation_parameters(source=flow_source, random="random") assert "random: Unknown field" in caplog.text @@ -90,7 +88,7 @@ def mock_get_arm_token(*args, **kwargs) -> str: assert user_tenant_id == mock_tid def test_eager_flow_creation(self, pf): - flow_source = eager_flow_test_dir / "simple_with_yaml" + flow_source = EAGER_FLOWS_DIR / "simple_with_yaml" with pytest.raises(UserErrorException) as e: pf.flows.create_or_update( flow=flow_source, diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_entity.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_entity.py similarity index 86% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_entity.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_entity.py index c539d8833c9..bee29031ae6 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_entity.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_entity.py @@ -7,20 +7,12 @@ from unittest.mock import Mock import pytest +from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR from promptflow._sdk.entities import Run from promptflow._utils.flow_utils import get_flow_lineage_id from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() -FLOWS_DIR = "./tests/test_configs/flows" -RUNS_DIR = "./tests/test_configs/runs" -DATAS_DIR = "./tests/test_configs/datas" - @pytest.mark.unittest class TestRun: diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py similarity index 98% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_operations.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py index bb42729deaf..249597935f0 100644 --- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_operations.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py @@ -6,6 +6,7 @@ from azure.ai.ml import ManagedIdentityConfiguration from azure.ai.ml.entities import IdentityConfiguration from pytest_mock import MockerFixture +from sdk_cli_azure_test.conftest import DATAS_DIR, EAGER_FLOWS_DIR, FLOWS_DIR from promptflow._sdk._errors import RunOperationParameterError, UploadUserError, UserAuthenticationError from promptflow._sdk._utils import parse_otel_span_status_code @@ -16,10 +17,6 @@ from promptflow.azure.operations._async_run_uploader import AsyncRunUploader from promptflow.exceptions import UserErrorException -FLOWS_DIR = "./tests/test_configs/flows" -DATAS_DIR = "./tests/test_configs/datas" -EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows" - @pytest.mark.unittest class TestRunOperations: diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_span.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_span.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_span.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_span.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_summary.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_summary.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_summary.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_summary.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_trace.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_trace.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_trace.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_trace.py diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_utils.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_utils.py similarity index 100% rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_utils.py rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_utils.py diff --git a/src/promptflow-core/promptflow/_constants.py b/src/promptflow-core/promptflow/_constants.py index 78c2956e8c8..ba6ab76fa08 100644 --- a/src/promptflow-core/promptflow/_constants.py +++ b/src/promptflow-core/promptflow/_constants.py @@ -277,6 +277,7 @@ class ConnectionProviderConfig: "(/providers/Microsoft.MachineLearningServices)?/workspaces/([^/]+)$" ) CONNECTION_DATA_CLASS_KEY = "DATA_CLASS" +AML_WORKSPACE_TEMPLATE = "azureml://subscriptions/{}/resourceGroups/{}/providers/Microsoft.MachineLearningServices/workspaces/{}" # noqa: E501 class AzureWorkspaceKind: diff --git a/src/promptflow-core/promptflow/_utils/flow_utils.py b/src/promptflow-core/promptflow/_utils/flow_utils.py index bfa46cf0706..1e9506852e6 100644 --- a/src/promptflow-core/promptflow/_utils/flow_utils.py +++ b/src/promptflow-core/promptflow/_utils/flow_utils.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- import hashlib +import itertools import json import os import re @@ -87,13 +88,19 @@ def resolve_flow_path( if flow_path.is_dir(): flow_folder = flow_path - dag_file_exist = (flow_folder / FLOW_DAG_YAML).is_file() - flex_file_exist = (flow_folder / FLOW_FLEX_YAML).is_file() - flow_file = FLOW_FLEX_YAML if flex_file_exist else FLOW_DAG_YAML - if dag_file_exist and flex_file_exist: + flow_file = FLOW_DAG_YAML + flow_file_list = [] + for flow_name, suffix in itertools.product([FLOW_DAG_YAML, FLOW_FLEX_YAML], [".yaml", ".yml"]): + flow_file_name = flow_name.replace(".yaml", suffix) + if (flow_folder / flow_file_name).is_file(): + flow_file_list.append(flow_file_name) + + if len(flow_file_list) == 1: + flow_file = flow_file_list[0] + elif len(flow_file_list) > 1: raise ValidationException( - f"Both {FLOW_DAG_YAML} and {FLOW_FLEX_YAML} exist in {flow_path}. " - f"Please specify a file or remove the extra YAML.", + f"Multiple files {', '.join(flow_file_list)} exist in {flow_path}. " + f"Please specify a file or remove the extra YAML file.", privacy_info=[str(flow_path)], ) elif flow_path.is_file() or flow_path.suffix.lower() in FLOW_FILE_SUFFIX: diff --git a/src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py b/src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py index 0f5f457a25e..fb05ffee1bc 100644 --- a/src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py +++ b/src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py @@ -6,6 +6,7 @@ from promptflow._constants import CONNECTION_NAME_PROPERTY, CONNECTION_SECRET_KEYS, CustomStrongTypeConnectionConfigs from promptflow._utils.utils import try_import +from promptflow.contracts.tool import ConnectionType from promptflow.contracts.types import Secret from ._connection_provider import ConnectionProvider @@ -47,7 +48,7 @@ def _build_connection(connection_dict: dict): secret_keys = connection_dict.get("secret_keys", []) secrets = {k: v for k, v in value.items() if k in secret_keys} configs = {k: v for k, v in value.items() if k not in secrets} - connection_value = connection_class(configs=configs, secrets=secrets, name=key) + connection_value = connection_class(configs=configs, secrets=secrets, name=name) if CustomStrongTypeConnectionConfigs.PROMPTFLOW_TYPE_KEY in configs: connection_value.custom_type = configs[CustomStrongTypeConnectionConfigs.PROMPTFLOW_TYPE_KEY] else: @@ -85,7 +86,7 @@ def _build_connections(cls, _dict: Mapping[str, dict]): def import_requisites(cls, _dict: Mapping[str, dict]): """Import connection required modules.""" modules = set() - for key, connection_dict in _dict.items(): + for _, connection_dict in _dict.items(): module = connection_dict.get("module") if module: modules.add(module) @@ -96,5 +97,9 @@ def import_requisites(cls, _dict: Mapping[str, dict]): def list(self): return [c for c in self._connections.values()] - def get(self, name: str, **kwargs) -> Any: - return self._connections.get(name) + def get(self, name: str) -> Any: + if isinstance(name, str): + return self._connections.get(name) + elif ConnectionType.is_connection_value(name): + return name + return None diff --git a/src/promptflow-core/promptflow/core/_connection_provider/_utils.py b/src/promptflow-core/promptflow/core/_connection_provider/_utils.py index 4ac642e76a1..6594c2dd1d1 100644 --- a/src/promptflow-core/promptflow/core/_connection_provider/_utils.py +++ b/src/promptflow-core/promptflow/core/_connection_provider/_utils.py @@ -3,9 +3,9 @@ # --------------------------------------------------------- import os -from promptflow._constants import PF_NO_INTERACTIVE_LOGIN +from promptflow._constants import PF_NO_INTERACTIVE_LOGIN, AzureWorkspaceKind from promptflow._utils.user_agent_utils import ClientUserAgentUtil -from promptflow.core._errors import MissingRequiredPackage +from promptflow.core._errors import MissingRequiredPackage, UnsupportedWorkspaceKind from promptflow.exceptions import ValidationException @@ -15,7 +15,7 @@ def check_required_packages(): import azure.identity # noqa: F401 except ImportError as e: raise MissingRequiredPackage( - message="Please install 'promptflow-core[azureml-serving]' to use workspace related features." + message="Please install 'promptflow-core[azureml-serving]' to use Azure related features." ) from e @@ -67,3 +67,14 @@ def interactive_credential_disabled(): def is_from_cli(): """Check if the current execution is from promptflow-cli.""" return "promptflow-cli" in ClientUserAgentUtil.get_user_agent() + + +def check_connection_provider_resource(resource_id: str, credential, pkg_name): + from .._utils import get_workspace_from_resource_id + + workspace = get_workspace_from_resource_id(resource_id, credential, pkg_name) + if workspace._kind not in [AzureWorkspaceKind.DEFAULT, AzureWorkspaceKind.PROJECT]: + raise UnsupportedWorkspaceKind( + message=f"Workspace kind {workspace._kind!r} is not supported. " + f"Please use either an azure machine learning workspace or an azure ai project." + ) diff --git a/src/promptflow-core/promptflow/core/_connection_provider/_workspace_connection_provider.py b/src/promptflow-core/promptflow/core/_connection_provider/_workspace_connection_provider.py index f7edb5f9c4a..e4fab554442 100644 --- a/src/promptflow-core/promptflow/core/_connection_provider/_workspace_connection_provider.py +++ b/src/promptflow-core/promptflow/core/_connection_provider/_workspace_connection_provider.py @@ -6,7 +6,7 @@ import requests -from promptflow._constants import ConnectionAuthMode +from promptflow._constants import AML_WORKSPACE_TEMPLATE, ConnectionAuthMode from promptflow._utils.retry_utils import http_retry_wrapper from promptflow.core._connection import CustomConnection, _Connection from promptflow.core._errors import ( @@ -16,6 +16,7 @@ MissingRequiredPackage, OpenURLFailed, OpenURLFailedUserError, + OpenURLNotFoundError, OpenURLUserAuthenticationError, UnknownConnectionType, UnsupportedConnectionAuthType, @@ -24,7 +25,12 @@ from ..._utils.credential_utils import get_default_azure_credential from ._connection_provider import ConnectionProvider -from ._utils import interactive_credential_disabled, is_from_cli, is_github_codespaces +from ._utils import ( + check_connection_provider_resource, + interactive_credential_disabled, + is_from_cli, + is_github_codespaces, +) GET_CONNECTION_URL = ( "/subscriptions/{sub}/resourcegroups/{rg}/providers/Microsoft.MachineLearningServices" @@ -76,6 +82,10 @@ def __init__( self.subscription_id = subscription_id self.resource_group_name = resource_group_name self.workspace_name = workspace_name + self.resource_id = AML_WORKSPACE_TEMPLATE.format( + self.subscription_id, self.resource_group_name, self.workspace_name + ) + self._workspace_checked = False @property def credential(self): @@ -131,7 +141,13 @@ def open_url(cls, token, url, action, host="management.azure.com", method="GET", f"Open url {{url}} failed with status code: {response.status_code}, action: {action}, reason: {{reason}}" ) if response.status_code == 403: - raise AccessDeniedError(operation=url, target=ErrorTarget.RUNTIME) + raise AccessDeniedError(operation=url, target=ErrorTarget.CORE) + elif response.status_code == 404: + raise OpenURLNotFoundError( + message_format=message_format, + url=url, + reason=response.reason, + ) elif 400 <= response.status_code < 500: raise OpenURLFailedUserError( message_format=message_format, @@ -404,6 +420,8 @@ def _build_list_connection_dict( raise OpenURLUserAuthenticationError(message=auth_error_message) except ClientAuthenticationError as e: raise UserErrorException(target=ErrorTarget.CORE, message=str(e), error=e) from e + except UserErrorException: + raise except Exception as e: raise SystemErrorException(target=ErrorTarget.CORE, message=str(e), error=e) from e @@ -417,6 +435,12 @@ def _build_list_connection_dict( return rest_list_connection_dict def list(self) -> List[_Connection]: + if not self._workspace_checked: + # Check workspace not 'hub' + check_connection_provider_resource( + resource_id=self.resource_id, credential=self.credential, pkg_name="promptflow-core[azureml-serving]" + ) + self._workspace_checked = True rest_list_connection_dict = self._build_list_connection_dict( subscription_id=self.subscription_id, resource_group_name=self.resource_group_name, @@ -432,6 +456,12 @@ def list(self) -> List[_Connection]: return connection_list def get(self, name: str, **kwargs) -> _Connection: + if not self._workspace_checked: + # Check workspace not 'hub' + check_connection_provider_resource( + resource_id=self.resource_id, credential=self.credential, pkg_name="promptflow-core[azureml-serving]" + ) + self._workspace_checked = True connection_dict = self._build_connection_dict( name, subscription_id=self.subscription_id, diff --git a/src/promptflow-core/promptflow/core/_errors.py b/src/promptflow-core/promptflow/core/_errors.py index 95380d0d689..c742de0afbd 100644 --- a/src/promptflow-core/promptflow/core/_errors.py +++ b/src/promptflow-core/promptflow/core/_errors.py @@ -100,6 +100,11 @@ def __init__(self, **kwargs): super().__init__(target=ErrorTarget.CORE, **kwargs) +class OpenURLNotFoundError(UserErrorException): + def __init__(self, **kwargs): + super().__init__(target=ErrorTarget.CORE, **kwargs) + + class UnknownConnectionType(UserErrorException): def __init__(self, **kwargs): super().__init__(target=ErrorTarget.CORE, **kwargs) @@ -127,6 +132,13 @@ def __init__(self, provider_config, **kwargs): super().__init__(target=ErrorTarget.CORE, message=message, **kwargs) +class UnsupportedWorkspaceKind(UserErrorException): + """Exception raised when workspace kind is not supported.""" + + def __init__(self, message, **kwargs): + super().__init__(target=ErrorTarget.CORE, message=message, **kwargs) + + class AccessDeniedError(UserErrorException): """Exception raised when run info can not be found in storage""" diff --git a/src/promptflow-core/promptflow/core/_serving/extension/azureml_extension.py b/src/promptflow-core/promptflow/core/_serving/extension/azureml_extension.py index c5ab797d0b9..9170993cf69 100644 --- a/src/promptflow-core/promptflow/core/_serving/extension/azureml_extension.py +++ b/src/promptflow-core/promptflow/core/_serving/extension/azureml_extension.py @@ -7,6 +7,7 @@ import re from typing import Any, Tuple +from promptflow._constants import AML_WORKSPACE_TEMPLATE from promptflow._utils.retry_utils import retry from promptflow.contracts.flow import Flow from promptflow.core._serving._errors import InvalidConnectionData, MissingConnectionProvider @@ -18,7 +19,6 @@ USER_AGENT = f"promptflow-cloud-serving/{__version__}" AML_DEPLOYMENT_RESOURCE_ID_REGEX = "/subscriptions/(.*)/resourceGroups/(.*)/providers/Microsoft.MachineLearningServices/workspaces/(.*)/onlineEndpoints/(.*)/deployments/(.*)" # noqa: E501 -AML_CONNECTION_PROVIDER_TEMPLATE = "azureml://subscriptions/{}/resourceGroups/{}/providers/Microsoft.MachineLearningServices/workspaces/{}" # noqa: E501 class AzureMLExtension(AppExtension): @@ -159,7 +159,7 @@ def _initialize_connection_provider(self): message="Missing connection provider, please check whether 'PROMPTFLOW_CONNECTION_PROVIDER' " "is in your environment variable list." ) # noqa: E501 - self.connection_provider = AML_CONNECTION_PROVIDER_TEMPLATE.format( + self.connection_provider = AML_WORKSPACE_TEMPLATE.format( self.subscription_id, self.resource_group, self.workspace_name ) # noqa: E501 diff --git a/src/promptflow-core/promptflow/executor/_service/contracts/execution_request.py b/src/promptflow-core/promptflow/executor/_service/contracts/execution_request.py index 5834699c3fb..5dacb86917c 100644 --- a/src/promptflow-core/promptflow/executor/_service/contracts/execution_request.py +++ b/src/promptflow-core/promptflow/executor/_service/contracts/execution_request.py @@ -22,6 +22,7 @@ class BaseExecutionRequest(BaseRequest): connections: Optional[Mapping[str, Any]] = None environment_variables: Optional[Mapping[str, Any]] = None flow_name: Optional[str] = None + flow_logs_folder: Optional[str] = None def get_run_mode(self): raise NotImplementedError(f"Request type {self.__class__.__name__} is not implemented.") diff --git a/src/promptflow-core/promptflow/executor/_service/utils/service_utils.py b/src/promptflow-core/promptflow/executor/_service/utils/service_utils.py index 6ab5861ee93..d6316c51727 100644 --- a/src/promptflow-core/promptflow/executor/_service/utils/service_utils.py +++ b/src/promptflow-core/promptflow/executor/_service/utils/service_utils.py @@ -17,7 +17,12 @@ def get_log_context(request: BaseExecutionRequest, enable_service_logger: bool = False) -> LogContext: run_mode = request.get_run_mode() credential_list = ConnectionManager(request.connections).get_secret_list() - log_context = LogContext(file_path=request.log_path, run_mode=run_mode, credential_list=credential_list) + log_context = LogContext( + file_path=request.log_path, + run_mode=run_mode, + credential_list=credential_list, + flow_logs_folder=request.flow_logs_folder, + ) if enable_service_logger: log_context.input_logger = service_logger return log_context diff --git a/src/promptflow-core/promptflow/executor/_tool_resolver.py b/src/promptflow-core/promptflow/executor/_tool_resolver.py index 41ae5130122..9458986cf12 100644 --- a/src/promptflow-core/promptflow/executor/_tool_resolver.py +++ b/src/promptflow-core/promptflow/executor/_tool_resolver.py @@ -13,7 +13,6 @@ from promptflow._constants import MessageFormatType from promptflow._core._errors import InvalidSource -from promptflow._core.connection_manager import ConnectionManager from promptflow._core.tool import STREAMING_OPTION_PARAMETER_ATTR from promptflow._core.tools_manager import BuiltinsManager, ToolLoader, connection_type_to_api_mapping from promptflow._utils.multimedia_utils import MultimediaProcessor @@ -23,6 +22,7 @@ get_prompt_param_name_from_func, ) from promptflow._utils.yaml_utils import load_yaml +from promptflow.connections import ConnectionProvider from promptflow.contracts.flow import InputAssignment, InputValueType, Node, ToolSource, ToolSourceType from promptflow.contracts.tool import ConnectionType, Tool, ToolType, ValueType from promptflow.contracts.types import AssistantDefinition, PromptTemplate @@ -60,7 +60,7 @@ class ToolResolver: def __init__( self, working_dir: Path, - connections: Optional[dict] = None, + connection_provider: Optional[ConnectionProvider] = None, package_tool_keys: Optional[List[str]] = None, message_format: str = MessageFormatType.BASIC, ): @@ -71,7 +71,7 @@ def __init__( pass self._tool_loader = ToolLoader(working_dir, package_tool_keys=package_tool_keys) self._working_dir = working_dir - self._connection_manager = ConnectionManager(connections) + self._connection_provider = connection_provider self._multimedia_processor = MultimediaProcessor.create(message_format) @classmethod @@ -83,7 +83,7 @@ def start_resolver( return resolver def _convert_to_connection_value(self, k: str, v: InputAssignment, node_name: str, conn_types: List[ValueType]): - connection_value = self._connection_manager.get(v.value) + connection_value = self._connection_provider.get(v.value) if not connection_value: raise ConnectionNotFound(f"Connection {v.value} not found for node {node_name!r} input {k!r}.") # Check if type matched @@ -108,7 +108,7 @@ def _convert_to_custom_strong_type_connection_value( if not conn_types: msg = f"Input '{k}' for node '{node_name}' has invalid types: {conn_types}." raise NodeInputValidationError(message=msg) - connection_value = self._connection_manager.get(v.value) + connection_value = self._connection_provider.get(v.value) if not connection_value: raise ConnectionNotFound(f"Connection {v.value} not found for node {node_name!r} input {k!r}.") @@ -476,7 +476,7 @@ def _remove_init_args(node_inputs: dict, init_args: dict): del node_inputs[k] def _get_llm_node_connection(self, node: Node): - connection = self._connection_manager.get(node.connection) + connection = self._connection_provider.get(node.connection) if connection is None: raise ConnectionNotFound( message_format="Connection '{connection}' of LLM node '{node_name}' is not found.", diff --git a/src/promptflow-core/promptflow/executor/flow_executor.py b/src/promptflow-core/promptflow/executor/flow_executor.py index 035885c014c..dcf086e596a 100644 --- a/src/promptflow-core/promptflow/executor/flow_executor.py +++ b/src/promptflow-core/promptflow/executor/flow_executor.py @@ -15,7 +15,7 @@ from pathlib import Path from threading import current_thread from types import GeneratorType -from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union import opentelemetry.trace as otel_trace from opentelemetry.trace.status import StatusCode @@ -41,9 +41,11 @@ from promptflow._utils.user_agent_utils import append_promptflow_package_ua from promptflow._utils.utils import get_int_env_var, transpose from promptflow._utils.yaml_utils import load_yaml +from promptflow.connections import ConnectionProvider from promptflow.contracts.flow import Flow, FlowInputDefinition, InputAssignment, InputValueType, Node from promptflow.contracts.run_info import FlowRunInfo, Status from promptflow.contracts.run_mode import RunMode +from promptflow.core._connection_provider._dict_connection_provider import DictConnectionProvider from promptflow.exceptions import PromptflowException from promptflow.executor import _input_assignment_parser from promptflow.executor._async_nodes_scheduler import AsyncNodesScheduler @@ -98,7 +100,7 @@ class FlowExecutor: def __init__( self, flow: Flow, - connections: dict, + connections: ConnectionProvider, run_tracker: RunTracker, cache_manager: AbstractCacheManager, loaded_tools: Mapping[str, Callable], @@ -113,7 +115,7 @@ def __init__( :param flow: The Flow object to execute. :type flow: ~promptflow.contracts.flow.Flow :param connections: The connections between nodes in the Flow. - :type connections: dict + :type connections: Union[dict, ConnectionProvider] :param run_tracker: The RunTracker object to track the execution of the Flow. :type run_tracker: ~promptflow._core.run_tracker.RunTracker :param cache_manager: The AbstractCacheManager object to manage caching of results. @@ -170,7 +172,7 @@ def __init__( def create( cls, flow_file: Path, - connections: dict, + connections: Union[dict, ConnectionProvider], working_dir: Optional[Path] = None, *, entry: Optional[str] = None, @@ -186,7 +188,7 @@ def create( :param flow_file: The path to the flow file. :type flow_file: Path :param connections: The connections to be used for the flow. - :type connections: dict + :type connections: Union[dict, ConnectionProvider] :param working_dir: The working directory to be used for the flow. Default is None. :type working_dir: Optional[str] :param func: The function to be used for the flow if .py is provided. Default is None. @@ -246,7 +248,7 @@ def create( def _create_from_flow( cls, flow: Flow, - connections: dict, + connections: Union[dict, ConnectionProvider], working_dir: Optional[Path], *, flow_file: Optional[Path] = None, @@ -262,6 +264,8 @@ def _create_from_flow( flow = flow._apply_default_node_variants() package_tool_keys = [node.source.tool for node in flow.nodes if node.source and node.source.tool] + if isinstance(connections, dict): + connections = DictConnectionProvider(connections) tool_resolver = ToolResolver(working_dir, connections, package_tool_keys, message_format=flow.message_format) with _change_working_dir(working_dir): @@ -393,6 +397,8 @@ def update_operation_context(): inputs = multimedia_processor.load_multimedia_data(node_referenced_flow_inputs, converted_flow_inputs_for_node) dependency_nodes_outputs = multimedia_processor.load_multimedia_data_recursively(dependency_nodes_outputs) package_tool_keys = [node.source.tool] if node.source and node.source.tool else [] + if isinstance(connections, dict): + connections = DictConnectionProvider(connections) tool_resolver = ToolResolver(working_dir, connections, package_tool_keys, message_format=flow.message_format) resolved_node = tool_resolver.resolve_tool_by_node(node) @@ -1329,6 +1335,7 @@ def execute_flow( run_aggregation: bool = True, enable_stream_output: bool = False, allow_generator_output: bool = False, # TODO: remove this + init_kwargs: Optional[dict] = None, **kwargs, ) -> LineResult: """Execute the flow, including aggregation nodes. @@ -1347,12 +1354,16 @@ def execute_flow( :type enable_stream_output: Optional[bool] :param run_id: Run id will be set in operation context and used for session. :type run_id: Optional[str] + :param init_kwargs: Initialization parameters for flex flow, only supported when flow is callable class. + :type init_kwargs: dict :param kwargs: Other keyword arguments to create flow executor. :type kwargs: Any :return: The line result of executing the flow. :rtype: ~promptflow.executor._result.LineResult """ - flow_executor = FlowExecutor.create(flow_file, connections, working_dir, raise_ex=False, **kwargs) + flow_executor = FlowExecutor.create( + flow_file, connections, working_dir, raise_ex=False, init_kwargs=init_kwargs, **kwargs + ) flow_executor.enable_streaming_for_llm_flow(lambda: enable_stream_output) with _change_working_dir(working_dir), _force_flush_tracer_provider(): # Execute nodes in the flow except the aggregation nodes diff --git a/src/promptflow-core/pyproject.toml b/src/promptflow-core/pyproject.toml index c38857195e6..2b873be545d 100644 --- a/src/promptflow-core/pyproject.toml +++ b/src/promptflow-core/pyproject.toml @@ -104,9 +104,13 @@ testpaths = ["tests"] [tool.coverage.run] concurrency = ["multiprocessing"] -source = ["promptflow"] +source = [ + "*/promptflow/*" +] omit = [ - "__init__.py", + "*/__init__.py", + "*/promptflow/core/_connection_provider/_models/*", + "*/promptflow/tracing/*" ] [tool.black] diff --git a/src/promptflow-devkit/CHANGELOG.md b/src/promptflow-devkit/CHANGELOG.md new file mode 100644 index 00000000000..baa72a7e2af --- /dev/null +++ b/src/promptflow-devkit/CHANGELOG.md @@ -0,0 +1,9 @@ +# Release History + +## 1.9.0 (Upcoming) + +### Features Added +- Added autocomplete feature for linux, reach [here](https://microsoft.github.io/promptflow/reference/pf-command-reference.html#autocomplete) for more details. + +### Bugs Fixed +- Fix run name missing directory name in some scenario of `pf.run`. diff --git a/src/promptflow-devkit/promptflow/_cli/_pf/_flow.py b/src/promptflow-devkit/promptflow/_cli/_pf/_flow.py index b289bb4104b..10d7978b876 100644 --- a/src/promptflow-devkit/promptflow/_cli/_pf/_flow.py +++ b/src/promptflow-devkit/promptflow/_cli/_pf/_flow.py @@ -256,6 +256,8 @@ def add_parser_test_flow(subparsers): pf flow test --flow my-awesome-flow --node node_name # Chat in the flow: pf flow test --flow my-awesome-flow --node node_name --interactive +# Test a flow with init kwargs: +pf flow test --flow my-awesome-flow --init key1=value1 key2=value2 """ # noqa: E501 add_param_flow = lambda parser: parser.add_argument( # noqa: E731 "--flow", type=str, required=True, help="the flow directory to test." @@ -297,6 +299,7 @@ def add_parser_test_flow(subparsers): add_param_config, add_param_detail, add_param_skip_browser, + add_param_init, ] + base_params if Configuration.get_instance().is_internal_features_enabled(): @@ -531,6 +534,7 @@ def _test_flow_standard(args, pf_client, inputs, environment_variables): stream_output=False, dump_test_result=True, output_path=args.detail, + init=list_of_dict_to_dict(args.init), ) # Print flow/node test result if isinstance(result, dict): diff --git a/src/promptflow-devkit/promptflow/_cli/_pf/_service.py b/src/promptflow-devkit/promptflow/_cli/_pf/_service.py index 3a9d4c5b935..abd576dd21e 100644 --- a/src/promptflow-devkit/promptflow/_cli/_pf/_service.py +++ b/src/promptflow-devkit/promptflow/_cli/_pf/_service.py @@ -3,6 +3,7 @@ # --------------------------------------------------------- import argparse +import contextlib import logging import os import platform @@ -22,12 +23,14 @@ ) from promptflow._sdk._service.app import create_app from promptflow._sdk._service.utils.utils import ( + add_executable_script_to_env_path, check_pfs_service_status, dump_port_to_config, get_current_env_pfs_file, get_pfs_version, get_port_from_config, get_started_service_info, + hint_stop_before_upgrade, is_port_in_use, is_run_from_built_binary, kill_exist_service, @@ -155,37 +158,13 @@ def start_service(args): if args.debug: os.environ[PF_SERVICE_DEBUG] = "true" - def validate_port(port, force_start): - if is_port_in_use(port): - if force_start: - message = f"Force restart the service on the port {port}." - print(message) - logger.warning(message) - kill_exist_service(port) - else: - logger.warning(f"Service port {port} is used.") - raise UserErrorException(f"Service port {port} is used.") - if is_run_from_built_binary(): - # For msi installer, use vbs to start pfs in a hidden window. But it doesn't support redirect output in the - # hidden window to terminal. So we redirect output to a file. And then print the file content to terminal in - # pfs.bat. - old_stdout = sys.stdout - old_stderr = sys.stderr + # For msi installer/executable, use sdk api to start pfs since it's not supported to invoke waitress by cli + # directly after packaged by Pyinstaller. parent_dir = os.path.dirname(sys.executable) - sys.stdout = open(os.path.join(parent_dir, "output.txt"), "w") - sys.stderr = sys.stdout - if port: - dump_port_to_config(port) - validate_port(port, args.force) - else: - port = get_port_from_config(create_if_not_exists=True) - validate_port(port, args.force) - - if is_run_from_built_binary(): - # For msi installer, use sdk api to start pfs since it's not supported to invoke waitress by cli directly - # after packaged by Pyinstaller. - try: + output_path = os.path.join(parent_dir, "output.txt") + with redirect_stdout_to_file(output_path): + port = validate_port(port, args.force) global app if app is None: app, _ = create_app() @@ -193,77 +172,122 @@ def validate_port(port, force_start): app.logger.setLevel(logging.DEBUG) else: app.logger.setLevel(logging.INFO) - message = f"Start Prompt Flow Service on {port}, version: {get_pfs_version()}." + message = f"Starting Prompt Flow Service on {port}, version: {get_pfs_version()}." app.logger.info(message) print(message) sys.stdout.flush() - waitress.serve(app, host="127.0.0.1", port=port, threads=PF_SERVICE_WORKER_NUM) - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr + waitress.serve(app, host="127.0.0.1", port=port, threads=PF_SERVICE_WORKER_NUM) else: - # Add executable script dir to PATH to make sure the subprocess can find the executable, especially in notebook - # environment which won't add it to system path automatically. - python_dir = os.path.dirname(sys.executable) - executable_dir = os.path.join(python_dir, "Scripts") if platform.system() == "Windows" else python_dir - if executable_dir not in os.environ["PATH"].split(os.pathsep): - os.environ["PATH"] = executable_dir + os.pathsep + os.environ["PATH"] - + port = validate_port(port, args.force) + add_executable_script_to_env_path() # Start a pfs process using detach mode. It will start a new process and create a new app. So we use environment # variable to pass the debug mode, since it will inherit parent process environment variable. if platform.system() == "Windows": - try: - import win32api - import win32con - import win32process - except ImportError as ex: - raise UserErrorException( - f"Please install pywin32 by 'pip install pywin32' and retry. prompt flow " - f"service start depends on pywin32.. {ex}" - ) - command = ( - f"waitress-serve --listen=127.0.0.1:{port} --threads={PF_SERVICE_WORKER_NUM} " - "promptflow._cli._pf._service:get_app" - ) - startupinfo = win32process.STARTUPINFO() - startupinfo.dwFlags |= win32process.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = win32con.SW_HIDE - process_attributes = None - thread_attributes = None - inherit_handles = False - creation_flags = win32con.CREATE_NEW_PROCESS_GROUP | win32con.DETACHED_PROCESS - environment = None - current_directory = None - process_handle, thread_handle, process_id, thread_id = win32process.CreateProcess( - None, - command, - process_attributes, - thread_attributes, - inherit_handles, - creation_flags, - environment, - current_directory, - startupinfo, - ) - - win32api.CloseHandle(process_handle) - win32api.CloseHandle(thread_handle) + _start_background_service_on_windows(port) else: - # Set host to localhost, only allow request from localhost. - cmd = [ - "waitress-serve", - f"--listen=127.0.0.1:{port}", - f"--threads={PF_SERVICE_WORKER_NUM}", - "promptflow._cli._pf._service:get_app", - ] - subprocess.Popen(cmd, stdout=subprocess.DEVNULL, start_new_session=True) + _start_background_service_on_unix(port) is_healthy = check_pfs_service_status(port) if is_healthy: message = f"Start Promptflow Service on port {port}, version: {get_pfs_version()}." print(message) logger.info(message) else: - logger.warning(f"Promptflow service start failed in {port}.") + logger.warning(f"Promptflow service start failed in {port}. {hint_stop_before_upgrade}") + + +def validate_port(port, force_start): + if port: + dump_port_to_config(port) + _validate_port(port, force_start) + else: + port = get_port_from_config(create_if_not_exists=True) + _validate_port(port, force_start) + return port + + +def _validate_port(port, force_start): + if is_port_in_use(port): + if force_start: + message = f"Force restart the service on the port {port}." + if is_run_from_built_binary(): + print(message) + logger.warning(message) + kill_exist_service(port) + else: + message = f"Service port {port} is used." + if is_run_from_built_binary(): + print(message) + logger.warning(message) + raise UserErrorException(f"Service port {port} is used.") + + +@contextlib.contextmanager +def redirect_stdout_to_file(path): + # For msi installer, use vbs to start pfs in a hidden window. But it doesn't support redirect output in the + # hidden window to terminal. So we redirect output to a file. And then print the file content to terminal in + # pfs.bat. + old_stdout = sys.stdout + old_stderr = sys.stderr + try: + with open(path, "w") as file: + sys.stdout = file + sys.stderr = file + yield + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + +def _start_background_service_on_windows(port): + try: + import win32api + import win32con + import win32process + except ImportError as ex: + raise UserErrorException( + f"Please install pywin32 by 'pip install pywin32' and retry. prompt flow " + f"service start depends on pywin32.. {ex}" + ) + command = ( + f"waitress-serve --listen=127.0.0.1:{port} --threads={PF_SERVICE_WORKER_NUM} " + "promptflow._cli._pf._service:get_app" + ) + logger.debug(f"Start Promptflow Service in Windows: {command}") + startupinfo = win32process.STARTUPINFO() + startupinfo.dwFlags |= win32process.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = win32con.SW_HIDE + process_attributes = None + thread_attributes = None + inherit_handles = False + creation_flags = win32con.CREATE_NEW_PROCESS_GROUP | win32con.DETACHED_PROCESS + environment = None + current_directory = None + process_handle, thread_handle, process_id, thread_id = win32process.CreateProcess( + None, + command, + process_attributes, + thread_attributes, + inherit_handles, + creation_flags, + environment, + current_directory, + startupinfo, + ) + + win32api.CloseHandle(process_handle) + win32api.CloseHandle(thread_handle) + + +def _start_background_service_on_unix(port): + # Set host to localhost, only allow request from localhost. + cmd = [ + "waitress-serve", + f"--listen=127.0.0.1:{port}", + f"--threads={PF_SERVICE_WORKER_NUM}", + "promptflow._cli._pf._service:get_app", + ] + logger.debug(f"Start Promptflow Service in Unix: {cmd}") + subprocess.Popen(cmd, stdout=subprocess.DEVNULL, start_new_session=True) def stop_service(): @@ -285,9 +309,13 @@ def show_service(): else: log_file = get_current_env_pfs_file(PF_SERVICE_LOG_FILE) if status: - status.update({"log_file": log_file.as_posix()}) + extra_info = {"log_file": log_file.as_posix(), "version": get_pfs_version()} + status.update(extra_info) print(status) return else: - logger.warning(f"Promptflow service is not started. log_file: {log_file.as_posix()}") + logger.warning( + f"Promptflow service is not started. The promptflow service log is located at {log_file.as_posix()} " + f"and promptflow version is {get_pfs_version()}." + ) sys.exit(1) diff --git a/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py b/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py index 8b434e97aa1..21dd6408aa4 100644 --- a/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py +++ b/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py @@ -1,6 +1,7 @@ import os from promptflow._cli._params import add_param_yes, base_params +from promptflow._cli._pf._service import stop_service from promptflow._cli._utils import activate_action, get_cli_sdk_logger from promptflow._utils.utils import prompt_y_n from promptflow.exceptions import UserErrorException @@ -41,7 +42,7 @@ def upgrade_version(args): from promptflow._constants import _ENV_PF_INSTALLER, CLI_PACKAGE_NAME from promptflow._sdk._utils import get_promptflow_sdk_version - from promptflow._utils.version_hint_utils import get_latest_version + from promptflow._sdk._version_hint_utils import get_latest_version installer = os.getenv(_ENV_PF_INSTALLER) or "" installer = installer.upper() @@ -73,6 +74,8 @@ def upgrade_version(args): if not confirmation: logger.debug("Upgrade stopped by user") return + # try to stop the service before upgrade + stop_service() if installer == "MSI": _upgrade_on_windows(yes) diff --git a/src/promptflow-devkit/promptflow/_sdk/_configuration.py b/src/promptflow-devkit/promptflow/_sdk/_configuration.py index d2136116904..f33291adbca 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_configuration.py +++ b/src/promptflow-devkit/promptflow/_sdk/_configuration.py @@ -17,10 +17,11 @@ HOME_PROMPT_FLOW_DIR, SERVICE_CONFIG_FILE, ) +from promptflow._sdk._errors import MissingAzurePackage from promptflow._sdk._utils import call_from_extension, gen_uuid_by_compute_info, read_write_by_user from promptflow._utils.logger_utils import get_cli_sdk_logger from promptflow._utils.yaml_utils import dump_yaml, load_yaml -from promptflow.exceptions import ErrorTarget, UserErrorException, ValidationException +from promptflow.exceptions import ErrorTarget, ValidationException logger = get_cli_sdk_logger() @@ -223,15 +224,7 @@ def _validate(key: str, value: str) -> None: validate_trace_provider(value) except ImportError: - msg = ( - '"promptflow[azure]" is required to validate trace provider, ' - 'please install it by running "pip install promptflow[azure]" with your version.' - ) - raise UserErrorException( - message=msg, - target=ErrorTarget.CONTROL_PLANE_SDK, - no_personal_data_message=msg, - ) + raise MissingAzurePackage() return def get_user_agent(self) -> Optional[str]: diff --git a/src/promptflow-devkit/promptflow/_sdk/_errors.py b/src/promptflow-devkit/promptflow/_sdk/_errors.py index 716f8c516fe..3ce415be56a 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_errors.py +++ b/src/promptflow-devkit/promptflow/_sdk/_errors.py @@ -257,3 +257,17 @@ class LineRunNotFoundError(SDKError): """Exception raised if line run cannot be found.""" pass + + +class MissingAzurePackage(SDKError): + """Exception raised if missing required package.""" + + def __init__( + self, + **kwargs, + ): + msg = ( + '"promptflow[azure]" is required for this functionality, ' + 'please install it by running "pip install promptflow-azure" with your version.' + ) + super().__init__(message=msg, no_personal_data_message=msg, **kwargs) diff --git a/src/promptflow-devkit/promptflow/_sdk/_orchestrator/test_submitter.py b/src/promptflow-devkit/promptflow/_sdk/_orchestrator/test_submitter.py index 49a63a335f1..da4219a2b7d 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_orchestrator/test_submitter.py +++ b/src/promptflow-devkit/promptflow/_sdk/_orchestrator/test_submitter.py @@ -189,6 +189,7 @@ def init( output_path: Optional[str] = None, session: Optional[str] = None, stream_output: bool = True, + init_kwargs: Optional[dict] = None, ): """ Create/Occupy dependent resources to execute the test within the context. @@ -208,6 +209,8 @@ def init( :type session: str :param stream_output: whether to return a generator for streaming output. :type stream_output: bool + :param init_kwargs: Initialization parameters for flex flow, only supported when flow is callable class. + :type init: init_kwargs :return: TestSubmitter instance. :rtype: TestSubmitter """ @@ -275,6 +278,7 @@ def init( log_path=log_path, enable_stream_output=stream_output, language=self.flow.language, + init_kwargs=init_kwargs, ) try: @@ -389,6 +393,7 @@ def flow_test( inputs: Mapping[str, Any], allow_generator_output: bool = False, # TODO: remove this run_id: str = None, + init_kwargs: Optional[dict] = None, ) -> LineResult: """ Submit a flow test. @@ -406,6 +411,8 @@ def flow_test( :type stream_output: bool :param run_id: Run id will be set in operation context and used for session :type run_id: str + :param init_kwargs: Initialization parameters for flex flow, only supported when flow is callable class. + :type init_kwargs: dict """ self._raise_if_not_within_init_context() if self.target_node: @@ -426,6 +433,7 @@ def flow_test( entry=self.entry, storage=self._storage, run_id=run_id, + init_kwargs=init_kwargs, ) else: from promptflow._utils.multimedia_utils import BasicMultimediaProcessor diff --git a/src/promptflow-devkit/promptflow/_sdk/_pf_client.py b/src/promptflow-devkit/promptflow/_sdk/_pf_client.py index d1d7759f847..cf578f92521 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_pf_client.py +++ b/src/promptflow-devkit/promptflow/_sdk/_pf_client.py @@ -317,6 +317,7 @@ def test( variant: str = None, node: str = None, environment_variables: dict = None, + init: Optional[dict] = None, ) -> dict: """Test flow or node. @@ -334,6 +335,8 @@ def test( The value reference to connection keys will be resolved to the actual value, and all environment variables specified will be set into os.environ. :type environment_variables: dict + :param init: Initialization parameters for flex flow, only supported when flow is callable class. + :type init: dict :return: The result of flow or node :rtype: dict """ @@ -351,7 +354,7 @@ def test( inputs = load_data(local_path=inputs)[0] return self.flows.test( - flow=flow, inputs=inputs, variant=variant, environment_variables=environment_variables, node=node + flow=flow, inputs=inputs, variant=variant, environment_variables=environment_variables, node=node, init=init ) @property diff --git a/src/promptflow-devkit/promptflow/_sdk/_service/app.py b/src/promptflow-devkit/promptflow/_sdk/_service/app.py index 1ce1e37fc2d..2da464cf505 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_service/app.py +++ b/src/promptflow-devkit/promptflow/_sdk/_service/app.py @@ -8,7 +8,7 @@ from logging.handlers import RotatingFileHandler from pathlib import WindowsPath -from flask import Blueprint, Flask, current_app, g, jsonify, request +from flask import Blueprint, Flask, current_app, g, jsonify, redirect, request, url_for from flask_cors import CORS from werkzeug.exceptions import HTTPException @@ -19,6 +19,7 @@ PF_SERVICE_MONITOR_SECOND, CreatedByFieldName, ) +from promptflow._sdk._errors import MissingAzurePackage from promptflow._sdk._service import Api from promptflow._sdk._service.apis.collector import trace_collector from promptflow._sdk._service.apis.connection import api as connection_api @@ -49,6 +50,10 @@ def heartbeat(): return jsonify(response) +def root(): + return redirect(url_for("serve_trace_ui")) + + def create_app(): app = Flask(__name__) @@ -57,6 +62,7 @@ def create_app(): # as there might be different ports in that scenario CORS(app) + app.add_url_rule("/", view_func=root) app.add_url_rule("/heartbeat", view_func=heartbeat) app.add_url_rule( "/v1/traces", view_func=lambda: trace_collector(get_created_by_info_with_cache, app.logger), methods=["POST"] @@ -204,6 +210,8 @@ def get_created_by_info_with_cache(): CreatedByFieldName.NAME: decoded_token.get("name", decoded_token.get("appid", "")), } ) + except ImportError: + raise MissingAzurePackage() except Exception as e: # This function is only target to be used in Flask app. current_app.logger.error(f"Failed to get created_by info, stop writing span. Exception: {e}") diff --git a/src/promptflow-devkit/promptflow/_sdk/_service/utils/utils.py b/src/promptflow-devkit/promptflow/_sdk/_service/utils/utils.py index 44e7edfff1a..4bcc19befba 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_service/utils/utils.py +++ b/src/promptflow-devkit/promptflow/_sdk/_service/utils/utils.py @@ -24,6 +24,7 @@ from promptflow._sdk._constants import ( DEFAULT_ENCODING, HOME_PROMPT_FLOW_DIR, + PF_SERVICE_HOUR_TIMEOUT, PF_SERVICE_PORT_DIT_NAME, PF_SERVICE_PORT_FILE, ) @@ -36,6 +37,21 @@ logger = get_cli_sdk_logger() +hint_stop_message = ( + f"You can stop the prompt flow tracing server with the following command:'\033[1mpf service stop\033[0m'.\n" + f"Alternatively, if no requests are made within {PF_SERVICE_HOUR_TIMEOUT} " + f"hours, it will automatically stop." +) +hint_stop_before_upgrade = ( + "Kindly reminder: If you have previously upgraded the promptflow package , please " + "double-confirm that you have run '\033[1mpf service stop\033[0m' to stop the promptflow" + "service before proceeding with the upgrade. Otherwise, you may encounter unexpected " + "environmental issues or inconsistencies between the version of running promptflow service " + "and the local promptflow version. Alternatively, you can use the " + "'\033[1mpf upgrade\033[0m' command to proceed with the upgrade process for the promptflow " + "package." +) + def local_user_only(func): @wraps(func) @@ -69,16 +85,17 @@ def get_port_from_config(create_if_not_exists=False): port_file_path.touch(mode=read_write_by_user(), exist_ok=True) else: port_file_path = get_current_env_pfs_file(PF_SERVICE_PORT_FILE) - with open(port_file_path, "r", encoding=DEFAULT_ENCODING) as f: + with open(port_file_path, "r+", encoding=DEFAULT_ENCODING) as f: service_config = load_yaml(f) or {} port = service_config.get("service", {}).get("port", None) - if not port and create_if_not_exists: - with open(port_file_path, "w", encoding=DEFAULT_ENCODING) as f: - # Set random port to ~/.promptflow/pf.yaml + if not port and create_if_not_exists: port = get_random_port() service_config["service"] = service_config.get("service", {}) service_config["service"]["port"] = port + logger.debug(f"Set port {port} to file {port_file_path}") + f.seek(0) # Move the file pointer to the beginning of the file dump_yaml(service_config, f) + f.truncate() # Remove any remaining content return port @@ -89,12 +106,15 @@ def dump_port_to_config(port): else: # Set port to ~/.promptflow/pfs/**_pf.port, if already have a port in file , will overwrite it. port_file_path = get_current_env_pfs_file(PF_SERVICE_PORT_FILE) - with open(port_file_path, "r", encoding=DEFAULT_ENCODING) as f: + with open(port_file_path, "r+", encoding=DEFAULT_ENCODING) as f: service_config = load_yaml(f) or {} - with open(port_file_path, "w", encoding=DEFAULT_ENCODING) as f: service_config["service"] = service_config.get("service", {}) - service_config["service"]["port"] = port - dump_yaml(service_config, f) + if service_config["service"].get("port", None) != port: + service_config["service"]["port"] = port + logger.debug(f"Set port {port} to file {port_file_path}") + f.seek(0) # Move the file pointer to the beginning of the file + dump_yaml(service_config, f) + f.truncate() # Remove any remaining content def is_port_in_use(port: int): @@ -187,21 +207,23 @@ def is_pfs_service_healthy(pfs_port) -> bool: return is_healthy except Exception: # pylint: disable=broad-except pass - logger.debug( - f"Promptflow service can't be reached through port {pfs_port}, will try to (force) start promptflow service." - ) + logger.debug(f"Failed to call prompt flow service api /heartbeat on port {pfs_port}.") return False -def check_pfs_service_status(pfs_port, time_delay=1, count_threshold=20) -> bool: +def check_pfs_service_status(pfs_port, time_delay=1, count_threshold=10) -> bool: cnt = 1 time.sleep(time_delay) is_healthy = is_pfs_service_healthy(pfs_port) while is_healthy is False and count_threshold > cnt: - logger.info( - f"Promptflow service is not ready. It has been tried for {cnt} times, will try at most {count_threshold} " - f"times." + message = ( + f"Waiting for the Promptflow service status to become healthy... It has been tried for {cnt} times, will " + f"try at most {count_threshold} times." ) + if cnt >= 3: + logger.warning(message) + else: + logger.info(message) cnt += 1 time.sleep(time_delay) is_healthy = is_pfs_service_healthy(pfs_port) @@ -289,9 +311,21 @@ def is_run_from_built_binary(): Allow customer to use environment variable to control the triggering. """ - return (not sys.executable.endswith("python.exe") and not sys.executable.endswith("python")) or os.environ.get( - PF_RUN_AS_BUILT_BINARY, "" - ).lower() == "true" + return ( + sys.executable.endswith("pfcli.exe") + or sys.executable.endswith("app.exe") + or sys.executable.endswith("app") + or os.environ.get(PF_RUN_AS_BUILT_BINARY, "").lower() == "true" + ) + + +def add_executable_script_to_env_path(): + # Add executable script dir to PATH to make sure the subprocess can find the executable, especially in notebook + # environment which won't add it to system path automatically. + python_dir = os.path.dirname(sys.executable) + executable_dir = os.path.join(python_dir, "Scripts") if platform.system() == "Windows" else python_dir + if executable_dir not in os.environ["PATH"].split(os.pathsep): + os.environ["PATH"] = executable_dir + os.pathsep + os.environ["PATH"] def encrypt_flow_path(flow_path): diff --git a/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py b/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py index b21ee3f8811..928fc082606 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py +++ b/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py @@ -234,7 +234,7 @@ def monitor(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): from promptflow._sdk._telemetry.telemetry import get_telemetry_logger - from promptflow._utils.version_hint_utils import HINT_ACTIVITY_NAME, check_latest_version, hint_for_update + from promptflow._sdk._version_hint_utils import HINT_ACTIVITY_NAME, check_latest_version, hint_for_update logger = get_telemetry_logger() diff --git a/src/promptflow-devkit/promptflow/_sdk/_tracing.py b/src/promptflow-devkit/promptflow/_sdk/_tracing.py index ffabf3fa453..209d336244f 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_tracing.py +++ b/src/promptflow-devkit/promptflow/_sdk/_tracing.py @@ -26,14 +26,16 @@ ) from promptflow._sdk._configuration import Configuration from promptflow._sdk._constants import ( - PF_SERVICE_HOUR_TIMEOUT, PF_TRACE_CONTEXT, PF_TRACE_CONTEXT_ATTR, AzureMLWorkspaceTriad, ContextAttributeKey, ) from promptflow._sdk._service.utils.utils import ( + add_executable_script_to_env_path, get_port_from_config, + hint_stop_before_upgrade, + hint_stop_message, is_pfs_service_healthy, is_port_in_use, is_run_from_built_binary, @@ -81,33 +83,44 @@ def _invoke_pf_svc() -> str: if is_run_from_built_binary(): interpreter_path = os.path.abspath(sys.executable) pf_path = os.path.join(os.path.dirname(interpreter_path), "pf") - if platform.system() == "Windows": - cmd_args = [pf_path, "service", "start", "--port", port] - else: - cmd_args = f"{pf_path} service start --port {port}" + cmd_args = [pf_path, "service", "start", "--port", port] else: - if platform.system() == "Windows": - cmd_args = ["pf", "service", "start", "--port", port] - else: - cmd_args = f"pf service start --port {port}" - hint_stop_message = ( - f"You can stop the Prompt flow Tracing Server with the following command:'\033[1mpf service stop\033[0m'.\n" - f"Alternatively, if no requests are made within {PF_SERVICE_HOUR_TIMEOUT} " - f"hours, it will automatically stop." - ) + cmd_args = ["pf", "service", "start", "--port", port] + if is_port_in_use(int(port)): if not is_pfs_service_healthy(port): cmd_args.append("--force") + logger.debug("Prompt flow service is not healthy, force to start...") else: print("Prompt flow Tracing Server has started...") print(hint_stop_message) return port - print("Starting Prompt flow Tracing Server...") - start_pfs = subprocess.Popen(cmd_args, shell=True) - # Wait for service to be started - start_pfs.wait() - logger.debug("Prompt flow service is serving on port %s", port) - print(hint_stop_message) + + add_executable_script_to_env_path() + print("Starting prompt flow Tracing Server...") + start_pfs = None + try: + start_pfs = subprocess.Popen(cmd_args, shell=platform.system() == "Windows", stderr=subprocess.PIPE) + # Wait for service to be started + start_pfs.wait(timeout=20) + except subprocess.TimeoutExpired: + logger.warning( + f"The starting prompt flow process did not finish within the timeout period. {hint_stop_before_upgrade}" + ) + except Exception as e: + logger.warning(f"An error occurred when starting prompt flow process: {e}. {hint_stop_before_upgrade}") + + # Check if there were any errors + if start_pfs is not None and start_pfs.returncode is not None and start_pfs.returncode != 0: + error_message = start_pfs.stderr.read().decode() + message = f"The starting prompt flow process returned an error: {error_message}. " + logger.warning(message) + elif not is_pfs_service_healthy(port): + # this branch is to check if the service is healthy for msi installer + logger.warning(f"Prompt flow service is not healthy. {hint_stop_before_upgrade}") + else: + logger.debug("Prompt flow service is serving on port %s", port) + print(hint_stop_message) return port diff --git a/src/promptflow-devkit/promptflow/_sdk/_utils.py b/src/promptflow-devkit/promptflow/_sdk/_utils.py index 794e5c36f90..1b89bb31134 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_utils.py +++ b/src/promptflow-devkit/promptflow/_sdk/_utils.py @@ -1029,8 +1029,10 @@ def can_accept_kwargs(func): def callable_to_entry_string(callable_obj: Callable) -> str: """Convert callable object to entry string.""" - if not isfunction(callable_obj): - raise UserErrorException(f"{callable_obj} is not function, only function is supported.") + if not isfunction(callable_obj) and not hasattr(callable_obj, "__call__"): + raise UserErrorException( + f"{callable_obj} is not function or callable object, only function or callable object are supported." + ) try: module_str = callable_obj.__module__ diff --git a/src/promptflow-core/promptflow/_utils/version_hint_utils.py b/src/promptflow-devkit/promptflow/_sdk/_version_hint_utils.py similarity index 96% rename from src/promptflow-core/promptflow/_utils/version_hint_utils.py rename to src/promptflow-devkit/promptflow/_sdk/_version_hint_utils.py index 1b5dd232381..28448fc5a43 100644 --- a/src/promptflow-core/promptflow/_utils/version_hint_utils.py +++ b/src/promptflow-devkit/promptflow/_sdk/_version_hint_utils.py @@ -101,9 +101,9 @@ def hint_for_update(): if last_hint_time is None or ( datetime.datetime.now() > last_hint_time + datetime.timedelta(days=HINT_INTERVAL_DAY) ): - from promptflow._sdk._utils import get_promptflow_sdk_version + from promptflow._sdk._utils import get_promptflow_devkit_version - cached_versions[CURRENT_VERSION] = get_promptflow_sdk_version() + cached_versions[CURRENT_VERSION] = get_promptflow_devkit_version() if LATEST_VERSION in cached_versions: from packaging.version import parse diff --git a/src/promptflow-devkit/promptflow/_sdk/entities/_flows/_flow_context_resolver.py b/src/promptflow-devkit/promptflow/_sdk/entities/_flows/_flow_context_resolver.py index fa67a1adbf9..59e347fbb6a 100644 --- a/src/promptflow-devkit/promptflow/_sdk/entities/_flows/_flow_context_resolver.py +++ b/src/promptflow-devkit/promptflow/_sdk/entities/_flows/_flow_context_resolver.py @@ -1,6 +1,7 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import copy import tempfile from functools import lru_cache from os import PathLike @@ -82,7 +83,7 @@ def _resolve_connections(self, flow_context: FlowContext) -> "FlowContextResolve overwrite_connections( flow_dag=self.flow_dag, - connections=flow_context.connections, + connections=copy.deepcopy(flow_context.connections), working_dir=self.working_dir, ) return self diff --git a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py index 76415c3e003..28176ec6e9e 100644 --- a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py +++ b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py @@ -39,7 +39,7 @@ RunStatus, RunTypes, ) -from promptflow._sdk._errors import InvalidRunError, InvalidRunStatusError +from promptflow._sdk._errors import InvalidRunError, InvalidRunStatusError, MissingAzurePackage from promptflow._sdk._orm import RunInfo as ORMRun from promptflow._sdk._utils import ( _sanitize_python_variable_name, @@ -515,7 +515,7 @@ def _format_display_name(self) -> str: def _get_flow_dir(self) -> Path: if not self._use_remote_flow: - flow = Path(self.flow) + flow = Path(str(self.flow)).resolve().absolute() if flow.is_dir(): return flow return flow.parent @@ -526,15 +526,18 @@ def _get_schema_cls(self): return RunSchema def _to_rest_object(self): - from azure.ai.ml._utils._storage_utils import AzureMLDatastorePathUri - - from promptflow.azure._restclient.flow.models import ( - BatchDataInput, - CreateExistingBulkRunRequest, - RunDisplayNameGenerationType, - SessionSetupModeEnum, - SubmitBulkRunRequest, - ) + try: + from azure.ai.ml._utils._storage_utils import AzureMLDatastorePathUri + + from promptflow.azure._restclient.flow.models import ( + BatchDataInput, + CreateExistingBulkRunRequest, + RunDisplayNameGenerationType, + SessionSetupModeEnum, + SubmitBulkRunRequest, + ) + except ImportError: + raise MissingAzurePackage() if self.run is not None: if isinstance(self.run, Run): diff --git a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py index 87d220a7863..daa55132863 100644 --- a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py +++ b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py @@ -15,7 +15,7 @@ from importlib.metadata import version from os import PathLike from pathlib import Path -from typing import Callable, Dict, Iterable, List, NoReturn, Tuple, Union +from typing import Callable, Dict, Iterable, List, NoReturn, Optional, Tuple, Union import pydash @@ -29,7 +29,6 @@ FLOW_TOOLS_JSON_GEN_TIMEOUT, LOCAL_MGMT_DB_PATH, SERVE_SAMPLE_JSON_PATH, - SignatureValueType, ) from promptflow._sdk._load_functions import load_flow from promptflow._sdk._orchestrator import TestSubmitter @@ -55,7 +54,6 @@ parse_variant, ) from promptflow._utils.yaml_utils import dump_yaml, load_yaml -from promptflow.contracts.tool import ValueType from promptflow.exceptions import ErrorTarget, UserErrorException @@ -75,7 +73,7 @@ def test( variant: str = None, node: str = None, environment_variables: dict = None, - entry: str = None, + init: Optional[dict] = None, **kwargs, ) -> dict: """Test flow or node. @@ -94,6 +92,8 @@ def test( The value reference to connection keys will be resolved to the actual value, and all environment variables specified will be set into os.environ. :type environment_variables: dict + :param init: Initialization parameters for flex flow, only supported when flow is callable class. + :type init: dict :return: The result of flow or node :rtype: dict """ @@ -111,6 +111,7 @@ def test( inputs=inputs, environment_variables=environment_variables, experiment=experiment, + init=init, **kwargs, ) elif is_prompty_flow(flow): @@ -126,6 +127,7 @@ def test( variant=variant, node=node, environment_variables=environment_variables, + init=init, **kwargs, ) dump_test_result = kwargs.get("dump_test_result", False) @@ -237,6 +239,7 @@ def _test( stream_log: bool = True, stream_output: bool = True, allow_generator_output: bool = True, + init: Optional[dict] = None, **kwargs, ): """Test flow or node. @@ -253,6 +256,7 @@ def _test( :param stream_log: Whether streaming the log. :param stream_output: Whether streaming the outputs. :param allow_generator_output: Whether return streaming output when flow has streaming output. + :param init: Initialization parameters for flex flow, only supported when flow is callable class. :return: Executor result """ inputs = inputs or {} @@ -275,6 +279,7 @@ def _test( output_path=output_path, stream_output=stream_output, session=session, + init_kwargs=init, ) as submitter: if isinstance(flow, FlexFlow) or isinstance(flow, Prompty): # TODO(2897153): support chat eager flow @@ -297,6 +302,7 @@ def _test( inputs=flow_inputs, allow_generator_output=allow_generator_output and is_chat_flow, run_id=run_id, + init_kwargs=init, ) @monitor_operation(activity_name="pf.flows._chat", activity_type=ActivityType.INTERNALCALL) @@ -1063,11 +1069,12 @@ def _infer_signature( raise UserErrorException("Entry must be a function or a class.") # signature is language irrelevant, so we apply json type system + # TODO: enable this mapping after service supports more types value_type_map = { - ValueType.INT.value: SignatureValueType.INT.value, - ValueType.DOUBLE.value: SignatureValueType.NUMBER.value, - ValueType.LIST.value: SignatureValueType.ARRAY.value, - ValueType.BOOL.value: SignatureValueType.BOOL.value, + # ValueType.INT.value: SignatureValueType.INT.value, + # ValueType.DOUBLE.value: SignatureValueType.NUMBER.value, + # ValueType.LIST.value: SignatureValueType.ARRAY.value, + # ValueType.BOOL.value: SignatureValueType.BOOL.value, } for port_type in ["inputs", "outputs", "init"]: if port_type not in flow_meta: diff --git a/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py b/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py index e23d66f9534..bd76ce8435c 100644 --- a/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py +++ b/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py @@ -5,6 +5,7 @@ from typing import List from promptflow._sdk._constants import MAX_LIST_CLI_RESULTS +from promptflow._sdk._errors import MissingAzurePackage from promptflow._sdk._telemetry import ActivityType, WorkspaceTelemetryMixin, monitor_operation from promptflow._sdk._utils import print_red_error from promptflow._sdk.entities._connection import _Connection @@ -37,7 +38,10 @@ def __init__(self, connection_provider, **kwargs): @property def _client(self): if self._pfazure_client is None: - from promptflow.azure._pf_client import PFClient as PFAzureClient + try: + from promptflow.azure._pf_client import PFClient as PFAzureClient + except ImportError: + raise MissingAzurePackage() self._pfazure_client = PFAzureClient( # TODO: disable interactive credential when starting as a service @@ -51,9 +55,12 @@ def _client(self): @classmethod def _get_credential(cls): - from azure.identity import DefaultAzureCredential, DeviceCodeCredential + try: + from azure.identity import DefaultAzureCredential, DeviceCodeCredential - from promptflow.azure._utils.general import get_arm_token + from promptflow.azure._utils.general import get_arm_token + except ImportError: + raise MissingAzurePackage() if is_from_cli(): try: @@ -105,7 +112,10 @@ def get(self, name: str, **kwargs) -> _Connection: if with_secrets: # Do not use pfazure_client here as it requires workspace read permission # Get secrets from arm only requires workspace listsecrets permission - from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + try: + from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + except ImportError: + raise MissingAzurePackage() return ArmConnectionOperations._direct_get( name, self._subscription_id, self._resource_group, self._workspace_name, self._credential diff --git a/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py b/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py index a50094264fc..c64cf2cc3a3 100644 --- a/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py +++ b/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py @@ -6,9 +6,10 @@ from promptflow._constants import LANGUAGE_KEY, ConnectionType, FlowLanguage from promptflow._proxy import ProxyFactory -from promptflow._sdk._constants import FlowType, SignatureValueType +from promptflow._sdk._constants import FlowType from promptflow._sdk.schemas._base import PatchedSchemaMeta, YamlFileSchema from promptflow._sdk.schemas._fields import NestedField +from promptflow.contracts.tool import ValueType class FlowInputSchema(metaclass=PatchedSchemaMeta): @@ -60,11 +61,21 @@ class FlowSchema(BaseFlowSchema): node_variants = fields.Dict(keys=fields.Str(), values=fields.Dict()) +ALLOWED_TYPES = [ + ValueType.STRING.value, + ValueType.INT.value, + ValueType.DOUBLE.value, + ValueType.BOOL.value, + ValueType.LIST.value, + ValueType.OBJECT.value, +] + + class FlexFlowInputSchema(FlowInputSchema): type = fields.Str( required=True, # TODO 3062609: Flex flow GPT-V support - validate=validate.OneOf(list(map(lambda x: x.value, SignatureValueType))), + validate=validate.OneOf(ALLOWED_TYPES), ) @@ -72,7 +83,7 @@ class FlexFlowInitSchema(FlowInputSchema): type = fields.Str( required=True, validate=validate.OneOf( - list(map(lambda x: x.value, SignatureValueType)) + ALLOWED_TYPES + list( map(lambda x: f"{x.value}Connection", filter(lambda x: x != ConnectionType._NOT_SET, ConnectionType)) ) @@ -83,7 +94,7 @@ class FlexFlowInitSchema(FlowInputSchema): class FlexFlowOutputSchema(FlowOutputSchema): type = fields.Str( required=True, - validate=validate.OneOf(list(map(lambda x: x.value, SignatureValueType))), + validate=validate.OneOf(ALLOWED_TYPES), ) diff --git a/src/promptflow-devkit/pyproject.toml b/src/promptflow-devkit/pyproject.toml index 96eb01e92c2..24aa28bb585 100644 --- a/src/promptflow-devkit/pyproject.toml +++ b/src/promptflow-devkit/pyproject.toml @@ -96,6 +96,7 @@ mock = "*" ipykernel = ">=6.27.1" papermill = ">=2.5.0" keyrings-alt = "*" +bs4 = "*" [build-system] requires = ["poetry-core>=1.5.0"] @@ -107,6 +108,7 @@ pf = "promptflow._cli.pf:main" [tool.pytest.ini_options] markers = [ "unittest", + "e2etest" ] # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action) # durations - list the slowest test durations @@ -123,8 +125,16 @@ addopts = """ testpaths = ["tests"] [tool.coverage.run] +source = [ + "*/promptflow/_cli/**", + "*/promptflow/_orchestrator/**", + "*/promptflow/_proxy/**", + "*/promptflow/_sdk/**", + "*/promptflow/batch/**", +] omit = [ - "__init__.py", + "*/__init__.py", + "*/promptflow/_sdk/_serving/*", ] [tool.black] diff --git a/src/promptflow-devkit/tests/_constants.py b/src/promptflow-devkit/tests/_constants.py new file mode 100644 index 00000000000..9929a01ff68 --- /dev/null +++ b/src/promptflow-devkit/tests/_constants.py @@ -0,0 +1,14 @@ +from pathlib import Path + +PROMPTFLOW_ROOT = Path(__file__).parent.parent.parent / "promptflow" +RUNTIME_TEST_CONFIGS_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/runtime") +CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() +ENV_FILE = (PROMPTFLOW_ROOT / ".env").resolve().absolute().as_posix() + +# below constants are used for pfazure and global config tests +DEFAULT_SUBSCRIPTION_ID = "96aede12-2f73-41cb-b983-6d11a904839b" +DEFAULT_RESOURCE_GROUP_NAME = "promptflow" +DEFAULT_WORKSPACE_NAME = "promptflow-eastus2euap" +DEFAULT_COMPUTE_INSTANCE_NAME = "ci-lin-cpu-sp" +DEFAULT_RUNTIME_NAME = "test-runtime-ci" +DEFAULT_REGISTRY_NAME = "promptflow-preview" diff --git a/src/promptflow-devkit/tests/conftest.py b/src/promptflow-devkit/tests/conftest.py new file mode 100644 index 00000000000..7b7d87f26f8 --- /dev/null +++ b/src/promptflow-devkit/tests/conftest.py @@ -0,0 +1,237 @@ +import importlib +import json +import os +import tempfile +from multiprocessing import Lock +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from _constants import ( + CONNECTION_FILE, + DEFAULT_RESOURCE_GROUP_NAME, + DEFAULT_SUBSCRIPTION_ID, + DEFAULT_WORKSPACE_NAME, + ENV_FILE, + PROMPTFLOW_ROOT, +) +from _pytest.monkeypatch import MonkeyPatch +from dotenv import load_dotenv +from filelock import FileLock +from pytest_mock import MockerFixture + +from promptflow._constants import PROMPTFLOW_CONNECTIONS +from promptflow._core.connection_manager import ConnectionManager +from promptflow._sdk.entities._connection import AzureOpenAIConnection +from promptflow._utils.context_utils import _change_working_dir + +load_dotenv() + + +@pytest.fixture(scope="session", autouse=True) +def modify_work_directory(): + os.chdir(PROMPTFLOW_ROOT) + + +@pytest.fixture(autouse=True, scope="session") +def mock_build_info(): + """Mock BUILD_INFO environment variable in pytest. + + BUILD_INFO is set as environment variable in docker image, but not in local test. + So we need to mock it in test senario. Rule - build_number is set as + ci- in CI pipeline, and set as local in local dev test.""" + if "BUILD_INFO" not in os.environ: + m = MonkeyPatch() + build_number = os.environ.get("BUILD_BUILDNUMBER", "") + buid_info = {"build_number": f"ci-{build_number}" if build_number else "local-pytest"} + m.setenv("BUILD_INFO", json.dumps(buid_info)) + yield m + + +@pytest.fixture +def dev_connections() -> dict: + with open(CONNECTION_FILE, "r") as f: + return json.load(f) + + +@pytest.fixture +def use_secrets_config_file(mocker: MockerFixture): + mocker.patch.dict(os.environ, {PROMPTFLOW_CONNECTIONS: CONNECTION_FILE}) + + +@pytest.fixture +def env_with_secrets_config_file(): + _lock = Lock() + with _lock: + with open(ENV_FILE, "w") as f: + f.write(f"{PROMPTFLOW_CONNECTIONS}={CONNECTION_FILE}\n") + yield ENV_FILE + if os.path.exists(ENV_FILE): + os.remove(ENV_FILE) + + +@pytest.fixture +def azure_open_ai_connection() -> AzureOpenAIConnection: + return ConnectionManager().get("azure_open_ai_connection") + + +@pytest.fixture +def temp_output_dir() -> str: + with tempfile.TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def prepare_symbolic_flow() -> str: + flows_dir = PROMPTFLOW_ROOT / "tests" / "test_configs" / "flows" + target_folder = flows_dir / "web_classification_with_symbolic" + source_folder = flows_dir / "web_classification" + + with _change_working_dir(target_folder): + + for file_name in os.listdir(source_folder): + if not Path(file_name).exists(): + os.symlink(source_folder / file_name, file_name) + return target_folder + + +@pytest.fixture(scope="session") +def install_custom_tool_pkg(): + # The tests could be running in parallel. Use a lock to prevent race conditions. + lock = FileLock("custom_tool_pkg_installation.lock") + with lock: + try: + import my_tool_package # noqa: F401 + + except ImportError: + import subprocess + import sys + + subprocess.check_call([sys.executable, "-m", "pip", "install", "test-custom-tools==0.0.2"]) + + +@pytest.fixture(scope="session") +def mock_list_func(): + """Mock function object for dynamic list testing.""" + + def my_list_func(prefix: str = "", size: int = 10, **kwargs): + return [ + { + "value": "fig0", + "display_value": "My_fig0", + "hyperlink": "https://www.bing.com/search?q=fig0", + "description": "this is 0 item", + }, + { + "value": "kiwi1", + "display_value": "My_kiwi1", + "hyperlink": "https://www.bing.com/search?q=kiwi1", + "description": "this is 1 item", + }, + ] + + return my_list_func + + +@pytest.fixture(scope="session") +def mock_module_with_list_func(mock_list_func): + """Mock module object for dynamic list testing.""" + mock_module = MagicMock() + mock_module.my_list_func = mock_list_func + mock_module.my_field = 1 + original_import_module = importlib.import_module # Save this to prevent recursion + + with patch.object(importlib, "import_module") as mock_import: + + def side_effect(module_name, *args, **kwargs): + if module_name == "my_tool_package.tools.tool_with_dynamic_list_input": + return mock_module + else: + return original_import_module(module_name, *args, **kwargs) + + mock_import.side_effect = side_effect + yield + + +@pytest.fixture(scope="session") +def mock_generated_by_func(): + """Mock function object for generated_by testing.""" + + def my_generated_by_func(index_type: str): + inputs = "" + if index_type == "Azure Cognitive Search": + inputs = {"index_type": index_type, "index": "index_1"} + elif index_type == "Workspace MLIndex": + inputs = {"index_type": index_type, "index": "index_2"} + + result = json.dumps(inputs) + return result + + return my_generated_by_func + + +@pytest.fixture(scope="session") +def mock_reverse_generated_by_func(): + """Mock function object for reverse_generated_by testing.""" + + def my_reverse_generated_by_func(index_json: str): + result = json.loads(index_json) + return result + + return my_reverse_generated_by_func + + +@pytest.fixture +def enable_logger_propagate(): + """This is for test cases that need to check the log output.""" + from promptflow._utils.logger_utils import get_cli_sdk_logger + + logger = get_cli_sdk_logger() + original_value = logger.propagate + logger.propagate = True + yield + logger.propagate = original_value + + +@pytest.fixture(scope="session") +def mock_module_with_for_retrieve_tool_func_result( + mock_list_func, mock_generated_by_func, mock_reverse_generated_by_func +): + """Mock module object for dynamic list testing.""" + mock_module_list_func = MagicMock() + mock_module_list_func.my_list_func = mock_list_func + mock_module_list_func.my_field = 1 + mock_module_generated_by = MagicMock() + mock_module_generated_by.generated_by_func = mock_generated_by_func + mock_module_generated_by.reverse_generated_by_func = mock_reverse_generated_by_func + mock_module_generated_by.my_field = 1 + original_import_module = importlib.import_module # Save this to prevent recursion + + with patch.object(importlib, "import_module") as mock_import: + + def side_effect(module_name, *args, **kwargs): + if module_name == "my_tool_package.tools.tool_with_dynamic_list_input": + return mock_module_list_func + elif module_name == "my_tool_package.tools.tool_with_generated_by_input": + return mock_module_generated_by + else: + return original_import_module(module_name, *args, **kwargs) + + mock_import.side_effect = side_effect + yield + + +# region pfazure constants +@pytest.fixture +def subscription_id() -> str: + return os.getenv("PROMPT_FLOW_SUBSCRIPTION_ID", DEFAULT_SUBSCRIPTION_ID) + + +@pytest.fixture +def resource_group_name() -> str: + return os.getenv("PROMPT_FLOW_RESOURCE_GROUP_NAME", DEFAULT_RESOURCE_GROUP_NAME) + + +@pytest.fixture +def workspace_name() -> str: + return os.getenv("PROMPT_FLOW_WORKSPACE_NAME", DEFAULT_WORKSPACE_NAME) diff --git a/src/promptflow/tests/sdk_cli_global_config_test/__init__.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_global_config_test/__init__.py rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/__init__.py diff --git a/src/promptflow/tests/sdk_cli_global_config_test/conftest.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/conftest.py similarity index 97% rename from src/promptflow/tests/sdk_cli_global_config_test/conftest.py rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/conftest.py index 672e866793f..736f1852322 100644 --- a/src/promptflow/tests/sdk_cli_global_config_test/conftest.py +++ b/src/promptflow-devkit/tests/sdk_cli_global_config_test/conftest.py @@ -6,8 +6,8 @@ import pytest from _constants import DEFAULT_RESOURCE_GROUP_NAME, DEFAULT_SUBSCRIPTION_ID, DEFAULT_WORKSPACE_NAME -from promptflow import PFClient from promptflow._sdk._configuration import Configuration +from promptflow.client import PFClient AZUREML_RESOURCE_PROVIDER = "Microsoft.MachineLearningServices" RESOURCE_ID_FORMAT = "/subscriptions/{}/resourceGroups/{}/providers/{}/workspaces/{}" diff --git a/src/promptflow/tests/sdk_cli_global_config_test/e2etests/__init__.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_global_config_test/e2etests/__init__.py rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/__init__.py diff --git a/src/promptflow/tests/sdk_cli_global_config_test/e2etests/test_global_config.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py similarity index 91% rename from src/promptflow/tests/sdk_cli_global_config_test/e2etests/test_global_config.py rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py index 30391b1fdb1..b7553962944 100644 --- a/src/promptflow/tests/sdk_cli_global_config_test/e2etests/test_global_config.py +++ b/src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py @@ -1,14 +1,13 @@ -from pathlib import Path - import mock import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._load_functions import load_flow from promptflow._sdk.entities._flows._flow_context_resolver import FlowContextResolver from promptflow.core._connection_provider._workspace_connection_provider import WorkspaceConnectionProvider -FLOWS_DIR = Path(__file__).parent.parent.parent / "test_configs" / "flows" -DATAS_DIR = Path(__file__).parent.parent.parent / "test_configs" / "datas" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests" / "test_configs" / "flows" +DATAS_DIR = PROMPTFLOW_ROOT / "tests" / "test_configs" / "datas" @pytest.mark.usefixtures("global_config") diff --git a/src/promptflow/tests/sdk_cli_test/.coveragerc b/src/promptflow-devkit/tests/sdk_cli_test/.coveragerc similarity index 100% rename from src/promptflow/tests/sdk_cli_test/.coveragerc rename to src/promptflow-devkit/tests/sdk_cli_test/.coveragerc diff --git a/src/promptflow/tests/sdk_cli_test/__init__.py b/src/promptflow-devkit/tests/sdk_cli_test/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/__init__.py rename to src/promptflow-devkit/tests/sdk_cli_test/__init__.py diff --git a/src/promptflow/tests/sdk_cli_test/conftest.py b/src/promptflow-devkit/tests/sdk_cli_test/conftest.py similarity index 96% rename from src/promptflow/tests/sdk_cli_test/conftest.py rename to src/promptflow-devkit/tests/sdk_cli_test/conftest.py index 19a61ceffd9..3c536329b93 100644 --- a/src/promptflow/tests/sdk_cli_test/conftest.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/conftest.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest +from _constants import CONNECTION_FILE, PROMPTFLOW_ROOT from mock import mock from pytest_mock import MockerFixture from sqlalchemy import create_engine @@ -41,14 +42,10 @@ def is_replay(): return False -PROMOTFLOW_ROOT = Path(__file__) / "../../.." -RUNTIME_TEST_CONFIGS_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/runtime") -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() -MODEL_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/flows") -EAGER_FLOW_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/eager_flows") +EAGER_FLOW_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/eager_flows") +MODEL_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/flows") -SRC_ROOT = PROMOTFLOW_ROOT / ".." -RECORDINGS_TEST_CONFIGS_ROOT = Path(SRC_ROOT / "promptflow-recording/recordings/local").resolve() +RECORDINGS_TEST_CONFIGS_ROOT = Path(PROMPTFLOW_ROOT / "../promptflow-recording/recordings/local").resolve() def pytest_configure(): diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/__init__.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/e2etests/__init__.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/__init__.py diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_chat_group.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_chat_group.py similarity index 90% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_chat_group.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_chat_group.py index 5af6973d288..34b6dd3fac1 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_chat_group.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_chat_group.py @@ -1,14 +1,10 @@ -from pathlib import Path - import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk.entities._chat_group._chat_group import ChatGroup from promptflow._sdk.entities._chat_group._chat_role import ChatRole -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -FLOWS_DIR = TEST_ROOT / "test_configs/flows" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" @pytest.mark.sdk_test diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py similarity index 98% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py index ba63383cb8e..c58db8bc851 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py @@ -17,6 +17,7 @@ import mock import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._cli._pf.entry import main from promptflow._constants import FLOW_FLEX_YAML, LINE_NUMBER_KEY, PF_USER_AGENT @@ -30,14 +31,14 @@ from promptflow._utils.yaml_utils import dump_yaml, load_yaml from promptflow.tracing._operation_context import OperationContext -FLOWS_DIR = "./tests/test_configs/flows" -EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows" -EXPERIMENT_DIR = "./tests/test_configs/experiments" -RUNS_DIR = "./tests/test_configs/runs" -CONNECTIONS_DIR = "./tests/test_configs/connections" -DATAS_DIR = "./tests/test_configs/datas" -TOOL_ROOT = "./tests/test_configs/tools" -PROMPTY_DIR = "./tests/test_configs/prompty" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" +EXPERIMENT_DIR = PROMPTFLOW_ROOT / "tests/test_configs/experiments" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" +CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" +TOOL_ROOT = PROMPTFLOW_ROOT / "tests/test_configs/tools" +PROMPTY_DIR = PROMPTFLOW_ROOT / "tests/test_configs/prompty" TARGET_URL = "https://www.youtube.com/watch?v=o5ZQyXaAv1g" @@ -1536,6 +1537,7 @@ def test_tool_init(self, capsys): outerr = capsys.readouterr() assert "Cannot find the icon path" in outerr.out + @pytest.mark.skip("Enable after promptflow-tool depend on core") def test_list_tool_cache(self, caplog, mocker): with tempfile.TemporaryDirectory() as temp_dir: package_name = "mock_tool_package_name" @@ -1565,6 +1567,7 @@ def test_list_tool_cache(self, caplog, mocker): assert "List tools meta from cache file" in caplog.text assert f"{package_name}.{func_name}.{func_name}" in tools_meta + @pytest.mark.skip("Enable after promptflow-tool depend on core") def test_tool_list(self, capsys): # List package tools in environment run_pf_command("tool", "list") @@ -1825,8 +1828,8 @@ def assert_flow_test(*args, **kwargs): def test_run_create_with_existing_run_folder(self): run_name = "web_classification_variant_0_20231205_120253_104100" # clean the run if exists - from promptflow import PFClient from promptflow._cli._utils import _try_delete_existing_run_record + from promptflow.client import PFClient pf = PFClient() _try_delete_existing_run_record(run_name) @@ -2362,6 +2365,21 @@ def test_pf_flow_save(self, pf): new_content = load_yaml(Path(temp_dir) / FLOW_FLEX_YAML) assert new_content == content + def test_flow_test_with_init(self, pf, capsys): + run_pf_command( + "flow", + "test", + "--flow", + f"{EAGER_FLOWS_DIR}/basic_callable_class", + "--inputs", + "func_input=input", + "--init", + "obj_input=val", + ) + stdout, _ = capsys.readouterr() + assert "obj_input" in stdout + assert "func_input" in stdout + def assert_batch_run_result(run, pf, assert_func): assert run.status == "Completed" diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli_perf.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli_perf.py similarity index 96% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_cli_perf.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli_perf.py index 9077a28897e..c57330b0dff 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli_perf.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli_perf.py @@ -10,14 +10,15 @@ from unittest import mock import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._cli._user_agent import USER_AGENT as CLI_USER_AGENT # noqa: E402 from promptflow._sdk._telemetry import log_activity from promptflow._utils.user_agent_utils import ClientUserAgentUtil -FLOWS_DIR = "./tests/test_configs/flows" -CONNECTIONS_DIR = "./tests/test_configs/connections" -DATAS_DIR = "./tests/test_configs/datas" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" def mock_log_activity(*args, **kwargs): diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_connection.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_connection.py similarity index 98% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_connection.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_connection.py index 14e4655bfaa..01b907a41ff 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_connection.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_connection.py @@ -1,9 +1,9 @@ import os import uuid -from pathlib import Path import pydash import pytest +from _constants import PROMPTFLOW_ROOT from mock import mock from promptflow._constants import ConnectionDefaultApiVersion @@ -12,10 +12,9 @@ from promptflow._sdk._pf_client import PFClient from promptflow._sdk.entities import AzureOpenAIConnection, CustomConnection, OpenAIConnection -_client = PFClient() - -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" CONNECTION_ROOT = TEST_ROOT / "test_configs/connections" +_client = PFClient() @pytest.mark.cli_test diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_csharp_cli.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_csharp_cli.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py similarity index 98% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py index ad1fe7812da..ea485d20f4e 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py @@ -1,8 +1,8 @@ import uuid -from pathlib import Path import pydash import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._constants import SCRUBBED_VALUE, CustomStrongTypeConnectionConfigs from promptflow._sdk._pf_client import PFClient @@ -17,7 +17,8 @@ class MyCustomConnection(CustomStrongTypeConnection): _client = PFClient() -TEST_ROOT = Path(__file__).parent.parent.parent + +TEST_ROOT = PROMPTFLOW_ROOT / "tests" CONNECTION_ROOT = TEST_ROOT / "test_configs/connections" diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_executable.py similarity index 88% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_executable.py index e10597382e7..eff466fa7b9 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_executable.py @@ -5,13 +5,14 @@ import mock import pytest +from _constants import PROMPTFLOW_ROOT from .test_cli import run_pf_command -FLOWS_DIR = "./tests/test_configs/flows" -RUNS_DIR = "./tests/test_configs/runs" -CONNECTIONS_DIR = "./tests/test_configs/connections" -DATAS_DIR = "./tests/test_configs/datas" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" +CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" @pytest.mark.usefixtures("use_secrets_config_file", "setup_local_connection", "install_custom_tool_pkg") diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_experiment.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_experiment.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_experiment.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_experiment.py index cf7a4513084..2f3c3fee50b 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_experiment.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_experiment.py @@ -9,6 +9,7 @@ from time import sleep import pytest +from _constants import PROMPTFLOW_ROOT from mock import mock from ruamel.yaml import YAML @@ -19,7 +20,7 @@ from promptflow._sdk._pf_client import PFClient from promptflow._sdk.entities._experiment import CommandNode, Experiment, ExperimentTemplate, FlowNode -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" EXP_ROOT = TEST_ROOT / "test_configs/experiments" FLOW_ROOT = TEST_ROOT / "test_configs/flows" EAGER_FLOW_ROOT = TEST_ROOT / "test_configs/eager_flows" diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_as_func.py similarity index 94% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_as_func.py index 53a1c71ee16..1607b6f7b4f 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_as_func.py @@ -10,18 +10,19 @@ import mock import pytest +from _constants import PROMPTFLOW_ROOT -from promptflow import load_flow from promptflow._sdk._errors import ConnectionNotFoundError, InvalidFlowError from promptflow._sdk.entities import CustomConnection from promptflow._sdk.entities._flows._flow_context_resolver import FlowContextResolver from promptflow._utils.flow_utils import dump_flow_dag, load_flow_dag +from promptflow.client import load_flow from promptflow.entities import FlowContext from promptflow.exceptions import UserErrorException -FLOWS_DIR = "./tests/test_configs/flows" -RUNS_DIR = "./tests/test_configs/runs" -DATAS_DIR = "./tests/test_configs/datas" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" @pytest.mark.usefixtures( @@ -326,3 +327,17 @@ def test_flow_with_default_variant(self, azure_open_ai_connection): ) # function can successfully run with connection override f(url="https://www.youtube.com/watch?v=o5ZQyXaAv1g") + + def test_flow_with_connection_override(self, azure_open_ai_connection): + f = load_flow(f"{FLOWS_DIR}/llm_tool_non_existing_connection") + with pytest.raises(ConnectionNotFoundError): + f(joke="joke") + f.context = FlowContext( + connections={ + "joke": {"connection": azure_open_ai_connection}, + } + ) + # function can successfully run with connection override + f(topic="joke") + # This should work on subsequent call not just first + f(topic="joke") diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_local_operations.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_local_operations.py similarity index 98% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_local_operations.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_local_operations.py index 9c789645cd9..6fcaec00308 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_local_operations.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_local_operations.py @@ -7,6 +7,7 @@ import mock import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._constants import FLOW_TOOLS_JSON, NODE_VARIANTS, PROMPT_FLOW_DIR_NAME, USE_VARIANTS from promptflow._utils.yaml_utils import load_yaml @@ -14,15 +15,12 @@ from promptflow.core._flow import Prompty from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() -FLOWS_DIR = "./tests/test_configs/flows" -EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows" -DATAS_DIR = "./tests/test_configs/datas" -PROMPTY_DIR = "./tests/test_configs/prompty" +TEST_ROOT = PROMPTFLOW_ROOT / "tests" +CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" +PROMPTY_DIR = PROMPTFLOW_ROOT / "tests/test_configs/prompty" def e2e_test_docker_build_and_run(output_path): diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py similarity index 98% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py index 3e48f0d26f4..4ec3a8024d2 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py @@ -8,10 +8,10 @@ import numpy as np import pandas as pd import pytest +from _constants import PROMPTFLOW_ROOT from marshmallow import ValidationError from pytest_mock import MockerFixture -from promptflow import PFClient from promptflow._constants import PROMPTFLOW_CONNECTIONS from promptflow._sdk._constants import ( FLOW_DIRECTORY_MACRO_IN_CONFIG, @@ -36,18 +36,16 @@ from promptflow._sdk.operations._local_storage_operations import LocalStorageOperations from promptflow._utils.context_utils import _change_working_dir, inject_sys_path from promptflow._utils.yaml_utils import load_yaml +from promptflow.client import PFClient from promptflow.connections import AzureOpenAIConnection from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() -FLOWS_DIR = "./tests/test_configs/flows" -EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows" -RUNS_DIR = "./tests/test_configs/runs" -DATAS_DIR = "./tests/test_configs/datas" +TEST_ROOT = PROMPTFLOW_ROOT / "tests" +CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" +RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs" +DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas" def my_entry(input1: str): @@ -236,7 +234,7 @@ def test_run_bulk_error(self, pf): # path not exist with pytest.raises(UserErrorException) as e: pf.run( - flow=f"{MODEL_ROOT}/not_exist", + flow=f"{FLOWS_DIR}/not_exist", data=f"{DATAS_DIR}/webClassification3.jsonl", column_mapping={"question": "${data.question}", "context": "${data.context}"}, variant="${summarize_text_content.variant_0}", @@ -1691,6 +1689,30 @@ def assert_func(details_dict): run = pf.runs.create_or_update(run=run) assert_batch_run_result(run, pf, assert_func) + def test_run_with_init_class(self, pf): + def assert_func(details_dict): + return details_dict["outputs.func_input"] == [ + "func_input", + "func_input", + "func_input", + "func_input", + ] and details_dict["outputs.obj_input"] == ["val", "val", "val", "val"] + + with inject_sys_path(f"{EAGER_FLOWS_DIR}/basic_callable_class"): + from simple_callable_class import MyFlow + + run = pf.run( + flow=MyFlow, + data=f"{EAGER_FLOWS_DIR}/basic_callable_class/inputs.jsonl", + init={"obj_input": "val"}, + # set code folder to avoid snapshot too big + code=f"{EAGER_FLOWS_DIR}/basic_callable_class", + ) + assert run.status == "Completed" + assert "error" not in run._to_dict() + + assert_batch_run_result(run, pf, assert_func) + def assert_batch_run_result(run: Run, pf: PFClient, assert_func): assert run.status == "Completed" diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_save.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_save.py similarity index 93% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_save.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_save.py index ae2e4311eb8..e5d53877147 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_save.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_save.py @@ -4,23 +4,19 @@ import shutil import sys import tempfile -from pathlib import Path from typing import Callable, TypedDict import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._pf_client import PFClient from promptflow._sdk.entities import AzureOpenAIConnection from promptflow.client import load_flow from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() -FLOWS_DIR = (TEST_ROOT / "test_configs/flows").resolve().absolute().as_posix() -EAGER_FLOWS_DIR = (TEST_ROOT / "test_configs/eager_flows").resolve().absolute().as_posix() +TEST_ROOT = PROMPTFLOW_ROOT / "tests" +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" +EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows" FLOW_RESULT_KEYS = ["category", "evidence"] _client = PFClient() @@ -74,7 +70,11 @@ def global_hello_int_return(text: str) -> int: def global_hello_strong_return(text: str) -> GlobalHello: - return len(text) + return GlobalHello(AzureOpenAIConnection("test")) + + +def global_hello_kwargs(text: str, **kwargs) -> str: + return f"Hello {text}!" @pytest.mark.usefixtures( @@ -166,7 +166,7 @@ class TestFlowSave: "type": "string", }, "length": { - "type": "integer", + "type": "int", }, }, }, @@ -204,16 +204,16 @@ class TestFlowSave: "type": "string", }, "i": { - "type": "integer", + "type": "int", }, "f": { - "type": "number", + "type": "double", }, "b": { - "type": "boolean", + "type": "bool", }, "li": { - "type": "array", + "type": "list", }, "d": { "type": "object", @@ -224,16 +224,16 @@ class TestFlowSave: "type": "string", }, "i": { - "type": "integer", + "type": "int", }, "f": { - "type": "number", + "type": "double", }, "b": { - "type": "boolean", + "type": "bool", }, "li": { - "type": "array", + "type": "list", }, "d": { "type": "object", @@ -244,16 +244,16 @@ class TestFlowSave: "type": "string", }, "i": { - "type": "integer", + "type": "int", }, "f": { - "type": "number", + "type": "double", }, "b": { - "type": "boolean", + "type": "bool", }, "l": { - "type": "array", + "type": "list", }, "d": { "type": "object", @@ -494,6 +494,17 @@ def test_pf_save_callable_function(self): }, id="inherited_typed_dict_output", ), + pytest.param( + global_hello_kwargs, + { + "inputs": { + "text": { + "type": "string", + } + }, + }, + id="kwargs", + ), ], ) def test_infer_signature( diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve.py index 125eaa03a75..e6c033e85dd 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve.py @@ -3,6 +3,7 @@ import re import pytest +from _constants import PROMPTFLOW_ROOT from opentelemetry import trace from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider @@ -15,6 +16,8 @@ from promptflow.exceptions import UserErrorException from promptflow.tracing._operation_context import OperationContext +TEST_CONFIGS = PROMPTFLOW_ROOT / "tests" / "test_configs" / "eager_flows" + @pytest.mark.usefixtures("recording_injection", "setup_local_connection") @pytest.mark.e2etest @@ -570,14 +573,13 @@ def test_eager_flow_serve_dataclass_output(simple_eager_flow_dataclass_output): def test_eager_flow_serve_non_json_serializable_output(mocker): with pytest.raises(UserErrorException, match="Parse interface for 'my_flow' failed:"): # instead of giving 400 response for all requests, we raise user error on serving now - from pathlib import Path from ..conftest import create_client_by_model create_client_by_model( "non_json_serializable_output", mocker, - model_root=Path(__file__).parent.parent.parent / "test_configs" / "eager_flows", + model_root=TEST_CONFIGS, ) diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py similarity index 95% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py index fe2edbd9271..ecda3ebbb51 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py @@ -7,6 +7,7 @@ import papermill import pytest +from _constants import PROMPTFLOW_ROOT from marshmallow import ValidationError from promptflow._sdk._constants import LOGGER_NAME @@ -14,11 +15,8 @@ from promptflow.core._utils import init_executable from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent -MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples" -CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() +TEST_ROOT = PROMPTFLOW_ROOT / "tests" +CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() FLOWS_DIR = (TEST_ROOT / "test_configs/flows").resolve().absolute().as_posix() EAGER_FLOWS_DIR = (TEST_ROOT / "test_configs/eager_flows").resolve().absolute().as_posix() FLOW_RESULT_KEYS = ["category", "evidence"] @@ -137,7 +135,7 @@ def test_pf_test_flow_with_variant(self): @pytest.mark.skip("TODO this test case failed in windows and Mac") def test_pf_test_with_additional_includes(self, caplog): - from promptflow import VERSION + from promptflow._sdk._version import VERSION print(VERSION) with caplog.at_level(level=logging.WARNING, logger=LOGGER_NAME): @@ -279,6 +277,12 @@ def test_eager_flow_test_with_yaml(self): result = _client._flows._test(flow=flow_path, inputs={"input_val": "val1"}) assert result.run_info.status.value == "Completed" + def test_eager_flow_test_with_yml(self): + clear_module_cache("entry") + flow_path = Path(f"{EAGER_FLOWS_DIR}/simple_with_yml/").absolute() + result = _client._flows._test(flow=flow_path, inputs={"input_val": "val1"}) + assert result.run_info.status.value == "Completed" + def test_eager_flow_test_with_primitive_output(self): clear_module_cache("entry") flow_path = Path(f"{EAGER_FLOWS_DIR}/primitive_output/").absolute() @@ -417,3 +421,13 @@ def test_eager_flow_multiple_stream_outputs_dataclass(self): assert is_dataclass(result.output) assert result.output.output1 == "0123456789" assert result.output.output2 == "0123456789" + + def test_flex_flow_with_init(self, pf): + + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_callable_class") + result1 = pf.test(flow=flow_path, inputs={"func_input": "input"}, init={"obj_input": "val"}) + assert result1["func_input"] == "input" + + result2 = pf.test(flow=flow_path, inputs={"func_input": "input"}, init={"obj_input": "val"}) + assert result2["func_input"] == "input" + assert result1["obj_id"] != result2["obj_id"] diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_orm.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_orm.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_orm.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_orm.py diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_prompty.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_prompty.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_prompty.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_prompty.py diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_telemetry.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_telemetry.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_telemetry.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_telemetry.py diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_tool.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_tool.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_tool.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_tool.py index a012c7a44e3..03016260639 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_tool.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_tool.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._core.tool import ToolProvider, tool from promptflow._core.tool_meta_generator import ToolValidationError, _serialize_tool @@ -12,8 +13,7 @@ from promptflow.entities import DynamicList, InputSetting from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" TOOL_ROOT = TEST_ROOT / "test_configs/tools" _client = PFClient() diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_trace.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_trace.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/e2etests/test_trace.py rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_trace.py index ebf2e2204ec..8e709df557c 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_trace.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_trace.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest +from _constants import PROMPTFLOW_ROOT from mock import mock from promptflow._constants import ( @@ -18,7 +19,7 @@ from promptflow._sdk._pf_client import PFClient from promptflow._sdk.entities._trace import Span -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" FLOWS_DIR = (TEST_ROOT / "test_configs/flows").resolve().absolute().as_posix() diff --git a/src/promptflow/tests/sdk_cli_test/unittests/__init__.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/unittests/__init__.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/__init__.py diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_chat_group.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_chat_group.py similarity index 95% rename from src/promptflow/tests/sdk_cli_test/unittests/test_chat_group.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_chat_group.py index eb8050a381a..2bb2d14f3b6 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_chat_group.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_chat_group.py @@ -1,15 +1,12 @@ -from pathlib import Path - import pytest +from _constants import PROMPTFLOW_ROOT from pytest_mock import MockFixture from promptflow._sdk._errors import ChatGroupError, ChatRoleError from promptflow._sdk.entities._chat_group._chat_group import ChatGroup from promptflow._sdk.entities._chat_group._chat_role import ChatRole -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." - -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" FLOWS_DIR = TEST_ROOT / "test_configs/flows" diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_cli_activity_name.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_cli_activity_name.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/unittests/test_cli_activity_name.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_cli_activity_name.py index 3cfbba2c8f2..e0723d40cf9 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_cli_activity_name.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_cli_activity_name.py @@ -1,4 +1,5 @@ import pytest + from promptflow._cli._pf.entry import get_parser_args from promptflow._cli._utils import _get_cli_activity_name diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_config.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_config.py similarity index 96% rename from src/promptflow/tests/sdk_cli_test/unittests/test_config.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_config.py index 768f042a3da..0e43a4979e5 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_config.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_config.py @@ -1,15 +1,15 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._configuration import Configuration, InvalidConfigValue from promptflow._sdk._constants import FLOW_DIRECTORY_MACRO_IN_CONFIG from promptflow._utils.user_agent_utils import ClientUserAgentUtil -CONFIG_DATA_ROOT = Path(__file__).parent.parent.parent / "test_configs" / "configs" +CONFIG_DATA_ROOT = PROMPTFLOW_ROOT / "tests" / "test_configs" / "configs" @pytest.fixture diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_connection.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_connection.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/unittests/test_connection.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_connection.py index 4520fb5f629..c128c41ca63 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_connection.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_connection.py @@ -2,11 +2,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- import os -from pathlib import Path from unittest.mock import patch import mock import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._cli._pf._connection import validate_and_interactive_get_secrets from promptflow._sdk._constants import SCRUBBED_VALUE, ConnectionAuthMode, CustomStrongTypeConnectionConfigs @@ -30,7 +30,7 @@ from promptflow.core._connection import RequiredEnvironmentVariablesNotSetError from promptflow.exceptions import UserErrorException -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" CONNECTION_ROOT = TEST_ROOT / "test_configs/connections" diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_experiment.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_experiment.py similarity index 97% rename from src/promptflow/tests/sdk_cli_test/unittests/test_experiment.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_experiment.py index f61a68133a0..ca2a9f115b0 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_experiment.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_experiment.py @@ -1,6 +1,5 @@ -from pathlib import Path - import pytest +from _constants import PROMPTFLOW_ROOT from ruamel.yaml import YAML from promptflow._sdk._errors import MultipleExperimentTemplateError, NoExperimentTemplateError @@ -8,7 +7,7 @@ from promptflow._sdk._orchestrator.experiment_orchestrator import ExperimentTemplateTestContext from promptflow._sdk.entities._experiment import Experiment, ExperimentData, ExperimentInput, FlowNode -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" EXP_ROOT = TEST_ROOT / "test_configs/experiments" FLOW_ROOT = TEST_ROOT / "test_configs/flows" diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_flow.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow.py similarity index 87% rename from src/promptflow/tests/sdk_cli_test/unittests/test_flow.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow.py index d9022df731b..85dafa3fd32 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_flow.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow.py @@ -6,8 +6,8 @@ import pytest from marshmallow import ValidationError -from promptflow import load_flow from promptflow._sdk.entities._flows import FlexFlow, Flow +from promptflow.client import load_flow from promptflow.exceptions import ValidationException FLOWS_DIR = Path("./tests/test_configs/flows") @@ -71,7 +71,13 @@ def test_multiple_flow_load(self): with pytest.raises(ValidationException) as e: load_flow(EAGER_FLOWS_DIR / "multiple_flow_yaml") - assert "Both flow.dag.yaml and flow.flex.yaml exist in " in str(e.value) + assert "Multiple files flow.dag.yaml, flow.flex.yaml exist in " in str(e.value) + + def test_multiple_flex_load(self): + with pytest.raises(ValidationException) as e: + load_flow(EAGER_FLOWS_DIR / "multiple_flex_yaml") + + assert "Multiple files flow.flex.yaml, flow.flex.yml exist in " in str(e.value) def test_specify_flow_load(self): load_flow(EAGER_FLOWS_DIR / "multiple_flow_yaml" / "flow.dag.yaml") diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_invoker.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_invoker.py similarity index 92% rename from src/promptflow/tests/sdk_cli_test/unittests/test_flow_invoker.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_invoker.py index 2a4bc7a7b8f..98810b982e1 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_invoker.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_invoker.py @@ -1,17 +1,16 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from promptflow._sdk._load_functions import load_flow from promptflow.core._serving._errors import UnexpectedConnectionProviderReturn, UnsupportedConnectionProvider from promptflow.core._serving.flow_invoker import FlowInvoker from promptflow.exceptions import UserErrorException -PROMOTFLOW_ROOT = Path(__file__).parent.parent.parent.parent -FLOWS_DIR = Path(PROMOTFLOW_ROOT / "tests/test_configs/flows") +FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows" EXAMPLE_FLOW_DIR = FLOWS_DIR / "web_classification" EXAMPLE_FLOW_FILE = EXAMPLE_FLOW_DIR / "flow.dag.yaml" EXAMPLE_FLOW = load_flow(EXAMPLE_FLOW_FILE) diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_serve.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_serve.py similarity index 69% rename from src/promptflow/tests/sdk_cli_test/unittests/test_flow_serve.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_serve.py index e7b8ad5f479..4988b7f02c6 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_serve.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_serve.py @@ -1,7 +1,7 @@ from pathlib import Path import pytest -from sdk_cli_test.conftest import MODEL_ROOT +from _constants import PROMPTFLOW_ROOT from promptflow._cli._pf._flow import _resolve_python_flow_additional_includes @@ -9,12 +9,17 @@ @pytest.mark.unittest def test_flow_serve_resolve_additional_includes(): # Assert flow path not changed if no additional includes - flow_path = (Path(MODEL_ROOT) / "web_classification").resolve().absolute().as_posix() + flow_path = (PROMPTFLOW_ROOT / "tests/test_configs/flows/web_classification").resolve().absolute().as_posix() resolved_flow_path = _resolve_python_flow_additional_includes(flow_path) assert flow_path == resolved_flow_path # Assert additional includes are resolved correctly - flow_path = (Path(MODEL_ROOT) / "web_classification_with_additional_include").resolve().absolute().as_posix() + flow_path = ( + (PROMPTFLOW_ROOT / "tests/test_configs/flows/web_classification_with_additional_include") + .resolve() + .absolute() + .as_posix() + ) resolved_flow_path = _resolve_python_flow_additional_includes(flow_path) assert (Path(resolved_flow_path) / "convert_to_dict.py").exists() diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_local_storage_operations.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_local_storage_operations.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/unittests/test_local_storage_operations.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_local_storage_operations.py diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_orm.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_orm.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/unittests/test_orm.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_orm.py diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_pf_client.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_pf_client.py similarity index 93% rename from src/promptflow/tests/sdk_cli_test/unittests/test_pf_client.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_pf_client.py index d4bd33175d8..d64a6311970 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_pf_client.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_pf_client.py @@ -3,8 +3,8 @@ # --------------------------------------------------------- import pytest -from promptflow import PFClient from promptflow._utils.user_agent_utils import ClientUserAgentUtil +from promptflow.client import PFClient @pytest.mark.sdk_test diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_run.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_run.py similarity index 99% rename from src/promptflow/tests/sdk_cli_test/unittests/test_run.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_run.py index 2943c2e660a..7dfd658cf00 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_run.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_run.py @@ -22,7 +22,6 @@ from promptflow._utils.yaml_utils import load_yaml from promptflow.exceptions import UserErrorException, ValidationException -PROMOTFLOW_ROOT = Path(__file__) / "../../../.." FLOWS_DIR = Path("./tests/test_configs/flows") EAGER_FLOWS_DIR = Path("./tests/test_configs/eager_flows") RUNS_DIR = Path("./tests/test_configs/runs") diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_tool.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_tool.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/unittests/test_tool.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_tool.py diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_trace.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_trace.py similarity index 100% rename from src/promptflow/tests/sdk_cli_test/unittests/test_trace.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_trace.py diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_utils.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_utils.py similarity index 91% rename from src/promptflow/tests/sdk_cli_test/unittests/test_utils.py rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_utils.py index e57fb41efa2..0d7a55a14b3 100644 --- a/src/promptflow/tests/sdk_cli_test/unittests/test_utils.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_utils.py @@ -9,7 +9,6 @@ import json import os import shutil -import subprocess import sys import tempfile import threading @@ -21,7 +20,7 @@ import mock import pandas as pd import pytest -from pip._vendor import tomli as toml +from _constants import PROMPTFLOW_ROOT from requests import Response from promptflow._cli._params import AppendToDictAction @@ -44,16 +43,16 @@ get_system_info, refresh_connections_dir, ) +from promptflow._sdk._version_hint_utils import check_latest_version from promptflow._utils.load_data import load_data from promptflow._utils.retry_utils import http_retry_wrapper, retry from promptflow._utils.utils import snake_to_camel -from promptflow._utils.version_hint_utils import check_latest_version from promptflow.core._utils import ( override_connection_config_with_environment_variable, resolve_connections_environment_variable_reference, ) -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" CONNECTION_ROOT = TEST_ROOT / "test_configs/connections" @@ -220,8 +219,8 @@ def mock_check_latest_version(): time.sleep(5) check_latest_version() - with patch("promptflow._utils.version_hint_utils.datetime") as mock_datetime, patch( - "promptflow._utils.version_hint_utils.check_latest_version", side_effect=mock_check_latest_version + with patch("promptflow._sdk._version_hint_utils.datetime") as mock_datetime, patch( + "promptflow._sdk._version_hint_utils.check_latest_version", side_effect=mock_check_latest_version ): from promptflow._sdk._telemetry import monitor_operation @@ -490,45 +489,3 @@ def test_gen_uuid_by_compute_info(self): system_info_hash = hashlib.sha256((host_name + system + machine).encode()).hexdigest() compute_info_hash = hashlib.sha256((mac_address + system_info_hash).encode()).hexdigest() assert str(uuid.uuid5(uuid.NAMESPACE_OID, compute_info_hash)) == gen_uuid_by_compute_info() - - def test_executable_package_match_toml_file(self): - def get_git_base_dir(): - return Path( - subprocess.run(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) - .stdout.decode("utf-8") - .strip() - ) - - def get_toml_dependencies(): - packages = ["promptflow-tracing", "promptflow-core", "promptflow-devkit"] - dependencies = [] - - for package in packages: - with open(get_git_base_dir() / "src" / package / "pyproject.toml", "rb") as file: - data = toml.load(file) - extra_package_names = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) - dependencies.extend(extra_package_names.keys()) - dependencies = [ - dependency - for dependency in dependencies - if not dependency.startswith("promptflow") and not dependency == "python" - ] - return dependencies - - all_packages = get_toml_dependencies() - - with open( - get_git_base_dir() - / "src" - / "promptflow-devkit" - / "promptflow" - / "_sdk" - / "data" - / "executable" - / "requirements.txt", - "r", - ) as f: - executable_all_packages = f.read().splitlines() - # check if all packages in requirements.txt are the same with pyproject.toml in devkit/core/tracinf packages. - # If not, maybe you need update requirements.txt - assert set(all_packages) == set(executable_all_packages) diff --git a/src/promptflow/tests/sdk_pfs_test/.coveragerc b/src/promptflow-devkit/tests/sdk_pfs_test/.coveragerc similarity index 100% rename from src/promptflow/tests/sdk_pfs_test/.coveragerc rename to src/promptflow-devkit/tests/sdk_pfs_test/.coveragerc diff --git a/src/promptflow/tests/sdk_pfs_test/__init__.py b/src/promptflow-devkit/tests/sdk_pfs_test/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_pfs_test/__init__.py rename to src/promptflow-devkit/tests/sdk_pfs_test/__init__.py diff --git a/src/promptflow/tests/sdk_pfs_test/conftest.py b/src/promptflow-devkit/tests/sdk_pfs_test/conftest.py similarity index 94% rename from src/promptflow/tests/sdk_pfs_test/conftest.py rename to src/promptflow-devkit/tests/sdk_pfs_test/conftest.py index 30dc613db4f..7ece485d6b5 100644 --- a/src/promptflow/tests/sdk_pfs_test/conftest.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/conftest.py @@ -5,7 +5,7 @@ import pytest from flask.app import Flask -from promptflow import PFClient +from promptflow.client import PFClient from .utils import PFSOperations diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/__init__.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/__init__.py similarity index 100% rename from src/promptflow/tests/sdk_pfs_test/e2etests/__init__.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/__init__.py diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_cli.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_cli.py similarity index 100% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_cli.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_cli.py diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_connection_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_connection_apis.py similarity index 92% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_connection_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_connection_apis.py index 4e002a74b70..11a99eada37 100644 --- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_connection_apis.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_connection_apis.py @@ -9,9 +9,9 @@ import mock import pytest -from promptflow import PFClient -from promptflow._sdk.entities import CustomConnection from promptflow._sdk._version import VERSION +from promptflow._sdk.entities import CustomConnection +from promptflow.client import PFClient from promptflow.recording.record_mode import is_replay from ..utils import PFSOperations, check_activity_end_telemetry @@ -89,14 +89,8 @@ def test_get_connection_specs(self, pfs_op: PFSOperations) -> None: @pytest.mark.skipif(is_replay(), reason="connection provider test, skip in non-live mode.") def test_get_connection_by_provicer(self, pfs_op, subscription_id, resource_group_name, workspace_name): target = "promptflow._sdk._pf_client.Configuration.get_connection_provider" - provider_url_target = ( - "promptflow._sdk.operations._local_azure_connection_operations." - "LocalAzureConnectionOperations._extract_workspace" - ) - mock_provider_url = (subscription_id, resource_group_name, workspace_name) - with mock.patch(target) as mocked_config, mock.patch(provider_url_target) as mocked_provider_url: + with mock.patch(target) as mocked_config: mocked_config.return_value = "azureml" - mocked_provider_url.return_value = mock_provider_url connections = pfs_op.list_connections(status_code=200).json assert len(connections) > 0 diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_experiment_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_experiment_apis.py similarity index 98% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_experiment_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_experiment_apis.py index d238bf4c329..1b24d9d693d 100644 --- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_experiment_apis.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_experiment_apis.py @@ -1,13 +1,13 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from ..utils import PFSOperations, check_activity_end_telemetry -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" EXPERIMENT_ROOT = TEST_ROOT / "test_configs/experiments" FLOW_ROOT = TEST_ROOT / "test_configs/flows" diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_flow_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_flow_apis.py similarity index 92% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_flow_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_flow_apis.py index d3ed544e51a..130026701ae 100644 --- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_flow_apis.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_flow_apis.py @@ -2,13 +2,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from ..utils import PFSOperations, check_activity_end_telemetry -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" FLOW_PATH = "./tests/test_configs/flows/print_env_var" IMAGE_PATH = "./tests/test_configs/datas/logo.jpg" FLOW_WITH_IMAGE_PATH = "./tests/test_configs/flows/chat_flow_with_image" diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_general_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_general_apis.py similarity index 81% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_general_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_general_apis.py index f03288477d8..41562b00c29 100644 --- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_general_apis.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_general_apis.py @@ -18,3 +18,7 @@ def test_heartbeat(self, pfs_op: PFSOperations) -> None: assert isinstance(response_json, dict) assert "promptflow" in response_json assert response_json["promptflow"] == get_pfs_version() + + def test_rootpage_redirect(self, pfs_op: PFSOperations) -> None: + response = pfs_op.root_page() + assert response.status_code == 302 diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_run_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_run_apis.py similarity index 99% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_run_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_run_apis.py index ccf9973c911..1640ad1af00 100644 --- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_run_apis.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_run_apis.py @@ -9,9 +9,9 @@ import pytest -from promptflow import PFClient from promptflow._sdk.entities import Run from promptflow._sdk.operations._local_storage_operations import LocalStorageOperations +from promptflow.client import PFClient from promptflow.contracts._run_management import RunMetadata from ..utils import PFSOperations, check_activity_end_telemetry diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py similarity index 100% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_trace.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_trace.py similarity index 100% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_trace.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_trace.py diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_ui_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_ui_apis.py similarity index 98% rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_ui_apis.py rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_ui_apis.py index 936012ddeda..1b3cc6ea3e2 100644 --- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_ui_apis.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_ui_apis.py @@ -8,11 +8,12 @@ from pathlib import Path import pytest +from _constants import PROMPTFLOW_ROOT from PIL import Image from ..utils import PFSOperations, check_activity_end_telemetry -TEST_ROOT = Path(__file__).parent.parent.parent +TEST_ROOT = PROMPTFLOW_ROOT / "tests" FLOW_PATH = "./tests/test_configs/flows/print_env_var" IMAGE_PATH = "./tests/test_configs/datas/logo.jpg" FLOW_WITH_IMAGE_PATH = "./tests/test_configs/flows/chat_flow_with_image" diff --git a/src/promptflow/tests/sdk_pfs_test/utils.py b/src/promptflow-devkit/tests/sdk_pfs_test/utils.py similarity index 99% rename from src/promptflow/tests/sdk_pfs_test/utils.py rename to src/promptflow-devkit/tests/sdk_pfs_test/utils.py index 141e0bbe4fa..168add856df 100644 --- a/src/promptflow/tests/sdk_pfs_test/utils.py +++ b/src/promptflow-devkit/tests/sdk_pfs_test/utils.py @@ -74,6 +74,9 @@ def remote_user_header(self, user_agent=None): def heartbeat(self): return self._client.get("/heartbeat") + def root_page(self): + return self._client.get("/") + # connection APIs def connection_operation_with_invalid_user(self, status_code=None): response = self._client.get(f"{self.CONNECTION_URL_PREFIX}/", headers={"X-Remote-User": "invalid_user"}) diff --git a/src/promptflow-devkit/tests/unittests/test.py b/src/promptflow-devkit/tests/unittests/test.py deleted file mode 100644 index 226a2ed4261..00000000000 --- a/src/promptflow-devkit/tests/unittests/test.py +++ /dev/null @@ -1,11 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -import pytest - - -@pytest.mark.unittest -class TestStartTrace: - def test_import(self): - assert True diff --git a/src/promptflow-recording/promptflow/recording/azure/utils.py b/src/promptflow-recording/promptflow/recording/azure/utils.py index 42077d1b681..b558ab812b9 100644 --- a/src/promptflow-recording/promptflow/recording/azure/utils.py +++ b/src/promptflow-recording/promptflow/recording/azure/utils.py @@ -99,8 +99,8 @@ def sanitize_azure_workspace_triad(value: str) -> str: flags=re.IGNORECASE, ) sanitized_ws = re.sub( - r"/(workspaces)/[-\w\._\(\)]+[/?]", - r"/\1/{}/".format("00000"), + r"/(workspaces)/[-\w\._\(\)]+([/?][-\w\._\(\)]*)", + r"/\1/{}\2".format("00000"), sanitized_rg, flags=re.IGNORECASE, ) @@ -108,7 +108,7 @@ def sanitize_azure_workspace_triad(value: str) -> str: # workspace name can be the last part of the string # e.g. xxx/Microsoft.MachineLearningServices/workspaces/ # apply a special handle here to sanitize - if sanitized_ws.startswith("https://"): + if sanitized_ws == sanitized_rg and sanitized_ws.startswith("https://"): split1, split2 = sanitized_ws.split("/")[-2:] if split1 == "workspaces": sanitized_ws = sanitized_ws.replace(split2, SanitizedValues.WORKSPACE_NAME) diff --git a/src/promptflow-recording/recordings/azure/test_arm_connection_operations_TestArmConnectionOperations_test_get_connection.yaml b/src/promptflow-recording/recordings/azure/test_arm_connection_operations_TestArmConnectionOperations_test_get_connection.yaml index 7ee0a697274..4d071dde742 100644 --- a/src/promptflow-recording/recordings/azure/test_arm_connection_operations_TestArmConnectionOperations_test_get_connection.yaml +++ b/src/promptflow-recording/recordings/azure/test_arm_connection_operations_TestArmConnectionOperations_test_get_connection.yaml @@ -9,21 +9,21 @@ interactions: Connection: - keep-alive User-Agent: - - promptflow-sdk/0.0.1 promptflow/0.0.1 azure-ai-ml/1.12.1 azsdk-python-mgmt-machinelearningservices/0.1.0 - Python/3.10.13 (Windows-10-10.0.22631-SP0) + - promptflow-azure-sdk/0.1.0b1 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.9.19 (Windows-10-10.0.22631-SP0) method: GET - uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api-version=2023-08-01-preview + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?api-version=2023-08-01-preview response: body: string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": - "eastus", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", - "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' + "eastus2euap", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": + "Basic", "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' headers: cache-control: - no-cache content-length: - - '3630' + - '3697' content-type: - application/json; charset=utf-8 expires: @@ -39,7 +39,7 @@ interactions: x-content-type-options: - nosniff x-request-time: - - '0.017' + - '0.019' status: code: 200 message: OK @@ -53,29 +53,21 @@ interactions: Connection: - keep-alive User-Agent: - - promptflow-sdk/0.0.1 promptflow/0.0.1 azure-ai-ml/1.12.1 azsdk-python-mgmt-machinelearningservices/0.1.0 - Python/3.10.13 (Windows-10-10.0.22631-SP0) + - azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.9.19 + (Windows-10-10.0.22631-SP0) method: GET - uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?api-version=2023-04-01-preview&count=30&isDefault=true&orderByAsc=false + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?api-version=2023-08-01-preview response: body: - string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", - "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", - "properties": {"description": null, "tags": null, "properties": null, "isDefault": - true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": - null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": - "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", - "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": - "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, - "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": - "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": - "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", - "lastModifiedByType": "Application"}}]}' + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2euap", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": + "Basic", "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' headers: cache-control: - no-cache content-length: - - '1372' + - '3697' content-type: - application/json; charset=utf-8 expires: @@ -91,7 +83,7 @@ interactions: x-content-type-options: - nosniff x-request-time: - - '0.070' + - '0.020' status: code: 200 message: OK @@ -116,18 +108,20 @@ interactions: "name": "azure_open_ai_connection", "type": "Microsoft.MachineLearningServices/workspaces/connections", "properties": {"authType": "ApiKey", "credentials": {"key": "_"}, "group": "AzureAI", "category": "AzureOpenAI", "expiryTime": null, "target": "_", "createdByWorkspaceArmId": - null, "isSharedToAll": false, "sharedUserList": [], "metadata": {"azureml.flow.connection_type": - "AzureOpenAI", "azureml.flow.module": "promptflow.connections", "ApiType": - "azure", "ApiVersion": "2023-07-01-preview", "ResourceId": null, "DeploymentApiVersion": - "2023-10-01-preview"}}, "systemData": {"createdAt": "2023-08-22T10:15:34.5762053Z", - "createdBy": "username@microsoft.com", "createdByType": "User", "lastModifiedAt": - "2023-08-22T10:15:34.5762053Z", "lastModifiedBy": "username@microsoft.com", - "lastModifiedByType": "User"}}' + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus2euap", + "useWorkspaceManagedIdentity": false, "isSharedToAll": false, "sharedUserList": + [], "metadata": {"azureml.flow.connection_type": "AzureOpenAI", "azureml.flow.module": + "promptflow.connections", "ApiType": "azure", "ApiVersion": "2023-07-01-preview", + "ResourceId": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.CognitiveServices/accounts/gpt-test-eus", + "DeploymentApiVersion": "2023-10-01-preview"}}, "systemData": {"createdAt": + "2024-03-22T14:56:56.0857069Z", "createdBy": "username@microsoft.com", "createdByType": + "User", "lastModifiedAt": "2024-03-22T14:56:56.0857069Z", "lastModifiedBy": + "username@microsoft.com", "lastModifiedByType": "User"}}' headers: cache-control: - no-cache content-length: - - '1246' + - '1579' content-type: - application/json; charset=utf-8 expires: @@ -143,7 +137,7 @@ interactions: x-content-type-options: - nosniff x-request-time: - - '0.072' + - '0.772' status: code: 200 message: OK @@ -168,16 +162,18 @@ interactions: "name": "custom_connection", "type": "Microsoft.MachineLearningServices/workspaces/connections", "properties": {"authType": "CustomKeys", "credentials": {"keys": {}}, "group": "AzureAI", "category": "CustomKeys", "expiryTime": null, "target": "_", "createdByWorkspaceArmId": - null, "isSharedToAll": false, "sharedUserList": [], "metadata": {"azureml.flow.connection_type": - "Custom", "azureml.flow.module": "promptflow.connections"}}, "systemData": - {"createdAt": "2023-06-19T20:56:12.0353964Z", "createdBy": "username@microsoft.com", - "createdByType": "User", "lastModifiedAt": "2023-06-19T20:56:12.0353964Z", - "lastModifiedBy": "username@microsoft.com", "lastModifiedByType": "User"}}' + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus2euap", + "useWorkspaceManagedIdentity": false, "isSharedToAll": false, "sharedUserList": + [], "metadata": {"azureml.flow.connection_type": "Custom", "azureml.flow.module": + "promptflow.connections"}}, "systemData": {"createdAt": "2023-12-18T09:44:22.1118582Z", + "createdBy": "username@microsoft.com", "createdByType": "User", "lastModifiedAt": + "2023-12-18T09:44:22.1118582Z", "lastModifiedBy": "username@microsoft.com", + "lastModifiedByType": "User"}}' headers: cache-control: - no-cache content-length: - - '1275' + - '1254' content-type: - application/json; charset=utf-8 expires: @@ -193,7 +189,7 @@ interactions: x-content-type-options: - nosniff x-request-time: - - '0.075' + - '0.090' status: code: 200 message: OK diff --git a/src/promptflow-recording/recordings/azure/test_workspace_connection_provider_TestWorkspaceConnectionProvider_test_list_connections.yaml b/src/promptflow-recording/recordings/azure/test_workspace_connection_provider_TestWorkspaceConnectionProvider_test_list_connections.yaml index 359dbc93463..5e1dbd617d2 100644 --- a/src/promptflow-recording/recordings/azure/test_workspace_connection_provider_TestWorkspaceConnectionProvider_test_list_connections.yaml +++ b/src/promptflow-recording/recordings/azure/test_workspace_connection_provider_TestWorkspaceConnectionProvider_test_list_connections.yaml @@ -10,9 +10,9 @@ interactions: - keep-alive User-Agent: - promptflow-azure-sdk/0.1.0b1 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 - Python/3.9.7 (Windows-10-10.0.22631-SP0) + Python/3.9.19 (Windows-10-10.0.22631-SP0) method: GET - uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api-version=2023-08-01-preview + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?api-version=2023-08-01-preview response: body: string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", @@ -39,7 +39,51 @@ interactions: x-content-type-options: - nosniff x-request-time: - - '0.021' + - '0.019' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.9.19 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?api-version=2023-08-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2euap", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": + "Basic", "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '3697' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.019' status: code: 200 message: OK @@ -172,7 +216,7 @@ interactions: x-content-type-options: - nosniff x-request-time: - - '0.029' + - '0.018' status: code: 200 message: OK diff --git a/src/promptflow-recording/recordings/local/node_cache.shelve.bak b/src/promptflow-recording/recordings/local/node_cache.shelve.bak index e857d4b15c4..66f374ec8e1 100644 --- a/src/promptflow-recording/recordings/local/node_cache.shelve.bak +++ b/src/promptflow-recording/recordings/local/node_cache.shelve.bak @@ -64,3 +64,4 @@ '168786c62c341d79c15261a538bca1c22826b988', (263168, 4250) 'f41378cf209103949aa13c85b2708f16611dfe2f', (267776, 4962) '22099320490973d1d4de67b18bec3ffa8b1a64e3', (272896, 11361) +'94217ac2bde6b4d503c6cf2cb6b6684f2558e17b', (284672, 1816) diff --git a/src/promptflow-recording/recordings/local/node_cache.shelve.dat b/src/promptflow-recording/recordings/local/node_cache.shelve.dat index 36b2ba504cd..efa68b5edee 100644 Binary files a/src/promptflow-recording/recordings/local/node_cache.shelve.dat and b/src/promptflow-recording/recordings/local/node_cache.shelve.dat differ diff --git a/src/promptflow-recording/recordings/local/node_cache.shelve.dir b/src/promptflow-recording/recordings/local/node_cache.shelve.dir index e857d4b15c4..66f374ec8e1 100644 --- a/src/promptflow-recording/recordings/local/node_cache.shelve.dir +++ b/src/promptflow-recording/recordings/local/node_cache.shelve.dir @@ -64,3 +64,4 @@ '168786c62c341d79c15261a538bca1c22826b988', (263168, 4250) 'f41378cf209103949aa13c85b2708f16611dfe2f', (267776, 4962) '22099320490973d1d4de67b18bec3ffa8b1a64e3', (272896, 11361) +'94217ac2bde6b4d503c6cf2cb6b6684f2558e17b', (284672, 1816) diff --git a/src/promptflow/CHANGELOG.md b/src/promptflow/CHANGELOG.md index bb0927bb648..b74e523a104 100644 --- a/src/promptflow/CHANGELOG.md +++ b/src/promptflow/CHANGELOG.md @@ -3,7 +3,10 @@ ## 1.9.0 (Upcoming) ### Features Added -- [CLI]: Added autocomplete feature. +- [promptflow-devkit]: Added autocomplete feature for linux, reach [here](https://microsoft.github.io/promptflow/reference/pf-command-reference.html#autocomplete) for more details. + +### Bugs Fixed +- [promptflow-devkit] Fix run name missing directory name in some scenario of `pf.run`. ### Others - [promptflow-core] Connection default api version changed: diff --git a/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py b/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py index 2c40c39daa7..6be0d8f36ef 100644 --- a/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py +++ b/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py @@ -10,11 +10,11 @@ from promptflow._core._errors import InvalidSource from promptflow._core.tools_manager import ToolLoader from promptflow._internal import tool -from promptflow.connections import CustomConnection, CustomStrongTypeConnection -from promptflow.connections import AzureOpenAIConnection +from promptflow.connections import AzureOpenAIConnection, CustomConnection, CustomStrongTypeConnection from promptflow.contracts.flow import InputAssignment, InputValueType, Node, ToolSource, ToolSourceType from promptflow.contracts.tool import AssistantDefinition, InputDefinition, Secret, Tool, ToolType, ValueType from promptflow.contracts.types import PromptTemplate +from promptflow.core._connection_provider._dict_connection_provider import DictConnectionProvider from promptflow.exceptions import UserErrorException from promptflow.executor._assistant_tool_invoker import ResolvedAssistantTool from promptflow.executor._errors import ( @@ -49,7 +49,7 @@ def mock_package_func(prompt: PromptTemplate, **kwargs): class TestToolResolver: @pytest.fixture def resolver(self): - return ToolResolver(working_dir=None, connections={}) + return ToolResolver(working_dir=None) def test_resolve_tool_by_node_with_diff_type(self, resolver, mocker): node = mocker.Mock(name="node", tool=None, inputs={}) @@ -181,7 +181,7 @@ def test_resolve_tool_by_node_with_invalid_template(self, resolver, mocker): assert expected_message in exec_info.value.message def test_convert_node_literal_input_types_with_invalid_case(self): - # Case 1: conn_name not in connections, should raise conn_name not found error + # Case 1: conn_name not in connection_provider, should raise conn_name not found error tool = Tool(name="mock", type="python", inputs={"conn": InputDefinition(type=["CustomConnection"])}) node = Node( name="mock", @@ -189,13 +189,13 @@ def test_convert_node_literal_input_types_with_invalid_case(self): inputs={"conn": InputAssignment(value="conn_name", value_type=InputValueType.LITERAL)}, ) with pytest.raises(ConnectionNotFound): - tool_resolver = ToolResolver(working_dir=None, connections={}) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider({})) tool_resolver._convert_node_literal_input_types(node, tool) - # Case 2: conn_name in connections, but type not matched + # Case 2: conn_name in connection_provider, but type not matched connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} with pytest.raises(NodeInputValidationError) as exe_info: - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider(connections)) tool_resolver._convert_node_literal_input_types(node, tool) message = "'AzureOpenAIConnection' is not supported, valid types ['CustomConnection']" assert message in exe_info.value.message, "Expected: {}, Actual: {}".format(message, exe_info.value.message) @@ -208,7 +208,7 @@ def test_convert_node_literal_input_types_with_invalid_case(self): inputs={"int_input": InputAssignment(value="invalid", value_type=InputValueType.LITERAL)}, ) with pytest.raises(NodeInputValidationError) as exe_info: - tool_resolver = ToolResolver(working_dir=None, connections={}) + tool_resolver = ToolResolver(working_dir=None) tool_resolver._convert_node_literal_input_types(node, tool) message = "value 'invalid' is not type int" assert message in exe_info.value.message, "Expected: {}, Actual: {}".format(message, exe_info.value.message) @@ -221,7 +221,7 @@ def test_convert_node_literal_input_types_with_invalid_case(self): inputs={"int_input": InputAssignment(value="invalid", value_type=InputValueType.LITERAL)}, ) with pytest.raises(ValueTypeUnresolved): - tool_resolver = ToolResolver(working_dir=None, connections={}) + tool_resolver = ToolResolver(working_dir=None) tool_resolver._convert_node_literal_input_types(node, tool) # Case 5: Literal value, invalid image in list @@ -233,7 +233,7 @@ def test_convert_node_literal_input_types_with_invalid_case(self): inputs={"list_input": InputAssignment(value=[invalid_image], value_type=InputValueType.LITERAL)}, ) with pytest.raises(NodeInputValidationError) as exe_info: - tool_resolver = ToolResolver(working_dir=None, connections={}) + tool_resolver = ToolResolver(working_dir=None) tool_resolver._convert_node_literal_input_types(node, tool) message = "Invalid base64 image" assert message in exe_info.value.message, "Expected: {}, Actual: {}".format(message, exe_info.value.message) @@ -250,7 +250,7 @@ def test_convert_node_literal_input_types_with_invalid_case(self): inputs={"assistant_definition": InputAssignment(value="invalid_path", value_type=InputValueType.LITERAL)}, ) with pytest.raises(NodeInputValidationError) as exe_info: - tool_resolver = ToolResolver(working_dir=Path(__file__).parent, connections={}) + tool_resolver = ToolResolver(working_dir=Path(__file__).parent) tool_resolver._convert_node_literal_input_types(node, tool) assert ( "Failed to load assistant definition" in exe_info.value.message @@ -258,6 +258,9 @@ def test_convert_node_literal_input_types_with_invalid_case(self): ), "Expected: {}, Actual: {}".format(message, exe_info.value.message) def test_resolve_llm_connection_to_inputs(self): + connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} + connection_provider = DictConnectionProvider(connections) + # Case 1: node.connection is not specified tool = Tool(name="mock", type="python", inputs={"conn": InputDefinition(type=["CustomConnection"])}) node = Node( @@ -265,9 +268,8 @@ def test_resolve_llm_connection_to_inputs(self): tool=tool, inputs={"conn": InputAssignment(value="conn_name", value_type=InputValueType.LITERAL)}, ) - connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} with pytest.raises(ConnectionNotFound): - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=connection_provider) tool_resolver._resolve_llm_connection_to_inputs(node, tool) # Case 2: node.connection is not found from connection manager @@ -278,9 +280,8 @@ def test_resolve_llm_connection_to_inputs(self): inputs={"conn": InputAssignment(value="conn_name", value_type=InputValueType.LITERAL)}, connection="conn_name1", ) - connections = {} with pytest.raises(ConnectionNotFound): - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider({})) tool_resolver._resolve_llm_connection_to_inputs(node, tool) # Case 3: Tool definition with bad input type list @@ -291,9 +292,8 @@ def test_resolve_llm_connection_to_inputs(self): inputs={"conn": InputAssignment(value="conn_name", value_type=InputValueType.LITERAL)}, connection="conn_name", ) - connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} with pytest.raises(InvalidConnectionType) as exe_info: - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=connection_provider) tool_resolver._resolve_llm_connection_to_inputs(node, tool) assert "Connection type can not be resolved for tool" in exe_info.value.message @@ -305,9 +305,8 @@ def test_resolve_llm_connection_to_inputs(self): inputs={"conn": InputAssignment(value="conn_name", value_type=InputValueType.LITERAL)}, connection="conn_name", ) - connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} with pytest.raises(InvalidConnectionType) as exe_info: - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=connection_provider) tool_resolver._resolve_llm_connection_to_inputs(node, tool) assert "Invalid connection" in exe_info.value.message @@ -323,9 +322,7 @@ def test_resolve_llm_connection_to_inputs(self): inputs={"conn": InputAssignment(value="conn_name", value_type=InputValueType.LITERAL)}, connection="conn_name", ) - connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} - - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=connection_provider) key, conn = tool_resolver._resolve_llm_connection_to_inputs(node, tool) assert key == "conn" assert isinstance(conn, AzureOpenAIConnection) @@ -346,7 +343,7 @@ def mock_llm_api_func(prompt: PromptTemplate, **kwargs): ) connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider(connections)) tool_resolver._tool_loader = tool_loader mocker.patch.object(tool_resolver, "_load_source_content", return_value="{{text}}![image]({{image}})") @@ -384,7 +381,7 @@ def mock_python_func(prompt: PromptTemplate, **kwargs): ) connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider(connections)) tool_resolver._tool_loader = tool_loader node = Node( @@ -420,7 +417,7 @@ def mock_python_func(input: AssistantDefinition): return_value=(mock_python_func, {}), ) - tool_resolver = ToolResolver(working_dir=Path(__file__).parent, connections={}) + tool_resolver = ToolResolver(working_dir=Path(__file__).parent) tool_resolver._tool_loader = tool_loader mocker.patch("builtins.open", mock_open()) mocker.patch( @@ -448,7 +445,7 @@ def test_resolve_package_node(self, mocker): ) connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider(connections)) tool_resolver._tool_loader = tool_loader node = Node( @@ -468,7 +465,7 @@ def test_resolve_package_node(self, mocker): assert resolved_tool.callable(**kwargs) == "Hello World!" def test_integrate_prompt_in_package_node(self, mocker): - tool_resolver = ToolResolver(working_dir=None, connections={}) + tool_resolver = ToolResolver(working_dir=None) mocker.patch.object( tool_resolver, "_load_source_content", @@ -501,7 +498,7 @@ def test_integrate_prompt_in_package_node(self, mocker): ) def test_convert_to_custom_strong_type_connection_value(self, conn_types: List[str], expected_type, mocker): connections = {"conn_name": {"type": "CustomConnection", "value": {"api_key": "mock", "api_base": "mock"}}} - tool_resolver = ToolResolver(working_dir=None, connections=connections) + tool_resolver = ToolResolver(working_dir=None, connection_provider=DictConnectionProvider(connections)) node = mocker.Mock(name="node", tool=None, inputs={}) node.type = ToolType.PYTHON @@ -565,7 +562,9 @@ def test_load_tools(self, predefined_inputs): # Test load tools connections = {"conn_name": {"type": "AzureOpenAIConnection", "value": {"api_key": "mock", "api_base": "mock"}}} - tool_resolver = ToolResolver(working_dir=Path(__file__).parent, connections=connections) + tool_resolver = ToolResolver( + working_dir=Path(__file__).parent, connection_provider=DictConnectionProvider(connections) + ) tool_resolver._resolve_assistant_tools("node_name", assistant_definitions) invoker = assistant_definitions._tool_invoker assert len(invoker._assistant_tools) == len(assistant_definitions.tools) == len(tool_definitions) @@ -623,7 +622,9 @@ def test_tool_with_connection_resolve(self, path): "value": {"api_key": "mock", "api_base": "mock"}, } } - tool_resolver = ToolResolver(working_dir=Path(ASSISTANT_DEFINITION_ROOT), connections=connections) + tool_resolver = ToolResolver( + working_dir=Path(ASSISTANT_DEFINITION_ROOT), connection_provider=DictConnectionProvider(connections) + ) assistant_definition = tool_resolver._convert_to_assistant_definition( assistant_definition_path=path, input_name="input_name", node_name="dummy_node" ) @@ -651,7 +652,7 @@ def test_tool_with_connection_resolve(self, path): @pytest.mark.parametrize("path", ["assistant_definition_without_functions.yaml"]) def test_code_interpreter_and_retrieval_tool_resolve(self, path): - tool_resolver = ToolResolver(working_dir=Path(ASSISTANT_DEFINITION_ROOT), connections={}) + tool_resolver = ToolResolver(working_dir=Path(ASSISTANT_DEFINITION_ROOT)) assistant_definition = tool_resolver._convert_to_assistant_definition( assistant_definition_path=path, input_name="input_name", node_name="dummy_node" ) @@ -680,7 +681,9 @@ def test_description_resolve(self, path): "value": {"api_key": "mock", "api_base": "mock"}, } } - tool_resolver = ToolResolver(working_dir=Path(ASSISTANT_DEFINITION_ROOT), connections=connections) + tool_resolver = ToolResolver( + working_dir=Path(ASSISTANT_DEFINITION_ROOT), connection_provider=DictConnectionProvider(connections) + ) assistant_definition = tool_resolver._convert_to_assistant_definition( assistant_definition_path=path, input_name="input_name", node_name="dummy_node" ) @@ -717,7 +720,9 @@ def test_types_resolve(self, path): "value": {"api_key": "mock", "api_base": "mock"}, } } - tool_resolver = ToolResolver(working_dir=Path(ASSISTANT_DEFINITION_ROOT), connections=connections) + tool_resolver = ToolResolver( + working_dir=Path(ASSISTANT_DEFINITION_ROOT), connection_provider=DictConnectionProvider(connections) + ) assistant_definition = tool_resolver._convert_to_assistant_definition( assistant_definition_path=path, input_name="input_name", node_name="dummy_node" ) @@ -774,7 +779,9 @@ def test_invalid_assistant_definition_path(self, path): "value": {"api_key": "mock", "api_base": "mock"}, } } - tool_resolver = ToolResolver(working_dir=Path(ASSISTANT_DEFINITION_ROOT), connections=connections) + tool_resolver = ToolResolver( + working_dir=Path(ASSISTANT_DEFINITION_ROOT), connection_provider=DictConnectionProvider(connections) + ) with pytest.raises(InvalidSource) as e: tool_resolver._convert_to_assistant_definition( assistant_definition_path=path, input_name="input_name", node_name="dummy_node" diff --git a/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/entry.py b/src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/entry.py similarity index 100% rename from src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/entry.py rename to src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/entry.py diff --git a/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/flow.flex.yaml b/src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/flow.flex.yaml similarity index 100% rename from src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/flow.flex.yaml rename to src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/flow.flex.yaml diff --git a/src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/flow.flex.yml b/src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/flow.flex.yml new file mode 100644 index 00000000000..7f255e988c9 --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/multiple_flex_yaml/flow.flex.yml @@ -0,0 +1 @@ +entry: entry:my_flow \ No newline at end of file diff --git a/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/flow.dag.yaml b/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/flow.dag.yaml deleted file mode 100644 index 0d20088a874..00000000000 --- a/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/flow.dag.yaml +++ /dev/null @@ -1,16 +0,0 @@ -inputs: - name: - type: string - default: hod -outputs: - result: - type: string - reference: ${hello_world.output} -nodes: -- name: hello_world - type: python - source: - type: code - path: hello_world.py - inputs: - name: ${inputs.name} diff --git a/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/hello_world.py b/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/hello_world.py deleted file mode 100644 index e0fa71d6796..00000000000 --- a/src/promptflow/tests/test_configs/eager_flows/mutiple_flow_yaml/hello_world.py +++ /dev/null @@ -1,6 +0,0 @@ -from promptflow import tool - - -@tool -def hello_world(name: str) -> str: - return f"Hello World {name}!" diff --git a/src/promptflow-azure/tests/unittests/test.py b/src/promptflow/tests/test_configs/eager_flows/simple_with_yml/entry.py similarity index 60% rename from src/promptflow-azure/tests/unittests/test.py rename to src/promptflow/tests/test_configs/eager_flows/simple_with_yml/entry.py index 226a2ed4261..7c62d1ce52b 100644 --- a/src/promptflow-azure/tests/unittests/test.py +++ b/src/promptflow/tests/test_configs/eager_flows/simple_with_yml/entry.py @@ -2,10 +2,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -import pytest - - -@pytest.mark.unittest -class TestStartTrace: - def test_import(self): - assert True +def my_flow(input_val: str = "gpt") -> str: + """Simple flow without yaml.""" + return f"Hello world! {input_val}" diff --git a/src/promptflow/tests/test_configs/eager_flows/simple_with_yml/flow.flex.yml b/src/promptflow/tests/test_configs/eager_flows/simple_with_yml/flow.flex.yml new file mode 100644 index 00000000000..7f255e988c9 --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/simple_with_yml/flow.flex.yml @@ -0,0 +1 @@ +entry: entry:my_flow \ No newline at end of file diff --git a/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/echo.py b/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/echo.py new file mode 100644 index 00000000000..5abdba40952 --- /dev/null +++ b/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/echo.py @@ -0,0 +1,6 @@ +from promptflow.core import tool + + +@tool +def echo(input: str) -> str: + return input diff --git a/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/flow.dag.yaml b/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/flow.dag.yaml new file mode 100644 index 00000000000..7eda4dcaf7f --- /dev/null +++ b/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/flow.dag.yaml @@ -0,0 +1,41 @@ +inputs: + topic: + type: string + default: hello world + is_chat_input: false + stream: + type: bool + default: false + is_chat_input: false +outputs: + joke: + type: string + reference: ${echo.output} +nodes: +- name: echo + type: python + source: + type: code + path: echo.py + inputs: + input: ${joke.output} + use_variants: false +- name: joke + type: llm + source: + type: code + path: joke.jinja2 + inputs: + deployment_name: gpt-35-turbo + temperature: 1 + top_p: 1 + max_tokens: 256 + presence_penalty: 0 + frequency_penalty: 0 + stream: ${inputs.stream} + topic: ${inputs.topic} + provider: AzureOpenAI + connection: azure_open_ai_missing_connection + api: chat + module: promptflow.tools.aoai + use_variants: false diff --git a/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/joke.jinja2 b/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/joke.jinja2 new file mode 100644 index 00000000000..de4a4a5ab93 --- /dev/null +++ b/src/promptflow/tests/test_configs/flows/llm_tool_non_existing_connection/joke.jinja2 @@ -0,0 +1,9 @@ +{# Prompt is a jinja2 template that generates prompt for LLM #} + +system: + +You are a bot can tell good jokes + +user: + +A joke about {{topic}} please