diff --git a/.cspell.json b/.cspell.json
index 1e86c090af7..35a190a2805 100644
--- a/.cspell.json
+++ b/.cspell.json
@@ -31,6 +31,8 @@
     "src/promptflow-azure/tests/**",
     "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
+    "src/promptflow-devkit/tests/**",
+    "src/promptflow-azure/tests/**",
     "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
     "src/promptflow-devkit/promptflow/_sdk/_service/static/index.html",
diff --git a/.github/actions/step_generate_configs/action.yml b/.github/actions/step_generate_configs/action.yml
index b0fc838cbe4..9e2a3480f36 100644
--- a/.github/actions/step_generate_configs/action.yml
+++ b/.github/actions/step_generate_configs/action.yml
@@ -12,6 +12,8 @@ runs:
     shell: pwsh
     run: |
       pip list
+      pip install azure-identity
+      pip install azure-keyvault
       echo "Generating connection config file..."
       python ./scripts/building/generate_connection_config.py `
           --target_folder ${{ inputs.targetFolder }}
diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml
index fe00424371c..3831e1d3186 100644
--- a/.github/workflows/promptflow-global-config-test.yml
+++ b/.github/workflows/promptflow-global-config-test.yml
@@ -4,16 +4,23 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request_target:
     paths:
-      - src/promptflow-core/*
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
+      - src/promptflow-tracing/**
+      - src/promptflow-azure/**
       - src/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-global-config-test.yml
   workflow_dispatch:
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+  AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
 jobs:
   authorize:
     environment:
@@ -41,70 +48,60 @@ jobs:
       uses: "./.github/actions/step_merge_main"
     - name: Display and Set Environment Variables
       run: |
-        if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then
-          export pyVersion="3.9";
-        elif [ "macos-latest" == "${{ matrix.os }}" ]; then
-          export pyVersion="3.10";
-        else
-          echo "Unsupported OS: ${{ matrix.os }}";
-          exit 1;
-        fi
+        export pyVersion="3.9"
         env | sort >> $GITHUB_OUTPUT
       id: display_env
       shell: bash -el {0}
-    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
-      uses: "./.github/actions/step_create_python_environment"
+    - uses: actions/setup-python@v5
       with:
-        pythonVersion: ${{ steps.display_env.outputs.pyVersion }}
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: ${{ env.packageSetupType }}
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Install dependency
-      shell: pwsh
+        python-version: ${{ steps.display_env.outputs.pyVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
+      working-directory: ${{ env.WORKING_DIRECTORY }}
       run: |
-        pip uninstall -y promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-tracing
-        echo "Installed promptflow-tracing"
-        pip uninstall -y promptflow-core
-        pip install ${{ github.workspace }}/src/promptflow-core
-        pip uninstall -y promptflow-devkit
-        pip install ${{ github.workspace }}/src/promptflow-devkit
-        pip uninstall -y promptflow-azure
-        pip install ${{ github.workspace }}/src/promptflow-azure
-        pip freeze
+        set -xe
+        poetry install --only test
+        poetry run pip install ${{ env.TRACING_DIRECTORY }}
+        poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
+        poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
+        poetry run pip install -e ${{ env.AZURE_DIRECTORY }}
+
+        echo "Need to install promptflow to avoid tool dependency issue"
+        poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
+        poetry run pip install ${{ env.TOOL_DIRECTORY }}
+        poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-azure
+        poetry run pip show promptflow-tools
     - name: Azure Login
       uses: azure/login@v1
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: Install Azure Login items
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+      run: |
+        pip install azure-identity
+        pip install azure-keyvault
     - name: Generate Configs
       uses: "./.github/actions/step_generate_configs"
       with:
-        targetFolder: ${{ env.testWorkingDirectory }}
-    - name: Get number of CPU cores
-      uses: SimenB/github-actions-cpu-cores@v1
-      id: cpu-cores
-    - name: Run Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+    - name: run devkit tests
       run: |
-        gci env:* | sort-object name
-        az account show
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_global_config_test `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -n ${{ steps.cpu-cores.outputs.count }} `
+        poetry run pytest ./tests/sdk_cli_global_config_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest"
+      working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results
       if: always()
       uses: actions/upload-artifact@v3
       with:
         name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }})
         path: |
-          ${{ env.testWorkingDirectory }}/*.xml
-          ${{ env.testWorkingDirectory }}/htmlcov/
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
   publish-test-results-global-config-test:
     needs: sdk_cli_global_config_tests
     runs-on: ubuntu-latest
@@ -117,7 +114,7 @@ jobs:
 
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha || github.ref }}
     - name: Publish Test Results
diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml
index 9215cbc760b..5d85fec622f 100644
--- a/.github/workflows/promptflow-sdk-cli-test.yml
+++ b/.github/workflows/promptflow-sdk-cli-test.yml
@@ -13,43 +13,15 @@ on:
       - src/promptflow-recording/**
   workflow_dispatch:
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
 jobs:
-  build:
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Display and Set Environment Variables
-      run: |
-        env | sort >> $GITHUB_OUTPUT
-      id: display_env
-      shell: bash -el {0}
-    - name: Python Setup - ubuntu-latest - Python Version 3.9
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: 3.9
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: promptflow_with_extra
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Upload Wheel
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: wheel
-        path: |
-          ${{ github.workspace }}/src/promptflow/dist/*.whl
-          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
   sdk_cli_tests:
-    needs: build
     strategy:
       fail-fast: false
       matrix:
@@ -59,39 +31,29 @@ jobs:
     steps:
     - name: set test mode
       run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Display and Set Environment Variables
-      run: |
-        env | sort >> $GITHUB_OUTPUT
-      id: display_env
-      shell: bash -el {0}
-    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: ${{ matrix.pythonVersion }}
-    - name: Download Artifacts
-      uses: actions/download-artifact@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        name: wheel
-        path: artifacts
-    - name: Install wheel
-      shell: pwsh
-      working-directory: artifacts
-      run: |
-        Set-PSDebug -Trace 1
-        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        pip install ${{ github.workspace }}/src/promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-core
-        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
-        pip install ${{ github.workspace }}/src/promptflow
-        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip freeze
-    - name: install recording
+        python-version: ${{ matrix.pythonVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
       run: |
-        pip install vcrpy
-        pip install .
-      working-directory: ${{ env.RECORD_DIRECTORY }}
+        set -xe
+        poetry install --only test
+        poetry run pip install ${{ env.TRACING_DIRECTORY }}
+        poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
+        poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
+
+        echo "Need to install promptflow to avoid tool dependency issue"
+        poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
+        poetry run pip install ${{ env.TOOL_DIRECTORY }}
+        poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-tools
+      working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Azure login (non pull_request workflow)
       if: github.event_name != 'pull_request'
       uses: azure/login@v1
@@ -101,62 +63,33 @@ jobs:
       if: github.event_name != 'pull_request'
       uses: "./.github/actions/step_generate_configs"
       with:
-        targetFolder: ${{ env.testWorkingDirectory }}
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
     - name: generate live test resources (pull_request workflow)
       if: github.event_name == 'pull_request'
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
+      working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
       run: |
         cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
-    - name: Run SDK CLI Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
-          -l eastus `
-          -m "unittest or e2etest" `
-          --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc `
-          -o "${{ env.testWorkingDirectory }}/test-results-sdk-cli.xml" `
-          --ignore-glob ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
-    - name: Install pf executable
-      shell: pwsh
-      working-directory: artifacts
+    - name: run devkit tests
       run: |
-        Set-PSDebug -Trace 1
-        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable]
-        pip freeze
-    - name: Run SDK CLI Executable Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -o "${{ env.testWorkingDirectory }}/test-results-sdk-cli-executable.xml"
-    - name: Run PFS Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_pfs_test `
-          -l eastus `
-          -m "e2etest" `
-          --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_pfs_test/.coveragerc `
-          -o "${{ env.testWorkingDirectory }}/test-results-pfs.xml"
+        poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \
+        --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
+      working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results
       if: always()
       uses: actions/upload-artifact@v3
       with:
         name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
         path: |
-          ${{ env.testWorkingDirectory }}/*.xml
-          ${{ env.testWorkingDirectory }}/htmlcov/
-          ${{ env.testWorkingDirectory }}/tests/sdk_cli_test/count.json
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
+          ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
+    - run: poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[executable]
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: run devkit executable tests
+      run: |
+        poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py
+      working-directory: ${{ env.WORKING_DIRECTORY }}
 
 
   publish-test-results-sdk-cli-test:
diff --git a/.github/workflows/sdk-cli-azure-test-pull-request.yml b/.github/workflows/sdk-cli-azure-test-pull-request.yml
index c509eceb3d5..9477d537416 100644
--- a/.github/workflows/sdk-cli-azure-test-pull-request.yml
+++ b/.github/workflows/sdk-cli-azure-test-pull-request.yml
@@ -6,56 +6,29 @@ name: sdk-cli-azure-test-pull-request
 on:
   pull_request:
     paths:
-      - src/promptflow-core/**
-      - src/promptflow-devkit/**
       - src/promptflow/**
       - scripts/building/**
-      - src/promptflow-tracing/**
       - .github/workflows/sdk-cli-azure-test-pull-request.yml
+      - src/promptflow-tracing/**
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
+      - src/promptflow-azure/**
 
 
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
   PROMPT_FLOW_TEST_MODE: "replay"
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 
 jobs:
-  build:
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Display and Set Environment Variables
-      run: |
-        env | sort >> $GITHUB_OUTPUT
-      id: display_env
-      shell: bash -el {0}
-    - name: Python Setup - ubuntu-latest - Python Version 3.9
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: 3.9
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: promptflow_with_extra
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Upload Wheel
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: wheel
-        path: |
-          ${{ github.workspace }}/src/promptflow/dist/*.whl
-          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
-
   sdk_cli_azure_test_replay:
-    needs: build
     strategy:
       fail-fast: false
       matrix:
@@ -71,47 +44,42 @@ jobs:
       - name: Display and Set Environment Variables
         run: env | sort >> $GITHUB_OUTPUT
 
-      - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-        uses: "./.github/actions/step_create_python_environment"
-        with:
-          pythonVersion: ${{ matrix.pythonVersion }}
-
-      - name: Download Artifacts
-        uses: actions/download-artifact@v3
+      - uses: actions/setup-python@v5
         with:
-          name: wheel
-          path: artifacts
-
-      - name: Install wheel
-        shell: pwsh
-        working-directory: artifacts
+          python-version: ${{ matrix.pythonVersion }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
         run: |
-          Set-PSDebug -Trace 1
-          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-          pip install ${{ github.workspace }}/src/promptflow-tracing
-          pip install ${{ github.workspace }}/src/promptflow-core
-          pip install ${{ github.workspace }}/src/promptflow-devkit
-          pip install ${{ github.workspace }}/src/promptflow-azure
-          gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
-          gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-          pip freeze
-
-      - name: install recording
+          set -xe
+          poetry install --only test
+          poetry run pip install ${{ env.TRACING_DIRECTORY }}
+          poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
+          poetry run pip install ${{ env.DEVKIT_DIRECTORY }}[pyarrow]
+          poetry run pip install -e ${{ env.WORKING_DIRECTORY }}
+
+          echo "Need to install promptflow to avoid tool dependency issue"
+          poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
+          poetry run pip install ${{ env.TOOL_DIRECTORY }}
+          poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
+          poetry run pip show promptflow-devkit
+          poetry run pip show promptflow-azure
+          poetry run pip show promptflow-tools
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+
+      - name: generate live test resources
+        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
         run: |
-          pip install vcrpy
-          pip install -e .
-        working-directory: ${{ env.RECORD_DIRECTORY }}
+          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
 
       - name: Run SDK CLI Azure Test (replay mode)
         shell: pwsh
-        working-directory: ${{ env.testWorkingDirectory }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
-          python "../../scripts/building/run_coverage_tests.py" `
-            -p promptflow `
-            -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test `
-            -l eastus `
-            -m "unittest or e2etest" `
-            --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc
+          poetry run pytest ./tests/sdk_cli_azure_test -p promptflow --cov=promptflow --cov-config=pyproject.toml `
+          --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest"
 
       - name: Upload Test Results
         if: always()
@@ -119,8 +87,8 @@ jobs:
         with:
           name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
           path: |
-            ${{ env.testWorkingDirectory }}/*.xml
-            ${{ env.testWorkingDirectory }}/htmlcov/
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
 
   publish-test-results-sdk-cli-azure-test:
     needs: sdk_cli_azure_test_replay
diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml
index f3aa9f2f821..a66a02b35a3 100644
--- a/.github/workflows/sdk-cli-perf-monitor-test.yml
+++ b/.github/workflows/sdk-cli-perf-monitor-test.yml
@@ -9,6 +9,7 @@ on:
       - src/promptflow/**
       - src/promptflow-core/**
       - src/promptflow-devkit/**
+      - src/promptflow-azure/**
       - scripts/building/**
       - .github/workflows/sdk-cli-perf-monitor-test.yml
 
@@ -19,20 +20,25 @@ on:
 
 
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
   PROMPT_FLOW_TEST_MODE: "replay"
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
-
 jobs:
   sdk_cli_perf_monitor_test:
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -50,63 +56,54 @@ jobs:
           export pyVersion="3.9";
           env | sort >> $GITHUB_OUTPUT
         id: display_env
-        shell: bash -el {0}
-
-      - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
-        uses: "./.github/actions/step_create_python_environment"
-        with:
-          pythonVersion: ${{ steps.display_env.outputs.pyVersion }}
 
-      - name: Build wheel
-        uses: "./.github/actions/step_sdk_setup"
-        with:
-          setupType: promptflow_with_extra
-          scriptPath: ${{ env.testWorkingDirectory }}
-      - name: Upload Wheel
-        if: always()
-        uses: actions/upload-artifact@v3
+      - uses: actions/setup-python@v5
         with:
-          name: wheel
-          path: |
-            ${{ github.workspace }}/src/promptflow/dist/*.whl
-            ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
-      - name: Download Artifacts
-        uses: actions/download-artifact@v3
+          python-version: ${{ steps.display_env.outputs.pyVersion }}
+      - uses: snok/install-poetry@v1
         with:
-          name: wheel
-          path: artifacts
-      - name: Install wheel
-        shell: pwsh
-        working-directory: artifacts
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - name: install test dependency group
+        working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
-          Set-PSDebug -Trace 1
-          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-          pip install ${{ github.workspace }}/src/promptflow-tracing
-          pip install ${{ github.workspace }}/src/promptflow-core
-          pip install ${{ github.workspace }}/src/promptflow-devkit
-          pip install ${{ github.workspace }}/src/promptflow-azure
-          gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[all]"}}
-          gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-          pip freeze
-
-      - name: install recording
-        run:
-          pip install vcrpy
-          pip install -e .
-        working-directory: ${{ env.RECORD_DIRECTORY }}
+          poetry install --with test
+      - run: |
+          source .venv/scripts/activate
+          pytest --version
+        if: runner.os == 'Windows'
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - run: |
+          source .venv/bin/activate
+          pytest --version
+        if: runner.os != 'Windows'
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - run: |
+          set -xe
+          poetry run pip install ../promptflow-tracing
+          poetry run pip install ../promptflow-core[azureml-serving]
+          poetry run pip install ../promptflow-devkit[pyarrow]
+          poetry run pip install ../promptflow-azure
+
+          echo "Need to install promptflow to avoid tool dependency issue"
+          poetry run pip install ../promptflow
+          poetry run pip install ../promptflow-tools
+          poetry run pip install ../promptflow-recording
+
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
+          poetry run pip show promptflow-devkit
+          poetry run pip show promptflow-azure
+          poetry run pip show promptflow-tools
+        working-directory: ${{ env.WORKING_DIRECTORY }}
 
       - name: Generate (mock) connections.json
         shell: pwsh
-        working-directory: ${{ env.testWorkingDirectory }}
+        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
         run: cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
 
       - name: Run Test
-        shell: pwsh
-        working-directory: ${{ env.testWorkingDirectory }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
-          gci env:* | sort-object name
-          python "../../scripts/building/run_coverage_tests.py" `
-            -p promptflow `
-            -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
-            -l eastus `
-            -m "perf_monitor_test"
+          poetry run pytest ./tests/sdk_cli_azure_test ../promptflow-azure/tests/sdk_cli_azure_test -n auto -m "perf_monitor_test"
+
diff --git a/scripts/check_enforcer/check_enforcer.py b/scripts/check_enforcer/check_enforcer.py
index 6755b95f77d..ab3974943d7 100644
--- a/scripts/check_enforcer/check_enforcer.py
+++ b/scripts/check_enforcer/check_enforcer.py
@@ -43,19 +43,27 @@
 # Copy from original yaml pipelines
 checks = {
     "sdk_cli_tests": [
+        "src/promptflow-core/**",
+        "src/promptflow-devkit/**",
         "src/promptflow/**",
+        "src/promptflow-tracing/**",
         "scripts/building/**",
         ".github/workflows/promptflow-sdk-cli-test.yml",
+        "src/promptflow-recording/**",
     ],
-    "sdk_cli_global_config_tests": [
-        "src/promptflow/**",
-        "scripts/building/**",
-        ".github/workflows/promptflow-global-config-test.yml",
-    ],
+    # "sdk_cli_global_config_tests": [
+    #     "src/promptflow/**",
+    #     "scripts/building/**",
+    #     ".github/workflows/promptflow-global-config-test.yml",
+    # ],
     "sdk_cli_azure_test_replay": [
         "src/promptflow/**",
         "scripts/building/**",
-        ".github/workflows/promptflow-sdk-cli-azure-test-pull-request.yml",
+        ".github/workflows/sdk-cli-azure-test-pull-request.yml",
+        "src/promptflow-tracing/**",
+        "src/promptflow-core/**",
+        "src/promptflow-devkit/**",
+        "src/promptflow-azure/**",
     ],
 }
 
diff --git a/src/promptflow-azure/pyproject.toml b/src/promptflow-azure/pyproject.toml
index 9f7b310c009..a950a1483ee 100644
--- a/src/promptflow-azure/pyproject.toml
+++ b/src/promptflow-azure/pyproject.toml
@@ -55,6 +55,11 @@ import-linter = "*"
 pytest = "*"
 pytest-cov = "*"
 pytest-xdist = "*"
+pytest-mock = "*"
+pytest-asyncio = "*"
+mock = "*"
+keyrings-alt = "*"
+bs4 = "*"
 
 [build-system]
 requires = ["poetry-core>=1.5.0"]
@@ -66,16 +71,12 @@ pfazure = "promptflow.azure._cli.entry:main"
 [tool.pytest.ini_options]
 markers = [
   "unittest",
+  "e2etest"
 ]
 # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action)
 # durations - list the slowest test durations
 addopts = """
 --junit-xml=test-results.xml \
---cov=promptflow \
---cov-config=pyproject.toml \
---cov-report=term \
---cov-report=html \
---cov-report=xml \
 --dist loadfile \
 --log-level=info \
 --log-format="%(asctime)s %(levelname)s %(message)s" \
@@ -87,8 +88,22 @@ addopts = """
 testpaths = ["tests"]
 
 [tool.coverage.run]
+source = [
+  "*/promptflow/azure/*"
+]
 omit = [
-  "__init__.py",
+  "*/__init__.py",
+  "*/promptflow/azure/_restclient/*",
+  "*/promptflow/azure/_models/*",
+  "*/promptflow/azure/_cli/*",
+  "*/promptflow/recording/*",
+  "*/promptflow/tracing/*",
+  "*/promptflow/tools/*",
+  "*/promptflow/executor/*",
+  "*/promptflow/core/*",
+  "*/promptflow/_sdk/_service/*",
+  "*/promptflow/_orchestrator/*",
+  "*/promptflow/_cli/*",
 ]
 
 [tool.black]
diff --git a/src/promptflow-azure/tests/_constants.py b/src/promptflow-azure/tests/_constants.py
new file mode 100644
index 00000000000..9929a01ff68
--- /dev/null
+++ b/src/promptflow-azure/tests/_constants.py
@@ -0,0 +1,14 @@
+from pathlib import Path
+
+PROMPTFLOW_ROOT = Path(__file__).parent.parent.parent / "promptflow"
+RUNTIME_TEST_CONFIGS_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/runtime")
+CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
+ENV_FILE = (PROMPTFLOW_ROOT / ".env").resolve().absolute().as_posix()
+
+# below constants are used for pfazure and global config tests
+DEFAULT_SUBSCRIPTION_ID = "96aede12-2f73-41cb-b983-6d11a904839b"
+DEFAULT_RESOURCE_GROUP_NAME = "promptflow"
+DEFAULT_WORKSPACE_NAME = "promptflow-eastus2euap"
+DEFAULT_COMPUTE_INSTANCE_NAME = "ci-lin-cpu-sp"
+DEFAULT_RUNTIME_NAME = "test-runtime-ci"
+DEFAULT_REGISTRY_NAME = "promptflow-preview"
diff --git a/src/promptflow-azure/tests/conftest.py b/src/promptflow-azure/tests/conftest.py
new file mode 100644
index 00000000000..290bb420cb2
--- /dev/null
+++ b/src/promptflow-azure/tests/conftest.py
@@ -0,0 +1,74 @@
+import json
+import os
+import tempfile
+from pathlib import Path
+
+import pytest
+from _constants import CONNECTION_FILE, PROMPTFLOW_ROOT
+from _pytest.monkeypatch import MonkeyPatch
+from dotenv import load_dotenv
+from pytest_mock import MockerFixture
+
+from promptflow._constants import PROMPTFLOW_CONNECTIONS
+from promptflow._core.connection_manager import ConnectionManager
+from promptflow._sdk.entities._connection import AzureOpenAIConnection
+from promptflow._utils.context_utils import _change_working_dir
+
+load_dotenv()
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_build_info():
+    """Mock BUILD_INFO environment variable in pytest.
+
+    BUILD_INFO is set as environment variable in docker image, but not in local test.
+    So we need to mock it in test senario. Rule - build_number is set as
+    ci-<BUILD_BUILDNUMBER> in CI pipeline, and set as local in local dev test."""
+    if "BUILD_INFO" not in os.environ:
+        m = MonkeyPatch()
+        build_number = os.environ.get("BUILD_BUILDNUMBER", "")
+        buid_info = {"build_number": f"ci-{build_number}" if build_number else "local-pytest"}
+        m.setenv("BUILD_INFO", json.dumps(buid_info))
+        yield m
+
+
+@pytest.fixture
+def use_secrets_config_file(mocker: MockerFixture):
+    mocker.patch.dict(os.environ, {PROMPTFLOW_CONNECTIONS: CONNECTION_FILE})
+
+
+@pytest.fixture
+def azure_open_ai_connection() -> AzureOpenAIConnection:
+    return ConnectionManager().get("azure_open_ai_connection")
+
+
+@pytest.fixture
+def temp_output_dir() -> str:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        yield temp_dir
+
+
+@pytest.fixture
+def prepare_symbolic_flow() -> str:
+    flows_dir = PROMPTFLOW_ROOT / "tests" / "test_configs" / "flows"
+    target_folder = flows_dir / "web_classification_with_symbolic"
+    source_folder = flows_dir / "web_classification"
+
+    with _change_working_dir(target_folder):
+
+        for file_name in os.listdir(source_folder):
+            if not Path(file_name).exists():
+                os.symlink(source_folder / file_name, file_name)
+    return target_folder
+
+
+@pytest.fixture
+def enable_logger_propagate():
+    """This is for test cases that need to check the log output."""
+    from promptflow._utils.logger_utils import get_cli_sdk_logger
+
+    logger = get_cli_sdk_logger()
+    original_value = logger.propagate
+    logger.propagate = True
+    yield
+    logger.propagate = original_value
diff --git a/src/promptflow/tests/sdk_cli_azure_test/__init__.py b/src/promptflow-azure/tests/sdk_cli_azure_test/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/__init__.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/_azure_utils.py b/src/promptflow-azure/tests/sdk_cli_azure_test/_azure_utils.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/_azure_utils.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/_azure_utils.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/conftest.py b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_azure_test/conftest.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py
index 40bc9231e54..0dde0a79cca 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/conftest.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py
@@ -44,13 +44,16 @@ def is_replay():
         return False
 
 
+from _constants import PROMPTFLOW_ROOT
+
 from ._azure_utils import get_cred
 
-FLOWS_DIR = "./tests/test_configs/flows"
-EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows"
-DATAS_DIR = "./tests/test_configs/datas"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
 AZUREML_RESOURCE_PROVIDER = "Microsoft.MachineLearningServices"
 RESOURCE_ID_FORMAT = "/subscriptions/{}/resourceGroups/{}/providers/{}/workspaces/{}"
+MODEL_ROOT = FLOWS_DIR
 
 
 def pytest_configure():
@@ -217,10 +220,6 @@ def runtime(runtime_name: str) -> str:
     return runtime_name
 
 
-PROMPTFLOW_ROOT = Path(__file__) / "../../.."
-MODEL_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/flows")
-
-
 @pytest.fixture
 def flow_serving_client_remote_connection(mocker: MockerFixture, remote_workspace_resource_id):
     from promptflow.core._serving.app import create_app as create_serving_app
@@ -445,7 +444,7 @@ def mock_get_user_identity_info(user_object_id: str, tenant_id: str) -> None:
 def created_flow(pf: PFClient, randstr: Callable[[str], str], variable_recorder) -> Flow:
     """Create a flow for test."""
     flow_display_name = randstr("flow_display_name")
-    flow_source = FLOWS_DIR + "/simple_hello_world/"
+    flow_source = FLOWS_DIR / "simple_hello_world"
     description = "test flow description"
     tags = {"owner": "sdk-test"}
     result = pf.flows.create_or_update(
@@ -610,5 +609,5 @@ def mock_check_latest_version() -> None:
     As CI uses docker, it will always trigger this check behavior, and we don't have recording for this;
     and this will hit many unknown issue with vcrpy.
     """
-    with patch("promptflow._utils.version_hint_utils.check_latest_version", new=lambda: None):
+    with patch("promptflow._sdk._version_hint_utils.check_latest_version", new=lambda: None):
         yield
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/__init__.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/__init__.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/classificationAccuracy.csv
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_arm_connection_operations.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py
similarity index 97%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py
index 2d247dec949..64393c271fc 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_azure_cli_perf.py
@@ -5,14 +5,12 @@
 from unittest import mock
 
 import pytest
+from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR
 
 from promptflow._cli._user_agent import USER_AGENT as CLI_USER_AGENT  # noqa: E402
 from promptflow._sdk._telemetry import log_activity
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
 
-FLOWS_DIR = "./tests/test_configs/flows"
-DATAS_DIR = "./tests/test_configs/datas"
-
 
 def mock_log_activity(*args, **kwargs):
     custom_message = "github run: https://github.com/microsoft/promptflow/actions/runs/{0}".format(
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli.py
similarity index 92%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli.py
index f586e8a87c3..298880ff3dc 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli.py
@@ -4,13 +4,13 @@
 from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
+from sdk_cli_azure_test.conftest import FLOWS_DIR
 
 from promptflow._cli._pf.entry import main
 
-FLOWS_DIR = "./tests/test_configs/flows"
-RUNS_DIR = "./tests/test_configs/runs"
-CONNECTIONS_DIR = "./tests/test_configs/connections"
-DATAS_DIR = "./tests/test_configs/datas"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
+CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections"
 
 
 # TODO: move this to a shared utility module
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py
similarity index 97%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py
index 3c313d74ce9..4975ea04498 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_cli_with_azure.py
@@ -8,7 +8,9 @@
 from typing import Callable
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from mock.mock import patch
+from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR
 
 from promptflow._constants import PF_USER_AGENT
 from promptflow._sdk.entities import Run
@@ -19,9 +21,7 @@
 
 from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
 
-FLOWS_DIR = "./tests/test_configs/flows"
-DATAS_DIR = "./tests/test_configs/datas"
-RUNS_DIR = "./tests/test_configs/runs"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
 
 
 # TODO: move this to a shared utility module
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_connection_operations.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py
similarity index 90%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py
index 1267edae5a9..7f11dcb2c8e 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_in_azure_ml.py
@@ -3,18 +3,13 @@
 
 import pydash
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._utils.yaml_utils import dump_yaml, load_yaml_string
 from promptflow.connections import AzureOpenAIConnection
 
 from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-
 
 def assert_dict_equals_with_skip_fields(item1, item2, skip_fields):
     for fot_key in skip_fields:
@@ -106,7 +101,7 @@ def test_flow_as_component(
         # keep the simplest test here, more tests are in azure-ai-ml
         from azure.ai.ml import load_component
 
-        flows_dir = "./tests/test_configs/flows"
+        flows_dir = PROMPTFLOW_ROOT / "tests/test_configs/flows"
 
         flow_func: Component = load_component(
             f"{flows_dir}/web_classification/flow.dag.yaml", params_override=[load_params]
@@ -114,10 +109,9 @@ def test_flow_as_component(
 
         # TODO: snapshot of flow component changed every time?
         created_component = ml_client.components.create_or_update(flow_func, is_anonymous=True)
+        spec_path = flows_dir / "saved_component_spec" / f"{request.node.callspec.id}.yaml"
 
-        update_saved_spec(
-            created_component, f"./tests/test_configs/flows/saved_component_spec/{request.node.callspec.id}.yaml"
-        )
+        update_saved_spec(created_component, spec_path.resolve().absolute().as_posix())
 
         component_dict = created_component._to_dict()
         slimmed_created_component_attrs = {key: pydash.get(component_dict, key) for key in expected_spec_attrs.keys()}
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py
similarity index 91%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py
index 103c74b318d..c2d33b1481a 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py
@@ -2,19 +2,15 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import json
-from pathlib import Path
 
 import pytest
+from sdk_cli_azure_test.conftest import FLOWS_DIR
 
 from promptflow.azure._entities._flow import Flow
 from promptflow.exceptions import UserErrorException
 
 from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
 
-tests_root_dir = Path(__file__).parent.parent.parent
-flow_test_dir = tests_root_dir / "test_configs/flows"
-data_dir = tests_root_dir / "test_configs/datas"
-
 
 @pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD)
 @pytest.mark.e2etest
@@ -64,7 +60,7 @@ def test_flow_test_with_config(self, remote_workspace_resource_id):
         from promptflow import PFClient
 
         client = PFClient(config={"connection.provider": remote_workspace_resource_id})
-        output = client.test(flow=flow_test_dir / "web_classification")
+        output = client.test(flow=FLOWS_DIR / "web_classification")
         assert output.keys() == {"category", "evidence"}
 
     @pytest.mark.usefixtures("mock_get_user_identity_info")
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_serve.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_run_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_run_operations.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py
index 33be3624166..e8874b2073b 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_run_operations.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py
@@ -16,8 +16,10 @@
 import pandas as pd
 import pydash
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from azure.ai.ml import ManagedIdentityConfiguration
 from azure.ai.ml.entities import IdentityConfiguration
+from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR
 
 from promptflow._constants import FLOW_FLEX_YAML
 from promptflow._sdk._configuration import Configuration
@@ -40,15 +42,8 @@
 
 from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-FLOWS_DIR = "./tests/test_configs/flows"
-EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows"
-RUNS_DIR = "./tests/test_configs/runs"
-DATAS_DIR = "./tests/test_configs/datas"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
 
 
 def create_registry_run(name: str, registry_name: str, runtime: str, pf: PFClient):
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_telemetry.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_telemetry.py
similarity index 88%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_telemetry.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_telemetry.py
index 08954802339..04d0f06cdb9 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_telemetry.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_telemetry.py
@@ -15,6 +15,8 @@
 
 import pydash
 import pytest
+from _constants import PROMPTFLOW_ROOT
+from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR
 
 from promptflow import load_run
 from promptflow._constants import PF_USER_AGENT
@@ -36,6 +38,9 @@
 
 from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
 
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
+
 
 @contextlib.contextmanager
 def cli_consent_config_overwrite(val):
@@ -65,10 +70,6 @@ def extension_consent_config_overwrite(val):
             config.set_config(key=Configuration.EXTENSION_COLLECT_TELEMETRY, value=True)
 
 
-RUNS_DIR = "./tests/test_configs/runs"
-FLOWS_DIR = "./tests/test_configs/flows"
-
-
 @pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD)
 @pytest.mark.usefixtures("mock_set_headers_with_user_aml_token", "single_worker_thread_pool", "vcr_recording")
 @pytest.mark.e2etest
@@ -125,47 +126,51 @@ def check_evelope():
             assert isinstance(custom_dimensions, dict)
             # Note: need privacy review if we add new fields.
             if "start" in envelope.data.base_data.name:
-                assert custom_dimensions.keys() == {
-                    "request_id",
-                    "activity_name",
-                    "activity_type",
-                    "subscription_id",
-                    "resource_group_name",
-                    "workspace_name",
-                    "level",
-                    "python_version",
-                    "user_agent",
-                    "installation_id",
-                    "first_call",
-                    "from_ci",
-                    "error_category",
-                    "error_type",
-                    "error_target",
-                    "error_message",
-                    "error_detail",
-                }
+                assert sorted(custom_dimensions.keys()) == sorted(
+                    {
+                        "request_id",
+                        "activity_name",
+                        "activity_type",
+                        "subscription_id",
+                        "resource_group_name",
+                        "workspace_name",
+                        "level",
+                        "python_version",
+                        "user_agent",
+                        "installation_id",
+                        "first_call",
+                        "from_ci",
+                        "error_category",
+                        "error_type",
+                        "error_target",
+                        "error_message",
+                        "error_detail",
+                    }
+                )
             elif "complete" in envelope.data.base_data.name:
-                assert custom_dimensions.keys() == {
-                    "request_id",
-                    "activity_name",
-                    "activity_type",
-                    "subscription_id",
-                    "resource_group_name",
-                    "workspace_name",
-                    "completion_status",
-                    "duration_ms",
-                    "level",
-                    "python_version",
-                    "user_agent",
-                    "installation_id",
-                    "first_call",
-                    "from_ci",
-                    "error_category",
-                    "error_type",
-                    "error_target",
-                    "error_message",
-                    "error_detail",
-                }
+                assert sorted(custom_dimensions.keys()) == sorted(
+                    {
+                        "request_id",
+                        "activity_name",
+                        "activity_type",
+                        "subscription_id",
+                        "resource_group_name",
+                        "workspace_name",
+                        "completion_status",
+                        "duration_ms",
+                        "level",
+                        "python_version",
+                        "user_agent",
+                        "installation_id",
+                        "first_call",
+                        "from_ci",
+                        "error_category",
+                        "error_type",
+                        "error_target",
+                        "error_message",
+                        "error_detail",
+                    }
+                )
             else:
                 raise ValueError("Invalid message: {}".format(envelope.data.base_data.name))
             assert envelope.data.base_data.name.startswith("pfazure.runs.get")
@@ -445,8 +450,8 @@ def check_evelope():
         ):
             flow_type = FlowType.DAG_FLOW
             pf.run(
-                flow="./tests/test_configs/flows/print_input_flow",
-                data="./tests/test_configs/datas/print_input_flow.jsonl",
+                flow=FLOWS_DIR / "print_input_flow",
+                data=DATAS_DIR / "print_input_flow.jsonl",
                 name=randstr("name"),
             )
             logger = get_telemetry_logger()
@@ -455,8 +460,8 @@ def check_evelope():
 
             flow_type = FlowType.FLEX_FLOW
             pf.run(
-                flow="./tests/test_configs/eager_flows/simple_with_req",
-                data="./tests/test_configs/datas/simple_eager_flow_data.jsonl",
+                flow=EAGER_FLOWS_DIR / "simple_with_req",
+                data=DATAS_DIR / "simple_eager_flow_data.jsonl",
                 name=randstr("name"),
             )
             logger.handlers[0].flush()
diff --git a/src/promptflow/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_workspace_connection_provider.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/__init__.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/__init__.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_blob_client.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_blob_client.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_blob_client.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_blob_client.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cli.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cli.py
similarity index 97%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_cli.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cli.py
index 25ec32e7b06..2cc95f4d4d9 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cli.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cli.py
@@ -8,13 +8,10 @@
 import pandas as pd
 import pytest
 from pytest_mock import MockFixture
+from sdk_cli_azure_test.conftest import FLOWS_DIR
 
 from promptflow._sdk._constants import VIS_PORTAL_URL_TMPL
 
-tests_root_dir = Path(__file__).parent.parent.parent
-flow_test_dir = tests_root_dir / "test_configs/flows"
-data_dir = tests_root_dir / "test_configs/datas"
-
 
 def run_pf_command(*args, cwd=None):
     from promptflow.azure._cli.entry import main
@@ -243,7 +240,7 @@ def test_flow_create(
 
         mocked = mocker.patch.object(FlowOperations, "create_or_update")
         mocked.return_value._to_dict.return_value = {"name": "test_run"}
-        flow_dir = Path(flow_test_dir, "web_classification").resolve().as_posix()
+        flow_dir = Path(FLOWS_DIR, "web_classification").resolve().as_posix()
         run_pf_command(
             "flow",
             "create",
@@ -269,7 +266,7 @@ def test_flow_create_with_unknown_field(self, mocker: MockFixture, operation_sco
 
         mocked = mocker.patch.object(FlowOperations, "create_or_update")
         mocked.return_value._to_dict.return_value = {"name": "test_run"}
-        flow_dir = Path(flow_test_dir, "web_classification").resolve().as_posix()
+        flow_dir = Path(FLOWS_DIR, "web_classification").resolve().as_posix()
         run_pf_command(
             "flow",
             "create",
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_collection.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_collection.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_collection.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_collection.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_config.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_config.py
similarity index 94%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_config.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_config.py
index 4d32e2cc6ab..caa19eed957 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_config.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_config.py
@@ -1,9 +1,9 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._configuration import ConfigFileNotFound, Configuration, InvalidConfigFile
 from promptflow._utils.context_utils import _change_working_dir
@@ -11,7 +11,8 @@
 AZUREML_RESOURCE_PROVIDER = "Microsoft.MachineLearningServices"
 RESOURCE_ID_FORMAT = "/subscriptions/{}/resourceGroups/{}/providers/{}/workspaces/{}"
 
-CONFIG_DATA_ROOT = Path(__file__).parent.parent.parent / "test_configs" / "configs"
+
+CONFIG_DATA_ROOT = PROMPTFLOW_ROOT / "tests/test_configs/configs"
 
 
 @pytest.fixture
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_cosmosdb_utils.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_exceptions.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_exceptions.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_exceptions.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_exceptions.py
index 1ce7c1206e7..607027349dd 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_exceptions.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_exceptions.py
@@ -12,8 +12,6 @@
 from promptflow.executor import FlowValidator
 from promptflow.executor._errors import InvalidNodeReference
 
-FLOWS_DIR = "./tests/test_configs/flows/print_input_flow"
-
 
 def is_match_error_detail(expected_info, actual_info):
     expected_info = re.sub(r"line \d+", r"", expected_info).replace("\n", "").replace(" ", "")
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_entity.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py
similarity index 95%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_entity.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py
index a1945ba230a..9ba0ce5629d 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_entity.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py
@@ -8,7 +8,9 @@
 from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from mock.mock import Mock
+from sdk_cli_azure_test.conftest import FLOWS_DIR
 
 from promptflow import load_run
 from promptflow._sdk._vendor import get_upload_files_from_folder
@@ -17,9 +19,7 @@
 from promptflow.azure._entities._flow import Flow
 from promptflow.exceptions import ValidationException
 
-tests_root_dir = Path(__file__).parent.parent.parent
-FLOWS_DIR = (tests_root_dir / "test_configs/flows").resolve()
-RUNS_DIR = (tests_root_dir / "test_configs/runs").resolve()
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
 
 
 def load_flow(source):
@@ -33,7 +33,7 @@ class TestFlow:
     @pytest.mark.skip(reason="TODO: add back when we bring back meta.yaml")
     def test_load_flow(self):
 
-        local_file = tests_root_dir / "test_configs/flows/meta_files/flow.meta.yaml"
+        local_file = FLOWS_DIR / "meta_files/flow.meta.yaml"
 
         flow = load_flow(source=local_file)
 
@@ -58,7 +58,7 @@ def test_load_flow(self):
     def test_load_flow_from_remote_storage(self):
         from promptflow.azure.operations._flow_operations import FlowOperations
 
-        local_file = tests_root_dir / "test_configs/flows/meta_files/remote_fs.meta.yaml"
+        local_file = FLOWS_DIR / "meta_files/remote_fs.meta.yaml"
 
         flow = load_flow(source=local_file)
 
@@ -79,7 +79,7 @@ def test_load_flow_from_remote_storage(self):
         }
 
     def test_ignore_files_in_flow(self):
-        local_file = tests_root_dir / "test_configs/flows/web_classification"
+        local_file = FLOWS_DIR / "web_classification"
         with tempfile.TemporaryDirectory() as temp:
             flow_path = Path(temp) / "flow"
             shutil.copytree(local_file, flow_path)
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_operations.py
similarity index 89%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_operations.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_operations.py
index 15831f80ea4..e1e55a27184 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_flow_operations.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_operations.py
@@ -1,20 +1,18 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from pathlib import Path
 from unittest.mock import patch
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
+from sdk_cli_azure_test.conftest import FLOWS_DIR
 
 from promptflow._sdk._constants import AzureFlowSource
 from promptflow._sdk._errors import FlowOperationError
 from promptflow.azure._entities._flow import Flow
 from promptflow.exceptions import UserErrorException
 
-tests_root_dir = Path(__file__).parent.parent.parent
-eager_flow_test_dir = tests_root_dir / "test_configs/eager_flows"
-flow_test_dir = tests_root_dir / "test_configs/flows"
-data_dir = tests_root_dir / "test_configs/datas"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
 
 
 @pytest.mark.unittest
@@ -23,7 +21,7 @@ def test_create_flow_with_invalid_parameters(self, pf):
         with pytest.raises(UserErrorException, match=r"fake_source does not exist."):
             pf.flows.create_or_update(flow="fake_source")
 
-        flow_source = flow_test_dir / "web_classification/"
+        flow_source = FLOWS_DIR / "web_classification/"
         with pytest.raises(UserErrorException, match="Not a valid string"):
             pf.flows.create_or_update(flow=flow_source, display_name=False)
 
@@ -42,7 +40,7 @@ def test_update_flow_with_invalid_parameters(self, pf):
 
     @pytest.mark.usefixtures("enable_logger_propagate")
     def test_create_flow_with_warnings(self, pf, caplog):
-        flow_source = flow_test_dir / "web_classification/"
+        flow_source = FLOWS_DIR / "web_classification/"
         pf.flows._validate_flow_creation_parameters(source=flow_source, random="random")
         assert "random: Unknown field" in caplog.text
 
@@ -90,7 +88,7 @@ def mock_get_arm_token(*args, **kwargs) -> str:
                 assert user_tenant_id == mock_tid
 
     def test_eager_flow_creation(self, pf):
-        flow_source = eager_flow_test_dir / "simple_with_yaml"
+        flow_source = EAGER_FLOWS_DIR / "simple_with_yaml"
         with pytest.raises(UserErrorException) as e:
             pf.flows.create_or_update(
                 flow=flow_source,
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_pf_client_azure.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_entity.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_entity.py
similarity index 86%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_entity.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_entity.py
index c539d8833c9..bee29031ae6 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_entity.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_entity.py
@@ -7,20 +7,12 @@
 from unittest.mock import Mock
 
 import pytest
+from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR
 
 from promptflow._sdk.entities import Run
 from promptflow._utils.flow_utils import get_flow_lineage_id
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-FLOWS_DIR = "./tests/test_configs/flows"
-RUNS_DIR = "./tests/test_configs/runs"
-DATAS_DIR = "./tests/test_configs/datas"
-
 
 @pytest.mark.unittest
 class TestRun:
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_operations.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py
index bb42729deaf..249597935f0 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_run_operations.py
+++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py
@@ -6,6 +6,7 @@
 from azure.ai.ml import ManagedIdentityConfiguration
 from azure.ai.ml.entities import IdentityConfiguration
 from pytest_mock import MockerFixture
+from sdk_cli_azure_test.conftest import DATAS_DIR, EAGER_FLOWS_DIR, FLOWS_DIR
 
 from promptflow._sdk._errors import RunOperationParameterError, UploadUserError, UserAuthenticationError
 from promptflow._sdk._utils import parse_otel_span_status_code
@@ -16,10 +17,6 @@
 from promptflow.azure.operations._async_run_uploader import AsyncRunUploader
 from promptflow.exceptions import UserErrorException
 
-FLOWS_DIR = "./tests/test_configs/flows"
-DATAS_DIR = "./tests/test_configs/datas"
-EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows"
-
 
 @pytest.mark.unittest
 class TestRunOperations:
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_span.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_span.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_span.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_span.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_summary.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_summary.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_summary.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_summary.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_trace.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_trace.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_trace.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_trace.py
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_utils.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_utils.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_azure_test/unittests/test_utils.py
rename to src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_utils.py
diff --git a/src/promptflow-azure/tests/unittests/test.py b/src/promptflow-azure/tests/unittests/test.py
deleted file mode 100644
index 226a2ed4261..00000000000
--- a/src/promptflow-azure/tests/unittests/test.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import pytest
-
-
-@pytest.mark.unittest
-class TestStartTrace:
-    def test_import(self):
-        assert True
diff --git a/src/promptflow-core/pyproject.toml b/src/promptflow-core/pyproject.toml
index 94a91cb52ea..e0e8c5d06e7 100644
--- a/src/promptflow-core/pyproject.toml
+++ b/src/promptflow-core/pyproject.toml
@@ -98,9 +98,13 @@ testpaths = ["tests"]
 
 [tool.coverage.run]
 concurrency = ["multiprocessing"]
-source = ["promptflow"]
+source = [
+  "*/promptflow/*"
+]
 omit = [
-  "__init__.py",
+  "*/__init__.py",
+  "*/promptflow/core/_connection_provider/_models/*",
+  "*/promptflow/tracing/*"
 ]
 
 [tool.black]
diff --git a/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py b/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py
index 41f90850748..21dd6408aa4 100644
--- a/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py
+++ b/src/promptflow-devkit/promptflow/_cli/_pf/_upgrade.py
@@ -42,7 +42,7 @@ def upgrade_version(args):
 
     from promptflow._constants import _ENV_PF_INSTALLER, CLI_PACKAGE_NAME
     from promptflow._sdk._utils import get_promptflow_sdk_version
-    from promptflow._utils.version_hint_utils import get_latest_version
+    from promptflow._sdk._version_hint_utils import get_latest_version
 
     installer = os.getenv(_ENV_PF_INSTALLER) or ""
     installer = installer.upper()
diff --git a/src/promptflow-devkit/promptflow/_sdk/_configuration.py b/src/promptflow-devkit/promptflow/_sdk/_configuration.py
index d2136116904..f33291adbca 100644
--- a/src/promptflow-devkit/promptflow/_sdk/_configuration.py
+++ b/src/promptflow-devkit/promptflow/_sdk/_configuration.py
@@ -17,10 +17,11 @@
     HOME_PROMPT_FLOW_DIR,
     SERVICE_CONFIG_FILE,
 )
+from promptflow._sdk._errors import MissingAzurePackage
 from promptflow._sdk._utils import call_from_extension, gen_uuid_by_compute_info, read_write_by_user
 from promptflow._utils.logger_utils import get_cli_sdk_logger
 from promptflow._utils.yaml_utils import dump_yaml, load_yaml
-from promptflow.exceptions import ErrorTarget, UserErrorException, ValidationException
+from promptflow.exceptions import ErrorTarget, ValidationException
 
 logger = get_cli_sdk_logger()
 
@@ -223,15 +224,7 @@ def _validate(key: str, value: str) -> None:
 
                 validate_trace_provider(value)
             except ImportError:
-                msg = (
-                    '"promptflow[azure]" is required to validate trace provider, '
-                    'please install it by running "pip install promptflow[azure]" with your version.'
-                )
-                raise UserErrorException(
-                    message=msg,
-                    target=ErrorTarget.CONTROL_PLANE_SDK,
-                    no_personal_data_message=msg,
-                )
+                raise MissingAzurePackage()
         return
 
     def get_user_agent(self) -> Optional[str]:
diff --git a/src/promptflow-devkit/promptflow/_sdk/_errors.py b/src/promptflow-devkit/promptflow/_sdk/_errors.py
index 716f8c516fe..3ce415be56a 100644
--- a/src/promptflow-devkit/promptflow/_sdk/_errors.py
+++ b/src/promptflow-devkit/promptflow/_sdk/_errors.py
@@ -257,3 +257,17 @@ class LineRunNotFoundError(SDKError):
     """Exception raised if line run cannot be found."""
 
     pass
+
+
+class MissingAzurePackage(SDKError):
+    """Exception raised if missing required package."""
+
+    def __init__(
+        self,
+        **kwargs,
+    ):
+        msg = (
+            '"promptflow[azure]" is required for this functionality, '
+            'please install it by running "pip install promptflow-azure" with your version.'
+        )
+        super().__init__(message=msg, no_personal_data_message=msg, **kwargs)
diff --git a/src/promptflow-devkit/promptflow/_sdk/_service/app.py b/src/promptflow-devkit/promptflow/_sdk/_service/app.py
index bf59e5cd815..2da464cf505 100644
--- a/src/promptflow-devkit/promptflow/_sdk/_service/app.py
+++ b/src/promptflow-devkit/promptflow/_sdk/_service/app.py
@@ -19,6 +19,7 @@
     PF_SERVICE_MONITOR_SECOND,
     CreatedByFieldName,
 )
+from promptflow._sdk._errors import MissingAzurePackage
 from promptflow._sdk._service import Api
 from promptflow._sdk._service.apis.collector import trace_collector
 from promptflow._sdk._service.apis.connection import api as connection_api
@@ -209,6 +210,8 @@ def get_created_by_info_with_cache():
                     CreatedByFieldName.NAME: decoded_token.get("name", decoded_token.get("appid", "")),
                 }
             )
+        except ImportError:
+            raise MissingAzurePackage()
         except Exception as e:
             # This function is only target to be used in Flask app.
             current_app.logger.error(f"Failed to get created_by info, stop writing span. Exception: {e}")
diff --git a/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py b/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py
index b21ee3f8811..928fc082606 100644
--- a/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py
+++ b/src/promptflow-devkit/promptflow/_sdk/_telemetry/activity.py
@@ -234,7 +234,7 @@ def monitor(f):
         @functools.wraps(f)
         def wrapper(self, *args, **kwargs):
             from promptflow._sdk._telemetry.telemetry import get_telemetry_logger
-            from promptflow._utils.version_hint_utils import HINT_ACTIVITY_NAME, check_latest_version, hint_for_update
+            from promptflow._sdk._version_hint_utils import HINT_ACTIVITY_NAME, check_latest_version, hint_for_update
 
             logger = get_telemetry_logger()
 
diff --git a/src/promptflow-core/promptflow/_utils/version_hint_utils.py b/src/promptflow-devkit/promptflow/_sdk/_version_hint_utils.py
similarity index 96%
rename from src/promptflow-core/promptflow/_utils/version_hint_utils.py
rename to src/promptflow-devkit/promptflow/_sdk/_version_hint_utils.py
index 1b5dd232381..28448fc5a43 100644
--- a/src/promptflow-core/promptflow/_utils/version_hint_utils.py
+++ b/src/promptflow-devkit/promptflow/_sdk/_version_hint_utils.py
@@ -101,9 +101,9 @@ def hint_for_update():
     if last_hint_time is None or (
         datetime.datetime.now() > last_hint_time + datetime.timedelta(days=HINT_INTERVAL_DAY)
     ):
-        from promptflow._sdk._utils import get_promptflow_sdk_version
+        from promptflow._sdk._utils import get_promptflow_devkit_version
 
-        cached_versions[CURRENT_VERSION] = get_promptflow_sdk_version()
+        cached_versions[CURRENT_VERSION] = get_promptflow_devkit_version()
         if LATEST_VERSION in cached_versions:
             from packaging.version import parse
 
diff --git a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py
index 76415c3e003..2e46536cb7d 100644
--- a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py
+++ b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py
@@ -39,7 +39,7 @@
     RunStatus,
     RunTypes,
 )
-from promptflow._sdk._errors import InvalidRunError, InvalidRunStatusError
+from promptflow._sdk._errors import InvalidRunError, InvalidRunStatusError, MissingAzurePackage
 from promptflow._sdk._orm import RunInfo as ORMRun
 from promptflow._sdk._utils import (
     _sanitize_python_variable_name,
@@ -526,15 +526,18 @@ def _get_schema_cls(self):
         return RunSchema
 
     def _to_rest_object(self):
-        from azure.ai.ml._utils._storage_utils import AzureMLDatastorePathUri
-
-        from promptflow.azure._restclient.flow.models import (
-            BatchDataInput,
-            CreateExistingBulkRunRequest,
-            RunDisplayNameGenerationType,
-            SessionSetupModeEnum,
-            SubmitBulkRunRequest,
-        )
+        try:
+            from azure.ai.ml._utils._storage_utils import AzureMLDatastorePathUri
+
+            from promptflow.azure._restclient.flow.models import (
+                BatchDataInput,
+                CreateExistingBulkRunRequest,
+                RunDisplayNameGenerationType,
+                SessionSetupModeEnum,
+                SubmitBulkRunRequest,
+            )
+        except ImportError:
+            raise MissingAzurePackage()
 
         if self.run is not None:
             if isinstance(self.run, Run):
diff --git a/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py b/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py
index e23d66f9534..bd76ce8435c 100644
--- a/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py
+++ b/src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py
@@ -5,6 +5,7 @@
 from typing import List
 
 from promptflow._sdk._constants import MAX_LIST_CLI_RESULTS
+from promptflow._sdk._errors import MissingAzurePackage
 from promptflow._sdk._telemetry import ActivityType, WorkspaceTelemetryMixin, monitor_operation
 from promptflow._sdk._utils import print_red_error
 from promptflow._sdk.entities._connection import _Connection
@@ -37,7 +38,10 @@ def __init__(self, connection_provider, **kwargs):
     @property
     def _client(self):
         if self._pfazure_client is None:
-            from promptflow.azure._pf_client import PFClient as PFAzureClient
+            try:
+                from promptflow.azure._pf_client import PFClient as PFAzureClient
+            except ImportError:
+                raise MissingAzurePackage()
 
             self._pfazure_client = PFAzureClient(
                 # TODO: disable interactive credential when starting as a service
@@ -51,9 +55,12 @@ def _client(self):
 
     @classmethod
     def _get_credential(cls):
-        from azure.identity import DefaultAzureCredential, DeviceCodeCredential
+        try:
+            from azure.identity import DefaultAzureCredential, DeviceCodeCredential
 
-        from promptflow.azure._utils.general import get_arm_token
+            from promptflow.azure._utils.general import get_arm_token
+        except ImportError:
+            raise MissingAzurePackage()
 
         if is_from_cli():
             try:
@@ -105,7 +112,10 @@ def get(self, name: str, **kwargs) -> _Connection:
         if with_secrets:
             # Do not use pfazure_client here as it requires workspace read permission
             # Get secrets from arm only requires workspace listsecrets permission
-            from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations
+            try:
+                from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations
+            except ImportError:
+                raise MissingAzurePackage()
 
             return ArmConnectionOperations._direct_get(
                 name, self._subscription_id, self._resource_group, self._workspace_name, self._credential
diff --git a/src/promptflow-devkit/pyproject.toml b/src/promptflow-devkit/pyproject.toml
index 96eb01e92c2..24aa28bb585 100644
--- a/src/promptflow-devkit/pyproject.toml
+++ b/src/promptflow-devkit/pyproject.toml
@@ -96,6 +96,7 @@ mock = "*"
 ipykernel = ">=6.27.1"
 papermill = ">=2.5.0"
 keyrings-alt = "*"
+bs4 = "*"
 
 [build-system]
 requires = ["poetry-core>=1.5.0"]
@@ -107,6 +108,7 @@ pf = "promptflow._cli.pf:main"
 [tool.pytest.ini_options]
 markers = [
   "unittest",
+  "e2etest"
 ]
 # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action)
 # durations - list the slowest test durations
@@ -123,8 +125,16 @@ addopts = """
 testpaths = ["tests"]
 
 [tool.coverage.run]
+source = [
+  "*/promptflow/_cli/**",
+  "*/promptflow/_orchestrator/**",
+  "*/promptflow/_proxy/**",
+  "*/promptflow/_sdk/**",
+  "*/promptflow/batch/**",
+]
 omit = [
-  "__init__.py",
+  "*/__init__.py",
+  "*/promptflow/_sdk/_serving/*",
 ]
 
 [tool.black]
diff --git a/src/promptflow-devkit/tests/_constants.py b/src/promptflow-devkit/tests/_constants.py
new file mode 100644
index 00000000000..9929a01ff68
--- /dev/null
+++ b/src/promptflow-devkit/tests/_constants.py
@@ -0,0 +1,14 @@
+from pathlib import Path
+
+PROMPTFLOW_ROOT = Path(__file__).parent.parent.parent / "promptflow"
+RUNTIME_TEST_CONFIGS_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/runtime")
+CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
+ENV_FILE = (PROMPTFLOW_ROOT / ".env").resolve().absolute().as_posix()
+
+# below constants are used for pfazure and global config tests
+DEFAULT_SUBSCRIPTION_ID = "96aede12-2f73-41cb-b983-6d11a904839b"
+DEFAULT_RESOURCE_GROUP_NAME = "promptflow"
+DEFAULT_WORKSPACE_NAME = "promptflow-eastus2euap"
+DEFAULT_COMPUTE_INSTANCE_NAME = "ci-lin-cpu-sp"
+DEFAULT_RUNTIME_NAME = "test-runtime-ci"
+DEFAULT_REGISTRY_NAME = "promptflow-preview"
diff --git a/src/promptflow-devkit/tests/conftest.py b/src/promptflow-devkit/tests/conftest.py
new file mode 100644
index 00000000000..7b7d87f26f8
--- /dev/null
+++ b/src/promptflow-devkit/tests/conftest.py
@@ -0,0 +1,237 @@
+import importlib
+import json
+import os
+import tempfile
+from multiprocessing import Lock
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from _constants import (
+    CONNECTION_FILE,
+    DEFAULT_RESOURCE_GROUP_NAME,
+    DEFAULT_SUBSCRIPTION_ID,
+    DEFAULT_WORKSPACE_NAME,
+    ENV_FILE,
+    PROMPTFLOW_ROOT,
+)
+from _pytest.monkeypatch import MonkeyPatch
+from dotenv import load_dotenv
+from filelock import FileLock
+from pytest_mock import MockerFixture
+
+from promptflow._constants import PROMPTFLOW_CONNECTIONS
+from promptflow._core.connection_manager import ConnectionManager
+from promptflow._sdk.entities._connection import AzureOpenAIConnection
+from promptflow._utils.context_utils import _change_working_dir
+
+load_dotenv()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def modify_work_directory():
+    os.chdir(PROMPTFLOW_ROOT)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_build_info():
+    """Mock BUILD_INFO environment variable in pytest.
+
+    BUILD_INFO is set as environment variable in docker image, but not in local test.
+    So we need to mock it in test senario. Rule - build_number is set as
+    ci-<BUILD_BUILDNUMBER> in CI pipeline, and set as local in local dev test."""
+    if "BUILD_INFO" not in os.environ:
+        m = MonkeyPatch()
+        build_number = os.environ.get("BUILD_BUILDNUMBER", "")
+        buid_info = {"build_number": f"ci-{build_number}" if build_number else "local-pytest"}
+        m.setenv("BUILD_INFO", json.dumps(buid_info))
+        yield m
+
+
+@pytest.fixture
+def dev_connections() -> dict:
+    with open(CONNECTION_FILE, "r") as f:
+        return json.load(f)
+
+
+@pytest.fixture
+def use_secrets_config_file(mocker: MockerFixture):
+    mocker.patch.dict(os.environ, {PROMPTFLOW_CONNECTIONS: CONNECTION_FILE})
+
+
+@pytest.fixture
+def env_with_secrets_config_file():
+    _lock = Lock()
+    with _lock:
+        with open(ENV_FILE, "w") as f:
+            f.write(f"{PROMPTFLOW_CONNECTIONS}={CONNECTION_FILE}\n")
+        yield ENV_FILE
+        if os.path.exists(ENV_FILE):
+            os.remove(ENV_FILE)
+
+
+@pytest.fixture
+def azure_open_ai_connection() -> AzureOpenAIConnection:
+    return ConnectionManager().get("azure_open_ai_connection")
+
+
+@pytest.fixture
+def temp_output_dir() -> str:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        yield temp_dir
+
+
+@pytest.fixture
+def prepare_symbolic_flow() -> str:
+    flows_dir = PROMPTFLOW_ROOT / "tests" / "test_configs" / "flows"
+    target_folder = flows_dir / "web_classification_with_symbolic"
+    source_folder = flows_dir / "web_classification"
+
+    with _change_working_dir(target_folder):
+
+        for file_name in os.listdir(source_folder):
+            if not Path(file_name).exists():
+                os.symlink(source_folder / file_name, file_name)
+    return target_folder
+
+
+@pytest.fixture(scope="session")
+def install_custom_tool_pkg():
+    # The tests could be running in parallel. Use a lock to prevent race conditions.
+    lock = FileLock("custom_tool_pkg_installation.lock")
+    with lock:
+        try:
+            import my_tool_package  # noqa: F401
+
+        except ImportError:
+            import subprocess
+            import sys
+
+            subprocess.check_call([sys.executable, "-m", "pip", "install", "test-custom-tools==0.0.2"])
+
+
+@pytest.fixture(scope="session")
+def mock_list_func():
+    """Mock function object for dynamic list testing."""
+
+    def my_list_func(prefix: str = "", size: int = 10, **kwargs):
+        return [
+            {
+                "value": "fig0",
+                "display_value": "My_fig0",
+                "hyperlink": "https://www.bing.com/search?q=fig0",
+                "description": "this is 0 item",
+            },
+            {
+                "value": "kiwi1",
+                "display_value": "My_kiwi1",
+                "hyperlink": "https://www.bing.com/search?q=kiwi1",
+                "description": "this is 1 item",
+            },
+        ]
+
+    return my_list_func
+
+
+@pytest.fixture(scope="session")
+def mock_module_with_list_func(mock_list_func):
+    """Mock module object for dynamic list testing."""
+    mock_module = MagicMock()
+    mock_module.my_list_func = mock_list_func
+    mock_module.my_field = 1
+    original_import_module = importlib.import_module  # Save this to prevent recursion
+
+    with patch.object(importlib, "import_module") as mock_import:
+
+        def side_effect(module_name, *args, **kwargs):
+            if module_name == "my_tool_package.tools.tool_with_dynamic_list_input":
+                return mock_module
+            else:
+                return original_import_module(module_name, *args, **kwargs)
+
+        mock_import.side_effect = side_effect
+        yield
+
+
+@pytest.fixture(scope="session")
+def mock_generated_by_func():
+    """Mock function object for generated_by testing."""
+
+    def my_generated_by_func(index_type: str):
+        inputs = ""
+        if index_type == "Azure Cognitive Search":
+            inputs = {"index_type": index_type, "index": "index_1"}
+        elif index_type == "Workspace MLIndex":
+            inputs = {"index_type": index_type, "index": "index_2"}
+
+        result = json.dumps(inputs)
+        return result
+
+    return my_generated_by_func
+
+
+@pytest.fixture(scope="session")
+def mock_reverse_generated_by_func():
+    """Mock function object for reverse_generated_by testing."""
+
+    def my_reverse_generated_by_func(index_json: str):
+        result = json.loads(index_json)
+        return result
+
+    return my_reverse_generated_by_func
+
+
+@pytest.fixture
+def enable_logger_propagate():
+    """This is for test cases that need to check the log output."""
+    from promptflow._utils.logger_utils import get_cli_sdk_logger
+
+    logger = get_cli_sdk_logger()
+    original_value = logger.propagate
+    logger.propagate = True
+    yield
+    logger.propagate = original_value
+
+
+@pytest.fixture(scope="session")
+def mock_module_with_for_retrieve_tool_func_result(
+    mock_list_func, mock_generated_by_func, mock_reverse_generated_by_func
+):
+    """Mock module object for dynamic list testing."""
+    mock_module_list_func = MagicMock()
+    mock_module_list_func.my_list_func = mock_list_func
+    mock_module_list_func.my_field = 1
+    mock_module_generated_by = MagicMock()
+    mock_module_generated_by.generated_by_func = mock_generated_by_func
+    mock_module_generated_by.reverse_generated_by_func = mock_reverse_generated_by_func
+    mock_module_generated_by.my_field = 1
+    original_import_module = importlib.import_module  # Save this to prevent recursion
+
+    with patch.object(importlib, "import_module") as mock_import:
+
+        def side_effect(module_name, *args, **kwargs):
+            if module_name == "my_tool_package.tools.tool_with_dynamic_list_input":
+                return mock_module_list_func
+            elif module_name == "my_tool_package.tools.tool_with_generated_by_input":
+                return mock_module_generated_by
+            else:
+                return original_import_module(module_name, *args, **kwargs)
+
+        mock_import.side_effect = side_effect
+        yield
+
+
+# region pfazure constants
+@pytest.fixture
+def subscription_id() -> str:
+    return os.getenv("PROMPT_FLOW_SUBSCRIPTION_ID", DEFAULT_SUBSCRIPTION_ID)
+
+
+@pytest.fixture
+def resource_group_name() -> str:
+    return os.getenv("PROMPT_FLOW_RESOURCE_GROUP_NAME", DEFAULT_RESOURCE_GROUP_NAME)
+
+
+@pytest.fixture
+def workspace_name() -> str:
+    return os.getenv("PROMPT_FLOW_WORKSPACE_NAME", DEFAULT_WORKSPACE_NAME)
diff --git a/src/promptflow/tests/sdk_cli_global_config_test/__init__.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_global_config_test/__init__.py
rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_global_config_test/conftest.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/conftest.py
similarity index 97%
rename from src/promptflow/tests/sdk_cli_global_config_test/conftest.py
rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/conftest.py
index 672e866793f..736f1852322 100644
--- a/src/promptflow/tests/sdk_cli_global_config_test/conftest.py
+++ b/src/promptflow-devkit/tests/sdk_cli_global_config_test/conftest.py
@@ -6,8 +6,8 @@
 import pytest
 from _constants import DEFAULT_RESOURCE_GROUP_NAME, DEFAULT_SUBSCRIPTION_ID, DEFAULT_WORKSPACE_NAME
 
-from promptflow import PFClient
 from promptflow._sdk._configuration import Configuration
+from promptflow.client import PFClient
 
 AZUREML_RESOURCE_PROVIDER = "Microsoft.MachineLearningServices"
 RESOURCE_ID_FORMAT = "/subscriptions/{}/resourceGroups/{}/providers/{}/workspaces/{}"
diff --git a/src/promptflow/tests/sdk_cli_global_config_test/e2etests/__init__.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_global_config_test/e2etests/__init__.py
rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_global_config_test/e2etests/test_global_config.py b/src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py
similarity index 91%
rename from src/promptflow/tests/sdk_cli_global_config_test/e2etests/test_global_config.py
rename to src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py
index 30391b1fdb1..b7553962944 100644
--- a/src/promptflow/tests/sdk_cli_global_config_test/e2etests/test_global_config.py
+++ b/src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py
@@ -1,14 +1,13 @@
-from pathlib import Path
-
 import mock
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._load_functions import load_flow
 from promptflow._sdk.entities._flows._flow_context_resolver import FlowContextResolver
 from promptflow.core._connection_provider._workspace_connection_provider import WorkspaceConnectionProvider
 
-FLOWS_DIR = Path(__file__).parent.parent.parent / "test_configs" / "flows"
-DATAS_DIR = Path(__file__).parent.parent.parent / "test_configs" / "datas"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests" / "test_configs" / "flows"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests" / "test_configs" / "datas"
 
 
 @pytest.mark.usefixtures("global_config")
diff --git a/src/promptflow/tests/sdk_cli_test/.coveragerc b/src/promptflow-devkit/tests/sdk_cli_test/.coveragerc
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/.coveragerc
rename to src/promptflow-devkit/tests/sdk_cli_test/.coveragerc
diff --git a/src/promptflow/tests/sdk_cli_test/__init__.py b/src/promptflow-devkit/tests/sdk_cli_test/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/__init__.py
rename to src/promptflow-devkit/tests/sdk_cli_test/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_test/conftest.py b/src/promptflow-devkit/tests/sdk_cli_test/conftest.py
similarity index 96%
rename from src/promptflow/tests/sdk_cli_test/conftest.py
rename to src/promptflow-devkit/tests/sdk_cli_test/conftest.py
index 19a61ceffd9..3c536329b93 100644
--- a/src/promptflow/tests/sdk_cli_test/conftest.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/conftest.py
@@ -6,6 +6,7 @@
 from unittest.mock import patch
 
 import pytest
+from _constants import CONNECTION_FILE, PROMPTFLOW_ROOT
 from mock import mock
 from pytest_mock import MockerFixture
 from sqlalchemy import create_engine
@@ -41,14 +42,10 @@ def is_replay():
         return False
 
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../.."
-RUNTIME_TEST_CONFIGS_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/runtime")
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-MODEL_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/flows")
-EAGER_FLOW_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/eager_flows")
+EAGER_FLOW_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/eager_flows")
+MODEL_ROOT = Path(PROMPTFLOW_ROOT / "tests/test_configs/flows")
 
-SRC_ROOT = PROMOTFLOW_ROOT / ".."
-RECORDINGS_TEST_CONFIGS_ROOT = Path(SRC_ROOT / "promptflow-recording/recordings/local").resolve()
+RECORDINGS_TEST_CONFIGS_ROOT = Path(PROMPTFLOW_ROOT / "../promptflow-recording/recordings/local").resolve()
 
 
 def pytest_configure():
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/__init__.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/e2etests/__init__.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_chat_group.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_chat_group.py
similarity index 90%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_chat_group.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_chat_group.py
index 5af6973d288..34b6dd3fac1 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_chat_group.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_chat_group.py
@@ -1,14 +1,10 @@
-from pathlib import Path
-
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk.entities._chat_group._chat_group import ChatGroup
 from promptflow._sdk.entities._chat_group._chat_role import ChatRole
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-FLOWS_DIR = TEST_ROOT / "test_configs/flows"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
 
 
 @pytest.mark.sdk_test
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py
index 32e6fdf8de3..c58db8bc851 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py
@@ -17,6 +17,7 @@
 
 import mock
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._cli._pf.entry import main
 from promptflow._constants import FLOW_FLEX_YAML, LINE_NUMBER_KEY, PF_USER_AGENT
@@ -30,14 +31,14 @@
 from promptflow._utils.yaml_utils import dump_yaml, load_yaml
 from promptflow.tracing._operation_context import OperationContext
 
-FLOWS_DIR = "./tests/test_configs/flows"
-EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows"
-EXPERIMENT_DIR = "./tests/test_configs/experiments"
-RUNS_DIR = "./tests/test_configs/runs"
-CONNECTIONS_DIR = "./tests/test_configs/connections"
-DATAS_DIR = "./tests/test_configs/datas"
-TOOL_ROOT = "./tests/test_configs/tools"
-PROMPTY_DIR = "./tests/test_configs/prompty"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
+EXPERIMENT_DIR = PROMPTFLOW_ROOT / "tests/test_configs/experiments"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
+CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
+TOOL_ROOT = PROMPTFLOW_ROOT / "tests/test_configs/tools"
+PROMPTY_DIR = PROMPTFLOW_ROOT / "tests/test_configs/prompty"
 
 TARGET_URL = "https://www.youtube.com/watch?v=o5ZQyXaAv1g"
 
@@ -1536,6 +1537,7 @@ def test_tool_init(self, capsys):
             outerr = capsys.readouterr()
             assert "Cannot find the icon path" in outerr.out
 
+    @pytest.mark.skip("Enable after promptflow-tool depend on core")
     def test_list_tool_cache(self, caplog, mocker):
         with tempfile.TemporaryDirectory() as temp_dir:
             package_name = "mock_tool_package_name"
@@ -1565,6 +1567,7 @@ def test_list_tool_cache(self, caplog, mocker):
             assert "List tools meta from cache file" in caplog.text
             assert f"{package_name}.{func_name}.{func_name}" in tools_meta
 
+    @pytest.mark.skip("Enable after promptflow-tool depend on core")
     def test_tool_list(self, capsys):
         # List package tools in environment
         run_pf_command("tool", "list")
@@ -1825,8 +1828,8 @@ def assert_flow_test(*args, **kwargs):
     def test_run_create_with_existing_run_folder(self):
         run_name = "web_classification_variant_0_20231205_120253_104100"
         # clean the run if exists
-        from promptflow import PFClient
         from promptflow._cli._utils import _try_delete_existing_run_record
+        from promptflow.client import PFClient
 
         pf = PFClient()
         _try_delete_existing_run_record(run_name)
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli_perf.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli_perf.py
similarity index 96%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_cli_perf.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli_perf.py
index 9077a28897e..c57330b0dff 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli_perf.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli_perf.py
@@ -10,14 +10,15 @@
 from unittest import mock
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._cli._user_agent import USER_AGENT as CLI_USER_AGENT  # noqa: E402
 from promptflow._sdk._telemetry import log_activity
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
 
-FLOWS_DIR = "./tests/test_configs/flows"
-CONNECTIONS_DIR = "./tests/test_configs/connections"
-DATAS_DIR = "./tests/test_configs/datas"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
 
 
 def mock_log_activity(*args, **kwargs):
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_connection.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_connection.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_connection.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_connection.py
index 14e4655bfaa..01b907a41ff 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_connection.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_connection.py
@@ -1,9 +1,9 @@
 import os
 import uuid
-from pathlib import Path
 
 import pydash
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from mock import mock
 
 from promptflow._constants import ConnectionDefaultApiVersion
@@ -12,10 +12,9 @@
 from promptflow._sdk._pf_client import PFClient
 from promptflow._sdk.entities import AzureOpenAIConnection, CustomConnection, OpenAIConnection
 
-_client = PFClient()
-
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 CONNECTION_ROOT = TEST_ROOT / "test_configs/connections"
+_client = PFClient()
 
 
 @pytest.mark.cli_test
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_csharp_cli.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_csharp_cli.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py
index ad1fe7812da..ea485d20f4e 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_custom_strong_type_connection.py
@@ -1,8 +1,8 @@
 import uuid
-from pathlib import Path
 
 import pydash
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._constants import SCRUBBED_VALUE, CustomStrongTypeConnectionConfigs
 from promptflow._sdk._pf_client import PFClient
@@ -17,7 +17,8 @@ class MyCustomConnection(CustomStrongTypeConnection):
 
 _client = PFClient()
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 CONNECTION_ROOT = TEST_ROOT / "test_configs/connections"
 
 
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_executable.py
similarity index 88%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_executable.py
index e10597382e7..eff466fa7b9 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_executable.py
@@ -5,13 +5,14 @@
 
 import mock
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from .test_cli import run_pf_command
 
-FLOWS_DIR = "./tests/test_configs/flows"
-RUNS_DIR = "./tests/test_configs/runs"
-CONNECTIONS_DIR = "./tests/test_configs/connections"
-DATAS_DIR = "./tests/test_configs/datas"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
+CONNECTIONS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/connections"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
 
 
 @pytest.mark.usefixtures("use_secrets_config_file", "setup_local_connection", "install_custom_tool_pkg")
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_experiment.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_experiment.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_experiment.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_experiment.py
index cf7a4513084..2f3c3fee50b 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_experiment.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_experiment.py
@@ -9,6 +9,7 @@
 from time import sleep
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from mock import mock
 from ruamel.yaml import YAML
 
@@ -19,7 +20,7 @@
 from promptflow._sdk._pf_client import PFClient
 from promptflow._sdk.entities._experiment import CommandNode, Experiment, ExperimentTemplate, FlowNode
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 EXP_ROOT = TEST_ROOT / "test_configs/experiments"
 FLOW_ROOT = TEST_ROOT / "test_configs/flows"
 EAGER_FLOW_ROOT = TEST_ROOT / "test_configs/eager_flows"
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_as_func.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_as_func.py
index 61bcc484a4c..1607b6f7b4f 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_as_func.py
@@ -10,18 +10,19 @@
 
 import mock
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
-from promptflow import load_flow
 from promptflow._sdk._errors import ConnectionNotFoundError, InvalidFlowError
 from promptflow._sdk.entities import CustomConnection
 from promptflow._sdk.entities._flows._flow_context_resolver import FlowContextResolver
 from promptflow._utils.flow_utils import dump_flow_dag, load_flow_dag
+from promptflow.client import load_flow
 from promptflow.entities import FlowContext
 from promptflow.exceptions import UserErrorException
 
-FLOWS_DIR = "./tests/test_configs/flows"
-RUNS_DIR = "./tests/test_configs/runs"
-DATAS_DIR = "./tests/test_configs/datas"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
 
 
 @pytest.mark.usefixtures(
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_local_operations.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_local_operations.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_local_operations.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_local_operations.py
index 9c789645cd9..6fcaec00308 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_local_operations.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_local_operations.py
@@ -7,6 +7,7 @@
 
 import mock
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._constants import FLOW_TOOLS_JSON, NODE_VARIANTS, PROMPT_FLOW_DIR_NAME, USE_VARIANTS
 from promptflow._utils.yaml_utils import load_yaml
@@ -14,15 +15,12 @@
 from promptflow.core._flow import Prompty
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-FLOWS_DIR = "./tests/test_configs/flows"
-EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows"
-DATAS_DIR = "./tests/test_configs/datas"
-PROMPTY_DIR = "./tests/test_configs/prompty"
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
+CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
+PROMPTY_DIR = PROMPTFLOW_ROOT / "tests/test_configs/prompty"
 
 
 def e2e_test_docker_build_and_run(output_path):
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py
index e832d519704..4ec3a8024d2 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py
@@ -8,10 +8,10 @@
 import numpy as np
 import pandas as pd
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from marshmallow import ValidationError
 from pytest_mock import MockerFixture
 
-from promptflow import PFClient
 from promptflow._constants import PROMPTFLOW_CONNECTIONS
 from promptflow._sdk._constants import (
     FLOW_DIRECTORY_MACRO_IN_CONFIG,
@@ -36,18 +36,16 @@
 from promptflow._sdk.operations._local_storage_operations import LocalStorageOperations
 from promptflow._utils.context_utils import _change_working_dir, inject_sys_path
 from promptflow._utils.yaml_utils import load_yaml
+from promptflow.client import PFClient
 from promptflow.connections import AzureOpenAIConnection
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-FLOWS_DIR = "./tests/test_configs/flows"
-EAGER_FLOWS_DIR = "./tests/test_configs/eager_flows"
-RUNS_DIR = "./tests/test_configs/runs"
-DATAS_DIR = "./tests/test_configs/datas"
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
+CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
+RUNS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/runs"
+DATAS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/datas"
 
 
 def my_entry(input1: str):
@@ -236,7 +234,7 @@ def test_run_bulk_error(self, pf):
         # path not exist
         with pytest.raises(UserErrorException) as e:
             pf.run(
-                flow=f"{MODEL_ROOT}/not_exist",
+                flow=f"{FLOWS_DIR}/not_exist",
                 data=f"{DATAS_DIR}/webClassification3.jsonl",
                 column_mapping={"question": "${data.question}", "context": "${data.context}"},
                 variant="${summarize_text_content.variant_0}",
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_save.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_save.py
similarity index 97%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_save.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_save.py
index c7e937374e0..e5d53877147 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_save.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_save.py
@@ -4,23 +4,19 @@
 import shutil
 import sys
 import tempfile
-from pathlib import Path
 from typing import Callable, TypedDict
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._pf_client import PFClient
 from promptflow._sdk.entities import AzureOpenAIConnection
 from promptflow.client import load_flow
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
-FLOWS_DIR = (TEST_ROOT / "test_configs/flows").resolve().absolute().as_posix()
-EAGER_FLOWS_DIR = (TEST_ROOT / "test_configs/eager_flows").resolve().absolute().as_posix()
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
+EAGER_FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/eager_flows"
 FLOW_RESULT_KEYS = ["category", "evidence"]
 
 _client = PFClient()
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve.py
index 125eaa03a75..e6c033e85dd 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve.py
@@ -3,6 +3,7 @@
 import re
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from opentelemetry import trace
 from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 from opentelemetry.sdk.trace import TracerProvider
@@ -15,6 +16,8 @@
 from promptflow.exceptions import UserErrorException
 from promptflow.tracing._operation_context import OperationContext
 
+TEST_CONFIGS = PROMPTFLOW_ROOT / "tests" / "test_configs" / "eager_flows"
+
 
 @pytest.mark.usefixtures("recording_injection", "setup_local_connection")
 @pytest.mark.e2etest
@@ -570,14 +573,13 @@ def test_eager_flow_serve_dataclass_output(simple_eager_flow_dataclass_output):
 def test_eager_flow_serve_non_json_serializable_output(mocker):
     with pytest.raises(UserErrorException, match="Parse interface for 'my_flow' failed:"):
         # instead of giving 400 response for all requests, we raise user error on serving now
-        from pathlib import Path
 
         from ..conftest import create_client_by_model
 
         create_client_by_model(
             "non_json_serializable_output",
             mocker,
-            model_root=Path(__file__).parent.parent.parent / "test_configs" / "eager_flows",
+            model_root=TEST_CONFIGS,
         )
 
 
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_serve_azureml_extension.py
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py
similarity index 98%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py
index 33781db39eb..1c27bdb8eb3 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py
@@ -7,6 +7,7 @@
 
 import papermill
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from marshmallow import ValidationError
 
 from promptflow._sdk._constants import LOGGER_NAME
@@ -14,11 +15,8 @@
 from promptflow.core._utils import init_executable
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
-MODEL_ROOT = TEST_ROOT / "test_configs/e2e_samples"
-CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
+CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
 FLOWS_DIR = (TEST_ROOT / "test_configs/flows").resolve().absolute().as_posix()
 EAGER_FLOWS_DIR = (TEST_ROOT / "test_configs/eager_flows").resolve().absolute().as_posix()
 FLOW_RESULT_KEYS = ["category", "evidence"]
@@ -137,7 +135,7 @@ def test_pf_test_flow_with_variant(self):
 
     @pytest.mark.skip("TODO this test case failed in windows and Mac")
     def test_pf_test_with_additional_includes(self, caplog):
-        from promptflow import VERSION
+        from promptflow._sdk._version import VERSION
 
         print(VERSION)
         with caplog.at_level(level=logging.WARNING, logger=LOGGER_NAME):
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_orm.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_orm.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_orm.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_orm.py
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_prompty.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_prompty.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_prompty.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_prompty.py
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_telemetry.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_telemetry.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_telemetry.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_telemetry.py
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_tool.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_tool.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_tool.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_tool.py
index a012c7a44e3..03016260639 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_tool.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_tool.py
@@ -5,6 +5,7 @@
 from unittest.mock import patch
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._core.tool import ToolProvider, tool
 from promptflow._core.tool_meta_generator import ToolValidationError, _serialize_tool
@@ -12,8 +13,7 @@
 from promptflow.entities import DynamicList, InputSetting
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 TOOL_ROOT = TEST_ROOT / "test_configs/tools"
 
 _client = PFClient()
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_trace.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_trace.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/e2etests/test_trace.py
rename to src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_trace.py
index ebf2e2204ec..8e709df557c 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_trace.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_trace.py
@@ -6,6 +6,7 @@
 from unittest.mock import patch
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from mock import mock
 
 from promptflow._constants import (
@@ -18,7 +19,7 @@
 from promptflow._sdk._pf_client import PFClient
 from promptflow._sdk.entities._trace import Span
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 FLOWS_DIR = (TEST_ROOT / "test_configs/flows").resolve().absolute().as_posix()
 
 
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/__init__.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/unittests/__init__.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/__init__.py
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_chat_group.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_chat_group.py
similarity index 95%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_chat_group.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_chat_group.py
index eb8050a381a..2bb2d14f3b6 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_chat_group.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_chat_group.py
@@ -1,15 +1,12 @@
-from pathlib import Path
-
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from pytest_mock import MockFixture
 
 from promptflow._sdk._errors import ChatGroupError, ChatRoleError
 from promptflow._sdk.entities._chat_group._chat_group import ChatGroup
 from promptflow._sdk.entities._chat_group._chat_role import ChatRole
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
-
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 FLOWS_DIR = TEST_ROOT / "test_configs/flows"
 
 
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_cli_activity_name.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_cli_activity_name.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_cli_activity_name.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_cli_activity_name.py
index 3cfbba2c8f2..e0723d40cf9 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_cli_activity_name.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_cli_activity_name.py
@@ -1,4 +1,5 @@
 import pytest
+
 from promptflow._cli._pf.entry import get_parser_args
 from promptflow._cli._utils import _get_cli_activity_name
 
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_config.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_config.py
similarity index 96%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_config.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_config.py
index 768f042a3da..0e43a4979e5 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_config.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_config.py
@@ -1,15 +1,15 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._configuration import Configuration, InvalidConfigValue
 from promptflow._sdk._constants import FLOW_DIRECTORY_MACRO_IN_CONFIG
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
 
-CONFIG_DATA_ROOT = Path(__file__).parent.parent.parent / "test_configs" / "configs"
+CONFIG_DATA_ROOT = PROMPTFLOW_ROOT / "tests" / "test_configs" / "configs"
 
 
 @pytest.fixture
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_connection.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_connection.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_connection.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_connection.py
index 4520fb5f629..c128c41ca63 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_connection.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_connection.py
@@ -2,11 +2,11 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import os
-from pathlib import Path
 from unittest.mock import patch
 
 import mock
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._cli._pf._connection import validate_and_interactive_get_secrets
 from promptflow._sdk._constants import SCRUBBED_VALUE, ConnectionAuthMode, CustomStrongTypeConnectionConfigs
@@ -30,7 +30,7 @@
 from promptflow.core._connection import RequiredEnvironmentVariablesNotSetError
 from promptflow.exceptions import UserErrorException
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 CONNECTION_ROOT = TEST_ROOT / "test_configs/connections"
 
 
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_experiment.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_experiment.py
similarity index 97%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_experiment.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_experiment.py
index f61a68133a0..ca2a9f115b0 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_experiment.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_experiment.py
@@ -1,6 +1,5 @@
-from pathlib import Path
-
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from ruamel.yaml import YAML
 
 from promptflow._sdk._errors import MultipleExperimentTemplateError, NoExperimentTemplateError
@@ -8,7 +7,7 @@
 from promptflow._sdk._orchestrator.experiment_orchestrator import ExperimentTemplateTestContext
 from promptflow._sdk.entities._experiment import Experiment, ExperimentData, ExperimentInput, FlowNode
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 EXP_ROOT = TEST_ROOT / "test_configs/experiments"
 FLOW_ROOT = TEST_ROOT / "test_configs/flows"
 
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_flow.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow.py
similarity index 80%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_flow.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow.py
index d9022df731b..1f71fc29e96 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_flow.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow.py
@@ -6,9 +6,8 @@
 import pytest
 from marshmallow import ValidationError
 
-from promptflow import load_flow
 from promptflow._sdk.entities._flows import FlexFlow, Flow
-from promptflow.exceptions import ValidationException
+from promptflow.client import load_flow
 
 FLOWS_DIR = Path("./tests/test_configs/flows")
 EAGER_FLOWS_DIR = Path("./tests/test_configs/eager_flows")
@@ -66,13 +65,3 @@ def test_flow_load_invalid(self, kwargs, error_message, exception_type):
             load_flow(**kwargs)
 
         assert error_message in str(e.value)
-
-    def test_multiple_flow_load(self):
-        with pytest.raises(ValidationException) as e:
-            load_flow(EAGER_FLOWS_DIR / "multiple_flow_yaml")
-
-        assert "Both flow.dag.yaml and flow.flex.yaml exist in " in str(e.value)
-
-    def test_specify_flow_load(self):
-        load_flow(EAGER_FLOWS_DIR / "multiple_flow_yaml" / "flow.dag.yaml")
-        load_flow(EAGER_FLOWS_DIR / "multiple_flow_yaml" / "flow.flex.yaml")
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_invoker.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_invoker.py
similarity index 92%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_flow_invoker.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_invoker.py
index 2a4bc7a7b8f..98810b982e1 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_invoker.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_invoker.py
@@ -1,17 +1,16 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._sdk._load_functions import load_flow
 from promptflow.core._serving._errors import UnexpectedConnectionProviderReturn, UnsupportedConnectionProvider
 from promptflow.core._serving.flow_invoker import FlowInvoker
 from promptflow.exceptions import UserErrorException
 
-PROMOTFLOW_ROOT = Path(__file__).parent.parent.parent.parent
-FLOWS_DIR = Path(PROMOTFLOW_ROOT / "tests/test_configs/flows")
+FLOWS_DIR = PROMPTFLOW_ROOT / "tests/test_configs/flows"
 EXAMPLE_FLOW_DIR = FLOWS_DIR / "web_classification"
 EXAMPLE_FLOW_FILE = EXAMPLE_FLOW_DIR / "flow.dag.yaml"
 EXAMPLE_FLOW = load_flow(EXAMPLE_FLOW_FILE)
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_serve.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_serve.py
similarity index 69%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_flow_serve.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_serve.py
index e7b8ad5f479..4988b7f02c6 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_flow_serve.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_flow_serve.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 
 import pytest
-from sdk_cli_test.conftest import MODEL_ROOT
+from _constants import PROMPTFLOW_ROOT
 
 from promptflow._cli._pf._flow import _resolve_python_flow_additional_includes
 
@@ -9,12 +9,17 @@
 @pytest.mark.unittest
 def test_flow_serve_resolve_additional_includes():
     # Assert flow path not changed if no additional includes
-    flow_path = (Path(MODEL_ROOT) / "web_classification").resolve().absolute().as_posix()
+    flow_path = (PROMPTFLOW_ROOT / "tests/test_configs/flows/web_classification").resolve().absolute().as_posix()
     resolved_flow_path = _resolve_python_flow_additional_includes(flow_path)
     assert flow_path == resolved_flow_path
 
     # Assert additional includes are resolved correctly
-    flow_path = (Path(MODEL_ROOT) / "web_classification_with_additional_include").resolve().absolute().as_posix()
+    flow_path = (
+        (PROMPTFLOW_ROOT / "tests/test_configs/flows/web_classification_with_additional_include")
+        .resolve()
+        .absolute()
+        .as_posix()
+    )
     resolved_flow_path = _resolve_python_flow_additional_includes(flow_path)
 
     assert (Path(resolved_flow_path) / "convert_to_dict.py").exists()
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_local_storage_operations.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_local_storage_operations.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_local_storage_operations.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_local_storage_operations.py
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_mlflow_dependencies.py
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_orm.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_orm.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_orm.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_orm.py
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_pf_client.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_pf_client.py
similarity index 93%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_pf_client.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_pf_client.py
index d4bd33175d8..d64a6311970 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_pf_client.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_pf_client.py
@@ -3,8 +3,8 @@
 # ---------------------------------------------------------
 import pytest
 
-from promptflow import PFClient
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
+from promptflow.client import PFClient
 
 
 @pytest.mark.sdk_test
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_run.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_run.py
similarity index 99%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_run.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_run.py
index 2943c2e660a..7dfd658cf00 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_run.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_run.py
@@ -22,7 +22,6 @@
 from promptflow._utils.yaml_utils import load_yaml
 from promptflow.exceptions import UserErrorException, ValidationException
 
-PROMOTFLOW_ROOT = Path(__file__) / "../../../.."
 FLOWS_DIR = Path("./tests/test_configs/flows")
 EAGER_FLOWS_DIR = Path("./tests/test_configs/eager_flows")
 RUNS_DIR = Path("./tests/test_configs/runs")
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_tool.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_tool.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_tool.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_tool.py
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_trace.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_trace.py
similarity index 100%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_trace.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_trace.py
diff --git a/src/promptflow/tests/sdk_cli_test/unittests/test_utils.py b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_utils.py
similarity index 91%
rename from src/promptflow/tests/sdk_cli_test/unittests/test_utils.py
rename to src/promptflow-devkit/tests/sdk_cli_test/unittests/test_utils.py
index e57fb41efa2..0d7a55a14b3 100644
--- a/src/promptflow/tests/sdk_cli_test/unittests/test_utils.py
+++ b/src/promptflow-devkit/tests/sdk_cli_test/unittests/test_utils.py
@@ -9,7 +9,6 @@
 import json
 import os
 import shutil
-import subprocess
 import sys
 import tempfile
 import threading
@@ -21,7 +20,7 @@
 import mock
 import pandas as pd
 import pytest
-from pip._vendor import tomli as toml
+from _constants import PROMPTFLOW_ROOT
 from requests import Response
 
 from promptflow._cli._params import AppendToDictAction
@@ -44,16 +43,16 @@
     get_system_info,
     refresh_connections_dir,
 )
+from promptflow._sdk._version_hint_utils import check_latest_version
 from promptflow._utils.load_data import load_data
 from promptflow._utils.retry_utils import http_retry_wrapper, retry
 from promptflow._utils.utils import snake_to_camel
-from promptflow._utils.version_hint_utils import check_latest_version
 from promptflow.core._utils import (
     override_connection_config_with_environment_variable,
     resolve_connections_environment_variable_reference,
 )
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 CONNECTION_ROOT = TEST_ROOT / "test_configs/connections"
 
 
@@ -220,8 +219,8 @@ def mock_check_latest_version():
             time.sleep(5)
             check_latest_version()
 
-        with patch("promptflow._utils.version_hint_utils.datetime") as mock_datetime, patch(
-            "promptflow._utils.version_hint_utils.check_latest_version", side_effect=mock_check_latest_version
+        with patch("promptflow._sdk._version_hint_utils.datetime") as mock_datetime, patch(
+            "promptflow._sdk._version_hint_utils.check_latest_version", side_effect=mock_check_latest_version
         ):
             from promptflow._sdk._telemetry import monitor_operation
 
@@ -490,45 +489,3 @@ def test_gen_uuid_by_compute_info(self):
         system_info_hash = hashlib.sha256((host_name + system + machine).encode()).hexdigest()
         compute_info_hash = hashlib.sha256((mac_address + system_info_hash).encode()).hexdigest()
         assert str(uuid.uuid5(uuid.NAMESPACE_OID, compute_info_hash)) == gen_uuid_by_compute_info()
-
-    def test_executable_package_match_toml_file(self):
-        def get_git_base_dir():
-            return Path(
-                subprocess.run(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE)
-                .stdout.decode("utf-8")
-                .strip()
-            )
-
-        def get_toml_dependencies():
-            packages = ["promptflow-tracing", "promptflow-core", "promptflow-devkit"]
-            dependencies = []
-
-            for package in packages:
-                with open(get_git_base_dir() / "src" / package / "pyproject.toml", "rb") as file:
-                    data = toml.load(file)
-                extra_package_names = data.get("tool", {}).get("poetry", {}).get("dependencies", {})
-                dependencies.extend(extra_package_names.keys())
-            dependencies = [
-                dependency
-                for dependency in dependencies
-                if not dependency.startswith("promptflow") and not dependency == "python"
-            ]
-            return dependencies
-
-        all_packages = get_toml_dependencies()
-
-        with open(
-            get_git_base_dir()
-            / "src"
-            / "promptflow-devkit"
-            / "promptflow"
-            / "_sdk"
-            / "data"
-            / "executable"
-            / "requirements.txt",
-            "r",
-        ) as f:
-            executable_all_packages = f.read().splitlines()
-        # check if all packages in requirements.txt are the same with pyproject.toml in devkit/core/tracinf packages.
-        # If not, maybe you need update requirements.txt
-        assert set(all_packages) == set(executable_all_packages)
diff --git a/src/promptflow/tests/sdk_pfs_test/.coveragerc b/src/promptflow-devkit/tests/sdk_pfs_test/.coveragerc
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/.coveragerc
rename to src/promptflow-devkit/tests/sdk_pfs_test/.coveragerc
diff --git a/src/promptflow/tests/sdk_pfs_test/__init__.py b/src/promptflow-devkit/tests/sdk_pfs_test/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/__init__.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/__init__.py
diff --git a/src/promptflow/tests/sdk_pfs_test/conftest.py b/src/promptflow-devkit/tests/sdk_pfs_test/conftest.py
similarity index 94%
rename from src/promptflow/tests/sdk_pfs_test/conftest.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/conftest.py
index 30dc613db4f..7ece485d6b5 100644
--- a/src/promptflow/tests/sdk_pfs_test/conftest.py
+++ b/src/promptflow-devkit/tests/sdk_pfs_test/conftest.py
@@ -5,7 +5,7 @@
 import pytest
 from flask.app import Flask
 
-from promptflow import PFClient
+from promptflow.client import PFClient
 
 from .utils import PFSOperations
 
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/__init__.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/__init__.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/__init__.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/__init__.py
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_cli.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_cli.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_cli.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_cli.py
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_connection_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_connection_apis.py
similarity index 92%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_connection_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_connection_apis.py
index 4e002a74b70..11a99eada37 100644
--- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_connection_apis.py
+++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_connection_apis.py
@@ -9,9 +9,9 @@
 import mock
 import pytest
 
-from promptflow import PFClient
-from promptflow._sdk.entities import CustomConnection
 from promptflow._sdk._version import VERSION
+from promptflow._sdk.entities import CustomConnection
+from promptflow.client import PFClient
 from promptflow.recording.record_mode import is_replay
 
 from ..utils import PFSOperations, check_activity_end_telemetry
@@ -89,14 +89,8 @@ def test_get_connection_specs(self, pfs_op: PFSOperations) -> None:
     @pytest.mark.skipif(is_replay(), reason="connection provider test, skip in non-live mode.")
     def test_get_connection_by_provicer(self, pfs_op, subscription_id, resource_group_name, workspace_name):
         target = "promptflow._sdk._pf_client.Configuration.get_connection_provider"
-        provider_url_target = (
-            "promptflow._sdk.operations._local_azure_connection_operations."
-            "LocalAzureConnectionOperations._extract_workspace"
-        )
-        mock_provider_url = (subscription_id, resource_group_name, workspace_name)
-        with mock.patch(target) as mocked_config, mock.patch(provider_url_target) as mocked_provider_url:
+        with mock.patch(target) as mocked_config:
             mocked_config.return_value = "azureml"
-            mocked_provider_url.return_value = mock_provider_url
             connections = pfs_op.list_connections(status_code=200).json
             assert len(connections) > 0
 
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_experiment_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_experiment_apis.py
similarity index 98%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_experiment_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_experiment_apis.py
index d238bf4c329..1b24d9d693d 100644
--- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_experiment_apis.py
+++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_experiment_apis.py
@@ -1,13 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from ..utils import PFSOperations, check_activity_end_telemetry
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 EXPERIMENT_ROOT = TEST_ROOT / "test_configs/experiments"
 FLOW_ROOT = TEST_ROOT / "test_configs/flows"
 
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_flow_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_flow_apis.py
similarity index 92%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_flow_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_flow_apis.py
index d3ed544e51a..130026701ae 100644
--- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_flow_apis.py
+++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_flow_apis.py
@@ -2,13 +2,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 
 from ..utils import PFSOperations, check_activity_end_telemetry
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 FLOW_PATH = "./tests/test_configs/flows/print_env_var"
 IMAGE_PATH = "./tests/test_configs/datas/logo.jpg"
 FLOW_WITH_IMAGE_PATH = "./tests/test_configs/flows/chat_flow_with_image"
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_general_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_general_apis.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_general_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_general_apis.py
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_run_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_run_apis.py
similarity index 99%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_run_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_run_apis.py
index ccf9973c911..1640ad1af00 100644
--- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_run_apis.py
+++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_run_apis.py
@@ -9,9 +9,9 @@
 
 import pytest
 
-from promptflow import PFClient
 from promptflow._sdk.entities import Run
 from promptflow._sdk.operations._local_storage_operations import LocalStorageOperations
+from promptflow.client import PFClient
 from promptflow.contracts._run_management import RunMetadata
 
 from ..utils import PFSOperations, check_activity_end_telemetry
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_telemetry_apis.py
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_trace.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_trace.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_trace.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_trace.py
diff --git a/src/promptflow/tests/sdk_pfs_test/e2etests/test_ui_apis.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_ui_apis.py
similarity index 98%
rename from src/promptflow/tests/sdk_pfs_test/e2etests/test_ui_apis.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_ui_apis.py
index 936012ddeda..1b3cc6ea3e2 100644
--- a/src/promptflow/tests/sdk_pfs_test/e2etests/test_ui_apis.py
+++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_ui_apis.py
@@ -8,11 +8,12 @@
 from pathlib import Path
 
 import pytest
+from _constants import PROMPTFLOW_ROOT
 from PIL import Image
 
 from ..utils import PFSOperations, check_activity_end_telemetry
 
-TEST_ROOT = Path(__file__).parent.parent.parent
+TEST_ROOT = PROMPTFLOW_ROOT / "tests"
 FLOW_PATH = "./tests/test_configs/flows/print_env_var"
 IMAGE_PATH = "./tests/test_configs/datas/logo.jpg"
 FLOW_WITH_IMAGE_PATH = "./tests/test_configs/flows/chat_flow_with_image"
diff --git a/src/promptflow/tests/sdk_pfs_test/utils.py b/src/promptflow-devkit/tests/sdk_pfs_test/utils.py
similarity index 100%
rename from src/promptflow/tests/sdk_pfs_test/utils.py
rename to src/promptflow-devkit/tests/sdk_pfs_test/utils.py
diff --git a/src/promptflow-devkit/tests/unittests/test.py b/src/promptflow-devkit/tests/unittests/test.py
deleted file mode 100644
index 226a2ed4261..00000000000
--- a/src/promptflow-devkit/tests/unittests/test.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import pytest
-
-
-@pytest.mark.unittest
-class TestStartTrace:
-    def test_import(self):
-        assert True