ci: promptflow runtime backward compatibility test

.github/workflows/promptflow-runtime-backward-compatibility-test.yml

@@ -0,0 +1,108 @@
+name: promptflow-runtime-backward-compatibility-test
+on:
+  schedule:
+    - cron: "40 18 * * *" # Every day starting at 2:40 BJT
+  pull_request:
+    paths:
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
+      - src/promptflow/**
+      - src/promptflow-tracing/**
+      - .github/workflows/promptflow-runtime-backward-compatibility-test.yml
+  workflow_dispatch:
+    inputs:
+      filepath:
+        description: file or paths you want to trigger a test
+        required: true
+        default: "./tests/sdk_cli_test ./tests/sdk_pfs_test"
+        type: string
+env:
+  IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+jobs:
+  sdk_cli_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        # reduce resource usage given the test results are irrelevant to python version
+        pythonVersion: ['3.10']
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: set test mode
+      run: |
+        echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV\
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.pythonVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
+      run: |
+        set -xe
+        poetry install -E pyarrow --with ci,test
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-tools
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: Azure login (non pull_request workflow)
+      if: github.event_name != 'pull_request'
+      uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: generate live test resources (non pull_request workflow)
+      if: github.event_name != 'pull_request'
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+    - name: generate live test resources (pull_request workflow)
+      if: github.event_name == 'pull_request'
+      working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+      run: |
+        cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
+    - name: run devkit tests
+      run: |
+        poetry run pip install promptflow==1.3.0
+        # we can't do most of the tests given tests are for latest version; but we can do pf --version to check whether 
+        # there are import errors
+        poetry run pf --version
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp"
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: Upload Test Results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        path: |
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
+          ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
+  publish-test-results-sdk-cli-test:
+    needs: sdk_cli_tests
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-sdk-cli-test.yml
+        testResultTitle: SDK CLI Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 40
+        context: test/sdk_cli