python: use Python Environment Tools for interpreter discovery

.github/actions/install-pet/action.yml

@@ -0,0 +1,27 @@
+name: "Setup Python Environment Tools"
+description: "Install and build microsoft/python-environment-tools"
+
+env:
+  PET_BRANCH: "release/latest"
+
+runs:
+  using: "composite"
+  steps:
+      - name: Checkout Python Environment Tools
+        uses: actions/checkout@v4
+        with:
+          repository: 'microsoft/python-environment-tools'
+          path: 'extensions/positron-python/python-env-tools'
+          branch: ${{ env.PET_BRANCH }}
+          sparse-checkout: |
+            crates
+            Cargo.toml
+            Cargo.lock
+          sparse-checkout-cone-mode: false
+
+      - name: Rust Tool Chain setup
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build Native Binaries
+        run: nox --noxfile extensions/positron-python/noxfile.py --session native_build
+        shell: bash

.github/actions/install-python/action.yml

@@ -34,6 +34,7 @@ runs:
         python -m pip install --upgrade pip
         python -m pip install -r requirements.txt
         python -m pip install ipykernel trcli
+        python -m pip install nox
 
     - name: Verify Python Version
       shell: bash

.github/actions/python-build-vsix/action.yml

@@ -0,0 +1,104 @@
+name: 'Build VSIX'
+description: "Build the extension's VSIX"
+
+inputs:
+  node_version:
+    description: 'Version of Node to install'
+    required: true
+  vsix_name:
+    description: 'Name to give the final VSIX'
+    required: true
+  artifact_name:
+    description: 'Name to give the artifact containing the VSIX'
+    required: true
+  cargo_target:
+    description: 'Cargo build target for the native build'
+    required: true
+  vsix_target:
+    description: 'vsix build target for the native build'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install Node
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ inputs.node_version }}
+        cache: 'npm'
+
+    - name: Rust Tool Chain setup
+      uses: dtolnay/rust-toolchain@stable
+
+    # Jedi LS depends on dataclasses which is not in the stdlib in Python 3.7.
+    - name: Use Python 3.8 for JediLSP
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.8
+        cache: 'pip'
+        cache-dependency-path: |
+          extensions/positron-python/requirements.txt
+          extensions/positron-python/python_files/jedilsp_requirements/requirements.txt
+
+    - name: Upgrade Pip
+      run: python -m pip install -U pip
+      shell: bash
+
+    # For faster/better builds of sdists.
+    - name: Install build pre-requisite
+      run: python -m pip install wheel nox
+      shell: bash
+
+    - name: Install Python Extension dependencies (jedi, etc.)
+      run: nox --noxfile extensions/positron-python/noxfile.py --session install_python_libs
+      shell: bash
+
+    - name: Add Rustup target
+      run: rustup target add ${{ inputs.cargo_target }}
+      shell: bash
+
+    - name: Build Native Binaries
+      run: nox --noxfile extensions/positron-python/noxfile.py --session native_build
+      shell: bash
+      env:
+        CARGO_TARGET: ${{ inputs.cargo_target }}
+
+    - name: Run npm ci
+      run: |
+        cd extensions/positron-python
+        npm ci --prefer-offline
+      shell: bash
+
+    - name: Update optional extension dependencies
+      run: |
+        cd extensions/positron-python
+        npm run addExtensionPackDependencies
+      shell: bash
+
+    - name: Build Webpack
+      run: |
+        cd extensions/positron-python
+        npx gulp clean
+        npx gulp prePublishBundle
+      shell: bash
+
+    - name: Build VSIX
+      run: |
+        cd extensions/positron-python
+        npx vsce package --target ${{ inputs.vsix_target }} --out positron-python-dev.vsix --pre-release
+      shell: bash
+
+    - name: Rename VSIX
+      working-directory: extensions/positron-python
+      # Move to a temp name in case the specified name happens to match the default name.
+      run: |
+        cp positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev-temp.vsix ${{ inputs.vsix_name }}
+      shell: bash
+
+    - name: Upload VSIX
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.artifact_name }}
+        path: extensions/positron-python/${{ inputs.vsix_name }}
+        if-no-files-found: error
+        retention-days: 2

.github/actions/python-smoke-tests/action.yml

@@ -0,0 +1,70 @@
+name: 'Smoke tests'
+description: 'Run smoke tests'
+
+inputs:
+  node_version:
+    description: 'Version of Node to install'
+    required: true
+  artifact_name:
+    description: 'Name of the artifact containing the VSIX'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Update working directory
+      run: cd extensions/positron-python
+      shell: bash
+
+    - name: Install Node
+      uses: actions/setup-node@v2
+      with:
+        node-version: ${{ inputs.node_version }}
+        cache: 'npm'
+
+    - name: Install Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+        cache: 'pip'
+        cache-dependency-path: |
+          extensions/positron-python/build/test-requirements.txt
+          extensions/positron-python/requirements.txt
+
+    - name: Install dependencies (npm ci)
+      run: npm ci --prefer-offline
+      shell: bash
+
+    - name: Install Python requirements
+      uses: brettcannon/pip-secure-install@v1
+      with:
+        options: '-t extensions/positron-python/python_files/lib/python --implementation py'
+
+    - name: pip install system test requirements
+      run: |
+        python -m pip install --upgrade -r extensions/positron-python/build/test-requirements.txt
+      shell: bash
+
+    # Bits from the VSIX are reused by smokeTest.ts to speed things up.
+    - name: Download VSIX
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.artifact_name }}
+
+    - name: Prepare for smoke tests
+      run: npx tsc -p ./
+      shell: bash
+
+    - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION
+      run: |
+        echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV
+        echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Run smoke tests
+      env:
+        DISPLAY: 10
+        INSTALL_JUPYTER_EXTENSION: true
+      uses: GabrielBB/[email protected]
+      with:
+        run: node --no-force-async-hooks-checks ./extensions/positron-python/out/test/smokeTest.js

.github/actions/setup-test-env/action.yml

@@ -47,6 +47,9 @@ runs:
       with:
         version: "3.10"
 
+    - name: Setup Python Environment Tools
+      uses: ./.github/actions/install-pet
+
     - name: Setup R
       uses: ./.github/actions/install-r
       with:

.github/workflows/positron-python-ci.yml

@@ -21,7 +21,9 @@ defaults:
 
 env:
   NODE_VERSION: '20.12.1'
+  PET_BRANCH: 'release/latest'
   PYTHON_VERSION: '3.10'
+  ARTIFACT_NAME_VSIX: positron-python-dev-vsix
   PROJECT_DIR: 'extensions/positron-python'
   PYTHON_SRC_DIR: 'extensions/positron-python/python_files'
   # Force a path with spaces and to test extension works in these scenarios
@@ -244,6 +246,18 @@ jobs:
         with:
           path: ${{ env.special-working-directory-relative }}
 
+      - name: Checkout Python Environment Tools
+        uses: actions/checkout@v4
+        with:
+          repository: 'microsoft/python-environment-tools'
+          path: ${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools
+          branch: ${{ env.PET_BRANCH }}
+          sparse-checkout: |
+            crates
+            Cargo.toml
+            Cargo.lock
+          sparse-checkout-cone-mode: false
+
       - name: Install Node ${{ env.NODE_VERSION }}
         uses: actions/setup-node@v3
         with:
@@ -267,7 +281,16 @@ jobs:
         run: npx @vscode/l10n-dev@latest export ./src
 
       - name: Install test requirements
-        run: python -m pip install --upgrade -r ./build/test-requirements.txt
+        run: |
+          python -m pip install --upgrade -r ./build/test-requirements.txt
+          python -m pip install nox
+
+      - name: Rust Tool Chain setup
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build Native Binaries
+        run: nox --session native_build
+        shell: bash
 
       - name: Install functional test requirements
         run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt
@@ -404,8 +427,122 @@ jobs:
         run: npm run test:functional
         if: matrix.test-suite == 'functional'
 
-      - name: Run smoke tests
-        env:
-          POSITRON_GITHUB_PAT: ${{ github.token }}
-        run: npx tsc && node ./out/test/smokeTest.js
-        if: matrix.test-suite == 'smoke'
+  build-vsix:
+    name: Create VSIX
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # - os: windows-latest
+          #   target: x86_64-pc-windows-msvc
+          #   vsix-target: win32-x64
+          # - os: windows-latest
+          #   target: aarch64-pc-windows-msvc
+          #   vsix-target: win32-arm64
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-musl
+            vsix-target: linux-x64
+          # - os: ubuntu-latest
+          #   target: x86_64-unknown-linux-musl
+          #   vsix-target: alpine-x64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Checkout Python Environment Tools
+        uses: actions/checkout@v4
+        with:
+          repository: 'microsoft/python-environment-tools'
+          branch: ${{ env.PET_BRANCH }}
+          path: '${{ env.PROJECT_DIR}}/python-env-tools'
+          sparse-checkout: |
+            crates
+            Cargo.toml
+            Cargo.lock
+          sparse-checkout-cone-mode: false
+
+      - name: Build VSIX
+        uses: ./.github/actions/python-build-vsix
+        with:
+          node_version: ${{ env.NODE_VERSION}}
+          vsix_name: 'positron-python-dev-${{ matrix.vsix-target }}.vsix'
+          artifact_name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}'
+          cargo_target: ${{ matrix.target }}
+          vsix_target: ${{ matrix.vsix-target }}
+          path: '${{ env.PROJECT_DIR}}/python-env-tools'
+
+  smoke-tests:
+    name: Smoke tests
+    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
+    runs-on: ${{ matrix.os }}
+    needs: [build-vsix]
+    strategy:
+      fail-fast: false
+      matrix:
+        # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
+        # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
+        include:
+          # - os: windows-latest
+          #   vsix-target: win32-x64
+          - os: ubuntu-latest
+            vsix-target: linux-x64
+
+    steps:
+      # Need the source to have the tests available.
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Checkout Python Environment Tools
+        uses: actions/checkout@v4
+        with:
+          repository: 'microsoft/python-environment-tools'
+          branch: ${{ env.PET_BRANCH }}
+          path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools'
+          sparse-checkout: |
+            crates
+            Cargo.toml
+            Cargo.lock
+          sparse-checkout-cone-mode: false
+
+      - name: Smoke tests
+        uses: ./.github/actions/python-smoke-tests
+        with:
+          node_version: ${{ env.NODE_VERSION }}
+          artifact_name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}'
+
+  native-tests:
+    name: Native Tests
+    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: ${{ env.special-working-directory }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
+        # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case.
+        os: [ubuntu-latest, windows-latest]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: ${{ env.special-working-directory-relative }}
+
+      - name: Checkout Python Environment Tools
+        uses: actions/checkout@v4
+        with:
+          repository: 'microsoft/python-environment-tools'
+          branch: ${{ env.PET_BRANCH }}
+          path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools'
+          sparse-checkout: |
+            crates
+            Cargo.toml
+            Cargo.lock
+          sparse-checkout-cone-mode: false
+
+      - name: Python Environment Tools tests
+        run: cargo test -- --nocapture
+        working-directory: ${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools