From 2d1658cb031448636e630db0bc177dfb491fefba Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Tue, 7 Jan 2025 13:10:45 -0500 Subject: [PATCH 01/26] install PET in CI --- .github/actions/install-pet/action.yml | 27 +++++++++++++++++++++++ .github/actions/setup-test-env/action.yml | 3 +++ .github/workflows/positron-python-ci.yml | 18 +++++++++++++++ extensions/positron-python/package.json | 2 +- 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 .github/actions/install-pet/action.yml diff --git a/.github/actions/install-pet/action.yml b/.github/actions/install-pet/action.yml new file mode 100644 index 00000000000..9dfe005e10a --- /dev/null +++ b/.github/actions/install-pet/action.yml @@ -0,0 +1,27 @@ +name: "Setup Python Environment Tools" +description: "Install and build microsoft/python-environment-tools" + +defaults: + run: + working-directory: 'extensions/positron-python' + +runs: + using: "composite" + steps: + - name: Checkout Python Environment Tools + uses: actions/checkout@v4 + with: + repository: 'microsoft/python-environment-tools' + path: 'python-env-tools' + sparse-checkout: | + crates + Cargo.toml + Cargo.lock + sparse-checkout-cone-mode: false + + - name: Rust Tool Chain setup + uses: dtolnay/rust-toolchain@stable + + - name: Build Native Binaries + run: nox --session native_build + shell: bash diff --git a/.github/actions/setup-test-env/action.yml b/.github/actions/setup-test-env/action.yml index 950922fce48..239beb6c611 100644 --- a/.github/actions/setup-test-env/action.yml +++ b/.github/actions/setup-test-env/action.yml @@ -47,6 +47,9 @@ runs: with: version: "3.10" + - name: Setup Python Environment Tools + uses: ./.github/actions/install-pet + - name: Setup R uses: ./.github/actions/install-r with: diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index d562341b13b..88a5a808c08 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -244,6 +244,17 @@ jobs: with: path: ${{ env.special-working-directory-relative }} + - name: Checkout Python Environment Tools + uses: actions/checkout@v4 + with: + repository: 'microsoft/python-environment-tools' + path: ${{ env.special-working-directory-relative }}/python-env-tools + sparse-checkout: | + crates + Cargo.toml + Cargo.lock + sparse-checkout-cone-mode: false + - name: Install Node ${{ env.NODE_VERSION }} uses: actions/setup-node@v3 with: @@ -269,6 +280,13 @@ jobs: - name: Install test requirements run: python -m pip install --upgrade -r ./build/test-requirements.txt + - name: Rust Tool Chain setup + uses: dtolnay/rust-toolchain@stable + + - name: Build Native Binaries + run: nox --session native_build + shell: bash + - name: Install functional test requirements run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt if: matrix.test-suite == 'functional' diff --git a/extensions/positron-python/package.json b/extensions/positron-python/package.json index 5365400f74d..c295d31f3ee 100644 --- a/extensions/positron-python/package.json +++ b/extensions/positron-python/package.json @@ -689,7 +689,7 @@ "type": "string" }, "python.locator": { - "default": "js", + "default": "native", "description": "%python.locator.description%", "enum": [ "js", From 72ce2829760aef2701d38fccf8cd24c8ee6e943b Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Tue, 7 Jan 2025 13:24:08 -0500 Subject: [PATCH 02/26] install nox into ci --- .github/actions/install-python/action.yml | 1 + .github/workflows/positron-python-ci.yml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-python/action.yml b/.github/actions/install-python/action.yml index 11cdc188f8c..e4d5568a9f2 100644 --- a/.github/actions/install-python/action.yml +++ b/.github/actions/install-python/action.yml @@ -34,6 +34,7 @@ runs: python -m pip install --upgrade pip python -m pip install -r requirements.txt python -m pip install ipykernel trcli + python -m pip install nox - name: Verify Python Version shell: bash diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 88a5a808c08..82b42d2ad6c 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -278,7 +278,9 @@ jobs: run: npx @vscode/l10n-dev@latest export ./src - name: Install test requirements - run: python -m pip install --upgrade -r ./build/test-requirements.txt + run: | + python -m pip install --upgrade -r ./build/test-requirements.txt + python -m pip install nox - name: Rust Tool Chain setup uses: dtolnay/rust-toolchain@stable From 5d4a4cf06cef3a50c86c464ac55e3969d1602991 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Tue, 7 Jan 2025 13:33:27 -0500 Subject: [PATCH 03/26] take into account different wd for ts tests --- .github/workflows/positron-python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 82b42d2ad6c..aeba1f14f0f 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -248,7 +248,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' - path: ${{ env.special-working-directory-relative }}/python-env-tools + path: ${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools sparse-checkout: | crates Cargo.toml From 3e317cbb488cdf473eb2fd45f691ba772ef52900 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Tue, 7 Jan 2025 15:21:33 -0500 Subject: [PATCH 04/26] specify which noxfile --- .github/actions/install-pet/action.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/install-pet/action.yml b/.github/actions/install-pet/action.yml index 9dfe005e10a..72364f4fc71 100644 --- a/.github/actions/install-pet/action.yml +++ b/.github/actions/install-pet/action.yml @@ -1,10 +1,6 @@ name: "Setup Python Environment Tools" description: "Install and build microsoft/python-environment-tools" -defaults: - run: - working-directory: 'extensions/positron-python' - runs: using: "composite" steps: @@ -12,7 +8,7 @@ runs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' - path: 'python-env-tools' + path: 'extensions/positron-python/python-env-tools' sparse-checkout: | crates Cargo.toml @@ -23,5 +19,5 @@ runs: uses: dtolnay/rust-toolchain@stable - name: Build Native Binaries - run: nox --session native_build + run: nox --noxfile extensions/positron-python/noxfile.py --session native_build shell: bash From 9c2ddba1547974df633c9e81f562667a857351a6 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Wed, 8 Jan 2025 17:03:15 -0500 Subject: [PATCH 05/26] use actions for smoke tests to build vsix --- .github/actions/python-build-vsix/action.yml | 99 +++++++++++++++++++ .github/actions/python-smoke-tests/action.yml | 70 +++++++++++++ .github/workflows/positron-python-ci.yml | 86 +++++++++++++++- 3 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 .github/actions/python-build-vsix/action.yml create mode 100644 .github/actions/python-smoke-tests/action.yml diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml new file mode 100644 index 00000000000..3cda28a19a1 --- /dev/null +++ b/.github/actions/python-build-vsix/action.yml @@ -0,0 +1,99 @@ +name: 'Build VSIX' +description: "Build the extension's VSIX" + +inputs: + node_version: + description: 'Version of Node to install' + required: true + vsix_name: + description: 'Name to give the final VSIX' + required: true + artifact_name: + description: 'Name to give the artifact containing the VSIX' + required: true + cargo_target: + description: 'Cargo build target for the native build' + required: true + vsix_target: + description: 'vsix build target for the native build' + required: true + +runs: + using: 'composite' + steps: + - name: Update working directory + run: cd extensions/positron-python + shell: bash + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node_version }} + cache: 'npm' + + - name: Rust Tool Chain setup + uses: dtolnay/rust-toolchain@stable + + # Jedi LS depends on dataclasses which is not in the stdlib in Python 3.7. + - name: Use Python 3.8 for JediLSP + uses: actions/setup-python@v5 + with: + python-version: 3.8 + cache: 'pip' + cache-dependency-path: | + requirements.txt + python_files/jedilsp_requirements/requirements.txt + + - name: Upgrade Pip + run: python -m pip install -U pip + shell: bash + + # For faster/better builds of sdists. + - name: Install build pre-requisite + run: python -m pip install wheel nox + shell: bash + + - name: Install Python Extension dependencies (jedi, etc.) + run: nox --session install_python_libs + shell: bash + + - name: Add Rustup target + run: rustup target add ${{ inputs.cargo_target }} + shell: bash + + - name: Build Native Binaries + run: nox --session native_build + shell: bash + env: + CARGO_TARGET: ${{ inputs.cargo_target }} + + - name: Run npm ci + run: npm ci --prefer-offline + shell: bash + + - name: Update optional extension dependencies + run: npm run addExtensionPackDependencies + shell: bash + + - name: Build Webpack + run: | + npx gulp clean + npx gulp prePublishBundle + shell: bash + + - name: Build VSIX + run: npx vsce package --target ${{ inputs.vsix_target }} --out ms-python-insiders.vsix --pre-release + shell: bash + + - name: Rename VSIX + # Move to a temp name in case the specified name happens to match the default name. + run: mv ms-python-insiders.vsix ms-python-temp.vsix && mv ms-python-temp.vsix ${{ inputs.vsix_name }} + shell: bash + + - name: Upload VSIX + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + path: ${{ inputs.vsix_name }} + if-no-files-found: error + retention-days: 7 diff --git a/.github/actions/python-smoke-tests/action.yml b/.github/actions/python-smoke-tests/action.yml new file mode 100644 index 00000000000..5e0cd68ddb8 --- /dev/null +++ b/.github/actions/python-smoke-tests/action.yml @@ -0,0 +1,70 @@ +name: 'Smoke tests' +description: 'Run smoke tests' + +inputs: + node_version: + description: 'Version of Node to install' + required: true + artifact_name: + description: 'Name of the artifact containing the VSIX' + required: true + +runs: + using: 'composite' + steps: + - name: Update working directory + run: cd extensions/positron-python + shell: bash + + - name: Install Node + uses: actions/setup-node@v2 + with: + node-version: ${{ inputs.node_version }} + cache: 'npm' + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + cache: 'pip' + cache-dependency-path: | + build/test-requirements.txt + requirements.txt + + - name: Install dependencies (npm ci) + run: npm ci --prefer-offline + shell: bash + + - name: Install Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./python_files/lib/python --implementation py' + + - name: pip install system test requirements + run: | + python -m pip install --upgrade -r build/test-requirements.txt + shell: bash + + # Bits from the VSIX are reused by smokeTest.ts to speed things up. + - name: Download VSIX + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + + - name: Prepare for smoke tests + run: npx tsc -p ./ + shell: bash + + - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION + run: | + echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV + echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV + shell: bash + + - name: Run smoke tests + env: + DISPLAY: 10 + INSTALL_JUPYTER_EXTENSION: true + uses: GabrielBB/xvfb-action@v1.7 + with: + run: node --no-force-async-hooks-checks ./out/test/smokeTest.js diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index aeba1f14f0f..44425e2044f 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -22,6 +22,7 @@ defaults: env: NODE_VERSION: '20.12.1' PYTHON_VERSION: '3.10' + ARTIFACT_NAME_VSIX: positron-python-dev-vsix PROJECT_DIR: 'extensions/positron-python' PYTHON_SRC_DIR: 'extensions/positron-python/python_files' # Force a path with spaces and to test extension works in these scenarios @@ -424,8 +425,83 @@ jobs: run: npm run test:functional if: matrix.test-suite == 'functional' - - name: Run smoke tests - env: - POSITRON_GITHUB_PAT: ${{ github.token }} - run: npx tsc && node ./out/test/smokeTest.js - if: matrix.test-suite == 'smoke' + build-vsix: + name: Create VSIX + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # - os: windows-latest + # target: x86_64-pc-windows-msvc + # vsix-target: win32-x64 + # - os: windows-latest + # target: aarch64-pc-windows-msvc + # vsix-target: win32-arm64 + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + vsix-target: linux-x64 + # - os: ubuntu-latest + # target: x86_64-unknown-linux-musl + # vsix-target: alpine-x64 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Checkout Python Environment Tools + uses: actions/checkout@v4 + with: + repository: 'microsoft/python-environment-tools' + path: 'python-env-tools' + sparse-checkout: | + crates + Cargo.toml + Cargo.lock + sparse-checkout-cone-mode: false + + - name: Build VSIX + uses: ./.github/actions/build-vsix + with: + node_version: ${{ env.NODE_VERSION}} + vsix_name: 'positron-python-dev-${{ matrix.vsix-target }}.vsix' + artifact_name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}' + cargo_target: ${{ matrix.target }} + vsix_target: ${{ matrix.vsix-target }} + + smoke-tests: + name: Smoke tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + needs: [build-vsix] + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. + include: + # - os: windows-latest + # vsix-target: win32-x64 + - os: ubuntu-latest + vsix-target: linux-x64 + + steps: + # Need the source to have the tests available. + - name: Checkout + uses: actions/checkout@v4 + + - name: Checkout Python Environment Tools + uses: actions/checkout@v4 + with: + repository: 'microsoft/python-environment-tools' + path: python-env-tools + sparse-checkout: | + crates + Cargo.toml + Cargo.lock + sparse-checkout-cone-mode: false + + - name: Smoke tests + uses: ./.github/actions/python-smoke-tests + with: + node_version: ${{ env.NODE_VERSION }} + artifact_name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}' From c8db8b09b2954348634b4eab3c6a36113bbe86d5 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Wed, 8 Jan 2025 17:14:35 -0500 Subject: [PATCH 06/26] correct path name --- .github/workflows/positron-python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 44425e2044f..9479566a4d1 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -460,7 +460,7 @@ jobs: sparse-checkout-cone-mode: false - name: Build VSIX - uses: ./.github/actions/build-vsix + uses: ./.github/actions/python-build-vsix with: node_version: ${{ env.NODE_VERSION}} vsix_name: 'positron-python-dev-${{ matrix.vsix-target }}.vsix' From 06d13d4017ee876922513da549f2dc14b668a06c Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Wed, 8 Jan 2025 17:20:52 -0500 Subject: [PATCH 07/26] cd into positron-python every time(?) --- .github/actions/python-build-vsix/action.yml | 31 ++++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 3cda28a19a1..8b1f23bc0e5 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -21,10 +21,6 @@ inputs: runs: using: 'composite' steps: - - name: Update working directory - run: cd extensions/positron-python - shell: bash - - name: Install Node uses: actions/setup-node@v4 with: @@ -41,8 +37,8 @@ runs: python-version: 3.8 cache: 'pip' cache-dependency-path: | - requirements.txt - python_files/jedilsp_requirements/requirements.txt + extensions/positron-python/requirements.txt + extensions/positron-python/python_files/jedilsp_requirements/requirements.txt - name: Upgrade Pip run: python -m pip install -U pip @@ -54,7 +50,7 @@ runs: shell: bash - name: Install Python Extension dependencies (jedi, etc.) - run: nox --session install_python_libs + run: nox --noxfile extensions/positron-python/noxfile.py --session install_python_libs shell: bash - name: Add Rustup target @@ -62,32 +58,41 @@ runs: shell: bash - name: Build Native Binaries - run: nox --session native_build + run: nox --noxfile extensions/positron-python/noxfile.py --session native_build shell: bash env: CARGO_TARGET: ${{ inputs.cargo_target }} - name: Run npm ci - run: npm ci --prefer-offline + run: | + cd extensions/positron-python + npm ci --prefer-offline shell: bash - name: Update optional extension dependencies - run: npm run addExtensionPackDependencies + run: | + cd extensions/positron-python + npm run addExtensionPackDependencies shell: bash - name: Build Webpack run: | + cd extensions/positron-python npx gulp clean npx gulp prePublishBundle shell: bash - name: Build VSIX - run: npx vsce package --target ${{ inputs.vsix_target }} --out ms-python-insiders.vsix --pre-release + run: | + cd extensions/positron-python + npx vsce package --target ${{ inputs.vsix_target }} --out positron-python-dev.vsix --pre-release shell: bash - name: Rename VSIX # Move to a temp name in case the specified name happens to match the default name. - run: mv ms-python-insiders.vsix ms-python-temp.vsix && mv ms-python-temp.vsix ${{ inputs.vsix_name }} + run: | + cd extensions/positron-python + mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} shell: bash - name: Upload VSIX @@ -96,4 +101,4 @@ runs: name: ${{ inputs.artifact_name }} path: ${{ inputs.vsix_name }} if-no-files-found: error - retention-days: 7 + retention-days: 2 From f1d2c1f013d54b195d06846fdb74572bc0b51b61 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Wed, 8 Jan 2025 17:24:31 -0500 Subject: [PATCH 08/26] format yml --- .github/actions/python-build-vsix/action.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 8b1f23bc0e5..891dcc6b4a3 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -65,14 +65,14 @@ runs: - name: Run npm ci run: | - cd extensions/positron-python - npm ci --prefer-offline + cd extensions/positron-python + npm ci --prefer-offline shell: bash - name: Update optional extension dependencies run: | - cd extensions/positron-python - npm run addExtensionPackDependencies + cd extensions/positron-python + npm run addExtensionPackDependencies shell: bash - name: Build Webpack @@ -84,15 +84,15 @@ runs: - name: Build VSIX run: | - cd extensions/positron-python - npx vsce package --target ${{ inputs.vsix_target }} --out positron-python-dev.vsix --pre-release + cd extensions/positron-python + npx vsce package --target ${{ inputs.vsix_target }} --out positron-python-dev.vsix --pre-release shell: bash - name: Rename VSIX # Move to a temp name in case the specified name happens to match the default name. run: | - cd extensions/positron-python - mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} + cd extensions/positron-python + mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} shell: bash - name: Upload VSIX From 8b247edb5dbd6505c255941e01963329398283ac Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Wed, 8 Jan 2025 17:28:35 -0500 Subject: [PATCH 09/26] place pet in project dir --- .github/workflows/positron-python-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 9479566a4d1..3379340eb1d 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -452,7 +452,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' - path: 'python-env-tools' + path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools' sparse-checkout: | crates Cargo.toml @@ -493,7 +493,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' - path: python-env-tools + path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools' sparse-checkout: | crates Cargo.toml From 8ba5db25c76b54c71a509f74c83a57114f979336 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 10:39:15 -0500 Subject: [PATCH 10/26] do not clone with special-working-dir --- .github/workflows/positron-python-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 3379340eb1d..21ed1c4d9f8 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -452,7 +452,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' - path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools' + path: '${{ env.PROJECT_DIR}}/python-env-tools' sparse-checkout: | crates Cargo.toml @@ -467,6 +467,7 @@ jobs: artifact_name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}' cargo_target: ${{ matrix.target }} vsix_target: ${{ matrix.vsix-target }} + path: '${{ env.PROJECT_DIR}}/python-env-tools' smoke-tests: name: Smoke tests From 77efe4032c89e28f4b504f21aaaa3a92728f0b6e Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 15:34:01 -0500 Subject: [PATCH 11/26] debug where vsix is --- .github/actions/python-build-vsix/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 891dcc6b4a3..53d390ff365 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -88,6 +88,10 @@ runs: npx vsce package --target ${{ inputs.vsix_target }} --out positron-python-dev.vsix --pre-release shell: bash + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 30 + - name: Rename VSIX # Move to a temp name in case the specified name happens to match the default name. run: | From 55b7461f300e47b312401fe19721dce152c2f70c Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 15:40:42 -0500 Subject: [PATCH 12/26] rm tmate session --- .github/actions/python-build-vsix/action.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 53d390ff365..73984d37438 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -88,14 +88,11 @@ runs: npx vsce package --target ${{ inputs.vsix_target }} --out positron-python-dev.vsix --pre-release shell: bash - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 30 - - name: Rename VSIX # Move to a temp name in case the specified name happens to match the default name. run: | - cd extensions/positron-python + pwd + ls mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} shell: bash From c0ddffb956f91203aacacc0230fd93a44ca05d17 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 15:59:08 -0500 Subject: [PATCH 13/26] move to ext dir --- .github/actions/python-build-vsix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 73984d37438..89ac87e77ff 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -91,7 +91,7 @@ runs: - name: Rename VSIX # Move to a temp name in case the specified name happens to match the default name. run: | - pwd + cd extensions/positron-python ls mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} shell: bash From b01e18939e4d0851036a27d8d5af6cb7669c9265 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 16:08:43 -0500 Subject: [PATCH 14/26] try quotes perhaps --- .github/actions/python-build-vsix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 89ac87e77ff..fc842aaa191 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -93,7 +93,7 @@ runs: run: | cd extensions/positron-python ls - mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} + mv "positron-python-dev.vsix" "positron-python-dev-temp.vsix" && mv "positron-python-dev.vsix" "${{ inputs.vsix_name }}" shell: bash - name: Upload VSIX From 383cf538c494078beff416f4768bdff0354b406b Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 16:19:51 -0500 Subject: [PATCH 15/26] try to move file again --- .github/actions/python-build-vsix/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index fc842aaa191..28aa3ac8f4c 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -92,8 +92,9 @@ runs: # Move to a temp name in case the specified name happens to match the default name. run: | cd extensions/positron-python - ls - mv "positron-python-dev.vsix" "positron-python-dev-temp.vsix" && mv "positron-python-dev.vsix" "${{ inputs.vsix_name }}" + ls -la + mv /home/runner/work/positron/positron/extensions/positron-python/positron-python-dev.vsix /home/runner/work/positron/positron/extensions/positron-python/positron-python-dev-temp.vsix + mv /home/runner/work/positron/positron/extensions/positron-python/positron-python-dev.vsix /home/runner/work/positron/positron/extensions/positron-python/${{ inputs.vsix_name }} shell: bash - name: Upload VSIX From c5a64c9cd6d5757ae8edebc33c9db9b7fa9620ce Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 16:57:09 -0500 Subject: [PATCH 16/26] using working-directory arg --- .github/actions/python-build-vsix/action.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 28aa3ac8f4c..a01c9b3467f 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -89,15 +89,13 @@ runs: shell: bash - name: Rename VSIX + working-directory: extensions/positron-python # Move to a temp name in case the specified name happens to match the default name. - run: | - cd extensions/positron-python - ls -la - mv /home/runner/work/positron/positron/extensions/positron-python/positron-python-dev.vsix /home/runner/work/positron/positron/extensions/positron-python/positron-python-dev-temp.vsix - mv /home/runner/work/positron/positron/extensions/positron-python/positron-python-dev.vsix /home/runner/work/positron/positron/extensions/positron-python/${{ inputs.vsix_name }} + run: mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} shell: bash - name: Upload VSIX + working-directory: extensions/positron-python uses: actions/upload-artifact@v4 with: name: ${{ inputs.artifact_name }} From 018cd9f4d2edbeac8bf2e1011a4ed22d71cf2d55 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 9 Jan 2025 17:05:01 -0500 Subject: [PATCH 17/26] no working dir for upload --- .github/actions/python-build-vsix/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index a01c9b3467f..248294af000 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -91,11 +91,11 @@ runs: - name: Rename VSIX working-directory: extensions/positron-python # Move to a temp name in case the specified name happens to match the default name. - run: mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} + run: | + mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} shell: bash - name: Upload VSIX - working-directory: extensions/positron-python uses: actions/upload-artifact@v4 with: name: ${{ inputs.artifact_name }} From 2d9ca1b280b15cf066004fef8d86f2b7beb9985a Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 10 Jan 2025 14:10:51 -0500 Subject: [PATCH 18/26] use cp instead of mv --- .github/actions/python-build-vsix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index 248294af000..aede3099581 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -92,7 +92,7 @@ runs: working-directory: extensions/positron-python # Move to a temp name in case the specified name happens to match the default name. run: | - mv positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev.vsix ${{ inputs.vsix_name }} + cp positron-python-dev.vsix positron-python-dev-temp.vsix && mv positron-python-dev-temp.vsix ${{ inputs.vsix_name }} shell: bash - name: Upload VSIX From c57dca8877abfb07c6e8e3886fe8d5b5e45e1e14 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 10 Jan 2025 14:25:08 -0500 Subject: [PATCH 19/26] upload vsix for smoke tests --- .github/actions/python-build-vsix/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/python-build-vsix/action.yml b/.github/actions/python-build-vsix/action.yml index aede3099581..79b9950dc4a 100644 --- a/.github/actions/python-build-vsix/action.yml +++ b/.github/actions/python-build-vsix/action.yml @@ -99,6 +99,6 @@ runs: uses: actions/upload-artifact@v4 with: name: ${{ inputs.artifact_name }} - path: ${{ inputs.vsix_name }} + path: extensions/positron-python/${{ inputs.vsix_name }} if-no-files-found: error retention-days: 2 From db5cf4cb3e010234c685068683a8f14b0d77a278 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 10 Jan 2025 15:58:43 -0500 Subject: [PATCH 20/26] update paths for smoke tests --- .github/actions/python-smoke-tests/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/python-smoke-tests/action.yml b/.github/actions/python-smoke-tests/action.yml index 5e0cd68ddb8..cc8aa8abb71 100644 --- a/.github/actions/python-smoke-tests/action.yml +++ b/.github/actions/python-smoke-tests/action.yml @@ -28,8 +28,8 @@ runs: python-version: '3.x' cache: 'pip' cache-dependency-path: | - build/test-requirements.txt - requirements.txt + extensions/positron-python/build/test-requirements.txt + extensions/positron-python/equirements.txt - name: Install dependencies (npm ci) run: npm ci --prefer-offline @@ -38,11 +38,11 @@ runs: - name: Install Python requirements uses: brettcannon/pip-secure-install@v1 with: - options: '-t ./python_files/lib/python --implementation py' + options: '-t extensions/positron-python/python_files/lib/python --implementation py' - name: pip install system test requirements run: | - python -m pip install --upgrade -r build/test-requirements.txt + python -m pip install --upgrade -r extensions/positron-python/build/test-requirements.txt shell: bash # Bits from the VSIX are reused by smokeTest.ts to speed things up. @@ -67,4 +67,4 @@ runs: INSTALL_JUPYTER_EXTENSION: true uses: GabrielBB/xvfb-action@v1.7 with: - run: node --no-force-async-hooks-checks ./out/test/smokeTest.js + run: node --no-force-async-hooks-checks ./extensions/positron-python/out/test/smokeTest.js From 69dbdb135308e6ffd4e5b96f6a9911b234a37cb4 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Mon, 13 Jan 2025 15:25:44 -0500 Subject: [PATCH 21/26] include version in interpreter name --- .github/actions/python-smoke-tests/action.yml | 2 +- extensions/positron-python/package.json | 2 +- extensions/positron-python/src/client/positron/runtime.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/python-smoke-tests/action.yml b/.github/actions/python-smoke-tests/action.yml index cc8aa8abb71..12a8f5a41a3 100644 --- a/.github/actions/python-smoke-tests/action.yml +++ b/.github/actions/python-smoke-tests/action.yml @@ -29,7 +29,7 @@ runs: cache: 'pip' cache-dependency-path: | extensions/positron-python/build/test-requirements.txt - extensions/positron-python/equirements.txt + extensions/positron-python/requirements.txt - name: Install dependencies (npm ci) run: npm ci --prefer-offline diff --git a/extensions/positron-python/package.json b/extensions/positron-python/package.json index c295d31f3ee..8eab78917e3 100644 --- a/extensions/positron-python/package.json +++ b/extensions/positron-python/package.json @@ -699,7 +699,7 @@ "onExP", "preview" ], - "scope": "machine", + "scope": "machine-overridable", "type": "string" }, "python.pipenvPath": { diff --git a/extensions/positron-python/src/client/positron/runtime.ts b/extensions/positron-python/src/client/positron/runtime.ts index 0d7c91cbae1..728ebbc01c3 100644 --- a/extensions/positron-python/src/client/positron/runtime.ts +++ b/extensions/positron-python/src/client/positron/runtime.ts @@ -66,7 +66,7 @@ export async function createPythonRuntimeMetadata( traceInfo(`createPythonRuntime: startup behavior: ${startupBehavior}`); // Get the Python version from sysVersion since only that includes alpha/beta info (e.g '3.12.0b1') - const pythonVersion = interpreter.sysVersion?.split(' ')[0] ?? '0.0.1'; + const pythonVersion = interpreter.sysVersion?.split(' ')[0] || interpreter.version?.raw || '0.0.1'; const envName = interpreter.envName ?? ''; const runtimeSource = interpreter.envType; From aed1167b3bf04325f68584dd59f92babaff4de32 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Mon, 13 Jan 2025 17:07:50 -0500 Subject: [PATCH 22/26] add pet specific tests --- .github/actions/install-pet/action.yml | 4 + .github/workflows/positron-python-ci.yml | 40 ++++++++ .github/workflows/positron-python-nightly.yml | 36 ++++++++ .../top-action-bar/interpreter-pet.test.ts | 92 +++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 test/e2e/areas/top-action-bar/interpreter-pet.test.ts diff --git a/.github/actions/install-pet/action.yml b/.github/actions/install-pet/action.yml index 72364f4fc71..8b746b14b57 100644 --- a/.github/actions/install-pet/action.yml +++ b/.github/actions/install-pet/action.yml @@ -1,6 +1,9 @@ name: "Setup Python Environment Tools" description: "Install and build microsoft/python-environment-tools" +env: + PET_BRANCH: "release/latest" + runs: using: "composite" steps: @@ -9,6 +12,7 @@ runs: with: repository: 'microsoft/python-environment-tools' path: 'extensions/positron-python/python-env-tools' + branch: ${{ env.PET_BRANCH }} sparse-checkout: | crates Cargo.toml diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 21ed1c4d9f8..4747c55f726 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -21,6 +21,7 @@ defaults: env: NODE_VERSION: '20.12.1' + PET_BRANCH: 'release/latest' PYTHON_VERSION: '3.10' ARTIFACT_NAME_VSIX: positron-python-dev-vsix PROJECT_DIR: 'extensions/positron-python' @@ -250,6 +251,7 @@ jobs: with: repository: 'microsoft/python-environment-tools' path: ${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools + branch: ${{ env.PET_BRANCH }} sparse-checkout: | crates Cargo.toml @@ -452,6 +454,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' + branch: ${{ env.PET_BRANCH }} path: '${{ env.PROJECT_DIR}}/python-env-tools' sparse-checkout: | crates @@ -494,6 +497,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'microsoft/python-environment-tools' + branch: ${{ env.PET_BRANCH }} path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools' sparse-checkout: | crates @@ -506,3 +510,39 @@ jobs: with: node_version: ${{ env.NODE_VERSION }} artifact_name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}' + + native-tests: + name: Native Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Checkout Python Environment Tools + uses: actions/checkout@v4 + with: + repository: 'microsoft/python-environment-tools' + branch: ${{ env.PET_BRANCH }} + path: '${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools' + sparse-checkout: | + crates + Cargo.toml + Cargo.lock + sparse-checkout-cone-mode: false + + - name: Python Environment Tools tests + run: cargo test -- --nocapture + working-directory: ${{ env.special-working-directory }}/python-env-tools diff --git a/.github/workflows/positron-python-nightly.yml b/.github/workflows/positron-python-nightly.yml index 1dcacbe3557..a8ecb69f80d 100644 --- a/.github/workflows/positron-python-nightly.yml +++ b/.github/workflows/positron-python-nightly.yml @@ -12,6 +12,7 @@ defaults: env: NODE_VERSION: '18.17.1' + PET_BRANCH: 'release/latest' PYTHON_VERSION: '3.10' PROJECT_DIR: 'extensions/positron-python' PYTHON_SRC_DIR: 'extensions/positron-python/python_files' @@ -128,6 +129,41 @@ jobs: name: ipykernel-test-output-${{ matrix.os }}-${{ matrix.python }} path: extensions/positron-python/python-test-results.xml + native-tests: + name: Native Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Checkout Python Environment Tools + uses: actions/checkout@v4 + with: + repository: 'microsoft/python-environment-tools' + path: ${{ env.special-working-directory-relative }}/python-env-tools + sparse-checkout: | + crates + Cargo.toml + Cargo.lock + sparse-checkout-cone-mode: false + + - name: Python Environment Tools tests + run: cargo test -- --nocapture + working-directory: ${{ env.special-working-directory }}/python-env-tools + slack-notification: name: 'Send Slack notification' runs-on: ubuntu-latest diff --git a/test/e2e/areas/top-action-bar/interpreter-pet.test.ts b/test/e2e/areas/top-action-bar/interpreter-pet.test.ts new file mode 100644 index 00000000000..89b35dc092d --- /dev/null +++ b/test/e2e/areas/top-action-bar/interpreter-pet.test.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2024 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + PositronConsole, + PositronInterpreterDropdown, +} from '../../../automation'; + +import { test, expect, tags } from '../_test.setup'; + +test.use({ + suiteId: __filename +}); + +test.describe('Interpreter Dropdown in Top Action Bar with PET', { tag: [tags.WEB, tags.TOP_ACTION_BAR] }, () => { + let interpreterDropdown: PositronInterpreterDropdown; + let positronConsole: PositronConsole; + + test.beforeAll(async function ({ app }) { + interpreterDropdown = app.workbench.positronInterpreterDropdown; + positronConsole = app.workbench.positronConsole; + }); + + test.beforeAll(async function ({ userSettings }) { + await userSettings.set([['python.locator', 'native']]); + }); + + test('Python interpreter starts and shows running [C707212]', async function ({ app }) { + const desiredPython = process.env.POSITRON_PY_VER_SEL!; + + + // Start a Python interpreter using the interpreter dropdown + await expect( + async () => + await interpreterDropdown.selectInterpreter('Python', desiredPython) + ).toPass({ timeout: 30_000 }); + + // Install ipykernel if prompted + if (await app.workbench.positronPopups.popupCurrentlyOpen()) { + await app.workbench.positronPopups.installIPyKernel(); + } + + // Wait for the console to be ready + await positronConsole.waitForReady('>>>', 10_000); + + // The interpreter selected in the dropdown matches the desired interpreter + const interpreterInfo = + await interpreterDropdown.getSelectedInterpreterInfo(); + expect(interpreterInfo?.version).toBeDefined(); + expect(interpreterInfo!.version).toContain(desiredPython); + expect(interpreterInfo!.path).toBeDefined(); + + // The interpreter dropdown should show the expected running indicators + await expect(async () => { + expect( + await interpreterDropdown.primaryInterpreterShowsRunning( + interpreterInfo!.path + ) + ).toBe(true); + }).toPass({ timeout: 30_000 }); + + // Close the interpreter dropdown. + await interpreterDropdown.closeInterpreterDropdown(); + }); + + test('Python interpreter restarts and shows running [C707213]', async function ({ python }) { + // Restart the active Python interpreter + await interpreterDropdown.restartPrimaryInterpreter('Python'); + + // Close the interpreter dropdown. + await interpreterDropdown.closeInterpreterDropdown(); + + // The console should indicate that the interpreter is restarting + await positronConsole.waitForConsoleContents('preparing for restart'); + await positronConsole.waitForConsoleContents('restarted'); + + // Wait for the console to be ready + await positronConsole.waitForReady('>>>', 10_000); + + // The interpreter dropdown should show the expected running indicators + await expect(async () => { + expect( + await interpreterDropdown.primaryInterpreterShowsRunning('Python') + ).toBe(true); + }).toPass({ timeout: 30_000 }); + + // Close the interpreter dropdown. + await interpreterDropdown.closeInterpreterDropdown(); + }); +}); From 5dcf7bffcfbcd4f78d4c0eb1f14baa569c19e541 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Mon, 13 Jan 2025 17:13:49 -0500 Subject: [PATCH 23/26] include PROJECT_DIR for cargo tests --- .github/workflows/positron-python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/positron-python-ci.yml b/.github/workflows/positron-python-ci.yml index 4747c55f726..66de01ff871 100644 --- a/.github/workflows/positron-python-ci.yml +++ b/.github/workflows/positron-python-ci.yml @@ -545,4 +545,4 @@ jobs: - name: Python Environment Tools tests run: cargo test -- --nocapture - working-directory: ${{ env.special-working-directory }}/python-env-tools + working-directory: ${{ env.special-working-directory }}/${{ env.PROJECT_DIR}}/python-env-tools From ad4dbb15795ce577509a1d5034643b1e54db5183 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Tue, 14 Jan 2025 15:14:58 -0500 Subject: [PATCH 24/26] scripts to download and unzip python-env-tools --- .../positron-python/scripts/install-pet.ts | 363 ++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 extensions/positron-python/scripts/install-pet.ts diff --git a/extensions/positron-python/scripts/install-pet.ts b/extensions/positron-python/scripts/install-pet.ts new file mode 100644 index 00000000000..10898688848 --- /dev/null +++ b/extensions/positron-python/scripts/install-pet.ts @@ -0,0 +1,363 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2023-2024 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as decompress from 'decompress'; +import * as fs from 'fs'; +import { IncomingMessage } from 'http'; +import * as https from 'https'; +import * as path from 'path'; +import { promisify } from 'util'; + +/** + * This script is a forked copy of the `install-kernel` script from the + * positron-r and positron-supervisor extension; it is responsible for downloading + * a copy of the Python Environment Tools repo and/or using a local version. + * + * In the future, we could consider some way to share this script between the + * extensions (note that some URLs, paths, and messages are different) or + * provide a shared library for downloading and installing binaries from Github + * releases. + */ + +// Promisify some filesystem functions. +const readFileAsync = promisify(fs.readFile); +const writeFileAsync = promisify(fs.writeFile); +const existsAsync = promisify(fs.exists); + +// Create a promisified version of https.get. We can't use the built-in promisify +// because the callback doesn't follow the promise convention of (error, result). +const httpsGetAsync = (opts: https.RequestOptions) => { + return new Promise((resolve, reject) => { + const req = https.get(opts, resolve); + req.once('error', reject); + }); +}; + +/** + * Gets the version of Python Environment Tool specified in package.json. + * + * @returns The version of Python Environment Tool specified in package.json, or null if it cannot be determined. + */ +async function getVersionFromPackageJson(): Promise { + try { + const packageJson = JSON.parse(await readFileAsync('package.json', 'utf-8')); + return packageJson.positron.externalDependencies?.pet || null; + } catch (error) { + throw new Error(`Error reading package.json: ${error}`); + } +} + +/** + * Gets the version of Python Environment Tools installed locally by reading a `VERSION` file that's written + * by this `install-kernel` script. + * + * @returns The version of Python Environment Tool installed locally, or null if PET is not installed. + */ +async function getLocalPetVersion(): Promise { + const versionFile = path.join('resources', 'pet', 'VERSION'); + try { + const petExists = await existsAsync(versionFile); + if (!petExists) { + return null; + } + return readFileAsync(versionFile, 'utf-8'); + } catch (error) { + throw new Error(`Error determining Python Environment Tools version: ${error}`); + } +} + +/** + * Helper to execute a command and return the stdout and stderr. + * + * @param command The command to execute. + * @param stdin Optional stdin to pass to the command. + * @returns A promise that resolves with the stdout and stderr of the command. + */ +async function executeCommand(command: string, stdin?: string): + Promise<{ stdout: string; stderr: string }> { + const { exec } = require('child_process'); + return new Promise((resolve, reject) => { + const process = exec(command, (error: any, stdout: string, stderr: string) => { + if (error) { + reject(error); + } else { + resolve({ stdout, stderr }); + } + }); + if (stdin) { + process.stdin.write(stdin); + process.stdin.end(); + } + }); +} + +/** + * Downloads the specified version of Python Environment Tool and replaces the local directory. + * + * @param version The version of Python Environment Tool to download. + * @param githubPat A Github Personal Access Token with the appropriate rights + * to download the release. + * @param gitCredential Whether the PAT originated from the `git credential` command. + */ +async function downloadAndReplacePet(version: string, + githubPat: string, + gitCredential: boolean): Promise { + + try { + const headers: Record = { + 'Accept': 'application/vnd.github.v3.raw', // eslint-disable-line + 'User-Agent': 'positron-pet-downloader' // eslint-disable-line + }; + // If we have a githubPat, set it for better rate limiting. + if (githubPat) { + headers.Authorization = `token ${githubPat}`; + } + const requestOptions: https.RequestOptions = { + headers, + method: 'GET', + protocol: 'https:', + hostname: 'api.github.com', + path: `/repos/microsoft/python-environment-tools/releases` + }; + + const response = await httpsGetAsync(requestOptions as any) as any; + + // Special handling for PATs originating from `git credential`. + if (gitCredential && response.statusCode === 200) { + // If the PAT hasn't been approved yet, do so now. This stores the credential in + // the system credential store (or whatever `git credential` uses on the system). + // Without this step, the user will be prompted for a username and password the + // next time they try to download PET. + const { stdout, stderr } = + await executeCommand('git credential approve', + `protocol=https\n` + + `host=github.com\n` + + `path=/repos/microsoft/python-environment-tools/releases\n` + + `username=\n` + + `password=${githubPat}\n`); + console.log(stdout); + if (stderr) { + console.warn(`Unable to approve PAT. You may be prompted for a username and ` + + `password the next time you download Python Environment Tools.`); + console.error(stderr); + } + } else if (gitCredential && response.statusCode > 400 && response.statusCode < 500) { + // This handles the case wherein we got an invalid PAT from `git credential`. In this + // case we need to clean up the PAT from the credential store, so that we don't + // continue to use it. + const { stdout, stderr } = + await executeCommand('git credential reject', + `protocol=https\n` + + `host=github.com\n` + + `path=/repos/microsoft/python-environment-tools/releases\n` + + `username=\n` + + `password=${githubPat}\n`); + console.log(stdout); + if (stderr) { + console.error(stderr); + throw new Error(`The stored PAT returned by 'git credential' is invalid, but\n` + + `could not be removed. Please manually remove the PAT from 'git credential'\n` + + `for the host 'github.com'`); + } + throw new Error(`The PAT returned by 'git credential' is invalid. Python Environment Tool cannot be\n` + + `downloaded.\n\n` + + `Check to be sure that your Personal Access Token:\n` + + '- Has the `repo` scope\n' + + '- Is not expired\n' + + '- Has been authorized for the "posit-dev" organization on Github (Configure SSO)\n'); + } + + let responseBody = ''; + + response.on('data', (chunk: any) => { + responseBody += chunk; + }); + + response.on('end', async () => { + if (response.statusCode !== 200) { + throw new Error(`Failed to download Python Environment Tool: HTTP ${response.statusCode}\n\n` + + `${responseBody}`); + } + const releases = JSON.parse(responseBody); + if (!Array.isArray(releases)) { + throw new Error(`Unexpected response from Github:\n\n` + + `${responseBody}`); + } + const release = releases.find((asset: any) => asset.tag_name == version); + if (!release) { + throw new Error(`Could not find Python Environment Tool ${version} in the releases.`); + } + const zipUrl = release.zipball_url; + if (!zipUrl) { + throw new Error(`Could not find Python Environment Tool with asset name ${version} in the release.`); + } + console.log(`Downloading Python Environment Tool ${version} from ${zipUrl}...`); + const url = new URL(zipUrl); + // Reset the Accept header to download the asset. + headers.Accept = 'application/json'; + const requestOptions: https.RequestOptions = { + headers, + method: 'GET', + protocol: url.protocol, + hostname: url.hostname, + path: url.pathname + }; + + let dlResponse = await httpsGetAsync(requestOptions) as any; + while (dlResponse.statusCode === 302) { + // Follow redirects. + dlResponse = await httpsGetAsync(dlResponse.headers.location) as any; + } + let binaryData = Buffer.alloc(0); + + dlResponse.on('data', (chunk: any) => { + binaryData = Buffer.concat([binaryData, chunk]); + }); + dlResponse.on('end', async () => { + const extensionParent = path.dirname(__dirname); + const petDir = path.join(extensionParent, 'python-env-tools'); + // Create the resources/pet directory if it doesn't exist. + if (!await existsAsync(petDir)) { + await fs.promises.mkdir(petDir); + } + + console.log(`Successfully downloaded PET ${version} (${binaryData.length} bytes).`); + const zipFileDest = path.join(petDir, 'pet.zip'); + await writeFileAsync(zipFileDest, binaryData); + + await decompress(zipFileDest, petDir, { strip: 1 }).then(files => { + console.log(`Successfully unzipped Python Environment Tool ${version}.`); + }); + + // Clean up the zipfile. + await fs.promises.unlink(zipFileDest); + + // Write a VERSION file with the version number. + await writeFileAsync(path.join('resources', 'pet', 'VERSION'), version); + + }); + }); + } catch (error) { + throw new Error(`Error downloading Pet: ${error}`); + } +} + +async function main() { + // Before we do any work, check to see if there is a locally built copy of + // the Python Environment Tool in the `pet / target` directory. If so, we'll assume + // that the user is a kernel developer and skip the download; this version + // will take precedence over any downloaded version. + const extensionParent = path.dirname(__dirname); + const petFolder = path.join(extensionParent, 'python-env-tools'); + if (fs.existsSync(petFolder)) { + console.log(`Using locally built PET in ${petFolder}.`); + // TODO: need this? + // Copy the locally built PET to the resources/PET directory. It won't + // be read from this directory at runtime, but we need to put it here + // so that `yarn gulp vscode` will package it up (the packaging step + // doesn't look for a sideloaded PET from an adjacent `pet` directory). + // fs.mkdirSync(path.join('resources', 'pet'), { recursive: true }); + // fs.copyFileSync(binary, path.join('resources', 'pet', kernelName)); + return; + } else { + console.log(`No locally built Python Environment Tool found in ${petFolder}; ` + + `checking downloaded version.`); + } + + const packageJsonVersion = await getVersionFromPackageJson(); + const localPetVersion = await getLocalPetVersion(); + + if (!packageJsonVersion) { + throw new Error('Could not determine PET version from package.json.'); + } + + console.log(`package.json version: ${packageJsonVersion} `); + console.log(`Downloaded PET version: ${localPetVersion ? localPetVersion : 'Not found'} `); + + if (packageJsonVersion === localPetVersion) { + console.log('Versions match. No action required.'); + return; + } + + // We need a Github Personal Access Token (PAT) to download PET. Because this is sensitive + // information, there are a lot of ways to set it. We try the following in order: + + // (1) The GITHUB_PAT environment variable. + // (2) The POSITRON_GITHUB_PAT environment variable. + // (3) The git config setting 'credential.https://api.github.com.token'. + // (4) The git credential store. + + // (1) Get the GITHUB_PAT from the environment. + let githubPat = process.env.GITHUB_PAT; + let gitCredential = false; + if (githubPat) { + console.log('Using Github PAT from environment variable GITHUB_PAT.'); + } else { + // (2) Try POSITRON_GITHUB_PAT (it's what the build script sets) + githubPat = process.env.POSITRON_GITHUB_PAT; + if (githubPat) { + console.log('Using Github PAT from environment variable POSITRON_GITHUB_PAT.'); + } + } + + // (3) If no GITHUB_PAT is set, try to get it from git config. This provides a + // convenient non-interactive way to set the PAT. + if (!githubPat) { + try { + const { stdout, stderr } = + await executeCommand('git config --get credential.https://api.github.com.token'); + githubPat = stdout.trim(); + if (githubPat) { + console.log(`Using Github PAT from git config setting ` + + `'credential.https://api.github.com.token'.`); + } + } catch (error) { + // We don't care if this fails; we'll try `git credential` next. + } + } + + // (4) If no GITHUB_PAT is set, try to get it from git credential. + if (!githubPat) { + // Explain to the user what's about to happen. + console.log(`Attempting to retrieve a Github Personal Access Token from git in order\n` + + `to download Python Environment Tool ${packageJsonVersion}. If you are prompted for a username and\n` + + `password, enter your Github username and a Personal Access Token with the\n` + + `'repo' scope. You can read about how to create a Personal Access Token here: \n` + + `\n` + + `https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens\n` + + `\n` + + `If you don't want to set up a Personal Access Token now, just press Enter twice to set \n` + + `a blank value for the password. Python Environment Tool will not be downloaded, but you will still be\n` + + `able to run Positron with Python support.\n` + + `\n` + + `You can set a PAT later by running yarn again and supplying the PAT at this prompt,\n` + + `or by running 'git config credential.https://api.github.com.token YOUR_GITHUB_PAT'\n`); + const { stdout, stderr } = + await executeCommand('git credential fill', + `protocol=https\n` + + `host=github.com\n` + + `path=/repos/posit-dev/pet/releases\n`); + + gitCredential = true; + // Extract the `password = ` line from the output. + const passwordLine = stdout.split('\n').find( + (line: string) => line.startsWith('password=')); + if (passwordLine) { + githubPat = passwordLine.split('=')[1]; + console.log(`Using Github PAT returned from 'git credential'.`); + } + } + + if (!githubPat) { + throw new Error(`No Github PAT was found. Unable to download PET ${packageJsonVersion}.\n` + + `You can still run Positron with Python Support.`); + } + + await downloadAndReplacePet(packageJsonVersion, githubPat, gitCredential); +} + +main().catch((error) => { + console.error('An error occurred:', error); +}); From 8fc277c349fff516282ffa85f6f3ce93e1805113 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Tue, 14 Jan 2025 15:51:33 -0500 Subject: [PATCH 25/26] script to build pet --- extensions/positron-python/package.json | 7 ++++++- extensions/positron-python/scripts/build-pet.ts | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 extensions/positron-python/scripts/build-pet.ts diff --git a/extensions/positron-python/package.json b/extensions/positron-python/package.json index 8eab78917e3..06dead15582 100644 --- a/extensions/positron-python/package.json +++ b/extensions/positron-python/package.json @@ -1972,5 +1972,10 @@ "webpack-require-from": "^1.8.6", "worker-loader": "^3.0.8", "yargs": "^15.3.1" - } + }, + "positron": { + "externalDependencies": { + "pet": "v2024.16.0" + } + } } diff --git a/extensions/positron-python/scripts/build-pet.ts b/extensions/positron-python/scripts/build-pet.ts new file mode 100644 index 00000000000..63d9ba5a85a --- /dev/null +++ b/extensions/positron-python/scripts/build-pet.ts @@ -0,0 +1,13 @@ +import { exec } from 'child_process'; + +exec('nox --session native_build', (error, stdout, stderr) => { + if (error) { + console.error(`Error: ${error.message}`); + return; + } + if (stderr) { + console.error(`Stderr: ${stderr}`); + return; + } + console.log(`Stdout: ${stdout}`); +}); From b6f7586887277c30eb520dbf445e658682c2b8ca Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Tue, 14 Jan 2025 16:30:52 -0500 Subject: [PATCH 26/26] copyright headers --- extensions/positron-python/.gitignore | 1 + extensions/positron-python/scripts/build-pet.ts | 5 +++++ extensions/positron-python/scripts/install-pet.ts | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/extensions/positron-python/.gitignore b/extensions/positron-python/.gitignore index 9a0ff28b190..98c08c1be17 100644 --- a/extensions/positron-python/.gitignore +++ b/extensions/positron-python/.gitignore @@ -53,6 +53,7 @@ tags # --- Start Positron --- python_files/positron/positron_ipykernel/tests/images python_files/positron/positron_ipykernel/_vendor/** +scripts/*.js # --- End Positron --- python-env-tools/** # coverage files produced as test output diff --git a/extensions/positron-python/scripts/build-pet.ts b/extensions/positron-python/scripts/build-pet.ts index 63d9ba5a85a..53e4a661c87 100644 --- a/extensions/positron-python/scripts/build-pet.ts +++ b/extensions/positron-python/scripts/build-pet.ts @@ -1,3 +1,8 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2024 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + import { exec } from 'child_process'; exec('nox --session native_build', (error, stdout, stderr) => { diff --git a/extensions/positron-python/scripts/install-pet.ts b/extensions/positron-python/scripts/install-pet.ts index 10898688848..73c7b3a5ec8 100644 --- a/extensions/positron-python/scripts/install-pet.ts +++ b/extensions/positron-python/scripts/install-pet.ts @@ -1,5 +1,5 @@ /*--------------------------------------------------------------------------------------------- - * Copyright (C) 2023-2024 Posit Software, PBC. All rights reserved. + * Copyright (C) 2024 Posit Software, PBC. All rights reserved. * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. *--------------------------------------------------------------------------------------------*/ @@ -28,7 +28,7 @@ const existsAsync = promisify(fs.exists); // Create a promisified version of https.get. We can't use the built-in promisify // because the callback doesn't follow the promise convention of (error, result). -const httpsGetAsync = (opts: https.RequestOptions) => { +export const httpsGetAsync = (opts: https.RequestOptions) => { return new Promise((resolve, reject) => { const req = https.get(opts, resolve); req.once('error', reject);