Allow bridged providers to opt into test sharding (#1293)

This PR rebases @blampe work on sharding for bridged providers #1217 with minor simplifications. An example PR applying this work is found here: pulumi/pulumi-awsx#1487 There should be no visible change for providers unless they opt in with "shards: 3" or similar. For providers that do opt in, the test job will stop splitting the build matrix by language and instead create a build matrix with as many workers as shards requested, and randomly assign integration tests to these workers. This permits better loading as typically language-based strategies end up being heavy on Node tests. It also permits having more workers than languages. This work is prerequisite to onboarding pulumi-eks to ci-mgmt via the simplified bridge template.
pulumi · Jan 16, 2025 · 30225ad · 30225ad
1 parent ae62d02
commit 30225ad
Show file tree

Hide file tree

Showing 11 changed files with 99 additions and 27 deletions.
diff --git a/provider-ci/internal/pkg/config.go b/provider-ci/internal/pkg/config.go
@@ -249,6 +249,9 @@ type Config struct {
 	// https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22parallel%3A%22&type=code
 	Parallel int `yaml:"parallel"`
 
+	// Shards controls how many jobs integration tests are distributed across.
+	Shards int `yaml:"shards"`
+
 	// Hybrid has no effect but is set by the docker provider.
 	// https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22hybrid%3A%22&type=code
 	Hybrid bool `yaml:"hybrid"`

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/test.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/test.yml
@@ -41,29 +41,46 @@ jobs:
         #{{- end }}#
         persist-credentials: false
     - name: Checkout p/examples
+      #{{- if not .Config.Shards }}#
       if: matrix.testTarget == 'pulumiExamples'
+      #{{- end }}#
       uses: #{{ .Config.ActionVersions.Checkout }}#
       with:
         repository: pulumi/examples
         path: p-examples
     - name: Setup tools
       uses: ./.github/actions/setup-tools
+      #{{- if not .Config.Shards }}#
       with:
         tools: pulumictl, pulumicli, ${{ matrix.language }}
+      #{{- end }}#
     - name: Prepare local workspace
       run: make prepare_local_workspace
     - name: Download bin
       uses: ./.github/actions/download-bin
+    #{{- if not .Config.Shards }}#
     - name: Download SDK
       uses: ./.github/actions/download-sdk
       with:
         language: ${{ matrix.language }}
     - name: Restore makefile progress
       run: make --touch provider schema build_${{ matrix.language }}
+    #{{- else }}#
+    #{{- range $_, $language := .Config.Languages }}#
+    - name: Download #{{ $language }}# SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: #{{ $language }}#
+    #{{- end }}#
+    - name: Restore makefile progress
+      run: make --touch provider schema build_sdks
+    #{{- end }}#
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
+    #{{- if not .Config.Shards }}#
       if: matrix.language == 'python'
+    #{{- end }}#
       run: |-
         pip3 install virtualenv==20.0.23
         pip3 install pipenv
@@ -104,6 +121,11 @@ jobs:
     - name: Run setup script
       run: #{{ index .Config.SetupScript }}#
     #{{- end }}#
+#{{- if .Config.Shards }}#
+    - name: Install prebuilt SDKs
+      run: make install_sdks
+#{{- end }}#
+#{{- if not .Config.Shards }}#
     - name: Install dependencies
       run: make install_${{ matrix.language}}_sdk
 #{{- if .Config.Actions.PreTest }}#
@@ -121,8 +143,29 @@ jobs:
     - name: Run pulumi/examples tests
       if: matrix.testTarget == 'pulumiExamples'
       run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -run TestPulumiExamples -parallel 4 .
+#{{- else }}#
+    - name: Generate test shards
+      run: |-
+        cd examples
+        go run github.com/pulumi/shard@861c9ce4aa851e98c19f8376892bf7e47238fa1b \
+            --total ${{ matrix.total-shards }} \
+            --index ${{ matrix.current-shard }} \
+            --output env >> "$GITHUB_ENV"
+    - name: Run tests
+      run: |
+        make GOTESTARGS="-test.run ${SHARD_TESTS} ${SHARD_PATHS}" test
+#{{- end }}#
     strategy:
       fail-fast: false
+#{{- if .Config.Shards }}#
+      matrix:
+        total-shards:
+          - #{{ .Config.Shards }}#
+        current-shard:
+          #{{- range $i, $_ := until .Config.Shards }}#
+          - #{{ $i }}#
+          #{{- end }}#
+#{{- else }}#
       matrix:
         language:
 #{{ .Config.Languages | toYaml | indent 8 }}#
@@ -131,3 +174,4 @@ jobs:
         #{{- else }}#
         testTarget: [local]
         #{{- end }}#
+#{{- end }}#
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/Makefile b/provider-ci/internal/pkg/templates/bridged-provider/Makefile
@@ -17,6 +17,7 @@ CODEGEN := pulumi-tfgen-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
+GOTESTARGS := ""
 WORKING_DIR := $(shell pwd)
 #{{- if .Config.GoBuildParallelism }}#
 PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p #{{ .Config.GoBuildParallelism }}#
@@ -280,7 +281,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h $(value GOTESTARGS)
 .PHONY: test
 
 #{{- if .Config.TestProviderCmd }}#

diff --git a/provider-ci/internal/pkg/templates/defaults.config.yaml b/provider-ci/internal/pkg/templates/defaults.config.yaml
@@ -141,6 +141,9 @@ integrationTestProvider: false
 # This is unused: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22testPulumiExamples%3A%22&type=code
 testPulumiExamples: false
 
+# How many shared to execute integration tests with. If omitted, shard behavior defaults to language-based sharding.
+shards: 0
+
 # runner defines the runs-on property for various stages of the build
 # These are not overridden by any providers: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22runner%3A%22&type=code
 runner:

diff --git a/provider-ci/test-providers/acme/Makefile b/provider-ci/test-providers/acme/Makefile
@@ -9,6 +9,7 @@ CODEGEN := pulumi-tfgen-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
+GOTESTARGS := ""
 WORKING_DIR := $(shell pwd)
 PULUMI_PROVIDER_BUILD_PARALLELISM ?=
 PULUMI_CONVERT := 0
@@ -229,7 +230,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h $(value GOTESTARGS)
 .PHONY: test
 test_provider_cmd = cd provider && go test -v -short \
 	-coverprofile="coverage.txt" \

diff --git a/provider-ci/test-providers/aws/Makefile b/provider-ci/test-providers/aws/Makefile
@@ -9,6 +9,7 @@ CODEGEN := pulumi-tfgen-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
+GOTESTARGS := ""
 WORKING_DIR := $(shell pwd)
 PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p 2
 PULUMI_CONVERT := 1
@@ -239,7 +240,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h $(value GOTESTARGS)
 .PHONY: test
 test_provider_cmd = cd provider && go test -v -short \
 	-coverprofile="coverage.txt" \

diff --git a/provider-ci/test-providers/cloudflare/Makefile b/provider-ci/test-providers/cloudflare/Makefile
@@ -9,6 +9,7 @@ CODEGEN := pulumi-tfgen-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
+GOTESTARGS := ""
 WORKING_DIR := $(shell pwd)
 PULUMI_PROVIDER_BUILD_PARALLELISM ?=
 PULUMI_CONVERT := 1
@@ -239,7 +240,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h $(value GOTESTARGS)
 .PHONY: test
 test_provider_cmd = cd provider && go test -v -short \
 	-coverprofile="coverage.txt" \

diff --git a/provider-ci/test-providers/docker/Makefile b/provider-ci/test-providers/docker/Makefile
@@ -9,6 +9,7 @@ CODEGEN := pulumi-tfgen-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
+GOTESTARGS := ""
 WORKING_DIR := $(shell pwd)
 PULUMI_PROVIDER_BUILD_PARALLELISM ?=
 PULUMI_CONVERT := 1
@@ -242,7 +243,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h $(value GOTESTARGS)
 .PHONY: test
 test_provider_cmd = cd provider && go test -v -short \
 	-coverprofile="coverage.txt" \

diff --git a/provider-ci/test-providers/eks/.ci-mgmt.yaml b/provider-ci/test-providers/eks/.ci-mgmt.yaml
@@ -24,3 +24,4 @@ env:
 template: generic
 freeDiskSpaceBeforeTest: true # TODO: https://github.com/pulumi/pulumi/issues/17718
 buildProviderPre: echo building-provider
+shards: 3
diff --git a/provider-ci/test-providers/eks/.github/workflows/test.yml b/provider-ci/test-providers/eks/.github/workflows/test.yml
@@ -58,29 +58,41 @@ jobs:
         ref: ${{ env.PR_COMMIT_SHA }}
         persist-credentials: false
     - name: Checkout p/examples
-      if: matrix.testTarget == 'pulumiExamples'
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         repository: pulumi/examples
         path: p-examples
     - name: Setup tools
       uses: ./.github/actions/setup-tools
-      with:
-        tools: pulumictl, pulumicli, ${{ matrix.language }}
     - name: Prepare local workspace
       run: make prepare_local_workspace
     - name: Download bin
       uses: ./.github/actions/download-bin
-    - name: Download SDK
+    - name: Download nodejs SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: nodejs
+    - name: Download python SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: python
+    - name: Download dotnet SDK
       uses: ./.github/actions/download-sdk
       with:
-        language: ${{ matrix.language }}
+        language: dotnet
+    - name: Download go SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: go
+    - name: Download java SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: java
     - name: Restore makefile progress
-      run: make --touch provider schema build_${{ matrix.language }}
+      run: make --touch provider schema build_sdks
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
-      if: matrix.language == 'python'
       run: |-
         pip3 install virtualenv==20.0.23
         pip3 install pipenv
@@ -93,21 +105,24 @@ jobs:
         role-duration-seconds: 7200
         role-session-name: eks@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
+    - name: Install prebuilt SDKs
+      run: make install_sdks
+    - name: Generate test shards
+      run: |-
+        cd examples
+        go run github.com/pulumi/shard@861c9ce4aa851e98c19f8376892bf7e47238fa1b \
+            --total ${{ matrix.total-shards }} \
+            --index ${{ matrix.current-shard }} \
+            --output env >> "$GITHUB_ENV"
     - name: Run tests
-      if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 .
-    - name: Run pulumi/examples tests
-      if: matrix.testTarget == 'pulumiExamples'
-      run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -run TestPulumiExamples -parallel 4 .
+      run: |
+        make GOTESTARGS="-test.run ${SHARD_TESTS} ${SHARD_PATHS}" test
     strategy:
       fail-fast: false
       matrix:
-        language:
-        - nodejs
-        - python
-        - dotnet
-        - go
-        - java
-        testTarget: [local]
+        total-shards:
+          - 3
+        current-shard:
+          - 0
+          - 1
+          - 2
diff --git a/provider-ci/test-providers/eks/Makefile b/provider-ci/test-providers/eks/Makefile
@@ -9,6 +9,7 @@ CODEGEN := pulumi-gen-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
+GOTESTARGS := ""
 WORKING_DIR := $(shell pwd)
 PULUMI_PROVIDER_BUILD_PARALLELISM ?=
 PULUMI_CONVERT := 0
@@ -229,7 +230,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h $(value GOTESTARGS)
 .PHONY: test
 test_provider_cmd = cd provider && go test -v -short \
 	-coverprofile="coverage.txt" \