test sharding

pulumi · Jan 7, 2025 · e7dd711 · e7dd711
1 parent 4f6a39f
commit e7dd711
Show file tree

Hide file tree

Showing 8 changed files with 7,187 additions and 1,097 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -8,6 +8,7 @@ on:
         required: false
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  TOTAL_SHARDS: 10
 
 jobs:
   test:
@@ -21,6 +22,48 @@ jobs:
         go-version: [1.22.x, 1.23.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
         feature-flags: ["DEFAULT", "PULUMI_TF_BRIDGE_ACCURATE_BRIDGE_PREVIEW"]
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+        exclude:
+          - platform: windows-latest
+            feature-flags: "PULUMI_TF_BRIDGE_ACCURATE_BRIDGE_PREVIEW"
+          - platform: macos-latest
+            feature-flags: "PULUMI_TF_BRIDGE_ACCURATE_BRIDGE_PREVIEW"
+          - platform: windows-latest
+            shard: 1
+          - platform: windows-latest
+            shard: 2
+          - platform: windows-latest
+            shard: 3
+          - platform: windows-latest
+            shard: 4
+          - platform: windows-latest
+            shard: 5
+          - platform: windows-latest
+            shard: 6
+          - platform: windows-latest
+            shard: 7
+          - platform: windows-latest
+            shard: 8
+          - platform: windows-latest
+            shard: 9
+          - platform: macos-latest
+            shard: 1
+          - platform: macos-latest
+            shard: 2
+          - platform: macos-latest
+            shard: 3
+          - platform: macos-latest
+            shard: 4
+          - platform: macos-latest
+            shard: 5
+          - platform: macos-latest
+            shard: 6
+          - platform: macos-latest
+            shard: 7
+          - platform: macos-latest
+            shard: 8
+          - platform: macos-latest
+            shard: 9
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Install pulumi
@@ -45,8 +88,17 @@ jobs:
         run: make build
       - name: Build PF
         run: cd pkg/pf && make build
+      - name: Shard tests
+        run: echo "SHARD_CMD=$(go run github.com/VenelinMartinov/sharder@9d09afeb1c053b4a0263fbaa5c3c9da1ca2d10e7 --total ${{ env.TOTAL_SHARDS }} --index ${{ matrix.shard }} --output testoutput.txt --format make)" >> $GITHUB_ENV
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+      - run: echo "$SHARD_CMD"
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+      - name: Test
+        run: make VERBOSE=true test_shard
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
       - name: Test
         run: make test
+        if: ${{ matrix.platform != 'ubuntu-latest' }}
       - name: Upload coverage reports to Codecov
         # If we have a CODECOV_TOKEN secret, then we upload it to get a coverage report.
         #

diff --git a/.gitignore b/.gitignore
@@ -14,4 +14,6 @@ testing/coverage.txt
 pf/coverage.out
 pf/tests/coverage.out
 
-pulumi-hcl-lint
+pulumi-hcl-lint
+
+schema-tpsdkv2.json
diff --git a/Makefile b/Makefile
@@ -33,6 +33,20 @@ test:: install_plugins
 	PULUMI_TERRAFORM_BRIDGE_TEST_PROVIDER=$(shell pwd)/bin/pulumi-terraform-bridge-test-provider \
 		go test -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 2h -parallel ${TESTPARALLELISM} ./...
 
+test_shard:: install_plugins
+	@mkdir -p bin
+	go build -o bin ./internal/testing/pulumi-terraform-bridge-test-provider
+	PULUMI_TERRAFORM_BRIDGE_TEST_PROVIDER=$(shell pwd)/bin/pulumi-terraform-bridge-test-provider \
+		go test -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 2h -parallel ${TESTPARALLELISM} ./... ${SHARD_CMD}
+
+test_profile:: install_plugins
+	@mkdir -p bin
+	go build -o bin ./internal/testing/pulumi-terraform-bridge-test-provider
+	rm -f testoutput.txt
+	PULUMI_TERRAFORM_BRIDGE_TEST_PROVIDER=$(shell pwd)/bin/pulumi-terraform-bridge-test-provider \
+		go test -count=1 -timeout 2h -parallel ${TESTPARALLELISM} ./... -json | \
+		jq -c -r 'select((.Action == "pass" or .Action == "fail") and .Test != null)' | tee testoutput.txt
+
 # Run tests while accepting current output as expected output "golden"
 # tests. In case where system behavior changes intentionally this can
 # be useful to run to review the differences with git diff.

diff --git a/pkg/pf/tests/provider_configure_test.go b/pkg/pf/tests/provider_configure_test.go
@@ -534,7 +534,7 @@ func TestConfigureBooleans(t *testing.T) {
 	}`)
 }
 
-func TestConfigureErrorReplacement(t *testing.T) {
+func TestConfigureErrorReplacementPF(t *testing.T) {
 	t.Parallel()
 	t.Run("replace_config_properties", func(t *testing.T) {
 		errString := `some error with "config_property" and "config" but not config`

diff --git a/pkg/tfbridge/provider_test.go b/pkg/tfbridge/provider_test.go
@@ -1988,7 +1988,7 @@ func TestConfigure(t *testing.T) {
 	})
 }
 
-func TestConfigureErrorReplacement(t *testing.T) {
+func TestConfigureErrorReplacementSDKv2(t *testing.T) {
 	t.Parallel()
 	t.Run("replace_config_properties", func(t *testing.T) {
 		p := testprovider.ProviderV2()