Remove goreleaser (#1016)

Stop using goreleaser, improve our release notes, speed up releases. - Fixes #1013 - Closes #816 - Partially addresses #731 - Related to #967 This should also save 3 hours of waiting per release of AWS. Build cross-platform provider binaries, in parallel, straight after prerequisites, then only calculate hashes & push to S3 and GitHub releases directly during the publish job. Specifics: - Use GitHub's own release notes generator which allows us to suppliment with our own context - specifically the schema change since the previous latest release. - Capture and restore the `schema-embed.json` to avoid having to run `tfgen` on every provider build. - Build the multi-platform build straight into the makefile for easier local testing. - Remove clearing disk space during publish as we're not doing any build work there any more. This does not include reworking tests to use the the provider binary being released - but this can be added later fairly easily.
pulumi · Jul 3, 2024 · e869944 · e869944
1 parent 25f2991
commit e869944
Show file tree

Hide file tree

Showing 34 changed files with 726 additions and 104 deletions.
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/build_provider.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/build_provider.yml
@@ -0,0 +1,65 @@
+name: "Build Provider"
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+        description: Version of the provider to build
+
+jobs:
+  build_provider:
+    name: Build ${{ matrix.platform.os }}-${{ matrix.platform.arch }}
+    runs-on: #{{ if .Config.runner.buildSdk }}##{{- .Config.runner.buildSdk }}##{{ else }}##{{- .Config.runner.default }}##{{ end }}#
+    env:
+      PROVIDER_VERSION: ${{ inputs.version }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+          - os: darwin
+            arch: amd64
+          - os: darwin
+            arch: arm64
+          - os: windows
+            arch: amd64
+    steps:
+      #{{- if .Config.freeDiskSpaceBeforeBuild }}#
+      # Run as first step so we don't delete things that have just been installed
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          tool-cache: false
+          swap-storage: false
+      #{{- end }}#
+      - name: Checkout Repo
+        uses: #{{ .Config.actionVersions.checkout }}#
+        #{{- if .Config.checkoutSubmodules }}#
+        with:
+          submodules: #{{ .Config.checkoutSubmodules }}#
+        #{{- end }}#
+      - name: Setup tools
+        uses: ./.github/actions/setup-tools
+        with:
+          tools: pulumictl, go
+      - name: Download schema-embed.json
+        uses: #{{ .Config.actionVersions.downloadArtifact }}#
+        with:
+          name: schema-embed.json
+          path: provider/cmd/pulumi-resource-#{{ .Config.provider }}#/schema-embed.json
+      - name: Prepare for build
+        # This installs plugins and prepares upstream
+        run: make upstream
+      - name: Build & package provider
+        run: make provider_dist-${{ matrix.platform.os }}-${{ matrix.platform.arch }}
+      - name: Upload artifacts
+        uses: #{{ .Config.actionVersions.uploadArtifact }}#
+        with:
+          name: pulumi-resource-#{{ .Config.provider }}#-v${{ inputs.version }}-${{ matrix.platform.os }}-${{ matrix.platform.arch }}.tar.gz
+          path: bin/pulumi-resource-#{{ .Config.provider }}#-v${{ inputs.version }}-${{ matrix.platform.os }}-${{ matrix.platform.arch }}.tar.gz
+          retention-days: 30
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/main.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/main.yml
@@ -11,6 +11,13 @@ jobs:
       is_pr: ${{ github.event_name == 'pull_request' }}
       is_automated: ${{ github.actor == 'dependabot[bot]' }}
 
+  build_provider:
+    uses: ./.github/workflows/build_provider.yml
+    needs: prerequisites
+    secrets: inherit
+    with:
+      version: ${{ needs.prerequisites.outputs.version }}
+
   build_sdk:
     name: build_sdk
     needs: prerequisites
@@ -77,6 +84,7 @@ jobs:
     name: publish
     needs:
       - prerequisites
+      - build_provider
       - test
       - license_check
       #{{- range $action, $_ := .Config.extraTests }}#

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/nightly-test.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/nightly-test.yml
@@ -12,6 +12,13 @@ jobs:
       is_pr: ${{ github.event_name == 'pull_request' }}
       is_automated: ${{ github.actor == 'dependabot[bot]' }}
 
+  build_provider:
+    uses: ./.github/workflows/build_provider.yml
+    needs: prerequisites
+    secrets: inherit
+    with:
+      version: ${{ needs.prerequisites.outputs.version }}
+
   build_sdk:
     name: build_sdk
     needs: prerequisites

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml
@@ -12,6 +12,13 @@ jobs:
       is_pr: ${{ github.event_name == 'pull_request' }}
       is_automated: ${{ github.actor == 'dependabot[bot]' }}
 
+  build_provider:
+    uses: ./.github/workflows/build_provider.yml
+    needs: prerequisites
+    secrets: inherit
+    with:
+      version: ${{ needs.prerequisites.outputs.version }}
+
   build_sdk:
     name: build_sdk
     needs: prerequisites
@@ -36,6 +43,7 @@ jobs:
     name: publish
     needs:
       - prerequisites
+      - build_provider
       - test
       - license_check
       #{{- range $action, $_ := .Config.extraTests }}#

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerequisites.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerequisites.yml
@@ -119,3 +119,10 @@ jobs:
 
     - name: Upload bin
       uses: ./.github/actions/upload-bin
+
+    - name: Upload schema-embed.json
+      uses: #{{ .Config.actionVersions.uploadArtifact }}#
+      with:
+        name: schema-embed.json
+        path: provider/cmd/pulumi-resource-#{{ .Config.provider }}#/schema-embed.json
+        retention-days: 30
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/publish.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/publish.yml
@@ -30,13 +30,6 @@ jobs:
     - name: Validate skipGoSdk
       if: inputs.skipGoSdk && inputs.isPrerelease == false
       run: echo "Can't skip Go SDK for stable releases. This is likely a bug in the calling workflow." && exit 1
-    - name: Free Disk Space (Ubuntu)
-      uses: jlumbroso/[email protected]
-      with:
-        # this might remove tools that are actually needed,
-        # if set to "true" but frees about 6 GB
-        tool-cache: false
-        swap-storage: false
     - name: Checkout Repo
       uses: #{{ .Config.actionVersions.checkout }}#
 #{{- if .Config.checkoutSubmodules }}#
@@ -46,7 +39,7 @@ jobs:
     - name: Setup tools
       uses: ./.github/actions/setup-tools
       with:
-        tools: pulumictl, pulumicli, go
+        tools: pulumictl, pulumicli, go, schema-tools
     - name: Configure AWS Credentials
       uses: #{{ .Config.actionVersions.configureAwsCredentials }}#
       with:
@@ -57,25 +50,44 @@ jobs:
         role-external-id: upload-pulumi-release
         role-session-name: #{{ .Config.provider }}#@githubActions
         role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE_ARN }}
-    - name: Run GoReleaser
+    - name: Create dist directory
+      run: mkdir -p dist
+    - name: Download provider assets
+      uses: #{{ .Config.actionVersions.downloadArtifact }}#
+      with:
+        pattern: pulumi-resource-#{{ .Config.provider }}#-v${{ inputs.version }}-*
+        path: dist
+        # Don't create a directory for each artifact
+        merge-multiple: true
+    - name: Calculate checksums
+      working-directory: dist
+      run: shasum ./*.tar.gz > pulumi-#{{ .Config.provider }}#_${{ inputs.version }}_checksums.txt
+    - name: Get Schema Change Summary
+      id: schema-summary
+      shell: bash
+      run: |
+        # Get latest stable release. Return only first column from result (tag).
+        LAST_VERSION=$(gh release view --repo pulumi/pulumi-#{{ .Config.provider }}# --json tagName -q .tagName)
+        {
+          echo 'summary<<EOF'
+          schema-tools compare --provider="#{{ .Config.provider }}#" --old-commit="$LAST_VERSION" --new-commit="--local-path=provider/cmd/pulumi-resource-#{{ .Config.provider }}#/schema.json"
+          echo 'EOF'
+        } >> "$GITHUB_OUTPUT"
+    - name: Upload Provider Binaries
+      run: aws s3 cp dist s3://get.pulumi.com/releases/plugins/ --recursive
+    - name: Create GH Release
+      uses: softprops/action-gh-release@v1
       if: inputs.isPrerelease == false
-      uses: #{{ .Config.actionVersions.goReleaser }}#
-      env:
-        GORELEASER_CURRENT_TAG: v${{ inputs.version }}
-        PROVIDER_VERSION: ${{ inputs.version }}
       with:
-        args: -p #{{ .Config.parallel }}# release --rm-dist --timeout #{{ .Config.timeout }}#m0s
-        version: latest
-    - name: Run GoReleaser (prerelease)
-      if: inputs.isPrerelease == true
-      uses: #{{ .Config.actionVersions.goReleaser }}#
+        tag_name: v${{ inputs.version }}
+        prerelease: ${{ inputs.isPrerelease }}
+        # We keep pre-releases as drafts so they're not visible until we manually publish them.
+        draft: ${{ inputs.isPrerelease }}
+        body: ${{ steps.schema-summary.outputs.summary }}
+        generate_release_notes: true
+        files: dist/*
       env:
-        GORELEASER_CURRENT_TAG: v${{ inputs.version }}
-        PROVIDER_VERSION: ${{ inputs.version }}
-      with:
-        args: -p #{{ .Config.parallel }}# -f .goreleaser.prerelease.yml --rm-dist --skip-validate --timeout
-          #{{ .Config.timeout }}#m0s
-        version: latest
+        GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
 
   publish_sdk:
     name: publish_sdk

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/release.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/release.yml
@@ -11,6 +11,13 @@ jobs:
       is_pr: ${{ github.event_name == 'pull_request' }}
       is_automated: ${{ github.actor == 'dependabot[bot]' }}
 
+  build_provider:
+    uses: ./.github/workflows/build_provider.yml
+    needs: prerequisites
+    secrets: inherit
+    with:
+      version: ${{ needs.prerequisites.outputs.version }}
+
   build_sdk:
     name: build_sdk
     needs: prerequisites
@@ -35,6 +42,7 @@ jobs:
     name: publish
     needs:
       - prerequisites
+      - build_provider
       - test
       - license_check
       #{{- range $action, $_ := .Config.extraTests }}#

diff --git a/...der-ci/internal/pkg/templates/bridged-provider/.github/workflows/run-acceptance-tests.yml b/...der-ci/internal/pkg/templates/bridged-provider/.github/workflows/run-acceptance-tests.yml
@@ -19,6 +19,13 @@ jobs:
       is_pr: ${{ github.event_name == 'pull_request' }}
       is_automated: ${{ github.actor == 'dependabot[bot]' }}
 
+  build_provider:
+    uses: ./.github/workflows/build_provider.yml
+    needs: prerequisites
+    secrets: inherit
+    with:
+      version: ${{ needs.prerequisites.outputs.version }}
+
   build_sdk:
     if: github.event_name == 'repository_dispatch' ||
       github.event.pull_request.head.repo.full_name == github.repository

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/Makefile b/provider-ci/internal/pkg/templates/bridged-provider/Makefile
@@ -16,6 +16,8 @@ TESTPARALLELISM := 10
 WORKING_DIR := $(shell pwd)
 #{{- if .Config.goBuildParallelism }}#
 PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p #{{ .Config.goBuildParallelism }}#
+#{{- else }}#
+PULUMI_PROVIDER_BUILD_PARALLELISM ?=
 #{{- end }}#
 #{{- if .Config.pulumiConvert }}#
 PULUMI_CONVERT := 1
@@ -30,6 +32,15 @@ PROVIDER_VERSION ?= #{{ index .Config "major-version" }}#.0.0-alpha.0+dev
 # Use this normalised version everywhere rather than the raw input to ensure consistency.
 VERSION_GENERIC = $(shell pulumictl convert-version --language generic --version "$(PROVIDER_VERSION)")
 
+LDFLAGS_PROJ_VERSION=-X $(PROJECT)/$(VERSION_PATH)=$(VERSION_GENERIC)#{{if .Config.providerVersion}}# -X #{{ .Config.providerVersion }}#=$(VERSION_GENERIC)#{{end}}#
+#{{- if .Config.providerVersion }}#
+LDFLAGS_UPSTREAM_VERSION=-X #{{ .Config.providerVersion }}#=v$(VERSION_GENERIC)
+#{{- else }}#
+LDFLAGS_UPSTREAM_VERSION=
+#{{- end }}#
+LDFLAGS_EXTRAS=#{{- range (index .Config "extra-ld-flags") }}# #{{ . }}# #{{- end }}#
+LDFLAGS=$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_UPSTREAM_VERSION) $(LDFLAGS_EXTRAS)
+
 development: install_plugins provider build_sdks install_sdks
 
 build: install_plugins provider build_sdks install_sdks
@@ -145,7 +156,7 @@ lint_provider.fix:
 # `cmd/pulumi-resource-#{{ .Config.provider }}#/schema.json` is valid and up to date.
 # To create a release ready binary, you should use `make provider`.
 provider_no_deps:
-	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION_GENERIC)#{{if .Config.providerVersion}}# -X #{{ .Config.providerVersion }}#=$(VERSION_GENERIC)#{{end}}#" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER))
+	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER))
 
 provider: tfgen provider_no_deps
 
@@ -172,7 +183,7 @@ tfgen_no_deps: tfgen_build_only
 	(cd provider && VERSION=$(VERSION_GENERIC) go generate cmd/$(PROVIDER)/main.go)
 
 tfgen_build_only:
-	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION_GENERIC)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
+	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
 
 upstream:
 ifneq ("$(wildcard upstream)","")
@@ -223,3 +234,40 @@ debug_tfgen:
 	dlv  --listen=:2345 --headless=true --api-version=2  exec $(WORKING_DIR)/bin/$(TFGEN) -- schema --out provider/cmd/$(PROVIDER)
 
 .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup#{{ if .Config.docsCmd }}# docs#{{end}}# help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider provider_no_deps test tfgen upstream upstream.finalize upstream.rebase ci-mgmt test_provider debug_tfgen tfgen_build_only
+
+# Provider cross-platform build & packaging
+
+# These targets assume that the schema-embed.json exists - it's generated by tfgen.
+# We disable CGO to ensure that the binary is statically linked.
+bin/linux-amd64/$(PROVIDER): TARGET := linux-amd64
+bin/linux-arm64/$(PROVIDER): TARGET := linux-arm64
+bin/darwin-amd64/$(PROVIDER): TARGET := darwin-amd64
+bin/darwin-arm64/$(PROVIDER): TARGET := darwin-arm64
+bin/windows-amd64/$(PROVIDER).exe: TARGET := windows-amd64
+bin/%/$(PROVIDER) bin/%/$(PROVIDER).exe: provider/cmd/$(PROVIDER)/schema-embed.json
+	@# check the TARGET is set
+	test $(TARGET)
+	cd provider && \
+		export GOOS=$$(echo "$(TARGET)" | cut -d "-" -f 1) && \
+		export GOARCH=$$(echo "$(TARGET)" | cut -d "-" -f 2) && \
+		export CGO_ENABLED=0 && \
+		go build -o "${WORKING_DIR}/$@" $(PULUMI_PROVIDER_BUILD_PARALLELISM) -ldflags "$(LDFLAGS)" "$(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER)"
+
+bin/$(PROVIDER)-v$(VERSION_GENERIC)-linux-amd64.tar.gz: bin/linux-amd64/$(PROVIDER)
+bin/$(PROVIDER)-v$(VERSION_GENERIC)-linux-arm64.tar.gz: bin/linux-arm64/$(PROVIDER)
+bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-amd64.tar.gz: bin/darwin-amd64/$(PROVIDER)
+bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-arm64.tar.gz: bin/darwin-arm64/$(PROVIDER)
+bin/$(PROVIDER)-v$(VERSION_GENERIC)-windows-amd64.tar.gz: bin/windows-amd64/$(PROVIDER).exe
+bin/$(PROVIDER)-v$(VERSION_GENERIC)-%.tar.gz:
+	@mkdir -p dist
+	@# $< is the last dependency (the binary path from above) e.g. bin/linux-amd64/pulumi-resource-xyz
+	@# $@ is the current target e.g. bin/pulumi-resource-xyz-v1.2.3-linux-amd64.tar.gz
+	tar --gzip -cf $@ README.md LICENSE -C $$(dirname $<) .
+
+provider_dist-linux-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-linux-amd64.tar.gz
+provider_dist-linux-arm64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-linux-arm64.tar.gz
+provider_dist-darwin-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-amd64.tar.gz
+provider_dist-darwin-arm64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-arm64.tar.gz
+provider_dist-windows-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-windows-amd64.tar.gz
+provider_dist: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64
+.PHONY: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64 provider_dist