From 2f474c6d558c9546a04751bca5b23e12cbb7de2e Mon Sep 17 00:00:00 2001 From: Andrea Frittoli Date: Thu, 26 Sep 2024 10:12:27 +0100 Subject: [PATCH] Migrate nightly builds and releases to ghcr.io As part of the work to reduce Tekton's infra spend, we are migrating nightly builds and new releases from gcr.io to ghcr.io, to reduce the expensive egress bandwith utilisation (https://github.com/tektoncd/plumbing/issues/2157). This PR introduces support for publishing container images to ghcr.io (YAML files are still published to Google Cloud storage). It also replicates a few improvements that have been done in the pipeline release pipeline before, such us: - Support for hashing the image path (for nicer path in GitHub) - Support for skipping build and test tasks, to be enabled for nightly builds to save CPU time Signed-off-by: Andrea Frittoli --- tekton/publish.yaml | 99 ++++++++++++++++++++++-------- tekton/release-pipeline.yaml | 116 ++++++++++++++++++++++++++++++----- 2 files changed, 174 insertions(+), 41 deletions(-) diff --git a/tekton/publish.yaml b/tekton/publish.yaml index 1f2042801..85ee2881d 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -15,6 +15,9 @@ spec: - name: interceptorImages description: List of cmd/* paths to be published as images in release manifest interceptors.yaml default: "interceptors" + - name: koExtraArgs + description: Extra args to be passed to ko + default: "--preserve-import-paths" - name: versionTag description: The vX.Y.Z version that the artifacts should be tagged with (including `v`) - name: imageRegistry @@ -25,6 +28,9 @@ spec: - name: imageRegistryRegions description: The target image registry regions default: "us eu asia" + - name: imageRegistryUser + description: Username to be used to login to the container registry + default: "_json_key" - name: releaseAsLatest description: Whether to tag and publish this release as Triggers' latest default: "true" @@ -49,14 +55,18 @@ spec: env: - name: "PROJECT_ROOT" value: "$(workspaces.source.path)" - - name: CONTAINER_REGISTY_CREDENTIALS + - name: CONTAINER_REGISTRY_CREDENTIALS value: "$(workspaces.release-secret.path)/$(params.serviceAccountPath)" - name: CONTAINER_REGISTRY value: "$(params.imageRegistry)/$(params.imageRegistryPath)" + - name: CONTAINER_REGISTRY_USER + value: "$(params.imageRegistryUser)" - name: REGIONS value: "$(params.imageRegistryRegions)" - name: OUTPUT_RELEASE_DIR value: "$(workspaces.output.path)/$(params.versionTag)" + - name: KO_EXTRA_ARGS + value: "$(params.koExtraArgs)" results: # IMAGES result is picked up by Tekton Chains to sign the release. # See https://github.com/tektoncd/plumbing/blob/main/docs/signing.md for more info. @@ -64,26 +74,26 @@ spec: steps: - name: container-registy-auth - image: gcr.io/go-containerregistry/crane:debug + image: cgr.dev/chainguard/crane:latest-dev@sha256:8ebcdd154abd06371886fee6583c7c9bbc4e88a2999c493266b1580f605e0e7c script: | - #!/busybox/sh + #!/bin/sh set -ex # Login to the container registry - DOCKER_CONFIG=$(cat ${CONTAINER_REGISTY_CREDENTIALS} | \ - crane auth login -u _json_key --password-stdin $(params.imageRegistry) 2>&1 | \ + DOCKER_CONFIG=$(cat ${CONTAINER_REGISTRY_CREDENTIALS} | \ + crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin $(params.imageRegistry) 2>&1 | \ sed 's,^.*logged in via \(.*\)$,\1,g') # Auth with account credentials for all regions. for region in ${REGIONS} do HOSTNAME=${region}.$(params.imageRegistry) - cat ${CONTAINER_REGISTY_CREDENTIALS} | crane auth login -u _json_key --password-stdin ${HOSTNAME} + cat ${CONTAINER_REGISTRY_CREDENTIALS} | crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin ${HOSTNAME} done cp ${DOCKER_CONFIG} /workspace/docker-config.json - name: run-ko - image: gcr.io/tekton-releases/dogfooding/ko@sha256:9ee3ae5273b1f55bf01ba71bd79b5a4a9d357c51c0fdabf1efec8bd7e7087983 + image: gcr.io/tekton-releases/dogfooding/ko@sha256:8c4dbc57bcfd4c0a68f62c42da3f22932b0f3f54d4724c65841ad78406bc09ad env: - name: KO_DOCKER_REPO value: $(params.imageRegistry)/$(params.imageRegistryPath) @@ -104,7 +114,7 @@ spec: # For each cmd/* directory, include a full gzipped tar of all source in # vendor/. This is overkill. Some deps' licenses require the source to be # included in the container image when they're used as a dependency. - # Rather than trying to determine which deps have this requirement (an(params.imageRegistryd + # Rather than trying to determine which deps have this requirement (and # probably get it wrong), we'll just targz up the whole vendor tree and # include it. As of 9/20/2019, this amounts to about 11MB of additional # data in each image. @@ -116,51 +126,92 @@ spec: fi done - # Rewrite "devel" to params.versionTag - sed -i -e 's/\(triggers.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${PROJECT_ROOT}/config/*.yaml - sed -i -e 's/\(triggers.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${PROJECT_ROOT}/config/interceptors/*.yaml # Publish images and create release.yaml mkdir -p $OUTPUT_RELEASE_DIR - ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml - ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f ${PROJECT_ROOT}/config/interceptors > $OUTPUT_RELEASE_DIR/interceptors.yaml + # Make a local git tag to make git status happy :) + # The real "tagging" will happen with the "create-release" pipeline. + git tag $(params.versionTag) + + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) \ + -t $(params.versionTag) ${KO_EXTRA_ARGS} \ + -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) ${KO_EXTRA_ARGS} \ + -t $(params.versionTag) \ + -f ${PROJECT_ROOT}/config/interceptors > $OUTPUT_RELEASE_DIR/interceptors.yaml + # Publish images and create release.notags.yaml # This is useful if your container runtime doesn't support the `image-reference:tag@digest` notation # This is currently the case for `cri-o` (and most likely others) - ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.notags.yaml - ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f ${PROJECT_ROOT}/config/interceptors > $OUTPUT_RELEASE_DIR/interceptors.notags.yaml + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) ${KO_EXTRA_ARGS} \ + -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.notags.yaml + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) ${KO_EXTRA_ARGS} \ + -f ${PROJECT_ROOT}/config/interceptors > $OUTPUT_RELEASE_DIR/interceptors.notags.yaml + # Rewrite "devel" to params.versionTag + sed -i -e 's/\(triggers.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.yaml + sed -i -e 's/\(triggers.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.notags.yaml + sed -i -e 's/\(triggers.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/interceptors.yaml + sed -i -e 's/\(triggers.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/interceptors.notags.yaml - name: koparse - image: gcr.io/tekton-releases/dogfooding/koparse:latest + image: gcr.io/tekton-releases/dogfooding/koparse@sha256:ae363d70e3c2fb75e96aaeb561dcea20383c27a47f0266c8179bbb72b89c2430 script: | set -ex - IMAGES_PATH=${CONTAINER_REGISTRY}/$(params.package) + # Find "--preserve-import-paths" in a list of args + function find_preserve_import_path() { + for arg in $@; do + if [[ "$arg" == "--preserve-import-paths" ]]; then + return 0 + fi + done + return 1 + } + + # If "--preserve-import-paths" is used, include "package" in the expected path + find_preserve_import_path \ + $(echo $KO_EXTRA_ARGS) && \ + PRESERVE_IMPORT_PATH="--preserve-path" || \ + PRESERVE_IMPORT_PATH="--no-preserve-path" for cmd in $(params.images) do - IMAGES="${IMAGES} ${IMAGES_PATH}/cmd/${cmd}:$(params.versionTag)" + IMAGES="${IMAGES} $(params.package)/cmd/${cmd}:$(params.versionTag)" done # Parse the built images from the release.yaml generated by ko koparse \ --path $OUTPUT_RELEASE_DIR/release.yaml \ - --base ${IMAGES_PATH} --images ${IMAGES} > /workspace/built_images + --base $(params.package) \ + --container-registry ${CONTAINER_REGISTRY} \ + --images ${IMAGES} \ + ${PRESERVE_IMPORT_PATH} > /workspace/built_images for cmd in $(params.interceptorImages) do - INTERCEPTOR_IMAGES="${INTERCEPTOR_IMAGES} ${IMAGES_PATH}/cmd/${cmd}:$(params.versionTag)" + INTERCEPTOR_IMAGES="${INTERCEPTOR_IMAGES} $(params.package)/cmd/${cmd}:$(params.versionTag)" done # Parse the built images from the interceptor.yaml generated by ko koparse \ --path $OUTPUT_RELEASE_DIR/interceptors.yaml \ - --base ${IMAGES_PATH} --images ${INTERCEPTOR_IMAGES} >> /workspace/built_images + --base $(params.package) \ + --container-registry ${CONTAINER_REGISTRY} \ + --images ${INTERCEPTOR_IMAGES} \ + ${PRESERVE_IMPORT_PATH} >> /workspace/built_images - name: tag-images - image: gcr.io/go-containerregistry/crane:debug + image: cgr.dev/chainguard/crane:latest-dev@sha256:8ebcdd154abd06371886fee6583c7c9bbc4e88a2999c493266b1580f605e0e7c script: | - #!/busybox/sh + #!/bin/sh set -ex # Setup docker-auth @@ -168,8 +219,6 @@ spec: mkdir -p ${DOCKER_CONFIG} cp /workspace/docker-config.json ${DOCKER_CONFIG}/config.json - REGIONS="us eu asia" - # Tag the images and put them in all the regions for IMAGE in $(cat /workspace/built_images) do diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index dab22b4fe..259d1c871 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -16,6 +16,12 @@ spec: - name: imageRegistryPath description: The path (project) in the image registry default: tekton-releases + - name: imageRegistryRegions + description: The target image registry regions + default: "us eu asia" + - name: imageRegistryUser + description: The user for the image registry credentials + default: _json_key - name: versionTag description: The X.Y.Z version that the artifacts should be tagged with - name: releaseBucket @@ -27,13 +33,23 @@ spec: - name: platforms description: Platforms to publish for the images (e.g. linux/amd64,linux/arm64) default: linux/amd64,linux/arm,linux/arm64,linux/s390x,linux/ppc64le + - name: koExtraArgs + description: Extra args to be passed to ko + default: "--preserve-import-paths" - name: serviceAccountPath description: The path to the service account file within the release-secret workspace + - name: serviceAccountImagesPath + description: The path to the service account file or credentials within the release-images-secret workspace + - name: runTests + description: If set to something other than "true", skip the build and test tasks + default: "true" workspaces: - name: workarea description: The workspace where the repo will be cloned. - name: release-secret - description: The secret that contains a service account authorized to push to the imageRegistry and to the output bucket + description: The secret that contains a service account authorized to push to the output bucket + - name: release-images-secret + description: The secret that contains a service account authorized to push to the imageRegistry results: - name: commit-sha description: the sha of the commit that was released @@ -53,7 +69,12 @@ spec: tasks: - name: git-clone taskRef: - name: git-clone + resolver: hub + params: + - name: name + value: git-clone + - name: version + value: "0.7" workspaces: - name: output workspace: workarea @@ -63,10 +84,19 @@ spec: value: https://$(params.package) - name: revision value: $(params.gitRevision) - - name: precheck + - name: prerelease-precheck runAfter: [git-clone] taskRef: - name: prerelease-checks + resolver: git + params: + - name: repo + value: plumbing + - name: org + value: tektoncd + - name: revision + value: aeed19e5a36f335ebfdc4b96fa78d1ce5bb4f7b8 + - name: pathInRepo + value: tekton/resources/release/base/prerelease_checks.yaml params: - name: package value: $(params.package) @@ -79,9 +109,18 @@ spec: workspace: workarea subpath: git - name: unit-tests - runAfter: [precheck] + runAfter: [prerelease-precheck] + when: + - cel: "'$(params.runTests)' == 'true'" taskRef: - name: golang-test + resolver: bundles + params: + - name: bundle + value: gcr.io/tekton-releases/catalog/upstream/golang-test:0.2 + - name: name + value: golang-test + - name: kind + value: task params: - name: package value: $(params.package) @@ -92,9 +131,18 @@ spec: workspace: workarea subpath: git - name: build - runAfter: [precheck] + runAfter: [prerelease-precheck] + when: + - cel: "'$(params.runTests)' == 'true'" taskRef: - name: golang-build + resolver: bundles + params: + - name: bundle + value: gcr.io/tekton-releases/catalog/upstream/golang-build:0.3 + - name: name + value: golang-build + - name: kind + value: task params: - name: package value: $(params.package) @@ -107,7 +155,16 @@ spec: - name: publish-images runAfter: [build, unit-tests] taskRef: - name: publish-triggers-release + resolver: git + params: + - name: repo + value: triggers + - name: org + value: tektoncd + - name: revision + value: $(params.gitRevision) + - name: pathInRepo + value: tekton/publish.yaml params: - name: package value: $(params.package) @@ -117,12 +174,18 @@ spec: value: $(params.imageRegistry) - name: imageRegistryPath value: $(params.imageRegistryPath) + - name: imageRegistryUser + value: $(params.imageRegistryUser) + - name: imageRegistryRegions + value: $(params.imageRegistryRegions) - name: releaseAsLatest value: $(params.releaseAsLatest) + - name: serviceAccountPath + value: $(params.serviceAccountImagesPath) - name: platforms value: $(params.platforms) - - name: serviceAccountPath - value: $(params.serviceAccountPath) + - name: koExtraArgs + value: $(params.koExtraArgs) workspaces: - name: source workspace: workarea @@ -131,11 +194,18 @@ spec: workspace: workarea subpath: bucket - name: release-secret - workspace: release-secret + workspace: release-images-secret - name: publish-to-bucket runAfter: [publish-images] taskRef: - name: gcs-upload + resolver: bundles + params: + - name: bundle + value: gcr.io/tekton-releases/catalog/upstream/gcs-upload:0.3 + - name: name + value: gcs-upload + - name: kind + value: task workspaces: - name: credentials workspace: release-secret @@ -156,7 +226,14 @@ spec: operator: in values: ["true"] taskRef: - name: gcs-upload + resolver: bundles + params: + - name: bundle + value: gcr.io/tekton-releases/catalog/upstream/gcs-upload:0.3 + - name: name + value: gcs-upload + - name: kind + value: task workspaces: - name: credentials workspace: release-secret @@ -170,6 +247,8 @@ spec: value: $(params.versionTag) - name: serviceAccountPath value: $(params.serviceAccountPath) + - name: deleteExtraFiles + value: "true" # Uses rsync to copy content into latest - name: report-bucket runAfter: [publish-to-bucket] params: @@ -192,9 +271,14 @@ spec: description: The full URL of the interceptors file (no tag) in the bucket steps: - name: create-results - image: alpine + image: docker.io/library/alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d + env: + - name: RELEASE_BUCKET + value: $(params.releaseBucket) + - name: VERSION_TAG + value: $(params.versionTag) script: | - BASE_URL=$(echo "$(params.releaseBucket)/previous/$(params.versionTag)") + BASE_URL=$(echo "${RELEASE_BUCKET}/previous/${VERSION_TAG}") # If the bucket is in the gs:// return the corresponding public https URL BASE_URL=$(echo ${BASE_URL} | sed 's,gs://,https://storage.googleapis.com/,g') echo "${BASE_URL}/release.yaml" > $(results.release.path)