From 38be56bf938194c857c51d58077d9f5b1117e6e2 Mon Sep 17 00:00:00 2001 From: paolino Date: Fri, 10 Jan 2025 17:35:37 +0000 Subject: [PATCH 1/7] Add new push-to-dockerhub script to replace the nix based one --- .../buildkite/release/push-to-dockerhub.sh | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 scripts/buildkite/release/push-to-dockerhub.sh diff --git a/scripts/buildkite/release/push-to-dockerhub.sh b/scripts/buildkite/release/push-to-dockerhub.sh new file mode 100755 index 00000000000..ce65039623b --- /dev/null +++ b/scripts/buildkite/release/push-to-dockerhub.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euox pipefail + +TRIGGERED_BY=$(buildkite-agent meta-data get base-build) +NEW_GIT_TAG=$(buildkite-agent meta-data get release-version) +TEST_RC=$(buildkite-agent meta-data get test-rc) + +if [ "$RELEASE" == "false" ]; then + if [ "$TEST_RC" == "TRUE" ]; then + TAG="test" + else + TAG="nightly" + fi +else + TAG=$NEW_GIT_TAG +fi + +main_build=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + -X GET "https://api.buildkite.com/v2/builds" \ + | jq ".[] | select(.meta_data.\"triggered-by\" == \"$TRIGGERED_BY\")" \ + | jq .number) + +mkdir -p artifacts + +artifact() { + local artifact_name=$1 + # shellcheck disable=SC2155 + local artifact_value=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + -X GET "https://api.buildkite.com/v2/organizations/cardano-foundation/pipelines/cardano-wallet/builds/$main_build/artifacts?per_page=100" \ + | jq -r " [.[] | select(.filename == \"$artifact_name\")][0] \ + | .download_url") + curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -L \ + -o "artifacts/$artifact_name" \ + "$artifact_value" + local image_name="cardanofoundation/cardano-wallet:$TAG" + docker login -u cfhal -p "$DOCKER_HUB_TOKEN" + docker load -i "artifacts/$artifact_name" + docker push "$image_name" +} + +artifact "cardano-wallet-$NEW_GIT_TAG-docker-image.tgz" \ No newline at end of file From a8b91b4ddf5e5d8bb30cf717b3cdb0a499a14121 Mon Sep 17 00:00:00 2001 From: paolino Date: Fri, 10 Jan 2025 17:37:17 +0000 Subject: [PATCH 2/7] Remove push to dockerhub step from main pipeline --- .buildkite/pipeline.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7bae58c38de..6520acf47b3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -625,22 +625,6 @@ steps: agents: system: x86_64-linux - - block: Docker Build - depends_on: [] - if: build.env("TEST_RC") == "TRUE" - key: docker-build-block - - - label: Push Docker Image - depends_on: - - docker-build - key: build-docker - command: - - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" - - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" - - "./docker-build-push" - agents: - system: x86_64-linux - - group: Docker Checks depends_on: - docker-artifacts From 7d76894940e5cd34b9f194282962965b31401d16 Mon Sep 17 00:00:00 2001 From: paolino Date: Fri, 10 Jan 2025 17:39:04 +0000 Subject: [PATCH 3/7] Add push to dockerhub step for nightly and test tag --- .buildkite/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.buildkite/release.yml b/.buildkite/release.yml index 4db0e0ccc49..72874918cc0 100644 --- a/.buildkite/release.yml +++ b/.buildkite/release.yml @@ -76,6 +76,14 @@ steps: env: RELEASE: false + - label: Push image to dockerhub + commands: + - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-dockerhub.sh + agents: + system: x86_64-linux + env: + RELEASE: false + - group: Release depends_on: nightly-or-test if: build.branch == "master" From 6be34774d3c0bbdcc5065eb94cfa7c6b280ee6f2 Mon Sep 17 00:00:00 2001 From: paolino Date: Fri, 10 Jan 2025 17:40:11 +0000 Subject: [PATCH 4/7] Replace old push to dockerhub step with a blocked one using the new script --- .buildkite/release.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.buildkite/release.yml b/.buildkite/release.yml index 72874918cc0..4eb38ed37c0 100644 --- a/.buildkite/release.yml +++ b/.buildkite/release.yml @@ -131,15 +131,20 @@ steps: env: RELEASE: true - - label: Push Docker Image - depends_on: - - create-release - command: - - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" - - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" - - "./docker-build-push" + - block: Push to dockerhub + key: push-dockerhub + depends_on: create-release + + - label: Push to dockerhub + depends_on: push-dockerhub + artifact_paths: + - ./artifacts/*.tgz + commands: + - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-dockerhub.sh agents: system: x86_64-linux + env: + RELEASE: true - label: Update Documentation Links depends_on: create-release From da789e878ced81dfc2553ad0deb86a56eab6f306 Mon Sep 17 00:00:00 2001 From: paolino Date: Mon, 13 Jan 2025 10:21:05 +0000 Subject: [PATCH 5/7] Remove nix based push to dockerhub script --- .buildkite/docker-build-push.nix | 112 ------------------------------- flake.nix | 6 +- 2 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 .buildkite/docker-build-push.nix diff --git a/.buildkite/docker-build-push.nix b/.buildkite/docker-build-push.nix deleted file mode 100644 index ccefc5a1294..00000000000 --- a/.buildkite/docker-build-push.nix +++ /dev/null @@ -1,112 +0,0 @@ -# This script will load nix-built docker images of cardano-wallet -# into the Docker daemon (must be running), and then push to the -# Docker Hub. Credentials for the hub must already be installed with -# "docker login". -# -# There is a little bit of bash logic to replace the default repo and -# tag from the nix-build (../nix/docker.nix). -# -# 1. So you can test this with your own Dockerhub account, the repo -# (default "cardanofoundation/cardano-wallet") is changed to match the -# currently logged in Docker user's credentials. -# -# 2. The tag (default "VERSION") is changed to reflect the -# branch which is being built under this Buildkite pipeline. -# -# - If this is a git tag build (i.e. release) then the docker tag -# is left as-is. -# - If this is a master branch build then the docker tag is set to -# "dev-master". -# - Anything else is not tagged and not pushed. -# -# 3. After pushing the image to the repo, the "latest" tags are updated. -# -# - "cardanofoundation/cardano-wallet:latest" should point to the most -# recent VERSION tag build (shelley backend). -# - -{ defaultNix ? import ../default.nix {} -, pkgs ? defaultNix.legacyPackages.pkgs -, dockerImage ? defaultNix.dockerImage - -# Build system's Nixpkgs. We use this so that we have the same docker -# version as the docker daemon. -, hostPkgs ? import {} - -# Dockerhub repository for image tagging. -, dockerHubRepoName ? null -}: - -with hostPkgs; -with hostPkgs.lib; - -let - images = [ (impureCreated dockerImage) ]; - - # Override Docker image, setting its creation date to the current - # time rather than the unix epoch. - impureCreated = image: - image.overrideAttrs (oldAttrs: { created = "now"; }) - // { inherit (image) version backend; }; - -in - writeScript "docker-build-push" ('' - #!${runtimeShell} - - set -euox pipefail - - export PATH=${lib.makeBinPath [ docker gnused ]} - - ${if dockerHubRepoName == null then '' - reponame=cardano-wallet - username="$(docker info | sed '/Username:/!d;s/.* //')" - fullrepo="$username/$reponame" - '' else '' - fullrepo="${dockerHubRepoName}" - ''} - - '' + concatMapStringsSep "\n" (image: '' - echo "Loading ${image.name}" - docker load -i "${image}" - - # Apply tagging scheme - orig_tag="${image.imageName}:${image.imageTag}" - git_tag="''${BUILDKITE_TAG:-}" - git_branch="''${BUILDKITE_BRANCH:-}" - tags=() - if [[ "$git_tag" =~ ^v20 ]]; then - tags+=( "${image.imageTag}" ) - tags+=( "latest" ) - elif [[ "$git_branch" =~ ^release-candidate ]]; then - tags+=( "release-candidate" ) - else - tags+=( "test") - fi - - echo - echo "Testing that entrypoint works" - set +e - docker run --rm "$orig_tag" version - docker_status="$?" - if [ "$docker_status" -eq 0 ]; then - echo "OK" - elif [ "$docker_status" -eq 125 ]; then - echo "Docker failed to run ... oh well." - echo "Continuing..." - else - echo "Entrypoint command failed with code $docker_status" - exit 1 - fi - set -e - echo - - for tag in ''${tags[@]}; do - tagged="$fullrepo:$tag" - if [ "$tagged" != "$orig_tag" ]; then - echo "Retagging with $tagged" - docker tag "$orig_tag" "$tagged" - fi - echo "Pushing $tagged" - docker push "$tagged" - done - '') images) diff --git a/flake.nix b/flake.nix index 79b0cf50ba0..96a8ffe645d 100644 --- a/flake.nix +++ b/flake.nix @@ -388,11 +388,7 @@ // rec { dockerImage = mkDockerImage (mkPackages walletProject.projectCross.musl64); - pushDockerImage = import ./.buildkite/docker-build-push.nix { - hostPkgs = import hostNixpkgs { inherit system; }; - inherit dockerImage; - inherit (config) dockerHubRepoName; - }; + } // (lib.optionalAttrs buildPlatform.isLinux { nixosTests = import ./nix/nixos/tests { inherit pkgs; From 051ab29ae493508bb1625ef122bd5e70aabb9c8a Mon Sep 17 00:00:00 2001 From: paolino Date: Mon, 13 Jan 2025 11:34:26 +0000 Subject: [PATCH 6/7] Remove concurrency from nightly/test and release step groups --- .buildkite/release.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.buildkite/release.yml b/.buildkite/release.yml index 4eb38ed37c0..5296549cb79 100644 --- a/.buildkite/release.yml +++ b/.buildkite/release.yml @@ -47,6 +47,8 @@ steps: RELEASE: false agents: system: x86_64-linux + concurrency: 1 + concurrency_group: push-swagger-nightly-or-test - label: Push nightly or test release tag key: push-nightly-or-test-tag @@ -66,6 +68,8 @@ steps: system: x86_64-linux env: RELEASE: false + concurrency: 1 + concurrency_group: push-release-nightly-or-test - label: Push nightly or test release artifacts depends_on: push-nightly-or-test-release @@ -75,6 +79,8 @@ steps: system: x86_64-linux env: RELEASE: false + concurrency: 1 + concurrency_group: push-release-nightly-or-test - label: Push image to dockerhub commands: @@ -83,6 +89,8 @@ steps: system: x86_64-linux env: RELEASE: false + concurrency: 1 + concurrency_group: push-dockerhub-nightly-or-test - group: Release depends_on: nightly-or-test @@ -101,6 +109,8 @@ steps: RELEASE: true agents: system: x86_64-linux + concurrency: 1 + concurrency_group: push-swagger-release - label: Push release tag key: push-release-tag @@ -121,6 +131,8 @@ steps: system: x86_64-linux env: RELEASE: true + concurrency: 1 + concurrency_group: push-release-release - label: Push release artifacts depends_on: push-release @@ -130,6 +142,8 @@ steps: system: x86_64-linux env: RELEASE: true + concurrency: 1 + concurrency_group: push-release-release - block: Push to dockerhub key: push-dockerhub @@ -145,6 +159,8 @@ steps: system: x86_64-linux env: RELEASE: true + concurrency: 1 + concurrency_group: push-dockerhub-release - label: Update Documentation Links depends_on: create-release From c164467b1e91194bc12fe154a0aa4736296498ab Mon Sep 17 00:00:00 2001 From: paolino Date: Mon, 13 Jan 2025 11:59:55 +0000 Subject: [PATCH 7/7] Add 'latest' tag push in case of release --- scripts/buildkite/release/push-to-dockerhub.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/buildkite/release/push-to-dockerhub.sh b/scripts/buildkite/release/push-to-dockerhub.sh index ce65039623b..8a2e7502628 100755 --- a/scripts/buildkite/release/push-to-dockerhub.sh +++ b/scripts/buildkite/release/push-to-dockerhub.sh @@ -5,6 +5,7 @@ set -euox pipefail TRIGGERED_BY=$(buildkite-agent meta-data get base-build) NEW_GIT_TAG=$(buildkite-agent meta-data get release-version) TEST_RC=$(buildkite-agent meta-data get test-rc) +CABAL_VERSION=$(buildkite-agent meta-data get release-cabal-version) if [ "$RELEASE" == "false" ]; then if [ "$TEST_RC" == "TRUE" ]; then @@ -23,6 +24,8 @@ main_build=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ mkdir -p artifacts +repo="cardanofoundation/cardano-wallet" + artifact() { local artifact_name=$1 # shellcheck disable=SC2155 @@ -33,10 +36,19 @@ artifact() { curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -L \ -o "artifacts/$artifact_name" \ "$artifact_value" - local image_name="cardanofoundation/cardano-wallet:$TAG" docker login -u cfhal -p "$DOCKER_HUB_TOKEN" docker load -i "artifacts/$artifact_name" - docker push "$image_name" + local image_name="$repo:$TAG" + if [ "$RELEASE" == "false" ]; then + local loaded_image_name="$repo:$CABAL_VERSION" + docker tag "$loaded_image_name" "$image_name" + docker push "$image_name" + else + local latest_image_name="$repo:latest" + docker push "$image_name" + docker tag "$image_name" "$latest_image_name" + docker push "$latest_image_name" + fi } artifact "cardano-wallet-$NEW_GIT_TAG-docker-image.tgz" \ No newline at end of file