From f32daa72b1998cdbb65a37803bd889602dfc8049 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sun, 19 Jan 2025 05:00:53 +0000 Subject: [PATCH 1/2] [docker] Enable image builds from a git worktree This change enables docker image builds from [git worktrees](https://git-scm.com/docs/git-worktree). This required discoverying the git commit used to configure the avalanchego build script in advance of image build and providing it as an argument to image build. Previously, it was impossible to build docker images from a git worktree: - the avalanchego build script needed to discover the git commit to configure the binary with for traceability - the build script assumed a .git directory to discover the git commit - the dockerfile copied the working tree into the image prior to running the avalanchego build script - if the working tree was for a git worktree, the resulting copy wouldn't have a .git directory to discovery the git commit from - instead of a regular .git directory, a non-primary worktree has a .git file referencing a path in the primary worktree's .git directory. --- Dockerfile | 2 ++ scripts/build_antithesis_images.sh | 6 ++++-- scripts/build_avalanche.sh | 1 + scripts/build_bootstrap_monitor.sh | 1 + scripts/build_image.sh | 9 +++++++++ scripts/build_tmpnetctl.sh | 1 + scripts/constants.sh | 18 ------------------ scripts/get_image_tag.sh | 15 +++++++++++++++ scripts/git_commit.sh | 12 ++++++++++++ scripts/lib_build_antithesis_images.sh | 4 ++++ scripts/tests.build_antithesis_images.sh | 2 +- scripts/tests.build_image.sh | 3 +++ 12 files changed, 53 insertions(+), 21 deletions(-) create mode 100755 scripts/get_image_tag.sh create mode 100644 scripts/git_commit.sh diff --git a/Dockerfile b/Dockerfile index 5a6b58f9c194..330e85826c2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,9 +40,11 @@ RUN [ -d ./build ] && rm -rf ./build/* || true # enabling cross-compilation. ARG RACE_FLAG="" ARG BUILD_SCRIPT=build.sh +ARG AVALANCHEGO_COMMIT="" RUN . ./build_env.sh && \ echo "{CC=$CC, TARGETPLATFORM=$TARGETPLATFORM, BUILDPLATFORM=$BUILDPLATFORM}" && \ export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \ + export AVALANCHEGO_COMMIT="${AVALANCHEGO_COMMIT}" && \ ./scripts/${BUILD_SCRIPT} ${RACE_FLAG} # Create this directory in the builder to avoid requiring anything to be executed in the diff --git a/scripts/build_antithesis_images.sh b/scripts/build_antithesis_images.sh index 3aa7469c58f2..748b733d6b51 100755 --- a/scripts/build_antithesis_images.sh +++ b/scripts/build_antithesis_images.sh @@ -19,6 +19,9 @@ fi # Directory above this script AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) +source "${AVALANCHE_PATH}"/scripts/constants.sh +source "${AVALANCHE_PATH}"/scripts/git_commit.sh + # Import common functions used to build images for antithesis test setups source "${AVALANCHE_PATH}"/scripts/lib_build_antithesis_images.sh @@ -28,7 +31,6 @@ IMAGE_PREFIX="${IMAGE_PREFIX:-}" IMAGE_TAG="${IMAGE_TAG:-}" if [[ -z "${IMAGE_TAG}" ]]; then # Default to tagging with the commit hash - source "${AVALANCHE_PATH}"/scripts/constants.sh IMAGE_TAG="${commit_hash}" fi @@ -56,7 +58,7 @@ function build_antithesis_images_for_avalanchego { fi build_antithesis_images "${GO_VERSION}" "${image_prefix}" "antithesis-${test_setup}" "${IMAGE_TAG}" "${IMAGE_TAG}" \ "${AVALANCHE_PATH}/tests/antithesis/${test_setup}/Dockerfile" "${uninstrumented_node_dockerfile}" \ - "${AVALANCHE_PATH}" "${node_only}" + "${AVALANCHE_PATH}" "${node_only}" "${git_commit}" } if [[ "${TEST_SETUP}" == "avalanchego" ]]; then diff --git a/scripts/build_avalanche.sh b/scripts/build_avalanche.sh index 74ec675e2363..1812b9f55f59 100755 --- a/scripts/build_avalanche.sh +++ b/scripts/build_avalanche.sh @@ -26,6 +26,7 @@ done AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Load the constants source "$AVALANCHE_PATH"/scripts/constants.sh +source "$AVALANCHE_PATH"/scripts/git_commit.sh build_args="$race" echo "Building AvalancheGo..." diff --git a/scripts/build_bootstrap_monitor.sh b/scripts/build_bootstrap_monitor.sh index 5a4a86c32faf..d52282c65574 100755 --- a/scripts/build_bootstrap_monitor.sh +++ b/scripts/build_bootstrap_monitor.sh @@ -6,6 +6,7 @@ set -euo pipefail AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Load the constants source "$AVALANCHE_PATH"/scripts/constants.sh +source "$AVALANCHE_PATH"/scripts/git_commit.sh echo "Building bootstrap-monitor..." go build -ldflags\ diff --git a/scripts/build_image.sh b/scripts/build_image.sh index 70c944193501..cb8d5dd91b51 100755 --- a/scripts/build_image.sh +++ b/scripts/build_image.sh @@ -35,6 +35,9 @@ FORCE_TAG_LATEST="${FORCE_TAG_LATEST:-}" # Load the constants source "$AVALANCHE_PATH"/scripts/constants.sh +source "$AVALANCHE_PATH"/scripts/git_commit.sh + +image_tag="$("${AVALANCHE_PATH}"/scripts/get_image_tag.sh)" if [[ -z "${SKIP_BUILD_RACE}" && $image_tag == *"-r" ]]; then echo "Branch name must not end in '-r'" @@ -69,6 +72,12 @@ DOCKER_CMD="docker buildx build ${*}" GO_VERSION="$(go list -m -f '{{.GoVersion}}')" DOCKER_CMD="${DOCKER_CMD} --build-arg GO_VERSION=${GO_VERSION}" +# Provide the git commit as a build argument to avoid requiring this +# to be discovered within the image. This enables image builds from +# git worktrees since a non-primary worktree won't have a .git +# directory to copy into the image. +DOCKER_CMD="${DOCKER_CMD} --build-arg AVALANCHEGO_COMMIT=${git_commit}" + if [[ "${DOCKER_IMAGE}" == *"/"* ]]; then # Default to pushing when the image name includes a slash which indicates the # use of a registry e.g. diff --git a/scripts/build_tmpnetctl.sh b/scripts/build_tmpnetctl.sh index 132cf09ee38d..936d4a70f28e 100755 --- a/scripts/build_tmpnetctl.sh +++ b/scripts/build_tmpnetctl.sh @@ -6,6 +6,7 @@ set -euo pipefail AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Load the constants source "$AVALANCHE_PATH"/scripts/constants.sh +source "$AVALANCHE_PATH"/scripts/git_commit.sh echo "Building tmpnetctl..." go build -ldflags\ diff --git a/scripts/constants.sh b/scripts/constants.sh index eeaeb08b5317..84bd72da7fa0 100755 --- a/scripts/constants.sh +++ b/scripts/constants.sh @@ -13,24 +13,6 @@ AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Direct # Where AvalancheGo binary goes avalanchego_path="$AVALANCHE_PATH/build/avalanchego" -# Image tag based on current branch (shared between image build and its test script) -# TODO: fix "fatal: No names found, cannot describe anything" in github CI -image_tag=$(git symbolic-ref -q --short HEAD || git describe --tags --exact-match || true) -if [[ -z $image_tag ]]; then - # Supply a default tag when one is not discovered - image_tag=ci_dummy -elif [[ "$image_tag" == */* ]]; then - # Slashes are not legal for docker image tags - replace with dashes - image_tag=$(echo "$image_tag" | tr '/' '-') -fi - -# Current commit (shared between image build and its test script) -# WARNING: this will use the most recent commit even if there are un-committed changes present -full_commit_hash="$(git --git-dir="$AVALANCHE_PATH/.git" rev-parse HEAD)" -commit_hash="${full_commit_hash::8}" - -git_commit=${AVALANCHEGO_COMMIT:-$( git rev-list -1 HEAD )} - # Static compilation static_ld_flags='' if [ "${STATIC_COMPILATION:-}" = 1 ] diff --git a/scripts/get_image_tag.sh b/scripts/get_image_tag.sh new file mode 100755 index 000000000000..3f33f5752fbb --- /dev/null +++ b/scripts/get_image_tag.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Emits an image tag derived from the current branch or tag + +IMAGE_TAG="$( git symbolic-ref -q --short HEAD || git describe --tags --exact-match || true )" +if [[ -z "${IMAGE_TAG}" ]]; then + # Supply a default tag when one is not discovered + IMAGE_TAG=ci_dummy +elif [[ "${IMAGE_TAG}" == */* ]]; then + # Slashes are not legal for docker image tags - replace with dashes + IMAGE_TAG="$( echo "${IMAGE_TAG}" | tr '/' '-' )" +fi +echo "${IMAGE_TAG}" diff --git a/scripts/git_commit.sh b/scripts/git_commit.sh new file mode 100644 index 000000000000..545e959afa1c --- /dev/null +++ b/scripts/git_commit.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Ignore warnings about variables appearing unused since this file is not the consumer of the variables it defines. +# shellcheck disable=SC2034 + +set -euo pipefail + +AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Directory above this script + +# WARNING: this will use the most recent commit even if there are un-committed changes present +git_commit="${AVALANCHEGO_COMMIT:-$(git --git-dir="${AVALANCHE_PATH}/.git" rev-parse HEAD)}" +commit_hash="${git_commit::8}" diff --git a/scripts/lib_build_antithesis_images.sh b/scripts/lib_build_antithesis_images.sh index 0ab36a0bcdf3..85a29adea889 100644 --- a/scripts/lib_build_antithesis_images.sh +++ b/scripts/lib_build_antithesis_images.sh @@ -39,6 +39,7 @@ function build_antithesis_images { local uninstrumented_node_dockerfile=$7 local target_path=$8 local node_only=${9:-} + local avalanchego_commit=${10:-} # Define image names if [[ -n "${image_prefix}" ]]; then @@ -65,6 +66,9 @@ function build_antithesis_images { --build-arg GO_VERSION=${go_version}\ --build-arg BUILDER_IMAGE_TAG=${image_tag}\ --build-arg BUILDER_WORKDIR=${builder_workdir}" + if [[ -n "${avalanchego_commit:-}" ]]; then + docker_cmd="${docker_cmd} --build-arg AVALANCHEGO_COMMIT=${avalanchego_commit}" + fi # By default the node image is intended to be local-only. AVALANCHEGO_NODE_IMAGE="antithesis-avalanchego-node:${node_image_tag}" diff --git a/scripts/tests.build_antithesis_images.sh b/scripts/tests.build_antithesis_images.sh index 1093295fa311..82eb6d1d33f2 100755 --- a/scripts/tests.build_antithesis_images.sh +++ b/scripts/tests.build_antithesis_images.sh @@ -19,7 +19,7 @@ set -euo pipefail AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Discover the default tag that will be used for the image -source "${AVALANCHE_PATH}"/scripts/constants.sh +source "${AVALANCHE_PATH}"/scripts/git_commit.sh export IMAGE_TAG="${commit_hash}" # Build the images for the specified test setup diff --git a/scripts/tests.build_image.sh b/scripts/tests.build_image.sh index f4dd81754165..46bdd848e880 100755 --- a/scripts/tests.build_image.sh +++ b/scripts/tests.build_image.sh @@ -12,6 +12,9 @@ set -euo pipefail AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) source "$AVALANCHE_PATH"/scripts/constants.sh +source "$AVALANCHE_PATH"/scripts/git_commit.sh + +image_tag="$("${AVALANCHE_PATH}"/scripts/get_image_tag.sh)" build_and_test() { local image_name=$1 From 6c4b75e3734ad7a7e5b40c93a1c9d0b0aedd15c4 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sun, 19 Jan 2025 21:07:10 +0000 Subject: [PATCH 2/2] fixup: Improve docs for docker builder deployment in build_image.sh --- scripts/build_image.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/build_image.sh b/scripts/build_image.sh index cb8d5dd91b51..fa129f1c57d8 100755 --- a/scripts/build_image.sh +++ b/scripts/build_image.sh @@ -17,10 +17,12 @@ set -euo pipefail # # sudo apt-get install qemu qemu-user-static # -# After installing qemu, it will also be necessary to start a new builder that can -# support multiplatform builds: +# After installing qemu, it will also be necessary to start a new builder that +# supports multiplatform builds and can use the host's network: # -# docker buildx create --use +# docker buildx create --use --driver-opt network=host +# +# Without `network=host`, the builder will timeout running `go mod download`. # # Reference: https://docs.docker.com/buildx/working-with-buildx/