From 1658c2e905519035090224ac30d61263f81a52a1 Mon Sep 17 00:00:00 2001 From: Shamil Mubarakshin <127750046+shamil-mubarakshin@users.noreply.github.com> Date: Fri, 29 Dec 2023 12:36:27 +0100 Subject: [PATCH] [ubuntu] Cleanup bash scripts (#9076) * [ubuntu] Cleanup bash scripts * Fix ms-repos lsb_release * Fix install-bicep url * Fix install-nvm --- images/ubuntu/scripts/build/cleanup.sh | 19 ++++---- .../scripts/build/configure-apt-mock.sh | 8 ++-- .../scripts/build/configure-apt-sources.sh | 1 + .../scripts/build/configure-environment.sh | 4 +- images/ubuntu/scripts/build/configure-snap.sh | 1 + .../ubuntu/scripts/build/configure-system.sh | 11 ++--- .../scripts/build/install-actions-cache.sh | 4 +- .../scripts/build/install-apt-common.sh | 3 ++ .../ubuntu/scripts/build/install-apt-vital.sh | 2 + images/ubuntu/scripts/build/install-azcopy.sh | 2 + .../ubuntu/scripts/build/install-azure-cli.sh | 1 + images/ubuntu/scripts/build/install-bazel.sh | 1 + images/ubuntu/scripts/build/install-bicep.sh | 5 +- images/ubuntu/scripts/build/install-cmake.sh | 2 + .../scripts/build/install-codeql-bundle.sh | 1 + .../scripts/build/install-container-tools.sh | 1 + .../scripts/build/install-docker-compose.sh | 11 +++-- images/ubuntu/scripts/build/install-docker.sh | 36 +++++++------- .../scripts/build/install-dotnetcore-sdk.sh | 48 ++++++++++--------- images/ubuntu/scripts/build/install-erlang.sh | 6 +-- .../ubuntu/scripts/build/install-firefox.sh | 19 ++++---- .../ubuntu/scripts/build/install-gfortran.sh | 2 + images/ubuntu/scripts/build/install-git.sh | 1 + .../scripts/build/install-github-cli.sh | 2 + .../scripts/build/install-google-chrome.sh | 44 ++++++++--------- .../ubuntu/scripts/build/install-haskell.sh | 20 ++++---- images/ubuntu/scripts/build/install-heroku.sh | 2 - images/ubuntu/scripts/build/install-hhvm.sh | 1 + .../scripts/build/install-java-tools.sh | 4 +- images/ubuntu/scripts/build/install-kotlin.sh | 1 + .../scripts/build/install-kubernetes-tools.sh | 18 ++++--- .../scripts/build/install-microsoft-edge.sh | 33 +++++++------ .../ubuntu/scripts/build/install-mongodb.sh | 11 ++--- images/ubuntu/scripts/build/install-mono.sh | 10 ++-- .../ubuntu/scripts/build/install-ms-repos.sh | 4 +- images/ubuntu/scripts/build/install-mysql.sh | 1 + images/ubuntu/scripts/build/install-nodejs.sh | 4 +- images/ubuntu/scripts/build/install-nvm.sh | 5 +- images/ubuntu/scripts/build/install-oc-cli.sh | 4 +- .../ubuntu/scripts/build/install-oras-cli.sh | 1 + images/ubuntu/scripts/build/install-packer.sh | 1 + .../ubuntu/scripts/build/install-phantomjs.sh | 10 ++-- images/ubuntu/scripts/build/install-php.sh | 7 ++- .../scripts/build/install-pipx-packages.sh | 22 ++++----- .../scripts/build/install-postgresql.sh | 6 +-- .../scripts/build/install-powershell.sh | 5 +- images/ubuntu/scripts/build/install-pypy.sh | 19 ++++---- images/ubuntu/scripts/build/install-python.sh | 2 + images/ubuntu/scripts/build/install-rlang.sh | 7 +-- images/ubuntu/scripts/build/install-ruby.sh | 47 +++++++++--------- .../ubuntu/scripts/build/install-selenium.sh | 12 ++--- .../scripts/build/install-sqlpackage.sh | 1 + images/ubuntu/scripts/build/install-swift.sh | 12 ++--- images/ubuntu/scripts/build/install-yq.sh | 9 ++-- images/ubuntu/scripts/build/install-zstd.sh | 4 +- images/ubuntu/scripts/helpers/os.sh | 4 -- 56 files changed, 275 insertions(+), 247 deletions(-) diff --git a/images/ubuntu/scripts/build/cleanup.sh b/images/ubuntu/scripts/build/cleanup.sh index c6cf5e5df882..d0fa9a0e6e01 100644 --- a/images/ubuntu/scripts/build/cleanup.sh +++ b/images/ubuntu/scripts/build/cleanup.sh @@ -26,20 +26,19 @@ find /var/log -type f -regex ".*\.[0-9]$" -delete # wipe log files find /var/log/ -type f -exec cp /dev/null {} \; -# after cleanup -after=$(df / -Pm | awk 'NR==2{print $4}') - -# display size - echo "Before: $before MB" - echo "After : $after MB" - echo "Delta : $(($after-$before)) MB" - # delete symlink for tests running rm -f /usr/local/bin/invoke_tests # remove apt mock prefix=/usr/local/bin - for tool in apt apt-get apt-fast apt-key;do - sudo rm -f $prefix/$tool + sudo rm -f $prefix/$tool done + +# after cleanup +after=$(df / -Pm | awk 'NR==2{print $4}') + +# display size +echo "Before: $before MB" +echo "After : $after MB" +echo "Delta : $(($after-$before)) MB" diff --git a/images/ubuntu/scripts/build/configure-apt-mock.sh b/images/ubuntu/scripts/build/configure-apt-mock.sh index 18af3ca002cc..e12b0f56d13b 100644 --- a/images/ubuntu/scripts/build/configure-apt-mock.sh +++ b/images/ubuntu/scripts/build/configure-apt-mock.sh @@ -7,9 +7,9 @@ prefix=/usr/local/bin -for real_tool in /usr/bin/apt /usr/bin/apt-get /usr/bin/apt-fast /usr/bin/apt-key;do - tool=`basename $real_tool` - cat >$prefix/$tool <$prefix/$tool < /dev/null -then - yarn cache clean +if yarn --version > /dev/null; then + yarn cache clean fi -if npm --version -then - npm cache clean --force + +if npm --version; then + npm cache clean --force fi diff --git a/images/ubuntu/scripts/build/install-actions-cache.sh b/images/ubuntu/scripts/build/install-actions-cache.sh index e50b94c9431f..22c2f71f5506 100644 --- a/images/ubuntu/scripts/build/install-actions-cache.sh +++ b/images/ubuntu/scripts/build/install-actions-cache.sh @@ -17,8 +17,8 @@ echo "Setting up ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE variable to ${ACTION_ARCHIV set_etc_environment_variable "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE" "${ACTION_ARCHIVE_CACHE_DIR}" # Download latest release from github.com/actions/action-versions and untar to /opt/actionarchivecache -downloadUrl=$(resolve_github_release_asset_url "actions/action-versions" "endswith(\"action-versions.tar.gz\")" "latest") -archive_path=$(download_with_retry "$downloadUrl") +download_url=$(resolve_github_release_asset_url "actions/action-versions" "endswith(\"action-versions.tar.gz\")" "latest") +archive_path=$(download_with_retry "$download_url") tar -xzf "$archive_path" -C $ACTION_ARCHIVE_CACHE_DIR invoke_tests "ActionArchiveCache" diff --git a/images/ubuntu/scripts/build/install-apt-common.sh b/images/ubuntu/scripts/build/install-apt-common.sh index 7a63390dec54..590c5e41d991 100644 --- a/images/ubuntu/scripts/build/install-apt-common.sh +++ b/images/ubuntu/scripts/build/install-apt-common.sh @@ -3,10 +3,13 @@ ## File: install-apt-common.sh ## Desc: Install basic command line utilities and dev packages ################################################################################ + +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh common_packages=$(get_toolset_value .apt.common_packages[]) cmd_packages=$(get_toolset_value .apt.cmd_packages[]) + for package in $common_packages $cmd_packages; do echo "Install $package" apt-get install -y --no-install-recommends $package diff --git a/images/ubuntu/scripts/build/install-apt-vital.sh b/images/ubuntu/scripts/build/install-apt-vital.sh index 69c18beac54f..26a3fa819be4 100644 --- a/images/ubuntu/scripts/build/install-apt-vital.sh +++ b/images/ubuntu/scripts/build/install-apt-vital.sh @@ -3,6 +3,8 @@ ## File: install-apt-vital.sh ## Desc: Install vital command line utilities ################################################################################ + +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh vital_packages=$(get_toolset_value .apt.vital_packages[]) diff --git a/images/ubuntu/scripts/build/install-azcopy.sh b/images/ubuntu/scripts/build/install-azcopy.sh index 83132cc46c27..6865db8304af 100644 --- a/images/ubuntu/scripts/build/install-azcopy.sh +++ b/images/ubuntu/scripts/build/install-azcopy.sh @@ -4,6 +4,7 @@ ## Desc: Install AzCopy ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Install AzCopy10 @@ -11,6 +12,7 @@ source $HELPER_SCRIPTS/install.sh archive_path=$(download_with_retry "https://azcopyvnext.azureedge.net/releases/release-10.21.2-20231106/azcopy_linux_amd64_10.21.2.tar.gz") tar xzf "$archive_path" --strip-components=1 -C /tmp install /tmp/azcopy /usr/local/bin/azcopy + # Create azcopy 10 alias for backward compatibility ln -sf /usr/local/bin/azcopy /usr/local/bin/azcopy10 diff --git a/images/ubuntu/scripts/build/install-azure-cli.sh b/images/ubuntu/scripts/build/install-azure-cli.sh index eef93fe18212..a2665b592850 100644 --- a/images/ubuntu/scripts/build/install-azure-cli.sh +++ b/images/ubuntu/scripts/build/install-azure-cli.sh @@ -7,6 +7,7 @@ # Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) curl -fsSL https://aka.ms/InstallAzureCLIDeb | sudo bash echo "azure-cli https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-linux?pivots=apt" >> $HELPER_SCRIPTS/apt-sources.txt + rm -f /etc/apt/sources.list.d/azure-cli.list rm -f /etc/apt/sources.list.d/azure-cli.list.save diff --git a/images/ubuntu/scripts/build/install-bazel.sh b/images/ubuntu/scripts/build/install-bazel.sh index 46d13998d22f..a9b066193315 100644 --- a/images/ubuntu/scripts/build/install-bazel.sh +++ b/images/ubuntu/scripts/build/install-bazel.sh @@ -4,6 +4,7 @@ ## Desc: Install Bazel and Bazelisk (A user-friendly launcher for Bazel) ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Install bazelisk diff --git a/images/ubuntu/scripts/build/install-bicep.sh b/images/ubuntu/scripts/build/install-bicep.sh index 840b95310d12..5cf996b96cf5 100644 --- a/images/ubuntu/scripts/build/install-bicep.sh +++ b/images/ubuntu/scripts/build/install-bicep.sh @@ -4,10 +4,13 @@ ## Desc: Install bicep cli ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Install Bicep CLI -bicep_binary_path=$(download_with_retry "https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64") +download_url=$(resolve_github_release_asset_url "Azure/bicep" "endswith(\"bicep-linux-x64\")" "latest") +bicep_binary_path=$(download_with_retry "${download_url}") + # Mark it as executable install "$bicep_binary_path" /usr/local/bin/bicep diff --git a/images/ubuntu/scripts/build/install-cmake.sh b/images/ubuntu/scripts/build/install-cmake.sh index 249f905e1483..3a53eec3e613 100644 --- a/images/ubuntu/scripts/build/install-cmake.sh +++ b/images/ubuntu/scripts/build/install-cmake.sh @@ -16,10 +16,12 @@ else # Download script to install CMake download_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"inux-x86_64.sh\")" "latest") curl -fsSL "${download_url}" -o cmakeinstall.sh + # Supply chain security - CMake hash_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"SHA-256.txt\")" "latest") external_hash=$(get_checksum_from_url "$hash_url" "linux-x86_64.sh" "SHA256") use_checksum_comparison "cmakeinstall.sh" "$external_hash" + # Install CMake and remove the install script chmod +x cmakeinstall.sh \ && ./cmakeinstall.sh --prefix=/usr/local --exclude-subdir \ diff --git a/images/ubuntu/scripts/build/install-codeql-bundle.sh b/images/ubuntu/scripts/build/install-codeql-bundle.sh index c9dd18120313..a92933aabfdf 100644 --- a/images/ubuntu/scripts/build/install-codeql-bundle.sh +++ b/images/ubuntu/scripts/build/install-codeql-bundle.sh @@ -4,6 +4,7 @@ ## Desc: Install CodeQL CLI Bundle to the toolcache. ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Retrieve the CLI version of the latest CodeQL bundle. diff --git a/images/ubuntu/scripts/build/install-container-tools.sh b/images/ubuntu/scripts/build/install-container-tools.sh index 130356945e82..f7374163719f 100644 --- a/images/ubuntu/scripts/build/install-container-tools.sh +++ b/images/ubuntu/scripts/build/install-container-tools.sh @@ -4,6 +4,7 @@ ## Desc: Install container tools: podman, buildah and skopeo onto the image ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/os.sh # diff --git a/images/ubuntu/scripts/build/install-docker-compose.sh b/images/ubuntu/scripts/build/install-docker-compose.sh index 3cd844adb2f2..4d152c77c391 100644 --- a/images/ubuntu/scripts/build/install-docker-compose.sh +++ b/images/ubuntu/scripts/build/install-docker-compose.sh @@ -8,13 +8,14 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -# Install docker-compose v1 from releases -URL="https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64" -curl -fsSL "${URL}" -o /tmp/docker-compose-v1 +# Download docker-compose v1 from releases +binary_path=$(download_with_retry "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64") # Supply chain security - Docker Compose v1 external_hash="f3f10cf3dbb8107e9ba2ea5f23c1d2159ff7321d16f0a23051d68d8e2547b323" -use_checksum_comparison "/tmp/docker-compose-v1" "${external_hash}" -install /tmp/docker-compose-v1 /usr/local/bin/docker-compose +use_checksum_comparison "${binary_path}" "${external_hash}" + +# Install docker-compose v1 +install "${binary_path}" "/usr/local/bin/docker-compose" invoke_tests "Tools" "Docker-compose v1" diff --git a/images/ubuntu/scripts/build/install-docker.sh b/images/ubuntu/scripts/build/install-docker.sh index cbbed19a85e0..039398bd4e3e 100644 --- a/images/ubuntu/scripts/build/install-docker.sh +++ b/images/ubuntu/scripts/build/install-docker.sh @@ -6,28 +6,28 @@ ################################################################################ # Source the helpers for use with the script -source $HELPER_SCRIPTS/os.sh source $HELPER_SCRIPTS/install.sh -repo_url="https://download.docker.com/linux/ubuntu" -gpg_key="/usr/share/keyrings/docker.gpg" -repo_path="/etc/apt/sources.list.d/docker.list" +REPO_URL="https://download.docker.com/linux/ubuntu" +GPG_KEY="/usr/share/keyrings/docker.gpg" +REPO_PATH="/etc/apt/sources.list.d/docker.list" -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o $gpg_key -echo "deb [arch=amd64 signed-by=$gpg_key] $repo_url $(get_os_version_label) stable" > $repo_path +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o $GPG_KEY +echo "deb [arch=amd64 signed-by=$GPG_KEY] $REPO_URL $(lsb_release -cs) stable" > $REPO_PATH apt-get update apt-get install --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin # Download docker compose v2 from releases URL=$(resolve_github_release_asset_url "docker/compose" "endswith(\"compose-linux-x86_64\")" "latest") -curl -fsSL "${URL}" -o /tmp/docker-compose +compose_binary_path=$(download_with_retry "${URL}" "/tmp/docker-compose-v2") + # Supply chain security - Docker Compose v2 compose_hash_url=$(resolve_github_release_asset_url "docker/compose" "endswith(\"checksums.txt\")" "latest") compose_external_hash=$(get_checksum_from_url "${compose_hash_url}" "compose-linux-x86_64" "SHA256") -use_checksum_comparison "/tmp/docker-compose" "${compose_external_hash}" -# Install docker compose v2 -install /tmp/docker-compose /usr/libexec/docker/cli-plugins/docker-compose +use_checksum_comparison "${compose_binary_path}" "${compose_external_hash}" +# Install docker compose v2 +install "${compose_binary_path}" /usr/libexec/docker/cli-plugins/docker-compose # docker from official repo introduced different GID generation: https://github.com/actions/runner-images/issues/8157 gid=$(cut -d ":" -f 3 /etc/group | grep "^1..$" | sort -n | tail -n 1 | awk '{ print $1+1 }') @@ -42,11 +42,11 @@ systemctl is-enabled --quiet docker.service || systemctl enable docker.service sleep 10 docker info -if [ "${DOCKERHUB_PULL_IMAGES:-yes}" == "yes" ]; then +if [[ "${DOCKERHUB_PULL_IMAGES:-yes}" == "yes" ]]; then # If credentials are provided, attempt to log into Docker Hub # with a paid account to avoid Docker Hub's rate limit. - if [ "${DOCKERHUB_LOGIN}" ] && [ "${DOCKERHUB_PASSWORD}" ]; then - docker login --username "${DOCKERHUB_LOGIN}" --password "${DOCKERHUB_PASSWORD}" + if [[ "${DOCKERHUB_LOGIN}" ]] && [[ "${DOCKERHUB_PASSWORD}" ]]; then + docker login --username "${DOCKERHUB_LOGIN}" --password "${DOCKERHUB_PASSWORD}" fi # Pull images @@ -65,19 +65,21 @@ fi # Download amazon-ecr-credential-helper aws_latest_release_url="https://api.github.com/repos/awslabs/amazon-ecr-credential-helper/releases/latest" -aws_helper_url=$(curl "${authString[@]}" -fsSL "${aws_latest_release_url}" | jq -r '.body' | awk -F'[()]' '/linux-amd64/ {print $2}') +aws_helper_url=$(curl -fsSL "${aws_latest_release_url}" | jq -r '.body' | awk -F'[()]' '/linux-amd64/ {print $2}') aws_helper_binary_path=$(download_with_retry "$aws_helper_url") + # Supply chain security - amazon-ecr-credential-helper aws_helper_external_hash=$(get_checksum_from_url "${aws_helper_url}.sha256" "docker-credential-ecr-login" "SHA256") use_checksum_comparison "$aws_helper_binary_path" "$aws_helper_external_hash" + # Install amazon-ecr-credential-helper install "$aws_helper_binary_path" "/usr/bin/docker-credential-ecr-login" # Cleanup custom repositories -rm $gpg_key -rm $repo_path +rm $GPG_KEY +rm $REPO_PATH invoke_tests "Tools" "Docker" -if [ "${DOCKERHUB_PULL_IMAGES:-yes}" == "yes" ]; then +if [[ "${DOCKERHUB_PULL_IMAGES:-yes}" == "yes" ]]; then invoke_tests "Tools" "Docker images" fi diff --git a/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh b/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh index 82c683d55dda..591a41729abb 100644 --- a/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh +++ b/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh @@ -4,23 +4,38 @@ ## Desc: Install .NET Core SDK ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/etc-environment.sh source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/os.sh +extract_dotnet_sdk() { + local archive_name=$1 + + set -e + destination="./tmp-$(basename -s .tar.gz $archive_name)" + + echo "Extracting $archive_name to $destination" + mkdir "$destination" && tar -C "$destination" -xzf "$archive_name" + rsync -qav --remove-source-files "$destination/shared/" /usr/share/dotnet/shared/ + rsync -qav --remove-source-files "$destination/host/" /usr/share/dotnet/host/ + rsync -qav --remove-source-files "$destination/sdk/" /usr/share/dotnet/sdk/ + rm -rf "$destination" "$archive_name" +} + # Ubuntu 20 doesn't support EOL versions -LATEST_DOTNET_PACKAGES=$(get_toolset_value '.dotnet.aptPackages[]') -DOTNET_VERSIONS=$(get_toolset_value '.dotnet.versions[]') -DOTNET_TOOLS=$(get_toolset_value '.dotnet.tools[].name') +latest_dotnet_packages=$(get_toolset_value '.dotnet.aptPackages[]') +dotnet_versions=$(get_toolset_value '.dotnet.versions[]') +dotnet_tools=$(get_toolset_value '.dotnet.tools[].name') # Disable telemetry export DOTNET_CLI_TELEMETRY_OPTOUT=1 +# Install .NET SDK from apt # There is a versions conflict, that leads to # Microsoft <-> Canonical repos dependencies mix up. # Give Microsoft's repo higher priority to avoid collisions. # See: https://github.com/dotnet/core/issues/7699 - cat << EOF > /etc/apt/preferences.d/dotnet Package: *net* Pin: origin packages.microsoft.com @@ -29,7 +44,7 @@ EOF apt-get update -for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do +for latest_package in ${latest_dotnet_packages[@]}; do echo "Determing if .NET Core ($latest_package) is installed" if ! dpkg -S $latest_package &> /dev/null; then echo "Could not find .NET Core ($latest_package), installing..." @@ -43,9 +58,10 @@ rm /etc/apt/preferences.d/dotnet apt-get update +# Install .NET SDK from home repository # Get list of all released SDKs from channels which are not end-of-life or preview sdks=() -for version in ${DOTNET_VERSIONS[@]}; do +for version in ${dotnet_versions[@]}; do release_url="https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/${version}/releases.json" releases=$(cat "$(download_with_retry "$release_url")") if [[ $version == "6.0" ]]; then @@ -56,19 +72,7 @@ for version in ${DOTNET_VERSIONS[@]}; do fi done -sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | sort -r | uniq -w 5) - -extract_dotnet_sdk() { - local ARCHIVE_NAME="$1" - set -e - dest="./tmp-$(basename -s .tar.gz $ARCHIVE_NAME)" - echo "Extracting $ARCHIVE_NAME to $dest" - mkdir "$dest" && tar -C "$dest" -xzf "$ARCHIVE_NAME" - rsync -qav --remove-source-files "$dest/shared/" /usr/share/dotnet/shared/ - rsync -qav --remove-source-files "$dest/host/" /usr/share/dotnet/host/ - rsync -qav --remove-source-files "$dest/sdk/" /usr/share/dotnet/sdk/ - rm -rf "$dest" "$ARCHIVE_NAME" -} +sorted_sdks=$(echo ${sdks[@]} | tr ' ' '\n' | sort -r | uniq -w 5) # Download/install additional SDKs in parallel export -f download_with_retry @@ -76,7 +80,7 @@ export -f extract_dotnet_sdk parallel --jobs 0 --halt soon,fail=1 \ 'url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/{}/dotnet-sdk-{}-linux-x64.tar.gz"; \ - download_with_retry $url' ::: "${sortedSdks[@]}" + download_with_retry $url' ::: "${sorted_sdks[@]}" find . -name "*.tar.gz" | parallel --halt soon,fail=1 'extract_dotnet_sdk {}' @@ -87,8 +91,8 @@ set_etc_environment_variable DOTNET_NOLOGO 1 set_etc_environment_variable DOTNET_MULTILEVEL_LOOKUP 0 prepend_etc_environment_path '$HOME/.dotnet/tools' -# install dotnet tools -for dotnet_tool in ${DOTNET_TOOLS[@]}; do +# Install .Net tools +for dotnet_tool in ${dotnet_tools[@]}; do echo "Installing dotnet tool $dotnet_tool" dotnet tool install $dotnet_tool --tool-path '/etc/skel/.dotnet/tools' done diff --git a/images/ubuntu/scripts/build/install-erlang.sh b/images/ubuntu/scripts/build/install-erlang.sh index e78439caef46..d5308a5cc238 100644 --- a/images/ubuntu/scripts/build/install-erlang.sh +++ b/images/ubuntu/scripts/build/install-erlang.sh @@ -18,9 +18,9 @@ apt-get update apt-get install --no-install-recommends esl-erlang # Install rebar3 -rebar3_url="https://github.com/erlang/rebar3/releases/latest/download/rebar3" -rebar3_binary_path=$(download_with_retry "$rebar3_url") -install "$rebar3_binary_path" /usr/local/bin/rebar3 +rebar3_url=$(resolve_github_release_asset_url "erlang/rebar3" "endswith(\"rebar3\")" "latest") +binary_path=$(download_with_retry "$rebar3_url") +install "$binary_path" /usr/local/bin/rebar3 # Clean up source list rm $source_list diff --git a/images/ubuntu/scripts/build/install-firefox.sh b/images/ubuntu/scripts/build/install-firefox.sh index b4a81dcba508..837c0f6a3bfc 100644 --- a/images/ubuntu/scripts/build/install-firefox.sh +++ b/images/ubuntu/scripts/build/install-firefox.sh @@ -5,28 +5,27 @@ ################################################################################ # Source the helpers for use with the script -source "$HELPER_SCRIPTS/install.sh" -source "$HELPER_SCRIPTS/os.sh" +source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/etc-environment.sh # Mozillateam PPA is added manually because sometimes # lanuchad portal sends empty answer when trying to add it automatically -repo_url="http://ppa.launchpad.net/mozillateam/ppa/ubuntu" -gpg_fingerprint="0ab215679c571d1c8325275b9bdb3d89ce49ec21" -gpg_key="/etc/apt/trusted.gpg.d/mozillateam_ubuntu_ppa.gpg" -repo_path="/etc/apt/sources.list.d/mozillateam-ubuntu-ppa-focal.list" +REPO_URL="http://ppa.launchpad.net/mozillateam/ppa/ubuntu" +GPG_FINGERPRINT="0ab215679c571d1c8325275b9bdb3d89ce49ec21" +GPG_KEY="/etc/apt/trusted.gpg.d/mozillateam_ubuntu_ppa.gpg" +REPO_PATH="/etc/apt/sources.list.d/mozillateam-ubuntu-ppa-focal.list" # Install Firefox -curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x${gpg_fingerprint}" | sudo gpg --dearmor -o $gpg_key -echo "deb $repo_url $(get_os_version_label) main" > $repo_path +curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x${GPG_FINGERPRINT}" | sudo gpg --dearmor -o $GPG_KEY +echo "deb $REPO_URL $(lsb_release -cs) main" > $REPO_PATH apt-get update apt-get install --target-release 'o=LP-PPA-mozillateam' -y firefox -rm $repo_path +rm $REPO_PATH # Document apt source repo's -echo "mozillateam $repo_url" >> $HELPER_SCRIPTS/apt-sources.txt +echo "mozillateam $REPO_URL" >> $HELPER_SCRIPTS/apt-sources.txt # add to gloabl system preferences for firefox locale en_US, because other browsers have en_US local. # Default firefox local is en_GB diff --git a/images/ubuntu/scripts/build/install-gfortran.sh b/images/ubuntu/scripts/build/install-gfortran.sh index 024a0311e978..04d848f6dd64 100644 --- a/images/ubuntu/scripts/build/install-gfortran.sh +++ b/images/ubuntu/scripts/build/install-gfortran.sh @@ -3,6 +3,8 @@ ## File: install-gfortran.sh ## Desc: Install GNU Fortran ################################################################################ + +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/os.sh diff --git a/images/ubuntu/scripts/build/install-git.sh b/images/ubuntu/scripts/build/install-git.sh index 385df085559f..f38c586e2b74 100644 --- a/images/ubuntu/scripts/build/install-git.sh +++ b/images/ubuntu/scripts/build/install-git.sh @@ -14,6 +14,7 @@ add-apt-repository $GIT_REPO -y apt-get update apt-get install git -y git --version + # Git version 2.35.2 introduces security fix that breaks action\checkout https://github.com/actions/checkout/issues/760 cat <> /etc/gitconfig [safe] diff --git a/images/ubuntu/scripts/build/install-github-cli.sh b/images/ubuntu/scripts/build/install-github-cli.sh index b38844756e96..bebdf9a349bd 100644 --- a/images/ubuntu/scripts/build/install-github-cli.sh +++ b/images/ubuntu/scripts/build/install-github-cli.sh @@ -12,10 +12,12 @@ source $HELPER_SCRIPTS/install.sh # Download GitHub CLI gh_cli_url=$(resolve_github_release_asset_url "cli/cli" "contains(\"linux\") and contains(\"amd64\") and endswith(\".deb\")" "latest") gh_cli_deb_path=$(download_with_retry "$gh_cli_url") + # Supply chain security - GitHub CLI hash_url=$(resolve_github_release_asset_url "cli/cli" "endswith(\"checksums.txt\")" "latest") external_hash=$(get_checksum_from_url "$hash_url" "linux_amd64.deb" "SHA256") use_checksum_comparison "$gh_cli_deb_path" "$external_hash" + # Install GitHub CLI apt install "$gh_cli_deb_path" diff --git a/images/ubuntu/scripts/build/install-google-chrome.sh b/images/ubuntu/scripts/build/install-google-chrome.sh index b8458cbe900a..b8f5016a0b21 100644 --- a/images/ubuntu/scripts/build/install-google-chrome.sh +++ b/images/ubuntu/scripts/build/install-google-chrome.sh @@ -36,51 +36,51 @@ get_chromium_revision() { # Download and install Google Chrome CHROME_DEB_URL="https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" -CHROME_DEB_PATH=$(download_with_retry "$CHROME_DEB_URL") -apt install "$CHROME_DEB_PATH" -f +chrome_deb_path=$(download_with_retry "$CHROME_DEB_URL") +apt install "$chrome_deb_path" -f set_etc_environment_variable "CHROME_BIN" "/usr/bin/google-chrome" # Remove Google Chrome repo rm -f /etc/cron.daily/google-chrome /etc/apt/sources.list.d/google-chrome.list /etc/apt/sources.list.d/google-chrome.list.save # Parse Google Chrome version -FULL_CHROME_VERSION=$(google-chrome --product-version) -CHROME_VERSION=${FULL_CHROME_VERSION%.*} -echo "Chrome version is $FULL_CHROME_VERSION" +full_chrome_version=$(google-chrome --product-version) +chrome_version=${full_chrome_version%.*} +echo "Chrome version is $full_chrome_version" # Get chrome versions information CHROME_PLATFORM="linux64" CHROME_VERSIONS_URL="https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build-with-downloads.json" -CHROME_VERSIONS_JSON=$(curl -fsSL "${CHROME_VERSIONS_URL}") +chrome_versions_json=$(curl -fsSL "${CHROME_VERSIONS_URL}") # Download and unpack the latest release of chromedriver -CHROMEDRIVER_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].version') -CHROMEDRIVER_URL=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].downloads.chromedriver[] | select(.platform=="'"${CHROME_PLATFORM}"'").url') +chromedriver_version=$(echo "${chrome_versions_json}" | jq -r '.builds["'"$chrome_version"'"].version') +chromedriver_url=$(echo "${chrome_versions_json}" | jq -r '.builds["'"$chrome_version"'"].downloads.chromedriver[] | select(.platform=="'"${CHROME_PLATFORM}"'").url') CHROMEDRIVER_DIR="/usr/local/share/chromedriver-linux64" -CHROMEDRIVER_BIN="$CHROMEDRIVER_DIR/chromedriver" +chromedriver_bin="$CHROMEDRIVER_DIR/chromedriver" -echo "Installing chromedriver version $CHROMEDRIVER_VERSION" -driver_archive_path=$(download_with_retry "$CHROMEDRIVER_URL") +echo "Installing chromedriver version $chromedriver_version" +driver_archive_path=$(download_with_retry "$chromedriver_url") unzip -qq "$driver_archive_path" -d /usr/local/share -chmod +x $CHROMEDRIVER_BIN -ln -s "$CHROMEDRIVER_BIN" /usr/bin/ +chmod +x $chromedriver_bin +ln -s "$chromedriver_bin" /usr/bin/ set_etc_environment_variable "CHROMEWEBDRIVER" "${CHROMEDRIVER_DIR}" # Download and unpack Chromium -CHROME_REVISION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].revision') -CHROMIUM_REVISION=$(get_chromium_revision $CHROME_REVISION) -CHROMIUM_URL="https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${CHROMIUM_REVISION}%2Fchrome-linux.zip?alt=media" +chrome_revision=$(echo "${chrome_versions_json}" | jq -r '.builds["'"$chrome_version"'"].revision') +chromium_revision=$(get_chromium_revision $chrome_revision) +chromium_url="https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${chromium_revision}%2Fchrome-linux.zip?alt=media" CHROMIUM_DIR="/usr/local/share/chromium" -CHROMIUM_BIN="${CHROMIUM_DIR}/chrome-linux/chrome" +chromium_bin="${CHROMIUM_DIR}/chrome-linux/chrome" -echo "Installing chromium revision $CHROMIUM_REVISION" -CHROMIUM_ARCHIVE_PATH=$(download_with_retry "$CHROMIUM_URL") +echo "Installing chromium revision $chromium_revision" +chromium_archive_path=$(download_with_retry "$chromium_url") mkdir $CHROMIUM_DIR -unzip -qq "$CHROMIUM_ARCHIVE_PATH" -d $CHROMIUM_DIR +unzip -qq "$chromium_archive_path" -d $CHROMIUM_DIR -ln -s $CHROMIUM_BIN /usr/bin/chromium -ln -s $CHROMIUM_BIN /usr/bin/chromium-browser +ln -s $chromium_bin /usr/bin/chromium +ln -s $chromium_bin /usr/bin/chromium-browser invoke_tests "Browsers" "Chrome" invoke_tests "Browsers" "Chromium" diff --git a/images/ubuntu/scripts/build/install-haskell.sh b/images/ubuntu/scripts/build/install-haskell.sh index a5710fb7340a..b249f0fc3607 100644 --- a/images/ubuntu/scripts/build/install-haskell.sh +++ b/images/ubuntu/scripts/build/install-haskell.sh @@ -21,15 +21,15 @@ curl --proto '=https' --tlsv1.2 -fsSL https://get-ghcup.haskell.org | sh > /dev/ export PATH="$ghcup_bin:$PATH" prepend_etc_environment_path $ghcup_bin -availableVersions=$(ghcup list -t ghc -r | grep -v "prerelease" | awk '{print $2}') - -# Get 2 latest Haskell Major.Minor versions -minorMajorVersions=$(echo "$availableVersions" | cut -d"." -f 1,2 | uniq | tail -n2) -for majorMinorVersion in $minorMajorVersions; do - fullVersion=$(echo "$availableVersions" | grep "$majorMinorVersion." | tail -n1) - echo "install ghc version $fullVersion..." - ghcup install ghc $fullVersion - ghcup set ghc $fullVersion +available_versions=$(ghcup list -t ghc -r | grep -v "prerelease" | awk '{print $2}') + +# Install 2 latest Haskell Major.Minor versions +major_minor_versions=$(echo "$available_versions" | cut -d"." -f 1,2 | uniq | tail -n2) +for major_minor_version in $major_minor_versions; do + full_version=$(echo "$available_versions" | grep "$major_minor_version." | tail -n1) + echo "install ghc version $full_version..." + ghcup install ghc $full_version + ghcup set ghc $full_version done echo "install cabal..." @@ -39,6 +39,6 @@ chmod -R 777 $GHCUP_INSTALL_BASE_PREFIX/.ghcup ln -s $GHCUP_INSTALL_BASE_PREFIX/.ghcup /etc/skel/.ghcup # Install the latest stable release of haskell stack -curl -fsSL https://get.haskellstack.org/ | sh +curl -fsSL https://get.haskellstack.org/ | bash invoke_tests "Haskell" diff --git a/images/ubuntu/scripts/build/install-heroku.sh b/images/ubuntu/scripts/build/install-heroku.sh index b432729ead77..dc53ed7bfb7f 100644 --- a/images/ubuntu/scripts/build/install-heroku.sh +++ b/images/ubuntu/scripts/build/install-heroku.sh @@ -4,8 +4,6 @@ ## Desc: Install Heroku CLI. Based on instructions found here: https://devcenter.heroku.com/articles/heroku-cli ################################################################################ -## Install Heroku CLI - # add heroku repository to apt echo "deb https://cli-assets.heroku.com/channels/stable/apt ./" > /etc/apt/sources.list.d/heroku.list diff --git a/images/ubuntu/scripts/build/install-hhvm.sh b/images/ubuntu/scripts/build/install-hhvm.sh index 49591a51c12e..0e1bdb25d43a 100644 --- a/images/ubuntu/scripts/build/install-hhvm.sh +++ b/images/ubuntu/scripts/build/install-hhvm.sh @@ -6,6 +6,7 @@ apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xB4112585D386EB94 add-apt-repository https://dl.hhvm.com/ubuntu + apt-get update apt-get -qq install -y hhvm diff --git a/images/ubuntu/scripts/build/install-java-tools.sh b/images/ubuntu/scripts/build/install-java-tools.sh index b1776b7940f0..c366d87b0c4e 100644 --- a/images/ubuntu/scripts/build/install-java-tools.sh +++ b/images/ubuntu/scripts/build/install-java-tools.sh @@ -4,8 +4,8 @@ ## Desc: Install Java and related tooling (Ant, Gradle, Maven) ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -source $HELPER_SCRIPTS/os.sh source $HELPER_SCRIPTS/etc-environment.sh create_java_environment_variable() { @@ -64,7 +64,7 @@ install_open_jdk() { # Add Addoptium PPA # apt-key is deprecated, dearmor and add manually wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor > /usr/share/keyrings/adoptium.gpg -echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb/ $(get_os_version_label) main" > /etc/apt/sources.list.d/adoptium.list +echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/adoptium.list # Get all the updates from enabled repositories. apt-get update diff --git a/images/ubuntu/scripts/build/install-kotlin.sh b/images/ubuntu/scripts/build/install-kotlin.sh index 2b7d2367e671..80edb8a33b9c 100644 --- a/images/ubuntu/scripts/build/install-kotlin.sh +++ b/images/ubuntu/scripts/build/install-kotlin.sh @@ -5,6 +5,7 @@ ## Supply chain security: Kotlin - checksum validation ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh KOTLIN_ROOT="/usr/share" diff --git a/images/ubuntu/scripts/build/install-kubernetes-tools.sh b/images/ubuntu/scripts/build/install-kubernetes-tools.sh index 50c36bd11c46..bd7ae7c74174 100644 --- a/images/ubuntu/scripts/build/install-kubernetes-tools.sh +++ b/images/ubuntu/scripts/build/install-kubernetes-tools.sh @@ -10,17 +10,19 @@ source $HELPER_SCRIPTS/install.sh # Download KIND kind_url=$(resolve_github_release_asset_url "kubernetes-sigs/kind" "endswith(\"kind-linux-amd64\")" "latest") -curl -fsSL -o /tmp/kind "${kind_url}" +kind_binary_path=$(download_with_retry "${kind_url}") + # Supply chain security - KIND kind_external_hash=$(get_checksum_from_url "${kind_url}.sha256sum" "kind-linux-amd64" "SHA256") -use_checksum_comparison "/tmp/kind" "${kind_external_hash}" +use_checksum_comparison "${kind_binary_path}" "${kind_external_hash}" + # Install KIND -sudo install /tmp/kind /usr/local/bin/kind +install "${kind_binary_path}" /usr/local/bin/kind ## Install kubectl -KUBECTL_MINOR_VERSION=$(curl -fsSL "https://dl.k8s.io/release/stable.txt" | cut -d'.' -f1,2 ) -curl -fsSL https://pkgs.k8s.io/core:/stable:/$KUBECTL_MINOR_VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg -echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/'$KUBECTL_MINOR_VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list +kubectl_minor_version=$(curl -fsSL "https://dl.k8s.io/release/stable.txt" | cut -d'.' -f1,2 ) +curl -fsSL https://pkgs.k8s.io/core:/stable:/$kubectl_minor_version/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/'$kubectl_minor_version'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list sudo apt-get update -y && sudo apt-get install -y kubectl rm -f /etc/apt/sources.list.d/kubernetes.list @@ -29,11 +31,13 @@ curl -fsSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 # Download minikube curl -fsSL -O https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + # Supply chain security - minikube minikube_hash=$(get_checksum_from_github_release "kubernetes/minikube" "linux-amd64" "latest" "SHA256") use_checksum_comparison "minikube-linux-amd64" "${minikube_hash}" + # Install minikube -sudo install minikube-linux-amd64 /usr/local/bin/minikube +install minikube-linux-amd64 /usr/local/bin/minikube # Install kustomize download_url="https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" diff --git a/images/ubuntu/scripts/build/install-microsoft-edge.sh b/images/ubuntu/scripts/build/install-microsoft-edge.sh index 221ccc5133ac..c46d5622a7cc 100644 --- a/images/ubuntu/scripts/build/install-microsoft-edge.sh +++ b/images/ubuntu/scripts/build/install-microsoft-edge.sh @@ -9,42 +9,41 @@ source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/etc-environment.sh REPO_URL="https://packages.microsoft.com/repos/edge" -gpg_key="/usr/share/keyrings/microsoft-edge.gpg" -repo_path="/etc/apt/sources.list.d/microsoft-edge.list" +GPG_KEY="/usr/share/keyrings/microsoft-edge.gpg" +REPO_PATH="/etc/apt/sources.list.d/microsoft-edge.list" - -wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > $gpg_key +wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > $GPG_KEY # Specify an arch as Microsoft repository supports armhf and arm64 as well -echo "deb [arch=amd64 signed-by=$gpg_key] $REPO_URL stable main" > $repo_path +echo "deb [arch=amd64 signed-by=$GPG_KEY] $REPO_URL stable main" > $REPO_PATH apt-get update apt-get install --no-install-recommends microsoft-edge-stable -rm $gpg_key -rm $repo_path +rm $GPG_KEY +rm $REPO_PATH echo "microsoft-edge $REPO_URL" >> $HELPER_SCRIPTS/apt-sources.txt # Install Microsoft Edge Webdriver EDGEDRIVER_DIR="/usr/local/share/edge_driver" -EDGEDRIVER_BIN="$EDGEDRIVER_DIR/msedgedriver" +edgedriver_bin="$EDGEDRIVER_DIR/msedgedriver" mkdir -p $EDGEDRIVER_DIR -EDGE_VERSION=$(microsoft-edge --version | cut -d' ' -f 3) -EDGE_VERSION_MAJOR=$(echo $EDGE_VERSION | cut -d'.' -f 1) +edge_version=$(microsoft-edge --version | cut -d' ' -f 3) +edge_version_major=$(echo $edge_version | cut -d'.' -f 1) -EDGE_DRIVER_VERSION_URL="https://msedgedriver.azureedge.net/LATEST_RELEASE_${EDGE_VERSION_MAJOR}_LINUX" +edgedriver_version_url="https://msedgedriver.azureedge.net/LATEST_RELEASE_${edge_version_major}_LINUX" # Convert a resulting file to normal UTF-8 -EDGE_DRIVER_LATEST_VERSION=$(curl -fsSL "$EDGE_DRIVER_VERSION_URL" | iconv -f utf-16 -t utf-8 | tr -d '\r') +edgedriver_latest_version=$(curl -fsSL "$edgedriver_version_url" | iconv -f utf-16 -t utf-8 | tr -d '\r') -EDGEDRIVER_URL="https://msedgedriver.azureedge.net/${EDGE_DRIVER_LATEST_VERSION}/edgedriver_linux64.zip" -EDGEDRIVER_ARCHIVE_PATH=$(download_with_retry "$EDGEDRIVER_URL") +edgedriver_url="https://msedgedriver.azureedge.net/${edgedriver_latest_version}/edgedriver_linux64.zip" +edgedriver_archive_path=$(download_with_retry "$edgedriver_url") -unzip -qq "$EDGEDRIVER_ARCHIVE_PATH" -d "$EDGEDRIVER_DIR" -chmod +x $EDGEDRIVER_BIN -ln -s $EDGEDRIVER_BIN /usr/bin +unzip -qq "$edgedriver_archive_path" -d "$EDGEDRIVER_DIR" +chmod +x $edgedriver_bin +ln -s $edgedriver_bin /usr/bin set_etc_environment_variable "EDGEWEBDRIVER" "${EDGEDRIVER_DIR}" diff --git a/images/ubuntu/scripts/build/install-mongodb.sh b/images/ubuntu/scripts/build/install-mongodb.sh index 7a47185347ac..266485926208 100644 --- a/images/ubuntu/scripts/build/install-mongodb.sh +++ b/images/ubuntu/scripts/build/install-mongodb.sh @@ -5,21 +5,20 @@ ################################################################################ # Source the helpers -source $HELPER_SCRIPTS/os.sh source $HELPER_SCRIPTS/install.sh REPO_URL="https://repo.mongodb.org/apt/ubuntu" -osLabel=$(get_os_version_label) -toolsetVersion=$(get_toolset_value '.mongodb.version') +os_label=$(lsb_release -cs) +toolset_version=$(get_toolset_value '.mongodb.version') # Install Mongo DB -wget -qO - https://www.mongodb.org/static/pgp/server-$toolsetVersion.asc | sudo apt-key add - +wget -qO - https://www.mongodb.org/static/pgp/server-$toolset_version.asc | sudo apt-key add - -echo "deb [ arch=amd64,arm64 ] $REPO_URL $osLabel/mongodb-org/$toolsetVersion multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-$toolsetVersion.list +echo "deb [ arch=amd64,arm64 ] $REPO_URL $os_label/mongodb-org/$toolset_version multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-$toolset_version.list sudo apt-get update sudo apt-get install -y mongodb-org -rm /etc/apt/sources.list.d/mongodb-org-$toolsetVersion.list +rm /etc/apt/sources.list.d/mongodb-org-$toolset_version.list echo "mongodb $REPO_URL" >> $HELPER_SCRIPTS/apt-sources.txt diff --git a/images/ubuntu/scripts/build/install-mono.sh b/images/ubuntu/scripts/build/install-mono.sh index e8be1f9c6a71..3abf40b14f41 100644 --- a/images/ubuntu/scripts/build/install-mono.sh +++ b/images/ubuntu/scripts/build/install-mono.sh @@ -4,25 +4,27 @@ ## Desc: Install Mono ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/os.sh -LSB_CODENAME=$(lsb_release -cs) +os_label=$(lsb_release -cs) # There are no packages for Ubuntu 22 in the repo, but developers confirmed that packages from Ubuntu 20 should work if is_ubuntu22; then - LSB_CODENAME="focal" + os_label="focal" fi # Test to see if the software in question is already installed, if not install it # wget "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF" -O out && sudo apt-key add out && rm out apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb https://download.mono-project.com/repo/ubuntu stable-$LSB_CODENAME main" | tee /etc/apt/sources.list.d/mono-official-stable.list +echo "deb https://download.mono-project.com/repo/ubuntu stable-$os_label main" | tee /etc/apt/sources.list.d/mono-official-stable.list apt-get update apt-get install -y --no-install-recommends apt-transport-https mono-complete nuget rm /etc/apt/sources.list.d/mono-official-stable.list rm -f /etc/apt/sources.list.d/mono-official-stable.list.save -echo "mono https://download.mono-project.com/repo/ubuntu stable-$LSB_CODENAME main" >> $HELPER_SCRIPTS/apt-sources.txt + +echo "mono https://download.mono-project.com/repo/ubuntu stable-$os_label main" >> $HELPER_SCRIPTS/apt-sources.txt invoke_tests "Tools" "Mono" diff --git a/images/ubuntu/scripts/build/install-ms-repos.sh b/images/ubuntu/scripts/build/install-ms-repos.sh index 26eb8f475852..09f04d7b808b 100644 --- a/images/ubuntu/scripts/build/install-ms-repos.sh +++ b/images/ubuntu/scripts/build/install-ms-repos.sh @@ -4,10 +4,10 @@ ## Desc: Install official Microsoft package repos for the distribution ################################################################################ -LSB_RELEASE=$(lsb_release -rs) +os_label=$(lsb_release -rs) # Install Microsoft repository -wget https://packages.microsoft.com/config/ubuntu/$LSB_RELEASE/packages-microsoft-prod.deb +wget https://packages.microsoft.com/config/ubuntu/$os_label/packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb # update diff --git a/images/ubuntu/scripts/build/install-mysql.sh b/images/ubuntu/scripts/build/install-mysql.sh index 9ff05155958a..d01f83daad8d 100644 --- a/images/ubuntu/scripts/build/install-mysql.sh +++ b/images/ubuntu/scripts/build/install-mysql.sh @@ -4,6 +4,7 @@ ## Desc: Install MySQL Client ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/os.sh # Mysql setting up root password diff --git a/images/ubuntu/scripts/build/install-nodejs.sh b/images/ubuntu/scripts/build/install-nodejs.sh index dd96adc85d6f..9cd007398eab 100644 --- a/images/ubuntu/scripts/build/install-nodejs.sh +++ b/images/ubuntu/scripts/build/install-nodejs.sh @@ -8,9 +8,9 @@ source $HELPER_SCRIPTS/install.sh # Install default Node.js -defaultVersion=$(get_toolset_value '.node.default') +default_version=$(get_toolset_value '.node.default') curl -fsSL https://raw.githubusercontent.com/tj/n/master/bin/n -o ~/n -bash ~/n $defaultVersion +bash ~/n $default_version # Install node modules node_modules=$(get_toolset_value '.node_modules[].name') diff --git a/images/ubuntu/scripts/build/install-nvm.sh b/images/ubuntu/scripts/build/install-nvm.sh index c63aaab3e9c6..def0d5dc3f63 100644 --- a/images/ubuntu/scripts/build/install-nvm.sh +++ b/images/ubuntu/scripts/build/install-nvm.sh @@ -9,9 +9,10 @@ source $HELPER_SCRIPTS/etc-environment.sh export NVM_DIR="/etc/skel/.nvm" mkdir $NVM_DIR -VERSION=$(curl -fsSL https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') -curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/$VERSION/install.sh | bash +nvm_version=$(curl -fsSL https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') +curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/$nvm_version/install.sh | bash set_etc_environment_variable "NVM_DIR" '$HOME/.nvm' + echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' | tee -a /etc/skel/.bash_profile [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" diff --git a/images/ubuntu/scripts/build/install-oc-cli.sh b/images/ubuntu/scripts/build/install-oc-cli.sh index f1094ff6a35c..8f8a06de49c6 100644 --- a/images/ubuntu/scripts/build/install-oc-cli.sh +++ b/images/ubuntu/scripts/build/install-oc-cli.sh @@ -4,11 +4,11 @@ ## Desc: Install the OC CLI ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Install the oc CLI -download_url="https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz" -archive_path=$(download_with_retry "$download_url") +archive_path=$(download_with_retry "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz") tar xzf "$archive_path" -C "/usr/local/bin" oc invoke_tests "CLI.Tools" "OC CLI" diff --git a/images/ubuntu/scripts/build/install-oras-cli.sh b/images/ubuntu/scripts/build/install-oras-cli.sh index cf01267056a2..119bea0b2511 100644 --- a/images/ubuntu/scripts/build/install-oras-cli.sh +++ b/images/ubuntu/scripts/build/install-oras-cli.sh @@ -5,6 +5,7 @@ ## Supply chain security: ORAS CLI - checksum validation ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Determine latest ORAS CLI version diff --git a/images/ubuntu/scripts/build/install-packer.sh b/images/ubuntu/scripts/build/install-packer.sh index 5fb0bd8b50e8..33d284869a14 100644 --- a/images/ubuntu/scripts/build/install-packer.sh +++ b/images/ubuntu/scripts/build/install-packer.sh @@ -4,6 +4,7 @@ ## Desc: Install packer ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # Install Packer diff --git a/images/ubuntu/scripts/build/install-phantomjs.sh b/images/ubuntu/scripts/build/install-phantomjs.sh index 2ed14a04800d..1eec7a1b1a75 100644 --- a/images/ubuntu/scripts/build/install-phantomjs.sh +++ b/images/ubuntu/scripts/build/install-phantomjs.sh @@ -11,16 +11,16 @@ source $HELPER_SCRIPTS/install.sh apt-get install -y chrpath libssl-dev libxft-dev libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev # Define the version and hash of PhantomJS to be installed -dir_name=phantomjs-2.1.1-linux-x86_64 -archive_hash="86dd9a4bf4aee45f1a84c9f61cf1947c1d6dce9b9e8d2a907105da7852460d2f" +DIR_NAME=phantomjs-2.1.1-linux-x86_64 +ARCHIVE_HASH="86dd9a4bf4aee45f1a84c9f61cf1947c1d6dce9b9e8d2a907105da7852460d2f" # Download the archive and verify its integrity using checksum comparison -download_url="https://bitbucket.org/ariya/phantomjs/downloads/$dir_name.tar.bz2" +download_url="https://bitbucket.org/ariya/phantomjs/downloads/$DIR_NAME.tar.bz2" archive_path=$(download_with_retry "$download_url") -use_checksum_comparison "$archive_path" "$archive_hash" +use_checksum_comparison "$archive_path" "$ARCHIVE_HASH" # Extract the archive and create a symbolic link to the executable tar xjf "$archive_path" -C /usr/local/share -ln -sf /usr/local/share/$dir_name/bin/phantomjs /usr/local/bin +ln -sf /usr/local/share/$DIR_NAME/bin/phantomjs /usr/local/bin invoke_tests "Tools" "Phantomjs" diff --git a/images/ubuntu/scripts/build/install-php.sh b/images/ubuntu/scripts/build/install-php.sh index 81409da4694b..d03494b50fb5 100644 --- a/images/ubuntu/scripts/build/install-php.sh +++ b/images/ubuntu/scripts/build/install-php.sh @@ -101,14 +101,13 @@ mkdir -p /etc/skel/.composer # Install phpunit (for PHP) wget -q -O phpunit https://phar.phpunit.de/phpunit-8.phar -chmod +x phpunit -mv phpunit /usr/local/bin/phpunit +install phpunit /usr/local/bin/phpunit # ubuntu 20.04 libzip-dev is libzip5 based and is not compatible libzip-dev of ppa:ondrej/php # see https://github.com/actions/runner-images/issues/1084 if is_ubuntu20; then - rm /etc/apt/sources.list.d/ondrej-*.list - apt-get update + rm /etc/apt/sources.list.d/ondrej-*.list + apt-get update fi invoke_tests "Common" "PHP" diff --git a/images/ubuntu/scripts/build/install-pipx-packages.sh b/images/ubuntu/scripts/build/install-pipx-packages.sh index 14262d128863..40356b6c3f77 100644 --- a/images/ubuntu/scripts/build/install-pipx-packages.sh +++ b/images/ubuntu/scripts/build/install-pipx-packages.sh @@ -3,6 +3,8 @@ ## File: install-pipx-packages.sh ## Desc: Install tools via pipx ################################################################################ + +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh export PATH="$PATH:/opt/pipx_bin" @@ -10,22 +12,14 @@ export PATH="$PATH:/opt/pipx_bin" pipx_packages=$(get_toolset_value ".pipx[] .package") for package in $pipx_packages; do - python_version=$(get_toolset_value ".pipx[] | select(.package == \"$package\") .python") - if [ "$python_version" != "null" ]; then - python_path="/opt/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version" - echo "Install $package into python $python_path" - pipx install $package --python $python_path - else - echo "Install $package into default python" - pipx install $package + echo "Install $package into default python" + pipx install $package - # https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html - # Install ansible into an existing ansible-core Virtual Environment - if [[ $package == "ansible-core" ]]; then - pipx inject $package ansible - fi + # https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html + # Install ansible into an existing ansible-core Virtual Environment + if [[ $package == "ansible-core" ]]; then + pipx inject $package ansible fi - done invoke_tests "Common" "PipxPackages" diff --git a/images/ubuntu/scripts/build/install-postgresql.sh b/images/ubuntu/scripts/build/install-postgresql.sh index 18b5bc037052..8285f610a133 100644 --- a/images/ubuntu/scripts/build/install-postgresql.sh +++ b/images/ubuntu/scripts/build/install-postgresql.sh @@ -12,15 +12,15 @@ REPO_URL="https://apt.postgresql.org/pub/repos/apt/" # Preparing repo for PostgreSQL wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /usr/share/keyrings/postgresql.gpg -echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] $REPO_URL $(get_os_version_label)-pgdg main" > /etc/apt/sources.list.d/pgdg.list +echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] $REPO_URL $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list # Fetch PostgreSQL version to install from the toolset -toolsetVersion=$(get_toolset_value '.postgresql.version') +toolset_version=$(get_toolset_value '.postgresql.version') # Install PostgreSQL echo "Install PostgreSQL" apt update -apt install postgresql-$toolsetVersion +apt install postgresql-$toolset_version echo "Install libpq-dev" apt-get install libpq-dev diff --git a/images/ubuntu/scripts/build/install-powershell.sh b/images/ubuntu/scripts/build/install-powershell.sh index b2a0069bcbaf..f1345375d1be 100644 --- a/images/ubuntu/scripts/build/install-powershell.sh +++ b/images/ubuntu/scripts/build/install-powershell.sh @@ -4,9 +4,10 @@ ## Desc: Install PowerShell Core ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -pwshversion=$(get_toolset_value .pwsh.version) +pwsh_version=$(get_toolset_value .pwsh.version) # Install Powershell -apt-get install -y powershell=$pwshversion* +apt-get install -y powershell=$pwsh_version* diff --git a/images/ubuntu/scripts/build/install-pypy.sh b/images/ubuntu/scripts/build/install-pypy.sh index 47dba4496e18..61808c14af8e 100644 --- a/images/ubuntu/scripts/build/install-pypy.sh +++ b/images/ubuntu/scripts/build/install-pypy.sh @@ -4,6 +4,7 @@ ## Desc: Install PyPy ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh # This function installs PyPy using the specified arguments: @@ -73,20 +74,20 @@ install_pypy() { } # Installation PyPy -pypyVersions=$(curl -fsSL https://downloads.python.org/pypy/versions.json) -toolsetVersions=$(get_toolset_value '.toolcache[] | select(.name | contains("PyPy")) | .versions[]') +pypy_versions_json=$(curl -fsSL https://downloads.python.org/pypy/versions.json) +toolset_versions=$(get_toolset_value '.toolcache[] | select(.name | contains("PyPy")) | .versions[]') -for toolsetVersion in $toolsetVersions; do - latestMajorPyPyVersion=$(echo $pypyVersions | - jq -r --arg toolsetVersion $toolsetVersion '.[] - | select((.python_version | startswith($toolsetVersion)) and .stable == true).files[] +for toolset_version in $toolset_versions; do + latest_major_pypy_version=$(echo $pypy_versions_json | + jq -r --arg toolset_version $toolset_version '.[] + | select((.python_version | startswith($toolset_version)) and .stable == true).files[] | select(.arch == "x64" and .platform == "linux").download_url' | head -1) - if [[ -z "$latestMajorPyPyVersion" ]]; then - echo "Failed to get PyPy version '$toolsetVersion'" + if [[ -z "$latest_major_pypy_version" ]]; then + echo "Failed to get PyPy version '$toolset_version'" exit 1 fi - install_pypy $latestMajorPyPyVersion + install_pypy $latest_major_pypy_version done chown -R "$SUDO_USER:$SUDO_USER" "$AGENT_TOOLSDIRECTORY/PyPy" diff --git a/images/ubuntu/scripts/build/install-python.sh b/images/ubuntu/scripts/build/install-python.sh index 846aa4206289..f96cfac97548 100644 --- a/images/ubuntu/scripts/build/install-python.sh +++ b/images/ubuntu/scripts/build/install-python.sh @@ -18,10 +18,12 @@ export PIPX_BIN_DIR=/opt/pipx_bin export PIPX_HOME=/opt/pipx python3 -m pip install pipx python3 -m pipx ensurepath + # Update /etc/environment set_etc_environment_variable "PIPX_BIN_DIR" $PIPX_BIN_DIR set_etc_environment_variable "PIPX_HOME" $PIPX_HOME prepend_etc_environment_path $PIPX_BIN_DIR + # Test pipx if ! command -v pipx; then echo "pipx was not installed or not found on PATH" diff --git a/images/ubuntu/scripts/build/install-rlang.sh b/images/ubuntu/scripts/build/install-rlang.sh index 7445d7de507e..42871cae9020 100644 --- a/images/ubuntu/scripts/build/install-rlang.sh +++ b/images/ubuntu/scripts/build/install-rlang.sh @@ -4,14 +4,11 @@ ## Desc: Install R ################################################################################ -# Source the helpers for use with the script -source $HELPER_SCRIPTS/os.sh - # install R -osLabel=$(get_os_version_label) +os_label=$(lsb_release -cs) wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | gpg --dearmor > /usr/share/keyrings/rlang.gpg -echo "deb [signed-by=/usr/share/keyrings/rlang.gpg] https://cloud.r-project.org/bin/linux/ubuntu $osLabel-cran40/" > /etc/apt/sources.list.d/rlang.list +echo "deb [signed-by=/usr/share/keyrings/rlang.gpg] https://cloud.r-project.org/bin/linux/ubuntu $os_label-cran40/" > /etc/apt/sources.list.d/rlang.list apt-get update apt-get install r-base diff --git a/images/ubuntu/scripts/build/install-ruby.sh b/images/ubuntu/scripts/build/install-ruby.sh index 5241a73c4cde..d3cd3f4e4894 100644 --- a/images/ubuntu/scripts/build/install-ruby.sh +++ b/images/ubuntu/scripts/build/install-ruby.sh @@ -4,15 +4,16 @@ ## Desc: Install Ruby requirements and ruby gems ################################################################################ +# Source the helpers for use with the script source $HELPER_SCRIPTS/os.sh source $HELPER_SCRIPTS/install.sh apt-get install ruby-full # Install ruby gems from toolset -gemsToInstall=$(get_toolset_value ".rubygems[] .name") -if [ -n "$gemsToInstall" ]; then - for gem in $gemsToInstall; do +gems_to_install=$(get_toolset_value ".rubygems[] .name") +if [[ -n "$gems_to_install" ]]; then + for gem in $gems_to_install; do echo "Installing gem $gem" gem install $gem done @@ -22,35 +23,35 @@ fi apt-get install -y libz-dev openssl libssl-dev echo "Install Ruby from toolset..." -PACKAGE_TAR_NAMES=$(curl -fsSL "https://api.github.com/repos/ruby/ruby-builder/releases/latest" | jq -r '.assets[].name') -TOOLSET_VERSIONS=$(get_toolset_value '.toolcache[] | select(.name | contains("Ruby")) | .versions[]') -PLATFORM_VERSION=$(get_toolset_value '.toolcache[] | select(.name | contains("Ruby")) | .platform_version') -RUBY_PATH="$AGENT_TOOLSDIRECTORY/Ruby" +package_tar_names=$(curl -fsSL "https://api.github.com/repos/ruby/ruby-builder/releases/latest" | jq -r '.assets[].name') +toolset_versions=$(get_toolset_value '.toolcache[] | select(.name | contains("Ruby")) | .versions[]') +platform_version=$(get_toolset_value '.toolcache[] | select(.name | contains("Ruby")) | .platform_version') +ruby_path="$AGENT_TOOLSDIRECTORY/Ruby" echo "Check if Ruby hostedtoolcache folder exist..." -if [ ! -d $RUBY_PATH ]; then - mkdir -p $RUBY_PATH +if [[ ! -d $ruby_path ]]; then + mkdir -p $ruby_path fi -for TOOLSET_VERSION in ${TOOLSET_VERSIONS[@]}; do - PACKAGE_TAR_NAME=$(echo "$PACKAGE_TAR_NAMES" | grep "^ruby-${TOOLSET_VERSION}-ubuntu-${PLATFORM_VERSION}.tar.gz$" | sort -V | tail -1) - RUBY_VERSION=$(echo "$PACKAGE_TAR_NAME" | cut -d'-' -f 2) - RUBY_VERSION_PATH="$RUBY_PATH/$RUBY_VERSION" +for toolset_version in ${toolset_versions[@]}; do + package_tar_name=$(echo "$package_tar_names" | grep "^ruby-${toolset_version}-ubuntu-${platform_version}.tar.gz$" | sort -V | tail -1) + ruby_version=$(echo "$package_tar_name" | cut -d'-' -f 2) + ruby_version_path="$ruby_path/$ruby_version" - echo "Create Ruby $RUBY_VERSION directory..." - mkdir -p $RUBY_VERSION_PATH + echo "Create Ruby $ruby_version directory..." + mkdir -p $ruby_version_path - echo "Downloading tar archive $PACKAGE_TAR_NAME" - DOWNLOAD_URL="https://github.com/ruby/ruby-builder/releases/download/toolcache/${PACKAGE_TAR_NAME}" - PACKAGE_ARCHIVE_PATH=$(download_with_retry "$DOWNLOAD_URL") + echo "Downloading tar archive $package_tar_name" + download_url="https://github.com/ruby/ruby-builder/releases/download/toolcache/${package_tar_name}" + package_archive_path=$(download_with_retry "$download_url") - echo "Expand '$PACKAGE_TAR_NAME' to the '$RUBY_VERSION_PATH' folder" - tar xf "$PACKAGE_ARCHIVE_PATH" -C $RUBY_VERSION_PATH + echo "Expand '$package_tar_name' to the '$ruby_version_path' folder" + tar xf "$package_archive_path" -C $ruby_version_path - COMPLETE_FILE_PATH="$RUBY_VERSION_PATH/x64.complete" - if [ ! -f $COMPLETE_FILE_PATH ]; then + complete_file_path="$ruby_version_path/x64.complete" + if [[ ! -f $complete_file_path ]]; then echo "Create complete file" - touch $COMPLETE_FILE_PATH + touch $complete_file_path fi done diff --git a/images/ubuntu/scripts/build/install-selenium.sh b/images/ubuntu/scripts/build/install-selenium.sh index 3b9360762c19..270c75f93673 100644 --- a/images/ubuntu/scripts/build/install-selenium.sh +++ b/images/ubuntu/scripts/build/install-selenium.sh @@ -8,17 +8,17 @@ source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/etc-environment.sh -SELENIUM_MAJOR_VERSION=$(get_toolset_value '.selenium.version') +selenium_major_version=$(get_toolset_value '.selenium.version') # Download Selenium server -SELENIUM_DOWNLOAD_URL=$(resolve_github_release_asset_url "SeleniumHQ/selenium" "contains(\"selenium-server-\") and endswith(\".jar\")" "$SELENIUM_MAJOR_VERSION\.+" "" "true") -SELENIUM_JAR_PATH=$(download_with_retry "$SELENIUM_DOWNLOAD_URL" "/usr/share/java/selenium-server.jar") +selenium_download_url=$(resolve_github_release_asset_url "SeleniumHQ/selenium" "contains(\"selenium-server-\") and endswith(\".jar\")" "$selenium_major_version\.+" "" "true") +selenium_jar_path=$(download_with_retry "$selenium_download_url" "/usr/share/java/selenium-server.jar") # Create an epmty file to retrive selenium version -SELENIUM_FULL_VERSION=$(echo $SELENIUM_DOWNLOAD_URL | awk -F"selenium-server-|.jar" '{print $2}') -touch "/usr/share/java/selenium-server-$SELENIUM_FULL_VERSION" +selenium_full_version=$(echo $selenium_download_url | awk -F"selenium-server-|.jar" '{print $2}') +touch "/usr/share/java/selenium-server-$selenium_full_version" # Add SELENIUM_JAR_PATH environment variable -set_etc_environment_variable "SELENIUM_JAR_PATH" "$SELENIUM_JAR_PATH" +set_etc_environment_variable "SELENIUM_JAR_PATH" "$selenium_jar_path" invoke_tests "Tools" "Selenium" diff --git a/images/ubuntu/scripts/build/install-sqlpackage.sh b/images/ubuntu/scripts/build/install-sqlpackage.sh index d8dcb89ed1f9..c002a68d538d 100644 --- a/images/ubuntu/scripts/build/install-sqlpackage.sh +++ b/images/ubuntu/scripts/build/install-sqlpackage.sh @@ -13,6 +13,7 @@ if is_ubuntu22; then libssl_deb_path=$(download_with_retry "http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb") libssl_hash="0b3251aee55db6e20d02f4b9a2b703c9874a85ab6a20b12f4870f52f91633d37" use_checksum_comparison "$libssl_deb_path" "$libssl_hash" + dpkg -i "$libssl_deb_path" fi diff --git a/images/ubuntu/scripts/build/install-swift.sh b/images/ubuntu/scripts/build/install-swift.sh index 43e3f2b16ee7..06b4860d58ee 100644 --- a/images/ubuntu/scripts/build/install-swift.sh +++ b/images/ubuntu/scripts/build/install-swift.sh @@ -32,16 +32,16 @@ rm ~/.gnupg/swift tar xzf "$archive_path" -C /tmp SWIFT_INSTALL_ROOT="/usr/share/swift" -SWIFT_BIN_ROOT="$SWIFT_INSTALL_ROOT/usr/bin" -SWIFT_LIB_ROOT="$SWIFT_INSTALL_ROOT/usr/lib" +swift_bin_root="$SWIFT_INSTALL_ROOT/usr/bin" +swift_lib_root="$SWIFT_INSTALL_ROOT/usr/lib" mv "/tmp/${swift_release_name}" $SWIFT_INSTALL_ROOT mkdir -p /usr/local/lib -ln -s "$SWIFT_BIN_ROOT/swift" /usr/local/bin/swift -ln -s "$SWIFT_BIN_ROOT/swiftc" /usr/local/bin/swiftc -ln -s "$SWIFT_LIB_ROOT/libsourcekitdInProc.so" /usr/local/lib/libsourcekitdInProc.so +ln -s "$swift_bin_root/swift" /usr/local/bin/swift +ln -s "$swift_bin_root/swiftc" /usr/local/bin/swiftc +ln -s "$swift_lib_root/libsourcekitdInProc.so" /usr/local/lib/libsourcekitdInProc.so -set_etc_environment_variable "SWIFT_PATH" "${SWIFT_BIN_ROOT}" +set_etc_environment_variable "SWIFT_PATH" "${swift_bin_root}" invoke_tests "Common" "Swift" diff --git a/images/ubuntu/scripts/build/install-yq.sh b/images/ubuntu/scripts/build/install-yq.sh index 07a192651212..9caf52a56802 100644 --- a/images/ubuntu/scripts/build/install-yq.sh +++ b/images/ubuntu/scripts/build/install-yq.sh @@ -9,14 +9,15 @@ source $HELPER_SCRIPTS/install.sh # Download yq -base_url="https://github.com/mikefarah/yq/releases/latest/download" -binary_path=$(download_with_retry "${base_url}/yq_linux_amd64") +yq_url=$(resolve_github_release_asset_url "mikefarah/yq" "endswith(\"yq_linux_amd64\")" "latest") +binary_path=$(download_with_retry "${yq_url}") # Supply chain security - yq -external_hash=$(get_checksum_from_url "${base_url}/checksums" "yq_linux_amd64 " "SHA256" "true" " " "19") +hash_url=$(resolve_github_release_asset_url "mikefarah/yq" "endswith(\"checksums\")" "latest") +external_hash=$(get_checksum_from_url "${hash_url}" "yq_linux_amd64 " "SHA256" "true" " " "19") use_checksum_comparison "$binary_path" "$external_hash" # Install yq -sudo install "$binary_path" /usr/bin/yq +install "$binary_path" /usr/bin/yq invoke_tests "Tools" "yq" diff --git a/images/ubuntu/scripts/build/install-zstd.sh b/images/ubuntu/scripts/build/install-zstd.sh index 84d5932ddb99..a980ad8917e4 100644 --- a/images/ubuntu/scripts/build/install-zstd.sh +++ b/images/ubuntu/scripts/build/install-zstd.sh @@ -26,13 +26,13 @@ make -C "/tmp/${release_name}/contrib/pzstd" all make -C "/tmp/${release_name}" zstd-release for copyprocess in zstd zstdless zstdgrep; do - cp "/tmp/${release_name}/programs/${copyprocess}" /usr/local/bin/ + cp "/tmp/${release_name}/programs/${copyprocess}" /usr/local/bin/ done cp "/tmp/${release_name}/contrib/pzstd/pzstd" /usr/local/bin/ for symlink in zstdcat zstdmt unzstd; do - ln -sf /usr/local/bin/zstd /usr/local/bin/${symlink} + ln -sf /usr/local/bin/zstd /usr/local/bin/${symlink} done invoke_tests "Tools" "Zstd" diff --git a/images/ubuntu/scripts/helpers/os.sh b/images/ubuntu/scripts/helpers/os.sh index c500ab582e83..09a2213c2f70 100644 --- a/images/ubuntu/scripts/helpers/os.sh +++ b/images/ubuntu/scripts/helpers/os.sh @@ -11,7 +11,3 @@ is_ubuntu20() { is_ubuntu22() { lsb_release -d | grep -q 'Ubuntu 22' } - -get_os_version_label() { - lsb_release -cs -}