Improve support for linux/arm64/v8 on most Ubuntu-based images (par…

…t 2) (#1630)

This is a follow-up to #1597. Docker images for Arm64-based PCs has been
added.

.changes/1630.json

@@ -0,0 +1,4 @@
+{
+  "type": "added",
+  "description": "Add docker platform support for `linux/arm64/v8` target for many Ubuntu-based targets"
+}

docker/apt-cross-essential.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -x
+set -euo pipefail
+
+# shellcheck disable=SC1091
+. lib.sh
+
+main() {
+    local narch
+    local -a packages
+
+    narch="$(dpkg --print-architecture)"
+    packages+=("libc6-dev-${TARGET_ARCH}-cross:${narch}")
+
+    # Install crossbuild-essential if CROSSBUILD_ESSENTIAL is set
+    if [ -n "${CROSSBUILD_ESSENTIAL:-}" ]; then
+        packages+=("crossbuild-essential-${TARGET_ARCH}:${narch}")
+    fi
+
+    if ! command -v "${CROSS_TOOLCHAIN_PREFIX}g++" &>/dev/null; then
+      packages+=("g++-${TARGET_TRIPLE}:${narch}")
+    fi
+
+    if ! command -v "${CROSS_TOOLCHAIN_PREFIX}gfortran" &>/dev/null; then
+      packages+=("gfortran-${TARGET_TRIPLE}:${narch}")
+    fi
+
+    install_packages "${packages[@]}"
+
+    rm "${0}"
+}
+
+main "${@}"

docker/cmake.sh

@@ -13,11 +13,33 @@ main() {
 
     local td
     td="$(mktemp -d)"
-
     pushd "${td}"
 
-    curl --retry 3 -sSfL "https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-linux-x86_64.sh" -o cmake.sh
+    local cmake_arch
+    local cmake_sha256
+
+    local narch
+    narch="$(dpkg --print-architecture)"
+
+    case "${narch}" in
+        amd64)
+            cmake_arch="linux-x86_64"
+            cmake_sha256="da2a9b18c3bfb136917fa1a579aa5316b01c1d6c111043d03f18877ff05bda30"
+            ;;
+        arm64)
+            cmake_arch="linux-aarch64"
+            cmake_sha256="86122bdfd030208aa36705ef421a218ccec52a14368020b2d67043af5e45490b"
+            ;;
+        *)
+            echo "Unsupported architecture: ${narch}" 1>&2
+            exit 1
+            ;;
+     esac
+
+    curl --retry 3 -sSfL "https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${cmake_arch}.sh" -o cmake.sh
+    sha256sum --check <<<"${cmake_sha256}  cmake.sh"
     sh cmake.sh --skip-license --prefix=/usr/local
+    cmake --version
 
     popd
 

docker/common.sh

@@ -6,12 +6,19 @@ set -euo pipefail
 # shellcheck disable=SC1091
 . lib.sh
 
-# For architectures except amd64 and i386, look for packages on ports.ubuntu.com instead.
+# For non-native architectures, look for packages on ports.ubuntu.com instead.
 # This is important if you enable additional architectures so you can install libraries to cross-compile against.
 # Look for 'dpkg --add-architecture' in the README for more details.
 if grep -i ubuntu /etc/os-release >/dev/null; then
-    sed 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch-=amd64,i386] http:\/\/ports.ubuntu.com\/ubuntu-ports\//g' /etc/apt/sources.list > /etc/apt/sources.list.d/ports.list
-    sed -i 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch=amd64,i386] http:\/\/\1.archive.ubuntu.com\/ubuntu\//g' /etc/apt/sources.list
+    NATIVE_ARCH=$(dpkg --print-architecture)
+
+    if [ "$NATIVE_ARCH" = "amd64" ]; then
+        sed 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch-=amd64,i386] http:\/\/ports.ubuntu.com\/ubuntu-ports\//g' /etc/apt/sources.list > /etc/apt/sources.list.d/ports.list
+        sed -i 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch=amd64,i386] http:\/\/\1.archive.ubuntu.com\/ubuntu\//g' /etc/apt/sources.list
+    else
+        sed -i "s/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch-=${NATIVE_ARCH}] http:\/\/ports.ubuntu.com\/ubuntu-ports\//g" /etc/apt/sources.list
+        sed -i "s/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch=${NATIVE_ARCH}] http:\/\/\1.archive.ubuntu.com\/ubuntu\//g" /etc/apt/sources.list
+    fi
 fi
 
 install_packages \

docker/linux-image.sh

@@ -198,10 +198,10 @@ main() {
     dpkg --add-architecture "${arch}" || echo "foreign-architecture ${arch}" >/etc/dpkg/dpkg.cfg.d/multiarch
 
     # Add Debian keys.
-    curl --retry 3 -sSfL 'https://ftp-master.debian.org/keys/archive-key-{7.0,8,9,10,11}.asc' -O
-    curl --retry 3 -sSfL 'https://ftp-master.debian.org/keys/archive-key-{8,9,10,11}-security.asc' -O
-    curl --retry 3 -sSfL 'https://ftp-master.debian.org/keys/release-{7,8,9,10,11}.asc' -O
-    curl --retry 3 -sSfL 'https://www.ports.debian.org/archive_{2020,2021,2022,2023,2024}.key' -O
+    curl --retry 3 -sSfL 'https://ftp-master.debian.org/keys/archive-key-{7.0,8,9,10,11,12}.asc' -O
+    curl --retry 3 -sSfL 'https://ftp-master.debian.org/keys/archive-key-{8,9,10,11,12}-security.asc' -O
+    curl --retry 3 -sSfL 'https://ftp-master.debian.org/keys/release-{7,8,9,10,11,12}.asc' -O
+    curl --retry 3 -sSfL 'https://www.ports.debian.org/archive_{2020,2021,2022,2023,2024,2025}.key' -O
 
     for key in *.asc *.key; do
         apt-key add "${key}"

docker/qemu.sh

@@ -19,7 +19,7 @@ build_static_libffi () {
     tar --strip-components=1 -xzf "v${version}.tar.gz"
     ./configure --prefix="$td"/lib --disable-builddir --disable-shared --enable-static
     make "-j$(nproc)"
-    install -m 644 ./.libs/libffi.a /usr/lib64/
+    install -m 644 ./.libs/libffi.a /usr/local/lib/
 
     popd
 
@@ -42,7 +42,7 @@ build_static_libmount () {
     tar --strip-components=1 -xJf "util-linux-${version_spec}.tar.xz"
     ./configure --disable-shared --enable-static --without-ncurses
     make "-j$(nproc)" mount blkid
-    install -m 644 ./.libs/*.a /usr/lib64/
+    install -m 644 ./.libs/*.a /usr/local/lib/
 
     popd
 
@@ -67,7 +67,7 @@ build_static_libattr() {
 
     ./configure
     make "-j$(nproc)"
-    install -m 644 ./libattr/.libs/libattr.a /usr/lib64/
+    install -m 644 ./libattr/.libs/libattr.a /usr/local/lib/
 
     yum remove -y gettext
 
@@ -87,7 +87,7 @@ build_static_libcap() {
     curl --retry 3 -sSfL "https://www.kernel.org/pub/linux/libs/security/linux-privs/libcap2/libcap-${version}.tar.xz" -O
     tar --strip-components=1 -xJf "libcap-${version}.tar.xz"
     make "-j$(nproc)"
-    install -m 644 libcap/libcap.a /usr/lib64/
+    install -m 644 libcap/libcap.a /usr/local/lib/
 
     popd
 
@@ -106,7 +106,7 @@ build_static_pixman() {
     tar --strip-components=1 -xzf "pixman-${version}.tar.gz"
     ./configure
     make "-j$(nproc)"
-    install -m 644 ./pixman/.libs/libpixman-1.a /usr/lib64/
+    install -m 644 ./pixman/.libs/libpixman-1.a /usr/local/lib/
 
     popd
 
@@ -125,7 +125,7 @@ build_static_slirp() {
     tar -xzf "libslirp-v${version}.tar.gz"
     meson setup -Ddefault_library=static libslirp-v${version} build
     ninja -C build
-    install -m 644 ./build/libslirp.a /usr/lib64/
+    install -m 644 ./build/libslirp.a /usr/local/lib/
 
     popd
 

src/docker/image.rs

@@ -191,6 +191,17 @@ impl ImagePlatform {
             format!("{}/{}", self.os, self.architecture)
         }
     }
+
+    /// Returns a string that can be used in codegen to represent this platform
+    pub fn to_codegen_string(&self) -> Option<&'static str> {
+        match self.target {
+            TargetTriple::X86_64UnknownLinuxGnu => Some("ImagePlatform::X86_64_UNKNOWN_LINUX_GNU"),
+            TargetTriple::Aarch64UnknownLinuxGnu => {
+                Some("ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU")
+            }
+            _ => None,
+        }
+    }
 }
 
 impl Default for ImagePlatform {
@@ -221,6 +232,14 @@ impl std::str::FromStr for ImagePlatform {
             value::{Error as SerdeError, StrDeserializer},
             IntoDeserializer,
         };
+
+        // Try to match the docker platform string first
+        match s {
+            "linux/amd64" => return Ok(Self::X86_64_UNKNOWN_LINUX_GNU),
+            "linux/arm64" | "linux/arm64/v8" => return Ok(Self::AARCH64_UNKNOWN_LINUX_GNU),
+            _ => {}
+        };
+
         if let Some((platform, toolchain)) = s.split_once('=') {
             let image_toolchain = toolchain.into();
             let (os, arch, variant) = if let Some((os, rest)) = platform.split_once('/') {
@@ -505,4 +524,24 @@ pub mod tests {
         assert_eq!(Os::from_target(&t!("x86_64-pc-windows-msvc"))?, Os::Windows);
         Ok(())
     }
+
+    #[test]
+    fn image_platform_from_docker_platform_str() -> Result<()> {
+        assert_eq!(
+            "linux/amd64".parse::<ImagePlatform>()?,
+            ImagePlatform::X86_64_UNKNOWN_LINUX_GNU
+        );
+
+        assert_eq!(
+            "linux/arm64".parse::<ImagePlatform>()?,
+            ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU
+        );
+
+        assert_eq!(
+            "linux/arm64/v8".parse::<ImagePlatform>()?,
+            ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU
+        );
+
+        Ok(())
+    }
 }

xtask/src/codegen.rs

@@ -1,4 +1,5 @@
 use clap::Args;
+use cross::docker::ImagePlatform;
 use eyre::Context;
 use std::fmt::Write;
 
@@ -39,10 +40,12 @@ pub static PROVIDED_IMAGES: &[ProvidedImage] = &["#,
                 .platforms()
                 .iter()
                 .map(|p| {
-                    format!(
-                        "ImagePlatform::{}",
-                        p.replace('-', "_").to_ascii_uppercase()
-                    )
+                    let image_platform: ImagePlatform =
+                        p.parse().expect("should be a valid platform");
+
+                    image_platform
+                        .to_codegen_string()
+                        .expect("should be a valid platform")
                 })
                 .collect::<Vec<_>>()
                 .join(", "),