scripts: cross-compilation implementation (compilation of 32-bit prog…

…rams for 64-bit arch)

- New variables have been implemented:
 - TERMUX_PKG_BUILD32 - enables or disables cross-compilation of the package
 - TERMUX_PKG_BUILD32DIR - path to compiled cross-package components
 - TERMUX_LIB64_PATH - 64-bit library path
 - TERMUX_LIB32_PATH - 32-bit library path
 - TERMUX_LIB_PATH - path to the library which changes depending on the compilation type (non-cross - TERMUX_LIB64_PATH, cross - TERMUX_LIB32_PATH)
- Implemented cross functions (termux_step_configure32, termux_step_make32 and termux_step_make_install32).
- The application environment setup scheme has been updated to ensure that cgct works properly. It now sets up temporary glibc, glibc32 packages, and libgcc symbolic links from cgct when needed. The updated scheme also allows forks to easily build glibc packages for their applications.
- The glibc package naming standard has been updated, now glibc packages can be named with the "glibc32" prefix.

build-package.sh

@@ -401,6 +401,14 @@ termux_check_package_in_building_packages_list() {
 	return $?
 }
 
+termux_set_crosses_arch() {
+	case $TERMUX_ARCH in
+		"aarch64") TERMUX_ARCH="arm";;
+		"x86_64") TERMUX_ARCH="i686";;
+		*) termux_error_exit "It is impossible to set crosses arch for ${TERMUX_ARCH} arch, because it is 32 bit. Only 64 bit arch are supported."
+	esac
+}
+
 # Special hook to prevent use of "sudo" inside package build scripts.
 # build-package.sh shouldn't perform any privileged operations.
 sudo() {
@@ -657,15 +665,33 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do
 		# to tools so need to run part of configure step
 		cd "$TERMUX_PKG_BUILDDIR"
 		termux_step_configure
+		if [ "$TERMUX_PKG_BUILD32" = "true" ]; then
+			(
+				termux_step_setup_build32_environment
+				termux_step_configure32
+			)
+		fi
 
 		if [ "$TERMUX_CONTINUE_BUILD" == "false" ]; then
 			cd "$TERMUX_PKG_BUILDDIR"
 			termux_step_post_configure
 		fi
 		cd "$TERMUX_PKG_BUILDDIR"
 		termux_step_make
+		if [ "$TERMUX_PKG_BUILD32" = "true" ]; then
+			(
+				termux_step_setup_build32_environment
+				termux_step_make32
+			)
+		fi
 		cd "$TERMUX_PKG_BUILDDIR"
 		termux_step_make_install
+		if [ "$TERMUX_PKG_BUILD32" = "true" ]; then
+			(
+				termux_step_setup_build32_environment
+				termux_step_make_install32
+			)
+		fi
 		cd "$TERMUX_PKG_BUILDDIR"
 		termux_step_post_make_install
 		termux_step_install_service_scripts

scripts/build/configure/termux_step_configure.sh

@@ -32,3 +32,7 @@ termux_step_configure() {
 		termux_step_configure_meson
 	fi
 }
+
+termux_step_configure32() {
+	termux_step_configure
+}

scripts/build/termux_step_make.sh

@@ -18,3 +18,7 @@ termux_step_make() {
 		fi
 	fi
 }
+
+termux_step_make32() {
+	termux_step_make
+}

scripts/build/termux_step_make_install.sh

@@ -34,3 +34,7 @@ termux_step_make_install() {
 			$TERMUX_PKG_EXTRA_CONFIGURE_ARGS
 	fi
 }
+
+termux_step_make_install32() {
+	termux_step_make_install
+}

scripts/build/termux_step_setup_build_folders.sh

@@ -3,6 +3,7 @@ termux_step_setup_build_folders() {
 	# permissions which makes them undeletable. We need to fix
 	# that.
 	[ -d "$TERMUX_PKG_BUILDDIR" ] && chmod +w -R "$TERMUX_PKG_BUILDDIR" || true
+	[ -d "$TERMUX_PKG_BUILD32DIR" ] && chmod +w -R "$TERMUX_PKG_BUILD32DIR" || true
 	[ -d "$TERMUX_PKG_SRCDIR" ] && chmod +w -R "$TERMUX_PKG_SRCDIR" || true
 	if [ "$TERMUX_SKIP_DEPCHECK" = false ] && \
 		   [ "$TERMUX_INSTALL_DEPS" = true ] && \
@@ -17,6 +18,7 @@ termux_step_setup_build_folders() {
 
 	# Cleanup old build state:
 	rm -Rf "$TERMUX_PKG_BUILDDIR" \
+		"$TERMUX_PKG_BUILD32DIR" \
 		"$TERMUX_PKG_SRCDIR"
 
 	# Cleanup old packaging state:

scripts/build/termux_step_setup_cgct_environment.sh

@@ -3,68 +3,108 @@
 termux_step_setup_cgct_environment() {
 	[ "$TERMUX_ON_DEVICE_BUILD" = "true" ] && return
 
-	if [ "$TERMUX_REPO_PACKAGE" != "$TERMUX_APP_PACKAGE" ]; then
-		echo "WARNING: It is not possible to install glibc core packages from the repo for operation of CGCT, you must install glibc packages for your application with the prefix '$TERMUX_PREFIX' yourself (core packages: glibc and linux-api-headers-glibc)."
-		return
+	termux_install_temporary_glibc
+	termux_set_links_to_libgcc_of_cgct
+
+	if [ "$TERMUX_PKG_BUILD32" = "true" ]; then
+		(
+			termux_set_crosses_arch
+			TERMUX_LIB_PATH="$TERMUX_LIB32_PATH"
+			termux_install_temporary_glibc
+			termux_set_links_to_libgcc_of_cgct
+		)
 	fi
+}
 
-	for PKG in gcc-libs-glibc glibc linux-api-headers-glibc; do
-		local PKG_DIR=$(ls ${TERMUX_SCRIPTDIR}/*/${PKG}/build.sh 2> /dev/null || \
-			ls ${TERMUX_SCRIPTDIR}/*/${PKG/-glibc/}/build.sh 2> /dev/null)
-		if [ -z "$PKG_DIR" ]; then
-			termux_error_exit "Could not find build.sh file for package '${PKG}'"
-		fi
-		local PKG_DIR_SPLIT=(${PKG_DIR//// })
+# The temporary glibc is a glibc used only during compilation of simple packages
+# or a full glibc that will be customized for the system and replace the temporary glibc
+termux_install_temporary_glibc() {
+	local PKG="glibc"
+	local cross_glibc=false
+	if [ "$TERMUX_ARCH" != "$TERMUX_REAL_ARCH" ]; then
+		cross_glibc=true
+		PKG+="32"
+	fi
 
-		local REPO_NAME=""
-		local LIST_PACKAGES_DIRECTORIES=(${TERMUX_PACKAGES_DIRECTORIES})
-		for idx in ${!LIST_PACKAGES_DIRECTORIES[@]}; do
-			if [ "${LIST_PACKAGES_DIRECTORIES[$idx]}" = "${PKG_DIR_SPLIT[-3]}" ]; then
-				REPO_NAME=$(echo "${TERMUX_REPO_URL[$idx]}" | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/%-%g')
-				if [ "$TERMUX_REPO_PKG_FORMAT" = "debian" ]; then
-					REPO_NAME+="-${TERMUX_REPO_DISTRIBUTION[$idx]}-Release"
-				elif [ "$TERMUX_REPO_PKG_FORMAT" = "pacman" ]; then
-					REPO_NAME+="-json"
-				fi
-				break
-			fi
-		done
-		if [ -z "$REPO_NAME" ]; then
-			termux_error_exit "Could not find '${PKG_DIR_SPLIT[-3]}' repo"
+	# Checking if temporary glibc needs to be installed
+	if [ -f "$TERMUX_BUILT_PACKAGES_DIRECTORY/$PKG-temporary-for-cgct" ]; then
+		return
+	fi
+	local PKG_DIR=$(ls ${TERMUX_SCRIPTDIR}/*/${PKG}/build.sh 2> /dev/null || \
+		ls ${TERMUX_SCRIPTDIR}/*/${PKG}/build.sh 2> /dev/null)
+	if [ -n "$PKG_DIR" ]; then
+		read DEP_ARCH DEP_VERSION DEP_VERSION_PAC <<< $(termux_extract_dep_info $PKG "${PKG_DIR/'/build.sh'/}")
+		if termux_package__is_package_version_built "$PKG" "$DEP_VERSION"; then
+			return
 		fi
+	fi
 
-		read DEP_ARCH DEP_VERSION DEP_VERSION_PAC <<< $(termux_extract_dep_info $PKG "${PKG_DIR/'/build.sh'/}")
+	[ ! "$TERMUX_QUIET_BUILD" = "true" ] && echo "Installing temporary '${PKG}' for the CGCT tool environment."
 
-		if ! termux_package__is_package_version_built "$PKG" "$DEP_VERSION" && [ ! -f "$TERMUX_BUILT_PACKAGES_DIRECTORY/$PKG-for-cgct" ]; then
-			[ ! "$TERMUX_QUIET_BUILD" = "true" ] && echo "Installing '${PKG}' for the CGCT tool environment."
+	local PREFIX_TMP_GLIBC="data/data/com.termux/files/usr/glibc"
+	local PATH_TMP_GLIBC="$TERMUX_COMMON_CACHEDIR/temporary_glibc_for_cgct"
+	mkdir -p "$PATH_TMP_GLIBC"
 
-			if [ ! -f "${TERMUX_COMMON_CACHEDIR}-${DEP_ARCH}/${REPO_NAME}" ]; then
-				TERMUX_INSTALL_DEPS=true termux_get_repo_files
-			fi
+	local GPKG_LINK="https://service.termux-pacman.dev/gpkg/$TERMUX_ARCH"
+	local GPKG_JSON="$PATH_TMP_GLIBC/gpkg-$TERMUX_ARCH.json"
+	termux_download "$GPKG_LINK/gpkg.json" \
+		"$GPKG_JSON" \
+		SKIP_CHECKSUM
+
+	# Installing temporary glibc
+	local GLIBC_PKG=$(jq -r '."glibc"."FILENAME"' "$GPKG_JSON")
+	if [ ! -f "$PATH_TMP_GLIBC/$GLIBC_PKG" ]; then
+		termux_download "$GPKG_LINK/$GLIBC_PKG" \
+			"$PATH_TMP_GLIBC/$GLIBC_PKG" \
+			$(jq -r '."glibc"."SHA256SUM"' "$GPKG_JSON")
+	fi
+
+	[ ! "$TERMUX_QUIET_BUILD" = true ] && echo "extracting temporary $PKG..."
 
-			if ! TERMUX_WITHOUT_DEPVERSION_BINDING=true termux_download_deb_pac $PKG $DEP_ARCH $DEP_VERSION $DEP_VERSION_PAC; then
-				termux_error_exit "Failed to download package '${PKG}'"
+	# Unpacking temporary glibc
+	tar -xJf "$PATH_TMP_GLIBC/$GLIBC_PKG" -C "$PATH_TMP_GLIBC" data
+	if [ "$cross_glibc" = "true" ]; then
+		# Installing lib32
+		mkdir "$TERMUX_LIB_PATH"
+		cp -r $PATH_TMP_GLIBC/$PREFIX_TMP_GLIBC/lib/* "$TERMUX_LIB_PATH"
+		# Installing include32
+		mkdir -p "$TERMUX_PREFIX/include32"
+		local hpath
+		for hpath in $(find "$PATH_TMP_GLIBC/$PREFIX_TMP_GLIBC/include" -type f); do
+			local h=$(sed "s|$PATH_TMP_GLIBC/$PREFIX_TMP_GLIBC/include/||g" <<< "$hpath")
+			if [ -f "$TERMUX_PREFIX/include/$h" ] && \
+				[ $(md5sum "$hpath" | awk '{printf $1}') = $(md5sum "$TERMUX_PREFIX/include/$h" | awk '{printf $1}') ]; then
+				rm "$hpath"
 			fi
+		done
+		find "$PATH_TMP_GLIBC/$PREFIX_TMP_GLIBC/include" -type d -empty -delete
+		cp -r $PATH_TMP_GLIBC/$PREFIX_TMP_GLIBC/include/* $TERMUX_PREFIX/include32
+		# Installing dynamic linker in lib
+		mkdir -p "$TERMUX_LIB64_PATH"
+		local ld_path=$(ls $TERMUX_LIB_PATH/ld-*)
+		ln -sr "${ld_path}" "$TERMUX_LIB64_PATH/$(basename ${ld_path})"
+	else
+		# Complete installation of glibc components
+		cp -r $PATH_TMP_GLIBC/$PREFIX_TMP_GLIBC/* "$TERMUX_PREFIX"
+	fi
+	# It is necessary to reconfigure the paths in libc.so
+	# for correct work of cross-compilation and compilation in forked projects
+	sed -i "s|/$PREFIX_TMP_GLIBC/lib/|$TERMUX_LIB_PATH/|g" "$TERMUX_LIB_PATH/libc.so"
 
-			[ ! "$TERMUX_QUIET_BUILD" = true ] && echo "extracting $PKG to $TERMUX_COMMON_CACHEDIR-$DEP_ARCH..."
-			(
-				cd $TERMUX_COMMON_CACHEDIR-$DEP_ARCH
-				if [ "$TERMUX_REPO_PKG_FORMAT" = "debian" ]; then
-					ar x ${PKG}_${DEP_VERSION}_${DEP_ARCH}.deb data.tar.xz
-					if tar -tf data.tar.xz|grep "^./$">/dev/null; then
-						tar -xf data.tar.xz --strip-components=1 \
-							--no-overwrite-dir -C /
-					else
-						tar -xf data.tar.xz --no-overwrite-dir -C /
-					fi
-				elif [ "$TERMUX_REPO_PKG_FORMAT" = "pacman" ]; then
-					tar -xJf "${PKG}-${DEP_VERSION_PAC}-${DEP_ARCH}.pkg.tar.xz" \
-						--anchored --exclude=.{BUILDINFO,PKGINFO,MTREE,INSTALL} \
-						--force-local --no-overwrite-dir -C /
-				fi
-			)
-			mkdir -p $TERMUX_BUILT_PACKAGES_DIRECTORY
-			echo "" > "$TERMUX_BUILT_PACKAGES_DIRECTORY/$PKG-for-cgct"
+	# Marking the installation of temporary glibc
+	rm -fr "$PATH_TMP_GLIBC/data"
+	mkdir -p "$TERMUX_BUILT_PACKAGES_DIRECTORY"
+	touch "$TERMUX_BUILT_PACKAGES_DIRECTORY/$PKG-temporary-for-cgct"
+}
+
+# Sets up symbolic links to libgcc* libraries (from cgct) in the application environment
+# to allow cgct to work properly, if necessary
+termux_set_links_to_libgcc_of_cgct() {
+	local libgcc_cgct
+	for libgcc_cgct in $(find "$CGCT_DIR/$TERMUX_ARCH/lib" -maxdepth 1 -type f -name 'libgcc*'); do
+		if [ ! -e "$TERMUX_LIB_PATH/$(basename $libgcc_cgct)" ]; then
+			ln -s "$libgcc_cgct" "$TERMUX_LIB_PATH"
 		fi
 	done
 }
+

scripts/build/termux_step_setup_toolchain.sh

@@ -24,3 +24,11 @@ termux_step_setup_toolchain() {
 		termux_setup_toolchain_gnu
 	fi
 }
+
+termux_step_setup_build32_environment() {
+	termux_set_crosses_arch
+	TERMUX_LIB_PATH="$TERMUX_LIB32_PATH"
+	termux_step_setup_arch_variables
+	termux_step_setup_toolchain
+	cd $TERMUX_PKG_BUILD32DIR
+}

scripts/build/termux_step_setup_variables.sh

@@ -44,6 +44,9 @@ termux_step_setup_variables() {
 			TERMUX_PKG_NAME="$(termux_package__add_prefix_glibc_to_package_name "${TERMUX_PKG_NAME}")"
 		fi
 	fi
+	TERMUX_LIB64_PATH="$TERMUX_PREFIX/lib"
+	TERMUX_LIB32_PATH="$TERMUX_PREFIX/lib32"
+	TERMUX_LIB_PATH="$TERMUX_LIB64_PATH"
 
 	if [ "$TERMUX_ON_DEVICE_BUILD" = "true" ]; then
 		# For on-device builds cross-compiling is not supported so we can
@@ -53,7 +56,7 @@ termux_step_setup_variables() {
 
 		if [ "$TERMUX_PACKAGE_LIBRARY" = "bionic" ]; then
 			# On-device builds without termux-exec are unsupported.
-			if ! grep -q "${TERMUX_PREFIX}/lib/libtermux-exec.so" <<< "${LD_PRELOAD-x}"; then
+			if ! grep -q "${TERMUX_LIB_PATH}/libtermux-exec.so" <<< "${LD_PRELOAD-x}"; then
 				termux_error_exit "On-device builds without termux-exec are not supported."
 			fi
 		fi
@@ -64,23 +67,9 @@ termux_step_setup_variables() {
 	# TERMUX_PKG_MAINTAINER should be explicitly set in build.sh of the package.
 	: "${TERMUX_PKG_MAINTAINER:="default"}"
 
-	if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then
-		TERMUX_ARCH_BITS=64
-	else
-		TERMUX_ARCH_BITS=32
-	fi
-
-	if [ "$TERMUX_PACKAGE_LIBRARY" = "bionic" ]; then
-		TERMUX_HOST_PLATFORM="${TERMUX_ARCH}-linux-android"
-	else
-		TERMUX_HOST_PLATFORM="${TERMUX_ARCH}-linux-gnu"
-	fi
-	if [ "$TERMUX_ARCH" = "arm" ]; then
-		TERMUX_HOST_PLATFORM="${TERMUX_HOST_PLATFORM}eabi"
-		if [ "$TERMUX_PACKAGE_LIBRARY" = "glibc" ]; then
-			TERMUX_HOST_PLATFORM="${TERMUX_HOST_PLATFORM}hf"
-		fi
-	fi
+	termux_step_setup_arch_variables
+	TERMUX_REAL_ARCH="$TERMUX_ARCH"
+	TERMUX_REAL_HOST_PLATFORM="$TERMUX_HOST_PLATFORM"
 
 	if [ "$TERMUX_PACKAGE_LIBRARY" = "bionic" ]; then
 		if [ "$TERMUX_ON_DEVICE_BUILD" = "false" ] && [ ! -d "$NDK" ]; then
@@ -139,6 +128,8 @@ termux_step_setup_variables() {
 	TERMUX_PKG_BUILDDIR=$TERMUX_TOPDIR/$TERMUX_PKG_NAME/build
 	TERMUX_PKG_BUILD_DEPENDS=""
 	TERMUX_PKG_BUILD_IN_SRC=false
+	TERMUX_PKG_BUILD32=false
+	TERMUX_PKG_BUILD32DIR=$TERMUX_TOPDIR/$TERMUX_PKG_NAME/build32
 	TERMUX_PKG_CONFFILES=""
 	TERMUX_PKG_CONFLICTS="" # https://www.debian.org/doc/debian-policy/ch-relationships.html#s-conflicts
 	TERMUX_PKG_DEPENDS=""
@@ -188,10 +179,30 @@ termux_step_setup_variables() {
 	TERMUX_PKG_PYTHON_BUILD_DEPS="" # python modules to be installed via build-pip
 	TERMUX_PKG_PYTHON_COMMON_DEPS="" # python modules to be installed via pip3 or build-pip
 	TERMUX_PYTHON_CROSSENV_PREFIX="$TERMUX_TOPDIR/python${TERMUX_PYTHON_VERSION}-crossenv-prefix-$TERMUX_PACKAGE_LIBRARY-$TERMUX_ARCH" # python modules dependency location (only used in non-devices)
-	TERMUX_PYTHON_HOME=$TERMUX_PREFIX/lib/python${TERMUX_PYTHON_VERSION} # location of python libraries
+	TERMUX_PYTHON_HOME=$TERMUX_LIB_PATH/python${TERMUX_PYTHON_VERSION} # location of python libraries
 	TERMUX_PKG_MESON_NATIVE=false
 	TERMUX_PKG_CMAKE_CROSSCOMPILING=true
 
 	unset CFLAGS CPPFLAGS LDFLAGS CXXFLAGS
 	unset TERMUX_MESON_ENABLE_SOVERSION # setenv to enable SOVERSION suffix for shared libs built with Meson
 }
+
+termux_step_setup_arch_variables() {
+	if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then
+		TERMUX_ARCH_BITS=64
+	else
+		TERMUX_ARCH_BITS=32
+	fi
+
+	if [ "$TERMUX_PACKAGE_LIBRARY" = "bionic" ]; then
+		TERMUX_HOST_PLATFORM="${TERMUX_ARCH}-linux-android"
+	else
+		TERMUX_HOST_PLATFORM="${TERMUX_ARCH}-linux-gnu"
+	fi
+	if [ "$TERMUX_ARCH" = "arm" ]; then
+		TERMUX_HOST_PLATFORM="${TERMUX_HOST_PLATFORM}eabi"
+		if [ "$TERMUX_PACKAGE_LIBRARY" = "glibc" ]; then
+			TERMUX_HOST_PLATFORM="${TERMUX_HOST_PLATFORM}hf"
+		fi
+	fi
+}

scripts/build/termux_step_start_build.sh

@@ -62,6 +62,16 @@ termux_step_start_build() {
 		fi
 	fi
 
+	if [ "$TERMUX_ON_DEVICE_BUILD" = "true" ] || [ "$TERMUX_ARCH" = "arm" ] || [ "$TERMUX_ARCH" = "i686" ]; then
+		TERMUX_PKG_BUILD32=false
+	fi
+	if [ "$TERMUX_PKG_BUILD_IN_SRC" = "true" ] && [ "$TERMUX_PKG_BUILD32" = "true" ]; then
+		termux_error_exit "It is not possible to build 32-bit and 64-bit versions of a package in one place, the build location must be separate."
+	fi
+	if [ "$TERMUX_PKG_BUILD32" = "true" ]; then
+		mkdir -p "$TERMUX_PKG_BUILD32DIR"
+	fi
+
 	echo "termux - building $TERMUX_PKG_NAME for arch $TERMUX_ARCH..."
 	test -t 1 && printf "\033]0;%s...\007" "$TERMUX_PKG_NAME"
 

scripts/build/toolchain/termux_setup_toolchain_gnu.sh

@@ -20,8 +20,11 @@ termux_setup_toolchain_gnu() {
 		export CXXFILT=$TERMUX_HOST_PLATFORM-c++filt
 	fi
 
-	if [ ! -d "$TERMUX_PREFIX/lib/" ]; then
-		termux_error_exit "glibc library directory was not found ('$TERMUX_PREFIX/lib/')"
+	if [ ! -d "$TERMUX_LIB_PATH/" ]; then
+		termux_error_exit "glibc library directory was not found ('$TERMUX_LIB_PATH/')"
+	fi
+	if [ ! -f "$TERMUX_LIB_PATH/libgcc_s.so" ] && [ ! -f "$TERMUX_LIB_PATH/libgcc_s.so.1" ]; then
+		termux_error_exit "libgcc not found, there is a risk of incorrect compiler operation"
 	fi
 	if [ ! -d "$TERMUX_PREFIX/include/" ]; then
 		termux_error_exit "glibc header directory was not found ('$TERMUX_PREFIX/include/')"
@@ -40,7 +43,7 @@ termux_setup_toolchain_gnu() {
 		CFLAGS+=" -march=i686"
 		export DYNAMIC_LINKER="ld-linux.so.2"
 	fi
-	export PATH_DYNAMIC_LINKER="$TERMUX_PREFIX/lib/$DYNAMIC_LINKER"
+	export PATH_DYNAMIC_LINKER="$TERMUX_LIB64_PATH/$DYNAMIC_LINKER"
 
 	if [ ! -f "$PATH_DYNAMIC_LINKER" ]; then
 		termux_error_exit "glibc dynamic linker was not found ('$PATH_DYNAMIC_LINKER')"