From 69d6500cb37ae6609629a95d094ae6448dabaa6a Mon Sep 17 00:00:00 2001 From: FooDeas Date: Sun, 20 Aug 2023 14:03:06 +0200 Subject: [PATCH 1/2] update --- update.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/update.sh b/update.sh index 7aab10e..229ba0a 100755 --- a/update.sh +++ b/update.sh @@ -5,9 +5,10 @@ ARCHIVE_KEYS=() ARCHIVE_KEYS+=("https://archive.raspbian.org;raspbian.public.key;A0DA38D0D76E8B5D638872819165938D90FDDD2E") ARCHIVE_KEYS+=("https://archive.raspberrypi.org/debian;raspberrypi.gpg.key;CF8A1AF502A2AA2D763BAE7E82B129927FA3303E") -ARCHIVE_KEYS+=("https://ftp-master.debian.org/keys;archive-key-10.asc;80D15823B7FD1561F9F7BCDDDC30D7C23CBBABEE") ARCHIVE_KEYS+=("https://ftp-master.debian.org/keys;archive-key-11.asc;1F89983E0081FDE018F3CC9673A4F27B8DD47936") ARCHIVE_KEYS+=("https://ftp-master.debian.org/keys;release-11.asc;A4285295FC7B1A81600062A9605C66F00D6C9793") +ARCHIVE_KEYS+=("https://ftp-master.debian.org/keys;archive-key-12.asc;B8B80B5B623EAB6AD8775C45B7C5D7D6350947F8") +ARCHIVE_KEYS+=("https://ftp-master.debian.org/keys;release-12.asc;4D64FEC119C2029067D6E791F8D2585B8783D481") mirror_raspbian=http://mirrordirector.raspbian.org/raspbian mirror_raspberrypi=http://archive.raspberrypi.org/debian @@ -15,7 +16,7 @@ mirror_debian=http://deb.debian.org/debian declare mirror_raspbian_cache declare mirror_raspberrypi_cache declare mirror_debian_cache -release=bullseye +release=bookworm packages=() @@ -239,8 +240,13 @@ download_package_lists() { fi echo -n > "${1}_Packages" - package_section=firmware - download_package_list "${1}" "${2}" + if [ "${1}" != "debian" ]; then + package_section=firmware + download_package_list "${1}" "${2}" + else + package_section=non-free-firmware + download_package_list "${1}" "${2}" + fi package_section=main download_package_list "${1}" "${2}" package_section=non-free From b651d0d035ade05f830cd2b4ef8900d3c7f421a7 Mon Sep 17 00:00:00 2001 From: Keith Michaels Date: Tue, 12 Mar 2024 23:01:35 -0400 Subject: [PATCH 2/2] Upgrade to Bookworm build: - add variables: cleanup (build.sh), debug_cache (update.sh) - /etc/mdev.conf: /dev/(null|zero|full|u?random) mode 666 (a=rw) fixes installer apt failure - output repo name for selected packages (update.sh) installer: - release: bookworm replaces bullseye - Debian Bookworm package changes: ntpsec replaces ntp, sysvinit-utils replaces lsb-base, systemd-resolved/systemd-timesyncd become presets - network time sync: remove ntpdate (python dep.), update rdate servers, http fallback: new http-date method replaces defunct method - add cdebootstrap_debug variable - resolve apt warnings, deprecated apt-key: install main repo gpg keys to /etc/apt/trusted.gpg.d/ doc: - preset packages reflect Debian Bookworm convention - add cdebootstrap_debug advanced install parameter --- build.sh | 18 ++-- doc/INSTALL_CUSTOM.md | 5 +- scripts/opt/raspberrypi-ua-netinst/install.sh | 85 +++++++++---------- update.sh | 33 ++++--- 4 files changed, 73 insertions(+), 68 deletions(-) diff --git a/build.sh b/build.sh index e2bb729..9839d7e 100755 --- a/build.sh +++ b/build.sh @@ -10,6 +10,9 @@ packages_dir=./packages resources_dir=./res scripts_dir=./scripts +# set cleanup=non-empty-value to remove temporary build files +cleanup=1 + libs_to_copy=() # update version and date @@ -163,7 +166,6 @@ function create_cpio { mkdir -p rootfs/usr/sbin/ mkdir -p rootfs/usr/share/{dpkg,keyrings,libc-bin} mkdir -p rootfs/var/lib/dpkg/{alternatives,info,parts,updates} - mkdir -p rootfs/var/lib/ntpdate mkdir -p rootfs/var/log/ mkdir -p rootfs/var/run/ @@ -255,7 +257,7 @@ function create_cpio { # busybox components cp_executable tmp/bin/busybox rootfs/bin cd rootfs && ln -s bin/busybox init; cd .. - echo "\$MODALIAS=.* 0:0 660 @/opt/busybox/bin/modprobe \"\$MODALIAS\"" > rootfs/etc/mdev.conf + echo -e "\$MODALIAS=.* 0:0 660 @/opt/busybox/bin/modprobe \"\$MODALIAS\"\n(null|zero|full|u?random) 0:0 666" > rootfs/etc/mdev.conf # bash-static components cp --preserve=xattr,timestamps tmp/bin/bash-static rootfs/bin @@ -356,7 +358,7 @@ function create_cpio { # iproute2 components cp_executable tmp/bin/ip rootfs/bin/ - # lsb-base components + # sysvinit-utils components cp --preserve=xattr,timestamps tmp/lib/lsb/init-functions rootfs/lib/lsb/ cp --preserve=xattr,timestamps tmp/lib/lsb/init-functions.d/00-verbose rootfs/lib/lsb/init-functions.d/ @@ -368,12 +370,6 @@ function create_cpio { # netcat-openbsd cp_executable tmp/bin/nc.openbsd rootfs/bin/nc - # ntpdate components - cp --preserve=xattr,timestamps tmp/etc/default/ntpdate rootfs/etc/default/ - sed -i s/NTPDATE_USE_NTP_CONF=yes/NTPDATE_USE_NTP_CONF=no/ rootfs/etc/default/ntpdate - cp_executable tmp/usr/sbin/ntpdate rootfs/usr/sbin/ - cp_executable tmp/usr/sbin/ntpdate-debian rootfs/usr/sbin/ - # raspberrypi.org GPG key cp --preserve=xattr,timestamps ../"${packages_dir}"/raspberrypi.gpg.key rootfs/usr/share/keyrings/ @@ -466,7 +462,7 @@ function create_cpio { INITRAMFS="../raspberrypi-ua-netinst.cpio.gz" (cd rootfs && find . | cpio -H newc -ov | gzip --best > $INITRAMFS) - rm -rf rootfs + [ "$cleanup" ] && rm -rf rootfs } # Run update if never run @@ -549,4 +545,4 @@ cd bootfs && zip -r -9 "../${zipfile}" ./*; cd .. mv "${zipfile}" ../ # clean up -rm -rf tmp +[ "$cleanup" ] && rm -rf tmp diff --git a/doc/INSTALL_CUSTOM.md b/doc/INSTALL_CUSTOM.md index 470a6ef..999e913 100644 --- a/doc/INSTALL_CUSTOM.md +++ b/doc/INSTALL_CUSTOM.md @@ -29,7 +29,7 @@ | Preset | Packages | |---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `base` | _\,apt,gnupg,kmod_ | -| `minimal` | _\,cpufrequtils,fake-hwclock,ifupdown,net-tools,ntp,openssh-server,dosfstools,raspberrypi-sys-mods_ | +| `minimal` | _\,cpufrequtils,fake-hwclock,ifupdown,net-tools,ntpsec,openssh-server,dosfstools,raspberrypi-sys-mods_ | | `server` | _\,systemd-sysv,vim-tiny,iputils-ping,wget,ca-certificates,rsyslog,cron,dialog,locales,tzdata,less,man-db,logrotate,bash-completion,console-setup,apt-utils,libraspberrypi-bin,raspi-copies-and-fills (raspi-copies-and-fills is not available on arm64)_ | Note that if the networking configuration is set to use DHCP, `isc-dhcp-client` will also be installed. @@ -39,7 +39,7 @@ Note that if the networking configuration is set to use DHCP, `isc-dhcp-client` | Preset | Packages | |---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `base` | _\,apt,kmod_ | -| `minimal` | _\,cpufrequtils,iproute2,openssh-server,dosfstools,raspberrypi-sys-mods_ | +| `minimal` | _\,cpufrequtils,iproute2,systemd-resolved,systemd-timesyncd,openssh-server,dosfstools,raspberrypi-sys-mods_ | | `server` | _\,systemd-sysv,vim-tiny,iputils-ping,wget,ca-certificates,rsyslog,cron,dialog,locales,tzdata,less,man-db,logrotate,bash-completion,console-setup,apt-utils,libraspberrypi-bin,raspi-copies-and-fills (raspi-copies-and-fills is not available on arm64)_ | Note that if the networking configuration is set to use DHCP, no additional packages will be installed as `systemd-networkd` provides DHCP client support. @@ -153,6 +153,7 @@ Note that if the networking configuration is set to use DHCP, no additional pack | `hwrng_support` | `1` | `0`/`1` | Install support for the ARM hardware random number generator. The default is enabled (1) on all presets. Users requiring a `base` install are advised that `hwrng_support=0` must be added in `installer-config.txt` if HWRNG support is undesirable. | | `watchdog_enable` | `0` | `0`/`1` | Set to "1" to enable and use the hardware watchdog. | | `cdebootstrap_cmdline` | | | | +| `cdebootstrap_debug` | `0` | `0`/`1` | Set to "1" to enable cdebootstrap verbose/debug output. | | `rootfs_mkfs_options` | | | | | `rootsize` | | | / partition size in megabytes, provide it in the form '+\M' (without quotes), leave empty to use all free space | | `timeserver` | `time.nist.gov` | | | diff --git a/scripts/opt/raspberrypi-ua-netinst/install.sh b/scripts/opt/raspberrypi-ua-netinst/install.sh index 02886fc..b5964b9 100644 --- a/scripts/opt/raspberrypi-ua-netinst/install.sh +++ b/scripts/opt/raspberrypi-ua-netinst/install.sh @@ -44,6 +44,7 @@ variables_reset() { userperms_admin= userperms_sound= cdebootstrap_cmdline= + cdebootstrap_debug= bootsize= bootoffset= rootsize= @@ -150,7 +151,7 @@ variables_set_defaults() { else variable_set "mirror" "http://mirrordirector.raspbian.org/raspbian/" fi - variable_set "release" "bullseye" + variable_set "release" "bookworm" variable_set "hostname" "pi" variable_set "rootpw" "raspbian" variable_set "root_ssh_pwlogin" "1" @@ -181,6 +182,7 @@ variables_set_defaults() { variable_set "installer_pkg_downloadretries" "5" variable_set "hwrng_support" "1" variable_set "watchdog_enable" "0" + variable_set "cdebootstrap_debug" "0" variable_set "quiet_boot" "0" variable_set "disable_raspberries" "0" variable_set "disable_splash" "0" @@ -1058,49 +1060,28 @@ PRE_NETWORK_DURATION=$(date +%s) date_set=false if [ "${date_set}" = "false" ]; then - # set time with ntpdate - echo -n "Set time using ntpdate... " - if ntpdate-debian -b &> /dev/null; then - echo "OK" - date_set=true - fi - - if [ "${date_set}" = "false" ]; then - echo "Failed to set time via ntpdate. Switched to rdate." - # failed to set time with ntpdate, fall back to rdate - # time server addresses taken from http://tf.nist.gov/tf-cgi/servers.cgi - timeservers="${timeserver}" - timeservers="${timeservers} time.nist.gov nist1.symmetricom.com" - timeservers="${timeservers} nist-time-server.eoni.com utcnist.colorado.edu" - timeservers="${timeservers} nist1-pa.ustiming.org nist.expertsmi.com" - timeservers="${timeservers} nist1-macon.macon.ga.us wolfnisttime.com" - timeservers="${timeservers} nist.time.nosc.us nist.netservicesgroup.com" - timeservers="${timeservers} nisttime.carsoncity.k12.mi.us nist1-lnk.binary.net" - timeservers="${timeservers} ntp-nist.ldsbc.edu utcnist2.colorado.edu" - timeservers="${timeservers} nist1-ny2.ustiming.org wwv.nist.gov" - echo -n "Set time using timeserver " - for ts in ${timeservers}; do - echo -n "'${ts}'... " - if rdate "${ts}" &> /dev/null; then - echo "OK" - date_set=true - break - fi - done - fi + # set time with rdate + # time server addresses taken from http://tf.nist.gov/tf-cgi/servers.cgi + echo -n "Set time using timeserver " + for ts in ${timeserver} time.nist.gov time-{a..e}-{g,b,wwv}.nist.gov utcnist{,2}.colorado.edu; do + echo -n "'${ts}'... " + if rdate "${ts}" &> /dev/null; then + echo "OK" + date_set=true + break + fi + done if [ "${date_set}" = "false" ]; then echo "Failed to set time via rdate. Switched to HTTP." # Try to set time via http to work behind proxies. - # Timeserver has to return the time in the format: YYYY-MM-DD HH:MM:SS. - timeservers_http="${timeserver_http}" - timeservers_http="${timeservers_http} http://chronic.herokuapp.com/utc/now?format=%25F+%25T" - timeservers_http="${timeservers_http} http://www.timeapi.org/utc/now?format=%25F+%25T" - echo -n "Set time using HTTP timeserver " + timeservers_http="${timeserver_http} deb.debian.org kernel.org example.com archive.org icann.org iana.org ietf.org" + date_re=$'.*^[[:space:]]*Date:([^\r\n]+)' + echo -n "Set time using HTTP Date header " for ts_http in ${timeservers_http}; do echo -n "'${ts_http}'... " - http_time="$(wget -q -O - "${ts_http}")" - if date -u -s "${http_time}" &> /dev/null; then + http_date="$(wget --method=HEAD -qSO- -t 2 -T 3 --max-redirect=0 "${ts_http}" 2>&1)" + if [[ $http_date =~ $date_re && -n "${BASH_REMATCH[1]//[[:space:]]}" ]] && date -s "${BASH_REMATCH[1]}" &> /dev/null; then echo "OK" date_set=true break @@ -1197,7 +1178,7 @@ fi # determine available releases mirror_base=http://archive.raspberrypi.org/debian/dists/ -release_fallback=bullseye +release_fallback=bookworm release_base="${release}" release_raspbian="${release}" if ! wget --spider "${mirror_base}/${release}/" &> /dev/null; then @@ -1264,13 +1245,13 @@ if [ -z "${cdebootstrap_cmdline}" ]; then # minimal minimal_packages="cpufrequtils,openssh-server,dosfstools" if [ "${init_system}" != "systemd" ] || [ "${use_systemd_services}" = "0" ]; then - minimal_packages="${minimal_packages},ntp" + minimal_packages="${minimal_packages},ntpsec" if [ -z "${rtc}" ]; then minimal_packages="${minimal_packages},fake-hwclock" fi minimal_packages="${minimal_packages},ifupdown,net-tools" else - minimal_packages="${minimal_packages},iproute2" + minimal_packages="${minimal_packages},iproute2,systemd-resolved,systemd-timesyncd" fi minimal_packages_postinstall="${base_packages_postinstall},${minimal_packages_postinstall},raspberrypi-sys-mods" if echo "${ifname}" | grep -q "wlan"; then @@ -1328,6 +1309,11 @@ if [ -z "${cdebootstrap_cmdline}" ]; then ;; esac + # enable cdebootstrap verbose output + if [ "${cdebootstrap_debug}" = "1" ]; then + cdebootstrap_cmdline="--verbose --debug ${cdebootstrap_cmdline}"; + fi + # add user defined syspackages if [ -n "${syspackages}" ]; then cdebootstrap_cmdline="${cdebootstrap_cmdline},${syspackages}" @@ -1419,6 +1405,7 @@ echo " usersysgroups = ${usersysgroups}" echo " userperms_admin = ${userperms_admin}" echo " userperms_sound = ${userperms_sound}" echo " cdebootstrap_cmdline = ${cdebootstrap_cmdline}" +echo " cdebootstrap_debug = ${cdebootstrap_debug}" echo " packages_postinstall = ${packages_postinstall}" echo " boot_volume_label = ${boot_volume_label}" echo " root_volume_label = ${root_volume_label}" @@ -2157,8 +2144,20 @@ if grep -l '__RELEASE__' /rootfs/etc/apt/sources.list > /dev/null; then else echo "OK" fi -echo -n " Adding raspberrypi.org GPG key to apt-key... " -(chroot /rootfs /usr/bin/apt-key add - &> /dev/null) < /usr/share/keyrings/raspberrypi.gpg.key || fail +echo -n " Checking Raspbian/Debian GPG key... " +if [ "$(chroot /rootfs /usr/bin/gpg --keyring "/usr/share/keyrings/raspbian-archive-keyring.gpg" --with-colons --fingerprint 2> /dev/null)" == \ + "$(chroot /rootfs /usr/bin/gpg --keyring "/etc/apt/trusted.gpg" --with-colons --fingerprint 2> /dev/null)" ]; then + # deprecated apt-key usage detected; remove legacy trusted.gpg keyring + echo -n "Moving key to /etc/apt/trusted.gpg.d/... " + (chroot /rootfs install -m 644 "/usr/share/keyrings/raspbian-archive-keyring.gpg" "/etc/apt/trusted.gpg.d/") || fail + rm "/rootfs/etc/apt/trusted.gpg" || fail +fi +echo "OK" + +echo -n " Adding raspberrypi.org GPG key to /etc/apt/trusted.gpg.d/... " +raspberrypi_gpg="/rootfs/etc/apt/trusted.gpg.d/raspberrypi.gpg" +(chroot /rootfs /usr/bin/gpg --dearmor - > "$raspberrypi_gpg") < /usr/share/keyrings/raspberrypi.gpg.key || fail +chmod 644 "$raspberrypi_gpg" || fail echo "OK" echo -n " Configuring RaspberryPi repository... " diff --git a/update.sh b/update.sh index 229ba0a..2ee3cb6 100755 --- a/update.sh +++ b/update.sh @@ -18,6 +18,9 @@ declare mirror_raspberrypi_cache declare mirror_debian_cache release=bookworm +# set debug_cache=non-empty-value to run this script using cached data in packages/ from a previous run. +debug_cache= + packages=() # programs @@ -39,10 +42,9 @@ packages+=("f2fs-tools") packages+=("gpgv") packages+=("ifupdown") packages+=("iproute2") -packages+=("lsb-base") +packages+=("sysvinit-utils") packages+=("netbase") packages+=("netcat-openbsd") -packages+=("ntpdate") packages+=("raspbian-archive-keyring") packages+=("debian-archive-keyring") packages+=("rng-tools5") @@ -61,6 +63,7 @@ packages_sha256= packages_done=() download_file() { + [ "$debug_cache" ] && return local download_source=$1 local download_target=$2 local progress_option @@ -137,7 +140,6 @@ check_key() { } setup_archive_keys() { - mkdir -m 0700 -p gnupg # Let gpg set itself up already in the 'gnupg' dir before we actually use it echo "Setting up gpg... " @@ -192,7 +194,7 @@ download_package_list() { if grep -q "${package_section}/binary-armhf/Packages${extension}" "${1}_Release"; then # Download Packages file - echo -e "\nDownloading ${package_section} package list..." + echo -e "\n${1}: Downloading ${package_section} package list..." if ! download_file "${2}/dists/$release/$package_section/binary-armhf/Packages${extension}" "tmp${extension}"; then echo -e "ERROR\nDownloading '${package_section}' package list failed! Exiting." cd .. @@ -227,7 +229,7 @@ download_package_list() { } download_package_lists() { - echo -e "\nDownloading Release file and its signature..." + echo -e "\n--- ${1} ---\nDownloading Release file and its signature..." download_file "${2}/dists/$release/Release" "${1}_Release" download_file "${2}/dists/$release/Release.gpg" "${1}_Release.gpg" echo -n "Verifying Release file... " @@ -254,7 +256,7 @@ download_package_lists() { } add_packages() { - echo -e "\nAdding required packages..." + echo -e "\n--- ${1} ---\nAdding required packages..." filter_package_list < "${1}_Packages" >"${1}_Packages.tmp" while true; do libs=() @@ -325,6 +327,7 @@ download_packages() { } download_remote_file() { + [ "$debug_cache" ] && return if [ "${4}" != "" ]; then echo -e "\nDownloading '${4}'..." else @@ -350,8 +353,10 @@ fi # Download packages ( - rm -rf packages/ - mkdir packages && cd packages + if [ ! "$debug_cache" ]; then + rm -rf packages/ + fi + mkdir -p packages && cd packages ## Add caching proxy if configured if [ -n "${mirror_raspbian_cache}" ]; then @@ -371,9 +376,11 @@ fi fi ## Download package list - download_package_lists raspberry "${mirror_raspberrypi}" - download_package_lists raspbian "${mirror_raspbian}" - download_package_lists debian "${mirror_debian}" + if [ ! "$debug_cache" ]; then + download_package_lists raspberry "${mirror_raspberrypi}" + download_package_lists raspbian "${mirror_raspbian}" + download_package_lists debian "${mirror_debian}" + fi ## Select packages for download packages_debs=() @@ -389,7 +396,9 @@ fi fi ## Download selected packages - download_packages + if [ ! "$debug_cache" ]; then + download_packages + fi ) || exit $? # Download additional resources