modules/coreboot: Cache coreboot toolchain archives and use mirrors

Download coreboot toolchain archives into packages/<arch> before coreboot tries to download them. This allows us to use mirrors to get the archives. We could also update the primary source this way if it goes down instead of patching coreboot itself (has happened for IASL). The archive versions and digests are retrieved from the coreboot module, so there isn't another copy of that info to maintain. That is done in bin/fetch_coreboot_crossgcc_archive.sh, which uses the existing fetch script to do the actual download, leveraging mirrors. bin/fetch_source_archive.sh supports using a SHA-1 digest instead of SHA-256, since coreboot has SHA-1 digests. It also checks if the file already exists (deleting the coreboot directory will cause it to be re-run, but the packages are already there and can be used from cache). The coreboot-4.11 IASL patch is updated to delete the outdated acpica archive digest (it already added the new one, but the old one was still there). bin/fetch_coreboot_crossgcc_archive.sh finds the archive version and digest from the digest files, so only one acpica file must be present. Signed-off-by: Jonathon Hall <jonathon.hall@puri.sm>
2024-12-18 20:47:55 +00:00 · 2024-01-08 14:05:03 -05:00 · 2024-01-08 14:05:03 -05:00 · f632897bb5
commit f632897bb5
parent 29203782f6
4 changed files with 203 additions and 10 deletions
--- a/bin/fetch_coreboot_crossgcc_archive.sh
+++ b/bin/fetch_coreboot_crossgcc_archive.sh
@ -0,0 +1,130 @@
+#! /usr/bin/env bash
+set -eo pipefail
+
+usage()
+{
+	cat <<USAGE_END
+usage:
+	$0 <coreboot-dir> <pkg-name> <pkgs-dir>
+	$0 --help
+
+	Downloads the source archive for <pkg-name> needed by coreboot's
+	crossgcc toolchain.  The package version and digest are found in
+	coreboot-dir.  The package is downloaded to <pkgs-dir>, then placed in
+	the coreboot directory.
+
+	Uses fetch_source_archive.sh, so mirrors are used and WGET can override
+	the path to wget.
+USAGE_END
+}
+
+if [ "$#" -lt 3 ]; then
+	usage
+	exit 1
+fi
+
+COREBOOT_DIR="$1"
+PKG_NAME="$2"
+PKGS_DIR="$3"
+
+# Get the result of a glob that should match a single thing, or die if it
+# doesn't match exactly one thing.
+single() {
+	if [ "$#" -eq 1 ]; then
+		if [ -f "$1" ]; then
+			echo "$1"
+		else
+			echo "$1: no matches" >&2
+			exit 1
+		fi
+	else
+		echo "multiple unexpected matches for glob:" "$@" >&2
+		exit 1
+	fi
+}
+
+# Delete prefix and suffix from a value
+delete_prefix_suffix() {
+	local value prefix suffix
+	value="$1"
+	prefix="$2"
+	suffix="$3"
+	value="${value/#$prefix/}"
+	value="${value/%$suffix/}"
+	echo "$value"
+}
+
+# Find the checksum file for this package
+
+# 'iasl' is special-cased.  Before coreboot 4.21, the archive was named
+# 'acpica-unix2-<ver>', and the original sources for those archives are gone.
+# Since coreboot 4.21, the archive is just named 'R<ver>.tar.gz', it lacks the
+# package name.
+# If we're fetching iasl, and this is an older release, look for the acpica
+# archive.
+if [ "$PKG_NAME" = iasl ] && [ -f "$COREBOOT_DIR/util/crossgcc/sum/"acpica-*.cksum ]; then
+	PKG_NAME=acpica
+fi
+# Otherwise, keep 'iasl' to look for the newer archive.
+
+# 'iasl' (4.21+) doesn't include the package name in the archive name, the
+# archive is just the release name
+if [ "$PKG_NAME" = "iasl" ]; then
+	PKG_CKSUM_FILE="$(single "$COREBOOT_DIR/util/crossgcc/sum/"R*.cksum)"
+else
+	PKG_CKSUM_FILE="$(single "$COREBOOT_DIR/util/crossgcc/sum/$PKG_NAME-"*.cksum)"
+fi
+
+PKG_BASENAME="$(basename "$PKG_CKSUM_FILE" .cksum)"
+# Get the base URL for the package.  This _is_ duplicated from coreboot's
+# buildgcc script, but these don't change much, and when they do we usually want
+# to use the newer source anyway for older versions of coreboot (e.g. Intel
+# broke all the iasl links - coreboot 90753398).
+case "$PKG_NAME" in
+	gmp)
+		PKG_BASEURL="https://ftpmirror.gnu.org/gmp/"
+		;;
+	mpfr)
+		PKG_BASEURL="https://ftpmirror.gnu.org/mpfr/"
+		;;
+	mpc)
+		PKG_BASEURL="https://ftpmirror.gnu.org/mpc/"
+		;;
+	gcc)
+		PKG_BASEURL="https://ftpmirror.gnu.org/gcc/gcc-$(delete_prefix_suffix "$PKG_BASENAME" gcc- .tar.xz)/"
+		;;
+	binutils)
+		PKG_BASEURL="https://ftpmirror.gnu.org/binutils/"
+		;;
+	nasm)
+		PKG_BASEURL="https://www.nasm.us/pub/nasm/releasebuilds/$(delete_prefix_suffix "$PKG_BASENAME" nasm- .tar.bz2)/"
+		;;
+	iasl)
+		PKG_BASEURL="https://github.com/acpica/acpica/archive/refs/tags/"
+		;;
+	acpica)
+		# Original acpica sources are gone.  Most of the older releases
+		# can be found here
+		PKG_BASEURL="https://distfiles.macports.org/acpica/"
+		# Version 20220331 (currently used by talos_2) isn't there, but
+		# there is an old link from Intel that is still up.  This is
+		# specific to this release.
+		if [ "$PKG_BASENAME" = acpica-unix2-20220331.tar.gz ]; then
+			PKG_BASEURL="https://downloadmirror.intel.com/774879/"
+		fi
+		;;
+esac
+
+PKG_DIGEST="$(cut -d' ' -f1 "$PKG_CKSUM_FILE")"
+
+BIN_DIR="$(dirname "${BASH_SOURCE[0]}")"
+
+# Download to packages/<arch>
+"$BIN_DIR/fetch_source_archive.sh" "$PKG_BASEURL$PKG_BASENAME" \
+	"$PKGS_DIR/coreboot-crossgcc-$PKG_BASENAME" "$PKG_DIGEST"
+
+# Copy to the tarballs directory so coreboot's toolchain build will use this
+# archive
+mkdir -p "$COREBOOT_DIR/util/crossgcc/tarballs"
+cp "$PKGS_DIR/coreboot-crossgcc-$PKG_BASENAME" \
+	"$COREBOOT_DIR/util/crossgcc/tarballs/$PKG_BASENAME"
--- a/bin/fetch_source_archive.sh
+++ b/bin/fetch_source_archive.sh
@ -10,11 +10,12 @@ usage()
 {
 	cat <<USAGE_END
 usage:
-	$0 <url> <file> <sha256sum>
+	$0 <url> <file> <digest>
 	$0 --help

 	Downloads <url> to <file>, falling back to package mirrors if the
-	primary source is not available or does match the expected sha256sum.
+	primary source is not available or does match the expected digest.  The
+	digest can be a SHA-256 (preferred) or SHA-1 digest.

 	Uses wget, export WGET to override the path to wget.
 USAGE_END
@ -27,20 +28,32 @@ fi

 URL="$1"
 FILE="$2"
-SHA256SUM="$3"
+DIGEST="$3"

 TMP_FILE="$2.tmp"

 WGET="${WGET:-wget}"

-rm -f "$FILE" "$TMP_FILE"
+case "$(echo -n "$DIGEST" | wc -c)" in
+	64)
+		SHASUM=sha256sum
+		;;
+	40)
+		# coreboot crossgcc archives have SHA-1 digests from coreboot
+		SHASUM=sha1sum
+		;;
+	*)
+		echo "Unknown digest for $FILE: $DIGEST" >&2
+		exit 1
+		;;
+esac

 download() {
 	local download_url
 	download_url="$1"
 	if ! "$WGET" -O "$TMP_FILE" "$download_url"; then
 		echo "Failed to download $download_url" >&2
-	elif ! echo "$SHA256SUM $TMP_FILE" | sha256sum --check -; then
+	elif ! echo "$DIGEST $TMP_FILE" | "$SHASUM" --check -; then
 		echo "File from $download_url does not match expected digest" >&2
 	else
 		mv "$TMP_FILE" "$FILE"	# Matches, keep this file
@ -50,6 +63,14 @@ download() {
 	return 1
 }

+# If the file exists already and the digest is correct, use the cached copy.
+if [ -f "$FILE" ] && (echo "$DIGEST $FILE" | "$SHASUM" --check -); then
+	echo "File $FILE is already cached" >&2
+	exit 0
+fi
+
+rm -f "$FILE" "$TMP_FILE"
+
 # Try the primary source
 download "$URL" && exit 0

--- a/modules/coreboot
+++ b/modules/coreboot
@ -14,7 +14,7 @@ else ifeq "$(CONFIG_TARGET_ARCH)" "ppc64"
 	# Linux), but coreboot is big-endian on PPC64.
 	COREBOOT_TARGET_CROSS := CROSS=$(CROSS)
 else
-	$(error "$(CONFIG_TARGET_ARCH) target isn't supported by this module")
+	$(error "$(CONFIG_TARGET_ARCH) target is not supported by this module")
 endif

 # Each coreboot version is defined as a separate module, but only the needed
@ -97,7 +97,7 @@ $(eval $(call coreboot_module,nitrokey,))
 ifeq "$(CONFIG_COREBOOT_VERSION)" ""
 	$(error "$(BOARD): does not specify coreboot version under CONFIG_COREBOOT_VERSION")
 else ifeq "$(coreboot-$(CONFIG_COREBOOT_VERSION)_dir)" ""
-	$(error "$(BOARD): coreboot version $(CONFIG_COREBOOT_VERSION) not known"
+	$(error "$(BOARD): coreboot version $(CONFIG_COREBOOT_VERSION) not known")
 endif 

 coreboot_module := coreboot-$(CONFIG_COREBOOT_VERSION)
@ -148,12 +148,49 @@ $(coreboot_module)_configure := \

 # Create a dependency from coreboot to the toolchain.  Use .heads-toolchain to
 # mark that the toolchain was built.
-COREBOOT_TOOLCHAIN=$(build)/$($(coreboot_toolchain_module)_base_dir)/.heads-toolchain
-$(COREBOOT_TOOLCHAIN): $(build)/$($(coreboot_toolchain_module)_base_dir)/.canary
+COREBOOT_TOOLCHAIN_DIR=$(build)/$($(coreboot_toolchain_module)_base_dir)
+$(COREBOOT_TOOLCHAIN_DIR)/.heads-toolchain: $(COREBOOT_TOOLCHAIN_DIR)/.canary
 	$(MAKE) -C "$(build)/$($(coreboot_toolchain_module)_base_dir)" CPUS=$(CPUS) "crossgcc-$(COREBOOT_TARGET)"
 	touch "$@"

-$(build)/$(coreboot_dir)/.configured: $(COREBOOT_TOOLCHAIN)
+$(build)/$(coreboot_dir)/.configured: $(COREBOOT_TOOLCHAIN_DIR)/.heads-toolchain
+
+## Toolchain packages ##
+# coreboot likes to download its own toolchain packages, but these sources can
+# go down or move also.  Download them ahead of the toolchain build with
+# fetch_source_archive.sh to leverage our mirrors.
+#
+# Create a task for each package.
+# $1 - package name (binutils/gcc/gmp/etc. - check coreboot/util/crossgcc/sum/)
+define coreboot-toolchain-pkg =
+# The package versions and digests aren't duplicated here, we get them
+# from the coreboot source.  This means downloading all packages
+# requires unpacking the coreboot source, but that's preferred over
+# maintaining a duplicate list here for each coreboot version.
+
+# Rule to download the source archive for $1 and place it in the
+# cooreboot directory.  Although there is a specific file produced here
+# (the package), we can't use it as the rule target because we don't
+# know the filename until coreboot is unpacked.
+$(COREBOOT_TOOLCHAIN_DIR)/.heads-crossgcc-pkg-$(1) : $(COREBOOT_TOOLCHAIN_DIR)/.canary
+	WGET="$(WGET)" bin/fetch_coreboot_crossgcc_archive.sh \
+		"$(COREBOOT_TOOLCHAIN_DIR)" "$(1)" "$(packages)"
+	touch "$$@"
+
+# The toolchain build requires this package
+$(COREBOOT_TOOLCHAIN_DIR)/.heads-toolchain: $(COREBOOT_TOOLCHAIN_DIR)/.heads-crossgcc-pkg-$(1)
+# The "packages" target depends on this target, so 'make packages' will
+# include these packages for seeding a mirror.
+packages: $(COREBOOT_TOOLCHAIN_DIR)/.heads-crossgcc-pkg-$(1)
+endef
+
+$(eval $(call coreboot-toolchain-pkg,gmp))
+$(eval $(call coreboot-toolchain-pkg,mpfr))
+$(eval $(call coreboot-toolchain-pkg,mpc))
+$(eval $(call coreboot-toolchain-pkg,binutils))
+$(eval $(call coreboot-toolchain-pkg,gcc))
+$(eval $(call coreboot-toolchain-pkg,nasm))
+$(eval $(call coreboot-toolchain-pkg,iasl))

 # Build with the cross toolchain from the toolchain module (which might be the
 # same coreboot module or a different one).
--- a/patches/coreboot-4.11/0070-crossgcc-iasl-2021-instead-of-2018_fix-old_coreboot-build-on-newer-platforms.patch
+++ b/patches/coreboot-4.11/0070-crossgcc-iasl-2021-instead-of-2018_fix-old_coreboot-build-on-newer-platforms.patch
@ -80,3 +80,8 @@ diff -ruN ./util/crossgcc/patches.orig/acpica-unix-20210105_iasl.patch ./util/cr
 }
 
 build_LLVM() {
+diff -ruN ./util/crossgcc/sum.orig/acpica-unix2-20190703.tar.gz.cksum ./util/crossgcc/sum/acpica-unix2-20190703.tar.gz.cksum
+--- ./util/crossgcc/sum.orig/acpica-unix2-20190703.tar.gz.cksum	2024-01-08 13:19:48.519641977 -0500
+++ ./util/crossgcc/sum/acpica-unix2-20190703.tar.gz.cksum	1969-12-31 19:00:00.000000000 -0500
+@@ -1 +0,0 @@
+-c5594944f933265a53695204a0672d0808e4a580  tarballs/acpica-unix2-20190703.tar.gz