Merge pull request #1576 from JonathonHall-Purism/package-mirrors

Makefile: Support mirrors for dependency source packages
2024-12-18 20:47:55 +00:00 · 2024-01-09 12:21:30 -05:00 · 2024-01-09 12:21:30 -05:00 · 025b4d0dfc
commit 025b4d0dfc
parent c9e067c721 93ccf25d24
5 changed files with 281 additions and 14 deletions
--- a/13
+++ b/13
@ -384,18 +384,15 @@ define define_module =
    # wget creates it early, so we have to cleanup if it fails
    $(packages)/$($1_tar):
 	$(call do,WGET,$($1_url),\
-		if ! $(WGET) -O "$$@.tmp" $($1_url) ; then \
-			exit 1 ; \
-		fi ; \
-		mv "$$@.tmp" "$$@" \
+		WGET="$(WGET)" bin/fetch_source_archive.sh "$($1_url)" "$$@" "$($1_hash)"
 	)
-    $(packages)/.$1-$($1_version)_verify: $(packages)/$($1_tar)
-	echo "$($1_hash)  $$^" | sha256sum --check -
-	@touch "$$@"
+
+    # Target to fetch all packages, for seeding mirrors
+    packages: $(packages)/$($1_tar)

    # Unpack the tar file and touch the canary so that we know
    # that the files are all present
-    $(build)/$($1_base_dir)/.canary: $(packages)/.$1-$($1_version)_verify
+    $(build)/$($1_base_dir)/.canary: $(packages)/$($1_tar)
 	mkdir -p "$$(dir $$@)"
 	tar -xf "$(packages)/$($1_tar)" $(or $($1_tar_opt),--strip 1) -C "$$(dir $$@)"
 	if [ -r patches/$($1_patch_name).patch ]; then \
--- a/bin/fetch_coreboot_crossgcc_archive.sh
+++ b/bin/fetch_coreboot_crossgcc_archive.sh
@ -0,0 +1,132 @@
+#! /usr/bin/env bash
+set -eo pipefail
+
+usage()
+{
+	cat <<USAGE_END
+usage:
+	$0 <coreboot-dir> <pkg-name> <pkgs-dir>
+	$0 --help
+
+	Downloads the source archive for <pkg-name> needed by coreboot's
+	crossgcc toolchain.  The package version and digest are found in
+	coreboot-dir.  The package is downloaded to <pkgs-dir>, then placed in
+	the coreboot directory.
+
+	Uses fetch_source_archive.sh, so mirrors are used and WGET can override
+	the path to wget.
+USAGE_END
+}
+
+if [ "$#" -lt 3 ]; then
+	usage
+	exit 1
+fi
+
+COREBOOT_DIR="$1"
+PKG_NAME="$2"
+PKGS_DIR="$(realpath "$3")" # Make sure it's an absolute path
+
+# Get the result of a glob that should match a single thing, or die if it
+# doesn't match exactly one thing.
+single() {
+	if [ "$#" -eq 1 ]; then
+		if [ -f "$1" ]; then
+			echo "$1"
+		else
+			echo "$1: no matches" >&2
+			exit 1
+		fi
+	else
+		echo "multiple unexpected matches for glob:" "$@" >&2
+		exit 1
+	fi
+}
+
+# Delete prefix and suffix from a value
+delete_prefix_suffix() {
+	local value prefix suffix
+	value="$1"
+	prefix="$2"
+	suffix="$3"
+	value="${value/#$prefix/}"
+	value="${value/%$suffix/}"
+	echo "$value"
+}
+
+# Find the checksum file for this package
+
+# 'iasl' is special-cased.  Before coreboot 4.21, the archive was named
+# 'acpica-unix2-<ver>', and the original sources for those archives are gone.
+# Since coreboot 4.21, the archive is just named 'R<ver>.tar.gz', it lacks the
+# package name.
+# If we're fetching iasl, and this is an older release, look for the acpica
+# archive.
+if [ "$PKG_NAME" = iasl ] && [ -f "$COREBOOT_DIR/util/crossgcc/sum/"acpica-*.cksum ]; then
+	PKG_NAME=acpica
+fi
+# Otherwise, keep 'iasl' to look for the newer archive.
+
+# 'iasl' (4.21+) doesn't include the package name in the archive name, the
+# archive is just the release name
+if [ "$PKG_NAME" = "iasl" ]; then
+	PKG_CKSUM_FILE="$(single "$COREBOOT_DIR/util/crossgcc/sum/"R*.cksum)"
+else
+	PKG_CKSUM_FILE="$(single "$COREBOOT_DIR/util/crossgcc/sum/$PKG_NAME-"*.cksum)"
+fi
+
+PKG_BASENAME="$(basename "$PKG_CKSUM_FILE" .cksum)"
+# Get the base URL for the package.  This _is_ duplicated from coreboot's
+# buildgcc script, but these don't change much, and when they do we usually want
+# to use the newer source anyway for older versions of coreboot (e.g. Intel
+# broke all the iasl links - coreboot 90753398).
+case "$PKG_NAME" in
+	gmp)
+		PKG_BASEURL="https://ftpmirror.gnu.org/gmp/"
+		;;
+	mpfr)
+		PKG_BASEURL="https://ftpmirror.gnu.org/mpfr/"
+		;;
+	mpc)
+		PKG_BASEURL="https://ftpmirror.gnu.org/mpc/"
+		;;
+	gcc)
+		PKG_BASEURL="https://ftpmirror.gnu.org/gcc/gcc-$(delete_prefix_suffix "$PKG_BASENAME" gcc- .tar.xz)/"
+		;;
+	binutils)
+		PKG_BASEURL="https://ftpmirror.gnu.org/binutils/"
+		;;
+	nasm)
+		PKG_BASEURL="https://www.nasm.us/pub/nasm/releasebuilds/$(delete_prefix_suffix "$PKG_BASENAME" nasm- .tar.bz2)/"
+		;;
+	iasl)
+		PKG_BASEURL="https://github.com/acpica/acpica/archive/refs/tags/"
+		;;
+	acpica)
+		# Original acpica sources are gone.  Most of the older releases
+		# can be found here
+		PKG_BASEURL="https://distfiles.macports.org/acpica/"
+		# Version 20220331 (currently used by talos_2) isn't there, but
+		# there is an old link from Intel that is still up.  This is
+		# specific to this release.
+		if [ "$PKG_BASENAME" = acpica-unix2-20220331.tar.gz ]; then
+			PKG_BASEURL="https://downloadmirror.intel.com/774879/"
+		fi
+		;;
+esac
+
+PKG_DIGEST="$(cut -d' ' -f1 "$PKG_CKSUM_FILE")"
+
+BIN_DIR="$(dirname "${BASH_SOURCE[0]}")"
+
+# Download to packages/<arch>
+"$BIN_DIR/fetch_source_archive.sh" "$PKG_BASEURL$PKG_BASENAME" \
+	"$PKGS_DIR/coreboot-crossgcc-$PKG_BASENAME" "$PKG_DIGEST"
+
+# Copy to the tarballs directory so coreboot's toolchain build will use this
+# archive
+mkdir -p "$COREBOOT_DIR/util/crossgcc/tarballs"
+(
+	cd "$COREBOOT_DIR/util/crossgcc/tarballs"
+	ln -s "$PKGS_DIR/coreboot-crossgcc-$PKG_BASENAME" "$PKG_BASENAME"
+)
--- a/bin/fetch_source_archive.sh
+++ b/bin/fetch_source_archive.sh
@ -0,0 +1,92 @@
+#! /usr/bin/env bash
+set -eo pipefail
+
+# Mirror URLs, make sure these end in slashes.
+BACKUP_MIRRORS=(
+	https://storage.puri.sm/heads-packages/
+)
+
+usage()
+{
+	cat <<USAGE_END
+usage:
+	$0 <url> <file> <digest>
+	$0 --help
+
+	Downloads <url> to <file>, falling back to package mirrors if the
+	primary source is not available or does match the expected digest.  The
+	digest can be a SHA-256 (preferred) or SHA-1 digest.
+
+	Uses wget, export WGET to override the path to wget.
+USAGE_END
+}
+
+if [ "$#" -lt 2 ]; then
+	usage
+	exit 1
+fi
+
+URL="$1"
+FILE="$2"
+DIGEST="$3"
+
+TMP_FILE="$2.tmp"
+
+WGET="${WGET:-wget}"
+
+case "$(echo -n "$DIGEST" | wc -c)" in
+	64)
+		SHASUM=sha256sum
+		;;
+	40)
+		# coreboot crossgcc archives have SHA-1 digests from coreboot
+		SHASUM=sha1sum
+		;;
+	*)
+		echo "Unknown digest for $FILE: $DIGEST" >&2
+		exit 1
+		;;
+esac
+
+download() {
+	local download_url
+	download_url="$1"
+	if ! "$WGET" -O "$TMP_FILE" "$download_url"; then
+		echo "Failed to download $download_url" >&2
+	elif ! echo "$DIGEST $TMP_FILE" | "$SHASUM" --check -; then
+		echo "File from $download_url does not match expected digest" >&2
+	else
+		mv "$TMP_FILE" "$FILE"	# Matches, keep this file
+		return 0
+	fi
+	rm -f "$TMP_FILE"	# Wasn't downloaded or failed check
+	return 1
+}
+
+# If the file exists already and the digest is correct, use the cached copy.
+if [ -f "$FILE" ] && (echo "$DIGEST $FILE" | "$SHASUM" --check -); then
+	echo "File $FILE is already cached" >&2
+	exit 0
+fi
+
+rm -f "$FILE" "$TMP_FILE"
+
+# Try the primary source
+download "$URL" && exit 0
+
+# Shuffle the mirrors so we try each equally
+readarray -t BACKUP_MIRRORS < <(shuf -e "${BACKUP_MIRRORS[@]}")
+
+# The mirrors use our archive names, which may differ from the primary source
+# (e.g. musl-cross-make archives are just <hash>.tar.gz, makes more sense to use
+# musl-cross-<hash>.tar.gz).  This also means mirrors can be seeded directly
+# from the packages/<arch>/ directories.
+archive="$(basename "$FILE")"
+echo "Try mirrors for $archive" >&2
+
+for mirror in "${BACKUP_MIRRORS[@]}"; do
+	download "$mirror$archive" && exit 0
+done
+
+# All mirrors failed
+exit 1
--- a/modules/coreboot
+++ b/modules/coreboot
@ -14,7 +14,7 @@ else ifeq "$(CONFIG_TARGET_ARCH)" "ppc64"
 	# Linux), but coreboot is big-endian on PPC64.
 	COREBOOT_TARGET_CROSS := CROSS=$(CROSS)
 else
-	$(error "$(CONFIG_TARGET_ARCH) target isn't supported by this module")
+	$(error "$(CONFIG_TARGET_ARCH) target is not supported by this module")
 endif

 # Each coreboot version is defined as a separate module, but only the needed
@ -61,6 +61,9 @@ endef
 coreboot-4.11_hash := 97fd859b4c39a25534fe33c30eb86e54a233952e08a024c55858d11598a8ad87
 coreboot-blobs-4.11_hash := aa7855c5bd385b3360dadc043ea6bc93f564e6e4840d9b3ee5b9e696bbd055db
 $(eval $(call coreboot_module,4.11,))
+# The coreboot 4.11 toolchain disables the Ada compiler.  None of the 4.11 boards need
+# libgfxinit, and the old Ada compiler no longer compiles with the Debian 12 host toolchain.
+coreboot-4.11_toolchain_build_args := BUILD_LANGUAGES=c

 coreboot-4.19_hash := 65ccb2f46535b996e0066a1b76f81c8cf1ff3e27df84b3f97d8ad7b3e7cf0a43
 coreboot-blobs-4.19_hash := 30214caed07b25f11e47bec022ff6234841376e36689eb674de2330a3e980cbc
@ -97,7 +100,7 @@ $(eval $(call coreboot_module,nitrokey,))
 ifeq "$(CONFIG_COREBOOT_VERSION)" ""
 	$(error "$(BOARD): does not specify coreboot version under CONFIG_COREBOOT_VERSION")
 else ifeq "$(coreboot-$(CONFIG_COREBOOT_VERSION)_dir)" ""
-	$(error "$(BOARD): coreboot version $(CONFIG_COREBOOT_VERSION) not known"
+	$(error "$(BOARD): coreboot version $(CONFIG_COREBOOT_VERSION) not known")
 endif 

 coreboot_module := coreboot-$(CONFIG_COREBOOT_VERSION)
@ -148,12 +151,50 @@ $(coreboot_module)_configure := \

 # Create a dependency from coreboot to the toolchain.  Use .heads-toolchain to
 # mark that the toolchain was built.
-COREBOOT_TOOLCHAIN=$(build)/$($(coreboot_toolchain_module)_base_dir)/.heads-toolchain
-$(COREBOOT_TOOLCHAIN): $(build)/$($(coreboot_toolchain_module)_base_dir)/.canary
-	$(MAKE) -C "$(build)/$($(coreboot_toolchain_module)_base_dir)" CPUS=$(CPUS) "crossgcc-$(COREBOOT_TARGET)"
+COREBOOT_TOOLCHAIN_DIR=$(build)/$($(coreboot_toolchain_module)_base_dir)
+$(COREBOOT_TOOLCHAIN_DIR)/.heads-toolchain: $(COREBOOT_TOOLCHAIN_DIR)/.canary
+	$(MAKE) -C "$(build)/$($(coreboot_toolchain_module)_base_dir)" CPUS=$(CPUS) "crossgcc-$(COREBOOT_TARGET)" \
+		$($(coreboot_toolchain_module)_toolchain_build_args)
 	touch "$@"

-$(build)/$(coreboot_dir)/.configured: $(COREBOOT_TOOLCHAIN)
+$(build)/$(coreboot_dir)/.configured: $(COREBOOT_TOOLCHAIN_DIR)/.heads-toolchain
+
+## Toolchain packages ##
+# coreboot likes to download its own toolchain packages, but these sources can
+# go down or move also.  Download them ahead of the toolchain build with
+# fetch_source_archive.sh to leverage our mirrors.
+#
+# Create a task for each package.
+# $1 - package name (binutils/gcc/gmp/etc. - check coreboot/util/crossgcc/sum/)
+define coreboot-toolchain-pkg =
+# The package versions and digests aren't duplicated here, we get them
+# from the coreboot source.  This means downloading all packages
+# requires unpacking the coreboot source, but that's preferred over
+# maintaining a duplicate list here for each coreboot version.
+
+# Rule to download the source archive for $1 and place it in the
+# cooreboot directory.  Although there is a specific file produced here
+# (the package), we can't use it as the rule target because we don't
+# know the filename until coreboot is unpacked.
+$(COREBOOT_TOOLCHAIN_DIR)/.heads-crossgcc-pkg-$(1) : $(COREBOOT_TOOLCHAIN_DIR)/.canary
+	WGET="$(WGET)" bin/fetch_coreboot_crossgcc_archive.sh \
+		"$(COREBOOT_TOOLCHAIN_DIR)" "$(1)" "$(packages)"
+	touch "$$@"
+
+# The toolchain build requires this package
+$(COREBOOT_TOOLCHAIN_DIR)/.heads-toolchain: $(COREBOOT_TOOLCHAIN_DIR)/.heads-crossgcc-pkg-$(1)
+# The "packages" target depends on this target, so 'make packages' will
+# include these packages for seeding a mirror.
+packages: $(COREBOOT_TOOLCHAIN_DIR)/.heads-crossgcc-pkg-$(1)
+endef
+
+$(eval $(call coreboot-toolchain-pkg,gmp))
+$(eval $(call coreboot-toolchain-pkg,mpfr))
+$(eval $(call coreboot-toolchain-pkg,mpc))
+$(eval $(call coreboot-toolchain-pkg,binutils))
+$(eval $(call coreboot-toolchain-pkg,gcc))
+$(eval $(call coreboot-toolchain-pkg,nasm))
+$(eval $(call coreboot-toolchain-pkg,iasl))

 # Build with the cross toolchain from the toolchain module (which might be the
 # same coreboot module or a different one).
--- a/patches/coreboot-4.11/0070-crossgcc-iasl-2021-instead-of-2018_fix-old_coreboot-build-on-newer-platforms.patch
+++ b/patches/coreboot-4.11/0070-crossgcc-iasl-2021-instead-of-2018_fix-old_coreboot-build-on-newer-platforms.patch
@ -80,3 +80,8 @@ diff -ruN ./util/crossgcc/patches.orig/acpica-unix-20210105_iasl.patch ./util/cr
 }
 
 build_LLVM() {
+diff -ruN ./util/crossgcc/sum.orig/acpica-unix2-20190703.tar.gz.cksum ./util/crossgcc/sum/acpica-unix2-20190703.tar.gz.cksum
+--- ./util/crossgcc/sum.orig/acpica-unix2-20190703.tar.gz.cksum	2024-01-08 13:19:48.519641977 -0500
+++ ./util/crossgcc/sum/acpica-unix2-20190703.tar.gz.cksum	1969-12-31 19:00:00.000000000 -0500
+@@ -1 +0,0 @@
+-c5594944f933265a53695204a0672d0808e4a580  tarballs/acpica-unix2-20190703.tar.gz