ExternalVendorCode/heads
1
0
Fork 0
mirror of https://github.com/linuxboot/heads.git synced 2024-12-18 20:47:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1734 from tlaurion/JonathonHall-Purism-module_checkout_submodules-flake_moreutils_for_timestamps_CircleCI_intermediary_arch_musl-cross-make_step

Browse Source 
Fix master's CircleCI builds (Bandwidth throttling making all boards depending on coreboot forks to exceed 1h step's max build time)
This commit is contained in:
Thierry Laurion 2024-07-26 15:29:32 -04:00 committed by GitHub
commit 60ec2b9b45
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 53 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
.circleci/config.yml
View File

@ -11,41 +11,41 @@ commands:
type: string type: string
steps: steps:
- run: - run:
name: Make Board (console logs not contiguous because multiple CPUS used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step) name: Make Board (console logs not contiguous because multiple CPUs used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step)
command: | command: |
echo "Sourcing /devenv.sh since docker entrypoint doesn't do it as expected" echo "Sourcing /devenv.sh since docker entrypoint doesn't do it as expected"
source /devenv.sh source /devenv.sh
rm -rf build/<<parameters.arch>>/<<parameters.target>>/* build/<<parameters.arch>>/log/* rm -rf build/<< parameters.arch >>/<< parameters.target >>/* build/<< parameters.arch >>/log/*
make V=1 BOARD=<<parameters.target>> <<parameters.subcommand>> || touch ./tmpDir/failed_build make V=1 BOARD=<< parameters.target >> << parameters.subcommand >> | ts || touch ./tmpDir/failed_build
no_output_timeout: 3h no_output_timeout: 3h
- run: - run:
name: Output hashes name: Output hashes
command: | command: |
cat build/<<parameters.arch>>/<<parameters.target>>/hashes.txt || echo "No hashes.txt for this build step..." cat build/<< parameters.arch >>/<< parameters.target >>/hashes.txt || echo "No hashes.txt for this build step..."
- run: - run:
name: Output sizes name: Output sizes
command: | command: |
cat build/<<parameters.arch>>/<<parameters.target>>/sizes.txt || echo "No sizes.txt for this build step..." cat build/<< parameters.arch >>/<< parameters.target >>/sizes.txt || echo "No sizes.txt for this build step..."
- run: - run:
name: Archiving build logs. name: Archiving build logs.
command: | command: |
tar zcvf build/<<parameters.arch>>/<<parameters.target>>/logs.tar.gz $(find build/ -name "*.log") tar zcvf build/<< parameters.arch >>/<< parameters.target >>/logs.tar.gz $(find build/ -name "*.log")
- run: - run:
name: Output build failing logs name: Output build failing logs
command: | command: |
if [[ -f ./tmpDir/failed_build ]]; then if [[ -f ./tmpDir/failed_build ]]; then
find "./build/<<parameters.arch>>/" -name "*.log" -type f -mmin -1 -exec tail -n +1 '{}' + find "./build/<< parameters.arch >>/" -name "*.log" -type f -mmin -1 -exec tail -n +1 '{}' +
exit 1 exit 1
else else
echo "Step hasn't failed. Continuing with next step..." echo "Step hasn't failed. Continuing with next step..."
fi fi
- store_artifacts: - store_artifacts:
path: build/<<parameters.arch>>/<<parameters.target>> path: build/<< parameters.arch >>/<< parameters.target >>
jobs: jobs:
prep_env: prep_env:
docker: docker:
- image: tlaurion/heads-dev-env:v0.2.0 - image: tlaurion/heads-dev-env:v0.2.1
resource_class: large resource_class: large
working_directory: ~/heads working_directory: ~/heads
steps: steps:
@ -73,7 +73,7 @@ jobs:
- restore_cache: - restore_cache:
# First matched/found key wins and following keys are not tried # First matched/found key wins and following keys are not tried
keys: keys:
# Cache for matching modules digest, validated to be exactly the same as in github current commit. # Cache for matching modules digest, validated to be exactly the same as in GitHub current commit.
# This cache was made on top of below caches, if previously existing. # This cache was made on top of below caches, if previously existing.
# If no module definition changed, we reuse this one # If no module definition changed, we reuse this one
- nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} - nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
@ -111,7 +111,7 @@ jobs:
build_and_persist: build_and_persist:
docker: docker:
- image: tlaurion/heads-dev-env:v0.2.0 - image: tlaurion/heads-dev-env:v0.2.1
resource_class: large resource_class: large
working_directory: ~/heads working_directory: ~/heads
parameters: parameters:
@ -126,20 +126,20 @@ jobs:
- attach_workspace: - attach_workspace:
at: ~/heads at: ~/heads
- build_board: - build_board:
arch: <<parameters.arch>> arch: << parameters.arch >>
target: <<parameters.target>> target: << parameters.target >>
subcommand: <<parameters.subcommand>> subcommand: << parameters.subcommand >>
- persist_to_workspace: - persist_to_workspace:
root: ~/heads root: ~/heads
paths: paths:
- packages/<<parameters.arch>> - packages/<< parameters.arch >>
- build/<<parameters.arch>> - build/<< parameters.arch >>
- crossgcc/<<parameters.arch>> - crossgcc/<< parameters.arch >>
- install/<<parameters.arch>> - install/<< parameters.arch >>
build: build:
docker: docker:
- image: tlaurion/heads-dev-env:v0.2.0 - image: tlaurion/heads-dev-env:v0.2.1
resource_class: large resource_class: large
working_directory: ~/heads working_directory: ~/heads
parameters: parameters:
@ -160,7 +160,7 @@ jobs:
save_cache: save_cache:
docker: docker:
- image: tlaurion/heads-dev-env:v0.2.0 - image: tlaurion/heads-dev-env:v0.2.1
resource_class: large resource_class: large
working_directory: ~/heads working_directory: ~/heads
steps: steps:
@ -176,25 +176,21 @@ jobs:
- crossgcc - crossgcc
- packages - packages
- save_cache: - save_cache:
# Generate cache for the same coreboot mnd musl-cross-make modules definition if hash is not previously existing # Generate cache for the same coreboot and musl-cross-make modules definition if hash is not previously existing
# CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names
key: nix-docker-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} key: nix-docker-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }}
paths: paths:
- build/ppc64/coreboot-talos_2 - build/ppc64/coreboot-talos_2
- build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1
- build/x86/coreboot-4.11 - build/x86/coreboot-4.11
- build/x86/coreboot-4.13 - build/x86/coreboot-24.02.01
- build/x86/coreboot-4.14 - build/x86/coreboot-dasharo
- build/x86/coreboot-4.15
- build/x86/coreboot-4.17
- build/x86/coreboot-4.22.01
- build/x86/coreboot-nitrokey
- build/x86/coreboot-purism - build/x86/coreboot-purism
- build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1
- crossgcc - crossgcc
- packages - packages
- save_cache: - save_cache:
#Generate cache for the exact same modules definitions if hash is not previously existing # Generate cache for the exact same modules definitions if hash is not previously existing
key: nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} key: nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
paths: paths:
- build - build
@ -208,16 +204,33 @@ workflows:
jobs: jobs:
- prep_env - prep_env
# This step builds musl-cross for x86 architecture, which will be used by subsequent x86 board builds
- build_and_persist:
name: x86-musl-cross
target: x230-hotp-maximized
subcommand: "musl-cross"
requires:
- prep_env
# This step builds musl-cross for ppc64 architecture, which will be used by subsequent ppc64 board builds
- build_and_persist:
name: ppc64-musl-cross
arch: ppc64
target: talos-2
subcommand: "musl-cross"
requires:
- prep_env
# Below, sequentially build one board for each coreboot version. # Below, sequentially build one board for each coreboot version.
# The last board in the sequence is the dependency for the parallel boards built at the end, and also save_cache. # The last board in the sequence is the dependency for the parallel boards built at the end, and also save_cache.
# coreboot 4.22.01 # coreboot 24.02.01
- build_and_persist: - build_and_persist:
name: x230-hotp-maximized name: x230-hotp-maximized
target: x230-hotp-maximized target: x230-hotp-maximized
subcommand: "" subcommand: ""
requires: requires:
- prep_env - x86-musl-cross
# coreboot purism # coreboot purism
- build_and_persist: - build_and_persist:
@ -237,7 +250,7 @@ workflows:
target: nitropad-nv41 target: nitropad-nv41
subcommand: "" subcommand: ""
requires: requires:
- prep_env - x86-musl-cross
# coreboot talos_2 # coreboot talos_2
- build_and_persist: - build_and_persist:
@ -246,7 +259,7 @@ workflows:
target: talos-2 target: talos-2
subcommand: "" subcommand: ""
requires: requires:
- prep_env - ppc64-musl-cross
# coreboot 4.11 # coreboot 4.11
- build_and_persist: - build_and_persist:
@ -254,7 +267,7 @@ workflows:
target: librem_l1um target: librem_l1um
subcommand: "" subcommand: ""
requires: requires:
- prep_env - x86-musl-cross
# Cache one workspace per architecture # Cache one workspace per architecture
# Make sure workspace caches are chainloaded and the last in chain for an arch is saved # Make sure workspace caches are chainloaded and the last in chain for an arch is saved
@ -263,10 +276,7 @@ workflows:
- talos-2 - talos-2
- librem_14 - librem_14
#
# Those onboarding new boards should add their entries below. # Those onboarding new boards should add their entries below.
#
# coreboot 4.22.01 boards # coreboot 4.22.01 boards
- build: - build:
name: x220-hotp-maximized name: x220-hotp-maximized
@ -513,31 +523,3 @@ workflows:
subcommand: "" subcommand: ""
requires: requires:
- librem_l1um - librem_l1um
# - build:
# name: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard
# target: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard
# subcommand: ""
# requires:
# - UNMAINTAINED_kgpe-d16_workstation
# - build:
# name: UNMAINTAINED_kgpe-d16_server
# target: UNMAINTAINED_kgpe-d16_server
# subcommand: ""
# requires:
# - UNMAINTAINED_kgpe-d16_workstation
# - build:
# name: UNMAINTAINED_kgpe-d16_server-whiptail
# target: UNMAINTAINED_kgpe-d16_server-whiptail
# subcommand: ""
# requires:
# - UNMAINTAINED_kgpe-d16_workstation
# - build:
# name: librem_l1um
# target: librem_l1um
# subcommand: ""
# requires:
# - librem_14

6
Makefile
View File

@ -378,7 +378,9 @@ define define_module =
# First time: # First time:
# Checkout the tree instead and create the canary file with repo and # Checkout the tree instead and create the canary file with repo and
# revision so that we know that the files are all present and their # revision so that we know that the files are all present and their
# version. # version. Submodules are _not_ checked out, because coreboot has
# many submodules that won't be used, let coreboot check out its own
# submodules during build
# #
# Other times: # Other times:
# If .canary contains the same repo and revision combination, do nothing. # If .canary contains the same repo and revision combination, do nothing.
@ -392,7 +394,7 @@ define define_module =
$(build)/$($1_base_dir)/.canary: FORCE $(build)/$($1_base_dir)/.canary: FORCE
if [ ! -e "$$@" ]; then \ if [ ! -e "$$@" ]; then \
git clone $($1_repo) "$(build)/$($1_base_dir)"; \ git clone $($1_repo) "$(build)/$($1_base_dir)"; \
git -C "$(build)/$($1_base_dir)" reset --hard $($1_commit_hash) && git submodule update --init --checkout; \ git -C "$(build)/$($1_base_dir)" reset --hard $($1_commit_hash); \
echo -n '$($1_repo)|$($1_commit_hash)' > "$$@"; \ echo -n '$($1_repo)|$($1_commit_hash)' > "$$@"; \
elif [ "$$$$(cat "$$@")" != '$($1_repo)|$($1_commit_hash)' ]; then \ elif [ "$$$$(cat "$$@")" != '$($1_repo)|$($1_commit_hash)' ]; then \
echo "Switching $1 to $($1_repo) at $($1_commit_hash)" && \ echo "Switching $1 to $($1_repo) at $($1_commit_hash)" && \

1
flake.nix
View File

@ -79,6 +79,7 @@
diffoscopeMinimal # Not sure exactly what is packed here, let's try. Might need diffoscope if something is missing diffoscopeMinimal # Not sure exactly what is packed here, let's try. Might need diffoscope if something is missing
gnupg #to inject public key inside of qemu create rom through inject_gpg target of targets/qemu.mk TODO: remove when pflash supported by flashrom + modify code gnupg #to inject public key inside of qemu create rom through inject_gpg target of targets/qemu.mk TODO: remove when pflash supported by flashrom + modify code
less # so 'git log' is usable less # so 'git log' is usable
moreutils # so that 'make 2>&1 | ts' can give timestamps
] ++ [ ] ++ [
# Tools for handling binary blobs in their compressed state. (blobs/xx30/vbios_[tw]530.sh) # Tools for handling binary blobs in their compressed state. (blobs/xx30/vbios_[tw]530.sh)
bundler bundler