name: Build sub target on: workflow_call: secrets: coverity_api_token: inputs: container_name: type: string default: tools target: required: true type: string subtarget: required: true type: string testing: type: boolean build_toolchain: type: boolean include_feeds: type: boolean build_full: type: boolean build_kernel: type: boolean build_all_modules: type: boolean build_all_kmods: type: boolean build_all_boards: type: boolean use_openwrt_container: type: boolean default: true coverity_project_name: type: string default: OpenWrt coverity_check_packages: type: string coverity_compiler_template_list: type: string default: >- arm-openwrt-linux-gcc coverity_force_compile_packages: type: string default: >- curl libnl mbedtls wolfssl openssl build_external_toolchain: type: boolean upload_external_toolchain: type: boolean use_ccache_cache: type: boolean default: true ccache_type: type: string default: kernel upload_ccache_cache: type: boolean permissions: contents: read jobs: setup_build: name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }} runs-on: ubuntu-latest outputs: owner_lc: ${{ steps.lower_owner.outputs.owner_lc }} container_tag: ${{ steps.determine_tools_container.outputs.container_tag }} container_name: ${{ steps.determine_tools_container.outputs.container_name }} steps: - name: Checkout uses: actions/checkout@v3 - name: Set lower case owner name id: lower_owner run: | OWNER_LC=$(echo "${{ github.repository_owner }}" \ | tr '[:upper:]' '[:lower:]') if [ ${{ inputs.use_openwrt_container }} == "true" ]; then OWNER_LC=openwrt fi echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT # Per branch tools container tag # By default stick to latest # For official test targetting openwrt stable branch # Get the branch or parse the tag and push dedicated tools containers # For local test to use the correct container for stable release testing # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]- - name: Determine tools container tag id: determine_tools_container run: | CONTAINER_NAME=${{ inputs.container_name }} CONTAINER_TAG=latest if [ -n "${{ github.base_ref }}" ]; then if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then CONTAINER_TAG="${{ github.base_ref }}" fi elif [ ${{ github.ref_type }} == "branch" ]; then if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then CONTAINER_TAG=${{ github.ref_name }} elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')" fi elif [ ${{ github.ref_type }} == "tag" ]; then if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')" fi fi if [ "$CONTAINER_NAME" = "toolchain" ]; then GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64) GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}" GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG" # Check if container exist if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG" else CONTAINER_NAME=tools fi fi echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG" echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT build: name: Build ${{ inputs.target }}/${{ inputs.subtarget }} needs: setup_build runs-on: ubuntu-latest container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }} permissions: contents: read packages: read actions: write steps: - name: Checkout master directory uses: actions/checkout@v3 with: path: openwrt - name: Checkout packages feed if: inputs.include_feeds == true uses: actions/checkout@v3 with: repository: openwrt/packages path: openwrt/feeds/packages - name: Checkout luci feed if: inputs.include_feeds == true uses: actions/checkout@v3 with: repository: openwrt/luci path: openwrt/feeds/luci - name: Checkout routing feed if: inputs.include_feeds == true uses: actions/checkout@v3 with: repository: openwrt/routing path: openwrt/feeds/routing - name: Checkout telephony feed if: inputs.include_feeds == true uses: actions/checkout@v3 with: repository: openwrt/telephony path: openwrt/feeds/telephony - name: Parse toolchain file if: inputs.build_toolchain == false id: parse-toolchain working-directory: openwrt run: | if [ -d /external-toolchain/ ]; then echo "toolchain-type=external_container" >> $GITHUB_OUTPUT exit 0 fi TOOLCHAIN_PATH=snapshots if [ -n "${{ github.base_ref }}" ]; then if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')" fi elif [ "${{ github.ref_type }}" = "branch" ]; then if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')" elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')" fi elif [ "${{ github.ref_type }}" = "tag" ]; then if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')" fi fi if [ -n "$major_ver" ]; then git fetch --tags -f latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)" if [ -n "$latest_tag" ]; then TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//') fi fi SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums" if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")" TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p') echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")" TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p') echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT else echo "toolchain-type=internal" >> $GITHUB_OUTPUT fi echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV" echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV" - name: Download and extract ccache cache from s3 id: restore-ccache-cache-s3 if: inputs.use_ccache_cache == true working-directory: openwrt run: | ENDPOINT=https://storage.googleapis.com BUCKET=openwrt-ci-cache CCACHE_TAR=ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar if curl -o /dev/null -s --head --fail $ENDPOINT/$BUCKET/$CCACHE_TAR; then wget -O - $ENDPOINT/$BUCKET/$CCACHE_TAR | tar -xf - echo "cache-hit=true" >> $GITHUB_OUTPUT fi - name: Fix permission run: | chown -R buildbot:buildbot openwrt - name: Prepare prebuilt tools shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | mkdir -p staging_dir build_dir ln -s /prebuilt_tools/staging_dir/host staging_dir/host ln -s /prebuilt_tools/build_dir/host build_dir/host ./scripts/ext-tools.sh --refresh - name: Update & Install feeds if: inputs.include_feeds == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | ./scripts/feeds update -a ./scripts/feeds install -a - name: Restore ccache cache id: restore-ccache-cache if: inputs.use_ccache_cache == true && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: actions/cache/restore@v3 with: path: openwrt/.ccache key: ccache-${{ inputs.ccache_type }}-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }} restore-keys: | ccache-${{ inputs.ccache_type }}-${{ inputs.target }}/${{ inputs.subtarget }}- - name: Import GPG keys shell: su buildbot -c "sh -e {0}" if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container' run: gpg --receive-keys 0xCD84BCED626471F1 0x1D53D1877742E911 0xCD54E82DADB3684D - name: Download external toolchain/sdk if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | wget https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz wget https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums.asc wget https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums gpg --with-fingerprint --verify sha256sums.asc sha256sum --check --ignore-missing sha256sums tar --xz -xf ${{ env.TOOLCHAIN_FILE }}.tar.xz rm ${{ env.TOOLCHAIN_FILE }}.tar.xz sha256sums - name: Configure testing kernel if: inputs.testing == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_TESTING_KERNEL=y >> .config - name: Configure all kernel modules if: inputs.build_all_kmods == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_ALL_KMODS=y >> .config - name: Configure all modules if: inputs.build_all_modules == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_ALL=y >> .config - name: Configure all boards if: inputs.build_all_boards == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_TARGET_MULTI_PROFILE=y >> .config echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config echo CONFIG_TARGET_ALL_PROFILES=y >> .config # ccache for some reason have problem detecting compiler type # with external toolchain. This cause the complete malfunction # of ccache with the result of tons of unsupported compiler # option error. # To fix this force compiler type to gcc. - name: Configure ccache and apply fixes if: inputs.use_ccache_cache == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt env: SYSTEM_CCACHE_CONF: staging_dir/host/etc/ccache.conf run: | touch $SYSTEM_CCACHE_CONF echo compiler_type=gcc >> $SYSTEM_CCACHE_CONF echo CONFIG_CCACHE=y >> .config - name: Configure external toolchain in container if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_DEVEL=y >> .config echo CONFIG_AUTOREMOVE=y >> .config ./scripts/ext-toolchain.sh \ --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \ --overwrite-config \ --config ${{ inputs.target }}/${{ inputs.subtarget }} - name: Configure external toolchain if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_DEVEL=y >> .config echo CONFIG_AUTOREMOVE=y >> .config ./scripts/ext-toolchain.sh \ --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \ --overwrite-config \ --config ${{ inputs.target }}/${{ inputs.subtarget }} - name: Adapt external sdk to external toolchain format if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain) TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin OPENWRT_DIR=$(pwd) # Find target name from toolchain info.mk GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/') cd $TOOLCHAIN_BIN # Revert sdk wrapper scripts applied to all the bins for app in $(find . -name "*.bin"); do TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/') rm $TARGET_APP mv .$TARGET_APP.bin $TARGET_APP done # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh for app in cc gcc g++ c++ cpp ld as ; do [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app done - name: Configure external toolchain with sdk if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_DEVEL=y >> .config echo CONFIG_AUTOREMOVE=y >> .config ./scripts/ext-toolchain.sh \ --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \ --overwrite-config \ --config ${{ inputs.target }}/${{ inputs.subtarget }} - name: Configure internal toolchain if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | echo CONFIG_DEVEL=y >> .config echo CONFIG_AUTOREMOVE=y >> .config echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config make defconfig - name: Show configuration shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: ./scripts/diffconfig.sh - name: Build tools shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Build toolchain shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Build Kernel if: inputs.build_kernel == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Build Kernel Kmods if: inputs.build_kernel == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Build everything if: inputs.build_full == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Build external toolchain if: inputs.build_external_toolchain == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Coverity prepare toolchain if: inputs.coverity_check_packages != '' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5 echo ' coverity.tar.gz' >> coverity.tar.gz.md5 md5sum -c coverity.tar.gz.md5 mkdir cov-analysis-linux64 tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH for template in ${{ inputs.coverity_compiler_template_list }}; do cov-configure --template --comptype gcc --compiler "$template" done - name: Clean and recompile packages with Coverity toolchain if: inputs.coverity_check_packages != '' shell: su buildbot -c "bash {0}" working-directory: openwrt run: | set -o pipefail -o errexit coverity_check_packages=(${{ inputs.coverity_check_packages }}) printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}" make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }}) printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}" make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}" export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh - name: Upload build to Coverity for analysis if: inputs.coverity_check_packages != '' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | tar czf cov-int.tar.gz ./cov-int curl \ --form token="${{ secrets.coverity_api_token }}" \ --form email="contact@openwrt.org" \ --form file=@cov-int.tar.gz \ --form version="${{ github.ref_name }}-${{ github.sha }}" \ --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \ "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}" - name: Upload logs if: failure() uses: actions/upload-artifact@v3 with: name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs path: "openwrt/logs" - name: Delete already present ccache cache if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: octokit/request-action@v2.x with: route: DELETE /repos/{repository}/actions/caches?key={key} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} INPUT_REPOSITORY: ${{ github.repository }} INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }} - name: Save ccache cache if: inputs.use_ccache_cache == true && github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: path: openwrt/.ccache key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }} - name: Archive ccache if: inputs.use_ccache_cache == true && github.event_name == 'push' && inputs.upload_ccache_cache == true shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: tar -cf ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar .ccache - name: Upload ccache cache if: inputs.use_ccache_cache == true && github.event_name == 'push' && inputs.upload_ccache_cache == true uses: actions/upload-artifact@v3 with: name: ${{ inputs.target }}-${{ inputs.subtarget }}-ccache-cache path: openwrt/ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar retention-days: 1 - name: Find external toolchain name id: get-toolchain-name if: inputs.upload_external_toolchain == true working-directory: openwrt run: | TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain) echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT - name: Upload prebuilt toolchain if: inputs.upload_external_toolchain == true uses: actions/upload-artifact@v3 with: name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }} retention-days: 1