openwrt/.github/workflows/build.yml

name: Build sub target

on:
  workflow_call:
    secrets:
      coverity_api_token:
    inputs:
      container_name:
        type: string
        default: tools
      target:
        required: true
        type: string
      subtarget:
        required: true
        type: string
      testing:
        type: boolean
      build_toolchain:
        type: boolean
      include_feeds:
        type: boolean
      build_full:
        type: boolean
      build_kernel:
        type: boolean
      build_all_modules:
        type: boolean
      build_all_kmods:
        type: boolean
      build_all_boards:
        type: boolean
      use_openwrt_container:
        type: boolean
        default: true
      coverity_project_name:
        type: string
        default: OpenWrt
      coverity_check_packages:
        type: string
      coverity_compiler_template_list:
        type: string
        default: >-
          arm-openwrt-linux-gcc          
      coverity_force_compile_packages:
        type: string
        default: >-
          curl
          libnl
          mbedtls
          wolfssl
          openssl          
      build_external_toolchain:
        type: boolean
      upload_external_toolchain:
        type: boolean

permissions:
  contents: read

jobs:
  setup_build:
    name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
    runs-on: ubuntu-latest
    outputs:
      owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
      container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
      container_name: ${{ steps.determine_tools_container.outputs.container_name }}

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set lower case owner name
        id: lower_owner
        run: |
          OWNER_LC=$(echo "${{ github.repository_owner }}" \
            | tr '[:upper:]' '[:lower:]')

          if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
            OWNER_LC=openwrt
          fi

          echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT          

      # Per branch tools container tag
      # By default stick to latest
      # For official test targetting openwrt stable branch
      # Get the branch or parse the tag and push dedicated tools containers
      # For local test to use the correct container for stable release testing
      # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
      - name: Determine tools container tag
        id: determine_tools_container
        run: |
          CONTAINER_NAME=${{ inputs.container_name }}
          CONTAINER_TAG=latest
          if [ -n "${{ github.base_ref }}" ]; then
            if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
              CONTAINER_TAG="${{ github.base_ref }}"
            fi
          elif [ ${{ github.ref_type }} == "branch" ]; then
            if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
              CONTAINER_TAG=${{ github.ref_name }}
            elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
              CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
            fi
          elif [ ${{ github.ref_type }} == "tag" ]; then
            if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
              CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
            fi
          fi

          if [ "$CONTAINER_NAME" = "toolchain" ]; then
            GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
            GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
            GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
            # Check if container exist
            if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
              CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
            else
              CONTAINER_NAME=tools
            fi
          fi

          echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
          echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
          echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT          

  build:
    name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
    needs: setup_build
    runs-on: ubuntu-latest

    container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}

    permissions:
      contents: read
      packages: read

    steps:
      - name: Checkout master directory
        uses: actions/checkout@v3
        with:
          path: openwrt

      - name: Checkout packages feed
        if: inputs.include_feeds == true
        uses: actions/checkout@v3
        with:
          repository: openwrt/packages
          path: openwrt/feeds/packages

      - name: Checkout luci feed
        if: inputs.include_feeds == true
        uses: actions/checkout@v3
        with:
          repository: openwrt/luci
          path: openwrt/feeds/luci

      - name: Checkout routing feed
        if: inputs.include_feeds == true
        uses: actions/checkout@v3
        with:
          repository: openwrt/routing
          path: openwrt/feeds/routing

      - name: Checkout telephony feed
        if: inputs.include_feeds == true
        uses: actions/checkout@v3
        with:
          repository: openwrt/telephony
          path: openwrt/feeds/telephony

      - name: Parse toolchain file
        if: inputs.build_toolchain == false
        id: parse-toolchain
        working-directory: openwrt
        run: |
          if [ -d /external-toolchain/ ]; then
            echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
            exit 0
          fi

          TOOLCHAIN_PATH=snapshots

          if [ -n "${{ github.base_ref }}" ]; then
            if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
              major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
            fi
          elif [ "${{ github.ref_type }}" = "branch" ]; then
            if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
              major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
            elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
              major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
            fi
          elif [ "${{ github.ref_type }}" = "tag" ]; then
            if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
              major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
            fi
          fi

          if [ -n "$major_ver" ]; then
            git fetch --tags -f
            latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
            if [ -n "$latest_tag" ]; then
              TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
            fi
          fi

          SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
          if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
            TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
            TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
            
            echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
          elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
            TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
            TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')

            echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
          else
            echo "toolchain-type=internal" >> $GITHUB_OUTPUT
          fi

          echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
          echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"          

      - name: Fix permission
        run: |
          chown -R buildbot:buildbot openwrt          

      - name: Prepare prebuilt tools
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          mkdir -p staging_dir build_dir
          ln -s /prebuilt_tools/staging_dir/host staging_dir/host
          ln -s /prebuilt_tools/build_dir/host build_dir/host

          ./scripts/ext-tools.sh --refresh          

      - name: Update & Install feeds
        if: inputs.include_feeds == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          ./scripts/feeds update -a
          ./scripts/feeds install -a          

      - name: Cache ccache
        uses: actions/cache@v3
        with:
          path: openwrt/.ccache
          key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }}
          restore-keys: |
            ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-            

      - name: Download external toolchain/sdk
        if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
            | tar --xz -xf -          

      - name: Configure testing kernel
        if: inputs.testing == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_TESTING_KERNEL=y >> .config          

      - name: Configure all kernel modules
        if: inputs.build_all_kmods == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_ALL_KMODS=y >> .config          

      - name: Configure all modules
        if: inputs.build_all_modules == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_ALL=y >> .config          

      - name: Configure all boards
        if: inputs.build_all_boards == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
          echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
          echo CONFIG_TARGET_ALL_PROFILES=y >> .config          

      - name: Configure external toolchain in container
        if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_DEVEL=y >> .config
          echo CONFIG_AUTOREMOVE=y >> .config
          echo CONFIG_CCACHE=y >> .config

          ./scripts/ext-toolchain.sh \
            --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
            --overwrite-config \
            --config ${{ inputs.target }}/${{ inputs.subtarget }}          

      - name: Configure external toolchain
        if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_DEVEL=y >> .config
          echo CONFIG_AUTOREMOVE=y >> .config
          echo CONFIG_CCACHE=y >> .config

          ./scripts/ext-toolchain.sh \
            --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
            --overwrite-config \
            --config ${{ inputs.target }}/${{ inputs.subtarget }}          

      - name: Adapt external sdk to external toolchain format
        if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
          TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
          OPENWRT_DIR=$(pwd)

          # Find target name from toolchain info.mk
          GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')

          cd $TOOLCHAIN_BIN

          # Revert sdk wrapper scripts applied to all the bins
          for app in $(find . -name "*.bin"); do
            TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
            rm $TARGET_APP
            mv .$TARGET_APP.bin $TARGET_APP
          done

          # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
          cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
          for app in cc gcc g++ c++ cpp ld as ; do
            [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
            ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
          done          

      - name: Configure external toolchain with sdk
        if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_DEVEL=y >> .config
          echo CONFIG_AUTOREMOVE=y >> .config
          echo CONFIG_CCACHE=y >> .config

          ./scripts/ext-toolchain.sh \
            --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
            --overwrite-config \
            --config ${{ inputs.target }}/${{ inputs.subtarget }}          

      - name: Configure internal toolchain
        if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          echo CONFIG_DEVEL=y >> .config
          echo CONFIG_AUTOREMOVE=y >> .config
          echo CONFIG_CCACHE=y >> .config

          echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
          echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config

          make defconfig          

      - name: Show configuration
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: ./scripts/diffconfig.sh

      - name: Build tools
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh

      - name: Build toolchain
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh

      - name: Build Kernel
        if: inputs.build_kernel == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh

      - name: Build Kernel Kmods
        if: inputs.build_kernel == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh

      - name: Build everything
        if: inputs.build_full == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh

      - name: Build external toolchain
        if: inputs.build_external_toolchain == true
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh

      - name: Coverity prepare toolchain
        if: inputs.coverity_check_packages != ''
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
          wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
          echo ' coverity.tar.gz' >> coverity.tar.gz.md5
          md5sum -c coverity.tar.gz.md5

          mkdir cov-analysis-linux64
          tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
          export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH

          for template in ${{ inputs.coverity_compiler_template_list }}; do
            cov-configure --template --comptype gcc --compiler "$template"
          done          

      - name: Clean and recompile packages with Coverity toolchain
        if: inputs.coverity_check_packages != ''
        shell: su buildbot -c "bash {0}"
        working-directory: openwrt
        run: |
          set -o pipefail -o errexit

          coverity_check_packages=(${{ inputs.coverity_check_packages }})
          printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
          make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh

          coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
          printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
          make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh

          printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
          export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
          cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh          

      - name: Upload build to Coverity for analysis
        if: inputs.coverity_check_packages != ''
        shell: su buildbot -c "sh -e {0}"
        working-directory: openwrt
        run: |
          tar czf cov-int.tar.gz ./cov-int
          curl \
            --form token="${{ secrets.coverity_api_token }}" \
            --form email="contact@openwrt.org" \
            --form file=@cov-int.tar.gz \
            --form version="${{ github.ref_name }}-${{ github.sha }}" \
            --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
            "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"          

      - name: Upload logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
          path: "openwrt/logs"

      - name: Find external toolchain name
        id: get-toolchain-name
        if: inputs.upload_external_toolchain == true
        working-directory: openwrt
        run: |
          TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
          echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT          

      - name: Upload prebuilt toolchain
        if: inputs.upload_external_toolchain == true
        uses: actions/upload-artifact@v3
        with:
          name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
          path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
          retention-days: 1