diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95d19f4c4e0..efaf7594033 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,6 +60,8 @@ on: ccache_type: type: string default: kernel + upload_ccache_cache: + type: boolean permissions: contents: read @@ -232,6 +234,20 @@ jobs: echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV" echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV" + - name: Download and extract ccache cache from s3 + id: restore-ccache-cache-s3 + if: inputs.use_ccache_cache == true + working-directory: openwrt + run: | + ENDPOINT=https://storage.googleapis.com + BUCKET=openwrt-ci-cache + CCACHE_TAR=ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar + + if curl -o /dev/null -s --head --fail $ENDPOINT/$BUCKET/$CCACHE_TAR; then + wget -O - $ENDPOINT/$BUCKET/$CCACHE_TAR | tar -xf - + echo "cache-hit=true" >> $GITHUB_OUTPUT + fi + - name: Fix permission run: | chown -R buildbot:buildbot openwrt @@ -256,7 +272,7 @@ jobs: - name: Restore ccache cache id: restore-ccache-cache - if: inputs.use_ccache_cache == true + if: inputs.use_ccache_cache == true && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: actions/cache/restore@v3 with: path: openwrt/.ccache @@ -498,7 +514,8 @@ jobs: path: "openwrt/logs" - name: Delete already present ccache cache - if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && github.event_name == 'push' + if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && + github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: octokit/request-action@v2.x with: route: DELETE /repos/{repository}/actions/caches?key={key} @@ -508,12 +525,29 @@ jobs: INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }} - name: Save ccache cache - if: inputs.use_ccache_cache == true && github.event_name == 'push' + if: inputs.use_ccache_cache == true && github.event_name == 'push' && + steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: path: openwrt/.ccache key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }} + - name: Archive ccache + if: inputs.use_ccache_cache == true && github.event_name == 'push' && + inputs.upload_ccache_cache == true + shell: su buildbot -c "sh -e {0}" + working-directory: openwrt + run: tar -cf ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar .ccache + + - name: Upload ccache cache + if: inputs.use_ccache_cache == true && github.event_name == 'push' && + inputs.upload_ccache_cache == true + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.target }}-${{ inputs.subtarget }}-ccache-cache + path: openwrt/ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar + retention-days: 1 + - name: Find external toolchain name id: get-toolchain-name if: inputs.upload_external_toolchain == true diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml index d9ab8581c9b..5f18a39c553 100644 --- a/.github/workflows/kernel.yml +++ b/.github/workflows/kernel.yml @@ -117,6 +117,7 @@ jobs: subtarget: ${{ matrix.subtarget }} build_kernel: true build_all_kmods: true + upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }} check-kernel-patches: name: Check Kernel patches @@ -133,3 +134,20 @@ jobs: target: ${{ matrix.target }} subtarget: ${{ matrix.subtarget }} + upload-ccache-cache-in-s3: + if: github.event_name == 'push' && github.repository_owner == 'openwrt' + name: Upload ccache cache to s3 + needs: [determine_targets, build] + strategy: + fail-fast: False + matrix: + include: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}} + secrets: + s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }} + s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }} + uses: ./.github/workflows/upload-file-s3.yml + with: + endpoint: https://storage.googleapis.com + bucket: openwrt-ci-cache + download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache + filename: ccache-kernel-${{ matrix.target }}-${{ matrix.subtarget }}.tar diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 58bbcd01311..fe2898cacdb 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -54,4 +54,27 @@ jobs: build_all_modules: true build_full: true ccache_type: packages + upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }} + + upload-ccache-cache-in-s3: + if: github.event_name == 'push' && github.repository_owner == 'openwrt' + name: Upload ccache cache to s3 + needs: build + strategy: + fail-fast: False + matrix: + include: + - target: malta + subtarget: be + - target: x86 + subtarget: 64 + secrets: + s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }} + s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }} + uses: ./.github/workflows/upload-file-s3.yml + with: + endpoint: https://storage.googleapis.com + bucket: openwrt-ci-cache + download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache + filename: ccache-packages-${{ matrix.target }}-${{ matrix.subtarget }}.tar diff --git a/.github/workflows/upload-file-s3.yml b/.github/workflows/upload-file-s3.yml new file mode 100644 index 00000000000..6bcb172468d --- /dev/null +++ b/.github/workflows/upload-file-s3.yml @@ -0,0 +1,46 @@ +name: Upload File to S3 + +on: + workflow_call: + secrets: + s3_access_key: + s3_secret_key: + inputs: + endpoint: + required: true + type: string + bucket: + required: true + type: string + download_id: + required: true + type: string + filename: + required: true + type: string + +jobs: + upload-file-in-s3: + name: Upload file in S3 + runs-on: ubuntu-latest + + steps: + - name: Install minio + run: | + curl https://dl.min.io/client/mc/release/linux-amd64/mc \ + --create-dirs \ + -o $GITHUB_WORKSPACE/minio-binaries/mc + + chmod +x $GITHUB_WORKSPACE/minio-binaries/mc + echo $GITHUB_WORKSPACE/minio-binaries/ >> $GITHUB_PATH + + - name: Setup minio + run: mc alias set s3 ${{ inputs.endpoint }} ${{ secrets.s3_access_key }} ${{ secrets.s3_secret_key }} + + - name: Download file + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.download_id }} + + - name: Upload file to s3 + run: mc cp ${{ inputs.filename }} s3/${{ inputs.bucket }}/