Reuse Agent artifacts if nothing in src/agent changes (#2115)

The agent build takes most of the CI runtime, so improve it by only rebuilding if the pre-reqs or something inside `src/agent` changes.

We will always skip the cache for builds on tags and from the `main` branch, so that version is stamped correctly there.

.github/workflows/ci.yml

@@ -24,7 +24,28 @@ jobs:
       run: |
         rustup default 1.62 
         rustup component add clippy rustfmt
+    - name: Get Rust version & build version
+      shell: bash
+      run: |
+        echo "::set-output name=RUST_VERSION::$(rustc --version)"
+        VERSION=$(src/ci/get-version.sh)
+        # it's a release build if version doesn't have a hyphen in it
+        IS_RELEASE_BUILD=$(if [[ "$VERSION" =~ '-' ]]; then echo 'false'; else echo 'true'; fi)
+        echo "::set-output name=RELEASE_BUILD::$IS_RELEASE_BUILD"
+      id: rust-version
+    - name: Rust artifact cache
+      id: cache-agent-artifacts
+      # don't cache the rust agent for relase builds as the version number will be incorrect
+      # don't cache on builds on the main branch for deployment to canary/nightly
+      if: steps.rust-version.outputs.RELEASE_BUILD == 'false' && github.ref_name != 'main'
+      uses: actions/cache@v3
+      with:
+        # if nothing has changed inside src/agent, we can reuse the artifacts directory
+        path: artifacts
+        key: agent-artifacts-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}-${{ hashFiles('src/agent/**/*') }}
+        # note: also including the ACTIONS_CACHE_KEY_DATE to rebuild if the Prereq Cache is invalidated
     - name: Rust Prereq Cache
+      if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
       uses: actions/cache@v2
       id: cache-rust-prereqs
       with:
@@ -32,27 +53,28 @@ jobs:
           ~/.cargo/registry
           ~/.cargo/git
           ~/.cargo/bin
-        key: rust-${{ runner.os }}-${{ env.ACTIONS_CACHE_KEY_DATE }}
+        key: rust-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}
     - name: Install Rust Prereqs
-      if: steps.cache-rust-prereqs.outputs.cache-hit != 'true'
+      if: steps.cache-rust-prereqs.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
       shell: bash
       run: src/ci/rust-prereqs.sh
     - name: Rust Compile Cache
+      if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
       uses: actions/cache@v2
       with:
         path: |
           sccache
           src/agent/target
-        key: agent-${{ runner.os }}-${{ hashFiles('src/agent/Cargo.lock') }}-${{ env.ACTIONS_CACHE_KEY_DATE }}
+        key: agent-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}-${{ hashFiles('src/agent/Cargo.lock') }}
         restore-keys: |
-          agent-${{ runner.os }}-${{ hashFiles('src/agent/Cargo.lock') }}-
-          agent-${{ runner.os }}-
+          agent-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}-
     - name: Linux Prereqs
       run: |
         sudo apt-get -y update
         sudo apt-get -y install libssl1.0-dev libunwind-dev
-      if: "${{ runner.os == 'Linux' }}"
+      if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
     - run: src/ci/agent.sh
+      if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
       shell: bash
     - uses: actions/upload-artifact@v2.2.2
       with:

src/api-service/__app__/onefuzzlib/workers/nodes.py

@@ -5,6 +5,7 @@
 
 import datetime
 import logging
+import os
 from typing import List, Optional, Tuple
 from uuid import UUID
 
@@ -131,6 +132,10 @@ class Node(BASE_NODE, ORMMixin):
 
     @classmethod
     def mark_outdated_nodes(cls) -> None:
+        # if outdated agents are allowed, do not attempt to update
+        if os.environ["ONEFUZZ_ALLOW_OUTDATED_AGENT"] == "true":
+            return
+
         # ony update 500 nodes at a time to mitigate timeout issues
         outdated = cls.search_outdated(exclude_update_scheduled=True, num_results=500)
         for node in outdated:
@@ -263,7 +268,7 @@ class Node(BASE_NODE, ORMMixin):
         from .pools import Pool
         from .scalesets import Scaleset
 
-        if self.is_outdated():
+        if self.is_outdated() and os.environ["ONEFUZZ_ALLOW_OUTDATED_AGENT"] != "true":
             logging.info(
                 "can_process_new_work agent and service versions differ, "
                 "stopping node. "