Merge pull request #816 from nsacyber/v3_issue_794_ci-error-check

Fixes CI test error checking

.ci/system-tests/container/pc_setup.sh

@@ -86,7 +86,7 @@ dnf install -y unzip &> /dev/null
 
 # Step 2: Unpack the dmi files.
 echo "dmi file used was $dmiZip"
-unzip -o "$dmiZip" -d $HIRS_CI_TEST_ROOT
+unzip -o "$dmiZip" -d $HIRS_CI_TEST_ROOT > /dev/null 2>&1
 
 # Step 3: Copy the platform cert to tcg folder and or upload it to the ACA
 if [[ ! -d $pcDir ]]; then
@@ -103,7 +103,7 @@ pushd $pcDir > /dev/null
       fi
       if [ "$UPLOAD_ARTIFACTS" = YES ]; then
         echo "Uploading $cert to $SERVER_PCERT_POST"
-        curl -k -F "file=@$cert" $SERVER_PCERT_POST
+        curl -k -F "file=@$cert" $SERVER_PCERT_POST > /dev/null 2>&1
       fi
     done
   fi

.ci/system-tests/container/rim_setup.sh

@@ -84,7 +84,7 @@ pushd $swidDir > /dev/null
       fi
       if [ "$UPLOAD_ARTIFACTS" = YES ]; then
         echo "Uploading $swidtag to $SERVER_RIM_POST"
-        curl -k -F "file=@$swidtag" $SERVER_RIM_POST
+        curl -k -F "file=@$swidtag" $SERVER_RIM_POST > /dev/null  2>&1
       fi
     done
   fi
@@ -102,7 +102,7 @@ pushd $rimDir > /dev/null
       fi
       if [ "$UPLOAD_ARTIFACTS" = YES ]; then
         echo "Uploading $rim to $SERVER_RIM_POST"
-        curl -k -F "file=@$rim" $SERVER_RIM_POST
+        curl -k -F "file=@$rim" $SERVER_RIM_POST > /dev/null  2>&1
       fi
     done
   fi

.ci/system-tests/run_system_tests.sh

@@ -6,7 +6,6 @@
 #    1. Uncomment the "cd ../.." line below to make working directory = /HIRS/
 #    2. Run with the desired HIRS branch as an argument (i.e. $./run_system_tests.sh main)
 ##########################################################################################
-#cd ../..
 
 # Setting variables
 aca_container=hirs-aca1
@@ -49,13 +48,3 @@ docker exec $tpm2_container sh -c "mkdir -p /HIRS/logs/provisioner/ && cp -ap hi
 echo "*** Exiting and removing Docker containers and network ..."
 docker compose -f ./.ci/docker/docker-compose-system-test.yml down -v
 
-# Return container exit code
-if [[ ${TEST_STATUS} == "0" ]]; then
-    echo "******** SUCCESS: System Tests for TPM 2.0 passed ********"
-    echo "TEST_STATUS=0" >> $GITHUB_ENV
-    exit 0;
-  else
-    echo "******** FAILURE: System Tests for TPM 2.0 failed ********"
-    echo "TEST_STATUS=1" >> $GITHUB_ENV
-    exit 1
-fi

.ci/system-tests/sys_test_common.sh

@@ -70,15 +70,15 @@ uploadTrustedCerts() {
   # Upload CA Cert from IBMTSS Tools
   echo "Uploading Trust Certificates to ${HIRS_ACA_HOSTNAME}:${HIRS_ACA_PORTAL_PORT}"
   echo "Uploading the EK Certificate CA(s)..."
-  docker exec -i $tpm2_container /bin/bash -c "curl -k -F 'file=@/ibmtss/utils/certificates/cacert.pem' $SERVER_CACERT_POST"
+  docker exec -i $tpm2_container /bin/bash -c "curl -k -F 'file=@/ibmtss/utils/certificates/cacert.pem' $SERVER_CACERT_POST" > /dev/null 2>&1
   echo "...done"
   # Upload Trusted Certs from HIRS
   echo "Uploading the Platform Certificate CA(s)..."
-  docker exec -i $aca_container /bin/bash -c "curl -k -F 'file=@$HIRS_CI_REPO_ROOT/.ci/setup/certs/ca.crt' https://localhost:${HIRS_ACA_PORTAL_PORT}/$HIRS_ACA_POST_POINT_TRUST"
+  docker exec -i $aca_container /bin/bash -c "curl -k -F 'file=@$HIRS_CI_REPO_ROOT/.ci/setup/certs/ca.crt' https://localhost:${HIRS_ACA_PORTAL_PORT}/$HIRS_ACA_POST_POINT_TRUST" > /dev/null 2>&1
   echo "...done"
   echo "Uploading the RIM CA(s)..."
-  docker exec -i $aca_container /bin/bash -c "curl -k -F 'file=@$HIRS_CI_REPO_ROOT/.ci/setup/certs/RIMCaCert.pem' https://localhost:${HIRS_ACA_PORTAL_PORT}/$HIRS_ACA_POST_POINT_TRUST"
+  docker exec -i $aca_container /bin/bash -c "curl -k -F 'file=@$HIRS_CI_REPO_ROOT/.ci/setup/certs/RIMCaCert.pem' https://localhost:${HIRS_ACA_PORTAL_PORT}/$HIRS_ACA_POST_POINT_TRUST" > /dev/null 2>&1
-  docker exec -i $aca_container /bin/bash -c "curl -k -F 'file=@$HIRS_CI_REPO_ROOT/.ci/setup/certs/RimSignCert.pem' https://localhost:${HIRS_ACA_PORTAL_PORT}/$HIRS_ACA_POST_POINT_TRUST"
+  docker exec -i $aca_container /bin/bash -c "curl -k -F 'file=@$HIRS_CI_REPO_ROOT/.ci/setup/certs/RimSignCert.pem' https://localhost:${HIRS_ACA_PORTAL_PORT}/$HIRS_ACA_POST_POINT_TRUST" > /dev/null 2>&1
   echo "...done"
 }
 
@@ -99,13 +99,16 @@ provisionTpm2() {
      else
         echo "Provisioning failed as expected."
      fi
-  else   # provisioning succeeded
+  elif [[ $provisionOutput == *"Provisioning successful"* ]]; then
-     if [[ $expected_result == "fail" ]]; then
+       if [[ $expected_result == "fail" ]]; then
-       ((failedTests++))
+          ((failedTests++))
-       echo "!!! Provisioning passed, but was expected to fail"
+         echo "!!! Provisioning passed, but was expected to fail."
-     else
+       else
-        echo "Provisioning passed as expected."
+         echo "Provisioning passed as expected."
-     fi
+       fi
+  else   # Unexpected output
+     ((failedTests++))
+       echo "Provisioning failed. Provisioner provided an unexpected output."
   fi
 }
 

.ci/system-tests/tests/platform_cert_tests.sh

@@ -48,7 +48,6 @@ fi
 
 #  Process Test Results, any single failure will send back a failed result.
 if [[ $failedTests != 0 ]]; then
-    export TEST_STATUS=1
     echo "****  $failedTests out of $totalTests Platform Certificate Tests Failed! ****"
     exit 1
   else

.ci/system-tests/tests/rim_system_tests.sh

@@ -51,7 +51,6 @@ fi
 
 #  Process Test Results, any single failure will send back a failed result.
 if [[ $failedTests != 0 ]]; then
-    export TEST_STATUS=1
     echo "****  $failedTests out of $totalTests ACA RIM Tests Failed! ****"
     exit 1
   else

.github/workflows/system_test.yml

@@ -8,8 +8,6 @@ on:
       - '*v3*'
       - 'main'
   workflow_dispatch:
-env:
-  TEST_STATUS: 0
 jobs:
   DockerTests:
     runs-on: ubuntu-latest
@@ -30,52 +28,52 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
           .ci/system-tests/setup_system_tests.sh ${GITHUB_REF#refs/heads/}
       - name: ACA POLICY TEST 1 - Test ACA default policy
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 1
       - name: ACA POLICY TEST 2 - Test EK cert Only Validation Policy without a EK Issuer Cert in the trust store
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 2
       - name: ACA POLICY TEST 3 - Test EK Only Validation Policy
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 3
       - name: ACA POLICY TEST 4 - Test PC Validation Policy with no PC
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 4
       - name: ACA POLICY TEST 5 - Test FW and PC Validation Policy with no PC
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 5
       - name: ACA POLICY TEST 6 - Test PC Validation Policy with valid PC with no Attribute Check
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 6
       - name: ACA POLICY TEST 7 - Test PC Validation Policy with valid PC with Attribute Check
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 7
       - name: ACA POLICY TEST 8 - Test PC with RIM Validation Policy with valid PC and RIM
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 8
       - name: ACA POLICY TEST 9 - Test valid PC and RIM with PC only uploaded
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 9
       - name: ACA POLICY TEST 10 - Test valid PC and RIM with RIM only uploaded
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/aca_policy_tests.sh 10
@@ -85,17 +83,17 @@ jobs:
       #   run: |
       #     .ci/system-tests/tests/aca_policy_tests.sh
       - name: ACA PLATFORM CERTIFICATE TEST 1 - Test a delta Platform Certificate that adds a new memory component
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/platform_cert_tests.sh 1
       - name: ACA PLATFORM CERTIFICATE TEST 2 - Test a Platform Certificate that is missing a memory component
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/platform_cert_tests.sh 2
       - name: ACA PLATFORM CERTIFICATE TEST 3 - Test a Delta Platform Certificate that has a wrong a memory component
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/platform_cert_tests.sh 3
@@ -105,17 +103,17 @@ jobs:
       #   run: |
       #     .ci/system-tests/tests/platform_cert_tests.sh
       - name: ACA RIM TEST 1 - Test a RIM from an OEM and a Supplemental RIM from a VAR
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/rim_system_tests.sh 1
       - name: ACA RIM TEST 2 - Test a RIM from an OEM with a bad reference measurement and a Supplemental RIM from a VAR
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/rim_system_tests.sh 2
       - name: ACA RIM TEST 3 - Test a RIM from an OEM and a Supplemental RIM from a VAR with a bad reference measurement
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           .ci/system-tests/tests/rim_system_tests.sh 3
@@ -125,38 +123,22 @@ jobs:
       #   run: |
       #     .ci/system-tests/tests/rim_system_tests.sh
       - name: Copy System Test Log files
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           echo "*** Extracting ACA and Provisioner.Net logs ..."
           docker exec hirs-aca1 bash -c "mkdir -p /HIRS/logs/aca/ && cp -arp /var/log/hirs/* /HIRS/logs/aca/"
           docker exec hirs-provisioner1-tpm2 bash -c "mkdir -p /HIRS/logs/provisioner/ && cp -ap hirs*.log /HIRS/logs/provisioner/ && chmod -R 777 /HIRS/logs"
       - name: Docker Compose Down
-        continue-on-error: true
+        if: always()
         shell: bash
         run: |
           echo "*** Exiting and removing Docker containers and network ..."
           docker compose -f .ci/docker/docker-compose-system-test.yml down -v
-          if [[ ${TEST_STATUS} == "0" ]]; then
-            echo "******** SUCCESS: System Tests for TPM 2.0 passed ********"
-            echo "TEST_STATUS=0" >> $GITHUB_ENV
-            exit 0;
-          else
-            echo "******** FAILURE: System Tests for TPM 2.0 failed ********"
-            echo "TEST_STATUS=1" >> $GITHUB_ENV
-            exit 1
-          fi
       - name: Archive System Test Log files
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: System_Test_Log_Files
           path: logs/
           if-no-files-found: ignore
-      - name: Check System Test results
-        if: success() || failure()
-        run: |
-          if [ ${TEST_STATUS} == "0" ]; then
-            exit 0;
-          else
-            exit 1;
-          fi