Adding system test for provision using Provisioner.NET (#748)

* Adding run script, Dockerfile, and container setup script for provisioning with TPM Simulator and Provisioner.Net * Replacing Docker container setup script with Docker Compose .yml file with respective configurations in related files * Adding system_test.yml for GitHub actions to run system tests upon V3 branch pushes. Modifying related files accordingly * Replacing V2 ACA with V3 ACA built from Rocky 9
2024-12-24 07:06:46 +00:00 · 2024-05-01 14:20:51 -04:00 · 2024-05-01 14:20:51 -04:00 · 02a44c2b5c
commit 02a44c2b5c
parent 937a9f295e
6 changed files with 245 additions and 119 deletions
--- a/.ci/docker/.env
+++ b/.ci/docker/.env
@ -6,7 +6,7 @@ IMA_ENABLED=false
 HIRS_ACA_PORTAL_IP=172.19.0.2
 HIRS_ACA_PORTAL_PORT=8443
 HIRS_BROKER_PORT=61616
-HIRS_ACA_PORTAL_CONTAINER_PORT=80
+HIRS_ACA_PORTAL_CONTAINER_PORT=8443

 HIRS_ACA_HOSTNAME=hirsaca

--- a/.ci/docker/Dockerfile.tpm2provisioner_dotnet
+++ b/.ci/docker/Dockerfile.tpm2provisioner_dotnet
@ -0,0 +1,100 @@
+FROM rockylinux:9
+LABEL org.opencontainers.image.vendor NSA Laboratory for Advanced Cybersecurity Research
+LABEL org.opencontainers.image.source https://github.com/nsacyber/hirs
+LABEL org.opencontainers.image.description Tools for testing the build and deployment of HIRS projects.
+
+# Purpose: This image is designed for HIRS Provisioner.Net testing on Rocky 9.
+# Date Modified: 4/15/24
+# Notes: The image to be built should be named "tpm2provisioner-dotnet-ci:latest".
+#        For local build, use this command from the /HIRS/ directory to build the image:
+#        $ docker build -f ./.ci/docker/Dockerfile.tpm2provisioner_dotnet -t tpm2provisioner-dotnet-ci:latest .
+
+# REF can be specified as a docker run environment variable to select the HIRS branch to work with
+ENV REF=main
+# BUILD, is an environment variable that if not empty, will attempt to run gradle bootWar on the cloned branch
+ENV BUILD=
+
+SHELL ["/bin/bash", "-c"]
+
+# Rocky 9 has a different channel for some apps
+RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb
+
+# Update and install OS-dependencies
+RUN dnf update -y
+# Dependencies were selected for these reasons:
+#   OS setup/Unknown direct impact for HIRS
+ENV HIRS_DNF_OS_SETUP="initscripts firewalld policycoreutils policycoreutils-python-utils net-tools"
+#   OS tools
+ENV HIRS_DNF_OS_TOOLS="git sudo vim wget"
+#   ACA compile
+ENV HIRS_DNF_ACA_COMPILE="java-17-openjdk-devel"
+#   ACA run
+ENV HIRS_DNF_ACA_RUN="mariadb-server"
+#   IBM TPM simulator compile
+ENV HIRS_DNF_TPM_COMPILE="tpm2-tools gcc cmake openssl-devel"
+#   IBM TSS compile
+ENV HIRS_DNF_TSS_COMPILE="autoconf automake libtool"
+#   .NET SDK
+ENV HIRS_DNF_DOTNET_SDK="dotnet-sdk-6.0"
+# Download and install all dependencies at one time
+RUN dnf -y install $(echo "$HIRS_DNF_OS_SETUP") $(echo "$HIRS_DNF_OS_TOOLS") $(echo "$HIRS_DNF_ACA_COMPILE") $(echo "$HIRS_DNF_ACA_RUN") $(echo "$HIRS_DNF_TPM_COMPILE") $(echo "$HIRS_DNF_TSS_COMPILE") $(echo "$HIRS_DNF_DOTNET_SDK")
+
+# Build IBM TPM Simulator
+RUN git clone https://github.com/kgoldman/ibmswtpm2 /ibmswtpm2
+WORKDIR /ibmswtpm2/src
+RUN make
+
+# Build IBM TPM tools
+RUN git clone https://github.com/kgoldman/ibmtss /ibmtss
+WORKDIR /ibmtss/utils
+RUN make -f makefiletpmc
+
+# The following script tests that the SW TPM and TSS were compiled in the docker image. And documents how to start the SW TPM after container launch using both IBM's tss and TPM2-TOOLS.
+RUN echo "#!/bin/bash" > /tmp/tpm_config && \
+    echo "/ibmswtpm2/src/tpm_server &" >> /tmp/tpm_config && \
+    echo "sleep 5" >> /tmp/tpm_config && \
+    echo "/ibmtss/utils/startup -c" >> /tmp/tpm_config && \
+    echo "tpm2_shutdown" >> /tmp/tpm_config && \
+    echo "tpm2_startup -c" >> /tmp/tpm_config && \
+    echo "/ibmtss/utils/shutdown -c" >> /tmp/tpm_config && \
+    bash /tmp/tpm_config && \
+    rm -rf /tmp/tpm_config
+
+# Checkout HIRS
+RUN git clone -b main https://github.com/nsacyber/HIRS.git /hirs
+
+# Run bootwar to cache build artifacts
+WORKDIR /hirs
+RUN ./gradlew bootWar
+
+# Install dotnet tools
+RUN dotnet tool install --global dotnet-deb
+RUN dotnet tool install --global dotnet-rpm
+RUN dotnet tool install --global dotnet-zip
+# Add dotnet PATHs
+ENV PATH="/root/.dotnet:/root/.dotnet/tools:$PATH"
+# Build .NET
+WORKDIR /hirs/HIRS_Provisioner.NET
+RUN dotnet restore
+WORKDIR /hirs/HIRS_Provisioner.NET/hirs
+RUN dotnet test
+RUN dotnet deb -r linux-x64 -c Release
+RUN dotnet rpm -r linux-x64 -c Release
+
+# The following script will clone and copy the referenced branch of HIRS off GitHub
+# If configured, run bootwar to cache build artifacts
+RUN echo "#!/bin/bash" > /tmp/auto_clone_branch && \
+    echo "cd /hirs" >> /tmp/auto_clone_branch && \
+    echo "git fetch origin && git pull origin main && git reset --hard" >> /tmp/auto_clone_branch && \
+    echo 'git checkout $1 && git reset --hard' >> /tmp/auto_clone_branch && \
+    echo 'if [ -n "${2}" ]; then ./gradlew bootWar; fi' >> /tmp/auto_clone_branch && \
+    echo "cd HIRS_Provisioner.NET/hirs" >> /tmp/auto_clone_branch && \
+    echo 'if [ -n "${2}" ]; then dotnet deb -r linux-x64 -c Release && dotnet rpm -r linux-x64 -c Release; fi' >> /tmp/auto_clone_branch && \
+    chmod 755 /tmp/auto_clone_branch
+
+# Reset working directory
+WORKDIR /hirs
+
+# Copy script for running TPM Simulator
+COPY ./.ci/setup/container/setup_tpm2provisioner_dotnet.sh /.ci/setup/container/
+RUN chmod 755 /.ci/setup/container/setup_tpm2provisioner_dotnet.sh
--- a/.ci/docker/docker-compose-system-test.yml
+++ b/.ci/docker/docker-compose-system-test.yml
@ -1,49 +1,29 @@
-version: "3.1"
-
 services:
   aca:
-      image: ghcr.io/nsacyber/hirs/aca-ci:latest
+      image: ghcr.io/nsacyber/hirs/aca-rocky:5445278
      container_name: hirs-aca1
      volumes:
         - ../../:/HIRS
      ports:
         - "${HIRS_ACA_PORTAL_PORT}:${HIRS_ACA_PORTAL_CONTAINER_PORT}"
-      entrypoint: /bin/bash -c
-      command: [tail -f /dev/null;]
      hostname: ${HIRS_ACA_HOSTNAME}
      networks:
         hirs_aca_system_tests:
            ipv4_address: ${HIRS_ACA_PORTAL_IP}
            aliases:
-              - ${HIRS_ACA_HOSTNAME}
+               - ${HIRS_ACA_HOSTNAME}

   tpmprovisioner:
-      image: ghcr.io/nsacyber/hirs/tpm2provisioner-ci:latest
+      image: ghcr.io/nsacyber/hirs/tpm2provisioner-dotnet-ci:latest
      container_name: hirs-provisioner1-tpm2
      depends_on:
         - aca
      volumes:
         - ../../:/HIRS
-         - ../system-tests/profiles/laptop/dmi:/sys/class/dmi:ro
-      entrypoint: /bin/bash -c
-      command: [tail -f /dev/null;]
-      devices:
-        - "/dev/mem:/dev/mem"
-      cap_add:
-        - sys_rawio
+      command: ["bash", "-c", "tail -f /dev/null;"]
      networks:
         hirs_aca_system_tests:
            ipv4_address: ${HIRS_ACA_PROVISIONER_TPM2_IP}
-      environment:
-         - HIRS_ACA_PROVISIONER_TPM2_IP=${HIRS_ACA_PROVISIONER_TPM2_IP}
-         - TPM_ENABLED=${TPM_ENABLED}
-         - IMA_ENABLED=${IMA_ENABLED}
-         - HIRS_ACA_PORTAL_IP=${HIRS_ACA_PORTAL_IP}
-         - HIRS_ACA_PORTAL_PORT=${HIRS_ACA_PORTAL_PORT}
-         - HIRS_BROKER_PORT=${HIRS_BROKER_PORT}
-         - HIRS_ACA_PORTAL_CONTAINER_PORT=${HIRS_ACA_PORTAL_CONTAINER_PORT}
-         - HIRS_ACA_HOSTNAME=${HIRS_ACA_HOSTNAME}
-         - HIRS_SUBNET=${HIRS_SUBNET}

 networks:
   hirs_aca_system_tests:
@ -51,4 +31,4 @@ networks:
      ipam:
         driver: default
         config:
-           - subnet: ${HIRS_SUBNET}
+            - subnet: ${HIRS_SUBNET}
--- a/.ci/setup/container/setup_tpm2provisioner_dotnet.sh
+++ b/.ci/setup/container/setup_tpm2provisioner_dotnet.sh
@ -0,0 +1,83 @@
+#!/bin/bash
+#########################################################################################
+#  Script to setup the TPM Provisioner.NET for System Tests
+#########################################################################################
+
+# Setting configurations
+. ./.ci/docker/.env
+
+set -a
+
+set -e
+echo "*** Setting up TPM emulator for the TPM2 Provisioner *** "
+
+# Wait for ACA to boot
+echo "*** Waiting for ACA to spin up at address ${HIRS_ACA_PORTAL_IP} on port ${HIRS_ACA_PORTAL_PORT} ..."
+  until [ "`curl --silent -I -k https://${HIRS_ACA_PORTAL_IP}:${HIRS_ACA_PORTAL_PORT}/HIRS_AttestationCAPortal | grep 'HTTP/1.1 200'`" != "" ]; do
+    sleep 1;
+  done
+  echo "*** ACA is up!"
+
+# Un-package Provisioner.NET RPM
+cd /
+yes | dnf install /hirs/HIRS_Provisioner.NET/hirs/bin/Release/net6.0/linux-x64/HIRS_Provisioner.NET.2.2.0.linux-x64.rpm 1> /dev/null
+
+# Start TPM simulator server
+./ibmswtpm2/src/tpm_server 1> /dev/null &
+echo "*** TPM Simulator Server has started"
+
+# Create EK Certificate
+cd /ibmtss/utils || exit
+./startup 1> /dev/null
+./createekcert -rsa 2048 -cakey cakey.pem -capwd rrrr -v 1> /dev/null
+cd / || exit
+echo "*** EK certificate has been created using IBMTSS CA Key"
+
+# Writing to Provisioner.Net configurations file for modified aca port and efi prefix
+cat <<APPSETTINGS_FILE > /usr/share/hirs/appsettings.json
+{
+  "auto_detect_tpm":  "TRUE",
+  "aca_address_port": "https://${HIRS_ACA_PORTAL_IP}:${HIRS_ACA_PORTAL_PORT}",
+  "efi_prefix": "/boot/efi",
+  "paccor_output_file": "",
+  "event_log_file":  "",
+  "hardware_manifest_collectors": "paccor_scripts",
+
+  "Serilog": {
+    "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ],
+    "Enrich": [ "FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId" ],
+    "MinimumLevel": {
+      "Default": "Debug",
+      "Override": {
+        "Microsoft": "Warning",
+        "System": "Warning"
+      }
+    },
+    "WriteTo": [
+      {
+        "Name": "Console",
+        "Args": {
+          "outputTemplate": "{Message}{NewLine}",
+          "theme": "Serilog.Sinks.SystemConsole.Themes.SystemConsoleTheme::Grayscale, Serilog.Sinks.Console",
+          "restrictedToMinimumLevel": "Information"
+        }
+      },
+      {
+        "Name": "File",
+        "Args": {
+          "path": "hirs.log",
+          "rollingInterval": "Day",
+          "retainedFileCountLimit": 5
+        }
+      }
+    ]
+  }
+}
+APPSETTINGS_FILE
+
+# Uploading CA Certificate to HIRS ACA Portal
+curl -k -s -F "file=@/ibmtss/utils/certificates/cacert.pem" https://${HIRS_ACA_PORTAL_IP}:${HIRS_ACA_PORTAL_PORT}/HIRS_AttestationCAPortal/portal/certificate-request/trust-chain/upload
+echo "*** CA Certificate has been uploaded to HIRS ACA Portal"
+
+# Starting Provisioning
+./usr/share/hirs/tpm_aca_provision --tcp --ip 127.0.0.1:2321 --sim
--- a/.ci/system-tests/run_system_tests.sh
+++ b/.ci/system-tests/run_system_tests.sh
@ -1,84 +1,47 @@
 #!/bin/bash
 #########################################################################################
 #    Script to run the System Tests  for HIRS TPM 2.0 Provisoner
-#
+#    Notes for running manually/locally (not from GitHub Actions)
+#    1. Uncomment the "cd ../.." line below to make working directory = /HIRS/
+#    2. Run with the desired HIRS branch as an argument (i.e. $./run_system_tests.sh main)
 #########################################################################################
+# cd ../..
+
+# Setting variables
 aca_container=hirs-aca1
 tpm2_container=hirs-provisioner1-tpm2
-testResult="passed";
-issuerCert=../setup/certs/ca.crt
-hirs_aca_log=/var/log/tomcat/HIRS_AttestationCA.log

-# Source files for Docker Variables and helper scripts
-. ./.ci/docker/.env
-
-set -a
-
-echo "********  Setting up for HIRS System Tests for TPM 2.0 ******** "
-
-# Expand linux dmi files to mount to the provisioner container to simulate device component
-unzip -q .ci/system-tests/profiles/laptop/laptop_dmi.zip -d .ci/system-tests/profiles/laptop/
 # Start System Testing Docker Environment
-pushd .ci/docker > /dev/null
+echo "********  Setting up for HIRS System Tests for TPM 2.0 ******** "
+docker compose -f ./.ci/docker/docker-compose-system-test.yml up -d

-docker-compose -f docker-compose-system-test.yml up -d
+# Switching to current/desired branch
+docker exec $tpm2_container sh -c "cd / && ./tmp/auto_clone_branch $1 1> /dev/null && cd hirs"

-popd > /dev/null
-pushd .ci/system-tests > /dev/null
-source sys_test_common.sh
-
-# Build, Package, and Install HIRS ACA (2+ minutes) then wait for systems tests...
-docker exec $aca_container sh -c "/HIRS/.ci/setup/container/setup_aca.sh"
-echo "ACA Loaded!"
-echo "ACA Container info: $(checkContainerStatus $aca_container)";
-
-# Install HIRS provioner and setup tpm2 emulator
-docker exec $tpm2_container /HIRS/.ci/setup/container/setup_tpm2provisioner.sh
-echo "TPM2 Provisioner Container info: $(checkContainerStatus $tpm2_container)";
-
-# ********* Execute system tests here, add tests as needed ************* 
-echo "******** Setup Complete Begin HIRS System Tests ******** "
-
-source tests/aca_policy_tests.sh
-source tests/platform_cert_tests.sh
-source tests/rim_system_tests.sh
+# Install HIRS Provisioner.Net and setup tpm2 simulator.
+# In doing so, tests a single provision between Provisioner.Net and ACA.
+docker exec $tpm2_container /.ci/setup/container/setup_tpm2provisioner_dotnet.sh

+# HERE is where other system tests will be called, including:
+# aca_policy_tests.sh, platform_cert_tests.sh, rim_system_tests.sh
 echo "******** HIRS System Tests Complete ******** "

-# collecting ACA logs for archiving
-echo "Collecting ACA logs ....."
-docker exec $aca_container mkdir -p /HIRS/logs/aca/;
-docker exec $aca_container cp -a /var/log/tomcat/. /HIRS/logs/aca/;
-docker exec $aca_container chmod -R 777 /HIRS/logs/;
-echo "Collecting provisioner logs"
-docker exec $tpm2_container mkdir -p /HIRS/logs/provisioner/;
-docker exec $tpm2_container cp -a /var/log/hirs/provisioner/. /HIRS/logs/provisioner/;
-docker exec $tpm2_container chmod -R 777 /HIRS/logs/;
+# Collecting ACA and Provisioner.Net logs for workflow artifact
+echo "*** Extracting ACA and Provisioner.Net logs ..."
+docker exec $aca_container sh -c "cd .. && mkdir -p /HIRS/logs/aca/ && cp -arp /var/log/hirs/* /HIRS/logs/aca/"
+docker exec $tpm2_container sh -c "cd .. && mkdir -p /HIRS/logs/provisioner/ && cp -ap hirs*.log /HIRS/logs/provisioner/ && chmod -R 777 /HIRS/logs"

-echo ""
-echo "===========HIRS Tests and Log collection complete ==========="
-
-echo ""
-echo "End of System Tests for TPM 2.0, cleaning up..."
-echo ""
 # Clean up services and network
-popd > /dev/null
-pushd .ci/docker > /dev/null
-docker-compose -f docker-compose-system-test.yml down -v
-popd > /dev/null
-# Clean up dangling containers
-echo "Cleaning up dangling containers..."
-echo ""
-docker container prune --force
-echo ""
-echo "New value of test status is ${TEST_STATUS}"
+echo "*** Exiting and removing Docker containers and network ..."
+docker compose -f ./.ci/docker/docker-compose-system-test.yml down -v
+
 # Return container exit code
 if [[ ${TEST_STATUS} == "0" ]]; then
-    echo "SUCCESS: System Tests for TPM 2.0 passed"
+    echo "******** SUCCESS: System Tests for TPM 2.0 passed ********"
    echo "TEST_STATUS=0" >> $GITHUB_ENV
    exit 0;
  else
-    echo "FAILURE: System Tests for TPM 2.0 failed"
+    echo "******** FAILURE: System Tests for TPM 2.0 failed ********"
    echo "TEST_STATUS=1" >> $GITHUB_ENV
    exit 1
 fi
--- a/.github/workflows/system_test.yml
+++ b/.github/workflows/system_test.yml
@ -1,12 +1,13 @@
-# Sets up and runs HIRS System tests
-
+# This workflow will build HIRS, run system tests, and create artifacts consisting of ACA and Provisioner logs.
+# Updated: 04/18/2024
+#
 name: HIRS System Tests
 on:
  push:
    branches:
-      - '*v2*'
-      - 'master'
-    workflow_dispatch:
+      - '*v3*'
+      - 'main'
+  workflow_dispatch:
 env:
  TEST_STATUS: 0
 jobs:
@ -18,28 +19,27 @@ jobs:
      contents: read
      packages: write
    steps:
-    - uses: actions/checkout@v2
-    - name: ACA TPM2 Tests
-      continue-on-error: true
-      shell: bash
-      run: |
-        sudo apt-get install -y curl
-        export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk-1.8.0*/jre/bin"
-        echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
-        # comment out the line above and uncomment the line below to run in a forked repo.
-        #echo "${{ secrets.PKG_PWD }}" | docker login ghcr.io -u $ --password-stdin
-        bash .ci/system-tests/run_system_tests.sh
-    - name: Archive System Test Log files
-      uses: actions/upload-artifact@v2
-      with:
-        name: System_Test_Log_Files
-        path: logs/
-        if-no-files-found: error
-    - name: Check System Test results
-      if: success() || failure()
-      run: |
-        if [ ${TEST_STATUS} == "0" ]; then
-          exit 0;
-        else
-          exit 1;
-        fi
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: ACA TPM2 Tests
+        continue-on-error: true
+        shell: bash
+        run: |
+          # If on a forked repo, ensure that it has a new secret for the PAT
+          # and replace secrets.GITHUB_TOKEN with the secret in the fork
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+          bash .ci/system-tests/run_system_tests.sh ${GITHUB_REF#refs/heads/}
+      - name: Archive System Test Log files
+        uses: actions/upload-artifact@v3
+        with:
+          name: System_Test_Log_Files
+          path: logs/
+          if-no-files-found: ignore
+      - name: Check System Test results
+        if: success() || failure()
+        run: |
+          if [ ${TEST_STATUS} == "0" ]; then
+            exit 0;
+          else
+            exit 1;
+          fi