ExternalVendorCode/HIRS
1
0
Fork 0
mirror of https://github.com/nsacyber/HIRS.git synced 2025-02-21 10:01:49 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

[#216] Upgrade docker images and system tests to support python3 (#219)

Browse Source 
* Updated docker images to use python3

* Updated docker compose files.

* Test Travis CI

* Test Travis CI

* Updated Dockerfiles.

* Updated .travis.yml

* Cleaned up scripts.

* Cleaned up scripts.

* Updated docker-compose files.

* Updated amazonlinux2 image with python3.

* Updated system test driver.

* Updated system test driver

* Cleaned up import in test driver.

* Updated setup scripts to output python version
This commit is contained in:
busaboy1340 2020-01-17 07:05:05 -05:00 committed by GitHub
parent 84a76608f3
commit da836a0862
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 79 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/docker/Dockerfile.amazonlinux2
View File

@ -3,7 +3,7 @@ FROM amazonlinux:2
# Install packages for building HIRS # Install packages for building HIRS
RUN yum -y update && yum clean all RUN yum -y update && yum clean all
RUN yum groupinstall -y "Development Tools" RUN yum groupinstall -y "Development Tools"
RUN yum install -y wget java-1.8.0-openjdk-devel protobuf-compiler rpm-build cmake make git gcc-c++ doxygen graphviz python libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel RUN yum install -y wget java-1.8.0-openjdk-devel protobuf-compiler rpm-build cmake make git gcc-c++ doxygen graphviz python3 libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel
# Install EPEL # Install EPEL
WORKDIR /tmp WORKDIR /tmp

2
.ci/docker/Dockerfile.centos7
View File

@ -2,7 +2,7 @@ FROM centos:7.5.1804
# Install packages for building HIRS # Install packages for building HIRS
RUN yum -y update && yum clean all RUN yum -y update && yum clean all
RUN yum install -y java-1.8.0-openjdk-devel protobuf-compiler rpm-build epel-release cmake make git gcc-c++ doxygen graphviz python libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel RUN yum install -y java-1.8.0-openjdk-devel protobuf-compiler rpm-build epel-release cmake make git gcc-c++ doxygen graphviz python3 libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel
RUN yum install -y cppcheck log4cplus-devel re2-devel RUN yum install -y cppcheck log4cplus-devel re2-devel
# Set Environment Variables # Set Environment Variables

2
.ci/docker/Dockerfile.tpm2provisioner
View File

@ -2,7 +2,7 @@ FROM hirs/hirs-ci:centos7
# Install packages for installing HIRS TPM2 Provisioner # Install packages for installing HIRS TPM2 Provisioner
RUN yum -y update && yum clean all RUN yum -y update && yum clean all
RUN yum install -y tpm2-tools libcurl procps-ng wget dbus python-requests && yum clean all RUN yum install -y tpm2-tools libcurl procps-ng wget dbus python-requests python2-future python36-future && yum clean all
# Install PACCOR for Device Info Gathering # Install PACCOR for Device Info Gathering
RUN mkdir paccor && pushd paccor && wget https://github.com/nsacyber/paccor/releases/download/v1.1.3r3/paccor-1.1.3-3.noarch.rpm && yum -y install paccor-*.rpm && popd RUN mkdir paccor && pushd paccor && wget https://github.com/nsacyber/paccor/releases/download/v1.1.3r3/paccor-1.1.3-3.noarch.rpm && yum -y install paccor-*.rpm && popd

2
.ci/docker/Dockerfile.tpmprovisioner
View File

@ -2,7 +2,7 @@ FROM hirs/hirs-ci:centos7
# Install packages for installing HIRS TPM Provisioner # Install packages for installing HIRS TPM Provisioner
RUN yum -y update && yum clean all RUN yum -y update && yum clean all
RUN yum install -y java-1.8.0-openjdk wget util-linux chkconfig sed systemd gmp-devel coreutils dmidecode bash autoconf autoconf-archive automake libtool pkgconfig m4 gcc-c++ openssl python-requests && yum clean all RUN yum install -y java-1.8.0-openjdk wget util-linux chkconfig sed systemd gmp-devel coreutils dmidecode bash autoconf autoconf-archive automake libtool pkgconfig m4 gcc-c++ openssl python-requests python2-future python36-future && yum clean all
# Install tpm_module for Communicating with TPM # Install tpm_module for Communicating with TPM
RUN mkdir tpm_module && pushd tpm_module && wget https://github.com/nsacyber/HIRS/releases/download/v1.0.4/tpm_module-1.0.4-1558547257.cedc93.x86_64.rpm && yum -y install tpm_module-*.rpm && popd RUN mkdir tpm_module && pushd tpm_module && wget https://github.com/nsacyber/HIRS/releases/download/v1.0.4/tpm_module-1.0.4-1558547257.cedc93.x86_64.rpm && yum -y install tpm_module-*.rpm && popd

1
.ci/setup/addFaultyComponentsForPBaseCertB.py
View File

@ -1,6 +1,7 @@
# Add faulty components to the PACCOR generated JSON componentsFile. # Add faulty components to the PACCOR generated JSON componentsFile.
# This will be used to create a bad platform certificate. # This will be used to create a bad platform certificate.
from __future__ import print_function
import json import json
import pprint import pprint

6
.ci/setup/createDeltaComponentsForPBaseCertA.py
View File

@ -8,10 +8,12 @@
# VARDeltaCertA2 - Bad Delta # VARDeltaCertA2 - Bad Delta
# VARDeltaCertA2Resolved - Good Delta # VARDeltaCertA2Resolved - Good Delta
import sys from __future__ import print_function
import json from builtins import range
import copy import copy
import json
import pprint import pprint
import sys
try: try:
minNumOfComponents = 3 minNumOfComponents = 3

5
.ci/setup/createDeltaComponentsForPBaseCertB.py
View File

@ -2,10 +2,11 @@
# SIDeltaCertB1 - Bad Delta # SIDeltaCertB1 - Bad Delta
# VARDeltaCertB1 - Good Delta # VARDeltaCertB1 - Good Delta
import sys from __future__ import print_function
import json
import copy import copy
import json
import pprint import pprint
import sys
try: try:
pcDir = '/var/hirs/pc_generation/' pcDir = '/var/hirs/pc_generation/'

8
.ci/setup/setup-tpm2provisioner-base-delta-bad.sh
View File

@ -69,7 +69,7 @@ function InitTpm2Emulator {
echo "Running PACCOR to generate local component information..." echo "Running PACCOR to generate local component information..."
# Use specific PACCOR script for system testing. # Use specific PACCOR script for system testing.
# Will provide default component SN#s when needed. # Will provide default component SN#s when needed.
cp -f /opt/paccor/scripts/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh cp -f /HIRS/.ci/system-tests/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh
/opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile /opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile
echo echo
@ -184,6 +184,12 @@ InitTpm2Emulator
# Update the hirs-site.config file # Update the hirs-site.config file
UpdateHirsSiteConfigFile UpdateHirsSiteConfigFile
# Set alias to use python3
echo "===========Python Version==========="
python3 --version
alias python='/usr/bin/python3.6'
alias
echo "" echo ""
echo "TPM 2.0 Emulator NV RAM list" echo "TPM 2.0 Emulator NV RAM list"
tpm2_nvlist tpm2_nvlist

8
.ci/setup/setup-tpm2provisioner-base-delta-good.sh
View File

@ -75,7 +75,7 @@ function InitTpm2Emulator {
echo "Running PACCOR to generate local component information..." echo "Running PACCOR to generate local component information..."
# Use specific PACCOR script for system testing. # Use specific PACCOR script for system testing.
# Will provide default component SN#s when needed. # Will provide default component SN#s when needed.
cp -f /opt/paccor/scripts/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh cp -f /HIRS/.ci/system-tests/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh
/opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile /opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile
# Split into JSON files needed to generate the certificates # Split into JSON files needed to generate the certificates
@ -223,6 +223,12 @@ InitTpm2Emulator
# Update the hirs-site.config file # Update the hirs-site.config file
UpdateHirsSiteConfigFile UpdateHirsSiteConfigFile
# Set alias to use python3
echo "===========Python Version==========="
python3 --version
alias python='/usr/bin/python3.6'
alias
echo "" echo ""
echo "TPM 2.0 Emulator NV RAM list" echo "TPM 2.0 Emulator NV RAM list"
tpm2_nvlist tpm2_nvlist

8
.ci/setup/setup-tpm2provisioner.sh
View File

@ -67,7 +67,7 @@ function InitTpm2Emulator {
echo "Running PACCOR to generate local component information..." echo "Running PACCOR to generate local component information..."
# Use specific PACCOR script for system testing. # Use specific PACCOR script for system testing.
# Will provide default component SN#s when needed. # Will provide default component SN#s when needed.
cp -f /opt/paccor/scripts/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh cp -f /HIRS/.ci/system-tests/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh
/opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile /opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile
/opt/paccor/scripts/referenceoptions.sh > $PC_DIR/optionsFile /opt/paccor/scripts/referenceoptions.sh > $PC_DIR/optionsFile
/opt/paccor/scripts/otherextensions.sh > $PC_DIR/extensionsFile /opt/paccor/scripts/otherextensions.sh > $PC_DIR/extensionsFile
@ -151,6 +151,12 @@ InitTpm2Emulator
# Update the hirs-site.config file # Update the hirs-site.config file
UpdateHirsSiteConfigFile UpdateHirsSiteConfigFile
# Set alias to use python3
echo "===========Python Version==========="
python3 --version
alias python='/usr/bin/python3.6'
alias
echo "" echo ""
echo "TPM 2.0 Emulator NV RAM list" echo "TPM 2.0 Emulator NV RAM list"
tpm2_nvlist tpm2_nvlist

6
.ci/setup/setup-tpmprovisioner.sh
View File

@ -144,5 +144,11 @@ UpdateHirsSiteConfigFile
# Update the logging.properties file # Update the logging.properties file
UpdateLoggingConfigFile UpdateLoggingConfigFile
# Set alias to use python3
echo "===========Python Version==========="
python3 --version
alias python='/usr/bin/python3.6'
alias
echo "" echo ""
echo "===========HIRS ACA TPM 1.2 Provisioner Setup Complete!===========" echo "===========HIRS ACA TPM 1.2 Provisioner Setup Complete!==========="

16
.ci/system-tests/system_test.py
View File

@ -1,20 +1,20 @@
# system_test.py - implements a group of tests that run appraisals on a client and server # system_test.py - implements a group of tests that run appraisals on a client and server.
# TODO: test_02-test_12 will need to be implemented when the additional HIRS
# TODO: test_01-test_11 will need to be implemented when the additional HIRS
# projects are imported to the new GitHub repo. The test code is commented out for now. # projects are imported to the new GitHub repo. The test code is commented out for now.
from __future__ import print_function
import logging import logging
import os import os
import sys import sys
import unittest import unittest
import urllib3 import urllib3
from system_test_core import HIRSPortal, AttestationCAPortal, collectors, \ from system_test_core import DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY, \
HIRSPortal, AttestationCAPortal, collectors, \
send_command, send_command_sha1sum, run_hirs_report, run_hirs_provisioner_tpm_1_2, \ send_command, send_command_sha1sum, run_hirs_report, run_hirs_provisioner_tpm_1_2, \
run_hirs_provisioner_tpm_2_0, parse_xml_with_stripped_namespaces, \ run_hirs_provisioner_tpm_2_0, parse_xml_with_stripped_namespaces, \
get_all_nodes_recursively, touch_random_file_and_remove, get_random_pcr_hex_value, \ get_all_nodes_recursively, touch_random_file_and_remove, get_random_pcr_hex_value, \
get_current_timestamp, is_ubuntu_client, is_tpm_2_0, is_tpm_1_2, \ get_current_timestamp, is_ubuntu_client, is_tpm_2_0, is_tpm_1_2, \
DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY, \
make_simple_ima_baseline, make_baseline_from_xml, \ make_simple_ima_baseline, make_baseline_from_xml, \
make_simple_ima_blacklist_baseline, \ make_simple_ima_blacklist_baseline, \
make_simple_ima_blacklist_baseline_with_hash, \ make_simple_ima_blacklist_baseline_with_hash, \
@ -611,9 +611,9 @@ class SystemTest(unittest.TestCase):
"""Test that running the TPM 1.2 hirs provisioner works""" """Test that running the TPM 1.2 hirs provisioner works"""
logging.info("***************** Beginning of initial TPM 1.2 provisioner run *****************") logging.info("***************** Beginning of initial TPM 1.2 provisioner run *****************")
# # Run the provisioner to ensure that it provisions successfully # Run the provisioner to ensure that it provisions successfully
# provisioner_out = run_hirs_provisioner_tpm_1_2(CLIENT) provisioner_out = run_hirs_provisioner_tpm_1_2(CLIENT)
# print("Initial TPM 1.2 provisioner run output: {0}".format(provisioner_out)) print("Initial TPM 1.2 provisioner run output: {0}".format(provisioner_out))
@collectors(['TPM'], COLLECTOR_LIST) @collectors(['TPM'], COLLECTOR_LIST)
@unittest.skipIf(not is_tpm_2_0(TPM_VERSION), "Skipping this test due to TPM Version " + TPM_VERSION) @unittest.skipIf(not is_tpm_2_0(TPM_VERSION), "Skipping this test due to TPM Version " + TPM_VERSION)

33
.ci/system-tests/system_test_Driver.py
View File

@ -1,30 +1,18 @@
# System Test Driver to help with debugging. # System Test Driver to help with debugging.
import binascii from __future__ import print_function
from ConfigParser import SafeConfigParser
import datetime
import json
import os
import shlex
import subprocess
import unittest
import re
import requests
import logging import logging
import pprint import os
import hashlib
import random
import uuid
import time
import sys import sys
import argparse import unittest
import urllib3
from system_test_core import HIRSPortal, AttestationCAPortal, collectors, \ from system_test_core import DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY, \
HIRSPortal, AttestationCAPortal, collectors, \
send_command, send_command_sha1sum, run_hirs_report, run_hirs_provisioner_tpm_1_2, \ send_command, send_command_sha1sum, run_hirs_report, run_hirs_provisioner_tpm_1_2, \
run_hirs_provisioner_tpm_2_0, parse_xml_with_stripped_namespaces, get_current_timestamp, \ run_hirs_provisioner_tpm_2_0, parse_xml_with_stripped_namespaces, get_current_timestamp, \
get_all_nodes_recursively, touch_random_file_and_remove, get_random_pcr_hex_value, \ get_all_nodes_recursively, touch_random_file_and_remove, get_random_pcr_hex_value, \
is_ubuntu_client, is_tpm_2_0, is_tpm_1_2, \ is_ubuntu_client, is_tpm_2_0, is_tpm_1_2 \
DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY
NUMBER_OF_PCRS = 24 NUMBER_OF_PCRS = 24
@ -36,7 +24,8 @@ else:
suffix = "" suffix = ""
# Change to point to your HIRS directory # Change to point to your HIRS directory
HOME_DIR = "/HIRS/" #HOME_DIR = "/HIRS/"
HOME_DIR = "/workspace/git/python2to3-dev-3/"
HIRS_ACA_PORTAL_IP="172.17.0.2" HIRS_ACA_PORTAL_IP="172.17.0.2"
TPM_VERSION="2.0" TPM_VERSION="2.0"
#TPM_VERSION="1.2" #TPM_VERSION="1.2"
@ -98,7 +87,7 @@ logging.info("The ACA Portal is: " + HIRS_ATTESTATION_CA_PORTAL_URL)
#Portal = HIRSPortal(HIRS_SERVER_URL) #Portal = HIRSPortal(HIRS_SERVER_URL)
AcaPortal = AttestationCAPortal(HIRS_ATTESTATION_CA_PORTAL_URL) AcaPortal = AttestationCAPortal(HIRS_ATTESTATION_CA_PORTAL_URL)
requests.packages.urllib3.disable_warnings() urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class SystemTest(unittest.TestCase): class SystemTest(unittest.TestCase):
@ -112,7 +101,7 @@ class SystemTest(unittest.TestCase):
def setUp(self): def setUp(self):
"""Set the systems tests state up for testing""" """Set the systems tests state up for testing"""
#AcaPortal.disable_supply_chain_validations() AcaPortal.disable_supply_chain_validations()
def tearDown(self): def tearDown(self):
"""Tears down the state for testing""" """Tears down the state for testing"""

38
.ci/system-tests/system_test_core.py
View File

@ -1,20 +1,24 @@
# Defines core methods shared amongst system test scripts # Defines core methods shared amongst system test scripts.
import sets from future import standard_library
import unittest standard_library.install_aliases()
import shlex from builtins import str
import subprocess from builtins import filter
import os from builtins import object
from io import StringIO
import binascii import binascii
import requests
import logging
import random
import time
import datetime import datetime
import json import json
import logging
import os
import pprint import pprint
import random
import requests
import shlex
import subprocess
import time
import unittest
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from StringIO import StringIO
DEFAULT_GROUP_NAME = "Default Group" DEFAULT_GROUP_NAME = "Default Group"
DEFAULT_TPM_POLICY = "Test TPM Policy" DEFAULT_TPM_POLICY = "Test TPM Policy"
@ -23,7 +27,7 @@ CACHED_XML_REPORT = None
APPRAISAL_SUCCESS_MESSAGE = "Appraisal passed" APPRAISAL_SUCCESS_MESSAGE = "Appraisal passed"
class HIRSPortal: class HIRSPortal(object):
def __init__(self, hirs_server_url): def __init__(self, hirs_server_url):
self.server_url = hirs_server_url self.server_url = hirs_server_url
@ -117,7 +121,7 @@ class HIRSPortal:
# (works for full or partial path) # (works for full or partial path)
return (record['hash']['digestString'] == sha_hash) and (filename in record['path']) return (record['hash']['digestString'] == sha_hash) and (filename in record['path'])
matching_records = filter(record_matcher, ima_records) matching_records = list(filter(record_matcher, ima_records))
return len(matching_records) > 0 return len(matching_records) > 0
def upload_payload(self, payload): def upload_payload(self, payload):
@ -275,7 +279,7 @@ class HIRSPortal:
def get_alerts(self): def get_alerts(self):
return self.request("get", "portal/alerts/list").json() return self.request("get", "portal/alerts/list").json()
class AttestationCAPortal: class AttestationCAPortal(object):
def __init__(self, hirs_server_url): def __init__(self, hirs_server_url):
self.server_url = hirs_server_url self.server_url = hirs_server_url
@ -362,8 +366,8 @@ def check_request_response(expected_status_codes, request_result, operation):
raise RuntimeError(message) raise RuntimeError(message)
def collectors(collectors, collector_list): def collectors(collectors, collector_list):
enabled_collectors = sets.Set(collector_list) enabled_collectors = set(collector_list)
tested_collectors = sets.Set(collectors) tested_collectors = set(collectors)
if tested_collectors.issubset(enabled_collectors): if tested_collectors.issubset(enabled_collectors):
return lambda func: func return lambda func: func
return unittest.skip("{0} collector isn't enabled".format(tested_collectors.difference(enabled_collectors))) return unittest.skip("{0} collector isn't enabled".format(tested_collectors.difference(enabled_collectors)))
@ -443,7 +447,7 @@ def parse_xml_with_stripped_namespaces(raw_xml_string):
for _, el in it: for _, el in it:
if '}' in el.tag: if '}' in el.tag:
el.tag = el.tag.split('}', 1)[1] # strip all namespaces el.tag = el.tag.split('}', 1)[1] # strip all namespaces
for at in el.attrib.keys(): # strip namespaces of attributes too for at in list(el.attrib.keys()): # strip namespaces of attributes too
if '}' in at: if '}' in at:
newat = at.split('}', 1)[1] newat = at.split('}', 1)[1]
el.attrib[newat] = el.attrib[at] el.attrib[newat] = el.attrib[at]