From da836a08623124c0cd67882f105e90364e65964b Mon Sep 17 00:00:00 2001 From: busaboy1340 <43374066+busaboy1340@users.noreply.github.com> Date: Fri, 17 Jan 2020 07:05:05 -0500 Subject: [PATCH] [#216] Upgrade docker images and system tests to support python3 (#219) * Updated docker images to use python3 * Updated docker compose files. * Test Travis CI * Test Travis CI * Updated Dockerfiles. * Updated .travis.yml * Cleaned up scripts. * Cleaned up scripts. * Updated docker-compose files. * Updated amazonlinux2 image with python3. * Updated system test driver. * Updated system test driver * Cleaned up import in test driver. * Updated setup scripts to output python version --- .ci/docker/Dockerfile.amazonlinux2 | 2 +- .ci/docker/Dockerfile.centos7 | 2 +- .ci/docker/Dockerfile.tpm2provisioner | 2 +- .ci/docker/Dockerfile.tpmprovisioner | 2 +- .ci/setup/addFaultyComponentsForPBaseCertB.py | 1 + .../createDeltaComponentsForPBaseCertA.py | 6 ++- .../createDeltaComponentsForPBaseCertB.py | 5 ++- .../setup-tpm2provisioner-base-delta-bad.sh | 8 +++- .../setup-tpm2provisioner-base-delta-good.sh | 8 +++- .ci/setup/setup-tpm2provisioner.sh | 8 +++- .ci/setup/setup-tpmprovisioner.sh | 6 +++ .ci/system-tests/system_test.py | 16 ++++---- .ci/system-tests/system_test_Driver.py | 33 ++++++---------- .ci/system-tests/system_test_core.py | 38 ++++++++++--------- 14 files changed, 79 insertions(+), 58 deletions(-) diff --git a/.ci/docker/Dockerfile.amazonlinux2 b/.ci/docker/Dockerfile.amazonlinux2 index 08dbe044..05e676c5 100644 --- a/.ci/docker/Dockerfile.amazonlinux2 +++ b/.ci/docker/Dockerfile.amazonlinux2 @@ -3,7 +3,7 @@ FROM amazonlinux:2 # Install packages for building HIRS RUN yum -y update && yum clean all RUN yum groupinstall -y "Development Tools" -RUN yum install -y wget java-1.8.0-openjdk-devel protobuf-compiler rpm-build cmake make git gcc-c++ doxygen graphviz python libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel +RUN yum install -y wget java-1.8.0-openjdk-devel protobuf-compiler rpm-build cmake make git gcc-c++ doxygen graphviz python3 libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel # Install EPEL WORKDIR /tmp diff --git a/.ci/docker/Dockerfile.centos7 b/.ci/docker/Dockerfile.centos7 index 90daba87..2f2d8108 100644 --- a/.ci/docker/Dockerfile.centos7 +++ b/.ci/docker/Dockerfile.centos7 @@ -2,7 +2,7 @@ FROM centos:7.5.1804 # Install packages for building HIRS RUN yum -y update && yum clean all -RUN yum install -y java-1.8.0-openjdk-devel protobuf-compiler rpm-build epel-release cmake make git gcc-c++ doxygen graphviz python libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel +RUN yum install -y java-1.8.0-openjdk-devel protobuf-compiler rpm-build epel-release cmake make git gcc-c++ doxygen graphviz python3 libssh2-devel openssl protobuf-devel tpm2-tss-devel tpm2-abrmd-devel trousers-devel libcurl-devel RUN yum install -y cppcheck log4cplus-devel re2-devel # Set Environment Variables diff --git a/.ci/docker/Dockerfile.tpm2provisioner b/.ci/docker/Dockerfile.tpm2provisioner index dbabff3d..e415a67b 100644 --- a/.ci/docker/Dockerfile.tpm2provisioner +++ b/.ci/docker/Dockerfile.tpm2provisioner @@ -2,7 +2,7 @@ FROM hirs/hirs-ci:centos7 # Install packages for installing HIRS TPM2 Provisioner RUN yum -y update && yum clean all -RUN yum install -y tpm2-tools libcurl procps-ng wget dbus python-requests && yum clean all +RUN yum install -y tpm2-tools libcurl procps-ng wget dbus python-requests python2-future python36-future && yum clean all # Install PACCOR for Device Info Gathering RUN mkdir paccor && pushd paccor && wget https://github.com/nsacyber/paccor/releases/download/v1.1.3r3/paccor-1.1.3-3.noarch.rpm && yum -y install paccor-*.rpm && popd diff --git a/.ci/docker/Dockerfile.tpmprovisioner b/.ci/docker/Dockerfile.tpmprovisioner index 4c6804e8..84df4600 100644 --- a/.ci/docker/Dockerfile.tpmprovisioner +++ b/.ci/docker/Dockerfile.tpmprovisioner @@ -2,7 +2,7 @@ FROM hirs/hirs-ci:centos7 # Install packages for installing HIRS TPM Provisioner RUN yum -y update && yum clean all -RUN yum install -y java-1.8.0-openjdk wget util-linux chkconfig sed systemd gmp-devel coreutils dmidecode bash autoconf autoconf-archive automake libtool pkgconfig m4 gcc-c++ openssl python-requests && yum clean all +RUN yum install -y java-1.8.0-openjdk wget util-linux chkconfig sed systemd gmp-devel coreutils dmidecode bash autoconf autoconf-archive automake libtool pkgconfig m4 gcc-c++ openssl python-requests python2-future python36-future && yum clean all # Install tpm_module for Communicating with TPM RUN mkdir tpm_module && pushd tpm_module && wget https://github.com/nsacyber/HIRS/releases/download/v1.0.4/tpm_module-1.0.4-1558547257.cedc93.x86_64.rpm && yum -y install tpm_module-*.rpm && popd diff --git a/.ci/setup/addFaultyComponentsForPBaseCertB.py b/.ci/setup/addFaultyComponentsForPBaseCertB.py index 36687e66..ebd13ccd 100644 --- a/.ci/setup/addFaultyComponentsForPBaseCertB.py +++ b/.ci/setup/addFaultyComponentsForPBaseCertB.py @@ -1,6 +1,7 @@ # Add faulty components to the PACCOR generated JSON componentsFile. # This will be used to create a bad platform certificate. +from __future__ import print_function import json import pprint diff --git a/.ci/setup/createDeltaComponentsForPBaseCertA.py b/.ci/setup/createDeltaComponentsForPBaseCertA.py index 08c7d5f6..01c265da 100644 --- a/.ci/setup/createDeltaComponentsForPBaseCertA.py +++ b/.ci/setup/createDeltaComponentsForPBaseCertA.py @@ -8,10 +8,12 @@ # VARDeltaCertA2 - Bad Delta # VARDeltaCertA2Resolved - Good Delta -import sys -import json +from __future__ import print_function +from builtins import range import copy +import json import pprint +import sys try: minNumOfComponents = 3 diff --git a/.ci/setup/createDeltaComponentsForPBaseCertB.py b/.ci/setup/createDeltaComponentsForPBaseCertB.py index 162a1206..e0b95538 100644 --- a/.ci/setup/createDeltaComponentsForPBaseCertB.py +++ b/.ci/setup/createDeltaComponentsForPBaseCertB.py @@ -2,10 +2,11 @@ # SIDeltaCertB1 - Bad Delta # VARDeltaCertB1 - Good Delta -import sys -import json +from __future__ import print_function import copy +import json import pprint +import sys try: pcDir = '/var/hirs/pc_generation/' diff --git a/.ci/setup/setup-tpm2provisioner-base-delta-bad.sh b/.ci/setup/setup-tpm2provisioner-base-delta-bad.sh index c2f4cad2..a6100c7c 100755 --- a/.ci/setup/setup-tpm2provisioner-base-delta-bad.sh +++ b/.ci/setup/setup-tpm2provisioner-base-delta-bad.sh @@ -69,7 +69,7 @@ function InitTpm2Emulator { echo "Running PACCOR to generate local component information..." # Use specific PACCOR script for system testing. # Will provide default component SN#s when needed. - cp -f /opt/paccor/scripts/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh + cp -f /HIRS/.ci/system-tests/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh /opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile echo @@ -184,6 +184,12 @@ InitTpm2Emulator # Update the hirs-site.config file UpdateHirsSiteConfigFile +# Set alias to use python3 +echo "===========Python Version===========" +python3 --version +alias python='/usr/bin/python3.6' +alias + echo "" echo "TPM 2.0 Emulator NV RAM list" tpm2_nvlist diff --git a/.ci/setup/setup-tpm2provisioner-base-delta-good.sh b/.ci/setup/setup-tpm2provisioner-base-delta-good.sh index b0fe9cdf..83f95e9a 100755 --- a/.ci/setup/setup-tpm2provisioner-base-delta-good.sh +++ b/.ci/setup/setup-tpm2provisioner-base-delta-good.sh @@ -75,7 +75,7 @@ function InitTpm2Emulator { echo "Running PACCOR to generate local component information..." # Use specific PACCOR script for system testing. # Will provide default component SN#s when needed. - cp -f /opt/paccor/scripts/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh + cp -f /HIRS/.ci/system-tests/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh /opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile # Split into JSON files needed to generate the certificates @@ -223,6 +223,12 @@ InitTpm2Emulator # Update the hirs-site.config file UpdateHirsSiteConfigFile +# Set alias to use python3 +echo "===========Python Version===========" +python3 --version +alias python='/usr/bin/python3.6' +alias + echo "" echo "TPM 2.0 Emulator NV RAM list" tpm2_nvlist diff --git a/.ci/setup/setup-tpm2provisioner.sh b/.ci/setup/setup-tpm2provisioner.sh index 23910c4e..67561928 100755 --- a/.ci/setup/setup-tpm2provisioner.sh +++ b/.ci/setup/setup-tpm2provisioner.sh @@ -67,7 +67,7 @@ function InitTpm2Emulator { echo "Running PACCOR to generate local component information..." # Use specific PACCOR script for system testing. # Will provide default component SN#s when needed. - cp -f /opt/paccor/scripts/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh + cp -f /HIRS/.ci/system-tests/allcomponents_hirs_system_tests.sh /opt/paccor/scripts/allcomponents.sh /opt/paccor/scripts/allcomponents.sh > $PC_DIR/componentsFile /opt/paccor/scripts/referenceoptions.sh > $PC_DIR/optionsFile /opt/paccor/scripts/otherextensions.sh > $PC_DIR/extensionsFile @@ -151,6 +151,12 @@ InitTpm2Emulator # Update the hirs-site.config file UpdateHirsSiteConfigFile +# Set alias to use python3 +echo "===========Python Version===========" +python3 --version +alias python='/usr/bin/python3.6' +alias + echo "" echo "TPM 2.0 Emulator NV RAM list" tpm2_nvlist diff --git a/.ci/setup/setup-tpmprovisioner.sh b/.ci/setup/setup-tpmprovisioner.sh index ba037824..81a7075d 100755 --- a/.ci/setup/setup-tpmprovisioner.sh +++ b/.ci/setup/setup-tpmprovisioner.sh @@ -144,5 +144,11 @@ UpdateHirsSiteConfigFile # Update the logging.properties file UpdateLoggingConfigFile +# Set alias to use python3 +echo "===========Python Version===========" +python3 --version +alias python='/usr/bin/python3.6' +alias + echo "" echo "===========HIRS ACA TPM 1.2 Provisioner Setup Complete!===========" diff --git a/.ci/system-tests/system_test.py b/.ci/system-tests/system_test.py index 62bacc09..1a2077b6 100644 --- a/.ci/system-tests/system_test.py +++ b/.ci/system-tests/system_test.py @@ -1,20 +1,20 @@ -# system_test.py - implements a group of tests that run appraisals on a client and server - -# TODO: test_01-test_11 will need to be implemented when the additional HIRS +# system_test.py - implements a group of tests that run appraisals on a client and server. +# TODO: test_02-test_12 will need to be implemented when the additional HIRS # projects are imported to the new GitHub repo. The test code is commented out for now. +from __future__ import print_function import logging import os import sys import unittest import urllib3 -from system_test_core import HIRSPortal, AttestationCAPortal, collectors, \ +from system_test_core import DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY, \ + HIRSPortal, AttestationCAPortal, collectors, \ send_command, send_command_sha1sum, run_hirs_report, run_hirs_provisioner_tpm_1_2, \ run_hirs_provisioner_tpm_2_0, parse_xml_with_stripped_namespaces, \ get_all_nodes_recursively, touch_random_file_and_remove, get_random_pcr_hex_value, \ get_current_timestamp, is_ubuntu_client, is_tpm_2_0, is_tpm_1_2, \ - DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY, \ make_simple_ima_baseline, make_baseline_from_xml, \ make_simple_ima_blacklist_baseline, \ make_simple_ima_blacklist_baseline_with_hash, \ @@ -611,9 +611,9 @@ class SystemTest(unittest.TestCase): """Test that running the TPM 1.2 hirs provisioner works""" logging.info("***************** Beginning of initial TPM 1.2 provisioner run *****************") -# # Run the provisioner to ensure that it provisions successfully -# provisioner_out = run_hirs_provisioner_tpm_1_2(CLIENT) -# print("Initial TPM 1.2 provisioner run output: {0}".format(provisioner_out)) + # Run the provisioner to ensure that it provisions successfully + provisioner_out = run_hirs_provisioner_tpm_1_2(CLIENT) + print("Initial TPM 1.2 provisioner run output: {0}".format(provisioner_out)) @collectors(['TPM'], COLLECTOR_LIST) @unittest.skipIf(not is_tpm_2_0(TPM_VERSION), "Skipping this test due to TPM Version " + TPM_VERSION) diff --git a/.ci/system-tests/system_test_Driver.py b/.ci/system-tests/system_test_Driver.py index e6af58fe..69d5e81c 100644 --- a/.ci/system-tests/system_test_Driver.py +++ b/.ci/system-tests/system_test_Driver.py @@ -1,30 +1,18 @@ # System Test Driver to help with debugging. -import binascii -from ConfigParser import SafeConfigParser -import datetime -import json -import os -import shlex -import subprocess -import unittest -import re -import requests +from __future__ import print_function import logging -import pprint -import hashlib -import random -import uuid -import time +import os import sys -import argparse +import unittest +import urllib3 -from system_test_core import HIRSPortal, AttestationCAPortal, collectors, \ +from system_test_core import DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY, \ + HIRSPortal, AttestationCAPortal, collectors, \ send_command, send_command_sha1sum, run_hirs_report, run_hirs_provisioner_tpm_1_2, \ run_hirs_provisioner_tpm_2_0, parse_xml_with_stripped_namespaces, get_current_timestamp, \ get_all_nodes_recursively, touch_random_file_and_remove, get_random_pcr_hex_value, \ - is_ubuntu_client, is_tpm_2_0, is_tpm_1_2, \ - DEFAULT_IMA_POLICY, DEFAULT_TPM_POLICY + is_ubuntu_client, is_tpm_2_0, is_tpm_1_2 \ NUMBER_OF_PCRS = 24 @@ -36,7 +24,8 @@ else: suffix = "" # Change to point to your HIRS directory -HOME_DIR = "/HIRS/" +#HOME_DIR = "/HIRS/" +HOME_DIR = "/workspace/git/python2to3-dev-3/" HIRS_ACA_PORTAL_IP="172.17.0.2" TPM_VERSION="2.0" #TPM_VERSION="1.2" @@ -98,7 +87,7 @@ logging.info("The ACA Portal is: " + HIRS_ATTESTATION_CA_PORTAL_URL) #Portal = HIRSPortal(HIRS_SERVER_URL) AcaPortal = AttestationCAPortal(HIRS_ATTESTATION_CA_PORTAL_URL) -requests.packages.urllib3.disable_warnings() +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) class SystemTest(unittest.TestCase): @@ -112,7 +101,7 @@ class SystemTest(unittest.TestCase): def setUp(self): """Set the systems tests state up for testing""" - #AcaPortal.disable_supply_chain_validations() + AcaPortal.disable_supply_chain_validations() def tearDown(self): """Tears down the state for testing""" diff --git a/.ci/system-tests/system_test_core.py b/.ci/system-tests/system_test_core.py index f2dc49b7..accba7ab 100644 --- a/.ci/system-tests/system_test_core.py +++ b/.ci/system-tests/system_test_core.py @@ -1,20 +1,24 @@ -# Defines core methods shared amongst system test scripts +# Defines core methods shared amongst system test scripts. -import sets -import unittest -import shlex -import subprocess -import os +from future import standard_library +standard_library.install_aliases() +from builtins import str +from builtins import filter +from builtins import object +from io import StringIO import binascii -import requests -import logging -import random -import time import datetime import json +import logging +import os import pprint +import random +import requests +import shlex +import subprocess +import time +import unittest import xml.etree.ElementTree as ET -from StringIO import StringIO DEFAULT_GROUP_NAME = "Default Group" DEFAULT_TPM_POLICY = "Test TPM Policy" @@ -23,7 +27,7 @@ CACHED_XML_REPORT = None APPRAISAL_SUCCESS_MESSAGE = "Appraisal passed" -class HIRSPortal: +class HIRSPortal(object): def __init__(self, hirs_server_url): self.server_url = hirs_server_url @@ -117,7 +121,7 @@ class HIRSPortal: # (works for full or partial path) return (record['hash']['digestString'] == sha_hash) and (filename in record['path']) - matching_records = filter(record_matcher, ima_records) + matching_records = list(filter(record_matcher, ima_records)) return len(matching_records) > 0 def upload_payload(self, payload): @@ -275,7 +279,7 @@ class HIRSPortal: def get_alerts(self): return self.request("get", "portal/alerts/list").json() -class AttestationCAPortal: +class AttestationCAPortal(object): def __init__(self, hirs_server_url): self.server_url = hirs_server_url @@ -362,8 +366,8 @@ def check_request_response(expected_status_codes, request_result, operation): raise RuntimeError(message) def collectors(collectors, collector_list): - enabled_collectors = sets.Set(collector_list) - tested_collectors = sets.Set(collectors) + enabled_collectors = set(collector_list) + tested_collectors = set(collectors) if tested_collectors.issubset(enabled_collectors): return lambda func: func return unittest.skip("{0} collector isn't enabled".format(tested_collectors.difference(enabled_collectors))) @@ -443,7 +447,7 @@ def parse_xml_with_stripped_namespaces(raw_xml_string): for _, el in it: if '}' in el.tag: el.tag = el.tag.split('}', 1)[1] # strip all namespaces - for at in el.attrib.keys(): # strip namespaces of attributes too + for at in list(el.attrib.keys()): # strip namespaces of attributes too if '}' in at: newat = at.split('}', 1)[1] el.attrib[newat] = el.attrib[at]