ExternalVendorCode/trick
1
0
Fork 0
mirror of https://github.com/nasa/trick.git synced 2024-12-18 20:57:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactored sim tests to run through TrickOps (#1390)

Browse Source
This commit is contained in:
Jacqueline Deans 2022-11-17 15:43:29 -06:00 committed by GitHub
parent f84c744a65
commit d9e7ac250d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 457 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/code_coverage.yml vendored
View File

@ -61,6 +61,9 @@ jobs:
export TRICK_SYSTEM_LDFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_SYSTEM_CFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_SYSTEM_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
cd share/trick/trickops/
python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
cd ../../../
make code-coverage
- name: Upload to Coveralls
uses: coverallsapp/github-action@master

4
.github/workflows/test_32.yml vendored
View File

@ -66,5 +66,9 @@ jobs:
make
- name: Run tests
run: |
cd share/trick/trickops/
yum install -y python3-devel
python3 -m venv .venv && . .venv/bin/activate && pip install --upgrade pip && pip3 install -r requirements.txt
cd ../../../
echo $MAKEFLAGS $CXXFLAGS $CFLAGS
make test

3
.github/workflows/test_32_oracle.yml vendored
View File

@ -71,6 +71,9 @@ jobs:
make
- name: Run tests
run: |
cd share/trick/trickops/
python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
cd ../../../
export CFLAGS="-m32"
export CXXFLAGS="-m32"
export MAKEFLAGS=-j`nproc`

7
.github/workflows/test_linux.yml vendored
View File

@ -67,6 +67,8 @@ jobs:
default-jdk
python2.7-dev
python3-dev
python3-pip
python3-venv
install_gtest: |
apt-get install -y libgtest-dev
cd /usr/src/gtest
@ -166,8 +168,9 @@ jobs:
make
- name: Test
run: |
export MAKEFLAGS=-j`nproc`
make test
cd share/trick/trickops/
python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
cd ../../../; make test
- name: Upload Tests
uses: actions/upload-artifact@v3.0.0
if: success() || failure() # run this step even if previous step failed

6
.github/workflows/test_macos.yml vendored
View File

@ -32,7 +32,9 @@ jobs:
export MAKEFLAGS=-j4
./configure --with-llvm=${GITHUB_WORKSPACE}/clang+llvm-14.0.6-x86_64-apple-darwin
make
- name: Run tests
- name: Test
run: |
cd share/trick/trickops/
python3 -m venv .venv && source .venv/bin/activate && pip3 install -r requirements.txt
export MAKEFLAGS=-j4
make test
cd ../../../; make test

3
.gitignore vendored
View File

@ -35,3 +35,6 @@ include/civet/
.vscode/
civet_server_error.log
server.pem
archive/
.venv
trickops_logs/

3
Makefile
View File

@ -301,8 +301,7 @@ $(DPX_UNIT_TEST_DIR):
sim_test:
@ $(MAKE) -C test
@ $(MAKE) -C trick_sims test
@ $(MAKE) -f test_overrides.mk sim_test
pytest:
make -C share/trick/pymods/trick

5
docs/developer_docs/Developer-Docs-Home.md
View File

@ -1,4 +1,9 @@
| [Home](/trick) → Developer Docs |
|------------------------------------------------------------------|
# Developer Documentation
Link documentation for Trick internals, processes, and plans here.
[Testing](Testing)

61
docs/developer_docs/Testing.md Normal file
View File

@ -0,0 +1,61 @@
| [Home](/trick) → [Developer Docs](Developer-Docs-Home) → Testing |
|------------------------------------------------------------------|
# Testing
Currently, Trick has a suite of unit and integration tests that run through a hodgepodge of GTest, Trick's internal unit test framework, Makefiles, and TrickOps.
Unit tests can be found in their respective `trick_source/[sim_services|utils]/*/test` directories, and primarily use the Gtest framework. These are run primarily through test targets in their Makefiles. See `trick_source/sim_services/MemoryManager/test` for an example of organization.
Integration tests take the form of Sims with some embedded tests. They live under `trick_sims/` and `test/`. A full list of sims that are used as part of the test suite are in [test_sims.yml](). These are run with TrickOps.
## Test suite dependencies
Gtest is required for the unit tests and some integration tests. See the [install guide](../documentation/install_guide/Install-Guide.md) for gtest installation.
TrickOps requires python3 and the packages `PyYAML` and `psutil` (updated list in [Trickops requirements.txt](https://github.com/nasa/trick/blob/master/share/trick/trickops/requirements.txt)). Install these in your python environment, or create a virtual environment as follows:
```
cd share/trick/trickops/
python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
cd ../../../
```
## Running the test suite
From trick home:
```
# Run everything
make test
# Run only integration tests
make sim_test
# Run only unit tests
make unit_test
```
Currently, TrickOps will redirect all console output from tests into logs under {TRICK_HOME}/trickops_logs/, and will also dump the output of failing logs to console after the test suite is finished.
## Coverage
Trick uses [Coveralls](https://coveralls.io/github/nasa/trick?branch=master) to host code coverage. Coverage is generated by running the test suite with gcov in CI, and then those files are uploaded to Coveralls.
To enable gcov in Trick, it must be cleaned and compiled with the following:
```
export CFLAGS="-fprofile-arcs -ftest-coverage -O0"
export CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
export LDFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_CFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_SYSTEM_LDFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_SYSTEM_CFLAGS="-fprofile-arcs -ftest-coverage -O0"
export TRICK_SYSTEM_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
```
After Trick has been rebuild with the instrumentation, run:
```
make code-coverage
```
This will generate, collect, and filter all the various coverage data collection files into `coverage.info`. This is the file that is uploaded to Coveralls in the [code_coverage.yml](https://github.com/nasa/trick/blob/master/.github/workflows/code_coverage.yml) Github Actions workflow.

1
docs/documentation/miscellaneous_trick_tools/TrickOps.md
View File

@ -64,6 +64,7 @@ SIM_abc: <-- required unique name for sim of interest, must start
- model_x by label within the framework, or for any other project-defined
- verification purpose
build_command: <-- optional literal cmd executed for SIM_build, defaults to trick-CP
binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe
size: <-- optional estimated size of successful build output file in bytes
runs: <-- optional dict of runs to be executed for this sim, where the
RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary

21
share/trick/trickops/TrickWorkflow.py
View File

@ -102,7 +102,7 @@ class TrickWorkflow(WorkflowCommon):
super().__init__(project_top_level=project_top_level, log_dir=log_dir, quiet=quiet)
# If not found in the config file, these defaults are used
self.defaults = {'cpus': 3, 'name': None, 'description': None,
'build_command': 'trick-CP', 'size': 2200000, 'env': None}
'build_command': 'trick-CP', 'size': 2200000, 'env': None, 'binary': 'S_main_{cpu}.exe'}
self.config_errors = False
self.compare_errors = False # True if comparison errors were found
self.sims = [] # list of Sim() instances, filled out from config file
@ -269,6 +269,7 @@ class TrickWorkflow(WorkflowCommon):
- model_x by label within the framework, or for any other project-defined
- verification purpose
build_command: <-- optional literal cmd executed for SIM_build, defaults to trick-CP
binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe
size: <-- optional estimated size of successful build output file in bytes
runs: <-- optional dict of runs to be executed for this sim, where the
RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary
@ -390,11 +391,22 @@ class TrickWorkflow(WorkflowCommon):
sanitized_labels =( [l for l in c[s]['labels'] if type_expected(
l, expected_type=str, extra_msg='Ignoring label "%s".' % l) ] )
self.config[s]['labels'] = sanitized_labels
# Check for binary argument
if 'binary' not in c[s] or not c[s]['binary']:
self.config[s]['binary'] = self.defaults['binary']
# Add the CPU format string (won't fail if there is no CPU placeholder)
self.config[s]['binary'] = self.config[s]['binary'].format(cpu=self.trick_host_cpu)
# Add the full path to the build command
self.config[s]['build_command'] = os.path.join(self.trick_dir, "bin", self.config[s]['build_command'])
# Create internal object to be populated with runs, valgrind runs, etc
thisSim = TrickWorkflow.Sim(name=s, sim_dir=self.config[s]['path'],
description=self.config[s]['description'], labels=self.config[s]['labels'],
prebuild_cmd=self.env, build_cmd=self.config[s]['build_command'],
cpus=self.cpus, size=self.config[s]['size'], log_dir=self.log_dir)
all_sim_paths.append(c[s]['path'])
# RUN sanity checks
if 'runs' in c[s]: # If it's there...
@ -442,7 +454,7 @@ class TrickWorkflow(WorkflowCommon):
self.config[s]['runs'][r]['returns'] = 0 # Default to zero
# Create internal object to be added to thisSim
thisRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r,
binary= 'S_main_' + self.trick_host_cpu + '.exe', prerun_cmd=self.env,
binary= self.config[s]['binary'], prerun_cmd=self.env,
returns=self.config[s]['runs'][r]['returns'],
valgrind_flags=None, log_dir=self.log_dir)
# Handle 'compare' option, if given, if not, assume 0
@ -502,7 +514,7 @@ class TrickWorkflow(WorkflowCommon):
else:
# Create internal object to be added to thisSim
vRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r,
binary= 'S_main_' + self.trick_host_cpu + '.exe',
binary= self.config[s]['binary'],
prerun_cmd=self.env, valgrind_flags=self.config[s]['valgrind']['flags'],
log_dir=self.log_dir)
thisSim.add_run(vRun)
@ -811,6 +823,7 @@ class TrickWorkflow(WorkflowCommon):
FileSizeJob()
Instance of FileSizeJob() job for compiling this sim
"""
if not self.build_job:
name = 'Build ' + self.sim_dir
self.build_job = FileSizeJob(name=name,
@ -1019,7 +1032,7 @@ class TrickWorkflow(WorkflowCommon):
including command-line arguments
ex: RUN_test/input.py, or RUN_test/input.py --my-arg
binary : str
Name of executable, usually S_min.. bu platform specific
Name of executable, usually S_main.. but platform specific
prerun_cmd : str
Optional string to execute immediately before sim run, e.g. env sourcing
valgrind_flags : str

3
share/trick/trickops/tests/trick_sims.yml
View File

@ -77,10 +77,13 @@ SIM_rti:
labels:
- unit_test
SIM_segments:
binary: 'T_main_*.exe'
path: test/SIM_segments
runs:
RUN_test/input.py:
SIM_stls:
binary: 'T_main_{cpu}_test.exe'
build_command: "trick-CP -t"
path: test/SIM_stls
labels:
- unit_test

23
share/trick/trickops/tests/ut_TrickWorkflow.py
View File

@ -117,6 +117,29 @@ class TrickWorkflowTestCase(unittest.TestCase):
sim = self.instance.get_sim(identifier='SIM_ball_L1')
run = sim.get_run('RUN_test/input.py')
def test_set_binary_with_cpu_formatter (self):
sim = self.instance.get_sim(identifier="SIM_stls")
cpu = self.instance.get_trick_host_cpu()
run = sim.get_run('RUN_test/unit_test.py')
print("Binary: " , run.binary)
self.assertEqual(run.binary, "T_main_" + cpu + "_test.exe")
def test_set_binary_without_cpu_formatter (self):
sim = self.instance.get_sim(identifier="SIM_segments")
run = sim.get_run('RUN_test/input.py')
print("Binary: " , run.binary)
self.assertEqual(run.binary, "T_main_*.exe")
def test_absolute_path_to_custom_build_cmd (self):
sim = self.instance.get_sim(identifier="SIM_stls")
absolute_path = os.path.join(self.instance.trick_dir, "bin", "trick-CP -t")
self.assertEqual(sim.build_cmd, absolute_path)
def test_absolute_path_to_default_build_cmd (self):
sim = self.instance.get_sim(identifier="SIM_segments")
absolute_path = os.path.join(self.instance.trick_dir, "bin", "trick-CP")
self.assertEqual(sim.build_cmd, absolute_path)
def test_get_unique_comparison_dirs(self):
ucd = self.instance.get_unique_comparison_dirs()
self.assertTrue(ucd[0] is not None)

16
test/Makefile Normal file
View File

@ -0,0 +1,16 @@
TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..)
export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
SIM_DIRECTORIES = $(wildcard SIM_*)
UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIM_DIRECTORIES)))
clean:
rm -f $(UNIT_TEST_RESULTS)
- for i in $(SIM_DIRECTORIES) ; do \
if [ -f "$$i/"[Mm]"akefile" ] ; then \
$(MAKE) -C $$i spotless ; \
fi \
done

9
test/SIM_stls/S_overrides.mk
View File

@ -4,15 +4,6 @@ TRICK_CXXFLAGS += -I./models
TRICK_CXXFLAGS += -std=c++11
test: setup
# This is a workaround so that the sim runs with RUN_test/setup.py before the actual unit test run.
# setup.py adds hardcoded data to the sim and dumps a checkpoint, and unit_test.py loads that checkpoint and checks all the values.
setup: T_main_${TRICK_HOST_CPU}.exe
echo "Running Setup Job"
./T_main_${TRICK_HOST_CPU}.exe RUN_test/setup.py
clean: checkpoint_clean
checkpoint_clean:

87
test/makefile
View File

@ -1,87 +0,0 @@
TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..)
export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
# Remove when issue #1147 is fixed: https://github.com/nasa/trick/issues/1147
unexport TRICK_PYTHON_PATH
SIMS_TO_COMPILE_ONLY = \
SIM_alloc_test \
SIM_anon_enum \
SIM_default_member_initializer \
SIM_delete_default_constructor \
SIM_demo_inputfile \
SIM_exclusion_mechanisms \
SIM_isystem \
SIM_measurement_units \
SIM_parse_s_define \
SIM_target_specific_variables \
SIM_test_abstract \
SIM_test_inherit \
SIM_test_ip2 \
SIM_threads_simple \
SIM_trickcomm
SIMS_TO_COMPILE_AND_RUN = \
SIM_demo_sdefine \
SIM_events \
SIM_exec_set_time_tic_value \
SIM_python_namespace \
SIM_rti \
SIM_stls \
SIM_test_dp \
SIM_test_dr \
SIM_test_icg_file_skipped \
SIM_test_io \
SIM_test_ip \
SIM_test_sched \
SIM_test_templates \
SIM_threads \
SIM_trickified
# SIM_stls will run twice. First run is RUN_test/setup.py, which is added as a dependency to test in its S_overrides.mk file
# The second run is the normal RUN_test/unit_test.py which is run through this makefile.
# Sims with problems, no purpose, or maybe shouldn't be a test
# SIM_leaks ( should be deleted )
# SIM_dynamic_sim_object ( not running, class won't instantiate )
# SIM_segments ( not a test, but a demo )
# This test is temporarily sitting out until fixed.
# SIM_test_varserv
EXECUTABLES = $(addsuffix /T_main_${TRICK_HOST_CPU}_test.exe, $(SIMS_TO_COMPILE_AND_RUN) $(SIMS_TO_COMPILE_ONLY))
UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIMS_TO_COMPILE_AND_RUN)))
test: $(EXECUTABLES) $(UNIT_TEST_RESULTS) data_record_results
clean:
rm -f $(UNIT_TEST_RESULTS)
- for i in $(SIMS_TO_COMPILE_AND_RUN) ; do \
if [ -f "$$i/"[Mm]"akefile" ] ; then \
$(MAKE) -C $$i spotless ; \
fi \
done
- for i in $(SIMS_TO_COMPILE_ONLY) ; do \
if [ -f "$$i/"[Mm]"akefile" ] ; then \
$(MAKE) -C $$i spotless ; \
fi \
done
$(EXECUTABLES):
@ cd $(@D) ; ${TRICK_HOME}/bin/trick-CP -t
$(UNIT_TEST_RESULTS): $(TRICK_HOME)/trick_test/%.xml : %/T_main_${TRICK_HOST_CPU}_test.exe
@ cd $* ; ./T_main_${TRICK_HOST_CPU}_test.exe RUN_test/unit_test.py
DR_RESULTS = $(TRICK_HOME)/test/SIM_test_dr/RUN_test
data_record_results: $(TRICK_HOME)/trick_test/SIM_test_dr.xml $(DR_RESULTS)
diff $(DR_RESULTS)/log_DR_bitfieldsASCII.csv $(DR_RESULTS)/Ref_Logs/log_DR_bitfieldsASCII_Master.csv
diff $(DR_RESULTS)/log_DR_typesASCII.csv $(DR_RESULTS)/Ref_Logs/log_DR_typesASCII_Master.csv
cmp -b $(DR_RESULTS)/log_DR_bitfieldsBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_bitfieldsBINARY.trk
ifeq (${TRICK_FORCE_32BIT}, 1)
cmp -b $(DR_RESULTS)/log_DR_typesBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_typesBINARY_32.trk
else
cmp -b $(DR_RESULTS)/log_DR_typesBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_typesBINARY.trk
endif

9
test_overrides.mk Normal file
View File

@ -0,0 +1,9 @@
TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
# Remove when issue #1147 is fixed: https://github.com/nasa/trick/issues/1147
unexport TRICK_PYTHON_PATH
sim_test:
python3 trickops.py ${TRICK_HOME}

235
test_sims.yml Normal file
View File

@ -0,0 +1,235 @@
# Compile only sims
SIM_alloc_test:
path: test/SIM_alloc_test
SIM_alloc_test:
path: test/SIM_alloc_test
SIM_anon_enum:
path: test/SIM_anon_enum
SIM_default_member_initializer:
path: test/SIM_default_member_initializer
SIM_delete_default_constructor:
path: test/SIM_delete_default_constructor
SIM_demo_inputfile:
path: test/SIM_demo_inputfile
SIM_exclusion_mechanisms:
path: test/SIM_exclusion_mechanisms
SIM_isystem:
path: test/SIM_isystem
SIM_measurement_units:
path: test/SIM_measurement_units
SIM_parse_s_define:
path: test/SIM_parse_s_define
# SIM_target_specific_variables: <-- This sim has to sit out on Mac until we fix the sprintf issue
# path: test/SIM_target_specific_variables
SIM_test_abstract:
path: test/SIM_test_abstract
SIM_test_inherit:
path: test/SIM_test_inherit
SIM_test_ip2:
path: test/SIM_test_ip2
SIM_threads_simple:
path: test/SIM_threads_simple
SIM_trickcomm:
path: test/SIM_trickcomm
SIM_satellite:
path: trick_sims/SIM_satellite
# Normal case compile and run sims
SIM_demo_sdefine:
path: test/SIM_demo_sdefine
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_events:
path: test/SIM_events
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_exec_set_time_tic_value:
path: test/SIM_exec_set_time_tic_value
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_python_namespace:
path: test/SIM_python_namespace
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_rti:
path: test/SIM_rti
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_test_dp:
path: test/SIM_test_dp
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_test_icg_file_skipped:
path: test/SIM_test_icg_file_skipped
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_test_io:
path: test/SIM_test_io
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_test_ip:
path: test/SIM_test_ip
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_test_sched:
path: test/SIM_test_sched
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_test_templates:
path: test/SIM_test_templates
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_threads:
path: test/SIM_threads
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_trickified:
path: test/SIM_trickified
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_ball_L1:
path: trick_sims/Ball/SIM_ball_L1
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_ball_L2:
path: trick_sims/Ball/SIM_ball_L2
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_ball_L3:
path: trick_sims/Ball/SIM_ball_L3
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
# SIM_amoeba:
# path: trick_sims/Cannon/SIM_amoeba
# build_command: "trick-CP -t"
# binary: "T_main_{cpu}_test.exe"
# runs:
# RUN_test/unit_test.py:
# returns: 0
SIM_cannon_aero:
path: trick_sims/Cannon/SIM_cannon_aero
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_cannon_analytic:
path: trick_sims/Cannon/SIM_cannon_analytic
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_cannon_eulercromer:
path: trick_sims/Cannon/SIM_cannon_eulercromer
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_cannon_numeric:
path: trick_sims/Cannon/SIM_cannon_numeric
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_cannon_jet:
path: trick_sims/Cannon/SIM_cannon_jet
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_Ball++_L1:
path: trick_sims/SIM_Ball++_L1
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
SIM_sun:
path: trick_sims/SIM_sun
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
# Special cases
# setup.py dumps a checkpoint
# unit_test.py loads that checkpoint and verifies the data
SIM_stls:
path: test/SIM_stls
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/setup.py:
returns: 0
RUN_test/unit_test.py:
returns: 0
SIM_test_dr:
path: test/SIM_test_dr
build_command: "trick-CP -t"
binary: "T_main_{cpu}_test.exe"
runs:
RUN_test/unit_test.py:
returns: 0
compare:
- test/SIM_test_dr/RUN_test/log_DR_bitfieldsASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsASCII_Master.csv
- test/SIM_test_dr/RUN_test/log_DR_typesASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_typesASCII_Master.csv
- test/SIM_test_dr/RUN_test/log_DR_bitfieldsBINARY.trk vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsBINARY.trk

61
trick_sims/makefile
View File

@ -1,59 +1,18 @@
TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..)
export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
# List out sims we want to compile
COMPILE_DIRS = \
Ball/SIM_ball_L1 \
Ball/SIM_ball_L2 \
Ball/SIM_ball_L3 \
Cannon/SIM_amoeba \
Cannon/SIM_cannon_aero \
Cannon/SIM_cannon_analytic \
Cannon/SIM_cannon_eulercromer \
Cannon/SIM_cannon_numeric \
Cannon/SIM_cannon_jet \
SIM_Ball++_L1 \
SIM_satellite \
SIM_sun
#SIM_monte \
NESTED_SIM_DIRECTORIES = $(wildcard */SIM_*)
SIM_DIRECTORIES = $(wildcard SIM_*) $(NESTED_SIM_DIRECTORIES)
# List out sims we want to run unit tests
TEST_DIRS = \
Ball/SIM_ball_L1 \
Ball/SIM_ball_L2 \
Ball/SIM_ball_L3 \
Cannon/SIM_amoeba \
Cannon/SIM_cannon_aero \
Cannon/SIM_cannon_analytic \
Cannon/SIM_cannon_eulercromer \
Cannon/SIM_cannon_numeric \
Cannon/SIM_cannon_jet \
SIM_Ball++_L1 \
SIM_sun
UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIM_DIRECTORIES)))
#SIM_monte \
EXECUTABLES = $(addsuffix /T_main_${TRICK_HOST_CPU}_test.exe, $(COMPILE_DIRS))
UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(TEST_DIRS)))
all:
@echo "This makefile is used with Trick's top level 'make test' command"
test: $(EXECUTABLES) $(UNIT_TEST_RESULTS)
clean:
rm -f $(UNIT_TEST_RESULTS)
- for i in $(COMPILE_DIRS) ; do \
if [ -f "$$i/"[Mm]"akefile" ] ; then \
$(MAKE) -C $$i spotless ; \
fi \
done
$(EXECUTABLES):
@ cd $(@D) ; ${TRICK_HOME}/bin/trick-CP -t ;
$(UNIT_TEST_RESULTS): $(TRICK_HOME)/trick_test/%.xml : %/T_main_${TRICK_HOST_CPU}_test.exe
@ cd $* ; ./T_main_${TRICK_HOST_CPU}_test.exe RUN_test/unit_test.py
- for i in $(SIM_DIRECTORIES) ; do \
if [ -f "$$i/"[Mm]"akefile" ] ; then \
$(MAKE) -C $$i spotless ; \
fi \
done

53
trickops.py Normal file
View File

@ -0,0 +1,53 @@
import sys
import os
sys.path.append(sys.argv[1] + "/share/trick/trickops")
from TrickWorkflow import *
from WorkflowCommon import Job
class SimTestWorkflow(TrickWorkflow):
def __init__( self, quiet, trick_top_level ):
# Create the trick_test directory if it doesn't already exist
if not os.path.exists(trick_top_level + "/trick_test"):
os.makedirs(trick_top_level + "/trick_test")
# Base Class initialize, this creates internal management structures
num_cpus = os.cpu_count() if os.cpu_count() is not None else 8
TrickWorkflow.__init__(self, project_top_level=(trick_top_level), log_dir=(trick_top_level +'/trickops_logs/'),
trick_dir=trick_top_level, config_file=(trick_top_level + "/test_sims.yml"), cpus=num_cpus, quiet=quiet)
def run( self ):
build_jobs = self.get_jobs(kind='build')
run_jobs = self.get_jobs(kind='run')
analysis_jobs = self.get_jobs(kind='analyze')
# This job dumps a checkpoint that is then read in and checked by RUN_test/unit_test.py in the same sim
# This is a workaround to ensure that this run goes first.
first_phase_job = self.get_sim('SIM_stls').get_run(input='RUN_test/setup.py').get_run_job()
run_jobs.remove(first_phase_job)
builds_status = self.execute_jobs(build_jobs, max_concurrent=self.cpus, header='Executing all sim builds.')
first_phase_run_status = self.execute_jobs([first_phase_job], max_concurrent=self.cpus, header="Executing required first phase runs.")
runs_status = self.execute_jobs(run_jobs, max_concurrent=self.cpus, header='Executing all sim runs.')
comparison_result = self.compare()
analysis_status = self.execute_jobs(analysis_jobs, max_concurrent=self.cpus, header='Executing all analysis.')
self.report() # Print Verbose report
self.status_summary() # Print a Succinct summary
# Dump failing logs
jobs = build_jobs + run_jobs
for job in jobs:
if job.get_status() == Job.Status.FAILED:
print("Failing job: ", job.name)
print ("*"*120)
print(open(job.log_file, "r").read())
print ("*"*120, "\n")
return (builds_status or runs_status or first_phase_run_status or self.config_errors or comparison_result or analysis_status)
if __name__ == "__main__":
should_be_quiet = os.getenv('CI') is not None
sys.exit(SimTestWorkflow(quiet=should_be_quiet, trick_top_level=sys.argv[1]).run())