Refactored sim tests to run through TrickOps (#1390)

.github/workflows/code_coverage.yml

@@ -61,6 +61,9 @@ jobs:
         export TRICK_SYSTEM_LDFLAGS="-fprofile-arcs -ftest-coverage -O0"
         export TRICK_SYSTEM_CFLAGS="-fprofile-arcs -ftest-coverage -O0"
         export TRICK_SYSTEM_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
+        cd share/trick/trickops/
+        python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
+        cd ../../../
         make code-coverage
     - name: Upload to Coveralls
       uses: coverallsapp/github-action@master

.github/workflows/test_32.yml

@@ -66,5 +66,9 @@ jobs:
         make
     - name: Run tests
       run: |
+        cd share/trick/trickops/
+        yum install -y python3-devel
+        python3 -m venv .venv && . .venv/bin/activate && pip install --upgrade pip && pip3 install -r requirements.txt        
+        cd ../../../
         echo $MAKEFLAGS $CXXFLAGS $CFLAGS 
         make test 

.github/workflows/test_32_oracle.yml

@@ -71,6 +71,9 @@ jobs:
         make
     - name: Run tests
       run: |
+        cd share/trick/trickops/
+        python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
+        cd ../../../
         export CFLAGS="-m32"
         export CXXFLAGS="-m32"
         export MAKEFLAGS=-j`nproc`

.github/workflows/test_linux.yml

@@ -67,6 +67,8 @@ jobs:
               default-jdk
               python2.7-dev
               python3-dev
+              python3-pip
+              python3-venv
             install_gtest: |
               apt-get install -y libgtest-dev
               cd /usr/src/gtest
@@ -166,8 +168,9 @@ jobs:
           make
       - name: Test
         run: |
-          export MAKEFLAGS=-j`nproc`
+          cd share/trick/trickops/
-          make test
+          python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
+          cd ../../../; make test
       - name: Upload Tests
         uses: actions/upload-artifact@v3.0.0
         if: success() || failure()    # run this step even if previous step failed

.github/workflows/test_macos.yml

@@ -32,7 +32,9 @@ jobs:
         export MAKEFLAGS=-j4
         ./configure --with-llvm=${GITHUB_WORKSPACE}/clang+llvm-14.0.6-x86_64-apple-darwin
         make
-    - name: Run tests
+    - name: Test
       run: |
+        cd share/trick/trickops/
+        python3 -m venv .venv && source .venv/bin/activate && pip3 install -r requirements.txt
         export MAKEFLAGS=-j4
-        make test
+        cd ../../../; make test

.gitignore

@@ -34,4 +34,7 @@ civetweb_clone/
 include/civet/
 .vscode/
 civet_server_error.log
-server.pem
+server.pem
+archive/
+.venv
+trickops_logs/

Makefile

@@ -301,8 +301,7 @@ $(DPX_UNIT_TEST_DIR):
 
 
 sim_test:
-	@ $(MAKE) -C test
+	@ $(MAKE) -f test_overrides.mk sim_test
-	@ $(MAKE) -C trick_sims test
 
 pytest:
 	make -C share/trick/pymods/trick

docs/developer_docs/Developer-Docs-Home.md

@@ -1,4 +1,9 @@
+| [Home](/trick) → Developer Docs |
+|------------------------------------------------------------------|
+
 # Developer Documentation
 
+
 Link documentation for Trick internals, processes, and plans here.
 
+[Testing](Testing)

docs/developer_docs/Testing.md

@@ -0,0 +1,61 @@
+| [Home](/trick) → [Developer Docs](Developer-Docs-Home) → Testing |
+|------------------------------------------------------------------|
+
+
+# Testing
+
+Currently, Trick has a suite of unit and integration tests that run through a hodgepodge of GTest, Trick's internal unit test framework, Makefiles, and TrickOps. 
+
+Unit tests can be found in their respective `trick_source/[sim_services|utils]/*/test` directories, and primarily use the Gtest framework. These are run primarily through test targets in their Makefiles. See `trick_source/sim_services/MemoryManager/test` for an example of organization.
+
+Integration tests take the form of Sims with some embedded tests. They live under `trick_sims/` and `test/`. A full list of sims that are used as part of the test suite are in [test_sims.yml](). These are run with TrickOps.
+
+## Test suite dependencies
+
+Gtest is required for the unit tests and some integration tests. See the [install guide](../documentation/install_guide/Install-Guide.md) for gtest installation.
+
+TrickOps requires python3 and the packages `PyYAML` and `psutil` (updated list in [Trickops requirements.txt](https://github.com/nasa/trick/blob/master/share/trick/trickops/requirements.txt)). Install these in your python environment, or create a virtual environment as follows:
+```
+cd share/trick/trickops/
+python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt
+cd ../../../
+```
+
+
+## Running the test suite
+From trick home:
+```
+# Run everything
+make test
+
+# Run only integration tests
+make sim_test
+
+# Run only unit tests
+make unit_test
+```
+
+Currently, TrickOps will redirect all console output from tests into logs under {TRICK_HOME}/trickops_logs/, and will also dump the output of failing logs to console after the test suite is finished.
+
+## Coverage
+Trick uses [Coveralls](https://coveralls.io/github/nasa/trick?branch=master) to host code coverage. Coverage is generated by running the test suite with gcov in CI, and then those files are uploaded to Coveralls.
+
+To enable gcov in Trick, it must be cleaned and compiled with the following:
+```
+export CFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export LDFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export TRICK_CFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export TRICK_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export TRICK_SYSTEM_LDFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export TRICK_SYSTEM_CFLAGS="-fprofile-arcs -ftest-coverage -O0"
+export TRICK_SYSTEM_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0"
+```
+
+After Trick has been rebuild with the instrumentation, run:
+```
+make code-coverage
+```
+This will generate, collect, and filter all the various coverage data collection files into `coverage.info`. This is the file that is uploaded to Coveralls in the [code_coverage.yml](https://github.com/nasa/trick/blob/master/.github/workflows/code_coverage.yml) Github Actions workflow.
+
+

docs/documentation/miscellaneous_trick_tools/TrickOps.md

@@ -64,6 +64,7 @@ SIM_abc:               <-- required unique name for sim of interest, must start
       - model_x            by label within the framework, or for any other project-defined
       - verification       purpose
   build_command:       <-- optional literal cmd executed for SIM_build, defaults to trick-CP
+  binary:              <-- optional name of sim binary, defaults to S_main_{cpu}.exe
   size:                <-- optional estimated size of successful build output file in bytes
   runs:                <-- optional dict of runs to be executed for this sim, where the
     RUN_1/input.py --foo:  dict keys are the literal arguments passed to the sim binary

share/trick/trickops/TrickWorkflow.py

@@ -102,7 +102,7 @@ class TrickWorkflow(WorkflowCommon):
         super().__init__(project_top_level=project_top_level, log_dir=log_dir, quiet=quiet)
         # If not found in the config file, these defaults are used
         self.defaults = {'cpus': 3, 'name': None, 'description': None,
-          'build_command': 'trick-CP', 'size': 2200000, 'env': None}
+          'build_command': 'trick-CP', 'size': 2200000, 'env': None, 'binary': 'S_main_{cpu}.exe'}
         self.config_errors = False
         self.compare_errors = False     # True if comparison errors were found
         self.sims = []                  # list of Sim() instances, filled out from config file
@@ -269,6 +269,7 @@ class TrickWorkflow(WorkflowCommon):
               - model_x            by label within the framework, or for any other project-defined
               - verification       purpose
           build_command:       <-- optional literal cmd executed for SIM_build, defaults to trick-CP
+          binary:              <-- optional name of sim binary, defaults to S_main_{cpu}.exe
           size:                <-- optional estimated size of successful build output file in bytes
           runs:                <-- optional dict of runs to be executed for this sim, where the
             RUN_1/input.py --foo:  dict keys are the literal arguments passed to the sim binary
@@ -390,11 +391,22 @@ class TrickWorkflow(WorkflowCommon):
             sanitized_labels =( [l for l in c[s]['labels'] if type_expected(
               l, expected_type=str, extra_msg='Ignoring label "%s".' % l) ] )
             self.config[s]['labels'] = sanitized_labels
+          # Check for binary argument
+          if 'binary' not in c[s] or not c[s]['binary']:
+            self.config[s]['binary'] = self.defaults['binary']
+
+          # Add the CPU format string (won't fail if there is no CPU placeholder)
+          self.config[s]['binary'] = self.config[s]['binary'].format(cpu=self.trick_host_cpu)
+
+          # Add the full path to the build command
+          self.config[s]['build_command'] = os.path.join(self.trick_dir, "bin", self.config[s]['build_command'])
+
           # Create internal object to be populated with runs, valgrind runs, etc
           thisSim = TrickWorkflow.Sim(name=s, sim_dir=self.config[s]['path'],
             description=self.config[s]['description'], labels=self.config[s]['labels'],
             prebuild_cmd=self.env, build_cmd=self.config[s]['build_command'],
             cpus=self.cpus, size=self.config[s]['size'], log_dir=self.log_dir)
+            
           all_sim_paths.append(c[s]['path'])
           # RUN sanity checks
           if 'runs' in c[s]:         # If it's there...
@@ -442,7 +454,7 @@ class TrickWorkflow(WorkflowCommon):
                   self.config[s]['runs'][r]['returns'] = 0 # Default to zero
                 # Create internal object to be added to thisSim
                 thisRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r,
-                              binary= 'S_main_' + self.trick_host_cpu + '.exe', prerun_cmd=self.env,
+                              binary= self.config[s]['binary'], prerun_cmd=self.env,
                               returns=self.config[s]['runs'][r]['returns'],
                               valgrind_flags=None, log_dir=self.log_dir)
                 # Handle 'compare' option, if given, if not, assume 0
@@ -502,7 +514,7 @@ class TrickWorkflow(WorkflowCommon):
                       else:
                         # Create internal object to be added to thisSim
                         vRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r,
-                          binary= 'S_main_' + self.trick_host_cpu + '.exe',
+                          binary= self.config[s]['binary'],
                           prerun_cmd=self.env, valgrind_flags=self.config[s]['valgrind']['flags'],
                           log_dir=self.log_dir)
                         thisSim.add_run(vRun)
@@ -811,6 +823,7 @@ class TrickWorkflow(WorkflowCommon):
             FileSizeJob()
                 Instance of FileSizeJob() job for compiling this sim
             """
+
             if not self.build_job:
                 name = 'Build ' + self.sim_dir
                 self.build_job = FileSizeJob(name=name,
@@ -1019,7 +1032,7 @@ class TrickWorkflow(WorkflowCommon):
                 including command-line arguments
                 ex: RUN_test/input.py, or RUN_test/input.py --my-arg
              binary : str
-                Name of executable, usually S_min.. bu platform specific
+                Name of executable, usually S_main.. but platform specific
              prerun_cmd : str
                 Optional string to execute immediately before sim run, e.g. env sourcing
              valgrind_flags : str

share/trick/trickops/tests/trick_sims.yml

@@ -77,10 +77,13 @@ SIM_rti:
     labels:
       - unit_test
 SIM_segments:
+    binary: 'T_main_*.exe'
     path: test/SIM_segments
     runs:
         RUN_test/input.py:
 SIM_stls:
+    binary: 'T_main_{cpu}_test.exe'
+    build_command: "trick-CP -t"
     path: test/SIM_stls
     labels:
       - unit_test

share/trick/trickops/tests/ut_TrickWorkflow.py

@@ -117,6 +117,29 @@ class TrickWorkflowTestCase(unittest.TestCase):
         sim = self.instance.get_sim(identifier='SIM_ball_L1')
         run = sim.get_run('RUN_test/input.py')
 
+    def test_set_binary_with_cpu_formatter (self):
+        sim = self.instance.get_sim(identifier="SIM_stls")
+        cpu = self.instance.get_trick_host_cpu()
+        run = sim.get_run('RUN_test/unit_test.py')
+        print("Binary: " , run.binary)
+        self.assertEqual(run.binary, "T_main_" + cpu + "_test.exe")
+
+    def test_set_binary_without_cpu_formatter (self):
+        sim = self.instance.get_sim(identifier="SIM_segments")
+        run = sim.get_run('RUN_test/input.py')
+        print("Binary: " , run.binary)
+        self.assertEqual(run.binary, "T_main_*.exe")
+
+    def test_absolute_path_to_custom_build_cmd (self):
+        sim = self.instance.get_sim(identifier="SIM_stls")
+        absolute_path = os.path.join(self.instance.trick_dir, "bin", "trick-CP -t")
+        self.assertEqual(sim.build_cmd, absolute_path)
+
+    def test_absolute_path_to_default_build_cmd (self):
+        sim = self.instance.get_sim(identifier="SIM_segments")
+        absolute_path = os.path.join(self.instance.trick_dir, "bin", "trick-CP")
+        self.assertEqual(sim.build_cmd, absolute_path)
+
     def test_get_unique_comparison_dirs(self):
         ucd = self.instance.get_unique_comparison_dirs()
         self.assertTrue(ucd[0] is not None)

test/Makefile

@@ -0,0 +1,16 @@
+TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..)
+export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
+include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
+
+
+SIM_DIRECTORIES = $(wildcard SIM_*)
+UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIM_DIRECTORIES)))
+
+
+clean:
+	rm -f $(UNIT_TEST_RESULTS)
+	- for i in $(SIM_DIRECTORIES) ; do \
+		if [ -f "$$i/"[Mm]"akefile" ] ; then \
+			$(MAKE) -C $$i spotless ; \
+		fi \
+	done

test/SIM_stls/S_overrides.mk

@@ -4,15 +4,6 @@ TRICK_CXXFLAGS += -I./models
 
 TRICK_CXXFLAGS += -std=c++11
 
-test: setup
-
-# This is a workaround so that the sim runs with RUN_test/setup.py before the actual unit test run.
-# setup.py adds hardcoded data to the sim and dumps a checkpoint, and unit_test.py loads that checkpoint and checks all the values.
-
-setup: T_main_${TRICK_HOST_CPU}.exe 
-	echo "Running Setup Job"
-	./T_main_${TRICK_HOST_CPU}.exe RUN_test/setup.py
-
 clean: checkpoint_clean
 
 checkpoint_clean:

test/makefile

@@ -1,87 +0,0 @@
-TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..)
-export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
-include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
-
-# Remove when issue #1147 is fixed: https://github.com/nasa/trick/issues/1147
-unexport TRICK_PYTHON_PATH
-
-SIMS_TO_COMPILE_ONLY = \
-	SIM_alloc_test \
-	SIM_anon_enum \
-	SIM_default_member_initializer \
-	SIM_delete_default_constructor \
-	SIM_demo_inputfile \
-	SIM_exclusion_mechanisms \
-	SIM_isystem \
-	SIM_measurement_units \
-	SIM_parse_s_define \
-	SIM_target_specific_variables \
-	SIM_test_abstract \
-	SIM_test_inherit \
-	SIM_test_ip2 \
-	SIM_threads_simple \
-	SIM_trickcomm
-
-SIMS_TO_COMPILE_AND_RUN = \
-	SIM_demo_sdefine \
-	SIM_events \
-	SIM_exec_set_time_tic_value \
-	SIM_python_namespace \
-	SIM_rti \
-	SIM_stls \
-	SIM_test_dp \
-	SIM_test_dr \
-	SIM_test_icg_file_skipped \
-	SIM_test_io \
-	SIM_test_ip \
-	SIM_test_sched \
-	SIM_test_templates \
-	SIM_threads \
-	SIM_trickified
-
-# SIM_stls will run twice. First run is RUN_test/setup.py, which is added as a dependency to test in its S_overrides.mk file
-# The second run is the normal RUN_test/unit_test.py which is run through this makefile.
-
-# Sims with problems, no purpose, or maybe shouldn't be a test
-# SIM_leaks ( should be deleted )
-# SIM_dynamic_sim_object ( not running, class won't instantiate )
-# SIM_segments  ( not a test, but a demo )
-
-# This test is temporarily sitting out until fixed.
-#    SIM_test_varserv
-
-EXECUTABLES = $(addsuffix /T_main_${TRICK_HOST_CPU}_test.exe, $(SIMS_TO_COMPILE_AND_RUN) $(SIMS_TO_COMPILE_ONLY))
-UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIMS_TO_COMPILE_AND_RUN)))
-
-test: $(EXECUTABLES) $(UNIT_TEST_RESULTS) data_record_results
-
-clean:
-	rm -f $(UNIT_TEST_RESULTS)
-	- for i in $(SIMS_TO_COMPILE_AND_RUN) ; do \
-		if [ -f "$$i/"[Mm]"akefile" ] ; then \
-			$(MAKE) -C $$i spotless ; \
-		fi \
-	done
-
-	- for i in $(SIMS_TO_COMPILE_ONLY) ; do \
-		if [ -f "$$i/"[Mm]"akefile" ] ; then \
-			$(MAKE) -C $$i spotless ; \
-		fi \
-	done
-
-$(EXECUTABLES):
-	@ cd $(@D) ; ${TRICK_HOME}/bin/trick-CP -t
-
-$(UNIT_TEST_RESULTS): $(TRICK_HOME)/trick_test/%.xml : %/T_main_${TRICK_HOST_CPU}_test.exe
-	@ cd $* ; ./T_main_${TRICK_HOST_CPU}_test.exe RUN_test/unit_test.py
-
-DR_RESULTS = $(TRICK_HOME)/test/SIM_test_dr/RUN_test
-data_record_results: $(TRICK_HOME)/trick_test/SIM_test_dr.xml $(DR_RESULTS)
-	diff $(DR_RESULTS)/log_DR_bitfieldsASCII.csv $(DR_RESULTS)/Ref_Logs/log_DR_bitfieldsASCII_Master.csv
-	diff $(DR_RESULTS)/log_DR_typesASCII.csv $(DR_RESULTS)/Ref_Logs/log_DR_typesASCII_Master.csv
-	cmp -b $(DR_RESULTS)/log_DR_bitfieldsBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_bitfieldsBINARY.trk
-ifeq (${TRICK_FORCE_32BIT}, 1)
-	cmp -b $(DR_RESULTS)/log_DR_typesBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_typesBINARY_32.trk
-else
-	cmp -b $(DR_RESULTS)/log_DR_typesBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_typesBINARY.trk
-endif

test_overrides.mk

@@ -0,0 +1,9 @@
+TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
+include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
+
+# Remove when issue #1147 is fixed: https://github.com/nasa/trick/issues/1147
+unexport TRICK_PYTHON_PATH
+
+sim_test:
+	python3 trickops.py ${TRICK_HOME}

test_sims.yml

@@ -0,0 +1,235 @@
+
+
+# Compile only sims
+SIM_alloc_test:
+  path: test/SIM_alloc_test
+SIM_alloc_test:
+  path: test/SIM_alloc_test
+SIM_anon_enum:
+  path: test/SIM_anon_enum
+SIM_default_member_initializer: 
+  path: test/SIM_default_member_initializer
+SIM_delete_default_constructor: 
+  path: test/SIM_delete_default_constructor
+SIM_demo_inputfile:
+  path: test/SIM_demo_inputfile
+SIM_exclusion_mechanisms:
+  path: test/SIM_exclusion_mechanisms
+SIM_isystem:
+  path: test/SIM_isystem
+SIM_measurement_units:
+  path: test/SIM_measurement_units
+SIM_parse_s_define:
+  path: test/SIM_parse_s_define
+# SIM_target_specific_variables:                            <-- This sim has to sit out on Mac until we fix the sprintf issue
+#   path: test/SIM_target_specific_variables
+SIM_test_abstract:
+  path: test/SIM_test_abstract
+SIM_test_inherit:
+  path: test/SIM_test_inherit
+SIM_test_ip2:
+  path: test/SIM_test_ip2
+SIM_threads_simple:
+  path: test/SIM_threads_simple
+SIM_trickcomm:
+  path: test/SIM_trickcomm
+SIM_satellite:
+  path: trick_sims/SIM_satellite
+
+# Normal case compile and run sims
+SIM_demo_sdefine: 
+  path: test/SIM_demo_sdefine
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_events:
+  path: test/SIM_events
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0 
+SIM_exec_set_time_tic_value:
+  path: test/SIM_exec_set_time_tic_value
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_python_namespace:
+  path: test/SIM_python_namespace
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_rti:
+  path: test/SIM_rti
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_test_dp:
+  path: test/SIM_test_dp
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_test_icg_file_skipped:
+  path: test/SIM_test_icg_file_skipped
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_test_io:
+  path: test/SIM_test_io
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_test_ip:
+  path: test/SIM_test_ip
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_test_sched:
+  path: test/SIM_test_sched
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_test_templates:
+  path: test/SIM_test_templates
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_threads:
+  path: test/SIM_threads
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_trickified:
+  path: test/SIM_trickified
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_ball_L1:
+  path: trick_sims/Ball/SIM_ball_L1
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_ball_L2:
+  path: trick_sims/Ball/SIM_ball_L2
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_ball_L3:
+  path: trick_sims/Ball/SIM_ball_L3
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+# SIM_amoeba:
+#   path: trick_sims/Cannon/SIM_amoeba
+#   build_command: "trick-CP -t"
+#   binary: "T_main_{cpu}_test.exe"
+#   runs:
+#     RUN_test/unit_test.py:
+#       returns: 0
+SIM_cannon_aero:
+  path: trick_sims/Cannon/SIM_cannon_aero
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_cannon_analytic:
+  path: trick_sims/Cannon/SIM_cannon_analytic
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_cannon_eulercromer:
+  path: trick_sims/Cannon/SIM_cannon_eulercromer
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_cannon_numeric:
+  path: trick_sims/Cannon/SIM_cannon_numeric
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_cannon_jet:
+  path: trick_sims/Cannon/SIM_cannon_jet
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_Ball++_L1:
+  path: trick_sims/SIM_Ball++_L1
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+SIM_sun:
+  path: trick_sims/SIM_sun
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+
+
+# Special cases
+
+# setup.py dumps a checkpoint
+# unit_test.py loads that checkpoint and verifies the data
+SIM_stls: 
+  path: test/SIM_stls
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/setup.py:
+      returns: 0
+    RUN_test/unit_test.py:
+      returns: 0
+
+SIM_test_dr:
+  path: test/SIM_test_dr
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test/unit_test.py:
+      returns: 0
+      compare:
+      - test/SIM_test_dr/RUN_test/log_DR_bitfieldsASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsASCII_Master.csv
+      - test/SIM_test_dr/RUN_test/log_DR_typesASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_typesASCII_Master.csv
+      - test/SIM_test_dr/RUN_test/log_DR_bitfieldsBINARY.trk vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsBINARY.trk
+

trick_sims/makefile

@@ -1,59 +1,18 @@
-
+TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..)
 export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU)
+include ${TRICK_HOME}/share/trick/makefiles/Makefile.common
 
-# List out sims we want to compile
-COMPILE_DIRS = \
-    Ball/SIM_ball_L1 \
-    Ball/SIM_ball_L2 \
-    Ball/SIM_ball_L3 \
-    Cannon/SIM_amoeba \
-    Cannon/SIM_cannon_aero \
-    Cannon/SIM_cannon_analytic \
-    Cannon/SIM_cannon_eulercromer \
-    Cannon/SIM_cannon_numeric \
-    Cannon/SIM_cannon_jet \
-    SIM_Ball++_L1 \
-    SIM_satellite \
-    SIM_sun
 
-    #SIM_monte \
+NESTED_SIM_DIRECTORIES = $(wildcard */SIM_*)
+SIM_DIRECTORIES = $(wildcard SIM_*) $(NESTED_SIM_DIRECTORIES)
 
-# List out sims we want to run unit tests
+UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIM_DIRECTORIES)))
-TEST_DIRS = \
-    Ball/SIM_ball_L1 \
-    Ball/SIM_ball_L2 \
-    Ball/SIM_ball_L3 \
-    Cannon/SIM_amoeba \
-    Cannon/SIM_cannon_aero \
-    Cannon/SIM_cannon_analytic \
-    Cannon/SIM_cannon_eulercromer \
-    Cannon/SIM_cannon_numeric \
-    Cannon/SIM_cannon_jet \
-    SIM_Ball++_L1 \
-    SIM_sun
 
-    #SIM_monte \
-
-EXECUTABLES = $(addsuffix /T_main_${TRICK_HOST_CPU}_test.exe, $(COMPILE_DIRS))
-UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(TEST_DIRS)))
-
-all:
-	@echo "This makefile is used with Trick's top level 'make test' command"
-
-test: $(EXECUTABLES) $(UNIT_TEST_RESULTS)
 
 clean:
 	rm -f $(UNIT_TEST_RESULTS)
-	- for i in $(COMPILE_DIRS) ; do \
+	- for i in $(SIM_DIRECTORIES) ; do \
-          if [ -f "$$i/"[Mm]"akefile" ] ; then \
+		if [ -f "$$i/"[Mm]"akefile" ] ; then \
-              $(MAKE) -C $$i spotless ; \
+			$(MAKE) -C $$i spotless ; \
-          fi \
+		fi \
-      done
+	done
-
-$(EXECUTABLES):
-	@ cd $(@D) ; ${TRICK_HOME}/bin/trick-CP -t ;
-
-$(UNIT_TEST_RESULTS): $(TRICK_HOME)/trick_test/%.xml : %/T_main_${TRICK_HOST_CPU}_test.exe
-	@ cd $* ; ./T_main_${TRICK_HOST_CPU}_test.exe RUN_test/unit_test.py
-
-

trickops.py

@@ -0,0 +1,53 @@
+import sys
+import os
+
+sys.path.append(sys.argv[1] + "/share/trick/trickops")
+
+from TrickWorkflow import *
+from WorkflowCommon import Job
+
+class SimTestWorkflow(TrickWorkflow):
+    def __init__( self, quiet, trick_top_level ):
+        # Create the trick_test directory if it doesn't already exist
+        if not os.path.exists(trick_top_level + "/trick_test"):
+          os.makedirs(trick_top_level + "/trick_test")
+
+        # Base Class initialize, this creates internal management structures
+        num_cpus = os.cpu_count() if os.cpu_count() is not None else 8
+        TrickWorkflow.__init__(self, project_top_level=(trick_top_level), log_dir=(trick_top_level +'/trickops_logs/'),
+            trick_dir=trick_top_level, config_file=(trick_top_level + "/test_sims.yml"), cpus=num_cpus, quiet=quiet)
+
+    def run( self ):
+
+      build_jobs      = self.get_jobs(kind='build')
+      run_jobs        = self.get_jobs(kind='run')
+      analysis_jobs   = self.get_jobs(kind='analyze')
+
+      # This job dumps a checkpoint that is then read in and checked by RUN_test/unit_test.py in the same sim
+      # This is a workaround to ensure that this run goes first.
+      first_phase_job = self.get_sim('SIM_stls').get_run(input='RUN_test/setup.py').get_run_job()
+      run_jobs.remove(first_phase_job)
+
+      builds_status = self.execute_jobs(build_jobs, max_concurrent=self.cpus, header='Executing all sim builds.')
+      first_phase_run_status = self.execute_jobs([first_phase_job], max_concurrent=self.cpus, header="Executing required first phase runs.")
+      runs_status   = self.execute_jobs(run_jobs,   max_concurrent=self.cpus, header='Executing all sim runs.')
+      comparison_result = self.compare()
+      analysis_status = self.execute_jobs(analysis_jobs, max_concurrent=self.cpus, header='Executing all analysis.')
+
+      self.report()           # Print Verbose report
+      self.status_summary()   # Print a Succinct summary
+
+      # Dump failing logs
+      jobs = build_jobs + run_jobs
+      for job in jobs:
+        if job.get_status() == Job.Status.FAILED:
+          print("Failing job: ", job.name)
+          print ("*"*120)
+          print(open(job.log_file, "r").read())
+          print ("*"*120, "\n")
+
+      return (builds_status or runs_status or first_phase_run_status or self.config_errors or comparison_result or analysis_status)
+
+if __name__ == "__main__":
+    should_be_quiet = os.getenv('CI') is not None
+    sys.exit(SimTestWorkflow(quiet=should_be_quiet, trick_top_level=sys.argv[1]).run())