diff --git a/.github/workflows/code_coverage.yml b/.github/workflows/code_coverage.yml index 59fadd1a..63f27163 100644 --- a/.github/workflows/code_coverage.yml +++ b/.github/workflows/code_coverage.yml @@ -61,6 +61,9 @@ jobs: export TRICK_SYSTEM_LDFLAGS="-fprofile-arcs -ftest-coverage -O0" export TRICK_SYSTEM_CFLAGS="-fprofile-arcs -ftest-coverage -O0" export TRICK_SYSTEM_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0" + cd share/trick/trickops/ + python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt + cd ../../../ make code-coverage - name: Upload to Coveralls uses: coverallsapp/github-action@master diff --git a/.github/workflows/test_32.yml b/.github/workflows/test_32.yml index c48aa067..15c20737 100644 --- a/.github/workflows/test_32.yml +++ b/.github/workflows/test_32.yml @@ -66,5 +66,9 @@ jobs: make - name: Run tests run: | + cd share/trick/trickops/ + yum install -y python3-devel + python3 -m venv .venv && . .venv/bin/activate && pip install --upgrade pip && pip3 install -r requirements.txt + cd ../../../ echo $MAKEFLAGS $CXXFLAGS $CFLAGS make test diff --git a/.github/workflows/test_32_oracle.yml b/.github/workflows/test_32_oracle.yml index 322ab853..794595aa 100644 --- a/.github/workflows/test_32_oracle.yml +++ b/.github/workflows/test_32_oracle.yml @@ -71,6 +71,9 @@ jobs: make - name: Run tests run: | + cd share/trick/trickops/ + python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt + cd ../../../ export CFLAGS="-m32" export CXXFLAGS="-m32" export MAKEFLAGS=-j`nproc` diff --git a/.github/workflows/test_linux.yml b/.github/workflows/test_linux.yml index 636117ba..109c0b56 100644 --- a/.github/workflows/test_linux.yml +++ b/.github/workflows/test_linux.yml @@ -67,6 +67,8 @@ jobs: default-jdk python2.7-dev python3-dev + python3-pip + python3-venv install_gtest: | apt-get install -y libgtest-dev cd /usr/src/gtest @@ -166,8 +168,9 @@ jobs: make - name: Test run: | - export MAKEFLAGS=-j`nproc` - make test + cd share/trick/trickops/ + python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt + cd ../../../; make test - name: Upload Tests uses: actions/upload-artifact@v3.0.0 if: success() || failure() # run this step even if previous step failed diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml index e1727b69..219fe8ef 100644 --- a/.github/workflows/test_macos.yml +++ b/.github/workflows/test_macos.yml @@ -32,7 +32,9 @@ jobs: export MAKEFLAGS=-j4 ./configure --with-llvm=${GITHUB_WORKSPACE}/clang+llvm-14.0.6-x86_64-apple-darwin make - - name: Run tests + - name: Test run: | + cd share/trick/trickops/ + python3 -m venv .venv && source .venv/bin/activate && pip3 install -r requirements.txt export MAKEFLAGS=-j4 - make test + cd ../../../; make test diff --git a/.gitignore b/.gitignore index b7d0bea1..2f71ded6 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,7 @@ civetweb_clone/ include/civet/ .vscode/ civet_server_error.log -server.pem \ No newline at end of file +server.pem +archive/ +.venv +trickops_logs/ \ No newline at end of file diff --git a/Makefile b/Makefile index 071946d4..c3548abf 100644 --- a/Makefile +++ b/Makefile @@ -301,8 +301,7 @@ $(DPX_UNIT_TEST_DIR): sim_test: - @ $(MAKE) -C test - @ $(MAKE) -C trick_sims test + @ $(MAKE) -f test_overrides.mk sim_test pytest: make -C share/trick/pymods/trick diff --git a/docs/developer_docs/Developer-Docs-Home.md b/docs/developer_docs/Developer-Docs-Home.md index 307d18e9..2be562e6 100644 --- a/docs/developer_docs/Developer-Docs-Home.md +++ b/docs/developer_docs/Developer-Docs-Home.md @@ -1,4 +1,9 @@ +| [Home](/trick) → Developer Docs | +|------------------------------------------------------------------| + # Developer Documentation + Link documentation for Trick internals, processes, and plans here. +[Testing](Testing) \ No newline at end of file diff --git a/docs/developer_docs/Testing.md b/docs/developer_docs/Testing.md new file mode 100644 index 00000000..9746e2ce --- /dev/null +++ b/docs/developer_docs/Testing.md @@ -0,0 +1,61 @@ +| [Home](/trick) → [Developer Docs](Developer-Docs-Home) → Testing | +|------------------------------------------------------------------| + + +# Testing + +Currently, Trick has a suite of unit and integration tests that run through a hodgepodge of GTest, Trick's internal unit test framework, Makefiles, and TrickOps. + +Unit tests can be found in their respective `trick_source/[sim_services|utils]/*/test` directories, and primarily use the Gtest framework. These are run primarily through test targets in their Makefiles. See `trick_source/sim_services/MemoryManager/test` for an example of organization. + +Integration tests take the form of Sims with some embedded tests. They live under `trick_sims/` and `test/`. A full list of sims that are used as part of the test suite are in [test_sims.yml](). These are run with TrickOps. + +## Test suite dependencies + +Gtest is required for the unit tests and some integration tests. See the [install guide](../documentation/install_guide/Install-Guide.md) for gtest installation. + +TrickOps requires python3 and the packages `PyYAML` and `psutil` (updated list in [Trickops requirements.txt](https://github.com/nasa/trick/blob/master/share/trick/trickops/requirements.txt)). Install these in your python environment, or create a virtual environment as follows: +``` +cd share/trick/trickops/ +python3 -m venv .venv && . .venv/bin/activate && pip3 install -r requirements.txt +cd ../../../ +``` + + +## Running the test suite +From trick home: +``` +# Run everything +make test + +# Run only integration tests +make sim_test + +# Run only unit tests +make unit_test +``` + +Currently, TrickOps will redirect all console output from tests into logs under {TRICK_HOME}/trickops_logs/, and will also dump the output of failing logs to console after the test suite is finished. + +## Coverage +Trick uses [Coveralls](https://coveralls.io/github/nasa/trick?branch=master) to host code coverage. Coverage is generated by running the test suite with gcov in CI, and then those files are uploaded to Coveralls. + +To enable gcov in Trick, it must be cleaned and compiled with the following: +``` +export CFLAGS="-fprofile-arcs -ftest-coverage -O0" +export CXXFLAGS="-fprofile-arcs -ftest-coverage -O0" +export LDFLAGS="-fprofile-arcs -ftest-coverage -O0" +export TRICK_CFLAGS="-fprofile-arcs -ftest-coverage -O0" +export TRICK_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0" +export TRICK_SYSTEM_LDFLAGS="-fprofile-arcs -ftest-coverage -O0" +export TRICK_SYSTEM_CFLAGS="-fprofile-arcs -ftest-coverage -O0" +export TRICK_SYSTEM_CXXFLAGS="-fprofile-arcs -ftest-coverage -O0" +``` + +After Trick has been rebuild with the instrumentation, run: +``` +make code-coverage +``` +This will generate, collect, and filter all the various coverage data collection files into `coverage.info`. This is the file that is uploaded to Coveralls in the [code_coverage.yml](https://github.com/nasa/trick/blob/master/.github/workflows/code_coverage.yml) Github Actions workflow. + + diff --git a/docs/documentation/miscellaneous_trick_tools/TrickOps.md b/docs/documentation/miscellaneous_trick_tools/TrickOps.md index 6ccef78a..38f549f4 100644 --- a/docs/documentation/miscellaneous_trick_tools/TrickOps.md +++ b/docs/documentation/miscellaneous_trick_tools/TrickOps.md @@ -64,6 +64,7 @@ SIM_abc: <-- required unique name for sim of interest, must start - model_x by label within the framework, or for any other project-defined - verification purpose build_command: <-- optional literal cmd executed for SIM_build, defaults to trick-CP + binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe size: <-- optional estimated size of successful build output file in bytes runs: <-- optional dict of runs to be executed for this sim, where the RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary diff --git a/share/trick/trickops/TrickWorkflow.py b/share/trick/trickops/TrickWorkflow.py index 8ab5076c..d67f5419 100644 --- a/share/trick/trickops/TrickWorkflow.py +++ b/share/trick/trickops/TrickWorkflow.py @@ -102,7 +102,7 @@ class TrickWorkflow(WorkflowCommon): super().__init__(project_top_level=project_top_level, log_dir=log_dir, quiet=quiet) # If not found in the config file, these defaults are used self.defaults = {'cpus': 3, 'name': None, 'description': None, - 'build_command': 'trick-CP', 'size': 2200000, 'env': None} + 'build_command': 'trick-CP', 'size': 2200000, 'env': None, 'binary': 'S_main_{cpu}.exe'} self.config_errors = False self.compare_errors = False # True if comparison errors were found self.sims = [] # list of Sim() instances, filled out from config file @@ -269,6 +269,7 @@ class TrickWorkflow(WorkflowCommon): - model_x by label within the framework, or for any other project-defined - verification purpose build_command: <-- optional literal cmd executed for SIM_build, defaults to trick-CP + binary: <-- optional name of sim binary, defaults to S_main_{cpu}.exe size: <-- optional estimated size of successful build output file in bytes runs: <-- optional dict of runs to be executed for this sim, where the RUN_1/input.py --foo: dict keys are the literal arguments passed to the sim binary @@ -390,11 +391,22 @@ class TrickWorkflow(WorkflowCommon): sanitized_labels =( [l for l in c[s]['labels'] if type_expected( l, expected_type=str, extra_msg='Ignoring label "%s".' % l) ] ) self.config[s]['labels'] = sanitized_labels + # Check for binary argument + if 'binary' not in c[s] or not c[s]['binary']: + self.config[s]['binary'] = self.defaults['binary'] + + # Add the CPU format string (won't fail if there is no CPU placeholder) + self.config[s]['binary'] = self.config[s]['binary'].format(cpu=self.trick_host_cpu) + + # Add the full path to the build command + self.config[s]['build_command'] = os.path.join(self.trick_dir, "bin", self.config[s]['build_command']) + # Create internal object to be populated with runs, valgrind runs, etc thisSim = TrickWorkflow.Sim(name=s, sim_dir=self.config[s]['path'], description=self.config[s]['description'], labels=self.config[s]['labels'], prebuild_cmd=self.env, build_cmd=self.config[s]['build_command'], cpus=self.cpus, size=self.config[s]['size'], log_dir=self.log_dir) + all_sim_paths.append(c[s]['path']) # RUN sanity checks if 'runs' in c[s]: # If it's there... @@ -442,7 +454,7 @@ class TrickWorkflow(WorkflowCommon): self.config[s]['runs'][r]['returns'] = 0 # Default to zero # Create internal object to be added to thisSim thisRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r, - binary= 'S_main_' + self.trick_host_cpu + '.exe', prerun_cmd=self.env, + binary= self.config[s]['binary'], prerun_cmd=self.env, returns=self.config[s]['runs'][r]['returns'], valgrind_flags=None, log_dir=self.log_dir) # Handle 'compare' option, if given, if not, assume 0 @@ -502,7 +514,7 @@ class TrickWorkflow(WorkflowCommon): else: # Create internal object to be added to thisSim vRun = TrickWorkflow.Run(sim_dir=self.config[s]['path'], input=r, - binary= 'S_main_' + self.trick_host_cpu + '.exe', + binary= self.config[s]['binary'], prerun_cmd=self.env, valgrind_flags=self.config[s]['valgrind']['flags'], log_dir=self.log_dir) thisSim.add_run(vRun) @@ -811,6 +823,7 @@ class TrickWorkflow(WorkflowCommon): FileSizeJob() Instance of FileSizeJob() job for compiling this sim """ + if not self.build_job: name = 'Build ' + self.sim_dir self.build_job = FileSizeJob(name=name, @@ -1019,7 +1032,7 @@ class TrickWorkflow(WorkflowCommon): including command-line arguments ex: RUN_test/input.py, or RUN_test/input.py --my-arg binary : str - Name of executable, usually S_min.. bu platform specific + Name of executable, usually S_main.. but platform specific prerun_cmd : str Optional string to execute immediately before sim run, e.g. env sourcing valgrind_flags : str diff --git a/share/trick/trickops/tests/trick_sims.yml b/share/trick/trickops/tests/trick_sims.yml index a2ccb36d..ca0cae72 100644 --- a/share/trick/trickops/tests/trick_sims.yml +++ b/share/trick/trickops/tests/trick_sims.yml @@ -77,10 +77,13 @@ SIM_rti: labels: - unit_test SIM_segments: + binary: 'T_main_*.exe' path: test/SIM_segments runs: RUN_test/input.py: SIM_stls: + binary: 'T_main_{cpu}_test.exe' + build_command: "trick-CP -t" path: test/SIM_stls labels: - unit_test diff --git a/share/trick/trickops/tests/ut_TrickWorkflow.py b/share/trick/trickops/tests/ut_TrickWorkflow.py index f25065c6..40723e9b 100644 --- a/share/trick/trickops/tests/ut_TrickWorkflow.py +++ b/share/trick/trickops/tests/ut_TrickWorkflow.py @@ -117,6 +117,29 @@ class TrickWorkflowTestCase(unittest.TestCase): sim = self.instance.get_sim(identifier='SIM_ball_L1') run = sim.get_run('RUN_test/input.py') + def test_set_binary_with_cpu_formatter (self): + sim = self.instance.get_sim(identifier="SIM_stls") + cpu = self.instance.get_trick_host_cpu() + run = sim.get_run('RUN_test/unit_test.py') + print("Binary: " , run.binary) + self.assertEqual(run.binary, "T_main_" + cpu + "_test.exe") + + def test_set_binary_without_cpu_formatter (self): + sim = self.instance.get_sim(identifier="SIM_segments") + run = sim.get_run('RUN_test/input.py') + print("Binary: " , run.binary) + self.assertEqual(run.binary, "T_main_*.exe") + + def test_absolute_path_to_custom_build_cmd (self): + sim = self.instance.get_sim(identifier="SIM_stls") + absolute_path = os.path.join(self.instance.trick_dir, "bin", "trick-CP -t") + self.assertEqual(sim.build_cmd, absolute_path) + + def test_absolute_path_to_default_build_cmd (self): + sim = self.instance.get_sim(identifier="SIM_segments") + absolute_path = os.path.join(self.instance.trick_dir, "bin", "trick-CP") + self.assertEqual(sim.build_cmd, absolute_path) + def test_get_unique_comparison_dirs(self): ucd = self.instance.get_unique_comparison_dirs() self.assertTrue(ucd[0] is not None) diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 00000000..f60dd273 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,16 @@ +TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..) +export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU) +include ${TRICK_HOME}/share/trick/makefiles/Makefile.common + + +SIM_DIRECTORIES = $(wildcard SIM_*) +UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIM_DIRECTORIES))) + + +clean: + rm -f $(UNIT_TEST_RESULTS) + - for i in $(SIM_DIRECTORIES) ; do \ + if [ -f "$$i/"[Mm]"akefile" ] ; then \ + $(MAKE) -C $$i spotless ; \ + fi \ + done diff --git a/test/SIM_stls/S_overrides.mk b/test/SIM_stls/S_overrides.mk index b05ddfc7..5587ab4f 100644 --- a/test/SIM_stls/S_overrides.mk +++ b/test/SIM_stls/S_overrides.mk @@ -4,15 +4,6 @@ TRICK_CXXFLAGS += -I./models TRICK_CXXFLAGS += -std=c++11 -test: setup - -# This is a workaround so that the sim runs with RUN_test/setup.py before the actual unit test run. -# setup.py adds hardcoded data to the sim and dumps a checkpoint, and unit_test.py loads that checkpoint and checks all the values. - -setup: T_main_${TRICK_HOST_CPU}.exe - echo "Running Setup Job" - ./T_main_${TRICK_HOST_CPU}.exe RUN_test/setup.py - clean: checkpoint_clean checkpoint_clean: diff --git a/test/makefile b/test/makefile deleted file mode 100644 index 8daf93fb..00000000 --- a/test/makefile +++ /dev/null @@ -1,87 +0,0 @@ -TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..) -export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU) -include ${TRICK_HOME}/share/trick/makefiles/Makefile.common - -# Remove when issue #1147 is fixed: https://github.com/nasa/trick/issues/1147 -unexport TRICK_PYTHON_PATH - -SIMS_TO_COMPILE_ONLY = \ - SIM_alloc_test \ - SIM_anon_enum \ - SIM_default_member_initializer \ - SIM_delete_default_constructor \ - SIM_demo_inputfile \ - SIM_exclusion_mechanisms \ - SIM_isystem \ - SIM_measurement_units \ - SIM_parse_s_define \ - SIM_target_specific_variables \ - SIM_test_abstract \ - SIM_test_inherit \ - SIM_test_ip2 \ - SIM_threads_simple \ - SIM_trickcomm - -SIMS_TO_COMPILE_AND_RUN = \ - SIM_demo_sdefine \ - SIM_events \ - SIM_exec_set_time_tic_value \ - SIM_python_namespace \ - SIM_rti \ - SIM_stls \ - SIM_test_dp \ - SIM_test_dr \ - SIM_test_icg_file_skipped \ - SIM_test_io \ - SIM_test_ip \ - SIM_test_sched \ - SIM_test_templates \ - SIM_threads \ - SIM_trickified - -# SIM_stls will run twice. First run is RUN_test/setup.py, which is added as a dependency to test in its S_overrides.mk file -# The second run is the normal RUN_test/unit_test.py which is run through this makefile. - -# Sims with problems, no purpose, or maybe shouldn't be a test -# SIM_leaks ( should be deleted ) -# SIM_dynamic_sim_object ( not running, class won't instantiate ) -# SIM_segments ( not a test, but a demo ) - -# This test is temporarily sitting out until fixed. -# SIM_test_varserv - -EXECUTABLES = $(addsuffix /T_main_${TRICK_HOST_CPU}_test.exe, $(SIMS_TO_COMPILE_AND_RUN) $(SIMS_TO_COMPILE_ONLY)) -UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIMS_TO_COMPILE_AND_RUN))) - -test: $(EXECUTABLES) $(UNIT_TEST_RESULTS) data_record_results - -clean: - rm -f $(UNIT_TEST_RESULTS) - - for i in $(SIMS_TO_COMPILE_AND_RUN) ; do \ - if [ -f "$$i/"[Mm]"akefile" ] ; then \ - $(MAKE) -C $$i spotless ; \ - fi \ - done - - - for i in $(SIMS_TO_COMPILE_ONLY) ; do \ - if [ -f "$$i/"[Mm]"akefile" ] ; then \ - $(MAKE) -C $$i spotless ; \ - fi \ - done - -$(EXECUTABLES): - @ cd $(@D) ; ${TRICK_HOME}/bin/trick-CP -t - -$(UNIT_TEST_RESULTS): $(TRICK_HOME)/trick_test/%.xml : %/T_main_${TRICK_HOST_CPU}_test.exe - @ cd $* ; ./T_main_${TRICK_HOST_CPU}_test.exe RUN_test/unit_test.py - -DR_RESULTS = $(TRICK_HOME)/test/SIM_test_dr/RUN_test -data_record_results: $(TRICK_HOME)/trick_test/SIM_test_dr.xml $(DR_RESULTS) - diff $(DR_RESULTS)/log_DR_bitfieldsASCII.csv $(DR_RESULTS)/Ref_Logs/log_DR_bitfieldsASCII_Master.csv - diff $(DR_RESULTS)/log_DR_typesASCII.csv $(DR_RESULTS)/Ref_Logs/log_DR_typesASCII_Master.csv - cmp -b $(DR_RESULTS)/log_DR_bitfieldsBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_bitfieldsBINARY.trk -ifeq (${TRICK_FORCE_32BIT}, 1) - cmp -b $(DR_RESULTS)/log_DR_typesBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_typesBINARY_32.trk -else - cmp -b $(DR_RESULTS)/log_DR_typesBINARY.trk $(DR_RESULTS)/Ref_Logs/log_DR_typesBINARY.trk -endif diff --git a/test_overrides.mk b/test_overrides.mk new file mode 100644 index 00000000..c80a7618 --- /dev/null +++ b/test_overrides.mk @@ -0,0 +1,9 @@ +TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU) +include ${TRICK_HOME}/share/trick/makefiles/Makefile.common + +# Remove when issue #1147 is fixed: https://github.com/nasa/trick/issues/1147 +unexport TRICK_PYTHON_PATH + +sim_test: + python3 trickops.py ${TRICK_HOME} diff --git a/test_sims.yml b/test_sims.yml new file mode 100644 index 00000000..dd0d5a53 --- /dev/null +++ b/test_sims.yml @@ -0,0 +1,235 @@ + + +# Compile only sims +SIM_alloc_test: + path: test/SIM_alloc_test +SIM_alloc_test: + path: test/SIM_alloc_test +SIM_anon_enum: + path: test/SIM_anon_enum +SIM_default_member_initializer: + path: test/SIM_default_member_initializer +SIM_delete_default_constructor: + path: test/SIM_delete_default_constructor +SIM_demo_inputfile: + path: test/SIM_demo_inputfile +SIM_exclusion_mechanisms: + path: test/SIM_exclusion_mechanisms +SIM_isystem: + path: test/SIM_isystem +SIM_measurement_units: + path: test/SIM_measurement_units +SIM_parse_s_define: + path: test/SIM_parse_s_define +# SIM_target_specific_variables: <-- This sim has to sit out on Mac until we fix the sprintf issue +# path: test/SIM_target_specific_variables +SIM_test_abstract: + path: test/SIM_test_abstract +SIM_test_inherit: + path: test/SIM_test_inherit +SIM_test_ip2: + path: test/SIM_test_ip2 +SIM_threads_simple: + path: test/SIM_threads_simple +SIM_trickcomm: + path: test/SIM_trickcomm +SIM_satellite: + path: trick_sims/SIM_satellite + +# Normal case compile and run sims +SIM_demo_sdefine: + path: test/SIM_demo_sdefine + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_events: + path: test/SIM_events + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_exec_set_time_tic_value: + path: test/SIM_exec_set_time_tic_value + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_python_namespace: + path: test/SIM_python_namespace + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_rti: + path: test/SIM_rti + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_test_dp: + path: test/SIM_test_dp + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_test_icg_file_skipped: + path: test/SIM_test_icg_file_skipped + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_test_io: + path: test/SIM_test_io + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_test_ip: + path: test/SIM_test_ip + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_test_sched: + path: test/SIM_test_sched + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_test_templates: + path: test/SIM_test_templates + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_threads: + path: test/SIM_threads + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_trickified: + path: test/SIM_trickified + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_ball_L1: + path: trick_sims/Ball/SIM_ball_L1 + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_ball_L2: + path: trick_sims/Ball/SIM_ball_L2 + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_ball_L3: + path: trick_sims/Ball/SIM_ball_L3 + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +# SIM_amoeba: +# path: trick_sims/Cannon/SIM_amoeba +# build_command: "trick-CP -t" +# binary: "T_main_{cpu}_test.exe" +# runs: +# RUN_test/unit_test.py: +# returns: 0 +SIM_cannon_aero: + path: trick_sims/Cannon/SIM_cannon_aero + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_cannon_analytic: + path: trick_sims/Cannon/SIM_cannon_analytic + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_cannon_eulercromer: + path: trick_sims/Cannon/SIM_cannon_eulercromer + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_cannon_numeric: + path: trick_sims/Cannon/SIM_cannon_numeric + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_cannon_jet: + path: trick_sims/Cannon/SIM_cannon_jet + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_Ball++_L1: + path: trick_sims/SIM_Ball++_L1 + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 +SIM_sun: + path: trick_sims/SIM_sun + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 + + +# Special cases + +# setup.py dumps a checkpoint +# unit_test.py loads that checkpoint and verifies the data +SIM_stls: + path: test/SIM_stls + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/setup.py: + returns: 0 + RUN_test/unit_test.py: + returns: 0 + +SIM_test_dr: + path: test/SIM_test_dr + build_command: "trick-CP -t" + binary: "T_main_{cpu}_test.exe" + runs: + RUN_test/unit_test.py: + returns: 0 + compare: + - test/SIM_test_dr/RUN_test/log_DR_bitfieldsASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsASCII_Master.csv + - test/SIM_test_dr/RUN_test/log_DR_typesASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_typesASCII_Master.csv + - test/SIM_test_dr/RUN_test/log_DR_bitfieldsBINARY.trk vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsBINARY.trk + diff --git a/trick_sims/makefile b/trick_sims/makefile index 26d43306..52917e52 100644 --- a/trick_sims/makefile +++ b/trick_sims/makefile @@ -1,59 +1,18 @@ - +TRICK_HOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))..) export TRICK_HOST_CPU := $(shell $(TRICK_HOME)/bin/trick-gte TRICK_HOST_CPU) +include ${TRICK_HOME}/share/trick/makefiles/Makefile.common -# List out sims we want to compile -COMPILE_DIRS = \ - Ball/SIM_ball_L1 \ - Ball/SIM_ball_L2 \ - Ball/SIM_ball_L3 \ - Cannon/SIM_amoeba \ - Cannon/SIM_cannon_aero \ - Cannon/SIM_cannon_analytic \ - Cannon/SIM_cannon_eulercromer \ - Cannon/SIM_cannon_numeric \ - Cannon/SIM_cannon_jet \ - SIM_Ball++_L1 \ - SIM_satellite \ - SIM_sun - #SIM_monte \ +NESTED_SIM_DIRECTORIES = $(wildcard */SIM_*) +SIM_DIRECTORIES = $(wildcard SIM_*) $(NESTED_SIM_DIRECTORIES) -# List out sims we want to run unit tests -TEST_DIRS = \ - Ball/SIM_ball_L1 \ - Ball/SIM_ball_L2 \ - Ball/SIM_ball_L3 \ - Cannon/SIM_amoeba \ - Cannon/SIM_cannon_aero \ - Cannon/SIM_cannon_analytic \ - Cannon/SIM_cannon_eulercromer \ - Cannon/SIM_cannon_numeric \ - Cannon/SIM_cannon_jet \ - SIM_Ball++_L1 \ - SIM_sun +UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(SIM_DIRECTORIES))) - #SIM_monte \ - -EXECUTABLES = $(addsuffix /T_main_${TRICK_HOST_CPU}_test.exe, $(COMPILE_DIRS)) -UNIT_TEST_RESULTS = $(addprefix $(TRICK_HOME)/trick_test/, $(addsuffix .xml, $(TEST_DIRS))) - -all: - @echo "This makefile is used with Trick's top level 'make test' command" - -test: $(EXECUTABLES) $(UNIT_TEST_RESULTS) clean: rm -f $(UNIT_TEST_RESULTS) - - for i in $(COMPILE_DIRS) ; do \ - if [ -f "$$i/"[Mm]"akefile" ] ; then \ - $(MAKE) -C $$i spotless ; \ - fi \ - done - -$(EXECUTABLES): - @ cd $(@D) ; ${TRICK_HOME}/bin/trick-CP -t ; - -$(UNIT_TEST_RESULTS): $(TRICK_HOME)/trick_test/%.xml : %/T_main_${TRICK_HOST_CPU}_test.exe - @ cd $* ; ./T_main_${TRICK_HOST_CPU}_test.exe RUN_test/unit_test.py - - + - for i in $(SIM_DIRECTORIES) ; do \ + if [ -f "$$i/"[Mm]"akefile" ] ; then \ + $(MAKE) -C $$i spotless ; \ + fi \ + done diff --git a/trickops.py b/trickops.py new file mode 100644 index 00000000..e64c0087 --- /dev/null +++ b/trickops.py @@ -0,0 +1,53 @@ +import sys +import os + +sys.path.append(sys.argv[1] + "/share/trick/trickops") + +from TrickWorkflow import * +from WorkflowCommon import Job + +class SimTestWorkflow(TrickWorkflow): + def __init__( self, quiet, trick_top_level ): + # Create the trick_test directory if it doesn't already exist + if not os.path.exists(trick_top_level + "/trick_test"): + os.makedirs(trick_top_level + "/trick_test") + + # Base Class initialize, this creates internal management structures + num_cpus = os.cpu_count() if os.cpu_count() is not None else 8 + TrickWorkflow.__init__(self, project_top_level=(trick_top_level), log_dir=(trick_top_level +'/trickops_logs/'), + trick_dir=trick_top_level, config_file=(trick_top_level + "/test_sims.yml"), cpus=num_cpus, quiet=quiet) + + def run( self ): + + build_jobs = self.get_jobs(kind='build') + run_jobs = self.get_jobs(kind='run') + analysis_jobs = self.get_jobs(kind='analyze') + + # This job dumps a checkpoint that is then read in and checked by RUN_test/unit_test.py in the same sim + # This is a workaround to ensure that this run goes first. + first_phase_job = self.get_sim('SIM_stls').get_run(input='RUN_test/setup.py').get_run_job() + run_jobs.remove(first_phase_job) + + builds_status = self.execute_jobs(build_jobs, max_concurrent=self.cpus, header='Executing all sim builds.') + first_phase_run_status = self.execute_jobs([first_phase_job], max_concurrent=self.cpus, header="Executing required first phase runs.") + runs_status = self.execute_jobs(run_jobs, max_concurrent=self.cpus, header='Executing all sim runs.') + comparison_result = self.compare() + analysis_status = self.execute_jobs(analysis_jobs, max_concurrent=self.cpus, header='Executing all analysis.') + + self.report() # Print Verbose report + self.status_summary() # Print a Succinct summary + + # Dump failing logs + jobs = build_jobs + run_jobs + for job in jobs: + if job.get_status() == Job.Status.FAILED: + print("Failing job: ", job.name) + print ("*"*120) + print(open(job.log_file, "r").read()) + print ("*"*120, "\n") + + return (builds_status or runs_status or first_phase_run_status or self.config_errors or comparison_result or analysis_status) + +if __name__ == "__main__": + should_be_quiet = os.getenv('CI') is not None + sys.exit(SimTestWorkflow(quiet=should_be_quiet, trick_top_level=sys.argv[1]).run())