Add test sim for interaction between data recording and checkpointing (#1398)

* Add test sim and documentation for interaction between data recording and checkpointing
* Dump failing logs from first phase jobs

docs/documentation/simulation_capabilities/Data-Record.md

@@ -417,4 +417,16 @@ GROUP "/" {
 }
 ```
 
+
+### Interaction with Checkpoints
+
+Data recording groups are able to be checkpointed, reloaded, and restarted without any interaction by the user. When a checkpoint is loaded that includes data recording,
+the data recording groups will be initiated and begin recording at the time in the checkpoint. For example, if a checkpoint was dumped when t=5, when the checkpoint is 
+loaded into another run, it will data record starting at t=5, no matter what time in the run it was loaded or whether the run was already data recording. Loading a checkpoint
+will overwrite any data recording files that were being recorded before the load. 
+
+Loading a checkpoint with different data recording groups than the current run will overwrite the current data recording groups.
+
+Refer to test/SIM_checkpoint_data_recording to see expected behavior in action. Overall, the loading a checkpoint should completely overwrite any other data recording the sim is currently doing, and the new recording will start at the time in the checkpoint. If you come across different behavior, please open an issue.
+
 [Continue to Checkpointing](Checkpoints)

test/SIM_checkpoint_data_recording/Modified_data/anotherfoo.dr

@@ -0,0 +1,18 @@
+global DR_GROUP_ID
+global drg
+try:
+    if DR_GROUP_ID >= 0:
+        DR_GROUP_ID += 1
+except NameError:
+    DR_GROUP_ID = 0
+    drg = []
+
+drg.append(trick.DRAscii("anotherfoo"))
+drg[DR_GROUP_ID].set_freq(trick.DR_Always)
+drg[DR_GROUP_ID].set_cycle(0.1)
+drg[DR_GROUP_ID].set_single_prec_only(False)
+drg[DR_GROUP_ID].add_variable("testSimObject.name")
+drg[DR_GROUP_ID].add_variable("testSimObject.my_foo.b")
+drg[DR_GROUP_ID].set_max_file_size(1 * 1073741824) # multiply converts GiB to B --Dr. Dre
+trick.add_data_record_group(drg[DR_GROUP_ID], trick.DR_Buffer)
+drg[DR_GROUP_ID].enable()

test/SIM_checkpoint_data_recording/Modified_data/foo.dr

@@ -0,0 +1,18 @@
+global DR_GROUP_ID
+global drg
+try:
+    if DR_GROUP_ID >= 0:
+        DR_GROUP_ID += 1
+except NameError:
+    DR_GROUP_ID = 0
+    drg = []
+
+drg.append(trick.DRAscii("foo"))
+drg[DR_GROUP_ID].set_freq(trick.DR_Always)
+drg[DR_GROUP_ID].set_cycle(0.1)
+drg[DR_GROUP_ID].set_single_prec_only(False)
+drg[DR_GROUP_ID].add_variable("testSimObject.my_foo.a")
+drg[DR_GROUP_ID].add_variable("testSimObject.my_foo.b")
+drg[DR_GROUP_ID].set_max_file_size(1 * 1073741824) # multiply converts GiB to B --Dr. Dre
+trick.add_data_record_group(drg[DR_GROUP_ID], trick.DR_Buffer)
+drg[DR_GROUP_ID].enable()

test/SIM_checkpoint_data_recording/Modified_data/foo2.dr

@@ -0,0 +1,17 @@
+global DR_GROUP_ID
+global drg
+try:
+    if DR_GROUP_ID >= 0:
+        DR_GROUP_ID += 1
+except NameError:
+    DR_GROUP_ID = 0
+    drg = []
+
+drg.append(trick.DRAscii("foo2"))
+drg[DR_GROUP_ID].set_freq(trick.DR_Always)
+drg[DR_GROUP_ID].set_cycle(1)
+drg[DR_GROUP_ID].set_single_prec_only(False)
+drg[DR_GROUP_ID].add_variable("testSimObject.my_foo.b")
+drg[DR_GROUP_ID].set_max_file_size(1 * 1073741824) # multiply converts GiB to B --Dr. Dre
+trick.add_data_record_group(drg[DR_GROUP_ID], trick.DR_Buffer)
+drg[DR_GROUP_ID].enable()

test/SIM_checkpoint_data_recording/Modified_data/realtime.py

@@ -0,0 +1,10 @@
+trick.frame_log_on()
+trick.real_time_enable()
+trick.exec_set_software_frame(0.1)
+trick.itimer_enable()
+
+trick.exec_set_enable_freeze(True)
+# trick.exec_set_freeze_command(True)
+
+# simControlPanel = trick.SimControlPanel()
+# trick.add_external_application(simControlPanel)

test/SIM_checkpoint_data_recording/README.md

@@ -0,0 +1,39 @@
+This test suite runs different data recording and checkpoint combinations in an effort to document and test expected behaviors.
+
+RUN_test1
+Checkpoint dumped at t=5 with data recording
+Run started without data recording
+Checkpoint loaded at t=5
+Expected: log_foo.csv contains data recorded from t=5+
+
+RUN_test2
+Checkpoint dumped at t=5 without data recording
+Run started with data recording
+Checkpoint loaded at t=5
+Expected: log_foo.csv does not exist
+
+RUN_test3
+Checkpoint dumped at t=5 with data recording
+Run started with data recording
+Checkpoint loaded at t=5
+Expected: log_foo.csv contains data recorded from t=5+
+
+RUN_test4
+Checkpoint dumped at t=2 with data recording
+Run started with data recording
+Checkpoint loaded at t=5
+Expected: log_foo.csv contains data recorded from t=2+
+
+RUN_test5
+Checkpoint dumped at t=7 with data recording
+Run started with data recording
+Checkpoint loaded at t=5
+Expected: log_foo.csv contains data recorded from t=7+
+
+RUN_test6
+Checkpoint dumped at t=7 with data recording group foo2
+Run started with data recording group 1
+Checkpoint loaded at t=5
+Expected: ?
+
+Overall: expectation is that what loads in from the checkpoint should take precedence and overwrite the file of the same name.

test/SIM_checkpoint_data_recording/RUN_test/dump.py

@@ -0,0 +1,14 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():
+    exec(open("Modified_data/foo.dr").read())
+    
+    trick.checkpoint(5.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test/input.py

@@ -0,0 +1,10 @@
+def main():
+
+    exec(open("Modified_data/realtime.py").read())
+
+    trick.exec_set_software_frame(0.10)
+    trick.exec_set_freeze_frame(0.10)
+    trick.stop(5.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test1/dump.py

@@ -0,0 +1,14 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():
+    exec(open("Modified_data/foo.dr").read())
+    
+    trick.checkpoint(5.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv

@@ -0,0 +1,51 @@
+sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1}
+                 5.1,6,12
+                 5.2,6,12
+                 5.3,6,12
+                 5.4,6,12
+                 5.5,6,12
+                 5.6,6,12
+                 5.7,6,12
+                 5.8,6,12
+                 5.9,6,12
+                   6,7,14
+                 6.1,7,14
+                 6.2,7,14
+                 6.3,7,14
+                 6.4,7,14
+                 6.5,7,14
+                 6.6,7,14
+                 6.7,7,14
+                 6.8,7,14
+                 6.9,7,14
+                   7,8,16
+                 7.1,8,16
+                 7.2,8,16
+                 7.3,8,16
+                 7.4,8,16
+                 7.5,8,16
+                 7.6,8,16
+                 7.7,8,16
+                 7.8,8,16
+                 7.9,8,16
+                   8,9,18
+                 8.1,9,18
+   8.199999999999999,9,18
+   8.300000000000001,9,18
+                 8.4,9,18
+                 8.5,9,18
+                 8.6,9,18
+   8.699999999999999,9,18
+   8.800000000000001,9,18
+                 8.9,9,18
+                   9,10,20
+                 9.1,10,20
+   9.199999999999999,10,20
+   9.300000000000001,10,20
+                 9.4,10,20
+                 9.5,10,20
+                 9.6,10,20
+   9.699999999999999,10,20
+   9.800000000000001,10,20
+                 9.9,10,20
+                  10,11,22

test/SIM_checkpoint_data_recording/RUN_test1/unit_test.py

@@ -0,0 +1,9 @@
+import trick
+
+
+def main():
+    trick.add_read(5.0, 'trick.load_checkpoint("RUN_test1/chkpnt_5.000000")') # This checkpoint has data recording
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test2/check_log.sh

@@ -0,0 +1,9 @@
+FILE=log_foo.csv
+if test -f "$FILE"; then
+    echo "$FILE exists."
+    exit 1
+else
+    # Expect the file not to exist.
+    echo "$FILE does not exist."
+    exit 0
+fi

test/SIM_checkpoint_data_recording/RUN_test2/dump.py

@@ -0,0 +1,12 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():    
+    trick.checkpoint(5.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test2/unit_test.py

@@ -0,0 +1,11 @@
+import trick
+
+def main():
+    exec(open("Modified_data/foo.dr").read())
+
+    trick.add_read(5.0, 'trick.load_checkpoint("RUN_test2/chkpnt_5.000000")') # this checkpoint does not contain data recording
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test3/dump.py

@@ -0,0 +1,14 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():
+    exec(open("Modified_data/foo.dr").read())
+    
+    trick.checkpoint(5.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv

@@ -0,0 +1,51 @@
+sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1}
+                 5.1,6,12
+                 5.2,6,12
+                 5.3,6,12
+                 5.4,6,12
+                 5.5,6,12
+                 5.6,6,12
+                 5.7,6,12
+                 5.8,6,12
+                 5.9,6,12
+                   6,7,14
+                 6.1,7,14
+                 6.2,7,14
+                 6.3,7,14
+                 6.4,7,14
+                 6.5,7,14
+                 6.6,7,14
+                 6.7,7,14
+                 6.8,7,14
+                 6.9,7,14
+                   7,8,16
+                 7.1,8,16
+                 7.2,8,16
+                 7.3,8,16
+                 7.4,8,16
+                 7.5,8,16
+                 7.6,8,16
+                 7.7,8,16
+                 7.8,8,16
+                 7.9,8,16
+                   8,9,18
+                 8.1,9,18
+   8.199999999999999,9,18
+   8.300000000000001,9,18
+                 8.4,9,18
+                 8.5,9,18
+                 8.6,9,18
+   8.699999999999999,9,18
+   8.800000000000001,9,18
+                 8.9,9,18
+                   9,10,20
+                 9.1,10,20
+   9.199999999999999,10,20
+   9.300000000000001,10,20
+                 9.4,10,20
+                 9.5,10,20
+                 9.6,10,20
+   9.699999999999999,10,20
+   9.800000000000001,10,20
+                 9.9,10,20
+                  10,11,22

test/SIM_checkpoint_data_recording/RUN_test3/unit_test.py

@@ -0,0 +1,12 @@
+import trick
+
+def main():
+    
+    exec(open("Modified_data/foo.dr").read())
+
+    trick.add_read(5.0, 'trick.load_checkpoint("RUN_test3/chkpnt_5.000000")') # contains data recording
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test4/dump.py

@@ -0,0 +1,14 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():
+    exec(open("Modified_data/foo.dr").read())
+    
+    trick.checkpoint(2.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv

@@ -0,0 +1,81 @@
+sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1}
+                 2.1,3,6
+                 2.2,3,6
+                 2.3,3,6
+                 2.4,3,6
+                 2.5,3,6
+                 2.6,3,6
+                 2.7,3,6
+                 2.8,3,6
+                 2.9,3,6
+                   3,4,8
+                 3.1,4,8
+                 3.2,4,8
+                 3.3,4,8
+                 3.4,4,8
+                 3.5,4,8
+                 3.6,4,8
+                 3.7,4,8
+                 3.8,4,8
+                 3.9,4,8
+                   4,5,10
+                 4.1,5,10
+                 4.2,5,10
+                 4.3,5,10
+                 4.4,5,10
+                 4.5,5,10
+                 4.6,5,10
+                 4.7,5,10
+                 4.8,5,10
+                 4.9,5,10
+                   5,6,12
+                 5.1,6,12
+                 5.2,6,12
+                 5.3,6,12
+                 5.4,6,12
+                 5.5,6,12
+                 5.6,6,12
+                 5.7,6,12
+                 5.8,6,12
+                 5.9,6,12
+                   6,7,14
+                 6.1,7,14
+                 6.2,7,14
+                 6.3,7,14
+                 6.4,7,14
+                 6.5,7,14
+                 6.6,7,14
+                 6.7,7,14
+                 6.8,7,14
+                 6.9,7,14
+                   7,8,16
+                 7.1,8,16
+                 7.2,8,16
+                 7.3,8,16
+                 7.4,8,16
+                 7.5,8,16
+                 7.6,8,16
+                 7.7,8,16
+                 7.8,8,16
+                 7.9,8,16
+                   8,9,18
+                 8.1,9,18
+   8.199999999999999,9,18
+   8.300000000000001,9,18
+                 8.4,9,18
+                 8.5,9,18
+                 8.6,9,18
+   8.699999999999999,9,18
+   8.800000000000001,9,18
+                 8.9,9,18
+                   9,10,20
+                 9.1,10,20
+   9.199999999999999,10,20
+   9.300000000000001,10,20
+                 9.4,10,20
+                 9.5,10,20
+                 9.6,10,20
+   9.699999999999999,10,20
+   9.800000000000001,10,20
+                 9.9,10,20
+                  10,11,22

test/SIM_checkpoint_data_recording/RUN_test4/unit_test.py

@@ -0,0 +1,12 @@
+import trick
+
+def main():
+    
+    exec(open("Modified_data/foo.dr").read())
+
+    trick.add_read(5.0, 'trick.load_checkpoint("RUN_test4/chkpnt_2.000000")') # contains data recording, starts at t=2
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test5/dump.py

@@ -0,0 +1,14 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():
+    exec(open("Modified_data/foo.dr").read())
+    
+    trick.checkpoint(7.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv

@@ -0,0 +1,31 @@
+sys.exec.out.time {s},testSimObject.my_foo.a {1},testSimObject.my_foo.b {1}
+                 7.1,8,16
+                 7.2,8,16
+                 7.3,8,16
+                 7.4,8,16
+                 7.5,8,16
+                 7.6,8,16
+                 7.7,8,16
+                 7.8,8,16
+                 7.9,8,16
+                   8,9,18
+                 8.1,9,18
+   8.199999999999999,9,18
+   8.300000000000001,9,18
+                 8.4,9,18
+                 8.5,9,18
+                 8.6,9,18
+   8.699999999999999,9,18
+   8.800000000000001,9,18
+                 8.9,9,18
+                   9,10,20
+                 9.1,10,20
+   9.199999999999999,10,20
+   9.300000000000001,10,20
+                 9.4,10,20
+                 9.5,10,20
+                 9.6,10,20
+   9.699999999999999,10,20
+   9.800000000000001,10,20
+                 9.9,10,20
+                  10,11,22

test/SIM_checkpoint_data_recording/RUN_test5/unit_test.py

@@ -0,0 +1,12 @@
+import trick
+
+def main():
+    
+    exec(open("Modified_data/foo.dr").read())
+
+    trick.add_read(5.0, 'trick.load_checkpoint("RUN_test5/chkpnt_7.000000")') # contains data recording, starts at t=7
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test6/dump.py

@@ -0,0 +1,14 @@
+import trick
+from trick.unit_test import *
+
+# This was just here for convenience to dump the checkpoints.
+
+def main():
+    exec(open("Modified_data/foo2.dr").read())
+    
+    trick.checkpoint(7.0)
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv

@@ -0,0 +1,4 @@
+sys.exec.out.time {s},testSimObject.my_foo.b {1}
+                   8,18
+                   9,20
+                  10,22

test/SIM_checkpoint_data_recording/RUN_test6/unit_test.py

@@ -0,0 +1,13 @@
+import trick
+
+def main():
+    
+    exec(open("Modified_data/foo.dr").read())
+
+    # trick.checkpoint(7.0)
+    trick.add_read(5.0, 'trick.load_checkpoint("RUN_test6/chkpnt_7.000000")') # contains data recording, starts at t=7
+
+    trick.stop(10.0)
+
+if __name__ == "__main__":
+    main()

test/SIM_checkpoint_data_recording/S_define

@@ -0,0 +1,16 @@
+#include "sim_objects/default_trick_sys.sm"
+##include "Foo.hh"
+
+class TestSimObject : public Trick::SimObject {
+
+    public:
+    Foo my_foo;
+
+    TestSimObject () {
+        ("initialization") my_foo.init ();
+        (1.0, "scheduled") my_foo.increment () ;
+    }
+
+};
+
+TestSimObject testSimObject;

test/SIM_checkpoint_data_recording/S_overrides.mk

@@ -0,0 +1,7 @@
+TRICK_CFLAGS += -I./models
+TRICK_CXXFLAGS += -I./models
+
+clean_logs:
+	rm RUN_test*/log_foo*
+
+clean: clean_logs

test/SIM_checkpoint_data_recording/models/Foo.cpp

@@ -0,0 +1 @@
+#include "Foo.hh"

test/SIM_checkpoint_data_recording/models/Foo.hh

@@ -0,0 +1,16 @@
+
+class Foo {
+    public:
+    int a; 
+    int b;
+
+    void init () {
+        a = 0;
+        b = 0;
+    }
+
+    void increment () {
+        a++;
+        b+=2;
+    }
+};

test_sims.yml

@@ -21,8 +21,8 @@ SIM_measurement_units:
   path: test/SIM_measurement_units
 SIM_parse_s_define:
   path: test/SIM_parse_s_define
-# SIM_target_specific_variables:                            <-- This sim has to sit out on Mac until we fix the sprintf issue
+SIM_target_specific_variables:                          
-#   path: test/SIM_target_specific_variables
+  path: test/SIM_target_specific_variables
 SIM_test_abstract:
   path: test/SIM_test_abstract
 SIM_test_inherit:
@@ -233,3 +233,46 @@ SIM_test_dr:
       - test/SIM_test_dr/RUN_test/log_DR_typesASCII.csv vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_typesASCII_Master.csv
       - test/SIM_test_dr/RUN_test/log_DR_bitfieldsBINARY.trk vs. test/SIM_test_dr/RUN_test/Ref_Logs/log_DR_bitfieldsBINARY.trk
 
+# All the dump.py runs dump a checkpoint
+# All the unit_test.py runs load that checkpoint and then compare against expected logs
+SIM_checkpoint_data_recording: 
+  path: test/SIM_checkpoint_data_recording
+  build_command: "trick-CP -t"
+  binary: "T_main_{cpu}_test.exe"
+  runs:
+    RUN_test1/dump.py:
+      returns: 0
+    RUN_test2/dump.py:
+      returns: 0
+    RUN_test3/dump.py:
+      returns: 0
+    RUN_test4/dump.py:
+      returns: 0
+    RUN_test5/dump.py:
+      returns: 0
+    # RUN_test6/dump.py:
+    #   returns: 0
+
+    RUN_test1/unit_test.py:
+      returns: 0
+      compare: 
+        - test/SIM_checkpoint_data_recording/RUN_test1/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test1/log_foo.csv
+    RUN_test2/unit_test.py:
+      returns: 0
+      analyze: './test/SIM_checkpoint_data_recording/RUN_test2/check_log.sh'
+    RUN_test3/unit_test.py:
+      returns: 0
+      compare: 
+        - test/SIM_checkpoint_data_recording/RUN_test3/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test3/log_foo.csv
+    RUN_test4/unit_test.py:
+      returns: 0
+      compare: 
+        - test/SIM_checkpoint_data_recording/RUN_test4/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test4/log_foo.csv
+    RUN_test5/unit_test.py:
+      returns: 0
+      compare: 
+        - test/SIM_checkpoint_data_recording/RUN_test5/ref_log_foo.csv vs. test/SIM_checkpoint_data_recording/RUN_test5/log_foo.csv
+    # RUN_test6/unit_test.py:
+    #   returns: 0
+    #   compare: 
+    #     - test/SIM_checkpoint_data_recording/RUN_test6/ref_log_foo2.csv vs. test/SIM_checkpoint_data_recording/RUN_test6/log_foo2.csv

trickops.py

@@ -23,13 +23,23 @@ class SimTestWorkflow(TrickWorkflow):
       run_jobs        = self.get_jobs(kind='run')
       analysis_jobs   = self.get_jobs(kind='analyze')
 
-      # This job dumps a checkpoint that is then read in and checked by RUN_test/unit_test.py in the same sim
+      # This job in SIM_stls dumps a checkpoint that is then read in and checked by RUN_test/unit_test.py in the same sim
       # This is a workaround to ensure that this run goes first.
-      first_phase_job = self.get_sim('SIM_stls').get_run(input='RUN_test/setup.py').get_run_job()
+      first_phase_jobs = []
-      run_jobs.remove(first_phase_job)
+      stl_dump_job = self.get_sim('SIM_stls').get_run(input='RUN_test/setup.py').get_run_job()
+      first_phase_jobs.append(stl_dump_job)
+      run_jobs.remove(stl_dump_job)
+
+      # Same with SIM_checkpoint_data_recording - half the runs dump checkpoints, the others read and verify.
+      # Make sure that the dump checkpoint runs go first.
+      num_dump_runs = int(len(self.get_sim('SIM_checkpoint_data_recording').get_runs())/2)
+      for i in range(num_dump_runs):
+        job = self.get_sim('SIM_checkpoint_data_recording').get_run(input=('RUN_test' + str(i+1) + '/dump.py')).get_run_job()
+        first_phase_jobs.append(job)
+        run_jobs.remove(job)
 
       builds_status = self.execute_jobs(build_jobs, max_concurrent=self.cpus, header='Executing all sim builds.')
-      first_phase_run_status = self.execute_jobs([first_phase_job], max_concurrent=self.cpus, header="Executing required first phase runs.")
+      first_phase_run_status = self.execute_jobs(first_phase_jobs, max_concurrent=self.cpus, header="Executing required first phase runs.")
       runs_status   = self.execute_jobs(run_jobs,   max_concurrent=self.cpus, header='Executing all sim runs.')
       comparison_result = self.compare()
       analysis_status = self.execute_jobs(analysis_jobs, max_concurrent=self.cpus, header='Executing all analysis.')
@@ -38,7 +48,7 @@ class SimTestWorkflow(TrickWorkflow):
       self.status_summary()   # Print a Succinct summary
 
       # Dump failing logs
-      jobs = build_jobs + run_jobs
+      jobs = build_jobs + first_phase_jobs + run_jobs
       for job in jobs:
         if job.get_status() == Job.Status.FAILED:
           print("Failing job: ", job.name)