From 27bf0b030bf97c886de1350a5c58dc6e03085711 Mon Sep 17 00:00:00 2001
From: Derek Bankieris <derek.r.bankieris@nasa.gov>
Date: Wed, 28 Feb 2018 14:13:05 -0600
Subject: [PATCH] Dequeue timed-out runs if the slave later returns

Fixes #569
---
 .../MonteCarlo/MonteCarlo_funcs.cpp           | 25 ++++++-------------
 .../MonteCarlo/MonteCarlo_receive_results.cpp |  9 ++++++-
 .../MonteCarlo/MonteCarlo_slave.cpp           |  8 +++---
 3 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/trick_source/sim_services/MonteCarlo/MonteCarlo_funcs.cpp b/trick_source/sim_services/MonteCarlo/MonteCarlo_funcs.cpp
index 107ae65d..c6597608 100644
--- a/trick_source/sim_services/MonteCarlo/MonteCarlo_funcs.cpp
+++ b/trick_source/sim_services/MonteCarlo/MonteCarlo_funcs.cpp
@@ -397,24 +397,15 @@ Trick::MonteRun *Trick::MonteCarlo::get_next_dispatch() {
     /** <ul><li> While there are remaining runs: */
     while (!runs.empty()) {
         MonteRun *curr_run = runs.front();
-        /** <ul><li> If this run hasn't been dispatched before: */
-        if (curr_run->num_tries == 0) {
-            /** <li> If it is in range, return it. </ul>*/
-            if (in_range(curr_run)) {
-                return curr_run;
-            /** <li> Otherwise, run the pre run jobs and dequeue it. */
-            } else {
-                if (verbosity >= ALL) {
-                    message_publish(MSG_WARNING, "Monte [Master] Run %d is out of range and has been skipped.\n", curr_run->id) ;
-                }
-                prepare_run(curr_run);
-            }
-        /**
-         * <li> If this run has been dispatched before, it may have been requeued due to a slave timeout for which the slave
-         * later returned results. In such a case, do not dispatch it again (return NULL). Otherwise, return it.
-         */
-        } else if (curr_run->exit_status == MonteRun::INCOMPLETE) {
+        /** <li> If it is in range, return it. </ul>*/
+        if (in_range(curr_run)) {
             return curr_run;
+        /** <li> Otherwise, run the pre run jobs and dequeue it. */
+        } else {
+            if (verbosity >= ALL) {
+                message_publish(MSG_WARNING, "Monte [Master] Run %d is out of range and has been skipped.\n", curr_run->id) ;
+            }
+            prepare_run(curr_run);
         }
     }
     return NULL;
diff --git a/trick_source/sim_services/MonteCarlo/MonteCarlo_receive_results.cpp b/trick_source/sim_services/MonteCarlo/MonteCarlo_receive_results.cpp
index 613889c2..998ff55c 100644
--- a/trick_source/sim_services/MonteCarlo/MonteCarlo_receive_results.cpp
+++ b/trick_source/sim_services/MonteCarlo/MonteCarlo_receive_results.cpp
@@ -81,7 +81,14 @@ void Trick::MonteCarlo::handle_run_data(Trick::MonteSlave& slave) {
     }
 
     /**
-     * <ul><li> This run may have already been resolved by another slave if
+     * <ul><li> Try to remove this run from the queue in case it was requeue by #check_timeouts.
+     * This covers the case in which the master determines that a slave has timed out, requeues
+     * the run, and then the slave reports results.
+     */
+    dequeue_run(slave.current_run);
+
+    /**
+     * <li> This run may have already been resolved by another slave if
      * this slave was marked as having timed out. If that is the case,
      * discard these results.
      */
diff --git a/trick_source/sim_services/MonteCarlo/MonteCarlo_slave.cpp b/trick_source/sim_services/MonteCarlo/MonteCarlo_slave.cpp
index 2fce326b..12d54f40 100644
--- a/trick_source/sim_services/MonteCarlo/MonteCarlo_slave.cpp
+++ b/trick_source/sim_services/MonteCarlo/MonteCarlo_slave.cpp
@@ -16,7 +16,7 @@ int Trick::MonteCarlo::execute_as_slave() {
         /** <ul><li> On a blocking read, wait for a MonteSlave::Command from the master. */
         if (tc_accept(&listen_device, &connection_device) != TC_SUCCESS) {
             if (verbosity >= ERROR) {
-                message_publish(MSG_ERROR, "Monte [%s:%d] Lost connection to Master.\nShutting down.\n",
+                message_publish(MSG_ERROR, "Monte [%s:%d] Lost connection to Master. Shutting down.\n",
                                 machine_name.c_str(), slave_id) ;
             }
             slave_shutdown();
@@ -24,7 +24,7 @@ int Trick::MonteCarlo::execute_as_slave() {
         int command;
         if (tc_read(&connection_device, (char *)&command, (int)sizeof(command)) != (int)sizeof(command)) {
             if (verbosity >= ERROR) {
-                message_publish(MSG_ERROR, "Monte [%s:%d] Lost connection to Master while receiving instructions.\nShutting down.\n",
+                message_publish(MSG_ERROR, "Monte [%s:%d] Lost connection to Master while receiving instructions. Shutting down.\n",
                                 machine_name.c_str(), slave_id) ;
             }
             slave_shutdown();
@@ -44,7 +44,7 @@ int Trick::MonteCarlo::execute_as_slave() {
             case MonteSlave::SHUTDOWN:
                 /** <li> MonteSlave::SHUTDOWN: Call #slave_shutdown. */
                 if (verbosity >= INFORMATIONAL) {
-                    message_publish(MSG_INFO, "Monte [%s:%d] Shutdown command received from Master.\nShutting down.\n",
+                    message_publish(MSG_INFO, "Monte [%s:%d] Shutdown command received from Master. Shutting down.\n",
                                     machine_name.c_str(), slave_id) ;
                 }
                 slave_shutdown();
@@ -60,7 +60,7 @@ int Trick::MonteCarlo::execute_as_slave() {
             default:
                 /** <li> default: Call #slave_shutdown. */
                 if (verbosity >= ERROR) {
-                message_publish(MSG_ERROR, "Monte [%s:%d] Unrecognized command %d received from Master.\nShutting down.\n",
+                message_publish(MSG_ERROR, "Monte [%s:%d] Unrecognized command %d received from Master. Shutting down.\n",
                                 machine_name.c_str(), slave_id, command) ;
                 }
                 slave_shutdown();