From af39d25a7df455b2b26cbc3b614f13b5610061fb Mon Sep 17 00:00:00 2001
From: bmc-msft <41130664+bmc-msft@users.noreply.github.com>
Date: Tue, 8 Jun 2021 13:37:10 -0400
Subject: [PATCH] reimage/delete expired nodes even with the debug_keep_node
 flag (#968)

Fixes #965
---
 .../__app__/onefuzzlib/workers/nodes.py       | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/api-service/__app__/onefuzzlib/workers/nodes.py b/src/api-service/__app__/onefuzzlib/workers/nodes.py
index fe8a4e93e..7f26d60de 100644
--- a/src/api-service/__app__/onefuzzlib/workers/nodes.py
+++ b/src/api-service/__app__/onefuzzlib/workers/nodes.py
@@ -264,6 +264,13 @@ class Node(BASE_NODE, ORMMixin):
             self.stop(done=True)
             return False
 
+        if self.is_too_old():
+            logging.info(
+                "can_schedule node is too old.  machine_id:%s", self.machine_id
+            )
+            self.stop(done=True)
+            return False
+
         if self.state in NodeState.ready_for_reset():
             logging.info(
                 "can_schedule node is set for reset.  machine_id:%s", self.machine_id
@@ -332,6 +339,14 @@ class Node(BASE_NODE, ORMMixin):
     def is_outdated(self) -> bool:
         return self.version != __version__
 
+    def is_too_old(self) -> bool:
+        return (
+            self.scaleset_id is not None
+            and self.timestamp is not None
+            and self.timestamp
+            < datetime.datetime.now(datetime.timezone.utc) - NODE_REIMAGE_TIME
+        )
+
     def send_message(self, message: NodeCommand) -> None:
         NodeMessage(
             machine_id=self.machine_id,
@@ -423,6 +438,14 @@ class Node(BASE_NODE, ORMMixin):
             },
             raw_unchecked_filter=time_filter,
         ):
+            if node.debug_keep_node:
+                logging.info(
+                    "removing debug_keep_node for expired node. "
+                    "scaleset_id:%s machine_id:%s",
+                    node.scaleset_id,
+                    node.machine_id,
+                )
+                node.debug_keep_node = False
             node.to_reimage()
 
     def set_state(self, state: NodeState) -> None: