update nodes and scalesets in a consistent order (#512)

src/api-service/__app__/timer_workers/__init__.py

@@ -33,22 +33,30 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None:  # noqa:
     # NOTE: Update pools first, such that scalesets impacted by pool updates
     # (such as shutdown or resize) happen during this iteration `timer_worker`
     # rather than the following iteration.
+
     pools = Pool.search()
     for pool in pools:
         if pool.state in PoolState.needs_work():
             logging.info("update pool: %s (%s)", pool.pool_id, pool.name)
             process_state_updates(pool)
-        elif pool.state in PoolState.available() and pool.autoscale:
+
+        if pool.state in PoolState.available() and pool.autoscale:
             autoscale_pool(pool)
 
+    # NOTE: Nodes, and Scalesets should be processed in a consistent order such
+    # during 'pool scale down' operations. This means that pools that are
+    # scaling down will more likely remove from the same scalesets over time.
+    # By more likely removing from the same scalesets, we are more likely to
+    # get to empty scalesets, which can safely be deleted.
+
     Node.mark_outdated_nodes()
     nodes = Node.search_states(states=NodeState.needs_work())
-    for node in nodes:
+    for node in sorted(nodes, key=lambda x: x.machine_id):
         logging.info("update node: %s", node.machine_id)
         process_state_updates(node)
 
     scalesets = Scaleset.search()
-    for scaleset in scalesets:
+    for scaleset in sorted(scalesets, key=lambda x: x.scaleset_id):
         process_scaleset(scaleset)
 
     events = get_events()