diff --git a/src/api-service/__app__/timer_workers/__init__.py b/src/api-service/__app__/timer_workers/__init__.py index 09bac76c0..b665d82ed 100644 --- a/src/api-service/__app__/timer_workers/__init__.py +++ b/src/api-service/__app__/timer_workers/__init__.py @@ -33,22 +33,30 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: # NOTE: Update pools first, such that scalesets impacted by pool updates # (such as shutdown or resize) happen during this iteration `timer_worker` # rather than the following iteration. + pools = Pool.search() for pool in pools: if pool.state in PoolState.needs_work(): logging.info("update pool: %s (%s)", pool.pool_id, pool.name) process_state_updates(pool) - elif pool.state in PoolState.available() and pool.autoscale: + + if pool.state in PoolState.available() and pool.autoscale: autoscale_pool(pool) + # NOTE: Nodes, and Scalesets should be processed in a consistent order such + # during 'pool scale down' operations. This means that pools that are + # scaling down will more likely remove from the same scalesets over time. + # By more likely removing from the same scalesets, we are more likely to + # get to empty scalesets, which can safely be deleted. + Node.mark_outdated_nodes() nodes = Node.search_states(states=NodeState.needs_work()) - for node in nodes: + for node in sorted(nodes, key=lambda x: x.machine_id): logging.info("update node: %s", node.machine_id) process_state_updates(node) scalesets = Scaleset.search() - for scaleset in scalesets: + for scaleset in sorted(scalesets, key=lambda x: x.scaleset_id): process_scaleset(scaleset) events = get_events()