removing nodes whose ground truth is not avail (#275)

This commit is contained in:
Anshuman Goel
2020-11-11 09:20:05 -08:00
committed by GitHub
parent 8678755b71
commit dec1a2d7b0

View File

@ -767,24 +767,31 @@ class Scaleset(BASE_SCALESET, ORMMixin):
to_reimage = [] to_reimage = []
to_delete = [] to_delete = []
nodes = Node.search_states( # ground truth of existing nodes
scaleset_id=self.scaleset_id, states=NodeState.ready_for_reset() azure_nodes = list_instance_ids(self.scaleset_id)
)
nodes = Node.search_states(scaleset_id=self.scaleset_id)
if not nodes: if not nodes:
logging.info("no nodes need updating: %s", self.scaleset_id) logging.info("no nodes need updating: %s", self.scaleset_id)
return False return False
# ground truth of existing nodes # Nodes do not exists in scalesets but in table due to unknown failure
azure_nodes = list_instance_ids(self.scaleset_id)
for node in nodes: for node in nodes:
if node.machine_id not in azure_nodes: if node.machine_id not in azure_nodes:
logging.info( logging.info(
"no longer in scaleset: %s:%s", self.scaleset_id, node.machine_id "no longer in scaleset: %s:%s", self.scaleset_id, node.machine_id
) )
node.delete() node.delete()
elif node.delete_requested:
nodes_to_reset = [x for x in nodes if x.state in NodeState.ready_for_reset()]
if len(nodes_to_reset) == 0:
logging.info("No needs are ready for resetting: %s", self.scaleset_id)
return False
for node in nodes_to_reset:
if node.delete_requested:
to_delete.append(node) to_delete.append(node)
else: else:
if ScalesetShrinkQueue(self.scaleset_id).should_shrink(): if ScalesetShrinkQueue(self.scaleset_id).should_shrink():