ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-16 20:08:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bug fixes for Pool Resize (#158)

Browse Source
This commit is contained in:
Anshuman Goel
2020-10-23 05:58:15 -07:00
committed by GitHub
parent 1c06d7085a
commit bfbc9f8c9e
1 changed files with 29 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/api-service/__app__/onefuzzlib/autoscale.py
View File

@ -80,12 +80,31 @@ def scale_up(pool: Pool, scalesets: List[Scaleset], nodes_needed: int) -> None:
def scale_down(scalesets: List[Scaleset], nodes_to_remove: int) -> None:
logging.info("Scaling down")
for scaleset in scalesets:
nodes = Node.search_states(
scaleset_id=scaleset.scaleset_id, states=[NodeState.free]
num_of_nodes = len(Node.search_states(scaleset_id=scaleset.scaleset_id))
if scaleset.size != num_of_nodes and scaleset.state not in [
ScalesetState.resize,
ScalesetState.shutdown,
ScalesetState.halt,
]:
scaleset.state = ScalesetState.resize
scaleset.save()
free_nodes = Node.search_states(
scaleset_id=scaleset.scaleset_id,
states=[NodeState.free],
)
nodes = []
for node in free_nodes:
if not node.delete_requested:
nodes.append(node)
logging.info(
"Scaleset: %s, #Free Nodes: %s" % (scaleset.scaleset_id, len(nodes))
)
if nodes and nodes_to_remove > 0:
max_nodes_remove = min(len(nodes), nodes_to_remove)
if max_nodes_remove >= scaleset.size and len(nodes) == scaleset.size:
# All nodes in scaleset are free. Can shutdown VMSS
if max_nodes_remove >= scaleset.size and len(nodes) >= scaleset.size:
scaleset.state = ScalesetState.shutdown
nodes_to_remove = nodes_to_remove - scaleset.size
scaleset.save()
@ -93,6 +112,7 @@ def scale_down(scalesets: List[Scaleset], nodes_to_remove: int) -> None:
node.set_shutdown()
continue
# Resize of VMSS needed
scaleset.size = scaleset.size - max_nodes_remove
nodes_to_remove = nodes_to_remove - max_nodes_remove
scaleset.state = ScalesetState.resize
@ -145,4 +165,10 @@ def autoscale_pool(pool: Pool) -> None:
# resizing scaleset or creating new scaleset.
scale_up(pool, scalesets, nodes_needed)
elif nodes_needed < 0:
for scaleset in scalesets:
nodes = Node.search_states(scaleset_id=scaleset.scaleset_id)
for node in nodes:
if node.delete_requested:
nodes_needed += 1
if nodes_needed < 0:
scale_down(scalesets, abs(nodes_needed))