try pushing updates to scaleset configs frequently until the push succeeds (#489)

src/api-service/__app__/onefuzzlib/azure/vmss.py

@@ -133,7 +133,7 @@ def check_can_update(name: UUID) -> Any:
     if vmss is None:
         raise UnableToUpdate
 
-    if vmss.provisioning_state != "Succeeded":
+    if vmss.provisioning_state == "Updating":
         raise UnableToUpdate
 
     return vmss

src/api-service/__app__/onefuzzlib/pools.py

@@ -1146,21 +1146,26 @@ class Scaleset(BASE_SCALESET, ORMMixin):
             self.nodes.append(node_state)
 
     def update_configs(self) -> None:
-        if self.state != ScalesetState.running:
-            logging.debug(
-                "scaleset not running, not updating configs: %s", self.scaleset_id
-            )
-            return
+        if not self.needs_config_update:
+            logging.debug("config update not needed: %s", self.scaleset_id)
+
+        logging.info("updating scaleset configs: %s", self.scaleset_id)
 
         pool = Pool.get_by_name(self.pool_name)
         if isinstance(pool, Error):
+            logging.error(
+                "unable to find pool during config update: %s - %s",
+                self.scaleset_id,
+                pool,
+            )
             self.set_failed(pool)
             return
 
-        logging.debug("updating scaleset configs: %s", self.scaleset_id)
         extensions = fuzz_extensions(pool, self)
         try:
             update_extensions(self.scaleset_id, extensions)
+            self.needs_config_update = False
+            self.save()
         except UnableToUpdate:
             logging.debug(
                 "unable to update configs, update already in progress: %s",

src/api-service/__app__/timer_daily/__init__.py

@@ -24,7 +24,8 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None:  # noqa:
     scalesets = Scaleset.search()
     for scaleset in scalesets:
         logging.info("updating scaleset configs: %s", scaleset.scaleset_id)
-        scaleset.update_configs()
+        scaleset.needs_config_update = True
+        scaleset.save()
 
     expired_webhook_logs = WebhookMessageLog.search_expired()
     for log_entry in expired_webhook_logs:

src/api-service/__app__/timer_workers/__init__.py

@@ -17,6 +17,8 @@ from ..onefuzzlib.pools import Node, Pool, Scaleset
 def process_scaleset(scaleset: Scaleset) -> None:
     logging.debug("checking scaleset for updates: %s", scaleset.scaleset_id)
 
+    scaleset.update_configs()
+
     # if the scaleset is touched during cleanup, don't continue to process it
     if scaleset.cleanup_nodes():
         logging.debug("scaleset needed cleanup: %s", scaleset.scaleset_id)

src/pytypes/onefuzztypes/models.py

@@ -621,6 +621,7 @@ class Scaleset(BaseModel):
     region: Region
     size: int
     spot_instances: bool
+    needs_config_update: bool = Field(default=False)
     error: Optional[Error]
     nodes: Optional[List[ScalesetNodeState]]
     client_id: Optional[UUID]