diff --git a/src/api-service/__app__/onefuzzlib/azure/vmss.py b/src/api-service/__app__/onefuzzlib/azure/vmss.py index 14d141b93..98f035c47 100644 --- a/src/api-service/__app__/onefuzzlib/azure/vmss.py +++ b/src/api-service/__app__/onefuzzlib/azure/vmss.py @@ -133,7 +133,7 @@ def check_can_update(name: UUID) -> Any: if vmss is None: raise UnableToUpdate - if vmss.provisioning_state != "Succeeded": + if vmss.provisioning_state == "Updating": raise UnableToUpdate return vmss diff --git a/src/api-service/__app__/onefuzzlib/pools.py b/src/api-service/__app__/onefuzzlib/pools.py index 638ae335e..d1d48d322 100644 --- a/src/api-service/__app__/onefuzzlib/pools.py +++ b/src/api-service/__app__/onefuzzlib/pools.py @@ -1146,21 +1146,26 @@ class Scaleset(BASE_SCALESET, ORMMixin): self.nodes.append(node_state) def update_configs(self) -> None: - if self.state != ScalesetState.running: - logging.debug( - "scaleset not running, not updating configs: %s", self.scaleset_id - ) - return + if not self.needs_config_update: + logging.debug("config update not needed: %s", self.scaleset_id) + + logging.info("updating scaleset configs: %s", self.scaleset_id) pool = Pool.get_by_name(self.pool_name) if isinstance(pool, Error): + logging.error( + "unable to find pool during config update: %s - %s", + self.scaleset_id, + pool, + ) self.set_failed(pool) return - logging.debug("updating scaleset configs: %s", self.scaleset_id) extensions = fuzz_extensions(pool, self) try: update_extensions(self.scaleset_id, extensions) + self.needs_config_update = False + self.save() except UnableToUpdate: logging.debug( "unable to update configs, update already in progress: %s", diff --git a/src/api-service/__app__/timer_daily/__init__.py b/src/api-service/__app__/timer_daily/__init__.py index 0621af705..57eed6631 100644 --- a/src/api-service/__app__/timer_daily/__init__.py +++ b/src/api-service/__app__/timer_daily/__init__.py @@ -24,7 +24,8 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: scalesets = Scaleset.search() for scaleset in scalesets: logging.info("updating scaleset configs: %s", scaleset.scaleset_id) - scaleset.update_configs() + scaleset.needs_config_update = True + scaleset.save() expired_webhook_logs = WebhookMessageLog.search_expired() for log_entry in expired_webhook_logs: diff --git a/src/api-service/__app__/timer_workers/__init__.py b/src/api-service/__app__/timer_workers/__init__.py index bf42dae2f..5f30cbf1a 100644 --- a/src/api-service/__app__/timer_workers/__init__.py +++ b/src/api-service/__app__/timer_workers/__init__.py @@ -17,6 +17,8 @@ from ..onefuzzlib.pools import Node, Pool, Scaleset def process_scaleset(scaleset: Scaleset) -> None: logging.debug("checking scaleset for updates: %s", scaleset.scaleset_id) + scaleset.update_configs() + # if the scaleset is touched during cleanup, don't continue to process it if scaleset.cleanup_nodes(): logging.debug("scaleset needed cleanup: %s", scaleset.scaleset_id) diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 4d5ebe1ab..bc8aa4765 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -621,6 +621,7 @@ class Scaleset(BaseModel): region: Region size: int spot_instances: bool + needs_config_update: bool = Field(default=False) error: Optional[Error] nodes: Optional[List[ScalesetNodeState]] client_id: Optional[UUID]