ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-16 11:58:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

try pushing updates to scaleset configs frequently until the push succeeds (#489)

Browse Source
This commit is contained in:
bmc-msft
2021-02-01 10:09:40 -05:00
committed by GitHub
parent e894a523ab
commit 0f70ffa3e2
5 changed files with 17 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/api-service/__app__/onefuzzlib/azure/vmss.py
View File

@ -133,7 +133,7 @@ def check_can_update(name: UUID) -> Any:
if vmss is None: if vmss is None:
raise UnableToUpdate raise UnableToUpdate
if vmss.provisioning_state != "Succeeded": if vmss.provisioning_state == "Updating":
raise UnableToUpdate raise UnableToUpdate
return vmss return vmss

17
src/api-service/__app__/onefuzzlib/pools.py
View File

@ -1146,21 +1146,26 @@ class Scaleset(BASE_SCALESET, ORMMixin):
self.nodes.append(node_state) self.nodes.append(node_state)
def update_configs(self) -> None: def update_configs(self) -> None:
if self.state != ScalesetState.running: if not self.needs_config_update:
logging.debug( logging.debug("config update not needed: %s", self.scaleset_id)
"scaleset not running, not updating configs: %s", self.scaleset_id
) logging.info("updating scaleset configs: %s", self.scaleset_id)
return
pool = Pool.get_by_name(self.pool_name) pool = Pool.get_by_name(self.pool_name)
if isinstance(pool, Error): if isinstance(pool, Error):
logging.error(
"unable to find pool during config update: %s - %s",
self.scaleset_id,
pool,
)
self.set_failed(pool) self.set_failed(pool)
return return
logging.debug("updating scaleset configs: %s", self.scaleset_id)
extensions = fuzz_extensions(pool, self) extensions = fuzz_extensions(pool, self)
try: try:
update_extensions(self.scaleset_id, extensions) update_extensions(self.scaleset_id, extensions)
self.needs_config_update = False
self.save()
except UnableToUpdate: except UnableToUpdate:
logging.debug( logging.debug(
"unable to update configs, update already in progress: %s", "unable to update configs, update already in progress: %s",

3
src/api-service/__app__/timer_daily/__init__.py
View File

@ -24,7 +24,8 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa:
scalesets = Scaleset.search() scalesets = Scaleset.search()
for scaleset in scalesets: for scaleset in scalesets:
logging.info("updating scaleset configs: %s", scaleset.scaleset_id) logging.info("updating scaleset configs: %s", scaleset.scaleset_id)
scaleset.update_configs() scaleset.needs_config_update = True
scaleset.save()
expired_webhook_logs = WebhookMessageLog.search_expired() expired_webhook_logs = WebhookMessageLog.search_expired()
for log_entry in expired_webhook_logs: for log_entry in expired_webhook_logs:

2
src/api-service/__app__/timer_workers/__init__.py
View File

@ -17,6 +17,8 @@ from ..onefuzzlib.pools import Node, Pool, Scaleset
def process_scaleset(scaleset: Scaleset) -> None: def process_scaleset(scaleset: Scaleset) -> None:
logging.debug("checking scaleset for updates: %s", scaleset.scaleset_id) logging.debug("checking scaleset for updates: %s", scaleset.scaleset_id)
scaleset.update_configs()
# if the scaleset is touched during cleanup, don't continue to process it # if the scaleset is touched during cleanup, don't continue to process it
if scaleset.cleanup_nodes(): if scaleset.cleanup_nodes():
logging.debug("scaleset needed cleanup: %s", scaleset.scaleset_id) logging.debug("scaleset needed cleanup: %s", scaleset.scaleset_id)

1
src/pytypes/onefuzztypes/models.py
View File

@ -621,6 +621,7 @@ class Scaleset(BaseModel):
region: Region region: Region
size: int size: int
spot_instances: bool spot_instances: bool
needs_config_update: bool = Field(default=False)
error: Optional[Error] error: Optional[Error]
nodes: Optional[List[ScalesetNodeState]] nodes: Optional[List[ScalesetNodeState]]
client_id: Optional[UUID] client_id: Optional[UUID]