diff --git a/src/api-service/__app__/onefuzzlib/proxy.py b/src/api-service/__app__/onefuzzlib/proxy.py index dd2e6196b..5e58ab3cc 100644 --- a/src/api-service/__app__/onefuzzlib/proxy.py +++ b/src/api-service/__app__/onefuzzlib/proxy.py @@ -169,6 +169,9 @@ class Proxy(ORMMixin): self.delete() def is_outdated(self) -> bool: + if self.state not in VmState.available(): + return True + if self.version != __version__: logging.info( PROXY_LOG_PREFIX + "mismatch version: proxy:%s service:%s state:%s", diff --git a/src/api-service/__app__/timer_daily/__init__.py b/src/api-service/__app__/timer_daily/__init__.py index 7f182fbe3..15b338e44 100644 --- a/src/api-service/__app__/timer_daily/__init__.py +++ b/src/api-service/__app__/timer_daily/__init__.py @@ -6,43 +6,13 @@ import logging import azure.functions as func -from onefuzztypes.enums import VmState -from onefuzztypes.events import EventProxyCreated -from ..onefuzzlib.events import get_events, send_event -from ..onefuzzlib.proxy import Proxy +from ..onefuzzlib.events import get_events from ..onefuzzlib.webhooks import WebhookMessageLog from ..onefuzzlib.workers.scalesets import Scaleset def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: F841 - proxy_list = Proxy.search() - # Marking Outdated Proxies. Subsequently, shutting down Outdated & Unused Proxies. - for proxy in proxy_list: - if proxy.is_outdated(): - logging.info("marking proxy in %s as outdated.", proxy.region) - proxy.outdated = True - proxy.save() - # Creating a new proxy if no proxy exists for a given region. - for proxy in proxy_list: - if proxy.outdated: - region_list = list( - filter( - lambda x: (x.region == proxy.region and not x.outdated), - proxy_list, - ) - ) - if not len(region_list): - logging.info("outdated proxy in %s, creating new one.", proxy.region) - new_proxy = Proxy(region=proxy.region) - new_proxy.save() - send_event( - EventProxyCreated(region=proxy.region, proxy_id=proxy.proxy_id) - ) - if not proxy.is_used(): - logging.info("stopping one proxy in %s.", proxy.region) - proxy.set_state(VmState.stopping) - scalesets = Scaleset.search() for scaleset in scalesets: logging.info("updating scaleset configs: %s", scaleset.scaleset_id) diff --git a/src/api-service/__app__/timer_proxy/__init__.py b/src/api-service/__app__/timer_proxy/__init__.py index 7d5f03a09..22a2dbd74 100644 --- a/src/api-service/__app__/timer_proxy/__init__.py +++ b/src/api-service/__app__/timer_proxy/__init__.py @@ -11,20 +11,27 @@ from onefuzztypes.enums import VmState from ..onefuzzlib.events import get_events from ..onefuzzlib.orm import process_state_updates from ..onefuzzlib.proxy import PROXY_LOG_PREFIX, Proxy +from ..onefuzzlib.workers.scalesets import Scaleset def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: F841 - # Reminder, proxies are created on-demand. If something is "wrong" with - # a proxy, the plan is: delete and recreate it. - for proxy in Proxy.search(): - if not proxy.is_alive(): - logging.error( - PROXY_LOG_PREFIX + "alive check failed, stopping: %s", proxy.region - ) - proxy.set_state(VmState.stopping) - proxy.save() - else: - proxy.save_proxy_config() + proxies = Proxy.search() + for proxy in proxies: + if proxy.state in VmState.available(): + # Note, outdated checked at the start, but set at the end of this loop. + # As this function is called via a timer, this works around a user + # requesting to use the proxy while this function is checking if it's + # out of date + if proxy.outdated and not proxy.is_used(): + proxy.set_state(VmState.stopping) + # If something is "wrong" with a proxy, delete & recreate it + elif not proxy.is_alive(): + logging.error( + PROXY_LOG_PREFIX + "alive check failed, stopping: %s", proxy.region + ) + proxy.set_state(VmState.stopping) + else: + proxy.save_proxy_config() if proxy.state in VmState.needs_work(): logging.info( @@ -34,6 +41,17 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: ) process_state_updates(proxy) + if proxy.is_outdated(): + proxy.outdated = True + proxy.save() + + # make sure there is a proxy for every currently active region + scalesets = Scaleset.search() + regions = set(x.region for x in scalesets) + for region in regions: + if all(x.outdated for x in proxies if x.region == region): + Proxy.get_or_create(region) + events = get_events() if events: dashboard.set(events)