mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-16 03:48:09 +00:00
Refactoring proxy lifetime to only shutdown when proxy is out-of-date. (#839)
## Summary of the Pull Request _What is this about?_ We'd like to refactor the proxy lifecycle to only delete when the proxy is out-of-date - i.e. when the proxy is older than 7 days or a mismatched version. I've changed two files, proxy.py and timer_daily\init.py to check for the version and timestamp before stopping a live proxy. ## PR Checklist * [ ] Applies to work item: #xxx * [ ] CLA signed. If not, go over [here](https://cla.opensource.microsoft.com/microsoft/onefuzz) and sign the CLI. * [ ] Tests added/passed * [ ] Requires documentation to be updated * [x] I've discussed this with core contributors already. If not checked, I'm ready to accept this work might be rejected in favor of a different grand plan. Issue number where discussion took place: #xxx ## Info on Pull Request _What does this include?_ Changes to two files: proxy.py: - get_or_create() edited to check if timestamp is >7 days. - Created is_outdated() to check version and timestamp for out-of-date proxy. timer_daily/init.py - Proxy check now includes is_outdated() before determining if a proxy should be shutdown. ## Validation Steps Performed Deploying test instance to determine if proxy lives past a single day.
This commit is contained in:
nharper285
GitHub
@ -7,6 +7,7 @@ import datetime
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Optional, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from azure.mgmt.compute.models import VirtualMachine
|
||||
from onefuzztypes.enums import ErrorCode, VmState
|
||||
@ -37,12 +38,17 @@ from .proxy_forward import ProxyForward
|
||||
PROXY_SKU = "Standard_B2s"
|
||||
PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
|
||||
PROXY_LOG_PREFIX = "scaleset-proxy: "
|
||||
PROXY_LIFESPAN = datetime.timedelta(days=7)
|
||||
|
||||
|
||||
# This isn't intended to ever be shared to the client, hence not being in
|
||||
# onefuzztypes
|
||||
class Proxy(ORMMixin):
|
||||
timestamp: Optional[datetime.datetime] = Field(alias="Timestamp")
|
||||
created_timestamp: datetime.datetime = Field(
|
||||
default_factory=datetime.datetime.utcnow
|
||||
)
|
||||
proxy_id: UUID = Field(default_factory=uuid4)
|
||||
region: Region
|
||||
state: VmState = Field(default=VmState.init)
|
||||
auth: Authentication = Field(default_factory=build_auth)
|
||||
@ -50,14 +56,15 @@ class Proxy(ORMMixin):
|
||||
error: Optional[Error]
|
||||
version: str = Field(default=__version__)
|
||||
heartbeat: Optional[ProxyHeartbeat]
|
||||
outdated: bool = Field(default=False)
|
||||
|
||||
@classmethod
|
||||
def key_fields(cls) -> Tuple[str, Optional[str]]:
|
||||
return ("region", None)
|
||||
return ("region", "proxy_id")
|
||||
|
||||
def get_vm(self) -> VM:
|
||||
vm = VM(
|
||||
name="proxy-%s" % self.region,
|
||||
name="proxy-%s-%s" % (self.region, self.proxy_id),
|
||||
region=self.region,
|
||||
sku=PROXY_SKU,
|
||||
image=PROXY_IMAGE,
|
||||
@ -104,7 +111,9 @@ class Proxy(ORMMixin):
|
||||
return
|
||||
|
||||
logging.error(PROXY_LOG_PREFIX + "vm failed: %s - %s", self.region, error)
|
||||
send_event(EventProxyFailed(region=self.region, error=error))
|
||||
send_event(
|
||||
EventProxyFailed(region=self.region, proxy_id=self.proxy_id, error=error)
|
||||
)
|
||||
self.error = error
|
||||
self.state = VmState.stopping
|
||||
self.save()
|
||||
@ -131,7 +140,7 @@ class Proxy(ORMMixin):
|
||||
return
|
||||
self.ip = ip
|
||||
|
||||
extensions = proxy_manager_extensions(self.region)
|
||||
extensions = proxy_manager_extensions(self.region, self.proxy_id)
|
||||
result = vm.add_extensions(extensions)
|
||||
if isinstance(result, Error):
|
||||
self.set_failed(result)
|
||||
@ -154,6 +163,29 @@ class Proxy(ORMMixin):
|
||||
logging.info(PROXY_LOG_PREFIX + "removing proxy: %s", self.region)
|
||||
self.delete()
|
||||
|
||||
def is_outdated(self) -> bool:
|
||||
if self.version != __version__:
|
||||
logging.info(
|
||||
PROXY_LOG_PREFIX + "mismatch version: proxy:%s service:%s state:%s",
|
||||
self.version,
|
||||
__version__,
|
||||
self.state,
|
||||
)
|
||||
return True
|
||||
if self.created_timestamp is not None:
|
||||
proxy_timestamp = self.created_timestamp
|
||||
if proxy_timestamp < (
|
||||
datetime.datetime.now(tz=datetime.timezone.utc) - PROXY_LIFESPAN
|
||||
):
|
||||
logging.info(
|
||||
PROXY_LOG_PREFIX
|
||||
+ "proxy older than 7 days:proxy-created:%s state:%s",
|
||||
self.created_timestamp,
|
||||
self.state,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_used(self) -> bool:
|
||||
if len(self.get_forwards()) == 0:
|
||||
logging.info(PROXY_LOG_PREFIX + "no forwards: %s", self.region)
|
||||
@ -194,7 +226,9 @@ class Proxy(ORMMixin):
|
||||
|
||||
def get_forwards(self) -> List[Forward]:
|
||||
forwards: List[Forward] = []
|
||||
for entry in ProxyForward.search_forward(region=self.region):
|
||||
for entry in ProxyForward.search_forward(
|
||||
region=self.region, proxy_id=self.proxy_id
|
||||
):
|
||||
if entry.endtime < datetime.datetime.now(tz=datetime.timezone.utc):
|
||||
entry.delete()
|
||||
else:
|
||||
@ -212,7 +246,7 @@ class Proxy(ORMMixin):
|
||||
proxy_config = ProxyConfig(
|
||||
url=get_file_sas_url(
|
||||
Container("proxy-configs"),
|
||||
"%s/config.json" % self.region,
|
||||
"%s/%s/config.json" % (self.region, self.proxy_id),
|
||||
StorageType.config,
|
||||
read=True,
|
||||
),
|
||||
@ -223,6 +257,7 @@ class Proxy(ORMMixin):
|
||||
),
|
||||
forwards=forwards,
|
||||
region=self.region,
|
||||
proxy_id=self.proxy_id,
|
||||
instance_telemetry_key=os.environ.get("APPINSIGHTS_INSTRUMENTATIONKEY"),
|
||||
microsoft_telemetry_key=os.environ.get("ONEFUZZ_TELEMETRY"),
|
||||
instance_id=get_instance_id(),
|
||||
@ -230,7 +265,7 @@ class Proxy(ORMMixin):
|
||||
|
||||
save_blob(
|
||||
Container("proxy-configs"),
|
||||
"%s/config.json" % self.region,
|
||||
"%s/%s/config.json" % (self.region, self.proxy_id),
|
||||
proxy_config.json(),
|
||||
StorageType.config,
|
||||
)
|
||||
@ -244,28 +279,22 @@ class Proxy(ORMMixin):
|
||||
|
||||
@classmethod
|
||||
def get_or_create(cls, region: Region) -> Optional["Proxy"]:
|
||||
proxy = Proxy.get(region)
|
||||
if proxy is not None:
|
||||
if proxy.version != __version__:
|
||||
logging.info(
|
||||
PROXY_LOG_PREFIX + "mismatch version: proxy:%s service:%s state:%s",
|
||||
proxy.version,
|
||||
__version__,
|
||||
proxy.state,
|
||||
)
|
||||
if proxy.state != VmState.stopping:
|
||||
# If the proxy is out-of-date, delete and re-create it
|
||||
proxy.state = VmState.stopping
|
||||
proxy.save()
|
||||
return None
|
||||
proxy_list = Proxy.search(query={"region": [region], "outdated": [False]})
|
||||
for proxy in proxy_list:
|
||||
if proxy.is_outdated():
|
||||
proxy.outdated = True
|
||||
proxy.save()
|
||||
continue
|
||||
if proxy.state not in VmState.available():
|
||||
continue
|
||||
return proxy
|
||||
|
||||
logging.info(PROXY_LOG_PREFIX + "creating proxy: region:%s", region)
|
||||
proxy = Proxy(region=region)
|
||||
proxy.save()
|
||||
send_event(EventProxyCreated(region=region))
|
||||
send_event(EventProxyCreated(region=region, proxy_id=proxy.proxy_id))
|
||||
return proxy
|
||||
|
||||
def delete(self) -> None:
|
||||
super().delete()
|
||||
send_event(EventProxyDeleted(region=self.region))
|
||||
send_event(EventProxyDeleted(region=self.region, proxy_id=self.proxy_id))
|
||||
|
||||
Reference in New Issue
Block a user