add node & task heartbeat events (#621)

This adds node & task heartbeats and makes the event data available as a structured data in the logs.
This commit is contained in:
bmc-msft
2021-03-02 17:04:39 -05:00
committed by GitHub
parent 7f66eeee0d
commit 4489036d9f
5 changed files with 538 additions and 3 deletions

View File

@ -8,10 +8,11 @@ import json
import logging
import azure.functions as func
from onefuzztypes.events import EventNodeHeartbeat
from onefuzztypes.models import NodeHeartbeatEntry
from pydantic import ValidationError
from ..onefuzzlib.events import get_events
from ..onefuzzlib.events import get_events, send_event
from ..onefuzzlib.workers.nodes import Node
@ -27,6 +28,13 @@ def main(msg: func.QueueMessage, dashboard: func.Out[str]) -> None:
return
node.heartbeat = datetime.datetime.utcnow()
node.save()
send_event(
EventNodeHeartbeat(
machine_id=node.machine_id,
scaleset_id=node.scaleset_id,
pool_name=node.pool_name,
)
)
except ValidationError:
logging.error("invalid node heartbeat: %s", raw)

View File

@ -8,10 +8,11 @@ import logging
from datetime import datetime
import azure.functions as func
from onefuzztypes.events import EventTaskHeartbeat
from onefuzztypes.models import Error, TaskHeartbeatEntry
from pydantic import ValidationError
from ..onefuzzlib.events import get_events
from ..onefuzzlib.events import get_events, send_event
from ..onefuzzlib.tasks.main import Task
@ -29,6 +30,11 @@ def main(msg: func.QueueMessage, dashboard: func.Out[str]) -> None:
if task:
task.heartbeat = datetime.utcnow()
task.save()
send_event(
EventTaskHeartbeat(
job_id=task.job_id, task_id=task.task_id, config=task.config
)
)
except ValidationError:
logging.error("invalid task heartbeat: %s", raw)