mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-21 13:51:19 +00:00
Expose coverage/exec_sec for libfuzzer targets via CLI (#325)
Adds debug subcommands to the SDK/CLI that simplify querying Application Insights for libfuzzer telemetry. Querying for the latest execs_sec for a job, by job_id fragment. ``` $ onefuzz debug job libfuzzer_execs_sec 88 --limit 1 [ { "execs_sec": "191035", "machine_id": "b2dbe720-4fd8-4342-957a-6cb0979d2187", "timestamp": "2020-11-18T00:08:53.98Z", "worker_id": "0" } ] ``` Querying for the latest coverage for a job, by job_id fragment. ``` $ onefuzz debug job libfuzzer_coverage 88 --limit 1 [ { "covered": "10", "features": "21", "rate": "0.47619047619047616", "timestamp": "2020-11-18T00:09:40.793Z" } ] ```
This commit is contained in:
@ -28,6 +28,7 @@ from .ssh import ssh_connect
|
|||||||
|
|
||||||
EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||||
ZERO_SHA256 = "0" * len(EMPTY_SHA256)
|
ZERO_SHA256 = "0" * len(EMPTY_SHA256)
|
||||||
|
DAY_TIMESPAN = "PT24H"
|
||||||
|
|
||||||
|
|
||||||
class DebugRepro(Command):
|
class DebugRepro(Command):
|
||||||
@ -213,6 +214,42 @@ class DebugTask(Command):
|
|||||||
scaleset_id, node_id = self._get_node(task_id, node_id)
|
scaleset_id, node_id = self._get_node(task_id, node_id)
|
||||||
return self.onefuzz.debug.scalesets.rdp(scaleset_id, node_id, duration=duration)
|
return self.onefuzz.debug.scalesets.rdp(scaleset_id, node_id, duration=duration)
|
||||||
|
|
||||||
|
def libfuzzer_coverage(
|
||||||
|
self,
|
||||||
|
task_id: UUID_EXPANSION,
|
||||||
|
timespan: str = DAY_TIMESPAN,
|
||||||
|
limit: Optional[int] = None,
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
Get the coverage for the specified task
|
||||||
|
|
||||||
|
:param task_id value: Task ID
|
||||||
|
:param str timespan: ISO 8601 duration format
|
||||||
|
:param int limit: Limit the number of records returned
|
||||||
|
"""
|
||||||
|
task = self.onefuzz.tasks.get(task_id)
|
||||||
|
query = f"where customDimensions.task_id == '{task.task_id}'"
|
||||||
|
return self.onefuzz.debug.logs._query_libfuzzer_coverage(query, timespan, limit)
|
||||||
|
|
||||||
|
def libfuzzer_execs_sec(
|
||||||
|
self,
|
||||||
|
task_id: UUID_EXPANSION,
|
||||||
|
timespan: str = DAY_TIMESPAN,
|
||||||
|
limit: Optional[int] = None,
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
Get the executions per second for the specified task
|
||||||
|
|
||||||
|
:param task_id value: Task ID
|
||||||
|
:param str timespan: ISO 8601 duration format
|
||||||
|
:param int limit: Limit the number of records returned
|
||||||
|
"""
|
||||||
|
task = self.onefuzz.tasks.get(task_id)
|
||||||
|
query = f"where customDimensions.task_id == '{task.task_id}'"
|
||||||
|
return self.onefuzz.debug.logs._query_libfuzzer_execs_sec(
|
||||||
|
query, timespan, limit
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DebugJobTask(Command):
|
class DebugJobTask(Command):
|
||||||
""" Debug a task for a specific job """
|
""" Debug a task for a specific job """
|
||||||
@ -258,6 +295,42 @@ class DebugJob(Command):
|
|||||||
super().__init__(onefuzz, logger)
|
super().__init__(onefuzz, logger)
|
||||||
self.task = DebugJobTask(onefuzz, logger)
|
self.task = DebugJobTask(onefuzz, logger)
|
||||||
|
|
||||||
|
def libfuzzer_coverage(
|
||||||
|
self,
|
||||||
|
job_id: UUID_EXPANSION,
|
||||||
|
timespan: str = DAY_TIMESPAN,
|
||||||
|
limit: Optional[int] = None,
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
Get the coverage for the specified job
|
||||||
|
|
||||||
|
:param job_id value: Job ID
|
||||||
|
:param str timespan: ISO 8601 duration format
|
||||||
|
:param int limit: Limit the number of records returned
|
||||||
|
"""
|
||||||
|
job = self.onefuzz.jobs.get(job_id)
|
||||||
|
query = f"where customDimensions.job_id == '{job.job_id}'"
|
||||||
|
return self.onefuzz.debug.logs._query_libfuzzer_coverage(query, timespan, limit)
|
||||||
|
|
||||||
|
def libfuzzer_execs_sec(
|
||||||
|
self,
|
||||||
|
job_id: UUID_EXPANSION,
|
||||||
|
timespan: str = DAY_TIMESPAN,
|
||||||
|
limit: Optional[int] = None,
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
Get the executions per second for the specified job
|
||||||
|
|
||||||
|
:param job_id value: Job ID
|
||||||
|
:param str timespan: ISO 8601 duration format
|
||||||
|
:param int limit: Limit the number of records returned
|
||||||
|
"""
|
||||||
|
job = self.onefuzz.jobs.get(job_id)
|
||||||
|
query = f"where customDimensions.job_id == '{job.job_id}'"
|
||||||
|
return self.onefuzz.debug.logs._query_libfuzzer_execs_sec(
|
||||||
|
query, timespan, limit
|
||||||
|
)
|
||||||
|
|
||||||
def download_files(self, job_id: UUID_EXPANSION, output: Directory) -> None:
|
def download_files(self, job_id: UUID_EXPANSION, output: Directory) -> None:
|
||||||
""" Download the containers by container type for each task in the specified job """
|
""" Download the containers by container type for each task in the specified job """
|
||||||
|
|
||||||
@ -306,7 +379,7 @@ class DebugLog(Command):
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
def query(
|
def query(
|
||||||
self, log_query: str, *, timespan: str = "PT24H", raw: bool = False
|
self, log_query: str, *, timespan: str = DAY_TIMESPAN, raw: bool = False
|
||||||
) -> Any:
|
) -> Any:
|
||||||
"""
|
"""
|
||||||
Perform an Application Insights query
|
Perform an Application Insights query
|
||||||
@ -326,6 +399,7 @@ class DebugLog(Command):
|
|||||||
app_id = self.onefuzz.info.get().insights_appid
|
app_id = self.onefuzz.info.get().insights_appid
|
||||||
if app_id is None:
|
if app_id is None:
|
||||||
raise Exception("instance does not have an insights_appid")
|
raise Exception("instance does not have an insights_appid")
|
||||||
|
self.logger.debug("query: %s", log_query)
|
||||||
raw_data = client.query.execute(
|
raw_data = client.query.execute(
|
||||||
app_id, body=QueryBody(query=log_query, timespan=timespan)
|
app_id, body=QueryBody(query=log_query, timespan=timespan)
|
||||||
)
|
)
|
||||||
@ -337,11 +411,17 @@ class DebugLog(Command):
|
|||||||
return raw_data
|
return raw_data
|
||||||
return self._convert(raw_data)
|
return self._convert(raw_data)
|
||||||
|
|
||||||
|
def _query_parts(
|
||||||
|
self, parts: List[str], timespan: str, *, raw: bool = False
|
||||||
|
) -> Any:
|
||||||
|
log_query = " | ".join(parts)
|
||||||
|
return self.query(log_query, timespan=timespan, raw=raw)
|
||||||
|
|
||||||
def keyword(
|
def keyword(
|
||||||
self,
|
self,
|
||||||
value: str,
|
value: str,
|
||||||
*,
|
*,
|
||||||
timespan: str = "PT24H",
|
timespan: str = DAY_TIMESPAN,
|
||||||
limit: Optional[int] = None,
|
limit: Optional[int] = None,
|
||||||
raw: bool = False,
|
raw: bool = False,
|
||||||
) -> Any:
|
) -> Any:
|
||||||
@ -368,10 +448,62 @@ class DebugLog(Command):
|
|||||||
if limit is not None:
|
if limit is not None:
|
||||||
components.append(f"take {limit}")
|
components.append(f"take {limit}")
|
||||||
|
|
||||||
log_query = " | ".join(components)
|
return self._query_parts(components, timespan=timespan, raw=raw)
|
||||||
self.logger.debug("query: %s", log_query)
|
|
||||||
|
|
||||||
return self.query(log_query, timespan=timespan)
|
def _query_libfuzzer_coverage(
|
||||||
|
self, query: str, timespan: str, limit: Optional[int] = None
|
||||||
|
) -> Any:
|
||||||
|
project_fields = [
|
||||||
|
"rate=customDimensions.rate",
|
||||||
|
"covered=customDimensions.covered",
|
||||||
|
"features=customDimensions.features",
|
||||||
|
"timestamp",
|
||||||
|
]
|
||||||
|
|
||||||
|
query_parts = [
|
||||||
|
"customEvents",
|
||||||
|
"where name == 'coverage_data'",
|
||||||
|
query,
|
||||||
|
"order by timestamp desc",
|
||||||
|
f"project {','.join(project_fields)}",
|
||||||
|
]
|
||||||
|
|
||||||
|
if limit:
|
||||||
|
query_parts.append(f"take {limit}")
|
||||||
|
|
||||||
|
results = self.onefuzz.debug.logs._query_parts(query_parts, timespan=timespan)
|
||||||
|
if "PrimaryResult" in results:
|
||||||
|
return results["PrimaryResult"]
|
||||||
|
return results
|
||||||
|
|
||||||
|
def _query_libfuzzer_execs_sec(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
timespan: str,
|
||||||
|
limit: Optional[int] = None,
|
||||||
|
) -> Any:
|
||||||
|
project_fields = [
|
||||||
|
"machine_id=customDimensions.machine_id",
|
||||||
|
"worker_id=customDimensions.worker_id",
|
||||||
|
"execs_sec=customDimensions.execs_sec",
|
||||||
|
"timestamp",
|
||||||
|
]
|
||||||
|
|
||||||
|
query_parts = [
|
||||||
|
"customEvents",
|
||||||
|
"where name == 'runtime_stats'",
|
||||||
|
query,
|
||||||
|
"where customDimensions.execs_sec > 0",
|
||||||
|
"order by timestamp desc",
|
||||||
|
f"project {','.join(project_fields)}",
|
||||||
|
]
|
||||||
|
if limit:
|
||||||
|
query_parts.append(f"take {limit}")
|
||||||
|
|
||||||
|
results = self.onefuzz.debug.logs._query_parts(query_parts, timespan=timespan)
|
||||||
|
if "PrimaryResult" in results:
|
||||||
|
return results["PrimaryResult"]
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
class DebugNotification(Command):
|
class DebugNotification(Command):
|
||||||
|
Reference in New Issue
Block a user