ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-09 08:41:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add job stopped task info (#648)

Browse Source
This commit is contained in:
bmc-msft 2021-03-09 10:06:06 -05:00 committed by GitHub
parent 18bf361d62
commit 0a3812d8bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 230 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
docs/webhook_events.md
View File

@ -345,7 +345,23 @@ Each event will be submitted via HTTP POST to the user provided URL.
"name": "example name",
"project": "example project"
},
"job_id": "00000000-0000-0000-0000-000000000000"
"job_id": "00000000-0000-0000-0000-000000000000",
"task_info": [
{
"error": {
"code": 468,
"errors": [
"example error message"
]
},
"task_id": "00000000-0000-0000-0000-000000000000",
"task_type": "libfuzzer_fuzz"
},
{
"task_id": "00000000-0000-0000-0000-000000000001",
"task_type": "libfuzzer_coverage"
}
]
}
```
@ -355,6 +371,54 @@ Each event will be submitted via HTTP POST to the user provided URL.
{
"additionalProperties": false,
"definitions": {
"Error": {
"properties": {
"code": {
"$ref": "#/definitions/ErrorCode"
},
"errors": {
"items": {
"type": "string"
},
"title": "Errors",
"type": "array"
}
},
"required": [
"code",
"errors"
],
"title": "Error",
"type": "object"
},
"ErrorCode": {
"description": "An enumeration.",
"enum": [
450,
451,
452,
453,
454,
455,
456,
457,
458,
459,
460,
461,
462,
463,
464,
465,
467,
468,
469,
470,
471,
472
],
"title": "ErrorCode"
},
"JobConfig": {
"properties": {
"build": {
@ -383,6 +447,42 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "JobConfig",
"type": "object"
},
"JobTaskStopped": {
"properties": {
"error": {
"$ref": "#/definitions/Error"
},
"task_id": {
"format": "uuid",
"title": "Task Id",
"type": "string"
},
"task_type": {
"$ref": "#/definitions/TaskType"
}
},
"required": [
"task_id",
"task_type"
],
"title": "JobTaskStopped",
"type": "object"
},
"TaskType": {
"description": "An enumeration.",
"enum": [
"libfuzzer_fuzz",
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
],
"title": "TaskType"
},
"UserInfo": {
"properties": {
"application_id": {
@ -413,6 +513,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Job Id",
"type": "string"
},
"task_info": {
"items": {
"$ref": "#/definitions/JobTaskStopped"
},
"title": "Task Info",
"type": "array"
},
"user_info": {
"$ref": "#/definitions/UserInfo"
}
@ -3488,6 +3595,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Job Id",
"type": "string"
},
"task_info": {
"items": {
"$ref": "#/definitions/JobTaskStopped"
},
"title": "Task Info",
"type": "array"
},
"user_info": {
"$ref": "#/definitions/UserInfo"
}
@ -3994,6 +4108,27 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "JobConfig",
"type": "object"
},
"JobTaskStopped": {
"properties": {
"error": {
"$ref": "#/definitions/Error"
},
"task_id": {
"format": "uuid",
"title": "Task Id",
"type": "string"
},
"task_type": {
"$ref": "#/definitions/TaskType"
}
},
"required": [
"task_id",
"task_type"
],
"title": "JobTaskStopped",
"type": "object"
},
"NodeState": {
"description": "An enumeration.",
"enum": [

20
src/api-service/__app__/onefuzzlib/jobs.py
View File

@ -8,7 +8,7 @@ from datetime import datetime, timedelta
from typing import List, Optional, Tuple
from onefuzztypes.enums import ErrorCode, JobState, TaskState
from onefuzztypes.events import EventJobCreated, EventJobStopped
from onefuzztypes.events import EventJobCreated, EventJobStopped, JobTaskStopped
from onefuzztypes.models import Error
from onefuzztypes.models import Job as BASE_JOB
@ -88,20 +88,26 @@ class Job(BASE_JOB, ORMMixin):
def stopping(self) -> None:
self.state = JobState.stopping
logging.info(JOB_LOG_PREFIX + "stopping: %s", self.job_id)
not_stopped = [
task
for task in Task.search(query={"job_id": [self.job_id]})
if task.state != TaskState.stopped
]
tasks = Task.search(query={"job_id": [self.job_id]})
not_stopped = [task for task in tasks if task.state != TaskState.stopped]
if not_stopped:
for task in not_stopped:
task.mark_stopping()
else:
self.state = JobState.stopped
task_info = [
JobTaskStopped(
task_id=x.task_id, error=x.error, task_type=x.config.task.type
)
for x in tasks
]
send_event(
EventJobStopped(
job_id=self.job_id, config=self.config, user_info=self.user_info
job_id=self.job_id,
config=self.config,
user_info=self.user_info,
task_info=task_info,
)
)
self.save()

8
src/ci/onefuzztypes.sh
View File

@ -12,11 +12,11 @@ pip install -r requirements-dev.txt
python setup.py sdist bdist_wheel
pip install -r requirements-lint.txt
black ./onefuzztypes --check
flake8 ./onefuzztypes
black ./onefuzztypes ./extra --check
flake8 ./onefuzztypes ./extra
bandit -r ./onefuzztypes
isort --profile black ./onefuzztypes --check
mypy ./onefuzztypes --ignore-missing-imports
isort --profile black ./onefuzztypes ./extra --check
mypy ./onefuzztypes ./extra --ignore-missing-imports
pytest -v tests
cp dist/*.* ../../artifacts/sdk

114
src/pytypes/extra/generate-docs.py
View File

@ -3,55 +3,57 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
from typing import Optional
from typing import List, Optional
from uuid import UUID
from onefuzztypes.primitives import Region, Container
from onefuzztypes.enums import (
TaskType,
ContainerType,
ErrorCode,
OS,
Architecture,
ContainerType,
ErrorCode,
NodeState,
)
from onefuzztypes.models import (
TaskConfig,
TaskDetails,
TaskContainers,
TaskState,
Error,
UserInfo,
JobConfig,
Report,
BlobRef,
TaskType,
)
from onefuzztypes.events import (
Event,
EventPing,
EventCrashReported,
EventFileAdded,
EventTaskCreated,
EventTaskStopped,
EventTaskFailed,
EventProxyCreated,
EventProxyDeleted,
EventProxyFailed,
EventPoolCreated,
EventPoolDeleted,
EventScalesetCreated,
EventScalesetFailed,
EventScalesetDeleted,
EventJobCreated,
EventJobStopped,
EventTaskStateUpdated,
EventNodeStateUpdated,
EventNodeCreated,
EventNodeDeleted,
EventNodeHeartbeat,
EventNodeStateUpdated,
EventPing,
EventPoolCreated,
EventPoolDeleted,
EventProxyCreated,
EventProxyDeleted,
EventProxyFailed,
EventScalesetCreated,
EventScalesetDeleted,
EventScalesetFailed,
EventTaskCreated,
EventTaskFailed,
EventTaskHeartbeat,
get_event_type,
EventTaskStateUpdated,
EventTaskStopped,
EventType,
JobTaskStopped,
get_event_type,
)
from onefuzztypes.models import (
BlobRef,
Error,
JobConfig,
Report,
TaskConfig,
TaskContainers,
TaskDetails,
UserInfo,
)
from onefuzztypes.primitives import Container, PoolName, Region
from onefuzztypes.webhooks import WebhookMessage
EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
@ -68,7 +70,7 @@ def typed(depth: int, title: str, content: str, data_type: str) -> None:
print(f"{'#' * depth} {title}\n\n```{data_type}\n{content}\n```\n")
def main():
def main() -> None:
task_config = TaskConfig(
job_id=UUID(int=0),
task=TaskDetails(
@ -79,13 +81,13 @@ def main():
target_options=[],
),
containers=[
TaskContainers(name="my-setup", type=ContainerType.setup),
TaskContainers(name="my-inputs", type=ContainerType.inputs),
TaskContainers(name="my-crashes", type=ContainerType.crashes),
TaskContainers(name=Container("my-setup"), type=ContainerType.setup),
TaskContainers(name=Container("my-inputs"), type=ContainerType.inputs),
TaskContainers(name=Container("my-crashes"), type=ContainerType.crashes),
],
tags={},
)
examples = [
examples: List[Event] = [
EventPing(ping_id=UUID(int=0)),
EventTaskCreated(
job_id=UUID(int=0),
@ -131,12 +133,15 @@ def main():
error=Error(code=ErrorCode.PROXY_FAILED, errors=["example error message"]),
),
EventPoolCreated(
pool_name="example", os=OS.linux, arch=Architecture.x86_64, managed=True
pool_name=PoolName("example"),
os=OS.linux,
arch=Architecture.x86_64,
managed=True,
),
EventPoolDeleted(pool_name="example"),
EventPoolDeleted(pool_name=PoolName("example")),
EventScalesetCreated(
scaleset_id=UUID(int=0),
pool_name="example",
pool_name=PoolName("example"),
vm_sku="Standard_D2s_v3",
image="Canonical:UbuntuServer:18.04-LTS:latest",
region=Region("eastus"),
@ -144,12 +149,12 @@ def main():
),
EventScalesetFailed(
scaleset_id=UUID(int=0),
pool_name="example",
pool_name=PoolName("example"),
error=Error(
code=ErrorCode.UNABLE_TO_RESIZE, errors=["example error message"]
),
),
EventScalesetDeleted(scaleset_id=UUID(int=0), pool_name="example"),
EventScalesetDeleted(scaleset_id=UUID(int=0), pool_name=PoolName("example")),
EventJobCreated(
job_id=UUID(int=0),
config=JobConfig(
@ -167,11 +172,26 @@ def main():
build="build 1",
duration=24,
),
task_info=[
JobTaskStopped(
task_id=UUID(int=0),
task_type=TaskType.libfuzzer_fuzz,
error=Error(
code=ErrorCode.TASK_FAILED, errors=["example error message"]
),
),
JobTaskStopped(
task_id=UUID(int=1),
task_type=TaskType.libfuzzer_coverage,
),
],
),
EventNodeCreated(machine_id=UUID(int=0), pool_name="example"),
EventNodeDeleted(machine_id=UUID(int=0), pool_name="example"),
EventNodeCreated(machine_id=UUID(int=0), pool_name=PoolName("example")),
EventNodeDeleted(machine_id=UUID(int=0), pool_name=PoolName("example")),
EventNodeStateUpdated(
machine_id=UUID(int=0), pool_name="example", state=NodeState.setting_up
machine_id=UUID(int=0),
pool_name=PoolName("example"),
state=NodeState.setting_up,
),
EventCrashReported(
container=Container("container-name"),
@ -196,11 +216,12 @@ def main():
),
),
EventFileAdded(container=Container("container-name"), filename="example.txt"),
EventNodeHeartbeat(machine_id=UUID(int=0), pool_name="example"),
EventNodeHeartbeat(machine_id=UUID(int=0), pool_name=PoolName("example")),
EventTaskHeartbeat(task_id=UUID(int=0), job_id=UUID(int=0), config=task_config),
]
for event in Event.__args__:
# works around `mypy` not handling that Union has `__args__`
for event in getattr(Event, "__args__", []):
seen = False
for value in examples:
if isinstance(value, event):
@ -227,7 +248,8 @@ def main():
layer(
1,
"Webhook Events",
"This document describes the basic webhook event subscriptions available in OneFuzz",
"This document describes the basic webhook event subscriptions "
"available in OneFuzz",
)
layer(
2,

11
src/pytypes/onefuzztypes/events.py
View File

@ -5,12 +5,12 @@
from datetime import datetime
from enum import Enum
from typing import Optional, Union
from typing import List, Optional, Union
from uuid import UUID, uuid4
from pydantic import BaseModel, Extra, Field
from .enums import OS, Architecture, NodeState, TaskState
from .enums import OS, Architecture, NodeState, TaskState, TaskType
from .models import AutoScaleConfig, Error, JobConfig, Report, TaskConfig, UserInfo
from .primitives import Container, PoolName, Region
from .responses import BaseResponse
@ -42,10 +42,17 @@ class EventJobCreated(BaseEvent):
user_info: Optional[UserInfo]
class JobTaskStopped(BaseModel):
task_id: UUID
task_type: TaskType
error: Optional[Error]
class EventJobStopped(BaseEvent):
job_id: UUID
config: JobConfig
user_info: Optional[UserInfo]
task_info: Optional[List[JobTaskStopped]]
class EventTaskCreated(BaseEvent):