ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-20 05:23:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Log redirection, service side (#1727)

Browse Source 
* Setting the service side of the log management
- a log is created or reused when e create a job
- when scheduling the task we send the log location to the agent
The expected log structure looks liek
{fuzzContainer}/logs/{job_id}/{task_id}/{machine_id}/1.log

* regenerate doces

* including job_id in the container name

* regenerating docs
removing bad doc file
This commit is contained in:
Cheick Keita
2022-03-29 11:47:20 -07:00
committed by GitHub
parent 424ffdb4b5
commit 7add51fd3a
7 changed files with 125 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
docs/webhook_events.md
View File

@ -147,7 +147,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -967,6 +968,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Duration",
"type": "integer"
},
"logs": {
"title": "Logs",
"type": "string"
},
"name": {
"title": "Name",
"type": "string"
@ -1110,7 +1115,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
},
@ -1126,6 +1132,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Duration",
"type": "integer"
},
"logs": {
"title": "Logs",
"type": "string"
},
"name": {
"title": "Name",
"type": "string"
@ -1722,7 +1732,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
}
@ -1908,7 +1919,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -2609,7 +2621,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
}
@ -2795,7 +2808,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -3250,7 +3264,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -3299,7 +3314,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
},
@ -3747,7 +3763,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -4169,7 +4186,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -4618,7 +4636,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -5197,7 +5216,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
@ -5258,7 +5278,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
},
@ -6024,6 +6045,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Duration",
"type": "integer"
},
"logs": {
"title": "Logs",
"type": "string"
},
"name": {
"title": "Name",
"type": "string"

29
src/api-service/__app__/jobs/__init__.py
View File

@ -4,10 +4,13 @@
# Licensed under the MIT License.
import azure.functions as func
from onefuzztypes.enums import ErrorCode, JobState
from onefuzztypes.enums import ContainerType, ErrorCode, JobState
from onefuzztypes.models import Error, JobConfig, JobTaskInfo
from onefuzztypes.primitives import Container
from onefuzztypes.requests import JobGet, JobSearch
from ..onefuzzlib.azure.containers import create_container
from ..onefuzzlib.azure.storage import StorageType
from ..onefuzzlib.endpoint_authorization import call_if_user
from ..onefuzzlib.jobs import Job
from ..onefuzzlib.request import not_ok, ok, parse_request
@ -52,6 +55,30 @@ def post(req: func.HttpRequest) -> func.HttpResponse:
job = Job(config=request, user_info=user_info)
job.save()
# create the job logs container
log_container_sas = create_container(
Container(f"logs-{job.job_id}"),
StorageType.corpus,
metadata={"container_type": ContainerType.logs.name},
)
if not log_container_sas:
return not_ok(
Error(
code=ErrorCode.UNABLE_TO_CREATE_CONTAINER,
errors=["unable to create logs container"],
),
context="logs",
)
sep_index = log_container_sas.find("?")
if sep_index > 0:
log_container = log_container_sas[:sep_index]
else:
log_container = log_container_sas
job.config.logs = log_container
job.save()
return ok(job)

34
src/api-service/__app__/onefuzzlib/azure/containers.py
View File

@ -109,11 +109,31 @@ def get_container_metadata(
return cast(Dict[str, str], result)
def create_container(
def add_container_sas_url(container_url: str) -> str:
parsed = urllib.parse.urlparse(container_url)
query = urllib.parse.parse_qs(parsed.query)
if "sig" in query:
return container_url
else:
account_name = parsed.netloc.split(".")[0]
account_key = get_storage_account_name_key_by_name(account_name)
sas_token = generate_container_sas(
account_name=account_name,
container_name=parsed.path.split("/")[1],
account_key=account_key,
permission=ContainerSasPermissions(
read=True, write=True, delete=True, list=True
),
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1),
)
return f"{container_url}?{sas_token}"
def get_or_create_container_client(
container: Container,
storage_type: StorageType,
metadata: Optional[Dict[str, str]],
) -> Optional[str]:
) -> Optional[ContainerClient]:
client = find_container(container, storage_type)
if client is None:
account = choose_account(storage_type)
@ -134,7 +154,17 @@ def create_container(
err,
)
return None
return client
def create_container(
container: Container,
storage_type: StorageType,
metadata: Optional[Dict[str, str]],
) -> Optional[str]:
client = get_or_create_container_client(container, storage_type, metadata)
if client is None:
return None
return get_container_sas_url_service(
client,
read=True,

23
src/api-service/__app__/onefuzzlib/tasks/config.py
View File

@ -7,13 +7,17 @@ import logging
import os
import pathlib
from typing import Dict, List, Optional
from uuid import UUID
from onefuzztypes.enums import Compare, ContainerPermission, ContainerType, TaskFeature
from onefuzztypes.models import TaskConfig, TaskDefinition, TaskUnitConfig
from onefuzztypes.models import Job, Task, TaskConfig, TaskDefinition, TaskUnitConfig
from onefuzztypes.primitives import Container
from ..azure.containers import blob_exists, container_exists, get_container_sas_url
from ..azure.containers import (
add_container_sas_url,
blob_exists,
container_exists,
get_container_sas_url,
)
from ..azure.creds import get_instance_id
from ..azure.queue import get_queue_sas
from ..azure.storage import StorageType
@ -255,18 +259,25 @@ def check_config(config: TaskConfig) -> None:
raise TaskConfigError(err)
def build_task_config(
job_id: UUID, task_id: UUID, task_config: TaskConfig
) -> TaskUnitConfig:
def build_task_config(job: Job, task: Task) -> TaskUnitConfig:
job_id = job.job_id
task_id = task.task_id
task_config = task.config
if task_config.task.type not in TASK_DEFINITIONS:
raise TaskConfigError("unsupported task type: %s" % task_config.task.type.name)
if job.config.logs is None:
raise TaskConfigError(
"Missing log container: job_id %s, task_id %s", job_id, task_id
)
definition = TASK_DEFINITIONS[task_config.task.type]
config = TaskUnitConfig(
job_id=job_id,
task_id=task_id,
logs=add_container_sas_url(job.config.logs),
task_type=task_config.task.type,
instance_telemetry_key=os.environ.get("APPINSIGHTS_INSTRUMENTATIONKEY"),
microsoft_telemetry_key=os.environ.get("ONEFUZZ_TELEMETRY"),

7
src/api-service/__app__/onefuzzlib/tasks/scheduler.py
View File

@ -14,6 +14,7 @@ from pydantic import BaseModel
from ..azure.containers import blob_exists, get_container_sas_url
from ..azure.storage import StorageType
from ..jobs import Job
from ..workers.pools import Pool
from .config import build_task_config, get_setup_container
from .main import Task
@ -116,7 +117,11 @@ def build_work_unit(task: Task) -> Optional[Tuple[BucketConfig, WorkUnit]]:
logging.info("scheduling task: %s", task.task_id)
task_config = build_task_config(task.job_id, task.task_id, task.config)
job = Job.get(task.job_id)
if not job:
raise Exception(f"invalid job_id {task.job_id} for task {task.task_id}")
task_config = build_task_config(job, task)
setup_container = get_setup_container(task.config)
setup_script = None

2
src/pytypes/onefuzztypes/enums.py
View File

@ -215,6 +215,7 @@ class ContainerType(Enum):
unique_inputs = "unique_inputs"
unique_reports = "unique_reports"
regression_reports = "regression_reports"
logs = "logs"
@classmethod
def reset_defaults(cls) -> List["ContainerType"]:
@ -266,6 +267,7 @@ class ErrorCode(Enum):
UNABLE_TO_UPDATE = 471
PROXY_FAILED = 472
INVALID_CONFIGURATION = 473
UNABLE_TO_CREATE_CONTAINER = 474
class HeartbeatType(Enum):

2
src/pytypes/onefuzztypes/models.py
View File

@ -120,6 +120,7 @@ class JobConfig(BaseModel):
name: str
build: str
duration: int = Field(ge=ONE_HOUR, le=SEVEN_DAYS)
logs: Optional[str]
class ReproConfig(BaseModel):
@ -336,6 +337,7 @@ class AgentConfig(BaseModel):
class TaskUnitConfig(BaseModel):
instance_id: UUID
logs: str
job_id: UUID
task_id: UUID
task_type: TaskType