Log redirection, service side (#1727)

* Setting the service side of the log management
- a log is created or reused when e create a job
- when scheduling the task we send the log location to the agent
The expected log structure looks liek
{fuzzContainer}/logs/{job_id}/{task_id}/{machine_id}/1.log

* regenerate doces

* including job_id in the container name

* regenerating docs
removing bad doc file

docs/webhook_events.md

@@ -147,7 +147,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -967,6 +968,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Duration",
                     "type": "integer"
                 },
+                "logs": {
+                    "title": "Logs",
+                    "type": "string"
+                },
                 "name": {
                     "title": "Name",
                     "type": "string"
@@ -1110,7 +1115,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 470,
                 471,
                 472,
-                473
+                473,
+                474
             ],
             "title": "ErrorCode"
         },
@@ -1126,6 +1132,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Duration",
                     "type": "integer"
                 },
+                "logs": {
+                    "title": "Logs",
+                    "type": "string"
+                },
                 "name": {
                     "title": "Name",
                     "type": "string"
@@ -1722,7 +1732,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 470,
                 471,
                 472,
-                473
+                473,
+                474
             ],
             "title": "ErrorCode"
         }
@@ -1908,7 +1919,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -2609,7 +2621,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 470,
                 471,
                 472,
-                473
+                473,
+                474
             ],
             "title": "ErrorCode"
         }
@@ -2795,7 +2808,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -3250,7 +3264,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -3299,7 +3314,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 470,
                 471,
                 472,
-                473
+                473,
+                474
             ],
             "title": "ErrorCode"
         },
@@ -3747,7 +3763,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -4169,7 +4186,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -4618,7 +4636,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -5197,7 +5216,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "tools",
                 "unique_inputs",
                 "unique_reports",
-                "regression_reports"
+                "regression_reports",
+                "logs"
             ],
             "title": "ContainerType"
         },
@@ -5258,7 +5278,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 470,
                 471,
                 472,
-                473
+                473,
+                474
             ],
             "title": "ErrorCode"
         },
@@ -6024,6 +6045,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Duration",
                     "type": "integer"
                 },
+                "logs": {
+                    "title": "Logs",
+                    "type": "string"
+                },
                 "name": {
                     "title": "Name",
                     "type": "string"

src/api-service/__app__/jobs/__init__.py

@@ -4,10 +4,13 @@
 # Licensed under the MIT License.
 
 import azure.functions as func
-from onefuzztypes.enums import ErrorCode, JobState
+from onefuzztypes.enums import ContainerType, ErrorCode, JobState
 from onefuzztypes.models import Error, JobConfig, JobTaskInfo
+from onefuzztypes.primitives import Container
 from onefuzztypes.requests import JobGet, JobSearch
 
+from ..onefuzzlib.azure.containers import create_container
+from ..onefuzzlib.azure.storage import StorageType
 from ..onefuzzlib.endpoint_authorization import call_if_user
 from ..onefuzzlib.jobs import Job
 from ..onefuzzlib.request import not_ok, ok, parse_request
@@ -52,6 +55,30 @@ def post(req: func.HttpRequest) -> func.HttpResponse:
 
     job = Job(config=request, user_info=user_info)
     job.save()
+
+    # create the job logs container
+    log_container_sas = create_container(
+        Container(f"logs-{job.job_id}"),
+        StorageType.corpus,
+        metadata={"container_type": ContainerType.logs.name},
+    )
+    if not log_container_sas:
+        return not_ok(
+            Error(
+                code=ErrorCode.UNABLE_TO_CREATE_CONTAINER,
+                errors=["unable to create logs container"],
+            ),
+            context="logs",
+        )
+    sep_index = log_container_sas.find("?")
+    if sep_index > 0:
+        log_container = log_container_sas[:sep_index]
+    else:
+        log_container = log_container_sas
+
+    job.config.logs = log_container
+    job.save()
+
     return ok(job)
 
 

src/api-service/__app__/onefuzzlib/azure/containers.py

@@ -109,11 +109,31 @@ def get_container_metadata(
     return cast(Dict[str, str], result)
 
 
-def create_container(
+def add_container_sas_url(container_url: str) -> str:
+    parsed = urllib.parse.urlparse(container_url)
+    query = urllib.parse.parse_qs(parsed.query)
+    if "sig" in query:
+        return container_url
+    else:
+        account_name = parsed.netloc.split(".")[0]
+        account_key = get_storage_account_name_key_by_name(account_name)
+        sas_token = generate_container_sas(
+            account_name=account_name,
+            container_name=parsed.path.split("/")[1],
+            account_key=account_key,
+            permission=ContainerSasPermissions(
+                read=True, write=True, delete=True, list=True
+            ),
+            expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1),
+        )
+        return f"{container_url}?{sas_token}"
+
+
+def get_or_create_container_client(
     container: Container,
     storage_type: StorageType,
     metadata: Optional[Dict[str, str]],
-) -> Optional[str]:
+) -> Optional[ContainerClient]:
     client = find_container(container, storage_type)
     if client is None:
         account = choose_account(storage_type)
@@ -134,7 +154,17 @@ def create_container(
                 err,
             )
             return None
+    return client
 
+
+def create_container(
+    container: Container,
+    storage_type: StorageType,
+    metadata: Optional[Dict[str, str]],
+) -> Optional[str]:
+    client = get_or_create_container_client(container, storage_type, metadata)
+    if client is None:
+        return None
     return get_container_sas_url_service(
         client,
         read=True,

src/api-service/__app__/onefuzzlib/tasks/config.py

@@ -7,13 +7,17 @@ import logging
 import os
 import pathlib
 from typing import Dict, List, Optional
-from uuid import UUID
 
 from onefuzztypes.enums import Compare, ContainerPermission, ContainerType, TaskFeature
-from onefuzztypes.models import TaskConfig, TaskDefinition, TaskUnitConfig
+from onefuzztypes.models import Job, Task, TaskConfig, TaskDefinition, TaskUnitConfig
 from onefuzztypes.primitives import Container
 
-from ..azure.containers import blob_exists, container_exists, get_container_sas_url
+from ..azure.containers import (
+    add_container_sas_url,
+    blob_exists,
+    container_exists,
+    get_container_sas_url,
+)
 from ..azure.creds import get_instance_id
 from ..azure.queue import get_queue_sas
 from ..azure.storage import StorageType
@@ -255,18 +259,25 @@ def check_config(config: TaskConfig) -> None:
             raise TaskConfigError(err)
 
 
-def build_task_config(
+def build_task_config(job: Job, task: Task) -> TaskUnitConfig:
-    job_id: UUID, task_id: UUID, task_config: TaskConfig
+    job_id = job.job_id
-) -> TaskUnitConfig:
+    task_id = task.task_id
+    task_config = task.config
 
     if task_config.task.type not in TASK_DEFINITIONS:
         raise TaskConfigError("unsupported task type: %s" % task_config.task.type.name)
 
+    if job.config.logs is None:
+        raise TaskConfigError(
+            "Missing log container:  job_id %s, task_id %s", job_id, task_id
+        )
+
     definition = TASK_DEFINITIONS[task_config.task.type]
 
     config = TaskUnitConfig(
         job_id=job_id,
         task_id=task_id,
+        logs=add_container_sas_url(job.config.logs),
         task_type=task_config.task.type,
         instance_telemetry_key=os.environ.get("APPINSIGHTS_INSTRUMENTATIONKEY"),
         microsoft_telemetry_key=os.environ.get("ONEFUZZ_TELEMETRY"),

src/api-service/__app__/onefuzzlib/tasks/scheduler.py

@@ -14,6 +14,7 @@ from pydantic import BaseModel
 
 from ..azure.containers import blob_exists, get_container_sas_url
 from ..azure.storage import StorageType
+from ..jobs import Job
 from ..workers.pools import Pool
 from .config import build_task_config, get_setup_container
 from .main import Task
@@ -116,7 +117,11 @@ def build_work_unit(task: Task) -> Optional[Tuple[BucketConfig, WorkUnit]]:
 
     logging.info("scheduling task: %s", task.task_id)
 
-    task_config = build_task_config(task.job_id, task.task_id, task.config)
+    job = Job.get(task.job_id)
+    if not job:
+        raise Exception(f"invalid job_id {task.job_id} for task {task.task_id}")
+
+    task_config = build_task_config(job, task)
 
     setup_container = get_setup_container(task.config)
     setup_script = None

src/pytypes/onefuzztypes/enums.py

@@ -215,6 +215,7 @@ class ContainerType(Enum):
     unique_inputs = "unique_inputs"
     unique_reports = "unique_reports"
     regression_reports = "regression_reports"
+    logs = "logs"
 
     @classmethod
     def reset_defaults(cls) -> List["ContainerType"]:
@@ -266,6 +267,7 @@ class ErrorCode(Enum):
     UNABLE_TO_UPDATE = 471
     PROXY_FAILED = 472
     INVALID_CONFIGURATION = 473
+    UNABLE_TO_CREATE_CONTAINER = 474
 
 
 class HeartbeatType(Enum):

src/pytypes/onefuzztypes/models.py

@@ -120,6 +120,7 @@ class JobConfig(BaseModel):
     name: str
     build: str
     duration: int = Field(ge=ONE_HOUR, le=SEVEN_DAYS)
+    logs: Optional[str]
 
 
 class ReproConfig(BaseModel):
@@ -336,6 +337,7 @@ class AgentConfig(BaseModel):
 
 class TaskUnitConfig(BaseModel):
     instance_id: UUID
+    logs: str
     job_id: UUID
     task_id: UUID
     task_type: TaskType