mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-21 21:54:26 +00:00
Added UserInfo Filter Logging Function (#661)
## Summary of the Pull Request _What is this about?_ Due to our GDPR privacy requirements, we decided that it would be best to completely purge personal identifiable information from our AppInsights telemetry and logging. Instead of just removing all of the logging statements with personal info, I created a filter function that logs telemetry after it's been run through a recursive scrubbing function. This PR includes this new scrubbing function. ## PR Checklist * [x] Applies to work item: #660 * [ ] CLA signed. If not, go over [here](https://cla.opensource.microsoft.com/microsoft/onefuzz) and sign the CLI. * [ ] Tests added/passed * [ ] Requires documentation to be updated * [x] I've discussed this with core contributors already. If not checked, I'm ready to accept this work might be rejected in favor of a different grand plan. Issue number where discussion took place: #xxx ## Info on Pull Request _What does this include?_ Includes changes to events.py in onefuzzlib. I've implemented functionality - log_event() - to recursively check Event structures for UserInfo before logging to AppInsights. ## Validation Steps Performed I run local tests using a script I created with test events. _How does someone test & validate?_ I can provide local testing script. If that is insufficient, I can write a unit test that will run against this code.
This commit is contained in:
@ -6,9 +6,11 @@
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from queue import Empty, Queue
|
from queue import Empty, Queue
|
||||||
from typing import Optional
|
from typing import List, Optional, Set
|
||||||
|
|
||||||
from onefuzztypes.events import Event, EventMessage, get_event_type
|
from onefuzztypes.events import Event, EventMessage, EventType, get_event_type
|
||||||
|
from onefuzztypes.models import UserInfo
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from .azure.creds import get_instance_id, get_instance_name
|
from .azure.creds import get_instance_id, get_instance_name
|
||||||
from .webhooks import Webhook
|
from .webhooks import Webhook
|
||||||
@ -33,9 +35,60 @@ def get_events() -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def log_event(event: Event, event_type: EventType) -> None:
|
||||||
|
scrubbed_event = filter_event(event, event_type)
|
||||||
|
logging.info("sending event: %s - %s", event_type, scrubbed_event)
|
||||||
|
|
||||||
|
|
||||||
|
def filter_event(event: Event, event_type: EventType) -> Event:
|
||||||
|
|
||||||
|
clone_event = event.copy(deep=True)
|
||||||
|
filter_event_recurs(clone_event)
|
||||||
|
|
||||||
|
return clone_event
|
||||||
|
|
||||||
|
|
||||||
|
def filter_event_recurs(clone_event: BaseModel, visited: Set[int] = set()) -> BaseModel:
|
||||||
|
|
||||||
|
if id(clone_event) in visited:
|
||||||
|
return clone_event
|
||||||
|
|
||||||
|
visited.add(id(clone_event))
|
||||||
|
|
||||||
|
for field in clone_event.__fields__:
|
||||||
|
field_data = getattr(clone_event, field)
|
||||||
|
|
||||||
|
if isinstance(field_data, UserInfo):
|
||||||
|
|
||||||
|
field_data = None
|
||||||
|
|
||||||
|
elif isinstance(field_data, List):
|
||||||
|
|
||||||
|
if len(field_data) > 0 and not isinstance(field_data[0], BaseModel):
|
||||||
|
continue
|
||||||
|
for data in field_data:
|
||||||
|
filter_event_recurs(data, visited)
|
||||||
|
|
||||||
|
elif isinstance(field_data, dict):
|
||||||
|
|
||||||
|
for key in field_data:
|
||||||
|
if not isinstance(field_data[key], BaseModel):
|
||||||
|
continue
|
||||||
|
filter_event_recurs(field_data[key], visited)
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
if isinstance(field_data, BaseModel):
|
||||||
|
filter_event_recurs(field_data, visited)
|
||||||
|
|
||||||
|
setattr(clone_event, field, field_data)
|
||||||
|
|
||||||
|
return clone_event
|
||||||
|
|
||||||
|
|
||||||
def send_event(event: Event) -> None:
|
def send_event(event: Event) -> None:
|
||||||
event_type = get_event_type(event)
|
event_type = get_event_type(event)
|
||||||
logging.info("sending event: %s - %s", event_type, event)
|
log_event(event, event_type)
|
||||||
event_message = EventMessage(
|
event_message = EventMessage(
|
||||||
event_type=event_type,
|
event_type=event_type,
|
||||||
event=event,
|
event=event,
|
||||||
|
76
src/api-service/tests/test_secret_filter.py
Normal file
76
src/api-service/tests/test_secret_filter.py
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
#
|
||||||
|
# Copyright (c) Microsoft Corporation.
|
||||||
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from onefuzztypes.enums import ContainerType, TaskType
|
||||||
|
from onefuzztypes.events import EventTaskCreated, get_event_type
|
||||||
|
from onefuzztypes.models import (
|
||||||
|
TaskConfig,
|
||||||
|
TaskContainers,
|
||||||
|
TaskDetails,
|
||||||
|
TaskPool,
|
||||||
|
UserInfo,
|
||||||
|
)
|
||||||
|
from onefuzztypes.primitives import Container, PoolName
|
||||||
|
|
||||||
|
from __app__.onefuzzlib.events import filter_event
|
||||||
|
|
||||||
|
|
||||||
|
class TestSecretFilter(unittest.TestCase):
|
||||||
|
def test_secret_filter(self) -> None:
|
||||||
|
job_id = uuid4()
|
||||||
|
task_id = uuid4()
|
||||||
|
application_id = uuid4()
|
||||||
|
object_id = uuid4()
|
||||||
|
upn = "testalias@contoso.com"
|
||||||
|
|
||||||
|
user_info = UserInfo(
|
||||||
|
application_id=application_id, object_id=object_id, upn=upn
|
||||||
|
)
|
||||||
|
|
||||||
|
task_config = TaskConfig(
|
||||||
|
job_id=job_id,
|
||||||
|
containers=[
|
||||||
|
TaskContainers(
|
||||||
|
type=ContainerType.inputs, name=Container("test-container")
|
||||||
|
)
|
||||||
|
],
|
||||||
|
tags={},
|
||||||
|
task=TaskDetails(
|
||||||
|
type=TaskType.libfuzzer_fuzz,
|
||||||
|
duration=12,
|
||||||
|
target_exe="fuzz.exe",
|
||||||
|
target_env={},
|
||||||
|
target_options=[],
|
||||||
|
),
|
||||||
|
pool=TaskPool(count=2, pool_name=PoolName("test-pool")),
|
||||||
|
)
|
||||||
|
|
||||||
|
test_event = EventTaskCreated(
|
||||||
|
job_id=job_id,
|
||||||
|
task_id=task_id,
|
||||||
|
config=task_config,
|
||||||
|
user_info=user_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
control_test_event = EventTaskCreated(
|
||||||
|
job_id=job_id,
|
||||||
|
task_id=task_id,
|
||||||
|
config=task_config,
|
||||||
|
user_info=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
test_event_type = get_event_type(test_event)
|
||||||
|
|
||||||
|
scrubbed_test_event = filter_event(test_event, test_event_type)
|
||||||
|
|
||||||
|
self.assertEqual(scrubbed_test_event, control_test_event)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
Reference in New Issue
Block a user