Added UserInfo Filter Logging Function (#661)

## Summary of the Pull Request

_What is this about?_
Due to our GDPR privacy requirements, we decided that it would be best to completely purge personal identifiable information from our AppInsights telemetry and logging. Instead of just removing all of the logging statements with personal info, I created a filter function that logs telemetry after it's been run through a recursive scrubbing function. This PR includes this new scrubbing function. 

## PR Checklist
* [x] Applies to work item: #660
* [ ] CLA signed. If not, go over [here](https://cla.opensource.microsoft.com/microsoft/onefuzz) and sign the CLI.
* [ ] Tests added/passed
* [ ] Requires documentation to be updated
* [x] I've discussed this with core contributors already. If not checked, I'm ready to accept this work might be rejected in favor of a different grand plan. Issue number where discussion took place: #xxx

## Info on Pull Request

_What does this include?_
Includes changes to events.py in onefuzzlib. I've implemented functionality - log_event() - to recursively check Event structures for UserInfo before logging to AppInsights. 

## Validation Steps Performed
I run local tests using a script I created with test events. 

_How does someone test & validate?_
I can provide local testing script. If that is insufficient, I can write a unit test that will run against this code.
This commit is contained in:
nharper285
2021-03-15 16:56:00 -07:00
committed by GitHub
parent 09e4afcbce
commit 7ebdeac537
2 changed files with 132 additions and 3 deletions

View File

@ -6,9 +6,11 @@
import json import json
import logging import logging
from queue import Empty, Queue from queue import Empty, Queue
from typing import Optional from typing import List, Optional, Set
from onefuzztypes.events import Event, EventMessage, get_event_type from onefuzztypes.events import Event, EventMessage, EventType, get_event_type
from onefuzztypes.models import UserInfo
from pydantic import BaseModel
from .azure.creds import get_instance_id, get_instance_name from .azure.creds import get_instance_id, get_instance_name
from .webhooks import Webhook from .webhooks import Webhook
@ -33,9 +35,60 @@ def get_events() -> Optional[str]:
return None return None
def log_event(event: Event, event_type: EventType) -> None:
scrubbed_event = filter_event(event, event_type)
logging.info("sending event: %s - %s", event_type, scrubbed_event)
def filter_event(event: Event, event_type: EventType) -> Event:
clone_event = event.copy(deep=True)
filter_event_recurs(clone_event)
return clone_event
def filter_event_recurs(clone_event: BaseModel, visited: Set[int] = set()) -> BaseModel:
if id(clone_event) in visited:
return clone_event
visited.add(id(clone_event))
for field in clone_event.__fields__:
field_data = getattr(clone_event, field)
if isinstance(field_data, UserInfo):
field_data = None
elif isinstance(field_data, List):
if len(field_data) > 0 and not isinstance(field_data[0], BaseModel):
continue
for data in field_data:
filter_event_recurs(data, visited)
elif isinstance(field_data, dict):
for key in field_data:
if not isinstance(field_data[key], BaseModel):
continue
filter_event_recurs(field_data[key], visited)
else:
if isinstance(field_data, BaseModel):
filter_event_recurs(field_data, visited)
setattr(clone_event, field, field_data)
return clone_event
def send_event(event: Event) -> None: def send_event(event: Event) -> None:
event_type = get_event_type(event) event_type = get_event_type(event)
logging.info("sending event: %s - %s", event_type, event) log_event(event, event_type)
event_message = EventMessage( event_message = EventMessage(
event_type=event_type, event_type=event_type,
event=event, event=event,

View File

@ -0,0 +1,76 @@
#!/usr/bin/env python
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import unittest
from uuid import uuid4
from onefuzztypes.enums import ContainerType, TaskType
from onefuzztypes.events import EventTaskCreated, get_event_type
from onefuzztypes.models import (
TaskConfig,
TaskContainers,
TaskDetails,
TaskPool,
UserInfo,
)
from onefuzztypes.primitives import Container, PoolName
from __app__.onefuzzlib.events import filter_event
class TestSecretFilter(unittest.TestCase):
def test_secret_filter(self) -> None:
job_id = uuid4()
task_id = uuid4()
application_id = uuid4()
object_id = uuid4()
upn = "testalias@contoso.com"
user_info = UserInfo(
application_id=application_id, object_id=object_id, upn=upn
)
task_config = TaskConfig(
job_id=job_id,
containers=[
TaskContainers(
type=ContainerType.inputs, name=Container("test-container")
)
],
tags={},
task=TaskDetails(
type=TaskType.libfuzzer_fuzz,
duration=12,
target_exe="fuzz.exe",
target_env={},
target_options=[],
),
pool=TaskPool(count=2, pool_name=PoolName("test-pool")),
)
test_event = EventTaskCreated(
job_id=job_id,
task_id=task_id,
config=task_config,
user_info=user_info,
)
control_test_event = EventTaskCreated(
job_id=job_id,
task_id=task_id,
config=task_config,
user_info=None,
)
test_event_type = get_event_type(test_event)
scrubbed_test_event = filter_event(test_event, test_event_type)
self.assertEqual(scrubbed_test_event, control_test_event)
if __name__ == "__main__":
unittest.main()