Add application-insights debug cli (#281)

src/api-service/__app__/info/__init__.py

@@ -9,6 +9,7 @@ from onefuzztypes.responses import Info
 from ..onefuzzlib.azure.creds import (
     get_base_region,
     get_base_resource_group,
+    get_insights_appid,
     get_instance_id,
     get_subscription,
 )
@@ -24,5 +25,6 @@ def main(req: func.HttpRequest) -> func.HttpResponse:
             subscription=get_subscription(),
             versions=versions(),
             instance_id=get_instance_id(),
+            insights_appid=get_insights_appid(),
         )
     )

src/api-service/__app__/onefuzzlib/azure/creds.py

@@ -82,6 +82,11 @@ def get_subscription() -> Any:  # should be str
     return parse_resource_id(os.environ["ONEFUZZ_DATA_STORAGE"])["subscription"]
 
 
+@cached
+def get_insights_appid() -> str:
+    return os.environ["APPINSIGHTS_APPID"]
+
+
 @cached
 def get_fuzz_storage() -> str:
     return os.environ["ONEFUZZ_DATA_STORAGE"]

src/cli/onefuzz/debug.py

@@ -3,15 +3,19 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+import json
 import logging
 import os
 import shutil
 import subprocess  # nosec
 import tempfile
-from typing import Any, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urlparse
 from uuid import UUID
 
+from azure.applicationinsights import ApplicationInsightsDataClient
+from azure.applicationinsights.models import QueryBody
+from azure.common.client_factory import get_azure_cli_credentials
 from onefuzztypes.enums import ContainerType, TaskType
 from onefuzztypes.models import BlobRef, NodeAssignment, Report, Task
 from onefuzztypes.primitives import Directory
@@ -262,6 +266,94 @@ class DebugJob(Command):
             subprocess.check_output([azcopy, "sync", to_download[name], outdir])
 
 
+class DebugLog(Command):
+    def _convert(self, raw_data: Any) -> Dict[str, List[Dict[str, Any]]]:
+        results = {}
+        for table in raw_data.tables:
+            result = []
+            for row in table.rows:
+                converted = {
+                    table.columns[x].name: y
+                    for (x, y) in enumerate(row)
+                    if y not in [None, ""]
+                }
+                if "customDimensions" in converted:
+                    converted["customDimensions"] = json.loads(
+                        converted["customDimensions"]
+                    )
+                result.append(converted)
+            results[table.name] = result
+        return results
+
+    def query(
+        self, log_query: str, *, timespan: str = "PT24H", raw: bool = False
+    ) -> Any:
+        """
+        Perform an Application Insights query
+
+        Queries should be well formed Kusto Queries.
+        Ref https://docs.microsoft.com/en-us/azure/data-explorer/kql-quick-reference
+
+        :param str log_query: Query to send to Application Insights
+        :param str timespan: ISO 8601 duration format
+        :param bool raw: Do not simplify the data result
+        """
+        creds, _ = get_azure_cli_credentials(
+            resource="https://api.applicationinsights.io"
+        )
+        client = ApplicationInsightsDataClient(creds)
+
+        app_id = self.onefuzz.info.get().insights_appid
+        if app_id is None:
+            raise Exception("instance does not have an insights_appid")
+        raw_data = client.query.execute(
+            app_id, body=QueryBody(query=log_query, timespan=timespan)
+        )
+        if "error" in raw_data.additional_properties:
+            raise Exception(
+                "Error performing query: %s" % raw_data.additional_properties["error"]
+            )
+        if raw:
+            return raw_data
+        return self._convert(raw_data)
+
+    def keyword(
+        self,
+        value: str,
+        *,
+        timespan: str = "PT24H",
+        limit: Optional[int] = None,
+        raw: bool = False,
+    ) -> Any:
+        """
+        Perform an Application Insights keyword query akin to "Transaction Search"
+
+        :param str value: Keyword to query Application Insights
+        :param str timespan: ISO 8601 duration format
+        :param int limit: Limit the number of records returned
+        :param bool raw: Do not simplify the data result
+        """
+
+        # See https://docs.microsoft.com/en-us/azure/data-explorer/kql-quick-reference
+
+        components = ["union isfuzzy=true exceptions, traces, customEvents"]
+
+        value = value.strip()
+        keywords = ['* has "%s"' % (x.replace('"', '\\"')) for x in value.split(" ")]
+        if keywords:
+            components.append("where " + " and ".join(keywords))
+
+        components.append("order by timestamp desc")
+
+        if limit is not None:
+            components.append(f"take {limit}")
+
+        log_query = " | ".join(components)
+        self.logger.debug("query: %s", log_query)
+
+        return self.query(log_query, timespan=timespan)
+
+
 class DebugNotification(Command):
     """ Debug notification integrations """
 
@@ -366,3 +458,4 @@ class Debug(Command):
         self.job = DebugJob(onefuzz, logger)
         self.notification = DebugNotification(onefuzz, logger)
         self.task = DebugTask(onefuzz, logger)
+        self.logs = DebugLog(onefuzz, logger)

src/cli/requirements.txt

@@ -8,7 +8,10 @@ asciimatics~=1.11.0
 dataclasses~=0.6
 pydantic~=1.6.1 --no-binary=pydantic
 memoization~=0.3.1
+msrestazure==0.6.4
 azure-storage-blob~=12.3
+azure-applicationinsights==0.1.0
+adal~=1.2.5
 tenacity==6.2.0
 docstring_parser==0.7.3
 # onefuzztypes version is set during build

src/deployment/azuredeploy.json

@@ -161,6 +161,10 @@
                             "name": "APPINSIGHTS_INSTRUMENTATIONKEY",
                             "value": "[reference(resourceId('microsoft.insights/components/', parameters('name')), '2015-05-01').InstrumentationKey]"
                         },
+                        {
+                            "name": "APPINSIGHTS_APPID",
+                            "value": "[reference(resourceId('microsoft.insights/components/', parameters('name')), '2015-05-01').AppId]"
+                        },
                         {
                             "name": "ONEFUZZ_TELEMETRY",
                             "value": "[variables('telemetry')]"

src/pytypes/onefuzztypes/responses.py

@@ -37,6 +37,7 @@ class Info(BaseResponse):
     subscription: str
     versions: Dict[str, Version]
     instance_id: Optional[UUID]
+    insights_appid: Optional[str]
 
 
 class ContainerInfoBase(BaseResponse):