diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index f3bf87d38..15357d7be 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -14,12 +14,11 @@ import tempfile import uuid import zipfile from datetime import datetime, timedelta -from data_migration import migrate from azure.common.client_factory import get_client_from_cli_profile from azure.common.credentials import get_cli_profile -from azure.cosmosdb.table.tableservice import TableService from azure.core.exceptions import ResourceExistsError +from azure.cosmosdb.table.tableservice import TableService from azure.graphrbac import GraphRbacManagementClient from azure.graphrbac.models import ( ApplicationCreateParameters, @@ -30,6 +29,10 @@ from azure.graphrbac.models import ( ResourceAccess, ServicePrincipalCreateParameters, ) +from azure.mgmt.applicationinsights import ApplicationInsightsManagementClient +from azure.mgmt.applicationinsights.models import ( + ApplicationInsightsComponentExportRequest, +) from azure.mgmt.eventgrid import EventGridManagementClient from azure.mgmt.eventgrid.models import ( EventSubscription, @@ -51,14 +54,14 @@ from azure.storage.blob import ( ) from azure.storage.queue import QueueServiceClient from msrest.serialization import TZ_UTC -from urllib3.util.retry import Retry +from data_migration import migrate from register_pool_application import ( add_application_password, authorize_application, - update_registration, get_application, register_application, + update_registration, ) USER_IMPERSONATION = "311a71cc-e848-46a1-bdf8-97ff7156d8e6" @@ -101,6 +104,7 @@ class Client: workbook_data, create_registration, migrations, + export_appinsights: bool, ): self.resource_group = resource_group self.arm_template = arm_template @@ -121,6 +125,7 @@ class Client: "authority": ONEFUZZ_CLI_AUTHORITY, } self.migrations = migrations + self.export_appinsights = export_appinsights if os.name == "nt": self.azcopy = os.path.join(self.tools, "win64", "azcopy.exe") @@ -274,7 +279,8 @@ class Client: if cli_app is None: logger.info( - "Could not find the default CLI application under the current subscription, creating a new one" + "Could not find the default CLI application under the current " + "subscription, creating a new one" ) app_info = register_application("onefuzz-cli", self.application_name) self.cli_config = { @@ -383,6 +389,70 @@ class Client: % json.dumps(result.as_dict(), indent=4, sort_keys=True), ) + def add_log_export(self): + if not self.export_appinsights: + logger.info("not exporting appinsights") + return + + container_name = "app-insights" + + logger.info("adding appinsight log export") + account_name = self.results["deploy"]["func-name"]["value"] + key = self.results["deploy"]["func-key"]["value"] + account_url = "https://%s.blob.core.windows.net" % account_name + client = BlobServiceClient(account_url, credential=key) + if container_name not in [x["name"] for x in client.list_containers()]: + client.create_container(container_name) + + expiry = datetime.utcnow() + timedelta(days=2 * 365) + + # NOTE: as this is a long-lived SAS url, it should not be logged and only + # used in the the later-on export_configurations.create() call + sas = generate_container_sas( + account_name, + container_name, + account_key=key, + permission=ContainerSasPermissions(write=True), + expiry=expiry, + ) + url = "%s/%s?%s" % (account_url, container_name, sas) + + record_types = ( + "Requests, Event, Exceptions, Metrics, PageViews, " + "PageViewPerformance, Rdd, PerformanceCounters, Availability" + ) + + req = ApplicationInsightsComponentExportRequest( + record_types=record_types, + destination_type="Blob", + is_enabled="true", + destination_address=url, + ) + + app_insight_client = get_client_from_cli_profile( + ApplicationInsightsManagementClient + ) + + to_delete = [] + for entry in app_insight_client.export_configurations.list( + self.resource_group, self.application_name + ): + if ( + entry.storage_name == account_name + and entry.container_name == container_name + ): + to_delete.append(entry.export_id) + + for export_id in to_delete: + logger.info("replacing existing export: %s", export_id) + app_insight_client.export_configurations.delete( + self.resource_group, self.application_name, export_id + ) + + app_insight_client.export_configurations.create( + self.resource_group, self.application_name, req + ) + def upload_tools(self): logger.info("uploading tools from %s", self.tools) account_name = self.results["deploy"]["func-name"]["value"] @@ -511,7 +581,8 @@ class Client: else "" ) logger.info( - "Update your CLI config via: onefuzz config --endpoint https://%s.azurewebsites.net --authority %s --client_id %s %s", + "Update your CLI config via: onefuzz config --endpoint " + "https://%s.azurewebsites.net --authority %s --client_id %s %s", self.application_name, self.cli_config["authority"], self.cli_config["client_id"], @@ -543,6 +614,7 @@ def main(): ("instance-specific-setup", Client.upload_instance_setup), ("third-party", Client.upload_third_party), ("api", Client.deploy_app), + ("export_appinsights", Client.add_log_export), ("update_registration", Client.update_registration), ] @@ -611,6 +683,11 @@ def main(): default=[], help="list of migration to apply to the azure table", ) + parser.add_argument( + "--export_appinsights", + action="store_true", + help="enable appinsight log export", + ) args = parser.parse_args() if shutil.which("func") is None: @@ -632,6 +709,7 @@ def main(): args.workbook_data, args.create_pool_registration, args.apply_migrations, + args.export_appinsights, ) if args.verbose: level = logging.DEBUG