Move the event grid topic creation and subscription to the deployment template (#1591)

* move the event grid subscription to the template

* change the name of the new subscription to prevent deleting the wrong subscription

* refactoring

* mypy fix

* format

* format

* remove old event grid before arm deployment

* fix deply

* attempt to fix check-pr issue

* fix interactive login in check-pr

* move the event grid subscription to the tempalte

* change the name of the new subscription to prevent deleting the wrong subscription

* refactoring

* mypy fix

* format

* format

* remove old event grid before arm deployment

* using resource Id

* fix type

* fix location

* revert changes in registration.py

* build fix attempt

* build fix

* revert ci changes

* remove file

* address comment

* address PR comments

* naming

* fix deplyment

src/deployment/azuredeploy.json

@@ -68,7 +68,8 @@
         "Virtual Machine Contributor": "9980e02c-c2be-4d73-94e8-173b1dc7cf3c",
         "Storage Blob Data Reader": "2a2b9908-6ea1-4ae2-8e65-a410df84e7d1",
         "Contributor": "b24988ac-6180-42a0-ab88-20f7382dd24c",  
-        "keyVaultName": "[concat('of-kv-', uniquestring(resourceGroup().id))]"
+        "keyVaultName": "[concat('of-kv-', uniquestring(resourceGroup().id))]",
+        "fuzz-blob-topic-name": "[concat('fuzz-blob-topic-', uniquestring(resourceGroup().id))]"
     },
     "functions": [
         {
@@ -881,6 +882,44 @@
                     }
                 ]
             }
+        },
+        {
+            "type": "Microsoft.EventGrid/systemTopics",
+            "apiVersion": "2021-12-01",
+            "name": "[variables('fuzz-blob-topic-name')]",
+            "location": "[resourceGroup().location]",
+            "properties": {
+                "source": "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]",
+                "topicType": "microsoft.storage.storageaccounts"
+            }
+        },
+        {
+            "type": "Microsoft.EventGrid/systemTopics/eventSubscriptions",
+            "apiVersion": "2021-12-01",
+            "name": "[concat(variables('fuzz-blob-topic-name'), '/onefuzz1_subscription')]",
+            "dependsOn": [
+                "[resourceId('Microsoft.EventGrid/systemTopics', variables('fuzz-blob-topic-name'))]"
+            ],
+            "properties": {
+                "destination": {
+                    "properties": {
+                        "resourceId": "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountNameFunc'))]",
+                        "queueName": "file-changes"
+                    },
+                    "endpointType": "StorageQueue"
+                },
+                "filter": {
+                    "includedEventTypes": [
+                        "Microsoft.Storage.BlobCreated",
+                        "Microsoft.Storage.BlobDeleted"
+                    ]
+                },
+                "eventDeliverySchema": "EventGridSchema",
+                "retryPolicy": {
+                    "maxDeliveryAttempts": 30,
+                    "eventTimeToLiveInMinutes": 1440
+                }
+            }
         }
     ],
     "outputs": {

src/deployment/deploy.py

@@ -20,6 +20,7 @@ from typing import Dict, List, Optional, Tuple, Union, cast
 from uuid import UUID
 
 from azure.common.credentials import get_cli_profile
+from azure.core.exceptions import ResourceNotFoundError
 from azure.cosmosdb.table.tableservice import TableService
 from azure.identity import AzureCliCredential
 from azure.mgmt.applicationinsights import ApplicationInsightsManagementClient
@@ -27,12 +28,6 @@ from azure.mgmt.applicationinsights.models import (
     ApplicationInsightsComponentExportRequest,
 )
 from azure.mgmt.eventgrid import EventGridManagementClient
-from azure.mgmt.eventgrid.models import (
-    EventSubscription,
-    EventSubscriptionFilter,
-    RetryPolicy,
-    StorageQueueEventSubscriptionDestination,
-)
 from azure.mgmt.resource import ResourceManagementClient, SubscriptionClient
 from azure.mgmt.resource.resources.models import (
     Deployment,
@@ -694,43 +689,59 @@ class Client:
             tenants.append(tenant)
         update_allowed_aad_tenants(config_client, tenants)
 
-    def create_eventgrid(self) -> None:
+    @staticmethod
-        logger.info("creating eventgrid subscription")
+    def event_subscription_exists(
-        src_resource_id = self.results["deploy"]["fuzz-storage"]["value"]
+        client: EventGridManagementClient, resource_id: str, subscription_name: str
-        dst_resource_id = self.results["deploy"]["func-storage"]["value"]
+    ) -> bool:
+        try:
+            client.event_subscriptions.get(resource_id, subscription_name)
+            return True
+        except ResourceNotFoundError:
+            return False
 
+    @staticmethod
+    def get_storage_account_id(
+        client: StorageManagementClient, resource_group: str, prefix: str
+    ) -> Optional[str]:
+        try:
+            storage_accounts = client.storage_accounts.list_by_resource_group(
+                resource_group
+            )
+            for storage_account in storage_accounts:
+                if storage_account.name.startswith(prefix):
+                    return str(storage_account.id)
+            return None
+        except ResourceNotFoundError:
+            return None
+
+    def remove_eventgrid(self) -> None:
         credential = AzureCliCredential()
-        client = StorageManagementClient(
+        storage_account_client = StorageManagementClient(
             credential, subscription_id=self.get_subscription_id()
         )
-        event_subscription_info = EventSubscription(
-            destination=StorageQueueEventSubscriptionDestination(
-                resource_id=dst_resource_id, queue_name="file-changes"
-            ),
-            filter=EventSubscriptionFilter(
-                included_event_types=[
-                    "Microsoft.Storage.BlobCreated",
-                    "Microsoft.Storage.BlobDeleted",
-                ]
-            ),
-            retry_policy=RetryPolicy(
-                max_delivery_attempts=30,
-                event_time_to_live_in_minutes=1440,
-            ),
-        )
 
-        client = EventGridManagementClient(
+        src_resource_id = Client.get_storage_account_id(
+            storage_account_client, self.resource_group, "fuzz"
+        )
+        if not src_resource_id:
+            return
+
+        event_grid_client = EventGridManagementClient(
             credential, subscription_id=self.get_subscription_id()
         )
-        result = client.event_subscriptions.begin_create_or_update(
+
-            src_resource_id, "onefuzz1", event_subscription_info
+        # Event subscription for version up to 5.1.0
-        ).result()
+        old_subscription_name = "onefuzz1"
-        if result.provisioning_state != "Succeeded":
+        old_subscription_exists = Client.event_subscription_exists(
-            raise Exception(
+            event_grid_client, src_resource_id, old_subscription_name
-                "eventgrid subscription failed: %s"
-                % json.dumps(result.as_dict(), indent=4, sort_keys=True),
         )
 
+        if old_subscription_exists:
+            logger.info("removing deprecated event subscription")
+            event_grid_client.event_subscriptions.begin_delete(
+                src_resource_id, old_subscription_name
+            ).wait()
+
     def add_instance_id(self) -> None:
         logger.info("setting instance_id log export")
 
@@ -1030,6 +1041,7 @@ def main() -> None:
     rbac_only_states = [
         ("check_region", Client.check_region),
         ("rbac", Client.setup_rbac),
+        ("eventgrid", Client.remove_eventgrid),
         ("arm", Client.deploy_template),
         ("assign_scaleset_identity_role", Client.assign_scaleset_identity_role),
         ("assign_user_access", Client.assign_user_access),
@@ -1038,7 +1050,6 @@ def main() -> None:
     full_deployment_states = rbac_only_states + [
         ("apply_migrations", Client.apply_migrations),
         ("set_instance_config", Client.set_instance_config),
-        ("eventgrid", Client.create_eventgrid),
         ("tools", Client.upload_tools),
         ("add_instance_id", Client.add_instance_id),
         ("instance-specific-setup", Client.upload_instance_setup),

src/utils/add-corpus-storage-accounts/add-corpus-storage-account.py

@@ -31,7 +31,7 @@ def get_base_event(
         resource_group, location
     ):
         if (
-            entry.name == "onefuzz1"
+            entry.name == "onefuzz1_subscription"
             and entry.type == "Microsoft.EventGrid/eventSubscriptions"
             and entry.event_delivery_schema == "EventGridSchema"
             and entry.destination.endpoint_type == "StorageQueue"