From 5be9c4dcee81ce3e29cc284235b53e4586625b70 Mon Sep 17 00:00:00 2001
From: bmc-msft <41130664+bmc-msft@users.noreply.github.com>
Date: Thu, 22 Jul 2021 14:10:20 -0400
Subject: [PATCH] relay SignalR integrations through a storage queue (#1100)

The SignalR integration from Azure Functions does not have automatic retry.  When the SignalR instance has issues, all other APIs fail.

To make the service resilient to SignalR outages, this bounces SignalR events through an Azure Storage queue.

NOTE: This PR does not remove the integration from all of the functions.  That is intended to be done as a follow-on PR.
---
 src/api-service/__app__/onefuzzlib/events.py  | 33 ++++++++-----------
 .../__app__/queue_signalr_events/__init__.py  | 12 +++++++
 .../queue_signalr_events/function.json        | 18 ++++++++++
 src/deployment/deploy.py                      |  1 +
 4 files changed, 45 insertions(+), 19 deletions(-)
 create mode 100644 src/api-service/__app__/queue_signalr_events/__init__.py
 create mode 100644 src/api-service/__app__/queue_signalr_events/function.json

diff --git a/src/api-service/__app__/onefuzzlib/events.py b/src/api-service/__app__/onefuzzlib/events.py
index 5a4da6e83..61ebbf402 100644
--- a/src/api-service/__app__/onefuzzlib/events.py
+++ b/src/api-service/__app__/onefuzzlib/events.py
@@ -3,36 +3,31 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-import json
 import logging
-from queue import Empty, Queue
-from typing import Optional
+from typing import List, Optional
 
 from onefuzztypes.events import Event, EventMessage, EventType, get_event_type
 from onefuzztypes.models import UserInfo
 from pydantic import BaseModel
 
 from .azure.creds import get_instance_id, get_instance_name
+from .azure.queue import send_message
+from .azure.storage import StorageType
 from .webhooks import Webhook
 
-EVENTS: Queue = Queue()
+
+class SignalREvent(BaseModel):
+    target: str
+    arguments: List[EventMessage]
+
+
+def queue_signalr_event(event_message: EventMessage) -> None:
+    message = SignalREvent(target="events", arguments=[event_message]).json().encode()
+    send_message("signalr-events", message, StorageType.config)
 
 
 def get_events() -> Optional[str]:
-    events = []
-
-    for _ in range(5):
-        try:
-            event = EVENTS.get(block=False)
-            events.append(json.loads(event))
-            EVENTS.task_done()
-        except Empty:
-            break
-
-    if events:
-        return json.dumps({"target": "events", "arguments": events})
-    else:
-        return None
+    return None
 
 
 def log_event(event: Event, event_type: EventType) -> None:
@@ -85,6 +80,6 @@ def send_event(event: Event) -> None:
     if event_message.event != event:
         event_message.event = event.copy(deep=True)
 
-    EVENTS.put(event_message.json())
+    queue_signalr_event(event_message)
     Webhook.send_event(event_message)
     log_event(event, event_type)
diff --git a/src/api-service/__app__/queue_signalr_events/__init__.py b/src/api-service/__app__/queue_signalr_events/__init__.py
new file mode 100644
index 000000000..bfac602a2
--- /dev/null
+++ b/src/api-service/__app__/queue_signalr_events/__init__.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+
+import azure.functions as func
+
+
+def main(msg: func.QueueMessage, dashboard: func.Out[str]) -> None:
+    body = msg.get_body().decode()
+    dashboard.set(body)
diff --git a/src/api-service/__app__/queue_signalr_events/function.json b/src/api-service/__app__/queue_signalr_events/function.json
new file mode 100644
index 000000000..45057d5da
--- /dev/null
+++ b/src/api-service/__app__/queue_signalr_events/function.json
@@ -0,0 +1,18 @@
+{
+  "scriptFile": "__init__.py",
+  "bindings": [
+    {
+      "name": "msg",
+      "type": "queueTrigger",
+      "direction": "in",
+      "queueName": "signalr-events",
+      "connection": "AzureWebJobsStorage"
+    },
+    {
+      "type": "signalR",
+      "direction": "out",
+      "name": "dashboard",
+      "hubName": "dashboard"
+    }
+  ]
+}
diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py
index d2a41c0b6..8ea932487 100644
--- a/src/deployment/deploy.py
+++ b/src/deployment/deploy.py
@@ -584,6 +584,7 @@ class Client:
             "proxy",
             "update-queue",
             "webhooks",
+            "signalr-events",
         ]:
             try:
                 client.create_queue(queue)