From 354798583ed45c8d3ef773a8f674fd744fe945d8 Mon Sep 17 00:00:00 2001
From: Michaela <git@michaela.lgbt>
Date: Sat, 9 Oct 2021 09:50:03 +1100
Subject: [PATCH] fix some listener bugs

---
 es.tf                                      |  65 +++++++++++++
 main.tf                                    | 106 ---------------------
 query/lambda_function.py                   | 102 +-------------------
 station-api-to-iot-core/lambda_function.py |  14 ++-
 4 files changed, 77 insertions(+), 210 deletions(-)

diff --git a/es.tf b/es.tf
index 73b6cc0..be247be 100644
--- a/es.tf
+++ b/es.tf
@@ -283,3 +283,68 @@ EOF
 resource "aws_iam_service_linked_role" "IAMServiceLinkedRole" {
   aws_service_name = "es.amazonaws.com"
 }
+
+
+resource "aws_iam_role_policy" "IAMPolicy" {
+  policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+               {
+            "Effect": "Allow",
+            "Action": "es:*",
+            "Resource": "arn:aws:es:us-east-1:${data.aws_caller_identity.current.account_id}:domain/sondes-v2"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "es:*",
+            "Resource": "arn:aws:es:us-east-1:${data.aws_caller_identity.current.account_id}:domain/sondes-v2/*"
+        }
+    ]
+}
+EOF
+  role   = aws_iam_role.auth_role.name
+}
+
+resource "aws_iam_role_policy" "IAMPolicy2" {
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "mobileanalytics:PutEvents",
+        "cognito-sync:*"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}
+EOF
+  role   = aws_iam_role.unauth_role.name
+}
+
+resource "aws_iam_role_policy" "IAMPolicy3" {
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "mobileanalytics:PutEvents",
+        "cognito-sync:*",
+        "cognito-identity:*"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}
+EOF
+  role   = aws_iam_role.auth_role.name
+}
\ No newline at end of file
diff --git a/main.tf b/main.tf
index bdd2b43..d43adc6 100644
--- a/main.tf
+++ b/main.tf
@@ -145,69 +145,6 @@ resource "aws_iam_policy" "IAMManagedPolicy4" {
 EOF
 }
 
-resource "aws_iam_role_policy" "IAMPolicy" {
-  policy = <<EOF
-{
-    "Version": "2012-10-17",
-    "Statement": [
-               {
-            "Effect": "Allow",
-            "Action": "es:*",
-            "Resource": "arn:aws:es:us-east-1:${data.aws_caller_identity.current.account_id}:domain/sondes-v2"
-        },
-        {
-            "Effect": "Allow",
-            "Action": "es:*",
-            "Resource": "arn:aws:es:us-east-1:${data.aws_caller_identity.current.account_id}:domain/sondes-v2/*"
-        }
-    ]
-}
-EOF
-  role   = aws_iam_role.IAMRole.name
-}
-
-resource "aws_iam_role_policy" "IAMPolicy2" {
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "mobileanalytics:PutEvents",
-        "cognito-sync:*"
-      ],
-      "Resource": [
-        "*"
-      ]
-    }
-  ]
-}
-EOF
-  role   = aws_iam_role.IAMRole2.name
-}
-
-resource "aws_iam_role_policy" "IAMPolicy3" {
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "mobileanalytics:PutEvents",
-        "cognito-sync:*",
-        "cognito-identity:*"
-      ],
-      "Resource": [
-        "*"
-      ]
-    }
-  ]
-}
-EOF
-  role   = aws_iam_role.IAMRole.name
-}
 
 resource "aws_iam_role_policy" "IAMPolicy4" {
   policy = <<EOF
@@ -409,24 +346,6 @@ resource "aws_lambda_function" "get_sondes" {
   }
 }
 
-resource "aws_lambda_function" "listeners" {
-  function_name    = "listeners"
-  handler          = "lambda_function.get_listeners"
-  filename         = "${path.module}/build/query.zip"
-  source_code_hash = data.archive_file.query.output_base64sha256
-  publish          = true
-  memory_size      = 256
-  role             = aws_iam_role.IAMRole5.arn
-  runtime          = "python3.9"
-  timeout          = 30
-  architectures    = ["arm64"]
-  environment {
-    variables = {
-      "ES" = "es.${local.domain_name}"
-    }
-  }
-}
-
 
 
 
@@ -564,13 +483,6 @@ resource "aws_lambda_permission" "get_sites" {
   source_arn    = "arn:aws:execute-api:us-east-1:${data.aws_caller_identity.current.account_id}:${aws_apigatewayv2_api.main.id}/*/*/sites"
 }
 
-resource "aws_lambda_permission" "listeners" {
-  action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.listeners.arn
-  principal     = "apigateway.amazonaws.com"
-  source_arn    = "arn:aws:execute-api:us-east-1:${data.aws_caller_identity.current.account_id}:${aws_apigatewayv2_api.main.id}/*/*/listeners"
-}
-
 
 resource "aws_lambda_permission" "predictions" {
   action        = "lambda:InvokeFunction"
@@ -666,14 +578,6 @@ resource "aws_apigatewayv2_route" "get_sites" {
   target             = "integrations/${aws_apigatewayv2_integration.get_sites.id}"
 }
 
-resource "aws_apigatewayv2_route" "listeners" {
-  api_id             = aws_apigatewayv2_api.main.id
-  api_key_required   = false
-  authorization_type = "NONE"
-  route_key          = "GET /listeners"
-  target             = "integrations/${aws_apigatewayv2_integration.listeners.id}"
-}
-
 
 
 resource "aws_apigatewayv2_route" "predictions" {
@@ -740,16 +644,6 @@ resource "aws_apigatewayv2_integration" "get_sites" {
   payload_format_version = "2.0"
 }
 
-resource "aws_apigatewayv2_integration" "listeners" {
-  api_id                 = aws_apigatewayv2_api.main.id
-  connection_type        = "INTERNET"
-  integration_method     = "POST"
-  integration_type       = "AWS_PROXY"
-  integration_uri        = aws_lambda_function.listeners.arn
-  timeout_milliseconds   = 30000
-  payload_format_version = "2.0"
-}
-
 
 resource "aws_apigatewayv2_integration" "predictions" {
   api_id                 = aws_apigatewayv2_api.main.id
diff --git a/query/lambda_function.py b/query/lambda_function.py
index d8b6b5c..80aec74 100644
--- a/query/lambda_function.py
+++ b/query/lambda_function.py
@@ -259,7 +259,7 @@ def get_listener_telemetry(event, context):
         )
 
     (duration, interval) = durations[duration_query]
-    if "uploader_callsign" in event["queryStringParameters"]:
+    if "queryStringParameters" in event and "uploader_callsign" in event["queryStringParameters"]:
         interval = 1
     lt = requested_time
     gte = requested_time - timedelta(0, duration)
@@ -303,6 +303,7 @@ def get_listener_telemetry(event, context):
             "bool": {
                 "filter": [
                     {"match_all": {}},
+                    {"exists": { "field": "uploader_position"}},
                     {
                         "range": {
                             "ts": {"gte": gte.isoformat(), "lt": lt.isoformat()}
@@ -395,105 +396,6 @@ def get_sites(event, context):
             
         }
 
-def get_listeners(event, context):
-
-    path = "listeners-*/_search"
-    payload = {
-        "timeout": "30s",
-        "aggs": {
-            "2": {
-                "terms": {
-                    "field": "uploader_callsign.keyword",
-                    "order": {"_key": "desc"},
-                    "size": 500,
-                },
-                "aggs": {
-                    "1": {
-                        "top_hits": {
-                            "_source": False,
-                            "size": 1,
-                            "docvalue_fields": [
-                                "uploader_position_elk",
-                                "uploader_alt",
-                                "uploader_antenna.keyword",
-                                "software_name.keyword",
-                                "software_version.keyword",
-                                "ts",
-                            ],
-                            "sort": [{"ts": {"order": "desc"}}],
-                        }
-                    }
-                },
-            }
-        },
-        "size": 0,
-        "query": {
-            "bool": {
-                "must": [],
-                "filter": [
-                    {"match_all": {}},
-                    {"exists": {"field": "uploader_position_elk"},},
-                    {"exists": {"field": "uploader_antenna.keyword"},},
-                    {"exists": {"field": "software_name.keyword"},},
-                    {"exists": {"field": "software_version.keyword"},},
-                    {"exists": {"field": "ts"},},
-                    {
-                        "range": {
-                            "ts": {
-                                "gte": "now-24h",
-                                "lte": "now+1m",
-                                "format": "strict_date_optional_time",
-                            }
-                        }
-                    },
-                    
-                ],
-                "should": [],
-                "must_not": [
-                    {"match_phrase": {"mobile": "true"}}, 
-                ],
-            }
-        },
-    }
-
-    results = es_request(payload, path, "GET")
-
-    output = [
-        {
-            "name": html.escape(listener["key"]),
-            "tdiff_hours": (
-                datetime.now(timezone.utc)
-                - datetime.fromisoformat(
-                    listener["1"]["hits"]["hits"][0]["fields"]["ts"][0].replace(
-                        "Z", "+00:00"
-                    )
-                )
-            ).seconds
-            / 60
-            / 60,
-            "lon": float(
-                listener["1"]["hits"]["hits"][0]["fields"]["uploader_position_elk"][0]
-                .replace(" ", "")
-                .split(",")[1]
-            ),
-            "lat": float(
-                listener["1"]["hits"]["hits"][0]["fields"]["uploader_position_elk"][0]
-                .replace(" ", "")
-                .split(",")[0]
-            ),
-            "alt": float(listener["1"]["hits"]["hits"][0]["fields"]["uploader_alt"][0]) if "uploader_alt" in listener["1"]["hits"]["hits"][0]["fields"] else 0,
-            "description": f"""\n
-                <font size=\"-2\"><BR>\n
-                    <B>Radio: {html.escape(listener["1"]["hits"]["hits"][0]["fields"]["software_name.keyword"][0])}-{html.escape(listener["1"]["hits"]["hits"][0]["fields"]["software_version.keyword"][0])}</B><BR>\n
-                    <B>Antenna: </B>{html.escape(listener["1"]["hits"]["hits"][0]["fields"]["uploader_antenna.keyword"][0])}<BR>\n
-                    <B>Last Contact: </B>{html.escape(listener["1"]["hits"]["hits"][0]["fields"]["ts"][0])} <BR>\n
-                </font>\n
-            """,
-        }
-        for listener in results["aggregations"]["2"]["buckets"]
-    ]
-    return json.dumps(output)
-
 
 def es_request(payload, path, method):
     # get aws creds
diff --git a/station-api-to-iot-core/lambda_function.py b/station-api-to-iot-core/lambda_function.py
index 62fc6d3..3760cd2 100644
--- a/station-api-to-iot-core/lambda_function.py
+++ b/station-api-to-iot-core/lambda_function.py
@@ -44,14 +44,20 @@ def lambda_handler(event, context):
         payload["user-agent"] = event["headers"]["user-agent"]
     if time_delta:
         payload["upload_time_delta"] = time_delta
-    (payload["uploader_alt"], payload["uploader_position_elk"]) = (
+
+    payload.pop("uploader_contact_email", None)
+
+    # clean up None reports
+
+    if "uploader_position" in payload and None in payload["uploader_position"]:
+        payload.pop("uploader_position", None)
+    
+    if "uploader_position" in payload:
+        (payload["uploader_alt"], payload["uploader_position_elk"]) = (
                 payload["uploader_position"][2],
                 f"{payload['uploader_position'][0]},{payload['uploader_position'][1]}",
             )
-    payload.pop("uploader_contact_email", None)
     index = datetime.datetime.utcnow().strftime("listeners-%Y-%m")
-    print(payload)
-    print(index)
     payload["ts"] = datetime.datetime.utcnow().isoformat()
 
     es_request(json.dumps(payload),f"{index}/_doc","POST")