hack in burst calc, add listeners to mqtt

cdn.tf

@@ -282,6 +282,22 @@ resource "aws_cloudfront_distribution" "sondehub" {
     origin_id   = "S3-${local.domain_name}"
     origin_path = ""
   }
+  origin {
+    connection_attempts = 3
+    connection_timeout  = 10
+    domain_name         = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
+    origin_id           = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
+    custom_origin_config {
+      http_port                = 80
+      https_port               = 443
+      origin_keepalive_timeout = 5
+      origin_protocol_policy   = "http-only"
+      origin_read_timeout      = 30
+      origin_ssl_protocols     = [
+            "TLSv1.2",
+        ]
+    }
+  }
   default_cache_behavior {
     allowed_methods = ["GET", "HEAD", "OPTIONS"]
     cached_methods = [
@@ -306,6 +322,7 @@ resource "aws_cloudfront_distribution" "sondehub" {
     target_origin_id       = "S3-${local.domain_name}"
     viewer_protocol_policy = "redirect-to-https"
   }
+
   ordered_cache_behavior {
 
     allowed_methods = ["GET", "HEAD"]
@@ -328,6 +345,48 @@ resource "aws_cloudfront_distribution" "sondehub" {
     target_origin_id       = "card"
     viewer_protocol_policy = "redirect-to-https"
   }
+  ordered_cache_behavior {
+    allowed_methods = ["GET", "HEAD", "OPTIONS"]
+    cached_methods = [
+      "HEAD",
+      "GET"
+    ]
+    forwarded_values {
+      cookies {
+        forward = "none"
+      }
+      query_string = false
+    }
+    compress    = true
+    default_ttl = 0
+    max_ttl                = 0
+    min_ttl                = 0
+    path_pattern           = "calc/*"
+    smooth_streaming       = false
+    target_origin_id       = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
+    viewer_protocol_policy = "redirect-to-https"
+  }
+  ordered_cache_behavior {
+    allowed_methods = ["GET", "HEAD", "OPTIONS"]
+    cached_methods = [
+      "HEAD",
+      "GET"
+    ]
+    forwarded_values {
+      cookies {
+        forward = "none"
+      }
+      query_string = false
+    }
+    compress    = true
+    default_ttl = 0
+    max_ttl                = 0
+    min_ttl                = 0
+    path_pattern           = "calc"
+    smooth_streaming       = false
+    target_origin_id       = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
+    viewer_protocol_policy = "redirect-to-https"
+  }
   ordered_cache_behavior {
     allowed_methods = ["GET", "HEAD", "OPTIONS"]
     cached_methods = [

ham_ingestion.tf

@@ -272,6 +272,7 @@ resource "aws_lambda_function" "ham_listener_put" {
   environment {
     variables = {
       "ES" = "es.${local.domain_name}"
+      "SNS_TOPIC" = aws_sns_topic.ham_listener_telem.arn
     }
   }
   tags = {
@@ -302,4 +303,68 @@ resource "aws_apigatewayv2_integration" "ham_listener_put" {
   integration_uri        = aws_lambda_function.ham_listener_put.arn
   timeout_milliseconds   = 30000
   payload_format_version = "2.0"
+}
+
+
+resource "aws_lambda_function" "ham_sns_to_mqtt_listener" {
+  function_name    = "ham-sns-to-mqtt-listener"
+  handler          = "sns_to_mqtt.lambda_handler"
+  s3_bucket        = aws_s3_bucket_object.lambda.bucket
+  s3_key           = aws_s3_bucket_object.lambda.key
+  source_code_hash = data.archive_file.lambda.output_base64sha256
+  publish          = true
+  memory_size      = 128
+  role             = aws_iam_role.basic_lambda_role.arn
+  runtime          = "python3.9"
+  timeout          = 3
+  architectures    = ["arm64"]
+  lifecycle {
+    ignore_changes = [environment]
+  }
+  tags = {
+    Name = "sns-to-mqtt"
+  }
+
+  vpc_config {
+    security_group_ids = [
+      "sg-05f795128b295c504",
+    ]
+    subnet_ids = [
+      aws_subnet.private["us-east-1b"].id
+    ]
+  }
+
+}
+
+resource "aws_lambda_permission" "ham_sns_to_mqtt_listener" {
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.ham_sns_to_mqtt_listener.arn
+  principal     = "sns.amazonaws.com"
+  source_arn    = aws_sns_topic.ham_listener_telem.arn
+}
+
+resource "aws_sns_topic" "ham_listener_telem" {
+  name            = "ham-listener-telem"
+  delivery_policy = <<EOF
+{
+  "http": {
+    "defaultHealthyRetryPolicy": {
+      "minDelayTarget": 5,
+      "maxDelayTarget": 30,
+      "numRetries": 100,
+      "numMaxDelayRetries": 0,
+      "numNoDelayRetries": 3,
+      "numMinDelayRetries": 0,
+      "backoffFunction": "linear"
+    },
+    "disableSubscriptionOverrides": false
+  }
+}
+EOF
+}
+
+resource "aws_sns_topic_subscription" "ham_listener_telem" {
+  topic_arn = aws_sns_topic.ham_listener_telem.arn
+  protocol  = "lambda"
+  endpoint  = aws_lambda_function.ham_sns_to_mqtt_listener.arn
 }

ingestion.tf

@@ -32,6 +32,7 @@ resource "aws_lambda_function" "station" {
   environment {
     variables = {
       "ES" = "es.${local.domain_name}"
+      "SNS_TOPIC" = aws_sns_topic.listener_telem.arn
     }
   }
   tags = {
@@ -109,6 +110,25 @@ resource "aws_sns_topic" "sonde_telem" {
 EOF
 }
 
+resource "aws_sns_topic" "listener_telem" {
+  name            = "listener-telem"
+  delivery_policy = <<EOF
+{
+  "http": {
+    "defaultHealthyRetryPolicy": {
+      "minDelayTarget": 5,
+      "maxDelayTarget": 30,
+      "numRetries": 100,
+      "numMaxDelayRetries": 0,
+      "numNoDelayRetries": 3,
+      "numMinDelayRetries": 0,
+      "backoffFunction": "linear"
+    },
+    "disableSubscriptionOverrides": false
+  }
+}
+EOF
+}
 
 // SNS to MQTT
 
@@ -147,4 +167,48 @@ resource "aws_lambda_permission" "sns_to_mqtt" {
   function_name = aws_lambda_function.station.arn
   principal     = "apigateway.amazonaws.com"
   source_arn    = aws_sns_topic.sonde_telem.arn
+}
+
+
+resource "aws_lambda_function" "sns_to_mqtt_listener" {
+  function_name    = "sns-to-mqtt-listener"
+  handler          = "sns_to_mqtt.lambda_handler"
+  s3_bucket        = aws_s3_bucket_object.lambda.bucket
+  s3_key           = aws_s3_bucket_object.lambda.key
+  source_code_hash = data.archive_file.lambda.output_base64sha256
+  publish          = true
+  memory_size      = 128
+  role             = aws_iam_role.basic_lambda_role.arn
+  runtime          = "python3.9"
+  timeout          = 3
+  architectures    = ["arm64"]
+  lifecycle {
+    ignore_changes = [environment]
+  }
+  tags = {
+    Name = "sns-to-mqtt"
+  }
+
+  vpc_config {
+    security_group_ids = [
+      "sg-05f795128b295c504",
+    ]
+    subnet_ids = [
+      aws_subnet.private["us-east-1b"].id
+    ]
+  }
+
+}
+
+resource "aws_lambda_permission" "sns_to_mqtt_listener" {
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.sns_to_mqtt_listener.arn
+  principal     = "sns.amazonaws.com"
+  source_arn    = aws_sns_topic.listener_telem.arn
+}
+
+resource "aws_sns_topic_subscription" "sns_to_mqtt_listener" {
+  topic_arn = aws_sns_topic.listener_telem.arn
+  protocol  = "lambda"
+  endpoint  = aws_lambda_function.sns_to_mqtt_listener.arn
 }

lambda/ham_listener_put/__init__.py

@@ -3,11 +3,34 @@ import zlib
 import base64
 import datetime
 from email.utils import parsedate
+import os
+import base64
+import gzip
+from io import BytesIO
+import boto3
 
 CALLSIGN_BLOCK_LIST = ["CHANGEME_RDZTTGO"]
 
 import es
 
+# Setup SNS
+
+def set_connection_header(request, operation_name, **kwargs):
+    request.headers['Connection'] = 'keep-alive'
+
+sns = boto3.client("sns",region_name="us-east-1")
+sns.meta.events.register('request-created.sns', set_connection_header)
+
+def post(payload):
+    compressed = BytesIO()
+    with gzip.GzipFile(fileobj=compressed, mode='w') as f:
+        f.write(json.dumps(payload).encode('utf-8'))
+    payload = base64.b64encode(compressed.getvalue()).decode("utf-8")
+    sns.publish(
+                TopicArn=os.getenv("SNS_TOPIC"),
+                Message=payload
+    )
+
 def lambda_handler(event, context):
     if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
         event["body"] = base64.b64decode(event["body"])
@@ -54,6 +77,7 @@ def lambda_handler(event, context):
     index = datetime.datetime.utcnow().strftime("ham-listeners-%Y-%m")
     payload["ts"] = datetime.datetime.utcnow().isoformat()
 
+    post([payload])
     es.request(json.dumps(payload),f"{index}/_doc","POST")
 
     return {"statusCode": 200, "body": "^v^ telm logged"}

lambda/station_api_to_iot_core/__init__.py

@@ -3,11 +3,34 @@ import zlib
 import base64
 import datetime
 from email.utils import parsedate
+import os
+import base64
+import gzip
+from io import BytesIO
+import boto3
 
 CALLSIGN_BLOCK_LIST = ["CHANGEME_RDZTTGO"]
 
 import es
 
+# Setup SNS
+
+def set_connection_header(request, operation_name, **kwargs):
+    request.headers['Connection'] = 'keep-alive'
+
+sns = boto3.client("sns",region_name="us-east-1")
+sns.meta.events.register('request-created.sns', set_connection_header)
+
+def post(payload):
+    compressed = BytesIO()
+    with gzip.GzipFile(fileobj=compressed, mode='w') as f:
+        f.write(json.dumps(payload).encode('utf-8'))
+    payload = base64.b64encode(compressed.getvalue()).decode("utf-8")
+    sns.publish(
+                TopicArn=os.getenv("SNS_TOPIC"),
+                Message=payload
+    )
+
 def lambda_handler(event, context):
     if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
         event["body"] = base64.b64decode(event["body"])
@@ -54,6 +77,7 @@ def lambda_handler(event, context):
     index = datetime.datetime.utcnow().strftime("listeners-%Y-%m")
     payload["ts"] = datetime.datetime.utcnow().isoformat()
 
+    post([payload])
     es.request(json.dumps(payload),f"{index}/_doc","POST")
 
     return {"statusCode": 200, "body": "^v^ telm logged"}