ExternalVendorCode/sondehub-infra
1
0
Fork 0
mirror of https://github.com/projecthorus/sondehub-infra.git synced 2025-02-20 17:12:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

hack in burst calc, add listeners to mqtt

Browse Source
This commit is contained in:
xss 2022-09-18 12:05:21 +10:00
parent fa651b2f2f
commit 422956c8e7
5 changed files with 236 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
cdn.tf
View File

@ -282,6 +282,22 @@ resource "aws_cloudfront_distribution" "sondehub" {
origin_id = "S3-${local.domain_name}"
origin_path = ""
}
origin {
connection_attempts = 3
connection_timeout = 10
domain_name = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
origin_id = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
custom_origin_config {
http_port = 80
https_port = 443
origin_keepalive_timeout = 5
origin_protocol_policy = "http-only"
origin_read_timeout = 30
origin_ssl_protocols = [
"TLSv1.2",
]
}
}
default_cache_behavior {
allowed_methods = ["GET", "HEAD", "OPTIONS"]
cached_methods = [
@ -306,6 +322,7 @@ resource "aws_cloudfront_distribution" "sondehub" {
target_origin_id = "S3-${local.domain_name}"
viewer_protocol_policy = "redirect-to-https"
}
ordered_cache_behavior {
allowed_methods = ["GET", "HEAD"]
@ -328,6 +345,48 @@ resource "aws_cloudfront_distribution" "sondehub" {
target_origin_id = "card"
viewer_protocol_policy = "redirect-to-https"
}
ordered_cache_behavior {
allowed_methods = ["GET", "HEAD", "OPTIONS"]
cached_methods = [
"HEAD",
"GET"
]
forwarded_values {
cookies {
forward = "none"
}
query_string = false
}
compress = true
default_ttl = 0
max_ttl = 0
min_ttl = 0
path_pattern = "calc/*"
smooth_streaming = false
target_origin_id = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
viewer_protocol_policy = "redirect-to-https"
}
ordered_cache_behavior {
allowed_methods = ["GET", "HEAD", "OPTIONS"]
cached_methods = [
"HEAD",
"GET"
]
forwarded_values {
cookies {
forward = "none"
}
query_string = false
}
compress = true
default_ttl = 0
max_ttl = 0
min_ttl = 0
path_pattern = "calc"
smooth_streaming = false
target_origin_id = "sondehub-burst-calc.s3-website-us-east-1.amazonaws.com"
viewer_protocol_policy = "redirect-to-https"
}
ordered_cache_behavior {
allowed_methods = ["GET", "HEAD", "OPTIONS"]
cached_methods = [

65
ham_ingestion.tf
View File

@ -272,6 +272,7 @@ resource "aws_lambda_function" "ham_listener_put" {
environment {
variables = {
"ES" = "es.${local.domain_name}"
"SNS_TOPIC" = aws_sns_topic.ham_listener_telem.arn
}
}
tags = {
@ -302,4 +303,68 @@ resource "aws_apigatewayv2_integration" "ham_listener_put" {
integration_uri = aws_lambda_function.ham_listener_put.arn
timeout_milliseconds = 30000
payload_format_version = "2.0"
}
resource "aws_lambda_function" "ham_sns_to_mqtt_listener" {
function_name = "ham-sns-to-mqtt-listener"
handler = "sns_to_mqtt.lambda_handler"
s3_bucket = aws_s3_bucket_object.lambda.bucket
s3_key = aws_s3_bucket_object.lambda.key
source_code_hash = data.archive_file.lambda.output_base64sha256
publish = true
memory_size = 128
role = aws_iam_role.basic_lambda_role.arn
runtime = "python3.9"
timeout = 3
architectures = ["arm64"]
lifecycle {
ignore_changes = [environment]
}
tags = {
Name = "sns-to-mqtt"
}
vpc_config {
security_group_ids = [
"sg-05f795128b295c504",
]
subnet_ids = [
aws_subnet.private["us-east-1b"].id
]
}
}
resource "aws_lambda_permission" "ham_sns_to_mqtt_listener" {
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.ham_sns_to_mqtt_listener.arn
principal = "sns.amazonaws.com"
source_arn = aws_sns_topic.ham_listener_telem.arn
}
resource "aws_sns_topic" "ham_listener_telem" {
name = "ham-listener-telem"
delivery_policy = <<EOF
{
"http": {
"defaultHealthyRetryPolicy": {
"minDelayTarget": 5,
"maxDelayTarget": 30,
"numRetries": 100,
"numMaxDelayRetries": 0,
"numNoDelayRetries": 3,
"numMinDelayRetries": 0,
"backoffFunction": "linear"
},
"disableSubscriptionOverrides": false
}
}
EOF
}
resource "aws_sns_topic_subscription" "ham_listener_telem" {
topic_arn = aws_sns_topic.ham_listener_telem.arn
protocol = "lambda"
endpoint = aws_lambda_function.ham_sns_to_mqtt_listener.arn
}

64
ingestion.tf
View File

@ -32,6 +32,7 @@ resource "aws_lambda_function" "station" {
environment {
variables = {
"ES" = "es.${local.domain_name}"
"SNS_TOPIC" = aws_sns_topic.listener_telem.arn
}
}
tags = {
@ -109,6 +110,25 @@ resource "aws_sns_topic" "sonde_telem" {
EOF
}
resource "aws_sns_topic" "listener_telem" {
name = "listener-telem"
delivery_policy = <<EOF
{
"http": {
"defaultHealthyRetryPolicy": {
"minDelayTarget": 5,
"maxDelayTarget": 30,
"numRetries": 100,
"numMaxDelayRetries": 0,
"numNoDelayRetries": 3,
"numMinDelayRetries": 0,
"backoffFunction": "linear"
},
"disableSubscriptionOverrides": false
}
}
EOF
}
// SNS to MQTT
@ -147,4 +167,48 @@ resource "aws_lambda_permission" "sns_to_mqtt" {
function_name = aws_lambda_function.station.arn
principal = "apigateway.amazonaws.com"
source_arn = aws_sns_topic.sonde_telem.arn
}
resource "aws_lambda_function" "sns_to_mqtt_listener" {
function_name = "sns-to-mqtt-listener"
handler = "sns_to_mqtt.lambda_handler"
s3_bucket = aws_s3_bucket_object.lambda.bucket
s3_key = aws_s3_bucket_object.lambda.key
source_code_hash = data.archive_file.lambda.output_base64sha256
publish = true
memory_size = 128
role = aws_iam_role.basic_lambda_role.arn
runtime = "python3.9"
timeout = 3
architectures = ["arm64"]
lifecycle {
ignore_changes = [environment]
}
tags = {
Name = "sns-to-mqtt"
}
vpc_config {
security_group_ids = [
"sg-05f795128b295c504",
]
subnet_ids = [
aws_subnet.private["us-east-1b"].id
]
}
}
resource "aws_lambda_permission" "sns_to_mqtt_listener" {
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.sns_to_mqtt_listener.arn
principal = "sns.amazonaws.com"
source_arn = aws_sns_topic.listener_telem.arn
}
resource "aws_sns_topic_subscription" "sns_to_mqtt_listener" {
topic_arn = aws_sns_topic.listener_telem.arn
protocol = "lambda"
endpoint = aws_lambda_function.sns_to_mqtt_listener.arn
}

24
lambda/ham_listener_put/__init__.py
View File

@ -3,11 +3,34 @@ import zlib
import base64
import datetime
from email.utils import parsedate
import os
import base64
import gzip
from io import BytesIO
import boto3
CALLSIGN_BLOCK_LIST = ["CHANGEME_RDZTTGO"]
import es
# Setup SNS
def set_connection_header(request, operation_name, **kwargs):
request.headers['Connection'] = 'keep-alive'
sns = boto3.client("sns",region_name="us-east-1")
sns.meta.events.register('request-created.sns', set_connection_header)
def post(payload):
compressed = BytesIO()
with gzip.GzipFile(fileobj=compressed, mode='w') as f:
f.write(json.dumps(payload).encode('utf-8'))
payload = base64.b64encode(compressed.getvalue()).decode("utf-8")
sns.publish(
TopicArn=os.getenv("SNS_TOPIC"),
Message=payload
)
def lambda_handler(event, context):
if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
event["body"] = base64.b64decode(event["body"])
@ -54,6 +77,7 @@ def lambda_handler(event, context):
index = datetime.datetime.utcnow().strftime("ham-listeners-%Y-%m")
payload["ts"] = datetime.datetime.utcnow().isoformat()
post([payload])
es.request(json.dumps(payload),f"{index}/_doc","POST")
return {"statusCode": 200, "body": "^v^ telm logged"}

24
lambda/station_api_to_iot_core/__init__.py
View File

@ -3,11 +3,34 @@ import zlib
import base64
import datetime
from email.utils import parsedate
import os
import base64
import gzip
from io import BytesIO
import boto3
CALLSIGN_BLOCK_LIST = ["CHANGEME_RDZTTGO"]
import es
# Setup SNS
def set_connection_header(request, operation_name, **kwargs):
request.headers['Connection'] = 'keep-alive'
sns = boto3.client("sns",region_name="us-east-1")
sns.meta.events.register('request-created.sns', set_connection_header)
def post(payload):
compressed = BytesIO()
with gzip.GzipFile(fileobj=compressed, mode='w') as f:
f.write(json.dumps(payload).encode('utf-8'))
payload = base64.b64encode(compressed.getvalue()).decode("utf-8")
sns.publish(
TopicArn=os.getenv("SNS_TOPIC"),
Message=payload
)
def lambda_handler(event, context):
if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
event["body"] = base64.b64decode(event["body"])
@ -54,6 +77,7 @@ def lambda_handler(event, context):
index = datetime.datetime.utcnow().strftime("listeners-%Y-%m")
payload["ts"] = datetime.datetime.utcnow().isoformat()
post([payload])
es.request(json.dumps(payload),f"{index}/_doc","POST")
return {"statusCode": 200, "body": "^v^ telm logged"}