From 81830c2d741f1c65893dd8bdee6c761ef364ef75 Mon Sep 17 00:00:00 2001 From: Michaela Wheeler Date: Sun, 22 Oct 2023 14:40:16 +1100 Subject: [PATCH] Move lambda functions to secrets manager (#126) Co-authored-by: xss --- .terraform.lock.hcl | 77 +++++--- ham_aprs.tf | 17 +- ham_ingestion.tf | 27 ++- ham_predictor.tf | 14 +- ingestion.tf | 23 ++- lambda/config_handler/README.md | 16 ++ lambda/config_handler/__init__.py | 37 ++++ lambda/config_handler/__main__.py | 45 +++++ lambda/ham_listener_put/__init__.py | 3 +- lambda/ham_predict_updater/__init__.py | 7 +- lambda/ham_put_api/__init__.py | 4 +- lambda/predict_updater/__init__.py | 8 +- lambda/recovery_ingest/__init__.py | 4 +- lambda/sns_to_mqtt/__init__.py | 13 +- lambda/sonde_api_to_iot_core/__init__.py | 4 +- lambda/station_api_to_iot_core/__init__.py | 3 +- lambda/ttn_helium/__init__.py | 3 +- lb.tf | 2 +- main.tf | 8 +- predictor.tf | 25 ++- recovered.tf | 12 +- secrets.tf | 38 ++++ sqs_to_elk.tf | 7 + vpc.tf | 102 +++++++--- websockets.tf | 206 ++++----------------- 25 files changed, 426 insertions(+), 279 deletions(-) create mode 100644 lambda/config_handler/README.md create mode 100644 lambda/config_handler/__init__.py create mode 100644 lambda/config_handler/__main__.py create mode 100644 secrets.tf diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index cf40aac..04f121e 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,42 +2,61 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/archive" { - version = "2.3.0" + version = "2.4.0" hashes = [ - "h1:NaDbOqAcA9d8DiAS5/6+5smXwN3/+twJGb3QRiz6pNw=", - "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", - "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", + "h1:cJokkjeH1jfpG4QEHdRx0t2j8rr52H33A7C/oX73Ok4=", + "zh:18e408596dd53048f7fc8229098d0e3ad940b92036a24287eff63e2caec72594", + "zh:392d4216ecd1a1fd933d23f4486b642a8480f934c13e2cae3c13b6b6a7e34a7b", + "zh:655dd1fa5ca753a4ace21d0de3792d96fff429445717f2ce31c125d19c38f3ff", + "zh:70dae36c176aa2b258331ad366a471176417a94dd3b4985a911b8be9ff842b00", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", - "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", - "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", - "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", - "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", - "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", - "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", - "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", - "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", + "zh:7d8c8e3925f1e21daf73f85983894fbe8868e326910e6df3720265bc657b9c9c", + "zh:a032ec0f0aee27a789726e348e8ad20778c3a1c9190ef25e7cff602c8d175f44", + "zh:b8e50de62ba185745b0fe9713755079ad0e9f7ac8638d204de6762cc36870410", + "zh:c8ad0c7697a3d444df21ff97f3473a8604c8639be64afe3f31b8ec7ad7571e18", + "zh:df736c5a2a7c3a82c5493665f659437a22f0baf8c2d157e45f4dd7ca40e739fc", + "zh:e8ffbf578a0977074f6d08aa8734e36c726e53dc79894cfc4f25fadc4f45f1df", + "zh:efea57ff23b141551f92b2699024d356c7ffd1a4ad62931da7ed7a386aef7f1f", ] } provider "registry.terraform.io/hashicorp/aws" { - version = "4.60.0" + version = "5.22.0" hashes = [ - "h1:XxVhnhtrRW3YueabP668hVZ3qL4th7pcWbx+ot/l864=", - "zh:1853d6bc89e289ac36c13485e8ff877c1be8485e22f545bb32c7a30f1d1856e8", - "zh:4321d145969e3b7ede62fe51bee248a15fe398643f21df9541eef85526bf3641", - "zh:4c01189cc6963abfe724e6b289a7c06d2de9c395011d8d54efa8fe1aac444e2e", - "zh:5934db7baa2eec0f9acb9c7f1c3dd3b3fe1e67e23dd4a49e9fe327832967b32b", - "zh:5fbedf5d55c6e04e34c32b744151e514a80308e7dec633a56b852829b41e4b5a", - "zh:651558e1446cc05061b75e6f5cc6e2959feb17615cd0ace6ec7a2bcc846321c0", - "zh:76875eb697916475e554af080f9d4d3cd1f7d5d58ecdd3317a844a30980f4eec", + "h1:XuU3tsGzElMt4Ti8SsM05pFllNMwSC4ScUxcfsOS140=", + "zh:09b8475cd519c945423b1e1183b71a4209dd2927e0d289a88c5abeecb53c1753", + "zh:2448e0c3ce9b991a5dd70f6a42d842366a6a2460cf63b31fb9bc5d2cc92ced19", + "zh:3b9fc2bf6714a9a9ab25eae3e56cead3d3917bc1b6d8b9fb3111c4198a790c72", + "zh:4fbd28ad5380529a36c54d7a96c9768df1288c625d28b8fa3a50d4fc2176ef0f", + "zh:54d550f190702a7edc2d459952d025e259a8c0b0ff7df3f15bbcc148539214bf", + "zh:638f406d084ac96f3a0b0a5ce8aa71a5a2a781a56ba96e3a235d3982b89eef0d", + "zh:69d4c175b13b6916b5c9398172cc384e7af46cb737b45870ab9907f12e82a28a", + "zh:81edec181a67255d25caf5e7ffe6d5e8f9373849b9e8f5e0705f277640abb18e", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a52528e6d6c945a6ac45b89e9a70a5435148e4c151241e04c231dd2acc4a8c80", - "zh:af5f94c69025f1c2466a3cf970d1e9bed72938ec33b976c8c067468b6707bb57", - "zh:b6692fad956c9d4ef4266519d9ac2ee9f699f8f2c21627625c9ed63814d41590", - "zh:b74311af5fa5ac6e4eb159c12cfb380dfe2f5cd8685da2eac8073475f398ae60", - "zh:cc5aa6f738baa42edacba5ef1ca0969e5a959422e4491607255f3f6142ba90ed", - "zh:dd1a7ff1b22f0036a76bc905a8229ce7ed0a7eb5a783d3a2586fb1bd920515c3", - "zh:e5ab40c4ad0f1c7bd4d5d834d1aa144e690d1a93329d73b3d37512715a638de9", + "zh:a66efb2b3cf7be8116728ae5782d7550f23f3719da2ed3c10228d29c44b7dc84", + "zh:ae754478d0bfa42195d16cf46091fab7c1c075ebc965d919338e36aed45add78", + "zh:e0603ad0061c43aa1cb52740b1e700b8afb55667d7ee01c1cc1ceb6f983d4c9d", + "zh:e4cb701d0185884eed0492a66eff17251f5b4971d30e81acd5e0a55627059fc8", + "zh:f7db2fcf69679925dde1ae326526242fd61ba1f83f614b1f6d9d68c925417e51", + "zh:fef331b9b62bc26d900ae937cc662281ff30794edf48aebfe8997d0e16835f6d", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.5.1" + hashes = [ + "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", ] } diff --git a/ham_aprs.tf b/ham_aprs.tf index a2dfecc..301971a 100644 --- a/ham_aprs.tf +++ b/ham_aprs.tf @@ -83,6 +83,13 @@ resource "aws_iam_role_policy" "aprsgw" { "Effect": "Allow", "Action": "sns:Publish", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] } @@ -90,8 +97,14 @@ EOF } resource "aws_ecs_cluster" "aprsgw" { - name = "aprsgw" - capacity_providers = ["FARGATE", "FARGATE_SPOT"] + name = "aprsgw" +} + + +resource "aws_ecs_cluster_capacity_providers" "aprsgw" { + cluster_name = aws_ecs_cluster.aprsgw.name + + capacity_providers = ["FARGATE"] } diff --git a/ham_ingestion.tf b/ham_ingestion.tf index a2f7028..0397043 100644 --- a/ham_ingestion.tf +++ b/ham_ingestion.tf @@ -69,6 +69,13 @@ resource "aws_iam_role_policy" "ham_sqs_to_elk" { "Effect": "Allow", "Action": "sqs:*", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] } @@ -226,9 +233,15 @@ resource "aws_lambda_function" "ham_sns_to_mqtt" { runtime = "python3.9" timeout = 3 architectures = ["arm64"] - lifecycle { - ignore_changes = [environment] + + environment { + variables = { + MQTT_BATCH = "batch-amateur" + MQTT_ID = "payload_callsign" + MQTT_PREFIX = "amateur" + } } + tags = { Name = "sns-to-mqtt" } @@ -318,9 +331,15 @@ resource "aws_lambda_function" "ham_sns_to_mqtt_listener" { runtime = "python3.9" timeout = 3 architectures = ["arm64"] - lifecycle { - ignore_changes = [environment] + + environment { + variables = { + MQTT_BATCH = "batch-amateur-listener" + MQTT_ID = "uploader_callsign" + MQTT_PREFIX = "amateur-listener" + } } + tags = { Name = "sns-to-mqtt" } diff --git a/ham_predictor.tf b/ham_predictor.tf index f1715ed..f0767d3 100644 --- a/ham_predictor.tf +++ b/ham_predictor.tf @@ -53,6 +53,13 @@ resource "aws_iam_role_policy" "ham_predict_updater" { "Effect": "Allow", "Action": "s3:*", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] } @@ -75,15 +82,14 @@ resource "aws_lambda_function" "ham_predict_updater" { reserved_concurrent_executions = 1 environment { variables = { - "ES" = aws_route53_record.es.fqdn + "ES" = aws_route53_record.es.fqdn + MQTT_HOST = "ws.v2.sondehub.org" # We go via the internet as this function isn't in a VPC + MQTT_PORT = "443" } } tags = { Name = "ham_predict_updater" } - lifecycle { - ignore_changes = [environment] - } } diff --git a/ingestion.tf b/ingestion.tf index 95d7abb..570c7e9 100644 --- a/ingestion.tf +++ b/ingestion.tf @@ -79,6 +79,13 @@ resource "aws_iam_role_policy" "ingestion_lambda_role" { ], "Effect": "Allow", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] @@ -241,8 +248,12 @@ resource "aws_lambda_function" "sns_to_mqtt" { runtime = "python3.9" timeout = 3 architectures = ["arm64"] - lifecycle { - ignore_changes = [environment] + environment { + variables = { + MQTT_BATCH = "batch" + MQTT_ID = "serial" + MQTT_PREFIX = "sondes" + } } tags = { Name = "sns-to-mqtt" @@ -279,8 +290,12 @@ resource "aws_lambda_function" "sns_to_mqtt_listener" { runtime = "python3.9" timeout = 3 architectures = ["arm64"] - lifecycle { - ignore_changes = [environment] + environment { + variables = { + MQTT_BATCH = "batch-listener" + MQTT_ID = "uploader_callsign" + MQTT_PREFIX = "listener" + } } tags = { Name = "sns-to-mqtt" diff --git a/lambda/config_handler/README.md b/lambda/config_handler/README.md new file mode 100644 index 0000000..4ab9ee6 --- /dev/null +++ b/lambda/config_handler/README.md @@ -0,0 +1,16 @@ +Helper class for getting config and secrets within SondeHub + + +## Example + +```python +import config_handler + +mqtt_password = config_handler.get("MQTT", "PASSWORD") +``` + +## Logic + +1. Checks environment variable for "{TOPIC}_{PARAMETER}" if it exists return that value +2. If that doesn't exist then we perform a `SecretsManager.Client.get_secret_value(SecretId={TOPIC})` +3. We then `json.loads()` this value and return the respective value. \ No newline at end of file diff --git a/lambda/config_handler/__init__.py b/lambda/config_handler/__init__.py new file mode 100644 index 0000000..10e985e --- /dev/null +++ b/lambda/config_handler/__init__.py @@ -0,0 +1,37 @@ +import os +import boto3 +import json + +def get(topic: str, parameter: str, default=None) -> str: + """ + Get's a configuration parameter. + + :param topic: The topic parameter (a logical grouping). When used with SecretsManager this is the secrets name + :param parameter: The parameter to look up. When used with SecretsManager this is the key in the json config + :param default: If the parameter isn't found return this value + :returns: The config or secret value as a string + :raises KeyError: raises a keyerror if the topic and parameter pair aren't found + """ + + # Try environment variables first + try: + return os.environ[f"{topic}_{parameter}"] + except KeyError: + pass + + # Try secrets manager + sm = boto3.client('secretsmanager') + try: + secret_data = json.loads(sm.get_secret_value(SecretId=topic)['SecretString']) + return secret_data[parameter] + except (KeyError, sm.exceptions.ResourceNotFoundException): + pass + except: + if default: + return default + raise + + if default: + return default + raise KeyError("Could not location a value for {topic} {parameter}") + \ No newline at end of file diff --git a/lambda/config_handler/__main__.py b/lambda/config_handler/__main__.py new file mode 100644 index 0000000..93b884b --- /dev/null +++ b/lambda/config_handler/__main__.py @@ -0,0 +1,45 @@ +import config_handler + +import unittest +from unittest.mock import MagicMock, call, patch + +# Mock AWS API calls +secret_call = { + 'ARN': 'arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3', + 'CreatedDate': 1523477145.713, + 'Name': 'MyTestDatabaseSecret', + 'SecretString': '{\n "PASSWORD":"test_password"\n}\n', + 'VersionId': 'EXAMPLE1-90ab-cdef-fedc-ba987SECRET1', + 'VersionStages': [ + 'AWSCURRENT', + ], + 'ResponseMetadata': { + '...': '...', + }, + } + +class TestConfigHandler(unittest.TestCase): + def test_env(self): + with patch.dict(config_handler.os.environ,{ "MQTT_PASSWORD": "test_password" }, clear=True): + return_value = config_handler.get("MQTT", "PASSWORD") + self.assertEqual(return_value, "test_password") + + @patch('botocore.client.BaseClient._make_api_call', return_value=secret_call) + def test_sm(self, MockApiCall): + with patch.dict(config_handler.os.environ,{}, clear=True): #ensure that local env variables don't influence the tests + return_value = config_handler.get("MQTT", "PASSWORD") + MockApiCall.assert_called() + self.assertEqual(return_value, "test_password") + + @patch('botocore.client.BaseClient._make_api_call', return_value=secret_call) + def test_not_found(self, MockApiCall): + with patch.dict(config_handler.os.environ,{}, clear=True): + self.assertRaises(KeyError, config_handler.get, "MQTT", "NOTPASSWORD") + + def test_default(self): + with patch.dict(config_handler.os.environ,{}, clear=True): + return_value = config_handler.get("MQTT", "PASSWORD", "test_password_abc") + self.assertEqual(return_value, "test_password_abc") + +if __name__ == '__main__': + unittest.main() diff --git a/lambda/ham_listener_put/__init__.py b/lambda/ham_listener_put/__init__.py index 75326ea..fb9c061 100644 --- a/lambda/ham_listener_put/__init__.py +++ b/lambda/ham_listener_put/__init__.py @@ -8,6 +8,7 @@ import base64 import gzip from io import BytesIO import boto3 +import config_handler CALLSIGN_BLOCK_LIST = ["CHANGEME_RDZTTGO"] @@ -27,7 +28,7 @@ def post(payload): f.write(json.dumps(payload).encode('utf-8')) payload = base64.b64encode(compressed.getvalue()).decode("utf-8") sns.publish( - TopicArn=os.getenv("SNS_TOPIC"), + TopicArn=config_handler.get("SNS","TOPIC"), Message=payload ) diff --git a/lambda/ham_predict_updater/__init__.py b/lambda/ham_predict_updater/__init__.py index 93db9d3..89b35ba 100644 --- a/lambda/ham_predict_updater/__init__.py +++ b/lambda/ham_predict_updater/__init__.py @@ -15,6 +15,7 @@ import functools import os import random import time +import config_handler TAWHIRI_SERVER = "tawhiri.v2.sondehub.org" @@ -64,9 +65,9 @@ def connect(): client.on_disconnect = on_disconnect client.on_publish = on_publish #client.tls_set() - client.username_pw_set(username=os.getenv("MQTT_USERNAME"), password=os.getenv("MQTT_PASSWORD")) - HOSTS = os.getenv("MQTT_HOST").split(",") - PORT = int(os.getenv("MQTT_PORT", default="8080")) + client.username_pw_set(config_handler.get("MQTT","USERNAME"), password=config_handler.get("MQTT","PASSWORD")) + HOSTS = config_handler.get("MQTT","HOST").split(",") + PORT = int(config_handler.get("MQTT","PORT", default="8080")) if PORT == 443: client.tls_set() HOST = random.choice(HOSTS) diff --git a/lambda/ham_put_api/__init__.py b/lambda/ham_put_api/__init__.py index 5f041e6..fad5f1b 100644 --- a/lambda/ham_put_api/__init__.py +++ b/lambda/ham_put_api/__init__.py @@ -5,7 +5,7 @@ import base64 import datetime from email.utils import parsedate import os - +import config_handler def set_connection_header(request, operation_name, **kwargs): request.headers['Connection'] = 'keep-alive' @@ -47,7 +47,7 @@ def telemetry_hide_filter(telemetry): def post(payload): sns.publish( - TopicArn=os.getenv("HAM_SNS_TOPIC"), + TopicArn=config_handler.get("HAM_SNS","TOPIC"), Message=json.dumps(payload) ) diff --git a/lambda/predict_updater/__init__.py b/lambda/predict_updater/__init__.py index 7d1a56d..7e2cf2f 100644 --- a/lambda/predict_updater/__init__.py +++ b/lambda/predict_updater/__init__.py @@ -14,7 +14,7 @@ import os import random import time import traceback - +import config_handler client = mqtt.Client(transport="websockets") connected_flag = False @@ -29,9 +29,9 @@ def connect(): client.on_disconnect = on_disconnect client.on_publish = on_publish #client.tls_set() - client.username_pw_set(username=os.getenv("MQTT_USERNAME"), password=os.getenv("MQTT_PASSWORD")) - HOSTS = os.getenv("MQTT_HOST").split(",") - PORT = int(os.getenv("MQTT_PORT", default="8080")) + client.username_pw_set(username=config_handler.get("MQTT","USERNAME"), password=config_handler.get("MQTT","PASSWORD")) + HOSTS = config_handler.get("MQTT","HOST").split(",") + PORT = int(config_handler.get("MQTT","PORT", default="8080")) if PORT == 443: client.tls_set() HOST = random.choice(HOSTS) diff --git a/lambda/recovery_ingest/__init__.py b/lambda/recovery_ingest/__init__.py index 03fab4d..3ade02a 100644 --- a/lambda/recovery_ingest/__init__.py +++ b/lambda/recovery_ingest/__init__.py @@ -1,9 +1,9 @@ from datetime import datetime import urllib.request import json -import os +import config_handler -apiKey = os.environ["radiosondy_apikey"] +apiKey = config_handler.get("RADIOSONDY","API_KEY") params = "?token={}&period=2".format(apiKey) url = "https://radiosondy.info/api/v1/sonde-logs{}".format(params) diff --git a/lambda/sns_to_mqtt/__init__.py b/lambda/sns_to_mqtt/__init__.py index 4240fa0..4ce626a 100644 --- a/lambda/sns_to_mqtt/__init__.py +++ b/lambda/sns_to_mqtt/__init__.py @@ -11,6 +11,7 @@ import boto3 import traceback import sys import uuid +import config_handler client = mqtt.Client(transport="websockets") @@ -69,8 +70,8 @@ def connect(): client.on_disconnect = on_disconnect client.on_publish = on_publish #client.tls_set() - client.username_pw_set(username=os.getenv("MQTT_USERNAME"), password=os.getenv("MQTT_PASSWORD")) - HOSTS = os.getenv("MQTT_HOST").split(",") + client.username_pw_set(username=config_handler.get("MQTT","USERNAME"), password=config_handler.get("MQTT","PASSWORD")) + HOSTS = config_handler.get("MQTT","HOST").split(",") HOST = random.choice(HOSTS) print(f"Connecting to {HOST}",None,log_stream_name) client.connect(HOST, 8080, 5) @@ -119,18 +120,18 @@ def lambda_handler(event, context): body = json.dumps(payload) - serial = payload[os.getenv("MQTT_ID")] + serial = payload[config_handler.get("MQTT","ID")] while not connected_flag: time.sleep(0.01) # wait until connected client.publish( - topic=f'{os.getenv("MQTT_PREFIX")}/{serial}', + topic=f'{config_handler.get("MQTT","PREFIX")}/{serial}', payload=body, qos=0, retain=False ) if serial not in cache: # low bandwidth feeds with just the first packet client.publish( - topic=f'{os.getenv("MQTT_PREFIX")}-new/{serial}', + topic=f'{config_handler.get("MQTT","PREFIX")}-new/{serial}', payload=body, qos=0, retain=False @@ -140,7 +141,7 @@ def lambda_handler(event, context): while len(cache) > MAX_CACHE: del cache[next(iter(cache))] client.publish( - topic=os.getenv("MQTT_BATCH"), + topic=config_handler.get("MQTT","BATCH"), payload=json.dumps(payloads), qos=0, retain=False diff --git a/lambda/sonde_api_to_iot_core/__init__.py b/lambda/sonde_api_to_iot_core/__init__.py index 21150bd..5557812 100644 --- a/lambda/sonde_api_to_iot_core/__init__.py +++ b/lambda/sonde_api_to_iot_core/__init__.py @@ -17,6 +17,8 @@ import base64 import gzip from io import BytesIO +import config_handler + logs = boto3.client('logs') sequenceToken = None @@ -366,7 +368,7 @@ def post(payload): f.write(json.dumps(payload).encode('utf-8')) payload = base64.b64encode(compressed.getvalue()).decode("utf-8") sns.publish( - TopicArn=os.getenv("SNS_TOPIC"), + TopicArn=config_handler.get("SNS","TOPIC"), Message=payload ) diff --git a/lambda/station_api_to_iot_core/__init__.py b/lambda/station_api_to_iot_core/__init__.py index 54fb95b..fa28e37 100644 --- a/lambda/station_api_to_iot_core/__init__.py +++ b/lambda/station_api_to_iot_core/__init__.py @@ -8,6 +8,7 @@ import base64 import gzip from io import BytesIO import boto3 +import config_handler CALLSIGN_BLOCK_LIST = ["CHANGEME_RDZTTGO"] @@ -27,7 +28,7 @@ def post(payload): f.write(json.dumps(payload).encode('utf-8')) payload = base64.b64encode(compressed.getvalue()).decode("utf-8") sns.publish( - TopicArn=os.getenv("SNS_TOPIC"), + TopicArn=config_handler.get("SNS","TOPIC"), Message=payload ) diff --git a/lambda/ttn_helium/__init__.py b/lambda/ttn_helium/__init__.py index 34f8303..5fe53d4 100644 --- a/lambda/ttn_helium/__init__.py +++ b/lambda/ttn_helium/__init__.py @@ -5,6 +5,7 @@ import base64 import datetime from email.utils import parsedate import os +import config_handler HELIUM_GW_VERSION = "2023.10.14" @@ -46,7 +47,7 @@ sns.meta.events.register('request-created.sns', set_connection_header) def post(payload): sns.publish( - TopicArn=os.getenv("HAM_SNS_TOPIC"), + TopicArn=config_handler.get("HAM_SNS","TOPIC"), Message=json.dumps(payload) ) diff --git a/lb.tf b/lb.tf index d77ddf5..be300d4 100644 --- a/lb.tf +++ b/lb.tf @@ -8,7 +8,7 @@ resource "aws_lb" "ws" { enable_deletion_protection = true - ip_address_type = "dualstack" + ip_address_type = "dualstack" } diff --git a/main.tf b/main.tf index 34a276a..2af9f6d 100644 --- a/main.tf +++ b/main.tf @@ -84,8 +84,14 @@ resource "aws_iam_role_policy" "basic_lambda_role" { ], "Effect": "Allow", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } - ] } EOF diff --git a/predictor.tf b/predictor.tf index d13b69d..d553f33 100644 --- a/predictor.tf +++ b/predictor.tf @@ -64,6 +64,13 @@ resource "aws_iam_role_policy" "predict_updater" { ], "Effect": "Allow", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] } @@ -86,15 +93,14 @@ resource "aws_lambda_function" "predict_updater" { reserved_concurrent_executions = 1 environment { variables = { - "ES" = aws_route53_record.es.fqdn + "ES" = aws_route53_record.es.fqdn + MQTT_HOST = "ws.v2.sondehub.org" + MQTT_PORT = "443" } } tags = { Name = "predict_updater" } - lifecycle { - ignore_changes = [environment] - } } @@ -380,8 +386,13 @@ resource "aws_ecr_repository" "tawhiri_downloader" { } resource "aws_ecs_cluster" "tawhiri" { - name = "Tawhiri" - capacity_providers = ["FARGATE", "FARGATE_SPOT"] + name = "Tawhiri" +} + +resource "aws_ecs_cluster_capacity_providers" "tawhiri" { + cluster_name = aws_ecs_cluster.tawhiri.name + + capacity_providers = ["FARGATE"] } @@ -410,7 +421,7 @@ resource "aws_ecs_service" "tawhiri" { task_definition = aws_ecs_task_definition.tawhiri.arn enable_ecs_managed_tags = true health_check_grace_period_seconds = 600 - iam_role = "aws-service-role" + iam_role = "/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS" launch_type = "FARGATE" platform_version = "LATEST" desired_count = 1 diff --git a/recovered.tf b/recovered.tf index a629c41..8e637f6 100644 --- a/recovered.tf +++ b/recovered.tf @@ -44,6 +44,13 @@ resource "aws_iam_role_policy" "recovered" { "Effect": "Allow", "Action": "es:*", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] } @@ -209,9 +216,8 @@ resource "aws_lambda_function" "recovery_ingest" { tags = { Name = "recovered_get" } - - lifecycle { - ignore_changes = [environment] + environment { + variables = {} } } diff --git a/secrets.tf b/secrets.tf new file mode 100644 index 0000000..48be954 --- /dev/null +++ b/secrets.tf @@ -0,0 +1,38 @@ +resource "aws_secretsmanager_secret" "mqtt" { + name = "MQTT" +} + +resource "aws_secretsmanager_secret_version" "mqtt" { + secret_id = aws_secretsmanager_secret.mqtt.id + secret_string = jsonencode( + { + HOST = join(",", local.websocket_host_addresses) + PASSWORD = random_password.mqtt.result + USERNAME = "write" + } + ) +} + +resource "random_password" "mqtt" { + length = 18 + special = false + lifecycle { + ignore_changes = [special] + } +} + +resource "aws_secretsmanager_secret" "radiosondy" { + name = "RADIOSONDY" +} + +resource "aws_secretsmanager_secret_version" "radiosondy" { + secret_id = aws_secretsmanager_secret.radiosondy.id + secret_string = jsonencode( + { + API_KEY = "" + } + ) + lifecycle { + ignore_changes = [secret_string] + } +} \ No newline at end of file diff --git a/sqs_to_elk.tf b/sqs_to_elk.tf index 3bdb9f8..8a083e6 100644 --- a/sqs_to_elk.tf +++ b/sqs_to_elk.tf @@ -48,6 +48,13 @@ resource "aws_iam_role_policy" "sqs_to_elk" { "Effect": "Allow", "Action": "sqs:*", "Resource": "*" + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] } diff --git a/vpc.tf b/vpc.tf index 63c1017..b2c4185 100644 --- a/vpc.tf +++ b/vpc.tf @@ -26,12 +26,12 @@ locals { "us-east-1f" = cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 12) } public_subnets = { - "us-east-1a" = ["172.31.80.0/20",cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 19)], - "us-east-1b" = ["172.31.16.0/20",cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 20)], - "us-east-1c" = ["172.31.32.0/20",cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 21)], - "us-east-1d" = ["172.31.0.0/20",cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 22)], - "us-east-1e" = ["172.31.48.0/20",cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 23)], - "us-east-1f" = ["172.31.64.0/20",cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 24)] + "us-east-1a" = ["172.31.80.0/20", cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 19)], + "us-east-1b" = ["172.31.16.0/20", cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 20)], + "us-east-1c" = ["172.31.32.0/20", cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 21)], + "us-east-1d" = ["172.31.0.0/20", cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 22)], + "us-east-1e" = ["172.31.48.0/20", cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 23)], + "us-east-1f" = ["172.31.64.0/20", cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 24)] } public_v6 = { "us-east-1a" = cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 1), @@ -45,11 +45,11 @@ locals { resource "aws_subnet" "private" { for_each = local.private_subnets - map_public_ip_on_launch = false - vpc_id = aws_vpc.main.id - cidr_block = each.value[0] - ipv6_cidr_block = each.value[1] - assign_ipv6_address_on_creation = true + map_public_ip_on_launch = false + vpc_id = aws_vpc.main.id + cidr_block = each.value[0] + ipv6_cidr_block = each.value[1] + assign_ipv6_address_on_creation = true tags = { Name = "${each.key} - private" } @@ -58,11 +58,11 @@ resource "aws_subnet" "private" { resource "aws_subnet" "public" { for_each = local.public_subnets - map_public_ip_on_launch = false - vpc_id = aws_vpc.main.id - cidr_block = each.value[0] - ipv6_cidr_block = each.value[1] - assign_ipv6_address_on_creation = true + map_public_ip_on_launch = false + vpc_id = aws_vpc.main.id + cidr_block = each.value[0] + ipv6_cidr_block = each.value[1] + assign_ipv6_address_on_creation = true tags = { Name = "${each.key} - public" @@ -72,31 +72,31 @@ resource "aws_subnet" "public" { resource "aws_subnet" "public_v6_only" { for_each = local.public_v6 - availability_zone = each.key + availability_zone = each.key enable_resource_name_dns_aaaa_record_on_launch = true - assign_ipv6_address_on_creation = true - vpc_id = aws_vpc.main.id - ipv6_native = true - ipv6_cidr_block = each.value + assign_ipv6_address_on_creation = true + vpc_id = aws_vpc.main.id + ipv6_native = true + ipv6_cidr_block = each.value tags = { Name = "${each.key} - public v6 only" } - + } resource "aws_subnet" "private_v6_only" { for_each = local.private_v6 - availability_zone = each.key + availability_zone = each.key enable_resource_name_dns_aaaa_record_on_launch = true - assign_ipv6_address_on_creation = true - vpc_id = aws_vpc.main.id - ipv6_native = true - ipv6_cidr_block = each.value + assign_ipv6_address_on_creation = true + vpc_id = aws_vpc.main.id + ipv6_native = true + ipv6_cidr_block = each.value tags = { Name = "${each.key} - private v6 only" } - + } resource "aws_route_table" "main" { @@ -157,4 +157,50 @@ resource "aws_route" "public_v6" { route_table_id = aws_route_table.public_v6.id destination_ipv6_cidr_block = "::/0" gateway_id = aws_internet_gateway.gw.id +} + +resource "aws_security_group" "vpcendpoint" { + name = "vpcendpoint" + description = "vpcendpoint" + ingress = [ + { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + description = "" + prefix_list_ids = [] + self = false + security_groups = [] + } + ] + egress = [ + { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + security_groups = [] + description = "" + prefix_list_ids = [] + self = false + } + ] + vpc_id = aws_vpc.main.id + +} + + +resource "aws_vpc_endpoint" "secretsmanager" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.us-east-1.secretsmanager" + vpc_endpoint_type = "Interface" + + security_group_ids = [ + aws_security_group.vpcendpoint.id, + ] + + private_dns_enabled = true } \ No newline at end of file diff --git a/websockets.tf b/websockets.tf index 7672bd2..b95445b 100644 --- a/websockets.tf +++ b/websockets.tf @@ -67,11 +67,11 @@ resource "aws_ecr_repository" "wsproxy" { // Subnet that is used to make discovery simple for the main ws server resource "aws_subnet" "ws_main" { - map_public_ip_on_launch = false - vpc_id = aws_vpc.main.id - cidr_block = "172.31.134.0/28" - ipv6_cidr_block = cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 128) - assign_ipv6_address_on_creation = true + map_public_ip_on_launch = false + vpc_id = aws_vpc.main.id + cidr_block = local.websocket_subnet + ipv6_cidr_block = cidrsubnet(aws_vpc.main.ipv6_cidr_block, 8, 128) + assign_ipv6_address_on_creation = true tags = { Name = "wsmain" @@ -83,6 +83,16 @@ resource "aws_route_table_association" "ws_main" { route_table_id = aws_route_table.main.id } +locals { + websocket_subnet = "172.31.134.0/28" + websocket_network_addresses = cidrsubnets(local.websocket_subnet, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4) + websocket_host_addresses = setsubtract( # calculates what the "remaining" IP addresses are in the VPC subnet after padding + [for x in slice(local.websocket_network_addresses, 4, 15) : replace(x, "/32", "")], + [for x in aws_network_interface.ws_pad : tolist(x.private_ips)[0]] + ) +} + + // so we need to ensure there is only as handful of IP addresses avaliable in the subnet, so we assign all the IPs to ENIs resource "aws_network_interface" "ws_pad" { count = 9 @@ -91,144 +101,6 @@ resource "aws_network_interface" "ws_pad" { description = "Do not delete. Padding to limit addresses" } -# resource "aws_ecs_task_definition" "ws_reader" { -# family = "ws-reader" -# container_definitions = jsonencode( -# [ -# { -# command = [ -# "s3", -# "sync", -# "s3://sondehub-ws-config/", -# "/config/", -# ] -# cpu = 0 -# environment = [] -# essential = false -# image = "amazon/aws-cli" -# logConfiguration = { -# logDriver = "awslogs" -# options = { -# awslogs-group = "/ecs/ws" -# awslogs-region = "us-east-1" -# awslogs-stream-prefix = "ecs" -# } -# } -# mountPoints = [ -# { -# containerPath = "/config" -# sourceVolume = "config" -# }, -# ] -# name = "config" -# portMappings = [] -# volumesFrom = [] -# }, -# { -# command = [] -# cpu = 0 -# dependsOn = [ -# { -# condition = "SUCCESS" -# containerName = "config" -# }, -# { -# condition = "SUCCESS" -# containerName = "config-move" -# }, -# ] -# environment = [] -# essential = true -# image = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-1.amazonaws.com/wsproxy:latest" -# logConfiguration = { -# logDriver = "awslogs" -# options = { -# awslogs-group = "/ecs/ws" -# awslogs-region = "us-east-1" -# awslogs-stream-prefix = "ecs" -# } -# } -# mountPoints = [ -# { -# containerPath = "/mosquitto/config" -# sourceVolume = "config" -# }, -# ] -# name = "mqtt" -# portMappings = [ -# { -# containerPort = 8080 -# hostPort = 8080 -# protocol = "tcp" -# }, -# { -# containerPort = 8883 -# hostPort = 8883 -# protocol = "tcp" -# }, -# ] -# ulimits = [ -# { -# hardLimit = 50000 -# name = "nofile" -# softLimit = 30000 -# }, -# ] -# volumesFrom = [] -# }, -# { -# command = [ -# "cp", -# "/config/mosquitto-reader.conf", -# "/config/mosquitto.conf", -# ] -# cpu = 0 -# dependsOn = [ -# { -# condition = "SUCCESS" -# containerName = "config" -# }, -# ] -# environment = [] -# essential = false -# image = "alpine" -# # logConfiguration = { -# # logDriver = "awslogs" -# # options = { -# # awslogs-group = "/ecs/ws-reader" -# # awslogs-region = "us-east-1" -# # awslogs-stream-prefix = "ecs" -# # } -# # } -# mountPoints = [ -# { -# containerPath = "/config" -# sourceVolume = "config" -# }, -# ] -# name = "config-move" -# portMappings = [] -# volumesFrom = [] -# }, -# ] -# ) -# cpu = "256" -# execution_role_arn = aws_iam_role.ecs_execution.arn -# memory = "512" -# network_mode = "awsvpc" -# requires_compatibilities = [ -# "FARGATE", -# ] - -# tags = {} -# task_role_arn = "arn:aws:iam::143841941773:role/ws" - - -# volume { -# name = "config" -# } -# } - resource "aws_ecs_task_definition" "ws_reader_ec2" { family = "ws_reader_ec2" container_definitions = jsonencode( @@ -466,8 +338,14 @@ resource "aws_ecs_task_definition" "ws" { } resource "aws_ecs_cluster" "ws" { - name = "ws" - capacity_providers = ["FARGATE", "FARGATE_SPOT"] + name = "ws" +} + +resource "aws_ecs_cluster_capacity_providers" "ws" { + cluster_name = aws_ecs_cluster.ws.name + + capacity_providers = ["FARGATE"] + } resource "aws_lb_target_group" "ws" { @@ -507,35 +385,6 @@ resource "aws_lb_target_group" "ws_reader" { } } -# resource "aws_ecs_service" "ws_reader" { -# name = "ws-reader" -# cluster = aws_ecs_cluster.ws.id -# task_definition = aws_ecs_task_definition.ws_reader.arn -# enable_ecs_managed_tags = true -# health_check_grace_period_seconds = 60 -# iam_role = "aws-service-role" -# launch_type = "FARGATE" -# platform_version = "LATEST" -# desired_count = 0 - -# load_balancer { -# container_name = "mqtt" -# container_port = 8080 -# target_group_arn = aws_lb_target_group.ws_reader.arn -# } - -# lifecycle { -# ignore_changes = [desired_count] -# } - -# network_configuration { -# assign_public_ip = true -# security_groups = [ -# aws_security_group.ws_reader.id -# ] -# subnets = values(aws_subnet.public)[*].id -# } -# } resource "aws_ecs_service" "ws_reader_ec2" { name = "ws-reader-ec2" @@ -555,7 +404,7 @@ resource "aws_ecs_service" "ws_writer" { task_definition = aws_ecs_task_definition.ws.arn enable_ecs_managed_tags = true health_check_grace_period_seconds = 60 - iam_role = "aws-service-role" + iam_role = "/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS" launch_type = "FARGATE" platform_version = "LATEST" desired_count = 1 @@ -810,6 +659,13 @@ resource "aws_iam_role_policy" "s3_config" { "arn:aws:s3:::sondehub-ws-config", "arn:aws:s3:::sondehub-ws-config/*" ] + }, + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": ["${aws_secretsmanager_secret.mqtt.arn}", "${aws_secretsmanager_secret.radiosondy.arn}"] } ] }