From 86a1d4f07cb6b7d5ee65bdbcce33b72e0bd93d2d Mon Sep 17 00:00:00 2001 From: Michaela Date: Mon, 4 Oct 2021 19:08:54 +1100 Subject: [PATCH 1/2] add in arm --- historic.tf | 4 +- main.tf | 101 ++++++++++++++------------------------- predictor.tf | 3 +- query/lambda_function.py | 9 +++- recovered.tf | 4 +- sqs_to_elk.tf | 4 +- 6 files changed, 52 insertions(+), 73 deletions(-) diff --git a/historic.tf b/historic.tf index 77b8aef..8193623 100644 --- a/historic.tf +++ b/historic.tf @@ -80,7 +80,7 @@ resource "aws_lambda_function" "historic_to_s3" { publish = true memory_size = 2048 role = aws_iam_role.historic.arn - runtime = "python3.8" + runtime = "python3.9" timeout = 60 reserved_concurrent_executions = 8 environment { @@ -97,7 +97,7 @@ resource "aws_lambda_function" "queue_data_update" { publish = true memory_size = 256 role = aws_iam_role.historic.arn - runtime = "python3.8" + runtime = "python3.9" timeout = 30 reserved_concurrent_executions = 1 environment { diff --git a/main.tf b/main.tf index 02b0009..67ab0f7 100644 --- a/main.tf +++ b/main.tf @@ -496,7 +496,7 @@ resource "aws_route53_record" "Route53RecordSet7" { data "archive_file" "api_to_iot" { type = "zip" - source_dir = "sonde-api-to-iot-core/" + source_dir = "sonde-api-to-iot-core/" output_path = "${path.module}/build/sonde-api-to-iot-core.zip" } @@ -537,20 +537,17 @@ resource "aws_lambda_function" "LambdaFunction" { filename = "${path.module}/build/sonde-api-to-iot-core.zip" source_code_hash = data.archive_file.api_to_iot.output_base64sha256 publish = true - memory_size = 256 + memory_size = 128 role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 30 + architectures = ["arm64"] environment { variables = { "IOT_ENDPOINT" = data.aws_iot_endpoint.endpoint.endpoint_address - "SNS_TOPIC" = aws_sns_topic.sonde_telem.arn + "SNS_TOPIC" = aws_sns_topic.sonde_telem.arn } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] } resource "aws_lambda_function" "station" { @@ -582,18 +579,14 @@ resource "aws_lambda_function" "get_sondes" { publish = true memory_size = 256 role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 30 - + architectures = ["arm64"] environment { variables = { "ES" = "es.${local.domain_name}" } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] } resource "aws_lambda_function" "listeners" { @@ -604,18 +597,14 @@ resource "aws_lambda_function" "listeners" { publish = true memory_size = 256 role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 30 - + architectures = ["arm64"] environment { variables = { "ES" = "es.${local.domain_name}" } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] } @@ -627,20 +616,16 @@ resource "aws_lambda_function" "predictions" { filename = "${path.module}/build/predictions.zip" source_code_hash = data.archive_file.predictions.output_base64sha256 publish = true - memory_size = 256 + memory_size = 128 role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 30 - + architectures = ["arm64"] environment { variables = { "ES" = "es.${local.domain_name}" } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] } resource "aws_lambda_function" "get_telem" { @@ -651,18 +636,14 @@ resource "aws_lambda_function" "get_telem" { publish = true memory_size = 256 role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 30 - + architectures = ["arm64"] environment { variables = { "ES" = "es.${local.domain_name}" } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] } resource "aws_lambda_function" "get_listener_telemetry" { @@ -673,12 +654,9 @@ resource "aws_lambda_function" "get_listener_telemetry" { publish = true memory_size = 256 role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 30 - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] + architectures = ["arm64"] environment { variables = { "ES" = "es.${local.domain_name}" @@ -694,41 +672,35 @@ resource "aws_lambda_function" "sign_socket" { publish = true memory_size = 128 role = aws_iam_role.sign_socket.arn - runtime = "python3.7" + runtime = "python3.9" timeout = 10 - + architectures = ["arm64"] environment { variables = { "IOT_ENDPOINT" = data.aws_iot_endpoint.endpoint.endpoint_address } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] } resource "aws_lambda_function" "history" { - function_name = "history" - handler = "lambda_function.history" - filename = "${path.module}/build/history.zip" - source_code_hash = data.archive_file.history.output_base64sha256 - publish = true - memory_size = 512 - role = aws_iam_role.IAMRole5.arn - runtime = "python3.7" - timeout = 30 + function_name = "history" + handler = "lambda_function.history" + filename = "${path.module}/build/history.zip" + source_code_hash = data.archive_file.history.output_base64sha256 + publish = true + memory_size = 512 + role = aws_iam_role.IAMRole5.arn + runtime = "python3.9" + timeout = 30 reserved_concurrent_executions = 4 + architectures = ["arm64"] environment { variables = { "ES" = "es.${local.domain_name}" } } - layers = [ - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:xray-python:1", - "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current.account_id}:layer:iot:3" - ] + } resource "aws_lambda_permission" "sign_socket" { @@ -821,7 +793,7 @@ resource "aws_s3_bucket" "S3Bucket" { } resource "aws_cloudwatch_log_group" "LogsLogGroup" { - name = "/aws/lambda/sonde-api-to-iot-core" + name = "/aws/lambda/sonde-api-to-iot-core" retention_in_days = 30 } @@ -1069,7 +1041,7 @@ resource "aws_acm_certificate" "CertificateManagerCertificate" { resource "aws_elasticsearch_domain" "ElasticsearchDomain" { domain_name = "sondes-v2" - elasticsearch_version = "7.9" + elasticsearch_version = "OpenSearch_1.0" cluster_config { dedicated_master_count = 3 dedicated_master_enabled = false @@ -1109,11 +1081,12 @@ EOF } advanced_options = { "rest.action.multi.allow_explicit_index" = "true" + "override_main_response_version" = "true" } ebs_options { ebs_enabled = true volume_type = "gp2" - volume_size = 400 + volume_size = 250 } log_publishing_options { cloudwatch_log_group_arn = "arn:aws:logs:us-east-1:143841941773:log-group:/aws/aes/domains/sondes-v2/application-logs" @@ -1128,7 +1101,7 @@ EOF log_publishing_options { cloudwatch_log_group_arn = "arn:aws:logs:us-east-1:143841941773:log-group:/aws/aes/domains/sondes-v2/search-logs" enabled = true - log_type = "SEARCH_SLOW_LOGS" + log_type = "SEARCH_SLOW_LOGS" } } data "aws_kms_key" "es" { @@ -1156,7 +1129,7 @@ resource "aws_cognito_identity_pool_roles_attachment" "CognitoIdentityPoolRoleAt } role_mapping { ambiguous_role_resolution = "AuthenticatedRole" - identity_provider = "cognito-idp.us-east-1.amazonaws.com/us-east-1_G4H7NMniM:5sngha3l291nb4784iid5hli48" + identity_provider = "cognito-idp.us-east-1.amazonaws.com/us-east-1_G4H7NMniM:62uut02s5ts991uhpf4fbuvrtj" type = "Token" } } @@ -1229,8 +1202,8 @@ resource "aws_cognito_user_pool_client" "CognitoUserPoolClient" { allowed_oauth_flows = ["code"] allowed_oauth_flows_user_pool_client = true allowed_oauth_scopes = ["email", "openid", "phone", "profile"] - callback_urls = ["https://es.${local.domain_name}/_plugin/kibana/app/kibana"] - logout_urls = ["https://es.${local.domain_name}/_plugin/kibana/app/kibana"] + callback_urls = ["https://es.${local.domain_name}/_dashboards/app/home"] + logout_urls = ["https://es.${local.domain_name}/_dashboards/app/home"] supported_identity_providers = ["COGNITO", "Google"] explicit_auth_flows = ["ALLOW_CUSTOM_AUTH", "ALLOW_REFRESH_TOKEN_AUTH", "ALLOW_USER_SRP_AUTH"] } \ No newline at end of file diff --git a/predictor.tf b/predictor.tf index 96c20b1..7182318 100644 --- a/predictor.tf +++ b/predictor.tf @@ -74,7 +74,8 @@ resource "aws_lambda_function" "predict_updater" { publish = true memory_size = 256 role = aws_iam_role.predict_updater.arn - runtime = "python3.8" + runtime = "python3.9" + architectures = ["arm64"] timeout = 60 reserved_concurrent_executions = 8 environment { diff --git a/query/lambda_function.py b/query/lambda_function.py index d2644b3..7a5552d 100644 --- a/query/lambda_function.py +++ b/query/lambda_function.py @@ -490,8 +490,13 @@ if __name__ == "__main__": # ) # ) print( - get_listeners( - {},{} + get_telem( + { + "queryStringParameters": { + "duration": "3d", + "serial": "P4120469" + }},{} + ) ) # print ( diff --git a/recovered.tf b/recovered.tf index 01f4bc7..7b30171 100644 --- a/recovered.tf +++ b/recovered.tf @@ -64,7 +64,7 @@ resource "aws_lambda_function" "recovered_get" { publish = true memory_size = 128 role = aws_iam_role.recovered.arn - runtime = "python3.8" + runtime = "python3.9" timeout = 30 reserved_concurrent_executions = 100 environment { @@ -82,7 +82,7 @@ resource "aws_lambda_function" "recovered_put" { publish = true memory_size = 128 role = aws_iam_role.recovered.arn - runtime = "python3.8" + runtime = "python3.9" timeout = 30 reserved_concurrent_executions = 100 environment { diff --git a/sqs_to_elk.tf b/sqs_to_elk.tf index 9368402..5f854d2 100644 --- a/sqs_to_elk.tf +++ b/sqs_to_elk.tf @@ -68,14 +68,14 @@ resource "aws_lambda_function" "sqs_to_elk" { publish = true memory_size = 128 role = aws_iam_role.sqs_to_elk.arn - runtime = "python3.8" + runtime = "python3.9" timeout = 5 reserved_concurrent_executions = 100 environment { variables = { "ES" = aws_route53_record.Route53RecordSet7.fqdn } - } + } } resource "aws_lambda_event_source_mapping" "sqs_to_elk" { From 345e10400dc16cb7e3f214c039234bd99437e158 Mon Sep 17 00:00:00 2001 From: Michaela Date: Mon, 4 Oct 2021 19:24:30 +1100 Subject: [PATCH 2/2] lower historic --- historic.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/historic.tf b/historic.tf index 8193623..9374b2f 100644 --- a/historic.tf +++ b/historic.tf @@ -82,7 +82,7 @@ resource "aws_lambda_function" "historic_to_s3" { role = aws_iam_role.historic.arn runtime = "python3.9" timeout = 60 - reserved_concurrent_executions = 8 + reserved_concurrent_executions = 2 environment { variables = { "ES" = aws_route53_record.Route53RecordSet7.fqdn