Align terraform

es.tf

@@ -8,7 +8,7 @@ resource "aws_elasticsearch_domain" "ElasticsearchDomain" {
     dedicated_master_enabled = false
     dedicated_master_type    = "t3.small.elasticsearch"
     instance_count           = 1
-    instance_type            = "r6g.large.elasticsearch"
+    instance_type            = "r6g.xlarge.elasticsearch"
     zone_awareness_enabled   = false
   }
   cognito_options {

ham_predictor.tf

@@ -131,6 +131,7 @@ resource "aws_lambda_function" "ham_predictions" {
   s3_bucket        = aws_s3_bucket_object.lambda.bucket
   s3_key           = aws_s3_bucket_object.lambda.key
   source_code_hash = data.archive_file.lambda.output_base64sha256
+  reserved_concurrent_executions = 10
   publish          = true
   memory_size      = 128
   role             = aws_iam_role.basic_lambda_role.arn

ham_query.tf

@@ -31,6 +31,7 @@ resource "aws_lambda_function" "ham_telem" {
   handler          = "query_ham.get_telem"
   s3_bucket        = aws_s3_bucket_object.lambda.bucket
   s3_key           = aws_s3_bucket_object.lambda.key
+  reserved_concurrent_executions = 10 
   source_code_hash = data.archive_file.lambda.output_base64sha256
   publish          = true
   memory_size      = 1024

predictor.tf

@@ -145,6 +145,7 @@ resource "aws_lambda_function" "predictions" {
   handler          = "predict.predict"
   s3_bucket        = aws_s3_bucket_object.lambda.bucket
   s3_key           = aws_s3_bucket_object.lambda.key
+  reserved_concurrent_executions = 10 
   source_code_hash = data.archive_file.lambda.output_base64sha256
   publish          = true
   memory_size      = 128
@@ -177,6 +178,7 @@ resource "aws_lambda_function" "reverse_predictions" {
   source_code_hash = data.archive_file.lambda.output_base64sha256
   publish          = true
   memory_size      = 128
+  reserved_concurrent_executions = 10
   role             = aws_iam_role.basic_lambda_role.arn
   runtime          = "python3.9"
   timeout          = 30
@@ -206,14 +208,14 @@ resource "aws_ecs_task_definition" "tawhiri" {
     [
       {
         healthCheck = {
-          retries = 3
+          retries = 10
           command = [
             "/usr/bin/python3.7",
             "-c",
             "import urllib.request; import json; import datetime; import sys; sys.exit(0) if len(json.loads(urllib.request.urlopen(f'http://localhost:8000/api/v1/?launch_latitude=51.77542999852449&launch_longitude=15.553199937567115&launch_datetime={datetime.datetime.now().strftime(\"%Y-%m-%dT%H:%M:%SZ\")}&launch_altitude=0&ascent_rate=5.00&burst_altitude=14030.77&descent_rate=5.28').read())['prediction'][0]['trajectory']) > 0 else sys.exit(1)"
           ]
           timeout     = 20
-          interval    = 60
+          interval    = 120
           startPeriod = 30
         }
         command = [
@@ -221,7 +223,7 @@ resource "aws_ecs_task_definition" "tawhiri" {
           "-b",
           "0.0.0.0:8000",
           "--workers=1",
-          "--timeout=10",
+          "--timeout=3",
           "--keep-alive=65",
           "--threads=20",
           "tawhiri.api:app"
@@ -230,6 +232,10 @@ resource "aws_ecs_task_definition" "tawhiri" {
           {
             containerName = "downloader"
             condition     = "SUCCESS"
+          },
+          {
+            containerName = "ruaumoko-copy"
+            condition     = "SUCCESS"
           }
         ]
         cpu         = 0
@@ -247,7 +253,7 @@ resource "aws_ecs_task_definition" "tawhiri" {
         mountPoints = [
           {
             containerPath = "/srv"
-            sourceVolume  = "srv"
+            sourceVolume  = "ruaumoko"
           },
           {
             containerPath = "/srv/tawhiri-datasets"
@@ -291,6 +297,34 @@ resource "aws_ecs_task_definition" "tawhiri" {
         name        = "downloader"
         volumesFrom = []
       },
+      {
+        cpu = 0
+        environment = []
+        essential = false
+        image     = "amazon/aws-cli"
+        command = [                
+          "s3",
+                "cp",
+                "s3://ruaumoko/ruaumoko-dataset",
+                "/ruaumoko/ruaumoko-dataset"
+                ]
+        logConfiguration = {
+          logDriver = "awslogs"
+          options = {
+            awslogs-group         = "/ecs/tawhiri"
+            awslogs-region        = "us-east-1"
+            awslogs-stream-prefix = "ecs"
+          }
+        }
+        mountPoints = [
+          {
+            containerPath = "/ruaumoko"
+            sourceVolume  = "ruaumoko"
+          },
+        ]
+        name        = "ruaumoko-copy"
+        volumesFrom = []
+      }
     ]
   )
   cpu                = "512"
@@ -308,92 +342,10 @@ resource "aws_ecs_task_definition" "tawhiri" {
   volume {
     name = "downloader"
   }
-
   volume {
-    name = "srv"
-
-    efs_volume_configuration {
-      file_system_id     = aws_efs_file_system.tawhiri.id
-      root_directory     = "/srv"
-      transit_encryption = "DISABLED"
-
-      authorization_config {
-        iam = "DISABLED"
-      }
-    }
-  }
-  lifecycle {
-    ignore_changes = [volume] # terraform has a bug that doesn't correctly deal with root_directory because I don't know why - so we ignore it
-  }
-}
-
-resource "aws_ecs_task_definition" "tawhiri_ruaumoko" {
-  family = "tawhiri-ruaumoko"
-  container_definitions = jsonencode(
-    [
-      {
-        cpu = 0
-        entryPoint = [
-          "/root/.local/bin/ruaumoko-download",
-        ]
-        environment = []
-        essential   = true
-        image       = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-1.amazonaws.com/tawhiri:latest"
-        logConfiguration = {
-          logDriver = "awslogs"
-          options = {
-            awslogs-group         = "/ecs/tawhiri-ruaumoko"
-            awslogs-region        = "us-east-1"
-            awslogs-stream-prefix = "ecs"
-          }
-        }
-        mountPoints = [
-          {
-            containerPath = "/srv"
-            sourceVolume  = "srv"
-          },
-        ]
     name = "ruaumoko"
-        portMappings = []
-        volumesFrom  = []
-      },
-    ]
-  )
-  cpu                = "1024"
-  execution_role_arn = "arn:aws:iam::143841941773:role/ecsTaskExecutionRole"
-  memory             = "2048"
-  network_mode       = "awsvpc"
-  requires_compatibilities = [
-    "FARGATE",
-  ]
-  tags          = {}
-  task_role_arn = "arn:aws:iam::143841941773:role/ecsTaskExecutionRole"
-
-
-  volume {
-    name = "srv"
-
-    efs_volume_configuration {
-      file_system_id     = aws_efs_file_system.tawhiri.id
-      root_directory     = "srv"
-      transit_encryption = "DISABLED"
-
-      authorization_config {
-        iam = "DISABLED"
   }
-    }
-  }
-}
 
-
-
-resource "aws_efs_file_system" "tawhiri" {
-  tags = {
-    Name = "Tawhiri"
-  }
-  lifecycle_policy {
-    transition_to_ia = "AFTER_7_DAYS"
-  }
 }
 
 resource "aws_ecr_repository" "tawhiri" {
@@ -449,7 +401,7 @@ resource "aws_ecs_service" "tawhiri" {
   platform_version                  = "LATEST"
   desired_count                     = 1
   enable_execute_command            = true
-
+  deployment_maximum_percent         = 400
   load_balancer {
     container_name   = "tawhiri"
     container_port   = 8000
@@ -476,7 +428,7 @@ resource "aws_appautoscaling_target" "tawhiri" {
   scalable_dimension = "ecs:service:DesiredCount"
   resource_id        = "service/Tawhiri/tawhiri"
   min_capacity       = 1
-  max_capacity       = 5
+  max_capacity       = 7
 }
 
 resource "aws_appautoscaling_policy" "tawhiri" {
@@ -491,9 +443,9 @@ resource "aws_appautoscaling_policy" "tawhiri" {
       predefined_metric_type = "ECSServiceAverageCPUUtilization"
     }
 
-    target_value       = 80
+    target_value       = 75
-    scale_in_cooldown  = 120
+    scale_in_cooldown  = 240
-    scale_out_cooldown = 120
+    scale_out_cooldown = 240
   }
 }
 

query.tf

@@ -7,6 +7,7 @@ resource "aws_lambda_function" "get_sondes" {
   source_code_hash = data.archive_file.lambda.output_base64sha256
   publish          = true
   memory_size      = 256
+  reserved_concurrent_executions = 10
   role             = aws_iam_role.basic_lambda_role.arn
   runtime          = "python3.9"
   timeout          = 30
@@ -101,6 +102,7 @@ resource "aws_lambda_function" "get_listener_stats" {
   s3_key           = aws_s3_bucket_object.lambda.key
   source_code_hash = data.archive_file.lambda.output_base64sha256
   publish          = true
+  reserved_concurrent_executions = 0
   memory_size      = 256
   role             = aws_iam_role.basic_lambda_role.arn
   runtime          = "python3.9"

websockets.tf

@@ -541,7 +541,7 @@ resource "aws_ecs_service" "ws_reader_ec2" {
   task_definition         = aws_ecs_task_definition.ws_reader_ec2.arn
   enable_ecs_managed_tags = true
   launch_type             = "EC2"
-  desired_count           = 6
+  desired_count           = 9
   placement_constraints {
     type = "distinctInstance"
   }