sondehub-infra/predictor.tf

resource "aws_iam_role" "predict_updater" {
  path                 = "/service-role/"
  name                 = "predict-updater"
  assume_role_policy   = data.aws_iam_policy_document.lambda_assume_role_policy.json
  max_session_duration = 3600
}

data "aws_iam_policy_document" "predict_updater" {
  statement {
    resources = ["arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:*"]
    actions   = ["logs:CreateLogGroup"]
  }

  statement {
    resources = ["arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"]

    actions = [
      "logs:CreateLogStream",
      "logs:PutLogEvents",
    ]
  }

  statement {
    resources = ["*"]
    actions   = ["es:*"]
  }

  statement {
    resources = ["*"]
    actions   = ["sqs:*"]
  }

  statement {
    resources = ["*"]
    actions   = ["s3:*"]
  }

  statement {
    resources = ["*"]

    actions = [
      "ec2:DescribeNetworkInterfaces",
      "ec2:CreateNetworkInterface",
      "ec2:DeleteNetworkInterface",
      "ec2:DescribeInstances",
      "ec2:AttachNetworkInterface",
    ]
  }

  statement {
    resources = [
      aws_secretsmanager_secret.mqtt.arn,
      aws_secretsmanager_secret.radiosondy.arn,
    ]

    actions = ["secretsmanager:GetSecretValue"]
  }
}


resource "aws_iam_role_policy" "predict_updater" {
  name   = "predict_updater"
  role   = aws_iam_role.predict_updater.name
  policy = data.aws_iam_policy_document.predict_updater.json
}


resource "aws_lambda_function" "predict_updater" {
  function_name                  = "predict_updater"
  handler                        = "predict_updater.predict"
  s3_bucket                      = aws_s3_bucket_object.lambda.bucket
  s3_key                         = aws_s3_bucket_object.lambda.key
  source_code_hash               = data.archive_file.lambda.output_base64sha256
  publish                        = true
  memory_size                    = 512
  role                           = aws_iam_role.predict_updater.arn
  runtime                        = "python3.9"
  architectures                  = ["arm64"]
  timeout                        = 300
  reserved_concurrent_executions = 1
  environment {
    variables = {
      "ES"      = aws_route53_record.es.fqdn
      MQTT_HOST = "ws.v2.sondehub.org"
      MQTT_PORT = "443"
    }
  }
  tags = {
    Name = "predict_updater"
  }
}


resource "aws_cloudwatch_event_rule" "predict_updater" {
  name        = "predict_updater"
  description = "predict_updater"

  schedule_expression = "rate(1 minute)"
}

resource "aws_cloudwatch_event_target" "predict_updater" {
  rule      = aws_cloudwatch_event_rule.predict_updater.name
  target_id = "SendToLambda"
  arn       = aws_lambda_function.predict_updater.arn
}

resource "aws_lambda_permission" "predict_updater" {
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.predict_updater.function_name
  principal     = "events.amazonaws.com"
  source_arn    = aws_cloudwatch_event_rule.predict_updater.arn
}

resource "aws_apigatewayv2_route" "predictions" {
  api_id             = aws_apigatewayv2_api.main.id
  api_key_required   = false
  authorization_type = "NONE"
  route_key          = "GET /predictions"
  target             = "integrations/${aws_apigatewayv2_integration.predictions.id}"
}

resource "aws_apigatewayv2_route" "reverse_predictions" {
  api_id             = aws_apigatewayv2_api.main.id
  api_key_required   = false
  authorization_type = "NONE"
  route_key          = "GET /predictions/reverse"
  target             = "integrations/${aws_apigatewayv2_integration.reverse_predictions.id}"
}

resource "aws_apigatewayv2_integration" "predictions" {
  api_id                 = aws_apigatewayv2_api.main.id
  connection_type        = "INTERNET"
  integration_method     = "POST"
  integration_type       = "AWS_PROXY"
  integration_uri        = aws_lambda_function.predictions.arn
  timeout_milliseconds   = 30000
  payload_format_version = "2.0"
}

resource "aws_apigatewayv2_integration" "reverse_predictions" {
  api_id                 = aws_apigatewayv2_api.main.id
  connection_type        = "INTERNET"
  integration_method     = "POST"
  integration_type       = "AWS_PROXY"
  integration_uri        = aws_lambda_function.reverse_predictions.arn
  timeout_milliseconds   = 30000
  payload_format_version = "2.0"
}

resource "aws_lambda_function" "predictions" {
  function_name                  = "predictions"
  handler                        = "predict.predict"
  s3_bucket                      = aws_s3_bucket_object.lambda.bucket
  s3_key                         = aws_s3_bucket_object.lambda.key
  reserved_concurrent_executions = 10
  source_code_hash               = data.archive_file.lambda.output_base64sha256
  publish                        = true
  memory_size                    = 128
  role                           = aws_iam_role.basic_lambda_role.arn
  runtime                        = "python3.9"
  timeout                        = 30
  architectures                  = ["arm64"]
  environment {
    variables = {
      "ES" = "es.${local.domain_name}"
    }
  }
  tags = {
    Name = "predictions"
  }
}
resource "aws_lambda_permission" "predictions" {
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.predictions.arn
  principal     = "apigateway.amazonaws.com"
  source_arn    = "arn:aws:execute-api:us-east-1:${data.aws_caller_identity.current.account_id}:${aws_apigatewayv2_api.main.id}/*/*/predictions"
}


resource "aws_lambda_function" "reverse_predictions" {
  function_name                  = "reverse-predictions"
  handler                        = "reverse_predict.predict"
  s3_bucket                      = aws_s3_bucket_object.lambda.bucket
  s3_key                         = aws_s3_bucket_object.lambda.key
  source_code_hash               = data.archive_file.lambda.output_base64sha256
  publish                        = true
  memory_size                    = 128
  reserved_concurrent_executions = 10
  role                           = aws_iam_role.basic_lambda_role.arn
  runtime                        = "python3.9"
  timeout                        = 30
  architectures                  = ["arm64"]
  environment {
    variables = {
      "ES" = "es.${local.domain_name}"
    }
  }
  tags = {
    Name = "reverse_predictions"
  }
}
resource "aws_lambda_permission" "reverse_predictions" {
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.reverse_predictions.arn
  principal     = "apigateway.amazonaws.com"
  source_arn    = "arn:aws:execute-api:us-east-1:${data.aws_caller_identity.current.account_id}:${aws_apigatewayv2_api.main.id}/*/*/predictions/reverse"
}




resource "aws_ecs_task_definition" "tawhiri" {
  family = "tawhiri"
  container_definitions = jsonencode(
    [
      {
        healthCheck = {
          retries = 10
          command = [
            "/usr/bin/python3.7",
            "-c",
            "import urllib.request; import json; import datetime; import sys; sys.exit(0) if len(json.loads(urllib.request.urlopen(f'http://localhost:8000/api/v1/?launch_latitude=51.77542999852449&launch_longitude=15.553199937567115&launch_datetime={datetime.datetime.now().strftime(\"%Y-%m-%dT%H:%M:%SZ\")}&launch_altitude=0&ascent_rate=5.00&burst_altitude=14030.77&descent_rate=5.28').read())['prediction'][0]['trajectory']) > 0 else sys.exit(1)"
          ]
          timeout     = 20
          interval    = 120
          startPeriod = 30
        }
        command = [
          "/root/.local/bin/gunicorn",
          "-b",
          "0.0.0.0:8000",
          "--workers=1",
          "--timeout=3",
          "--keep-alive=65",
          "--threads=20",
          "tawhiri.api:app"
        ]
        dependsOn = [
          {
            containerName = "downloader"
            condition     = "SUCCESS"
          },
          {
            containerName = "ruaumoko-copy"
            condition     = "SUCCESS"
          }
        ]
        cpu         = 0
        environment = []
        essential   = true
        image       = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-1.amazonaws.com/tawhiri:latest"
        logConfiguration = {
          logDriver = "awslogs"
          options = {
            awslogs-group         = "/ecs/tawhiri"
            awslogs-region        = "us-east-1"
            awslogs-stream-prefix = "ecs"
          }
        }
        mountPoints = [
          {
            containerPath = "/srv"
            sourceVolume  = "ruaumoko"
          },
          {
            containerPath = "/srv/tawhiri-datasets"
            sourceVolume  = "downloader"
          }
        ]
        name = "tawhiri"
        portMappings = [
          {
            containerPort = 8000
            hostPort      = 8000
            protocol      = "tcp"
          },
        ]
        volumesFrom = []
      },
      {
        cpu = 0
        environment = [
          {
            name  = "TZ"
            value = "UTC"
          }
        ]
        essential = false
        image     = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-1.amazonaws.com/tawhiri-downloader:latest"
        logConfiguration = {
          logDriver = "awslogs"
          options = {
            awslogs-group         = "/ecs/tawhiri"
            awslogs-region        = "us-east-1"
            awslogs-stream-prefix = "ecs"
          }
        }
        mountPoints = [
          {
            containerPath = "/srv/tawhiri-datasets"
            sourceVolume  = "downloader"
          },
        ]
        name        = "downloader"
        volumesFrom = []
      },
      {
        cpu         = 0
        environment = []
        essential   = false
        image       = "amazon/aws-cli"
        command = [
          "s3",
          "cp",
          "s3://ruaumoko/ruaumoko-dataset",
          "/ruaumoko/ruaumoko-dataset"
        ]
        logConfiguration = {
          logDriver = "awslogs"
          options = {
            awslogs-group         = "/ecs/tawhiri"
            awslogs-region        = "us-east-1"
            awslogs-stream-prefix = "ecs"
          }
        }
        mountPoints = [
          {
            containerPath = "/ruaumoko"
            sourceVolume  = "ruaumoko"
          },
        ]
        name        = "ruaumoko-copy"
        volumesFrom = []
      }
    ]
  )
  cpu                = "512"
  execution_role_arn = aws_iam_role.ecs_execution.arn
  memory             = "1024"
  network_mode       = "awsvpc"
  requires_compatibilities = [
    "FARGATE",
  ]
  tags          = {}
  task_role_arn = aws_iam_role.ecs_execution.arn



  volume {
    name = "downloader"
  }
  volume {
    name = "ruaumoko"
  }

}

resource "aws_ecr_repository" "tawhiri" {
  name                 = "tawhiri"
  image_tag_mutability = "MUTABLE"

  image_scanning_configuration {
    scan_on_push = true
  }
}
resource "aws_ecr_repository" "tawhiri_downloader" {
  name                 = "tawhiri-downloader"
  image_tag_mutability = "MUTABLE"

  image_scanning_configuration {
    scan_on_push = true
  }
}

resource "aws_ecs_cluster" "tawhiri" {
  name = "Tawhiri"
}

resource "aws_ecs_cluster_capacity_providers" "tawhiri" {
  cluster_name = aws_ecs_cluster.tawhiri.name

  capacity_providers = ["FARGATE"]
}


resource "aws_lb_target_group" "tawhiri" {
  name        = "tawhiri"
  port        = 8000
  protocol    = "HTTP"
  vpc_id      = aws_vpc.main.id
  target_type = "ip"
  health_check {
    enabled             = true
    healthy_threshold   = 2
    unhealthy_threshold = 10
    interval            = 20
    matcher             = "200"
    path                = "/api/datasetcheck"
    port                = "traffic-port"
    protocol            = "HTTP"
    timeout             = 12
  }
}

resource "aws_ecs_service" "tawhiri" {
  name                              = "tawhiri"
  cluster                           = aws_ecs_cluster.tawhiri.id
  task_definition                   = aws_ecs_task_definition.tawhiri.arn
  enable_ecs_managed_tags           = true
  health_check_grace_period_seconds = 600
  iam_role                          = "/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS"
  launch_type                       = "FARGATE"
  platform_version                  = "LATEST"
  desired_count                     = 1
  enable_execute_command            = true
  deployment_maximum_percent        = 400
  load_balancer {
    container_name   = "tawhiri"
    container_port   = 8000
    target_group_arn = aws_lb_target_group.tawhiri.arn
  }

  lifecycle {
    ignore_changes = [desired_count, task_definition]
  }

  network_configuration {
    assign_public_ip = true
    security_groups = [
      aws_security_group.tawhiri_efs.id,
      aws_security_group.tawhiri.id
    ]
    subnets = [aws_subnet.public["us-east-1b"].id]
  }
}


resource "aws_appautoscaling_target" "tawhiri" {
  service_namespace  = "ecs"
  scalable_dimension = "ecs:service:DesiredCount"
  resource_id        = "service/Tawhiri/tawhiri"
  min_capacity       = 1
  max_capacity       = 7
}

resource "aws_appautoscaling_policy" "tawhiri" {
  name               = "cpu"
  service_namespace  = aws_appautoscaling_target.tawhiri.service_namespace
  scalable_dimension = aws_appautoscaling_target.tawhiri.scalable_dimension
  resource_id        = aws_appautoscaling_target.tawhiri.resource_id
  policy_type        = "TargetTrackingScaling"

  target_tracking_scaling_policy_configuration {
    predefined_metric_specification {
      predefined_metric_type = "ECSServiceAverageCPUUtilization"
    }

    target_value       = 75
    scale_in_cooldown  = 240
    scale_out_cooldown = 240
  }
}

resource "aws_security_group" "tawhiri_efs" {
  name = "tawhiri-efs"
  ingress = [
    {
      from_port        = 2049
      to_port          = 2049
      protocol         = "tcp"
      cidr_blocks      = []
      ipv6_cidr_blocks = []
      description      = ""
      prefix_list_ids  = []
      self             = true
      security_groups  = [aws_vpc.main.default_security_group_id]
    }
  ]
  egress = [
    {
      from_port        = 0
      to_port          = 0
      protocol         = "-1"
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
      description      = ""
      prefix_list_ids  = []
      self             = false
      security_groups  = []
    }
  ]
  vpc_id = aws_vpc.main.id

  lifecycle {
    ignore_changes = [description, name]
  }

}

resource "aws_security_group" "tawhiri" {
  name = "tawhiri"
  ingress = [
    {
      from_port        = 8000
      to_port          = 8000
      protocol         = "tcp"
      cidr_blocks      = []
      ipv6_cidr_blocks = []
      description      = ""
      prefix_list_ids  = []
      self             = true
      security_groups  = [aws_security_group.tawhiri_alb.id, aws_security_group.lb.id]
    }
  ]
  egress = [
    {
      from_port        = 0
      to_port          = 0
      protocol         = "-1"
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
      description      = ""
      prefix_list_ids  = []
      self             = false
      security_groups  = []
    }
  ]
  vpc_id = aws_vpc.main.id

  lifecycle {
    ignore_changes = [description, name]
  }

}


resource "aws_security_group" "tawhiri_alb" {
  name = "tawhiri-alb"
  egress = [
    {
      from_port        = 0
      to_port          = 0
      protocol         = "-1"
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
      description      = ""
      prefix_list_ids  = []
      self             = false
      security_groups  = []
    }
  ]
  ingress = [
    {
      from_port        = 443
      to_port          = 443
      protocol         = "tcp"
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
      description      = ""
      prefix_list_ids  = []
      self             = false
      security_groups  = []
    }
  ]
  vpc_id = aws_vpc.main.id

  lifecycle {
    ignore_changes = [description, name]
  }

}



resource "aws_route53_record" "tawhiri_A" {
  name = "tawhiri"
  type = "A"
  alias {
    name                   = "dualstack.${aws_lb.ws.dns_name}."
    zone_id                = aws_lb.ws.zone_id
    evaluate_target_health = true
  }
  zone_id = aws_route53_zone.Route53HostedZone.zone_id
}

resource "aws_route53_record" "tawhiri_AAAA" {
  name = "tawhiri"
  type = "AAAA"
  alias {
    name                   = "dualstack.${aws_lb.ws.dns_name}."
    zone_id                = aws_lb.ws.zone_id
    evaluate_target_health = true
  }
  zone_id = aws_route53_zone.Route53HostedZone.zone_id
}

resource "aws_iam_role" "predictor_update_trigger_lambda" {
  path                 = "/service-role/"
  assume_role_policy   = data.aws_iam_policy_document.lambda_assume_role_policy.json
  max_session_duration = 3600
}

data "aws_iam_policy_document" "predictor_update_trigger_lambda" {
  statement {
    resources = ["*"]
    actions   = ["ecs:UpdateService"]
  }

  statement {
    resources = ["arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:*"]
    actions   = ["logs:CreateLogGroup"]
  }

  statement {
    resources = ["arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"]

    actions = [
      "logs:CreateLogStream",
      "logs:PutLogEvents",
    ]
  }
}

resource "aws_iam_role_policy" "predictor_update_trigger_lambda" {
  policy = data.aws_iam_policy_document.predictor_update_trigger_lambda.json
  role   = aws_iam_role.predictor_update_trigger_lambda.name
}

resource "aws_lambda_function" "predictor_update_trigger_lambda" {
  function_name    = "tawhiri-updater"
  handler          = "tawhiri_updater.handler"
  s3_bucket        = aws_s3_bucket_object.lambda.bucket
  s3_key           = aws_s3_bucket_object.lambda.key
  source_code_hash = data.archive_file.lambda.output_base64sha256
  publish          = true
  memory_size      = 128
  role             = aws_iam_role.predictor_update_trigger_lambda.arn
  runtime          = "python3.9"
  timeout          = 3
  tags = {
    Name = "tawhiri-updater"
  }
}

resource "aws_lambda_permission" "predictor_update_trigger_lambda" {
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.predictor_update_trigger_lambda.function_name
  principal     = "sns.amazonaws.com"
  source_arn    = "arn:aws:sns:us-east-1:123901341784:NewGFSObject"
}

resource "aws_sns_topic_subscription" "predictor_update_trigger_lambda" {
  topic_arn = "arn:aws:sns:us-east-1:123901341784:NewGFSObject"
  protocol  = "lambda"
  endpoint  = aws_lambda_function.predictor_update_trigger_lambda.arn
}

# sns subscription