move history to use s3 backend

history/lambda_function.py

@@ -9,10 +9,11 @@ import json
 import os
 from datetime import datetime, timedelta, timezone
 import sys, traceback
+import uuid
 
 HOST = os.getenv("ES")
 # get current sondes, filter by date, location
-
+s3 = boto3.resource('s3')
 def history(event, context):
     path = "telm-*/_search"
     payload = {
@@ -49,10 +50,14 @@ def history(event, context):
     
     results = es_request(payload, path, "POST")
     output = [
-        data["1"]["hits"]["hits"][0]["_source"] 
+        {k: v for k, v in data["1"]["hits"]["hits"][0]["_source"].items() if k != 'user-agent' and k != 'upload_time_delta'}
+        
         for data in results["aggregations"]["3"]["buckets"] 
     ]
-    return json.dumps(output)
+    s3 = boto3.resource('s3')
+    object = s3.Object('sondehub-open-data', 'export/' + str(uuid.uuid4()))
+    object.put(Body=json.dumps(output).encode('utf-8'), ACL='public-read', ContentType='application/json')
+    return {"statusCode": 302, "headers": {"Location": f'https://{object.bucket_name}.s3.amazonaws.com/{object.key}'}}
 
 
 
@@ -75,7 +80,7 @@ def es_request(payload, path, method):
 if __name__ == "__main__":
     print(
         history(
-            {"pathParameters": {"serial": "S4720140"}}, {}
+            {"pathParameters": {"serial": "S3530044"}}, {}
         )
     )
 

main.tf

@@ -309,6 +309,12 @@ resource "aws_iam_role_policy" "IAMPolicy4" {
             "Effect": "Allow",
             "Action": "iot:*",
             "Resource": "*"
+        },
+        {
+            "Sid": "VisualEditor1",
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": "*"
         }
     ]
 }

query/lambda_function.py

@@ -314,13 +314,7 @@ def datanew(event, context):
             for frame in sonde["3"]["buckets"]:
                 try:
                     frame_data = frame["1"]["hits"]["hits"][0]["_source"]
-                    uploaders = {
-                        html.escape(x["_source"]['uploader_callsign']) : {
-                            "snr" : x["_source"]["snr"] if "snr" in x["_source"]  else None,
-                            "rssi" : x["_source"]["rssi"] if "rssi" in x["_source"] else None
-                        }
-                        for x in frame["1"]["hits"]["hits"]
-                    }
+                    uploaders = {}
                     
                     # Use subtype if it exists, else just use the basic type.
                     if "subtype" in frame_data:
@@ -382,7 +376,13 @@ def datanew(event, context):
                     )
                 except:
                     traceback.print_exc(file=sys.stdout)
-
+            output["positions"]["position"][-1]["callsign"] = {
+                html.escape(x["_source"]['uploader_callsign']) : {
+                    "snr" : x["_source"]["snr"] if "snr" in x["_source"]  else None,
+                    "rssi" : x["_source"]["rssi"] if "rssi" in x["_source"] else None
+                }
+                for x in frame["1"]["hits"]["hits"]
+            }
 
     # get chase cars
 
@@ -607,7 +607,8 @@ if __name__ == "__main__":
              "queryStringParameters": {
                  "type": "positions",
                  "mode": "1day",
-                 "position_id": "0"
+                 "position_id": "0",
+                 "vehicles": "S3530044"
              }
             },
             {},