Add TTN parser

lambda/ttn_helium/__init__.py

@@ -6,7 +6,7 @@ import datetime
 from email.utils import parsedate
 import os
 
-HELIUM_GW_VERSION = "2023.10.08"
+HELIUM_GW_VERSION = "2023.10.14"
 
 # Mappings between input (Helium) field names, and field names fed into SondeHub-Amateur
 FIELD_MAPPINGS = [
@@ -46,14 +46,14 @@ sns = boto3.client("sns",region_name="us-east-1")
 sns.meta.events.register('request-created.sns', set_connection_header)
 
 
-
 def post(payload):
     sns.publish(
                 TopicArn=os.getenv("HAM_SNS_TOPIC"),
                 Message=json.dumps(payload)
     )
 
-def upload(event, context):
+
+def upload_helium(event, context):
     if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
         event["body"] = base64.b64decode(event["body"])
     if (
@@ -146,14 +146,130 @@ def upload(event, context):
                 continue
     
     #print(to_sns)
-
     post(to_sns)
     return errors, warnings
 
 
-def lambda_handler(event, context):
+def upload_ttn(event, context):
+    if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
+        event["body"] = base64.b64decode(event["body"])
+    if (
+        "content-encoding" in event["headers"]
+        and event["headers"]["content-encoding"] == "gzip"
+    ):
+        event["body"] = zlib.decompress(event["body"], 16 + zlib.MAX_WBITS)
+
+    payloads = json.loads(event["body"])
+    to_sns = []
+    errors = []
+    warnings = []
+
+
+    # If only have one object, turn it into a single-entry list.
+    if type(payloads) == dict:
+        payloads = [payloads]
+
+    # Iterate over list:
+    for payload in payloads:
+
+        try:
+            telem = {
+                'software_name': 'SondeHub-Amateur TTN Gateway',
+                'software_version': HELIUM_GW_VERSION
+            }
+
+            #
+            # Extract mandatory fields.
+            #
+            # Name -> Payload Callsign
+            telem['payload_callsign'] = payload['end_device_ids']['application_ids']['application_id']
+
+            # Time
+            telem['datetime'] = payload['received_at']
+
+            # Positional and other data
+            telem_data = payload["uplink_message"]["decoded_payload"]
+
+            # Work through all accepted field names and map them
+            # into the output structure.
+            for _field in FIELD_MAPPINGS:
+                _input = _field[0]
+                _output = _field[1]
+
+                if _input in telem_data:
+                    telem[_output] = telem_data[_input]
+
+            # Position field, required by OpenSearch
+            # If lat/lon are not in the telemetry, then this will error
+            telem["position"] = f'{telem["lat"]},{telem["lon"]}'
+
+            # We also need altitude as a minimum
+            if 'alt' not in telem:
+                raise IOError("No altitude field")
+        
+        except Exception as e:
+            errors.append({
+                "error_message": f"Error parsing telemetry data - {str(e)}",
+                "payload": payload
+            })
+            continue
+
+        # Now iterate through the receiving stations
+        for hotspot in payload['uplink_message']['rx_metadata']:
+            try:
+                hotspot_telem = telem.copy()
+
+                hotspot_telem['uploader_callsign'] = hotspot['gateway_ids']['gateway_id']
+
+                # Frequency and modulation metadata is common to all packets
+                # Frequency is in Hz
+                hotspot_telem['frequency'] = float(payload['uplink_message']['settings']['frequency'])/1e6
+
+                # Construct the lora modulation details.
+                _bw = int( int(payload['uplink_message']['settings']['data_rate']['lora']['bandwidth']) / 1000)
+                _sf = int(payload['uplink_message']['settings']['data_rate']['lora']['spreading_factor'])
+                _cr = payload['uplink_message']['settings']['data_rate']['lora']['coding_rate'].replace('/','')
+                hotspot_telem['modulation'] = f"TTN (SF{_sf}BW{_bw}CR{_cr})"
+
+                # SNR and RSSI is unique to each receiver
+                hotspot_telem['snr'] = hotspot['snr']
+                hotspot_telem['rssi'] = hotspot['rssi']
+                # There is also a channel_rssi field that we could include...
+
+                # Can't seem to trust the timestamp in the per-receiver metadata
+                # Example input has some very wrong timestamps in it.
+                hotspot_telem['time_received'] = payload['received_at']
+                
+                try:
+                    hotspot_telem['uploader_position'] = f'{hotspot["location"]["latitude"]},{hotspot["location"]["longitude"]}'
+                    if 'altitude' in hotspot["location"]:
+                        hotspot_telem['uploader_alt'] = hotspot["location"]["altitude"]
+                    else:
+                        hotspot_telem['uploader_alt'] = 0
+                except:
+                    pass
+
+            
+                to_sns.append(hotspot_telem)
+
+            except Exception as e:
+                errors.append({
+                "error_message": f"Error parsing hotspot data - {str(e)}",
+                "payload": payload
+                })
+                continue
+    
+    #print(to_sns)
+    post(to_sns)
+    return errors, warnings
+
+
+def lambda_handler(event, context, ttn_source=False):
     try:
-        errors, warnings = upload(event, context)
+        if ttn_source:
+            errors, warnings = upload_ttn(event, context)
+        else:
+            errors, warnings = upload_helium(event, context)
     except zlib.error:
         return {"statusCode": 400, "body": "Could not decompress"}
     except json.decoder.JSONDecodeError:
@@ -184,5 +300,6 @@ def lambda_handler(event, context):
 
 def lambda_handler_helium(event, context):
     return lambda_handler(event, context)
+
 def lambda_handler_ttn(event, context):
-    return lambda_handler(event, context)
+    return lambda_handler(event, context, ttn_source=True)

lambda/ttn_helium/__main__.py

@@ -7,7 +7,7 @@ from io import BytesIO
 
 import sys
 
-filename = "./helium/test_data.json"
+filename = "./ttn_helium/helium_test_data.json"
 
 _f = open(filename, 'r')
 _json = json.loads(_f.read())
@@ -62,4 +62,64 @@ payload = {
     "body": bbody,
     "isBase64Encoded": True,
 }
+
 print(lambda_handler_helium(payload, {}))
+
+
+filename = "./ttn_helium/ttn_test_data.json"
+
+_f = open(filename, 'r')
+_json = json.loads(_f.read())
+
+body = _json
+
+compressed = BytesIO()
+with gzip.GzipFile(fileobj=compressed, mode='w') as f:
+    f.write(json.dumps(body).encode('utf-8'))
+compressed.seek(0)
+bbody = base64.b64encode(compressed.read()).decode("utf-8")
+
+
+
+payload = {
+    "version": "2.0",
+    "routeKey": "POST /ttn",
+    "rawPath": "/ttn",
+    "rawQueryString": "",
+    "headers": {
+        "accept": "*/*",
+        "accept-encoding": "gzip, deflate",
+        "content-encoding": "gzip",
+        "content-length": "2135",
+        "content-type": "application/json",
+        "host": "api.v2.sondehub.org",
+        "user-agent": "autorx-1.4.1-beta4",
+        "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a",
+        "x-forwarded-for": "103.107.130.22",
+        "x-forwarded-port": "443",
+        "x-forwarded-proto": "https",
+        "date": "Sun, 31 Jan 2021 00:21:45 GMT",
+    },
+    "requestContext": {
+        "accountId": "143841941773",
+        "apiId": "r03szwwq41",
+        "domainName": "api.v2.sondehub.org",
+        "domainPrefix": "api",
+        "http": {
+            "method": "POST",
+            "path": "/helium",
+            "protocol": "HTTP/1.1",
+            "sourceIp": "103.107.130.22",
+            "userAgent": "everybody-needs-to-get-a-blimp",
+        },
+        "requestId": "Z_NJvh0RoAMEJaw=",
+        "routeKey": "PUT /sondes/telemetry",
+        "stage": "$default",
+        "time": "31/Jan/2021:00:10:25 +0000",
+        "timeEpoch": 1612051825409,
+    },
+    "body": bbody,
+    "isBase64Encoded": True,
+}
+
+print(lambda_handler_ttn(payload, {}))

lambda/ttn_helium/ttn_test_data.json

@@ -0,0 +1,105 @@
+{
+    "end_device_ids": {
+        "device_id": "eui-70b3d57ed0061adc",
+        "application_ids": {
+            "application_id": "pipico-lorawan-test"
+        },
+        "dev_eui": "70B3D57ED0061ADC",
+        "join_eui": "6081F9B609A7673F",
+        "dev_addr": "260BF141"
+    },
+    "correlation_ids": [
+        "gs:uplink:01HCDRPD9W9N0VFH82N5W21CCM"
+    ],
+    "received_at": "2023-10-10T21:43:10.096176654Z",
+    "uplink_message": {
+        "session_key_id": "AYsbay8ar1O0vhiICduTrw==",
+        "f_port": 1,
+        "f_cnt": 34,
+        "frm_payload": "AYgH7jEANY4AF74CZwD6AwIB3gQACgUCAHMGAJgHcyd7CHEArwBF/7UJAgAG",
+        "decoded_payload": {
+            "AnalogInput": 0.06,
+            "accel_x": 0.175,
+            "accel_y": 0.069,
+            "accel_z": -0.075,
+            "altitude": 60.78,
+            "battery": 4.78,
+            "ext_pressure": 1010.7,
+            "heading": 152,
+            "latitude": 51.9729,
+            "longitude": 1.371,
+            "sats": 10,
+            "speed": 1.15,
+            "temp": 25
+        },
+        "rx_metadata": [
+            {
+                "gateway_ids": {
+                    "gateway_id": "eui-00800000a0004f01",
+                    "eui": "00800000A0004F01"
+                },
+                "time": "2023-10-10T21:43:09.862Z",
+                "timestamp": 1514901724,
+                "rssi": -113,
+                "channel_rssi": -113,
+                "snr": -9,
+                "location": {
+                    "latitude": 51.9673559,
+                    "longitude": 1.35287071,
+                    "source": "SOURCE_REGISTRY"
+                },
+                "uplink_token": "CiIKIAoUZXVpLTAwODAwMDAwYTAwMDRmMDESCACAAACgAE8BENyhrtIFGgwI7YqXqQYQw/HmpQMg4La3uYvc6gEqDAjtipepBhCAp4SbAw==",
+                "channel_index": 4,
+                "gps_time": "2023-10-10T21:43:09.862Z",
+                "received_at": "2023-10-10T21:43:09.884586691Z"
+            },
+            {
+                "gateway_ids": {
+                    "gateway_id": "steves-ttig868",
+                    "eui": "58A0CBFFFE801F80"
+                },
+                "time": "2023-10-10T21:43:09.846003055Z",
+                "timestamp": 2173659356,
+                "rssi": -64,
+                "channel_rssi": -64,
+                "snr": 9.25,
+                "location": {
+                    "latitude": 51.972983407143836,
+                    "longitude": 1.3708904385566714,
+                    "altitude": 15,
+                    "source": "SOURCE_REGISTRY"
+                },
+                "uplink_token": "ChwKGgoOc3RldmVzLXR0aWc4NjgSCFigy//+gB+AENzRvYwIGgwI7YqXqQYQvv3+twMg4LbGwaE/",
+                "received_at": "2023-10-10T21:43:09.901949354Z"
+            }
+        ],
+        "settings": {
+            "data_rate": {
+                "lora": {
+                    "bandwidth": 125000,
+                    "spreading_factor": 10,
+                    "coding_rate": "4/5"
+                }
+            },
+            "frequency": "867300000",
+            "timestamp": 1514901724,
+            "time": "2023-10-10T21:43:09.862Z"
+        },
+        "received_at": "2023-10-10T21:43:09.885956045Z",
+        "consumed_airtime": "0.657408s",
+        "locations": {
+            "frm-payload": {
+                "latitude": 51.9729,
+                "longitude": 1.371,
+                "altitude": 60,
+                "source": "SOURCE_GPS"
+            }
+        },
+        "network_ids": {
+            "net_id": "000013",
+            "tenant_id": "ttn",
+            "cluster_id": "eu1",
+            "cluster_address": "eu1.cloud.thethings.network"
+        }
+    }
+}