From 819d78fd59da9195fb2d6ae17e09c3014641eb97 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 25 Aug 2023 14:12:19 +0930 Subject: [PATCH] Add initial test of helium gw (#119) Co-authored-by: Mark Jessop --- lambda/helium/__init__.py | 162 ++++++++++++++++++++++++++++++++++- lambda/helium/__main__.py | 65 ++++++++++++++ lambda/helium/test_data.json | 152 ++++++++++++++++++++++++++++++++ 3 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 lambda/helium/__main__.py create mode 100644 lambda/helium/test_data.json diff --git a/lambda/helium/__init__.py b/lambda/helium/__init__.py index 2f7fc36..d44aadf 100644 --- a/lambda/helium/__init__.py +++ b/lambda/helium/__init__.py @@ -1,6 +1,164 @@ import json +import boto3 +import zlib +import base64 +import datetime +from email.utils import parsedate +import os + +HELIUM_GW_VERSION = "2023.08.25" + +def set_connection_header(request, operation_name, **kwargs): + request.headers['Connection'] = 'keep-alive' + +sns = boto3.client("sns",region_name="us-east-1") +sns.meta.events.register('request-created.sns', set_connection_header) + + + +def post(payload): + sns.publish( + TopicArn=os.getenv("HAM_SNS_TOPIC"), + Message=json.dumps(payload) + ) + +def upload(event, context): + if "isBase64Encoded" in event and event["isBase64Encoded"] == True: + event["body"] = base64.b64decode(event["body"]) + if ( + "content-encoding" in event["headers"] + and event["headers"]["content-encoding"] == "gzip" + ): + event["body"] = zlib.decompress(event["body"], 16 + zlib.MAX_WBITS) + + payloads = json.loads(event["body"]) + to_sns = [] + errors = [] + warnings = [] + + + # If only have one object, turn it into a single-entry list. + if type(payloads) == dict: + payloads = [payloads] + + # Iterate over list: + for payload in payloads: + + try: + telem = { + 'software_name': 'SondeHub-Amateur Helium Gateway', + 'software_version': HELIUM_GW_VERSION + } + + # + # Extract mandatory fields. + # + # Name -> Payload Callsign + telem['payload_callsign'] = payload['name'] + + # Time + telem['datetime'] = datetime.datetime.utcfromtimestamp(payload["reported_at"]/1000.0).isoformat() + "Z" + + # Positional and other data + telem_data = payload["decoded"]["payload"] + + # Position + telem["position"] = f'{telem_data["latitude"]},{telem_data["longitude"]}' + telem["lat"] = telem_data['latitude'] + telem["lon"] = telem_data['longitude'] + telem["alt"] = telem_data['altitude'] + + # + # Other optional fields + # + if 'sats' in telem_data: + telem["sats"] = telem_data["sats"] + + if 'battery' in telem_data: + telem["batt"] = telem_data["battery"] + + if 'batt' in telem_data: + telem["batt"] = telem_data["batt"] + + if 'speed' in telem_data: + telem['speed'] = telem_data['speed'] + + # Base64-encoded raw and payload packet data + if 'raw_packet' in payload: + telem['raw'] = payload['raw_packet'] + + if 'payload' in payload: + telem['raw_payload'] = payload['payload'] + + except Exception as e: + errors.append({ + "error_message": f"Error parsing telemetry data - {str(e)}", + "payload": payload + }) + continue + + # Now iterate through the receiving stations + for hotspot in payload['hotspots']: + try: + hotspot_telem = telem.copy() + + hotspot_telem['uploader_callsign'] = hotspot['name'] + hotspot_telem['modulation'] = f"Helium ({hotspot['spreading']})" + hotspot_telem['snr'] = hotspot['snr'] + hotspot_telem['rssi'] = hotspot['rssi'] + hotspot_telem['frequency'] = hotspot['frequency'] + hotspot_telem['time_received'] = datetime.datetime.utcfromtimestamp(hotspot["reported_at"]/1000.0).isoformat() + "Z" + + try: + hotspot_telem['uploader_position'] = f'{hotspot["lat"]},{hotspot["long"]}' + hotspot_telem['uploader_alt'] = 0 + except: + pass + + + to_sns.append(hotspot_telem) + + except Exception as e: + errors.append({ + "error_message": f"Error parsing hotspot data - {str(e)}", + "payload": payload + }) + continue + + #print(to_sns) + + post(to_sns) + return errors, warnings -#place holder def lambda_handler(event, context): - print(json.dumps(event)) \ No newline at end of file + try: + errors, warnings = upload(event, context) + except zlib.error: + return {"statusCode": 400, "body": "Could not decompress"} + except json.decoder.JSONDecodeError: + return {"statusCode": 400, "body": "Not valid json"} + error_message = { + "message": "some or all payloads could not be processed or have warnings", + "errors": errors, + "warnings": warnings + } + if errors or warnings: + output = { + "statusCode": 202, + "body": json.dumps(error_message), + "headers": { + "content-type": "application/json" + } + } + print({ + "statusCode": 202, + "body": error_message, + "headers": { + "content-type": "application/json" + } + }) + return output + else: + return {"statusCode": 200, "body": "^v^ telm logged"} + diff --git a/lambda/helium/__main__.py b/lambda/helium/__main__.py new file mode 100644 index 0000000..ca41b81 --- /dev/null +++ b/lambda/helium/__main__.py @@ -0,0 +1,65 @@ +from . import * +import json +import base64 +import gzip +import uuid +from io import BytesIO + +import sys + +filename = "./helium/test_data.json" + +_f = open(filename, 'r') +_json = json.loads(_f.read()) + +body = _json + +compressed = BytesIO() +with gzip.GzipFile(fileobj=compressed, mode='w') as f: + f.write(json.dumps(body).encode('utf-8')) +compressed.seek(0) +bbody = base64.b64encode(compressed.read()).decode("utf-8") + + + +payload = { + "version": "2.0", + "routeKey": "POST /helium", + "rawPath": "/helium", + "rawQueryString": "", + "headers": { + "accept": "*/*", + "accept-encoding": "gzip, deflate", + "content-encoding": "gzip", + "content-length": "2135", + "content-type": "application/json", + "host": "api.v2.sondehub.org", + "user-agent": "autorx-1.4.1-beta4", + "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", + "x-forwarded-for": "103.107.130.22", + "x-forwarded-port": "443", + "x-forwarded-proto": "https", + "date": "Sun, 31 Jan 2021 00:21:45 GMT", + }, + "requestContext": { + "accountId": "143841941773", + "apiId": "r03szwwq41", + "domainName": "api.v2.sondehub.org", + "domainPrefix": "api", + "http": { + "method": "POST", + "path": "/helium", + "protocol": "HTTP/1.1", + "sourceIp": "103.107.130.22", + "userAgent": "everybody-needs-to-get-a-blimp", + }, + "requestId": "Z_NJvh0RoAMEJaw=", + "routeKey": "PUT /sondes/telemetry", + "stage": "$default", + "time": "31/Jan/2021:00:10:25 +0000", + "timeEpoch": 1612051825409, + }, + "body": bbody, + "isBase64Encoded": True, +} +print(lambda_handler(payload, {})) diff --git a/lambda/helium/test_data.json b/lambda/helium/test_data.json new file mode 100644 index 0000000..0f16a05 --- /dev/null +++ b/lambda/helium/test_data.json @@ -0,0 +1,152 @@ +{ + "app_eui": "6081F9D18EBE47A6", + "decoded": { + "payload": { + "accuracy": 2.5, + "altitude": 87, + "battery": 4.08, + "latitude": -34.86380010031462, + "longitude": 138.63449803796397, + "sats": 17, + "speed": 0 + }, + "status": "success" + }, + "dev_eui": "6081F936179F6236", + "devaddr": "C4000048", + "fcnt": 5, + "hotspots": [ + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "11ZLMhbdUGEnjKfz4NbZQDu6yRZume6i4rd1xHQZrATYNJ4A382", + "lat": -34.86238643402376, + "long": 138.6337803107603, + "name": "tall-crimson-cougar", + "reported_at": 1692863544622, + "rssi": -106.0, + "snr": 8.5, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "11Dw4nfrktdR7ZpVsmiV491a41fPbUBZUY2byRn4YyP5wjEEHKR", + "lat": -34.8625638088662, + "long": 138.63461613735896, + "name": "delightful-vermilion-locust", + "reported_at": 1692863544625, + "rssi": -105.0, + "snr": 6.0, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "112fnhggsq1zuKoPzBQrx6ChZz9ZfHVhWKpwSSjuGbpYBBUZMe5X", + "lat": -34.863049545718404, + "long": 138.6246860443024, + "name": "long-nylon-snail", + "reported_at": 1692863544759, + "rssi": -116.0, + "snr": -3.5, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "11zHDsZaBKGZY9gDSqesLSMq5SDb9AnRm3kfgaJNZD4yc7cd5ev", + "lat": -34.83846034981166, + "long": 138.72593370209742, + "name": "bitter-infrared-cottonmouth", + "reported_at": 1692863544759, + "rssi": -124.0, + "snr": -12.199999809265137, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "11fbAkWdaX8gugFCqW5PiiWBbh5xgeJRZm5kiohbabs1hAQYR5n", + "lat": -34.92916696507037, + "long": 138.67657464635212, + "name": "lone-tangelo-rook", + "reported_at": 1692863544759, + "rssi": -130.0, + "snr": -12.0, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "112pcZDGWgtzEVfLYAT5SCXKssGcuNenza7J3bAqbs23Tm3qszSt", + "lat": -34.92555952825897, + "long": 138.61067667012273, + "name": "exotic-honey-moose", + "reported_at": 1692863544763, + "rssi": -131.0, + "snr": -14.199999809265137, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "112p6dymRpfjC5APJGTsgYBQgc4wRTq9udarrdDbkD94VaG3wxpT", + "lat": -34.86347548370978, + "long": 138.62211153540594, + "name": "savory-jetblack-griffin", + "reported_at": 1692863544763, + "rssi": -119.0, + "snr": -2.5, + "spreading": "SF9BW125", + "status": "success" + }, + { + "channel": 7, + "frequency": 918.2, + "hold_time": 0, + "id": "11WKJZ7QDhiXP5R4HMckmZ4iUkfbKQRuoeSFHNkejTKgM8soGEv", + "lat": -34.86997577863362, + "long": 138.63593499123186, + "name": "sour-opal-hamster", + "reported_at": 1692863544763, + "rssi": -80.0, + "snr": 11.199999809265137, + "spreading": "SF9BW125", + "status": "success" + } + ], + "id": "3f2dc194-e7a2-4db6-a964-b4e8dd775258", + "metadata": { + "adr_allowed": false, + "cf_list_enabled": false, + "multi_buy": 9999, + "organization_id": "9814b595-88c9-4985-a42f-3ab2271565bb", + "preferred_hotspots": [], + "rx_delay": 1, + "rx_delay_actual": 1, + "rx_delay_state": "rx_delay_established" + }, + "name": "VK5ALG_CubeCell_Mapper_01", + "payload": "Tmp64pWjAFcA0BE=", + "payload_size": 11, + "port": 2, + "raw_packet": "QMQAAEgABQAC1UItFfr2XKni69IDqlhw", + "replay": false, + "reported_at": 1692863544622, + "type": "uplink", + "uuid": "030745c7-0348-4046-b4ce-b545344428d4" +} \ No newline at end of file