From 512e36660750d660c0860070da1561e5452c5205 Mon Sep 17 00:00:00 2001 From: Michaela Wheeler Date: Mon, 20 Dec 2021 17:02:02 +1100 Subject: [PATCH] refactor --- historic.tf | 33 +- ingestion.tf | 24 +- lambda/__init__.py | 0 lambda/es/__init__.py | 34 + .../historic_es_to_s3/__init__.py | 75 +- lambda/historic_es_to_s3/__main__.py | 23 + .../history/__init__.py | 59 +- lambda/history/__main__.py | 6 + .../predict/__init__.py | 70 +- lambda/predict/__main__.py | 7 + .../predict_updater}/README.md | 0 .../predict_updater/__init__.py | 102 +- lambda/predict_updater/__main__.py | 43 + .../query/__init__.py | 120 +- lambda/query/__main__.py | 66 + .../queue_data_update/__init__.py | 50 +- lambda/queue_data_update/__main__.py | 2 + .../recovered/__init__.py | 116 +- lambda/recovered/__main__.py | 60 + .../reverse-predict/__init__.py | 72 +- lambda/reverse-predict/__main__.py | 19 + lambda/sign-websocket/__init__.py | 2 + lambda/sign-websocket/__main__.py | 2 + lambda/sns-to-mqtt/__init__.py | 72 + .../sns-to-mqtt/__main__.py | 74 +- lambda/sns-to-mqtt/vendor/paho/__init__.py | 0 .../sns-to-mqtt/vendor/paho/mqtt/__init__.py | 5 + lambda/sns-to-mqtt/vendor/paho/mqtt/client.py | 3858 +++++++++++++++++ .../sns-to-mqtt/vendor/paho/mqtt/matcher.py | 78 + .../vendor/paho/mqtt/packettypes.py | 43 + .../vendor/paho/mqtt/properties.py | 409 ++ .../sns-to-mqtt/vendor/paho/mqtt/publish.py | 232 + .../vendor/paho/mqtt/reasoncodes.py | 191 + .../sns-to-mqtt/vendor/paho/mqtt/subscribe.py | 266 ++ .../vendor/paho/mqtt/subscribeoptions.py | 110 + .../sonde-api-to-iot-core/__init__.py | 44 - lambda/sonde-api-to-iot-core/__main__.py | 42 + .../sqs-to-elk/__init__.py | 31 +- lambda/sqs-to-elk/__main__.py | 15 + lambda/station-api-to-iot-core/__init__.py | 56 + lambda/station-api-to-iot-core/__main__.py | 43 + lambda/tawhiri-updater/__init__.py | 21 + .../tawhiri-updater/__main__.py | 25 +- main.tf | 6 + predictor.tf | 48 +- query.tf | 40 +- recovered.tf | 23 +- sign-websocket/lambda_function.py | 13 - sonde-to-s3/README.md | 1 - sonde-to-s3/lambda_function.py | 106 - sqs_to_elk.tf | 12 +- station-api-to-iot-core/lambda_function.py | 147 - websockets.tf | 12 +- 53 files changed, 5802 insertions(+), 1206 deletions(-) create mode 100644 lambda/__init__.py create mode 100644 lambda/es/__init__.py rename historic/historic_es_to_s3/index.py => lambda/historic_es_to_s3/__init__.py (69%) create mode 100644 lambda/historic_es_to_s3/__main__.py rename history/lambda_function.py => lambda/history/__init__.py (54%) create mode 100644 lambda/history/__main__.py rename predict/lambda_function.py => lambda/predict/__init__.py (60%) create mode 100644 lambda/predict/__main__.py rename {predict_updater => lambda/predict_updater}/README.md (100%) rename predict_updater/lambda_function.py => lambda/predict_updater/__init__.py (90%) create mode 100644 lambda/predict_updater/__main__.py rename query/lambda_function.py => lambda/query/__init__.py (79%) create mode 100644 lambda/query/__main__.py rename historic/queue_data_update/index.py => lambda/queue_data_update/__init__.py (51%) create mode 100644 lambda/queue_data_update/__main__.py rename recovered/lambda_function.py => lambda/recovered/__init__.py (74%) create mode 100644 lambda/recovered/__main__.py rename reverse-predict/lambda_function.py => lambda/reverse-predict/__init__.py (64%) create mode 100644 lambda/reverse-predict/__main__.py create mode 100644 lambda/sign-websocket/__init__.py create mode 100644 lambda/sign-websocket/__main__.py create mode 100644 lambda/sns-to-mqtt/__init__.py rename sns-to-mqtt/lambda_function.py => lambda/sns-to-mqtt/__main__.py (56%) create mode 100644 lambda/sns-to-mqtt/vendor/paho/__init__.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/__init__.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/client.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/matcher.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/packettypes.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/properties.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/publish.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/reasoncodes.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/subscribe.py create mode 100644 lambda/sns-to-mqtt/vendor/paho/mqtt/subscribeoptions.py rename sonde-api-to-iot-core/lambda_function.py => lambda/sonde-api-to-iot-core/__init__.py (83%) create mode 100644 lambda/sonde-api-to-iot-core/__main__.py rename sqs-to-elk/lambda_function.py => lambda/sqs-to-elk/__init__.py (54%) create mode 100644 lambda/sqs-to-elk/__main__.py create mode 100644 lambda/station-api-to-iot-core/__init__.py create mode 100644 lambda/station-api-to-iot-core/__main__.py create mode 100644 lambda/tawhiri-updater/__init__.py rename tawhiri-updater/index.py => lambda/tawhiri-updater/__main__.py (70%) delete mode 100644 sign-websocket/lambda_function.py delete mode 100644 sonde-to-s3/README.md delete mode 100644 sonde-to-s3/lambda_function.py delete mode 100644 station-api-to-iot-core/lambda_function.py diff --git a/historic.tf b/historic.tf index 3e4e63e..400ab67 100644 --- a/historic.tf +++ b/historic.tf @@ -1,14 +1,4 @@ -data "archive_file" "historic_to_s3" { - type = "zip" - source_file = "historic/historic_es_to_s3/index.py" - output_path = "${path.module}/build/historic_to_s3.zip" -} -data "archive_file" "queue_data_update" { - type = "zip" - source_file = "historic/queue_data_update/index.py" - output_path = "${path.module}/build/queue_data_update.zip" -} resource "aws_iam_role" "historic" { path = "/service-role/" @@ -74,9 +64,9 @@ EOF resource "aws_lambda_function" "historic_to_s3" { function_name = "historic_to_s3" - handler = "index.handler" - filename = "${path.module}/build/historic_to_s3.zip" - source_code_hash = data.archive_file.historic_to_s3.output_base64sha256 + handler = "historic_es_to_s3.handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 3096 role = aws_iam_role.historic.arn @@ -91,9 +81,9 @@ resource "aws_lambda_function" "historic_to_s3" { } resource "aws_lambda_function" "queue_data_update" { function_name = "queue_data_update" - handler = "index.handler" - filename = "${path.module}/build/queue_data_update.zip" - source_code_hash = data.archive_file.queue_data_update.output_base64sha256 + handler = "queue_data_update.handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 256 role = aws_iam_role.historic.arn @@ -216,17 +206,12 @@ EOF role = aws_iam_role.history.name } -data "archive_file" "history" { - type = "zip" - source_file = "history/lambda_function.py" - output_path = "${path.module}/build/history.zip" -} resource "aws_lambda_function" "history" { function_name = "history" - handler = "lambda_function.history" - filename = "${path.module}/build/history.zip" - source_code_hash = data.archive_file.history.output_base64sha256 + handler = "history.history" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 512 role = aws_iam_role.basic_lambda_role.arn diff --git a/ingestion.tf b/ingestion.tf index 712ecbd..479301d 100644 --- a/ingestion.tf +++ b/ingestion.tf @@ -1,20 +1,8 @@ -data "archive_file" "api_to_iot" { - type = "zip" - source_dir = "sonde-api-to-iot-core/" - output_path = "${path.module}/build/sonde-api-to-iot-core.zip" -} - -data "archive_file" "station_api_to_iot" { - type = "zip" - source_file = "station-api-to-iot-core/lambda_function.py" - output_path = "${path.module}/build/station-api-to-iot-core.zip" -} - resource "aws_lambda_function" "upload_telem" { function_name = "sonde-api-to-iot-core" - handler = "lambda_function.lambda_handler" - filename = "${path.module}/build/sonde-api-to-iot-core.zip" - source_code_hash = data.archive_file.api_to_iot.output_base64sha256 + handler = "sonde-api-to-iot-core.lambda_handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.basic_lambda_role.arn @@ -30,9 +18,9 @@ resource "aws_lambda_function" "upload_telem" { resource "aws_lambda_function" "station" { function_name = "station-api-to-iot-core" - handler = "lambda_function.lambda_handler" - filename = "${path.module}/build/station-api-to-iot-core.zip" - source_code_hash = data.archive_file.station_api_to_iot.output_base64sha256 + handler = "station-api-to-iot-core.lambda_handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.basic_lambda_role.arn diff --git a/lambda/__init__.py b/lambda/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/es/__init__.py b/lambda/es/__init__.py new file mode 100644 index 0000000..3bc3548 --- /dev/null +++ b/lambda/es/__init__.py @@ -0,0 +1,34 @@ +import boto3 +import gzip +from botocore.awsrequest import AWSRequest +from botocore.endpoint import URLLib3Session +from botocore.auth import SigV4Auth +from io import BytesIO +import json +import os + +es_session = URLLib3Session() +ES_HOST = os.getenv("ES") + +def request(payload, path, method, params=None): + + compressed = BytesIO() + with gzip.GzipFile(fileobj=compressed, mode='w') as f: + f.write(payload.encode('utf-8')) + payload = compressed.getvalue() + + headers = {"Host": ES_HOST, "Content-Type": "application/json", + "Content-Encoding": "gzip"} + + request = AWSRequest( + method=method, url=f"https://{ES_HOST}/{path}", data=payload, headers=headers, params=params + ) + SigV4Auth(boto3.Session().get_credentials(), + "es", "us-east-1").add_auth(request) + + r = es_session.send(request.prepare()) + + if r.status_code != 200 and r.status_code != 201: + raise RuntimeError + + return json.loads(r.text) diff --git a/historic/historic_es_to_s3/index.py b/lambda/historic_es_to_s3/__init__.py similarity index 69% rename from historic/historic_es_to_s3/index.py rename to lambda/historic_es_to_s3/__init__.py index 65e106a..cf9fc5e 100644 --- a/historic/historic_es_to_s3/index.py +++ b/lambda/historic_es_to_s3/__init__.py @@ -1,54 +1,13 @@ import json -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import boto3 -import botocore.credentials -import os import gzip from botocore.exceptions import ClientError -from io import BytesIO +import es -HOST = os.getenv("ES") BUCKET = "sondehub-history" s3 = boto3.resource('s3') -http_session = URLLib3Session() - -from multiprocessing import Process - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) - -def es_request(payload, path, method, params=None): - # get aws creds - session = boto3.Session() - - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(payload.encode('utf-8')) - payload = compressed.getvalue() - - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - - request = AWSRequest( - method=method, url=f"https://{HOST}/{path}", data=payload, headers=headers, params=params - ) - SigV4Auth(boto3.Session().get_credentials(), - "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - return json.loads(r.text) - - def fetch_es(serial): payload = { "size": 10000, @@ -74,7 +33,7 @@ def fetch_es(serial): } } data = [] - response = es_request(json.dumps(payload), + response = es.request(json.dumps(payload), "telm-*/_search", "POST", params={"scroll": "1m"}) try: data += [x["_source"] for x in response['hits']['hits']] @@ -84,13 +43,13 @@ def fetch_es(serial): scroll_id = response['_scroll_id'] scroll_ids = [scroll_id] while response['hits']['hits']: - response = es_request(json.dumps({"scroll": "1m", "scroll_id": scroll_id }), + response = es.request(json.dumps({"scroll": "1m", "scroll_id": scroll_id }), "_search/scroll", "POST") scroll_id = response['_scroll_id'] scroll_ids.append(scroll_id) data += [x["_source"] for x in response['hits']['hits']] for scroll_id in scroll_ids: - scroll_delete = es_request(json.dumps({"scroll_id": scroll_id }), + scroll_delete = es.request(json.dumps({"scroll_id": scroll_id }), "_search/scroll", "DELETE") print(scroll_delete) return data @@ -184,7 +143,7 @@ def fetch_launch_sites(): } } - response = es_request(json.dumps(payload), + response = es.request(json.dumps(payload), "reverse-prediction-*/_search", "POST") data = { x['key'] : x for x in response['aggregations']['2']['buckets']} output = {} @@ -269,27 +228,3 @@ def handler(event, context): write_s3(serial, data, launch_sites) print(f"{serial} done") - -if __name__ == "__main__": - print(handler( - { - "Records": [ - { - "messageId": "3b5853b3-369c-40bf-8746-130c918fbb5c", - "receiptHandle": "AQEBg+/MIA2rSNmlrpXvk7pbi26kgIzqhairaHWGSpMgLzf2T54PLUmG+eG6CDOv35e42scDH0gppmS9RTQVu8D161oHYohhd1+0S4LtFJdgXr3At86NBIky5+y1A/wVuUm1FNQSvGKIDRgBhCgcCzLkEMNYIWWeDmg2ez2SCPu/3hmY5sc7eC32uqz5es9PspgQXOTykmoNv/q37iy2RBRDdu51Tq7yIxEr+Us3qkQrddAJ7qsA0l09aRL+/PJe1V/7MMN3CFyBATpRP/G3Gjn0Iuu4i2UhkRx2pF+0Hj8yhhHbqTMcw5sbbGIWMdsMXFQKUCHNx6HPkbuwIWo0TsINQjY7IXeZM/mNq65xC4avSlctJ/9BMzOBtFwbnRPZfHmlS5Al2nF1Vu3RecFGbTm1nQ==", - "body": "R0230678", - "attributes": { - "ApproximateReceiveCount": "1", - "SentTimestamp": "1627873604999", - "SenderId": "AROASC7NF3EG5DNHEPSYZ:queue_data_update", - "ApproximateFirstReceiveTimestamp": "1627873751266" - }, - "messageAttributes": {}, - "md5OfBody": "b3d67879b6a2e7f3abd62d404e53f71f", - "md5OfMessageAttributes": None, - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:143841941773:update-history", - "awsRegion": "us-east-1" - } - ] -}, {})) diff --git a/lambda/historic_es_to_s3/__main__.py b/lambda/historic_es_to_s3/__main__.py new file mode 100644 index 0000000..dd3f08b --- /dev/null +++ b/lambda/historic_es_to_s3/__main__.py @@ -0,0 +1,23 @@ +from . import * +print(handler( + { + "Records": [ + { + "messageId": "3b5853b3-369c-40bf-8746-130c918fbb5c", + "receiptHandle": "AQEBg+/MIA2rSNmlrpXvk7pbi26kgIzqhairaHWGSpMgLzf2T54PLUmG+eG6CDOv35e42scDH0gppmS9RTQVu8D161oHYohhd1+0S4LtFJdgXr3At86NBIky5+y1A/wVuUm1FNQSvGKIDRgBhCgcCzLkEMNYIWWeDmg2ez2SCPu/3hmY5sc7eC32uqz5es9PspgQXOTykmoNv/q37iy2RBRDdu51Tq7yIxEr+Us3qkQrddAJ7qsA0l09aRL+/PJe1V/7MMN3CFyBATpRP/G3Gjn0Iuu4i2UhkRx2pF+0Hj8yhhHbqTMcw5sbbGIWMdsMXFQKUCHNx6HPkbuwIWo0TsINQjY7IXeZM/mNq65xC4avSlctJ/9BMzOBtFwbnRPZfHmlS5Al2nF1Vu3RecFGbTm1nQ==", + "body": "R0230678", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1627873604999", + "SenderId": "AROASC7NF3EG5DNHEPSYZ:queue_data_update", + "ApproximateFirstReceiveTimestamp": "1627873751266" + }, + "messageAttributes": {}, + "md5OfBody": "b3d67879b6a2e7f3abd62d404e53f71f", + "md5OfMessageAttributes": None, + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:143841941773:update-history", + "awsRegion": "us-east-1" + } + ] +}, {})) \ No newline at end of file diff --git a/history/lambda_function.py b/lambda/history/__init__.py similarity index 54% rename from history/lambda_function.py rename to lambda/history/__init__.py index 6c16ccf..ae36a16 100644 --- a/history/lambda_function.py +++ b/lambda/history/__init__.py @@ -1,34 +1,9 @@ import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import json -import os from datetime import datetime, timedelta, timezone -import sys, traceback import uuid -import gzip -from io import BytesIO +import es -# TODO , HEAD S3 object, if it's less than 24 hours check ES, else 302 to bucket - -HOST = os.getenv("ES") -# get current sondes, filter by date, location - -from multiprocessing import Process - -http_session = URLLib3Session() - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) def history(event, context): @@ -82,7 +57,7 @@ def history(event, context): }, } - results = es_request(payload, path, "POST") + results = es.request(json.dumps(payload), path, "POST") output = [ {k: v for k, v in data["1"]["hits"]["hits"][0]["_source"].items() if k != 'user-agent' and k != 'upload_time_delta'} @@ -95,35 +70,5 @@ def history(event, context): -def es_request(payload, path, method): - # get aws creds - session = boto3.Session() - - params = json.dumps(payload) - - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - - - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - return json.loads(r.text) - - -if __name__ == "__main__": - print( - history( - {"pathParameters": {"serial": "T1510227"}}, {} - ) - ) - - diff --git a/lambda/history/__main__.py b/lambda/history/__main__.py new file mode 100644 index 0000000..d7c6c62 --- /dev/null +++ b/lambda/history/__main__.py @@ -0,0 +1,6 @@ +from . import * +print( + history( + {"pathParameters": {"serial": "T1510227"}}, {} + ) +) \ No newline at end of file diff --git a/predict/lambda_function.py b/lambda/predict/__init__.py similarity index 60% rename from predict/lambda_function.py rename to lambda/predict/__init__.py index 6c6f8d6..95320d7 100644 --- a/predict/lambda_function.py +++ b/lambda/predict/__init__.py @@ -1,34 +1,11 @@ -import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import json -import os -from datetime import datetime, timedelta, timezone -import sys -import traceback -import http.client -import math import logging import gzip from io import BytesIO import base64 +import es -HOST = os.getenv("ES") -http_session = URLLib3Session() -from multiprocessing import Process - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) def predict(event, context): @@ -109,7 +86,7 @@ def predict(event, context): # for single sonde allow longer predictions payload['query']['bool']['filter'].pop(0) logging.debug("Start ES Request") - results = es_request(payload, path, "GET") + results = es.request(json.dumps(payload), path, "GET") logging.debug("Finished ES Request") output = [] for sonde in results['aggregations']['2']['buckets']: @@ -144,46 +121,3 @@ def predict(event, context): } } - -def es_request(payload, path, method): - # get aws creds - session = boto3.Session() - - params = json.dumps(payload) - - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - - headers = {"Host": HOST, "Content-Type": "application/json", - "Content-Encoding": "gzip"} - request = AWSRequest( - method=method, url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), - "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - return json.loads(r.text) - - -if __name__ == "__main__": - # print(get_sondes({"queryStringParameters":{"lat":"-28.22717","lon":"153.82996","distance":"50000"}}, {})) - # mode: 6hours - # type: positions - # format: json - # max_positions: 0 - # position_id: 0 - # vehicles: RS_*;*chase - print(predict( - {"queryStringParameters": { - "vehicles": "" - }}, {} - )) - - -# get list of sondes, serial, lat,lon, alt - # and current rate -# for each one, request http://predict.cusf.co.uk/api/v1/?launch_latitude=-37.8136&launch_longitude=144.9631&launch_datetime=2021-02-22T00:15:18.513413Z&launch_altitude=30000&ascent_rate=5&burst_altitude=30000.1&descent_rate=5 - # have to set the burst alt slightly higher than the launch diff --git a/lambda/predict/__main__.py b/lambda/predict/__main__.py new file mode 100644 index 0000000..9e4b3c4 --- /dev/null +++ b/lambda/predict/__main__.py @@ -0,0 +1,7 @@ +from . import * + +print(predict( + {"queryStringParameters": { + "vehicles": "" + }}, {} + )) \ No newline at end of file diff --git a/predict_updater/README.md b/lambda/predict_updater/README.md similarity index 100% rename from predict_updater/README.md rename to lambda/predict_updater/README.md diff --git a/predict_updater/lambda_function.py b/lambda/predict_updater/__init__.py similarity index 90% rename from predict_updater/lambda_function.py rename to lambda/predict_updater/__init__.py index ece01ee..6e3d841 100644 --- a/predict_updater/lambda_function.py +++ b/lambda/predict_updater/__init__.py @@ -1,37 +1,13 @@ -from json.encoder import py_encode_basestring_ascii -import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import json -import os -from datetime import datetime, timedelta, timezone -import sys, traceback +from datetime import datetime import http.client import math import logging -import gzip -from io import BytesIO from math import radians, degrees, sin, cos, atan2, sqrt, pi - -HOST = os.getenv("ES") -http_session = URLLib3Session() - -from multiprocessing import Process - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) +import es + -# # FLIGHT PROFILE DEFAULTS # @@ -408,7 +384,7 @@ def get_reverse_predictions(): } } logging.debug("Start ES Request") - results = es_request(json.dumps(payload), path, "POST") + results = es.request(json.dumps(payload), path, "POST") logging.debug("Finished ES Request") return { x['_source']['serial'] : x['_source'] for x in results['hits']['hits']} @@ -427,7 +403,7 @@ def get_launch_sites(): "size": 10000 } logging.debug("Start ES Request") - results = es_request(json.dumps(payload), path, "POST") + results = es.request(json.dumps(payload), path, "POST") logging.debug("Finished ES Request") return {x['_source']['station']: x['_source'] for x in results['hits']['hits']} @@ -438,7 +414,7 @@ def bulk_upload_es(index_prefix,payloads): body += "{\"index\":{}}\n" + json.dumps(payload) + "\n" body += "\n" date_prefix = datetime.now().strftime("%Y-%m") - result = es_request(body, f"{index_prefix}-{date_prefix}/_doc/_bulk", "POST") + result = es.request(body, f"{index_prefix}-{date_prefix}/_doc/_bulk", "POST") if 'errors' in result and result['errors'] == True: error_types = [x['index']['error']['type'] for x in result['items'] if 'error' in x['index']] # get all the error types @@ -583,7 +559,7 @@ def predict(event, context): "size": 0 } logging.debug("Start ES Request") - results = es_request(json.dumps(payload), path, "GET") + results = es.request(json.dumps(payload), path, "GET") logging.debug("Finished ES Request") @@ -821,69 +797,5 @@ def predict(event, context): logging.debug("Finished") return -def es_request(params, path, method): - # get aws creds - session = boto3.Session() - - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method=method, url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - - if r.status_code != 200: - raise RuntimeError - return json.loads(r.text) - - -if __name__ == "__main__": - - # Predictor test - # conn = http.client.HTTPSConnection("tawhiri.v2.sondehub.org") - # _now = datetime.utcnow().isoformat() + "Z" - - # _ascent = get_standard_prediction(conn, _now, -34.0, 138.0, 10.0, burst_altitude=26000) - # print(f"Got {len(_ascent)} data points for ascent prediction.") - # _descent = get_standard_prediction(conn, _now, -34.0, 138.0, 24000.0, burst_altitude=24000.5) - # print(f"Got {len(_descent)} data points for descent prediction.") - - # test = predict( - # {},{} - # ) - #print(get_launch_sites()) - #print(get_reverse_predictions()) - # for _serial in test: - # print(f"{_serial['serial']}: {len(_serial['data'])}") - - - logging.basicConfig( - format="%(asctime)s %(levelname)s:%(message)s", level=logging.DEBUG - ) - - print(predict( - {},{} - )) - # bulk_upload_es("reverse-prediction",[{ - # "datetime" : "2021-10-04", - # "data" : { }, - # "serial" : "R12341234", - # "station" : "-2", - # "subtype" : "RS41-SGM", - # "ascent_rate" : "5", - # "alt" : 1000, - # "position" : [ - # 1, - # 2 - # ], - # "type" : "RS41" - # }] - # ) - diff --git a/lambda/predict_updater/__main__.py b/lambda/predict_updater/__main__.py new file mode 100644 index 0000000..c9ab7c4 --- /dev/null +++ b/lambda/predict_updater/__main__.py @@ -0,0 +1,43 @@ +from . import * + +# Predictor test +# conn = http.client.HTTPSConnection("tawhiri.v2.sondehub.org") +# _now = datetime.utcnow().isoformat() + "Z" + +# _ascent = get_standard_prediction(conn, _now, -34.0, 138.0, 10.0, burst_altitude=26000) +# print(f"Got {len(_ascent)} data points for ascent prediction.") +# _descent = get_standard_prediction(conn, _now, -34.0, 138.0, 24000.0, burst_altitude=24000.5) +# print(f"Got {len(_descent)} data points for descent prediction.") + +# test = predict( +# {},{} +# ) +#print(get_launch_sites()) +#print(get_reverse_predictions()) +# for _serial in test: +# print(f"{_serial['serial']}: {len(_serial['data'])}") + + +logging.basicConfig( + format="%(asctime)s %(levelname)s:%(message)s", level=logging.DEBUG +) + +print(predict( + {},{} + )) +# bulk_upload_es("reverse-prediction",[{ +# "datetime" : "2021-10-04", +# "data" : { }, +# "serial" : "R12341234", +# "station" : "-2", +# "subtype" : "RS41-SGM", +# "ascent_rate" : "5", +# "alt" : 1000, +# "position" : [ +# 1, +# 2 +# ], +# "type" : "RS41" +# }] +# ) + diff --git a/query/lambda_function.py b/lambda/query/__init__.py similarity index 79% rename from query/lambda_function.py rename to lambda/query/__init__.py index 8d765e8..77f2623 100644 --- a/query/lambda_function.py +++ b/lambda/query/__init__.py @@ -1,35 +1,9 @@ -import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import json -import os from datetime import datetime, timedelta, timezone -import sys, traceback -import re -import html import base64 import gzip from io import BytesIO - -from multiprocessing import Process - -http_session = URLLib3Session() - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) - -HOST = os.getenv("ES") -# get current sondes, filter by date, location - +import es def get_sondes(event, context): path = "telm-*/_search" @@ -93,7 +67,7 @@ def get_sondes(event, context): {"range": {"datetime": {"gte": "now-1d", "lte": "now+1m"}}} ) - results = es_request(payload, path, "POST") + results = es.request(json.dumps(payload), path, "POST") buckets = results["aggregations"]["2"]["buckets"] sondes = { bucket["1"]["hits"]["hits"][0]["_source"]["serial"]: bucket["1"]["hits"][ @@ -206,7 +180,7 @@ def get_telem(event, context): } } ) - results = es_request(payload, path, "POST") + results = es.request(json.dumps(payload), path, "POST") output = { sonde["key"]: { data["key_as_string"]: dict(data["1"]["hits"]["hits"][0]["_source"], @@ -335,7 +309,7 @@ def get_listener_telemetry(event, context): } } ) - results = es_request(payload, path, "POST") + results = es.request(json.dumps(payload), path, "POST") output = { sonde["key"]: { data["key_as_string"]: data["1"]["hits"]["hits"][0]["_source"] @@ -389,7 +363,7 @@ def get_sites(event, context): } } ) - results = es_request(payload, path, "POST") + results = es.request(json.dumps(payload), path, "POST") output = {x['_source']['station']: x['_source'] for x in results['hits']['hits']} compressed = BytesIO() @@ -410,89 +384,5 @@ def get_sites(event, context): } -def es_request(payload, path, method): - # get aws creds - session = boto3.Session() - - params = json.dumps(payload) - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - return json.loads(r.text) -if __name__ == "__main__": - #print(get_sondes({"queryStringParameters":{"lat":"-32.7933","lon":"151.8358","distance":"5000", "last":"604800"}}, {})) - # mode: 6hours - # type: positions - # format: json - # max_positions: 0 - # position_id: 0 - # vehicles: RS_*;*chase -# print( -# datanew( -# { -# "queryStringParameters": { -# "mode": "single", -# "format": "json", -# "position_id": "S1443103-2021-07-20T12:46:19.040000Z" -# } -# }, -# {}, -# ) -# ) - # print(get_sites({},{})) - print( - get_telem( - { - "queryStringParameters": { - "duration": "1d", - "serial": "S4430086" - }},{} - - ) - ) - # print ( - # get_chase( - # {"queryStringParameters": { - # "duration": "1d" - # } - # }, - # {} - # ) - # ) - - - # print( - # datanew( - # { - # "queryStringParameters": { - # "type": "positions", - # "mode": "3hours", - # "position_id": "0" - # } - # }, - # {}, - # ) - # ) - # print( - # get_telem( - # { - # "queryStringParameters":{ - # # "serial": "S3210639", - # "duration": "3h", - # # "datetime": "2021-07-26T06:49:29.001000Z" - # } - # }, {} - # ) - # ) - diff --git a/lambda/query/__main__.py b/lambda/query/__main__.py new file mode 100644 index 0000000..ab4d13e --- /dev/null +++ b/lambda/query/__main__.py @@ -0,0 +1,66 @@ +from . import * + +#print(get_sondes({"queryStringParameters":{"lat":"-32.7933","lon":"151.8358","distance":"5000", "last":"604800"}}, {})) +# mode: 6hours +# type: positions +# format: json +# max_positions: 0 +# position_id: 0 +# vehicles: RS_*;*chase +# print( +# datanew( +# { +# "queryStringParameters": { +# "mode": "single", +# "format": "json", +# "position_id": "S1443103-2021-07-20T12:46:19.040000Z" +# } +# }, +# {}, +# ) +# ) +# print(get_sites({},{})) +print( + get_telem( + { + "queryStringParameters": { + "duration": "1d", + # "serial": "S4430086" + }},{} + + ) + ) +# print ( +# get_chase( +# {"queryStringParameters": { +# "duration": "1d" +# } +# }, +# {} +# ) +# ) + + +# print( +# datanew( +# { +# "queryStringParameters": { +# "type": "positions", +# "mode": "3hours", +# "position_id": "0" +# } +# }, +# {}, +# ) +# ) +# print( +# get_telem( +# { +# "queryStringParameters":{ +# # "serial": "S3210639", +# "duration": "3h", +# # "datetime": "2021-07-26T06:49:29.001000Z" +# } +# }, {} +# ) +# ) diff --git a/historic/queue_data_update/index.py b/lambda/queue_data_update/__init__.py similarity index 51% rename from historic/queue_data_update/index.py rename to lambda/queue_data_update/__init__.py index 2a99e12..e7d2de5 100644 --- a/historic/queue_data_update/index.py +++ b/lambda/queue_data_update/__init__.py @@ -1,30 +1,8 @@ import json import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth - -import zlib -import base64 -import datetime import os -import gzip -from io import BytesIO +import es -from multiprocessing import Process - -http_session = URLLib3Session() - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) HOST = os.getenv("ES") @@ -35,27 +13,6 @@ def batch(iterable, n=1): for ndx in range(0, l, n): yield iterable[ndx:min(ndx + n, l)] -def es_request(payload, path, method): - session = boto3.Session() - - params = json.dumps(payload) - - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - - request = AWSRequest( - method=method, url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), - "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - return json.loads(r.text) - def handler(event, context): query = { @@ -88,7 +45,7 @@ def handler(event, context): } } - results = es_request(query, "telm-*/_search", "POST") + results = es.request(json.dumps(query), "telm-*/_search", "POST") serials = [ x['key'] for x in results['aggregations']['serials']['buckets'] ] for serial_batch in batch(serials, 10): sqs.send_message_batch( @@ -103,7 +60,6 @@ def handler(event, context): return [ x['key'] for x in results['aggregations']['serials']['buckets'] ] #TODO add to SQS queue -if __name__ == "__main__": - print(handler({}, {})) + # this script will find list of sondes seen in the last 48 hours and add them to the queue to be updated (including the first and last date they were seen) diff --git a/lambda/queue_data_update/__main__.py b/lambda/queue_data_update/__main__.py new file mode 100644 index 0000000..d8d89ca --- /dev/null +++ b/lambda/queue_data_update/__main__.py @@ -0,0 +1,2 @@ +from . import * +print(handler({}, {})) \ No newline at end of file diff --git a/recovered/lambda_function.py b/lambda/recovered/__init__.py similarity index 74% rename from recovered/lambda_function.py rename to lambda/recovered/__init__.py index 556bd0a..72603a8 100644 --- a/recovered/lambda_function.py +++ b/lambda/recovered/__init__.py @@ -1,57 +1,9 @@ -from multiprocessing import Process import json -import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import zlib import base64 from datetime import datetime, timedelta -import os -from io import BytesIO -import gzip - -HOST = os.getenv("ES") -http_session = URLLib3Session() - -# get aws creds -aws_session = boto3.Session() - - -def mirror(path, params): - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", - "Content-Type": "application/json", "Content-Encoding": "gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(aws_session.get_credentials(), - "es", "us-east-1").add_auth(request) - session = URLLib3Session() - r = session.send(request.prepare()) - - -def es_request(payload, path, method): - - params = json.dumps(payload) - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - - headers = {"Host": HOST, "Content-Type": "application/json", - "Content-Encoding": "gzip"} - - request = AWSRequest( - method="POST", url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(aws_session.get_credentials(), - "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,params)).start() - r = http_session.send(request.prepare()) - return json.loads(r.text) - +import es def getSonde(serial): query = { @@ -90,7 +42,7 @@ def getSonde(serial): } } } - results = es_request(query, "telm-*/_search", "POST") + results = es.request(json.dumps(query), "telm-*/_search", "POST") return results["aggregations"]["1"]["hits"]["hits"] @@ -132,7 +84,7 @@ def getRecovered(serial): } } } - results = es_request(query, "recovered*/_search", "POST") + results = es.request(json.dumps(query), "recovered*/_search", "POST") return results["aggregations"]["1"]["hits"]["hits"] @@ -175,7 +127,7 @@ def put(event, context): recovered['position'] = [recovered['lon'], recovered['lat']] - result = es_request(recovered, "recovered/_doc", "POST") + result = es.request(json.dumps(recovered), "recovered/_doc", "POST") # add in elasticsearch extra position field return {"statusCode": 200, "body": json.dumps({"message": "telm logged. Have a good day ^_^"})} @@ -282,7 +234,7 @@ def get(event, context): } if serials: query["query"]["bool"]["minimum_should_match"] = 1 - results = es_request(query, "recovered*/_search", "POST") + results = es.request(json.dumps(query), "recovered*/_search", "POST") output = [x['1']['hits']['hits'][0]["_source"] for x in results['aggregations']['2']['buckets']] return {"statusCode": 200, "body": json.dumps(output)} @@ -398,7 +350,7 @@ def stats(event, context): } } } - results = es_request(query, "recovered*/_search", "POST") + results = es.request(json.dumps(query), "recovered*/_search", "POST") output = { "total": 0, @@ -435,59 +387,3 @@ def stats(event, context): return {"statusCode": 200, "body": json.dumps(output)} -if __name__ == "__main__": - payload = { - "version": "2.0", - "routeKey": "PUT /recovered", - "rawPath": "/recovered", - "rawQueryString": "", - "queryStringParameters": { - # "datetime": "2021-12-20T00:00", - "duration": 1000000 - }, - "headers": { - "accept": "*/*", - "accept-encoding": "deflate", - "content-encoding": "", - "content-length": "2135", - "content-type": "application/json", - "host": "api.v2.sondehub.org", - "user-agent": "autorx-1.4.1-beta4", - "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", - "x-forwarded-for": "103.107.130.22", - "x-forwarded-port": "443", - "x-forwarded-proto": "https", - "date": "Sun, 31 Jan 2021 00:21:45 GMT", - }, - "requestContext": { - "accountId": "143841941773", - "apiId": "r03szwwq41", - "domainName": "api.v2.sondehub.org", - "domainPrefix": "api", - "http": { - "method": "PUT", - "path": "/sondes/telemetry", - "protocol": "HTTP/1.1", - "sourceIp": "103.107.130.22", - "userAgent": "autorx-1.4.1-beta4", - }, - "requestId": "Z_NJvh0RoAMEJaw=", - "routeKey": "PUT /sondes/telemetry", - "stage": "$default", - "time": "31/Jan/2021:00:10:25 +0000", - "timeEpoch": 1612051825409, - }, - "body": json.dumps({ - "datetime": "2021-06-06T01:10:07.629Z", - "serial": "string", - "lat": 0, - "lon": 0, - "alt": 0, - "recovered": True, - "recovered_by": "string", - "description": "string" - }), - "isBase64Encoded": False, - } - # print(put(payload, {})) - print(stats(payload, {})) diff --git a/lambda/recovered/__main__.py b/lambda/recovered/__main__.py new file mode 100644 index 0000000..690f00d --- /dev/null +++ b/lambda/recovered/__main__.py @@ -0,0 +1,60 @@ +from . import * + +payload = { + "version": "2.0", + "routeKey": "PUT /recovered", + "rawPath": "/recovered", + "rawQueryString": "", + "queryStringParameters": { + # "datetime": "2021-12-20T00:00", + # "duration": 1000000 + "lat": "-32.7933", + "lon": "151.835", + "distance": "30000000" + }, + "headers": { + "accept": "*/*", + "accept-encoding": "deflate", + "content-encoding": "", + "content-length": "2135", + "content-type": "application/json", + "host": "api.v2.sondehub.org", + "user-agent": "autorx-1.4.1-beta4", + "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", + "x-forwarded-for": "103.107.130.22", + "x-forwarded-port": "443", + "x-forwarded-proto": "https", + "date": "Sun, 31 Jan 2021 00:21:45 GMT", + }, + "requestContext": { + "accountId": "143841941773", + "apiId": "r03szwwq41", + "domainName": "api.v2.sondehub.org", + "domainPrefix": "api", + "http": { + "method": "PUT", + "path": "/sondes/telemetry", + "protocol": "HTTP/1.1", + "sourceIp": "103.107.130.22", + "userAgent": "autorx-1.4.1-beta4", + }, + "requestId": "Z_NJvh0RoAMEJaw=", + "routeKey": "PUT /sondes/telemetry", + "stage": "$default", + "time": "31/Jan/2021:00:10:25 +0000", + "timeEpoch": 1612051825409, + }, + "body": json.dumps({ + "datetime": "2021-06-06T01:10:07.629Z", + "serial": "string", + "lat": 0, + "lon": 0, + "alt": 0, + "recovered": True, + "recovered_by": "string", + "description": "string" + }), + "isBase64Encoded": False, +} +# print(put(payload, {})) +print(get(payload, {})) \ No newline at end of file diff --git a/reverse-predict/lambda_function.py b/lambda/reverse-predict/__init__.py similarity index 64% rename from reverse-predict/lambda_function.py rename to lambda/reverse-predict/__init__.py index 30d0f0e..4773e40 100644 --- a/reverse-predict/lambda_function.py +++ b/lambda/reverse-predict/__init__.py @@ -1,34 +1,10 @@ -import boto3 -import botocore.credentials -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth import json -import os from datetime import datetime, timedelta, timezone -import sys -import traceback -import http.client -import math import logging import gzip from io import BytesIO import base64 - -HOST = os.getenv("ES") - -from multiprocessing import Process -http_session = URLLib3Session() - - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - r = http_session.send(request.prepare()) +import es def predict(event, context): path = "reverse-prediction-*/_search" @@ -145,7 +121,7 @@ def predict(event, context): # for single sonde allow longer predictions payload['query']['bool']['filter'].pop(0) logging.debug("Start ES Request") - results = es_request(payload, path, "GET") + results = es.request(json.dumps(payload), path, "GET") logging.debug("Finished ES Request") output = {x['1']['hits']['hits'][0]['_source']['serial']: x['1']['hits']['hits'][0]['_source'] for x in results['aggregations']['2']['buckets']} @@ -166,47 +142,3 @@ def predict(event, context): } -def es_request(payload, path, method): - # get aws creds - session = boto3.Session() - - params = json.dumps(payload) - - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(params.encode('utf-8')) - params = compressed.getvalue() - - headers = {"Host": HOST, "Content-Type": "application/json", - "Content-Encoding": "gzip"} - request = AWSRequest( - method=method, url=f"https://{HOST}/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), - "es", "us-east-1").add_auth(request) - - #p = Process(target=mirror, args=(path,params)).start() - session = URLLib3Session() - r = session.send(request.prepare()) - return json.loads(r.text) - - -if __name__ == "__main__": - # print(get_sondes({"queryStringParameters":{"lat":"-28.22717","lon":"153.82996","distance":"50000"}}, {})) - # mode: 6hours - # type: positions - # format: json - # max_positions: 0 - # position_id: 0 - # vehicles: RS_*;*chase - print(predict( - {"queryStringParameters": { - "vehicles": "" - }}, {} - )) - - -# get list of sondes, serial, lat,lon, alt - # and current rate -# for each one, request http://predict.cusf.co.uk/api/v1/?launch_latitude=-37.8136&launch_longitude=144.9631&launch_datetime=2021-02-22T00:15:18.513413Z&launch_altitude=30000&ascent_rate=5&burst_altitude=30000.1&descent_rate=5 - # have to set the burst alt slightly higher than the launch diff --git a/lambda/reverse-predict/__main__.py b/lambda/reverse-predict/__main__.py new file mode 100644 index 0000000..8fa1235 --- /dev/null +++ b/lambda/reverse-predict/__main__.py @@ -0,0 +1,19 @@ +from . import * +# print(get_sondes({"queryStringParameters":{"lat":"-28.22717","lon":"153.82996","distance":"50000"}}, {})) +# mode: 6hours +# type: positions +# format: json +# max_positions: 0 +# position_id: 0 +# vehicles: RS_*;*chase +print(predict( + {"queryStringParameters": { + "vehicles": "" + }}, {} + )) + + +# get list of sondes, serial, lat,lon, alt +# and current rate +# for each one, request http://predict.cusf.co.uk/api/v1/?launch_latitude=-37.8136&launch_longitude=144.9631&launch_datetime=2021-02-22T00:15:18.513413Z&launch_altitude=30000&ascent_rate=5&burst_altitude=30000.1&descent_rate=5 +# have to set the burst alt slightly higher than the launch \ No newline at end of file diff --git a/lambda/sign-websocket/__init__.py b/lambda/sign-websocket/__init__.py new file mode 100644 index 0000000..d1ae183 --- /dev/null +++ b/lambda/sign-websocket/__init__.py @@ -0,0 +1,2 @@ +def lambda_handler(event, context): + return {"statusCode": 200, "body": "wss://ws-reader.v2.sondehub.org/"} diff --git a/lambda/sign-websocket/__main__.py b/lambda/sign-websocket/__main__.py new file mode 100644 index 0000000..8df5719 --- /dev/null +++ b/lambda/sign-websocket/__main__.py @@ -0,0 +1,2 @@ +from . import * +print(lambda_handler({}, {})) \ No newline at end of file diff --git a/lambda/sns-to-mqtt/__init__.py b/lambda/sns-to-mqtt/__init__.py new file mode 100644 index 0000000..8db3b37 --- /dev/null +++ b/lambda/sns-to-mqtt/__init__.py @@ -0,0 +1,72 @@ +import sys +sys.path.append("vendor") +import json +import os +import paho.mqtt.client as mqtt +import time + +client = mqtt.Client(transport="websockets") + +connected_flag = False + +def connect(): + client.on_connect = on_connect + client.on_disconnect = on_disconnect + client.on_publish = on_publish + client.tls_set() + client.username_pw_set(username=os.getenv("MQTT_USERNAME"), password=os.getenv("MQTT_PASSWORD")) + client.connect(os.getenv("MQTT_HOST"), 443, 5) + client.loop_start() + print("loop started") + +def on_disconnect(client, userdata, rc): + global connected_flag + print("disconnected") + connected_flag=False #set flag + +def on_connect(client, userdata, flags, rc): + global connected_flag + if rc==0: + print("connected") + connected_flag=True #set flag + else: + print("Bad connection Returned code=",rc) + +def on_publish(client, userdata, mid): + pass + +connect() + +def lambda_handler(event, context): + client.loop(timeout=0.05, max_packets=1) # make sure it reconnects + for record in event['Records']: + sns_message = record["Sns"] + if type(json.loads(sns_message["Message"])) == dict: + incoming_payloads = [json.loads(sns_message["Message"])] + else: + incoming_payloads = json.loads(sns_message["Message"]) + + #send only the first, last and every 5th packet + payloads = [incoming_payloads[0]] + incoming_payloads[1:-1:5][1:] + [incoming_payloads[-1]] + for payload in payloads: + + body = json.dumps(payload) + + serial = payload['serial'] + while not connected_flag: + time.sleep(0.01) # wait until connected + client.publish( + topic=f'sondes/{serial}', + payload=body, + qos=0, + retain=False + ) + client.publish( + topic=f'batch', + payload=json.dumps(payloads), + qos=0, + retain=False + ) + time.sleep(0.05) # give paho mqtt 100ms to send messages this could be improved on but paho mqtt is a pain to interface with + + diff --git a/sns-to-mqtt/lambda_function.py b/lambda/sns-to-mqtt/__main__.py similarity index 56% rename from sns-to-mqtt/lambda_function.py rename to lambda/sns-to-mqtt/__main__.py index 8745fc7..3961a93 100644 --- a/sns-to-mqtt/lambda_function.py +++ b/lambda/sns-to-mqtt/__main__.py @@ -1,76 +1,4 @@ -import sys -sys.path.append("vendor/lib/python3.9/site-packages") -import json -import os -import paho.mqtt.client as mqtt -import time - -client = mqtt.Client(transport="websockets") - -connected_flag = False - -def connect(): - client.on_connect = on_connect - client.on_disconnect = on_disconnect - client.on_publish = on_publish - client.tls_set() - client.username_pw_set(username=os.getenv("MQTT_USERNAME"), password=os.getenv("MQTT_PASSWORD")) - client.connect(os.getenv("MQTT_HOST"), 443, 5) - client.loop_start() - print("loop started") - -def on_disconnect(client, userdata, rc): - global connected_flag - print("disconnected") - connected_flag=False #set flag - -def on_connect(client, userdata, flags, rc): - global connected_flag - if rc==0: - print("connected") - connected_flag=True #set flag - else: - print("Bad connection Returned code=",rc) - -def on_publish(client, userdata, mid): - pass - -connect() - -def lambda_handler(event, context): - client.loop(timeout=0.05, max_packets=1) # make sure it reconnects - for record in event['Records']: - sns_message = record["Sns"] - if type(json.loads(sns_message["Message"])) == dict: - incoming_payloads = [json.loads(sns_message["Message"])] - else: - incoming_payloads = json.loads(sns_message["Message"]) - - #send only the first, last and every 5th packet - payloads = [incoming_payloads[0]] + incoming_payloads[1:-1:5][1:] + [incoming_payloads[-1]] - for payload in payloads: - - body = json.dumps(payload) - - serial = payload['serial'] - while not connected_flag: - time.sleep(0.01) # wait until connected - client.publish( - topic=f'sondes/{serial}', - payload=body, - qos=0, - retain=False - ) - client.publish( - topic=f'batch', - payload=json.dumps(payloads), - qos=0, - retain=False - ) - time.sleep(0.05) # give paho mqtt 100ms to send messages this could be improved on but paho mqtt is a pain to interface with - - - +from . import * # test event ########### if __name__ == "__main__": diff --git a/lambda/sns-to-mqtt/vendor/paho/__init__.py b/lambda/sns-to-mqtt/vendor/paho/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/__init__.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/__init__.py new file mode 100644 index 0000000..d16f17f --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/__init__.py @@ -0,0 +1,5 @@ +__version__ = "1.5.1" + + +class MQTTException(Exception): + pass diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/client.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/client.py new file mode 100644 index 0000000..1085970 --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/client.py @@ -0,0 +1,3858 @@ +# Copyright (c) 2012-2019 Roger Light and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# and Eclipse Distribution License v1.0 which accompany this distribution. +# +# The Eclipse Public License is available at +# http://www.eclipse.org/legal/epl-v10.html +# and the Eclipse Distribution License is available at +# http://www.eclipse.org/org/documents/edl-v10.php. +# +# Contributors: +# Roger Light - initial API and implementation +# Ian Craggs - MQTT V5 support + +from .subscribeoptions import SubscribeOptions +from .reasoncodes import ReasonCodes +from .properties import Properties +from .matcher import MQTTMatcher +import logging +import hashlib +import string +import base64 +import uuid +import time +import threading +import sys +import struct +""" +This is an MQTT client module. MQTT is a lightweight pub/sub messaging +protocol that is easy to implement and suitable for low powered devices. +""" +import collections +import errno +import os +import platform +import select +import socket + +ssl = None +try: + import ssl +except ImportError: + pass + +socks = None +try: + import socks +except ImportError: + pass + +try: + # Python 3 + from urllib import request as urllib_dot_request + from urllib import parse as urllib_dot_parse +except ImportError: + # Python 2 + import urllib as urllib_dot_request + import urlparse as urllib_dot_parse + + +try: + # Use monotonic clock if available + time_func = time.monotonic +except AttributeError: + time_func = time.time + +try: + import dns.resolver +except ImportError: + HAVE_DNS = False +else: + HAVE_DNS = True + + +if platform.system() == 'Windows': + EAGAIN = errno.WSAEWOULDBLOCK +else: + EAGAIN = errno.EAGAIN + +MQTTv31 = 3 +MQTTv311 = 4 +MQTTv5 = 5 + +if sys.version_info[0] >= 3: + # define some alias for python2 compatibility + unicode = str + basestring = str + +# Message types +CONNECT = 0x10 +CONNACK = 0x20 +PUBLISH = 0x30 +PUBACK = 0x40 +PUBREC = 0x50 +PUBREL = 0x60 +PUBCOMP = 0x70 +SUBSCRIBE = 0x80 +SUBACK = 0x90 +UNSUBSCRIBE = 0xA0 +UNSUBACK = 0xB0 +PINGREQ = 0xC0 +PINGRESP = 0xD0 +DISCONNECT = 0xE0 +AUTH = 0xF0 + +# Log levels +MQTT_LOG_INFO = 0x01 +MQTT_LOG_NOTICE = 0x02 +MQTT_LOG_WARNING = 0x04 +MQTT_LOG_ERR = 0x08 +MQTT_LOG_DEBUG = 0x10 +LOGGING_LEVEL = { + MQTT_LOG_DEBUG: logging.DEBUG, + MQTT_LOG_INFO: logging.INFO, + MQTT_LOG_NOTICE: logging.INFO, # This has no direct equivalent level + MQTT_LOG_WARNING: logging.WARNING, + MQTT_LOG_ERR: logging.ERROR, +} + +# CONNACK codes +CONNACK_ACCEPTED = 0 +CONNACK_REFUSED_PROTOCOL_VERSION = 1 +CONNACK_REFUSED_IDENTIFIER_REJECTED = 2 +CONNACK_REFUSED_SERVER_UNAVAILABLE = 3 +CONNACK_REFUSED_BAD_USERNAME_PASSWORD = 4 +CONNACK_REFUSED_NOT_AUTHORIZED = 5 + +# Connection state +mqtt_cs_new = 0 +mqtt_cs_connected = 1 +mqtt_cs_disconnecting = 2 +mqtt_cs_connect_async = 3 + +# Message state +mqtt_ms_invalid = 0 +mqtt_ms_publish = 1 +mqtt_ms_wait_for_puback = 2 +mqtt_ms_wait_for_pubrec = 3 +mqtt_ms_resend_pubrel = 4 +mqtt_ms_wait_for_pubrel = 5 +mqtt_ms_resend_pubcomp = 6 +mqtt_ms_wait_for_pubcomp = 7 +mqtt_ms_send_pubrec = 8 +mqtt_ms_queued = 9 + +# Error values +MQTT_ERR_AGAIN = -1 +MQTT_ERR_SUCCESS = 0 +MQTT_ERR_NOMEM = 1 +MQTT_ERR_PROTOCOL = 2 +MQTT_ERR_INVAL = 3 +MQTT_ERR_NO_CONN = 4 +MQTT_ERR_CONN_REFUSED = 5 +MQTT_ERR_NOT_FOUND = 6 +MQTT_ERR_CONN_LOST = 7 +MQTT_ERR_TLS = 8 +MQTT_ERR_PAYLOAD_SIZE = 9 +MQTT_ERR_NOT_SUPPORTED = 10 +MQTT_ERR_AUTH = 11 +MQTT_ERR_ACL_DENIED = 12 +MQTT_ERR_UNKNOWN = 13 +MQTT_ERR_ERRNO = 14 +MQTT_ERR_QUEUE_SIZE = 15 + +MQTT_CLIENT = 0 +MQTT_BRIDGE = 1 + +# For MQTT V5, use the clean start flag only on the first successful connect +MQTT_CLEAN_START_FIRST_ONLY = 3 + +sockpair_data = b"0" + + +class WebsocketConnectionError(ValueError): + pass + + +class WouldBlockError(Exception): + pass + + +def error_string(mqtt_errno): + """Return the error string associated with an mqtt error number.""" + if mqtt_errno == MQTT_ERR_SUCCESS: + return "No error." + elif mqtt_errno == MQTT_ERR_NOMEM: + return "Out of memory." + elif mqtt_errno == MQTT_ERR_PROTOCOL: + return "A network protocol error occurred when communicating with the broker." + elif mqtt_errno == MQTT_ERR_INVAL: + return "Invalid function arguments provided." + elif mqtt_errno == MQTT_ERR_NO_CONN: + return "The client is not currently connected." + elif mqtt_errno == MQTT_ERR_CONN_REFUSED: + return "The connection was refused." + elif mqtt_errno == MQTT_ERR_NOT_FOUND: + return "Message not found (internal error)." + elif mqtt_errno == MQTT_ERR_CONN_LOST: + return "The connection was lost." + elif mqtt_errno == MQTT_ERR_TLS: + return "A TLS error occurred." + elif mqtt_errno == MQTT_ERR_PAYLOAD_SIZE: + return "Payload too large." + elif mqtt_errno == MQTT_ERR_NOT_SUPPORTED: + return "This feature is not supported." + elif mqtt_errno == MQTT_ERR_AUTH: + return "Authorisation failed." + elif mqtt_errno == MQTT_ERR_ACL_DENIED: + return "Access denied by ACL." + elif mqtt_errno == MQTT_ERR_UNKNOWN: + return "Unknown error." + elif mqtt_errno == MQTT_ERR_ERRNO: + return "Error defined by errno." + elif mqtt_errno == MQTT_ERR_QUEUE_SIZE: + return "Message queue full." + else: + return "Unknown error." + + +def connack_string(connack_code): + """Return the string associated with a CONNACK result.""" + if connack_code == CONNACK_ACCEPTED: + return "Connection Accepted." + elif connack_code == CONNACK_REFUSED_PROTOCOL_VERSION: + return "Connection Refused: unacceptable protocol version." + elif connack_code == CONNACK_REFUSED_IDENTIFIER_REJECTED: + return "Connection Refused: identifier rejected." + elif connack_code == CONNACK_REFUSED_SERVER_UNAVAILABLE: + return "Connection Refused: broker unavailable." + elif connack_code == CONNACK_REFUSED_BAD_USERNAME_PASSWORD: + return "Connection Refused: bad user name or password." + elif connack_code == CONNACK_REFUSED_NOT_AUTHORIZED: + return "Connection Refused: not authorised." + else: + return "Connection Refused: unknown reason." + + +def base62(num, base=string.digits + string.ascii_letters, padding=1): + """Convert a number to base-62 representation.""" + assert num >= 0 + digits = [] + while num: + num, rest = divmod(num, 62) + digits.append(base[rest]) + digits.extend(base[0] for _ in range(len(digits), padding)) + return ''.join(reversed(digits)) + + +def topic_matches_sub(sub, topic): + """Check whether a topic matches a subscription. + + For example: + + foo/bar would match the subscription foo/# or +/bar + non/matching would not match the subscription non/+/+ + """ + matcher = MQTTMatcher() + matcher[sub] = True + try: + next(matcher.iter_match(topic)) + return True + except StopIteration: + return False + + +def _socketpair_compat(): + """TCP/IP socketpair including Windows support""" + listensock = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_IP) + listensock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listensock.bind(("127.0.0.1", 0)) + listensock.listen(1) + + iface, port = listensock.getsockname() + sock1 = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_IP) + sock1.setblocking(0) + try: + sock1.connect(("127.0.0.1", port)) + except socket.error as err: + if err.errno != errno.EINPROGRESS and err.errno != errno.EWOULDBLOCK and err.errno != EAGAIN: + raise + sock2, address = listensock.accept() + sock2.setblocking(0) + listensock.close() + return (sock1, sock2) + + +class MQTTMessageInfo(object): + """This is a class returned from Client.publish() and can be used to find + out the mid of the message that was published, and to determine whether the + message has been published, and/or wait until it is published. + """ + + __slots__ = 'mid', '_published', '_condition', 'rc', '_iterpos' + + def __init__(self, mid): + self.mid = mid + self._published = False + self._condition = threading.Condition() + self.rc = 0 + self._iterpos = 0 + + def __str__(self): + return str((self.rc, self.mid)) + + def __iter__(self): + self._iterpos = 0 + return self + + def __next__(self): + return self.next() + + def next(self): + if self._iterpos == 0: + self._iterpos = 1 + return self.rc + elif self._iterpos == 1: + self._iterpos = 2 + return self.mid + else: + raise StopIteration + + def __getitem__(self, index): + if index == 0: + return self.rc + elif index == 1: + return self.mid + else: + raise IndexError("index out of range") + + def _set_as_published(self): + with self._condition: + self._published = True + self._condition.notify() + + def wait_for_publish(self): + """Block until the message associated with this object is published.""" + if self.rc == MQTT_ERR_QUEUE_SIZE: + raise ValueError('Message is not queued due to ERR_QUEUE_SIZE') + with self._condition: + while not self._published: + self._condition.wait() + + def is_published(self): + """Returns True if the message associated with this object has been + published, else returns False.""" + if self.rc == MQTT_ERR_QUEUE_SIZE: + raise ValueError('Message is not queued due to ERR_QUEUE_SIZE') + with self._condition: + return self._published + + +class MQTTMessage(object): + """ This is a class that describes an incoming or outgoing message. It is + passed to the on_message callback as the message parameter. + + Members: + + topic : String/bytes. topic that the message was published on. + payload : String/bytes the message payload. + qos : Integer. The message Quality of Service 0, 1 or 2. + retain : Boolean. If true, the message is a retained message and not fresh. + mid : Integer. The message id. + properties: Properties class. In MQTT v5.0, the properties associated with the message. + + On Python 3, topic must be bytes. + """ + + __slots__ = 'timestamp', 'state', 'dup', 'mid', '_topic', 'payload', 'qos', 'retain', 'info', 'properties' + + def __init__(self, mid=0, topic=b""): + self.timestamp = 0 + self.state = mqtt_ms_invalid + self.dup = False + self.mid = mid + self._topic = topic + self.payload = b"" + self.qos = 0 + self.retain = False + self.info = MQTTMessageInfo(mid) + + def __eq__(self, other): + """Override the default Equals behavior""" + if isinstance(other, self.__class__): + return self.mid == other.mid + return False + + def __ne__(self, other): + """Define a non-equality test""" + return not self.__eq__(other) + + @property + def topic(self): + return self._topic.decode('utf-8') + + @topic.setter + def topic(self, value): + self._topic = value + + +class Client(object): + """MQTT version 3.1/3.1.1/5.0 client class. + + This is the main class for use communicating with an MQTT broker. + + General usage flow: + + * Use connect()/connect_async() to connect to a broker + * Call loop() frequently to maintain network traffic flow with the broker + * Or use loop_start() to set a thread running to call loop() for you. + * Or use loop_forever() to handle calling loop() for you in a blocking + * function. + * Use subscribe() to subscribe to a topic and receive messages + * Use publish() to send messages + * Use disconnect() to disconnect from the broker + + Data returned from the broker is made available with the use of callback + functions as described below. + + Callbacks + ========= + + A number of callback functions are available to receive data back from the + broker. To use a callback, define a function and then assign it to the + client: + + def on_connect(client, userdata, flags, rc, properties=None): + print("Connection returned " + str(rc)) + + client.on_connect = on_connect + + All of the callbacks as described below have a "client" and an "userdata" + argument. "client" is the Client instance that is calling the callback. + "userdata" is user data of any type and can be set when creating a new client + instance or with user_data_set(userdata). + + If you wish to suppress exceptions within a callback, you should set + `client.suppress_exceptions = True` + + The callbacks: + + on_connect(client, userdata, flags, rc, properties=None): called when the broker responds to our connection + request. + flags is a dict that contains response flags from the broker: + flags['session present'] - this flag is useful for clients that are + using clean session set to 0 only. If a client with clean + session=0, that reconnects to a broker that it has previously + connected to, this flag indicates whether the broker still has the + session information for the client. If 1, the session still exists. + The value of rc determines success or not: + 0: Connection successful + 1: Connection refused - incorrect protocol version + 2: Connection refused - invalid client identifier + 3: Connection refused - server unavailable + 4: Connection refused - bad username or password + 5: Connection refused - not authorised + 6-255: Currently unused. + + on_disconnect(client, userdata, rc): called when the client disconnects from the broker. + The rc parameter indicates the disconnection state. If MQTT_ERR_SUCCESS + (0), the callback was called in response to a disconnect() call. If any + other value the disconnection was unexpected, such as might be caused by + a network error. + + on_disconnect(client, userdata, rc, properties): called when the MQTT V5 client disconnects from the broker. + When using MQTT V5, the broker can send a disconnect message to the client. The + message can contain a reason code and MQTT V5 properties. The properties parameter could be + None if they do not exist in the disconnect message. + + on_message(client, userdata, message): called when a message has been received on a + topic that the client subscribes to. The message variable is a + MQTTMessage that describes all of the message parameters. + + on_publish(client, userdata, mid): called when a message that was to be sent using the + publish() call has completed transmission to the broker. For messages + with QoS levels 1 and 2, this means that the appropriate handshakes have + completed. For QoS 0, this simply means that the message has left the + client. The mid variable matches the mid variable returned from the + corresponding publish() call, to allow outgoing messages to be tracked. + This callback is important because even if the publish() call returns + success, it does not always mean that the message has been sent. + + on_subscribe(client, userdata, mid, granted_qos, properties=None): called when the broker responds to a + subscribe request. The mid variable matches the mid variable returned + from the corresponding subscribe() call. The granted_qos variable is a + list of integers that give the QoS level the broker has granted for each + of the different subscription requests. + + on_unsubscribe(client, userdata, mid): called when the broker responds to an unsubscribe + request. The mid variable matches the mid variable returned from the + corresponding unsubscribe() call. + + on_log(client, userdata, level, buf): called when the client has log information. Define + to allow debugging. The level variable gives the severity of the message + and will be one of MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING, + MQTT_LOG_ERR, and MQTT_LOG_DEBUG. The message itself is in buf. + + on_socket_open(client, userdata, sock): Called when the socket has been opened. Use this + to register the socket with an external event loop for reading. + + on_socket_close(client, userdata, sock): Called when the socket is about to be closed. + Use this to unregister a socket from an external event loop for reading. + + on_socket_register_write(client, userdata, sock): Called when a write operation to the + socket failed because it would have blocked, e.g. output buffer full. Use this to + register the socket with an external event loop for writing. + + on_socket_unregister_write(client, userdata, sock): Called when a write operation to the + socket succeeded after it had previously failed. Use this to unregister the socket + from an external event loop for writing. + """ + + def __init__(self, client_id="", clean_session=None, userdata=None, + protocol=MQTTv311, transport="tcp"): + """client_id is the unique client id string used when connecting to the + broker. If client_id is zero length or None, then the behaviour is + defined by which protocol version is in use. If using MQTT v3.1.1, then + a zero length client id will be sent to the broker and the broker will + generate a random for the client. If using MQTT v3.1 then an id will be + randomly generated. In both cases, clean_session must be True. If this + is not the case a ValueError will be raised. + + clean_session is a boolean that determines the client type. If True, + the broker will remove all information about this client when it + disconnects. If False, the client is a persistent client and + subscription information and queued messages will be retained when the + client disconnects. + Note that a client will never discard its own outgoing messages on + disconnect. Calling connect() or reconnect() will cause the messages to + be resent. Use reinitialise() to reset a client to its original state. + The clean_session argument only applies to MQTT versions v3.1.1 and v3.1. + It is not accepted if the MQTT version is v5.0 - use the clean_start + argument on connect() instead. + + userdata is user defined data of any type that is passed as the "userdata" + parameter to callbacks. It may be updated at a later point with the + user_data_set() function. + + The protocol argument allows explicit setting of the MQTT version to + use for this client. Can be paho.mqtt.client.MQTTv311 (v3.1.1), + paho.mqtt.client.MQTTv31 (v3.1) or paho.mqtt.client.MQTTv5 (v5.0), + with the default being v3.1.1. + + Set transport to "websockets" to use WebSockets as the transport + mechanism. Set to "tcp" to use raw TCP, which is the default. + """ + + if protocol == MQTTv5: + if clean_session != None: + raise ValueError('Clean session is not used for MQTT 5.0') + else: + if clean_session == None: + clean_session = True + if not clean_session and (client_id == "" or client_id is None): + raise ValueError( + 'A client id must be provided if clean session is False.') + self._clean_session = clean_session + + if transport.lower() not in ('websockets', 'tcp'): + raise ValueError( + 'transport must be "websockets" or "tcp", not %s' % transport) + self._transport = transport.lower() + self._protocol = protocol + self._userdata = userdata + self._sock = None + self._sockpairR, self._sockpairW = (None, None,) + self._sockpairR, self._sockpairW = _socketpair_compat() + self._keepalive = 60 + self._message_retry = 20 + self._last_retry_check = 0 + self._client_mode = MQTT_CLIENT + # [MQTT-3.1.3-4] Client Id must be UTF-8 encoded string. + if client_id == "" or client_id is None: + if protocol == MQTTv31: + self._client_id = base62(uuid.uuid4().int, padding=22) + else: + self._client_id = b"" + else: + self._client_id = client_id + if isinstance(self._client_id, unicode): + self._client_id = self._client_id.encode('utf-8') + + self._username = None + self._password = None + self._in_packet = { + "command": 0, + "have_remaining": 0, + "remaining_count": [], + "remaining_mult": 1, + "remaining_length": 0, + "packet": b"", + "to_process": 0, + "pos": 0} + self._out_packet = collections.deque() + self._current_out_packet = None + self._last_msg_in = time_func() + self._last_msg_out = time_func() + self._reconnect_min_delay = 1 + self._reconnect_max_delay = 120 + self._reconnect_delay = None + self._ping_t = 0 + self._last_mid = 0 + self._state = mqtt_cs_new + self._out_messages = collections.OrderedDict() + self._in_messages = collections.OrderedDict() + self._max_inflight_messages = 20 + self._inflight_messages = 0 + self._max_queued_messages = 0 + self._connect_properties = None + self._will_properties = None + self._will = False + self._will_topic = b"" + self._will_payload = b"" + self._will_qos = 0 + self._will_retain = False + self._on_message_filtered = MQTTMatcher() + self._host = "" + self._port = 1883 + self._bind_address = "" + self._bind_port = 0 + self._proxy = {} + self._in_callback_mutex = threading.Lock() + self._callback_mutex = threading.RLock() + self._out_packet_mutex = threading.Lock() + self._current_out_packet_mutex = threading.RLock() + self._msgtime_mutex = threading.Lock() + self._out_message_mutex = threading.RLock() + self._in_message_mutex = threading.Lock() + self._reconnect_delay_mutex = threading.Lock() + self._mid_generate_mutex = threading.Lock() + self._thread = None + self._thread_terminate = False + self._ssl = False + self._ssl_context = None + # Only used when SSL context does not have check_hostname attribute + self._tls_insecure = False + self._logger = None + self._registered_write = False + # No default callbacks + self._on_log = None + self._on_connect = None + self._on_subscribe = None + self._on_message = None + self._on_publish = None + self._on_unsubscribe = None + self._on_disconnect = None + self._on_socket_open = None + self._on_socket_close = None + self._on_socket_register_write = None + self._on_socket_unregister_write = None + self._websocket_path = "/mqtt" + self._websocket_extra_headers = None + # for clean_start == MQTT_CLEAN_START_FIRST_ONLY + self._mqttv5_first_connect = True + self.suppress_exceptions = False # For callbacks + + def __del__(self): + self._reset_sockets() + + def _sock_recv(self, bufsize): + try: + return self._sock.recv(bufsize) + except socket.error as err: + if self._ssl and err.errno == ssl.SSL_ERROR_WANT_READ: + raise WouldBlockError() + if self._ssl and err.errno == ssl.SSL_ERROR_WANT_WRITE: + self._call_socket_register_write() + raise WouldBlockError() + if err.errno == EAGAIN: + raise WouldBlockError() + raise + + def _sock_send(self, buf): + try: + return self._sock.send(buf) + except socket.error as err: + if self._ssl and err.errno == ssl.SSL_ERROR_WANT_READ: + raise WouldBlockError() + if self._ssl and err.errno == ssl.SSL_ERROR_WANT_WRITE: + self._call_socket_register_write() + raise WouldBlockError() + if err.errno == EAGAIN: + self._call_socket_register_write() + raise WouldBlockError() + raise + + def _sock_close(self): + """Close the connection to the server.""" + if not self._sock: + return + + try: + sock = self._sock + self._sock = None + self._call_socket_unregister_write(sock) + self._call_socket_close(sock) + finally: + # In case a callback fails, still close the socket to avoid leaking the file descriptor. + sock.close() + + def _reset_sockets(self): + self._sock_close() + + if self._sockpairR: + self._sockpairR.close() + self._sockpairR = None + if self._sockpairW: + self._sockpairW.close() + self._sockpairW = None + + def reinitialise(self, client_id="", clean_session=True, userdata=None): + self._reset_sockets() + + self.__init__(client_id, clean_session, userdata) + + def ws_set_options(self, path="/mqtt", headers=None): + """ Set the path and headers for a websocket connection + + path is a string starting with / which should be the endpoint of the + mqtt connection on the remote server + + headers can be either a dict or a callable object. If it is a dict then + the extra items in the dict are added to the websocket headers. If it is + a callable, then the default websocket headers are passed into this + function and the result is used as the new headers. + """ + self._websocket_path = path + + if headers is not None: + if isinstance(headers, dict) or callable(headers): + self._websocket_extra_headers = headers + else: + raise ValueError( + "'headers' option to ws_set_options has to be either a dictionary or callable") + + def tls_set_context(self, context=None): + """Configure network encryption and authentication context. Enables SSL/TLS support. + + context : an ssl.SSLContext object. By default this is given by + `ssl.create_default_context()`, if available. + + Must be called before connect() or connect_async().""" + if self._ssl_context is not None: + raise ValueError('SSL/TLS has already been configured.') + + # Assume that have SSL support, or at least that context input behaves like ssl.SSLContext + # in current versions of Python + + if context is None: + if hasattr(ssl, 'create_default_context'): + context = ssl.create_default_context() + else: + raise ValueError('SSL/TLS context must be specified') + + self._ssl = True + self._ssl_context = context + + # Ensure _tls_insecure is consistent with check_hostname attribute + if hasattr(context, 'check_hostname'): + self._tls_insecure = not context.check_hostname + + def tls_set(self, ca_certs=None, certfile=None, keyfile=None, cert_reqs=None, tls_version=None, ciphers=None): + """Configure network encryption and authentication options. Enables SSL/TLS support. + + ca_certs : a string path to the Certificate Authority certificate files + that are to be treated as trusted by this client. If this is the only + option given then the client will operate in a similar manner to a web + browser. That is to say it will require the broker to have a + certificate signed by the Certificate Authorities in ca_certs and will + communicate using TLS v1, but will not attempt any form of + authentication. This provides basic network encryption but may not be + sufficient depending on how the broker is configured. + By default, on Python 2.7.9+ or 3.4+, the default certification + authority of the system is used. On older Python version this parameter + is mandatory. + + certfile and keyfile are strings pointing to the PEM encoded client + certificate and private keys respectively. If these arguments are not + None then they will be used as client information for TLS based + authentication. Support for this feature is broker dependent. Note + that if either of these files in encrypted and needs a password to + decrypt it, Python will ask for the password at the command line. It is + not currently possible to define a callback to provide the password. + + cert_reqs allows the certificate requirements that the client imposes + on the broker to be changed. By default this is ssl.CERT_REQUIRED, + which means that the broker must provide a certificate. See the ssl + pydoc for more information on this parameter. + + tls_version allows the version of the SSL/TLS protocol used to be + specified. By default TLS v1 is used. Previous versions (all versions + beginning with SSL) are possible but not recommended due to possible + security problems. + + ciphers is a string specifying which encryption ciphers are allowable + for this connection, or None to use the defaults. See the ssl pydoc for + more information. + + Must be called before connect() or connect_async().""" + if ssl is None: + raise ValueError('This platform has no SSL/TLS.') + + if not hasattr(ssl, 'SSLContext'): + # Require Python version that has SSL context support in standard library + raise ValueError( + 'Python 2.7.9 and 3.2 are the minimum supported versions for TLS.') + + if ca_certs is None and not hasattr(ssl.SSLContext, 'load_default_certs'): + raise ValueError('ca_certs must not be None.') + + # Create SSLContext object + if tls_version is None: + tls_version = ssl.PROTOCOL_TLSv1 + # If the python version supports it, use highest TLS version automatically + if hasattr(ssl, "PROTOCOL_TLS"): + tls_version = ssl.PROTOCOL_TLS + context = ssl.SSLContext(tls_version) + + # Configure context + if certfile is not None: + context.load_cert_chain(certfile, keyfile) + + if cert_reqs == ssl.CERT_NONE and hasattr(context, 'check_hostname'): + context.check_hostname = False + + context.verify_mode = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs + + if ca_certs is not None: + context.load_verify_locations(ca_certs) + else: + context.load_default_certs() + + if ciphers is not None: + context.set_ciphers(ciphers) + + self.tls_set_context(context) + + if cert_reqs != ssl.CERT_NONE: + # Default to secure, sets context.check_hostname attribute + # if available + self.tls_insecure_set(False) + else: + # But with ssl.CERT_NONE, we can not check_hostname + self.tls_insecure_set(True) + + def tls_insecure_set(self, value): + """Configure verification of the server hostname in the server certificate. + + If value is set to true, it is impossible to guarantee that the host + you are connecting to is not impersonating your server. This can be + useful in initial server testing, but makes it possible for a malicious + third party to impersonate your server through DNS spoofing, for + example. + + Do not use this function in a real system. Setting value to true means + there is no point using encryption. + + Must be called before connect() and after either tls_set() or + tls_set_context().""" + + if self._ssl_context is None: + raise ValueError( + 'Must configure SSL context before using tls_insecure_set.') + + self._tls_insecure = value + + # Ensure check_hostname is consistent with _tls_insecure attribute + if hasattr(self._ssl_context, 'check_hostname'): + # Rely on SSLContext to check host name + # If verify_mode is CERT_NONE then the host name will never be checked + self._ssl_context.check_hostname = not value + + def proxy_set(self, **proxy_args): + """Configure proxying of MQTT connection. Enables support for SOCKS or + HTTP proxies. + + Proxying is done through the PySocks library. Brief descriptions of the + proxy_args parameters are below; see the PySocks docs for more info. + + (Required) + proxy_type: One of {socks.HTTP, socks.SOCKS4, or socks.SOCKS5} + proxy_addr: IP address or DNS name of proxy server + + (Optional) + proxy_rdns: boolean indicating whether proxy lookup should be performed + remotely (True, default) or locally (False) + proxy_username: username for SOCKS5 proxy, or userid for SOCKS4 proxy + proxy_password: password for SOCKS5 proxy + + Must be called before connect() or connect_async().""" + if socks is None: + raise ValueError("PySocks must be installed for proxy support.") + elif not self._proxy_is_valid(proxy_args): + raise ValueError("proxy_type and/or proxy_addr are invalid.") + else: + self._proxy = proxy_args + + def enable_logger(self, logger=None): + """ Enables a logger to send log messages to """ + if logger is None: + if self._logger is not None: + # Do not replace existing logger + return + logger = logging.getLogger(__name__) + self._logger = logger + + def disable_logger(self): + self._logger = None + + def connect(self, host, port=1883, keepalive=60, bind_address="", bind_port=0, + clean_start=MQTT_CLEAN_START_FIRST_ONLY, properties=None): + """Connect to a remote broker. + + host is the hostname or IP address of the remote broker. + port is the network port of the server host to connect to. Defaults to + 1883. Note that the default port for MQTT over SSL/TLS is 8883 so if you + are using tls_set() the port may need providing. + keepalive: Maximum period in seconds between communications with the + broker. If no other messages are being exchanged, this controls the + rate at which the client will send ping messages to the broker. + clean_start: (MQTT v5.0 only) True, False or MQTT_CLEAN_START_FIRST_ONLY. + Sets the MQTT v5.0 clean_start flag always, never or on the first successful connect only, + respectively. MQTT session data (such as outstanding messages and subscriptions) + is cleared on successful connect when the clean_start flag is set. + properties: (MQTT v5.0 only) the MQTT v5.0 properties to be sent in the + MQTT connect packet. + """ + + if self._protocol == MQTTv5: + self._mqttv5_first_connect = True + else: + if clean_start != MQTT_CLEAN_START_FIRST_ONLY: + raise ValueError("Clean start only applies to MQTT V5") + if properties != None: + raise ValueError("Properties only apply to MQTT V5") + + self.connect_async(host, port, keepalive, + bind_address, bind_port, clean_start, properties) + return self.reconnect() + + def connect_srv(self, domain=None, keepalive=60, bind_address="", + clean_start=MQTT_CLEAN_START_FIRST_ONLY, properties=None): + """Connect to a remote broker. + + domain is the DNS domain to search for SRV records; if None, + try to determine local domain name. + keepalive, bind_address, clean_start and properties are as for connect() + """ + + if HAVE_DNS is False: + raise ValueError( + 'No DNS resolver library found, try "pip install dnspython" or "pip3 install dnspython3".') + + if domain is None: + domain = socket.getfqdn() + domain = domain[domain.find('.') + 1:] + + try: + rr = '_mqtt._tcp.%s' % domain + if self._ssl: + # IANA specifies secure-mqtt (not mqtts) for port 8883 + rr = '_secure-mqtt._tcp.%s' % domain + answers = [] + for answer in dns.resolver.query(rr, dns.rdatatype.SRV): + addr = answer.target.to_text()[:-1] + answers.append( + (addr, answer.port, answer.priority, answer.weight)) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers): + raise ValueError("No answer/NXDOMAIN for SRV in %s" % (domain)) + + # FIXME: doesn't account for weight + for answer in answers: + host, port, prio, weight = answer + + try: + return self.connect(host, port, keepalive, bind_address, clean_start, properties) + except Exception: + pass + + raise ValueError("No SRV hosts responded") + + def connect_async(self, host, port=1883, keepalive=60, bind_address="", bind_port=0, + clean_start=MQTT_CLEAN_START_FIRST_ONLY, properties=None): + """Connect to a remote broker asynchronously. This is a non-blocking + connect call that can be used with loop_start() to provide very quick + start. + + host is the hostname or IP address of the remote broker. + port is the network port of the server host to connect to. Defaults to + 1883. Note that the default port for MQTT over SSL/TLS is 8883 so if you + are using tls_set() the port may need providing. + keepalive: Maximum period in seconds between communications with the + broker. If no other messages are being exchanged, this controls the + rate at which the client will send ping messages to the broker. + clean_start: (MQTT v5.0 only) True, False or MQTT_CLEAN_START_FIRST_ONLY. + Sets the MQTT v5.0 clean_start flag always, never or on the first successful connect only, + respectively. MQTT session data (such as outstanding messages and subscriptions) + is cleared on successful connect when the clean_start flag is set. + properties: (MQTT v5.0 only) the MQTT v5.0 properties to be sent in the + MQTT connect packet. Use the Properties class. + """ + if host is None or len(host) == 0: + raise ValueError('Invalid host.') + if port <= 0: + raise ValueError('Invalid port number.') + if keepalive < 0: + raise ValueError('Keepalive must be >=0.') + if bind_address != "" and bind_address is not None: + if sys.version_info < (2, 7) or (3, 0) < sys.version_info < (3, 2): + raise ValueError('bind_address requires Python 2.7 or 3.2.') + if bind_port < 0: + raise ValueError('Invalid bind port number.') + + self._host = host + self._port = port + self._keepalive = keepalive + self._bind_address = bind_address + self._bind_port = bind_port + self._clean_start = clean_start + self._connect_properties = properties + self._state = mqtt_cs_connect_async + + + def reconnect_delay_set(self, min_delay=1, max_delay=120): + """ Configure the exponential reconnect delay + + When connection is lost, wait initially min_delay seconds and + double this time every attempt. The wait is capped at max_delay. + Once the client is fully connected (e.g. not only TCP socket, but + received a success CONNACK), the wait timer is reset to min_delay. + """ + with self._reconnect_delay_mutex: + self._reconnect_min_delay = min_delay + self._reconnect_max_delay = max_delay + self._reconnect_delay = None + + def reconnect(self): + """Reconnect the client after a disconnect. Can only be called after + connect()/connect_async().""" + if len(self._host) == 0: + raise ValueError('Invalid host.') + if self._port <= 0: + raise ValueError('Invalid port number.') + + self._in_packet = { + "command": 0, + "have_remaining": 0, + "remaining_count": [], + "remaining_mult": 1, + "remaining_length": 0, + "packet": b"", + "to_process": 0, + "pos": 0} + + with self._out_packet_mutex: + self._out_packet = collections.deque() + + with self._current_out_packet_mutex: + self._current_out_packet = None + + with self._msgtime_mutex: + self._last_msg_in = time_func() + self._last_msg_out = time_func() + + self._ping_t = 0 + self._state = mqtt_cs_new + + self._sock_close() + + # Put messages in progress in a valid state. + self._messages_reconnect_reset() + + sock = self._create_socket_connection() + + if self._ssl: + # SSL is only supported when SSLContext is available (implies Python >= 2.7.9 or >= 3.2) + + verify_host = not self._tls_insecure + try: + # Try with server_hostname, even it's not supported in certain scenarios + sock = self._ssl_context.wrap_socket( + sock, + server_hostname=self._host, + do_handshake_on_connect=False, + ) + except ssl.CertificateError: + # CertificateError is derived from ValueError + raise + except ValueError: + # Python version requires SNI in order to handle server_hostname, but SNI is not available + sock = self._ssl_context.wrap_socket( + sock, + do_handshake_on_connect=False, + ) + else: + # If SSL context has already checked hostname, then don't need to do it again + if (hasattr(self._ssl_context, 'check_hostname') and + self._ssl_context.check_hostname): + verify_host = False + + sock.settimeout(self._keepalive) + sock.do_handshake() + + if verify_host: + ssl.match_hostname(sock.getpeercert(), self._host) + + if self._transport == "websockets": + sock.settimeout(self._keepalive) + sock = WebsocketWrapper(sock, self._host, self._port, self._ssl, + self._websocket_path, self._websocket_extra_headers) + + self._sock = sock + self._sock.setblocking(0) + self._registered_write = False + self._call_socket_open() + + return self._send_connect(self._keepalive) + + def loop(self, timeout=1.0, max_packets=1): + """Process network events. + + This function must be called regularly to ensure communication with the + broker is carried out. It calls select() on the network socket to wait + for network events. If incoming data is present it will then be + processed. Outgoing commands, from e.g. publish(), are normally sent + immediately that their function is called, but this is not always + possible. loop() will also attempt to send any remaining outgoing + messages, which also includes commands that are part of the flow for + messages with QoS>0. + + timeout: The time in seconds to wait for incoming/outgoing network + traffic before timing out and returning. + max_packets: Not currently used. + + Returns MQTT_ERR_SUCCESS on success. + Returns >0 on error. + + A ValueError will be raised if timeout < 0""" + if timeout < 0.0: + raise ValueError('Invalid timeout.') + + with self._current_out_packet_mutex: + with self._out_packet_mutex: + if self._current_out_packet is None and len(self._out_packet) > 0: + self._current_out_packet = self._out_packet.popleft() + + if self._current_out_packet: + wlist = [self._sock] + else: + wlist = [] + + # used to check if there are any bytes left in the (SSL) socket + pending_bytes = 0 + if hasattr(self._sock, 'pending'): + pending_bytes = self._sock.pending() + + # if bytes are pending do not wait in select + if pending_bytes > 0: + timeout = 0.0 + + # sockpairR is used to break out of select() before the timeout, on a + # call to publish() etc. + rlist = [self._sock, self._sockpairR] + try: + socklist = select.select(rlist, wlist, [], timeout) + except TypeError: + # Socket isn't correct type, in likelihood connection is lost + return MQTT_ERR_CONN_LOST + except ValueError: + # Can occur if we just reconnected but rlist/wlist contain a -1 for + # some reason. + return MQTT_ERR_CONN_LOST + except Exception: + # Note that KeyboardInterrupt, etc. can still terminate since they + # are not derived from Exception + return MQTT_ERR_UNKNOWN + + if self._sock in socklist[0] or pending_bytes > 0: + rc = self.loop_read(max_packets) + if rc or self._sock is None: + return rc + + if self._sockpairR in socklist[0]: + # Stimulate output write even though we didn't ask for it, because + # at that point the publish or other command wasn't present. + socklist[1].insert(0, self._sock) + # Clear sockpairR - only ever a single byte written. + try: + self._sockpairR.recv(1) + except socket.error as err: + if err.errno != EAGAIN: + raise + + if self._sock in socklist[1]: + rc = self.loop_write(max_packets) + if rc or self._sock is None: + return rc + + return self.loop_misc() + + def publish(self, topic, payload=None, qos=0, retain=False, properties=None): + """Publish a message on a topic. + + This causes a message to be sent to the broker and subsequently from + the broker to any clients subscribing to matching topics. + + topic: The topic that the message should be published on. + payload: The actual message to send. If not given, or set to None a + zero length message will be used. Passing an int or float will result + in the payload being converted to a string representing that number. If + you wish to send a true int/float, use struct.pack() to create the + payload you require. + qos: The quality of service level to use. + retain: If set to true, the message will be set as the "last known + good"/retained message for the topic. + properties: (MQTT v5.0 only) the MQTT v5.0 properties to be included. + Use the Properties class. + + Returns a MQTTMessageInfo class, which can be used to determine whether + the message has been delivered (using info.is_published()) or to block + waiting for the message to be delivered (info.wait_for_publish()). The + message ID and return code of the publish() call can be found at + info.mid and info.rc. + + For backwards compatibility, the MQTTMessageInfo class is iterable so + the old construct of (rc, mid) = client.publish(...) is still valid. + + rc is MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN if the + client is not currently connected. mid is the message ID for the + publish request. The mid value can be used to track the publish request + by checking against the mid argument in the on_publish() callback if it + is defined. + + A ValueError will be raised if topic is None, has zero length or is + invalid (contains a wildcard), except if the MQTT version used is v5.0. + For v5.0, a zero length topic can be used when a Topic Alias has been set. + + A ValueError will be raised if qos is not one of 0, 1 or 2, or if + the length of the payload is greater than 268435455 bytes.""" + if self._protocol != MQTTv5: + if topic is None or len(topic) == 0: + raise ValueError('Invalid topic.') + + topic = topic.encode('utf-8') + + if self._topic_wildcard_len_check(topic) != MQTT_ERR_SUCCESS: + raise ValueError('Publish topic cannot contain wildcards.') + + if qos < 0 or qos > 2: + raise ValueError('Invalid QoS level.') + + if isinstance(payload, unicode): + local_payload = payload.encode('utf-8') + elif isinstance(payload, (bytes, bytearray)): + local_payload = payload + elif isinstance(payload, (int, float)): + local_payload = str(payload).encode('ascii') + elif payload is None: + local_payload = b'' + else: + raise TypeError( + 'payload must be a string, bytearray, int, float or None.') + + if len(local_payload) > 268435455: + raise ValueError('Payload too large.') + + local_mid = self._mid_generate() + + if qos == 0: + info = MQTTMessageInfo(local_mid) + rc = self._send_publish( + local_mid, topic, local_payload, qos, retain, False, info, properties) + info.rc = rc + return info + else: + message = MQTTMessage(local_mid, topic) + message.timestamp = time_func() + message.payload = local_payload + message.qos = qos + message.retain = retain + message.dup = False + message.properties = properties + + with self._out_message_mutex: + if self._max_queued_messages > 0 and len(self._out_messages) >= self._max_queued_messages: + message.info.rc = MQTT_ERR_QUEUE_SIZE + return message.info + + if local_mid in self._out_messages: + message.info.rc = MQTT_ERR_QUEUE_SIZE + return message.info + + self._out_messages[message.mid] = message + if self._max_inflight_messages == 0 or self._inflight_messages < self._max_inflight_messages: + self._inflight_messages += 1 + if qos == 1: + message.state = mqtt_ms_wait_for_puback + elif qos == 2: + message.state = mqtt_ms_wait_for_pubrec + + rc = self._send_publish(message.mid, topic, message.payload, message.qos, message.retain, + message.dup, message.info, message.properties) + + # remove from inflight messages so it will be send after a connection is made + if rc is MQTT_ERR_NO_CONN: + self._inflight_messages -= 1 + message.state = mqtt_ms_publish + + message.info.rc = rc + return message.info + else: + message.state = mqtt_ms_queued + message.info.rc = MQTT_ERR_SUCCESS + return message.info + + def username_pw_set(self, username, password=None): + """Set a username and optionally a password for broker authentication. + + Must be called before connect() to have any effect. + Requires a broker that supports MQTT v3.1. + + username: The username to authenticate with. Need have no relationship to the client id. Must be unicode + [MQTT-3.1.3-11]. + Set to None to reset client back to not using username/password for broker authentication. + password: The password to authenticate with. Optional, set to None if not required. If it is unicode, then it + will be encoded as UTF-8. + """ + + # [MQTT-3.1.3-11] User name must be UTF-8 encoded string + self._username = None if username is None else username.encode('utf-8') + self._password = password + if isinstance(self._password, unicode): + self._password = self._password.encode('utf-8') + + def enable_bridge_mode(self): + """Sets the client in a bridge mode instead of client mode. + + Must be called before connect() to have any effect. + Requires brokers that support bridge mode. + + Under bridge mode, the broker will identify the client as a bridge and + not send it's own messages back to it. Hence a subsciption of # is + possible without message loops. This feature also correctly propagates + the retain flag on the messages. + + Currently Mosquitto and RSMB support this feature. This feature can + be used to create a bridge between multiple broker. + """ + self._client_mode = MQTT_BRIDGE + + def is_connected(self): + """Returns the current status of the connection + + True if connection exists + False if connection is closed + """ + return self._state == mqtt_cs_connected + + def disconnect(self, reasoncode=None, properties=None): + """Disconnect a connected client from the broker. + reasoncode: (MQTT v5.0 only) a ReasonCodes instance setting the MQTT v5.0 + reasoncode to be sent with the disconnect. It is optional, the receiver + then assuming that 0 (success) is the value. + properties: (MQTT v5.0 only) a Properties instance setting the MQTT v5.0 properties + to be included. Optional - if not set, no properties are sent. + """ + self._state = mqtt_cs_disconnecting + + if self._sock is None: + return MQTT_ERR_NO_CONN + + return self._send_disconnect(reasoncode, properties) + + def subscribe(self, topic, qos=0, options=None, properties=None): + """Subscribe the client to one or more topics. + + This function may be called in three different ways (and a further three for MQTT v5.0): + + Simple string and integer + ------------------------- + e.g. subscribe("my/topic", 2) + + topic: A string specifying the subscription topic to subscribe to. + qos: The desired quality of service level for the subscription. + Defaults to 0. + options and properties: Not used. + + Simple string and subscribe options (MQTT v5.0 only) + ---------------------------------------------------- + e.g. subscribe("my/topic", options=SubscribeOptions(qos=2)) + + topic: A string specifying the subscription topic to subscribe to. + qos: Not used. + options: The MQTT v5.0 subscribe options. + properties: a Properties instance setting the MQTT v5.0 properties + to be included. Optional - if not set, no properties are sent. + + String and integer tuple + ------------------------ + e.g. subscribe(("my/topic", 1)) + + topic: A tuple of (topic, qos). Both topic and qos must be present in + the tuple. + qos and options: Not used. + properties: Only used for MQTT v5.0. A Properties instance setting the + MQTT v5.0 properties. Optional - if not set, no properties are sent. + + String and subscribe options tuple (MQTT v5.0 only) + --------------------------------------------------- + e.g. subscribe(("my/topic", SubscribeOptions(qos=1))) + + topic: A tuple of (topic, SubscribeOptions). Both topic and subscribe + options must be present in the tuple. + qos and options: Not used. + properties: a Properties instance setting the MQTT v5.0 properties + to be included. Optional - if not set, no properties are sent. + + List of string and integer tuples + --------------------------------- + e.g. subscribe([("my/topic", 0), ("another/topic", 2)]) + + This allows multiple topic subscriptions in a single SUBSCRIPTION + command, which is more efficient than using multiple calls to + subscribe(). + + topic: A list of tuple of format (topic, qos). Both topic and qos must + be present in all of the tuples. + qos, options and properties: Not used. + + List of string and subscribe option tuples (MQTT v5.0 only) + ----------------------------------------------------------- + e.g. subscribe([("my/topic", SubscribeOptions(qos=0), ("another/topic", SubscribeOptions(qos=2)]) + + This allows multiple topic subscriptions in a single SUBSCRIPTION + command, which is more efficient than using multiple calls to + subscribe(). + + topic: A list of tuple of format (topic, SubscribeOptions). Both topic and subscribe + options must be present in all of the tuples. + qos and options: Not used. + properties: a Properties instance setting the MQTT v5.0 properties + to be included. Optional - if not set, no properties are sent. + + The function returns a tuple (result, mid), where result is + MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the + client is not currently connected. mid is the message ID for the + subscribe request. The mid value can be used to track the subscribe + request by checking against the mid argument in the on_subscribe() + callback if it is defined. + + Raises a ValueError if qos is not 0, 1 or 2, or if topic is None or has + zero string length, or if topic is not a string, tuple or list. + """ + topic_qos_list = None + + if isinstance(topic, tuple): + if self._protocol == MQTTv5: + topic, options = topic + if not isinstance(options, SubscribeOptions): + raise ValueError( + 'Subscribe options must be instance of SubscribeOptions class.') + else: + topic, qos = topic + + if isinstance(topic, basestring): + if qos < 0 or qos > 2: + raise ValueError('Invalid QoS level.') + if self._protocol == MQTTv5: + if options == None: + # if no options are provided, use the QoS passed instead + options = SubscribeOptions(qos=qos) + elif qos != 0: + raise ValueError( + 'Subscribe options and qos parameters cannot be combined.') + if not isinstance(options, SubscribeOptions): + raise ValueError( + 'Subscribe options must be instance of SubscribeOptions class.') + topic_qos_list = [(topic.encode('utf-8'), options)] + else: + if topic is None or len(topic) == 0: + raise ValueError('Invalid topic.') + topic_qos_list = [(topic.encode('utf-8'), qos)] + elif isinstance(topic, list): + topic_qos_list = [] + if self._protocol == MQTTv5: + for t, o in topic: + if not isinstance(o, SubscribeOptions): + # then the second value should be QoS + if o < 0 or o > 2: + raise ValueError('Invalid QoS level.') + o = SubscribeOptions(qos=o) + topic_qos_list.append((t.encode('utf-8'), o)) + else: + for t, q in topic: + if q < 0 or q > 2: + raise ValueError('Invalid QoS level.') + if t is None or len(t) == 0 or not isinstance(t, basestring): + raise ValueError('Invalid topic.') + topic_qos_list.append((t.encode('utf-8'), q)) + + if topic_qos_list is None: + raise ValueError("No topic specified, or incorrect topic type.") + + if any(self._filter_wildcard_len_check(topic) != MQTT_ERR_SUCCESS for topic, _ in topic_qos_list): + raise ValueError('Invalid subscription filter.') + + if self._sock is None: + return (MQTT_ERR_NO_CONN, None) + + return self._send_subscribe(False, topic_qos_list, properties) + + def unsubscribe(self, topic, properties=None): + """Unsubscribe the client from one or more topics. + + topic: A single string, or list of strings that are the subscription + topics to unsubscribe from. + properties: (MQTT v5.0 only) a Properties instance setting the MQTT v5.0 properties + to be included. Optional - if not set, no properties are sent. + + Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS + to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not + currently connected. + mid is the message ID for the unsubscribe request. The mid value can be + used to track the unsubscribe request by checking against the mid + argument in the on_unsubscribe() callback if it is defined. + + Raises a ValueError if topic is None or has zero string length, or is + not a string or list. + """ + topic_list = None + if topic is None: + raise ValueError('Invalid topic.') + if isinstance(topic, basestring): + if len(topic) == 0: + raise ValueError('Invalid topic.') + topic_list = [topic.encode('utf-8')] + elif isinstance(topic, list): + topic_list = [] + for t in topic: + if len(t) == 0 or not isinstance(t, basestring): + raise ValueError('Invalid topic.') + topic_list.append(t.encode('utf-8')) + + if topic_list is None: + raise ValueError("No topic specified, or incorrect topic type.") + + if self._sock is None: + return (MQTT_ERR_NO_CONN, None) + + return self._send_unsubscribe(False, topic_list, properties) + + def loop_read(self, max_packets=1): + """Process read network events. Use in place of calling loop() if you + wish to handle your client reads as part of your own application. + + Use socket() to obtain the client socket to call select() or equivalent + on. + + Do not use if you are using the threaded interface loop_start().""" + if self._sock is None: + return MQTT_ERR_NO_CONN + + max_packets = len(self._out_messages) + len(self._in_messages) + if max_packets < 1: + max_packets = 1 + + for _ in range(0, max_packets): + if self._sock is None: + return MQTT_ERR_NO_CONN + rc = self._packet_read() + if rc > 0: + return self._loop_rc_handle(rc) + elif rc == MQTT_ERR_AGAIN: + return MQTT_ERR_SUCCESS + return MQTT_ERR_SUCCESS + + def loop_write(self, max_packets=1): + """Process write network events. Use in place of calling loop() if you + wish to handle your client writes as part of your own application. + + Use socket() to obtain the client socket to call select() or equivalent + on. + + Use want_write() to determine if there is data waiting to be written. + + Do not use if you are using the threaded interface loop_start().""" + if self._sock is None: + return MQTT_ERR_NO_CONN + + max_packets = len(self._out_packet) + 1 + if max_packets < 1: + max_packets = 1 + + try: + for _ in range(0, max_packets): + rc = self._packet_write() + if rc > 0: + return self._loop_rc_handle(rc) + elif rc == MQTT_ERR_AGAIN: + return MQTT_ERR_SUCCESS + return MQTT_ERR_SUCCESS + finally: + if self.want_write(): + self._call_socket_register_write() + else: + self._call_socket_unregister_write() + + def want_write(self): + """Call to determine if there is network data waiting to be written. + Useful if you are calling select() yourself rather than using loop(). + """ + if self._current_out_packet or len(self._out_packet) > 0: + return True + else: + return False + + def loop_misc(self): + """Process miscellaneous network events. Use in place of calling loop() if you + wish to call select() or equivalent on. + + Do not use if you are using the threaded interface loop_start().""" + if self._sock is None: + return MQTT_ERR_NO_CONN + + now = time_func() + self._check_keepalive() + if self._last_retry_check + 1 < now: + # Only check once a second at most + self._message_retry_check() + self._last_retry_check = now + + if self._ping_t > 0 and now - self._ping_t >= self._keepalive: + # client->ping_t != 0 means we are waiting for a pingresp. + # This hasn't happened in the keepalive time so we should disconnect. + self._sock_close() + + if self._state == mqtt_cs_disconnecting: + rc = MQTT_ERR_SUCCESS + else: + rc = 1 + + self._do_on_disconnect(rc) + + return MQTT_ERR_CONN_LOST + + return MQTT_ERR_SUCCESS + + def max_inflight_messages_set(self, inflight): + """Set the maximum number of messages with QoS>0 that can be part way + through their network flow at once. Defaults to 20.""" + if inflight < 0: + raise ValueError('Invalid inflight.') + self._max_inflight_messages = inflight + + def max_queued_messages_set(self, queue_size): + """Set the maximum number of messages in the outgoing message queue. + 0 means unlimited.""" + if queue_size < 0: + raise ValueError('Invalid queue size.') + if not isinstance(queue_size, int): + raise ValueError('Invalid type of queue size.') + self._max_queued_messages = queue_size + return self + + def message_retry_set(self, retry): + """Set the timeout in seconds before a message with QoS>0 is retried. + 20 seconds by default.""" + if retry < 0: + raise ValueError('Invalid retry.') + + self._message_retry = retry + + def user_data_set(self, userdata): + """Set the user data variable passed to callbacks. May be any data type.""" + self._userdata = userdata + + def will_set(self, topic, payload=None, qos=0, retain=False, properties=None): + """Set a Will to be sent by the broker in case the client disconnects unexpectedly. + + This must be called before connect() to have any effect. + + topic: The topic that the will message should be published on. + payload: The message to send as a will. If not given, or set to None a + zero length message will be used as the will. Passing an int or float + will result in the payload being converted to a string representing + that number. If you wish to send a true int/float, use struct.pack() to + create the payload you require. + qos: The quality of service level to use for the will. + retain: If set to true, the will message will be set as the "last known + good"/retained message for the topic. + properties: (MQTT v5.0 only) a Properties instance setting the MQTT v5.0 properties + to be included with the will message. Optional - if not set, no properties are sent. + + Raises a ValueError if qos is not 0, 1 or 2, or if topic is None or has + zero string length. + """ + if topic is None or len(topic) == 0: + raise ValueError('Invalid topic.') + + if qos < 0 or qos > 2: + raise ValueError('Invalid QoS level.') + + if properties != None and not isinstance(properties, Properties): + raise ValueError( + "The properties argument must be an instance of the Properties class.") + + if isinstance(payload, unicode): + self._will_payload = payload.encode('utf-8') + elif isinstance(payload, (bytes, bytearray)): + self._will_payload = payload + elif isinstance(payload, (int, float)): + self._will_payload = str(payload).encode('ascii') + elif payload is None: + self._will_payload = b"" + else: + raise TypeError( + 'payload must be a string, bytearray, int, float or None.') + + self._will = True + self._will_topic = topic.encode('utf-8') + self._will_qos = qos + self._will_retain = retain + self._will_properties = properties + + def will_clear(self): + """ Removes a will that was previously configured with will_set(). + + Must be called before connect() to have any effect.""" + self._will = False + self._will_topic = b"" + self._will_payload = b"" + self._will_qos = 0 + self._will_retain = False + + def socket(self): + """Return the socket or ssl object for this client.""" + return self._sock + + def loop_forever(self, timeout=1.0, max_packets=1, retry_first_connection=False): + """This function call loop() for you in an infinite blocking loop. It + is useful for the case where you only want to run the MQTT client loop + in your program. + + loop_forever() will handle reconnecting for you. If you call + disconnect() in a callback it will return. + + + timeout: The time in seconds to wait for incoming/outgoing network + traffic before timing out and returning. + max_packets: Not currently used. + retry_first_connection: Should the first connection attempt be retried on failure. + + Raises socket.error on first connection failures unless retry_first_connection=True + """ + + run = True + + while run: + if self._thread_terminate is True: + break + + if self._state == mqtt_cs_connect_async: + try: + self.reconnect() + except (socket.error, OSError, WebsocketConnectionError): + if not retry_first_connection: + raise + self._easy_log( + MQTT_LOG_DEBUG, "Connection failed, retrying") + self._reconnect_wait() + else: + break + + while run: + rc = MQTT_ERR_SUCCESS + while rc == MQTT_ERR_SUCCESS: + rc = self.loop(timeout, max_packets) + # We don't need to worry about locking here, because we've + # either called loop_forever() when in single threaded mode, or + # in multi threaded mode when loop_stop() has been called and + # so no other threads can access _current_out_packet, + # _out_packet or _messages. + if (self._thread_terminate is True + and self._current_out_packet is None + and len(self._out_packet) == 0 + and len(self._out_messages) == 0): + rc = 1 + run = False + + def should_exit(): + return self._state == mqtt_cs_disconnecting or run is False or self._thread_terminate is True + + if should_exit(): + run = False + else: + self._reconnect_wait() + + if should_exit(): + run = False + else: + try: + self.reconnect() + except (socket.error, OSError, WebsocketConnectionError): + self._easy_log( + MQTT_LOG_DEBUG, "Connection failed, retrying") + + return rc + + def loop_start(self): + """This is part of the threaded client interface. Call this once to + start a new thread to process network traffic. This provides an + alternative to repeatedly calling loop() yourself. + """ + if self._thread is not None: + return MQTT_ERR_INVAL + + self._thread_terminate = False + self._thread = threading.Thread(target=self._thread_main) + self._thread.daemon = True + self._thread.start() + + def loop_stop(self, force=False): + """This is part of the threaded client interface. Call this once to + stop the network thread previously created with loop_start(). This call + will block until the network thread finishes. + + The force parameter is currently ignored. + """ + if self._thread is None: + return MQTT_ERR_INVAL + + self._thread_terminate = True + if threading.current_thread() != self._thread: + self._thread.join() + self._thread = None + + @property + def on_log(self): + """If implemented, called when the client has log information. + Defined to allow debugging.""" + return self._on_log + + @on_log.setter + def on_log(self, func): + """ Define the logging callback implementation. + + Expected signature is: + log_callback(client, userdata, level, buf) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + level: gives the severity of the message and will be one of + MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING, + MQTT_LOG_ERR, and MQTT_LOG_DEBUG. + buf: the message itself + """ + self._on_log = func + + @property + def on_connect(self): + """If implemented, called when the broker responds to our connection + request.""" + return self._on_connect + + @on_connect.setter + def on_connect(self, func): + """ Define the connect callback implementation. + + Expected signature for MQTT v3.1 and v3.1.1 is: + connect_callback(client, userdata, flags, rc, properties=None) + + and for MQTT v5.0: + connect_callback(client, userdata, flags, reasonCode, properties) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + flags: response flags sent by the broker + rc: the connection result + reasonCode: the MQTT v5.0 reason code: an instance of the ReasonCode class. + ReasonCode may be compared to interger. + properties: the MQTT v5.0 properties returned from the broker. An instance + of the Properties class. + For MQTT v3.1 and v3.1.1 properties is not provided but for compatibility + with MQTT v5.0, we recommand adding properties=None. + + flags is a dict that contains response flags from the broker: + flags['session present'] - this flag is useful for clients that are + using clean session set to 0 only. If a client with clean + session=0, that reconnects to a broker that it has previously + connected to, this flag indicates whether the broker still has the + session information for the client. If 1, the session still exists. + + The value of rc indicates success or not: + 0: Connection successful + 1: Connection refused - incorrect protocol version + 2: Connection refused - invalid client identifier + 3: Connection refused - server unavailable + 4: Connection refused - bad username or password + 5: Connection refused - not authorised + 6-255: Currently unused. + """ + with self._callback_mutex: + self._on_connect = func + + @property + def on_subscribe(self): + """If implemented, called when the broker responds to a subscribe + request.""" + return self._on_subscribe + + @on_subscribe.setter + def on_subscribe(self, func): + """ Define the suscribe callback implementation. + + Expected signature for MQTT v3.1.1 and v3.1 is: + subscribe_callback(client, userdata, mid, granted_qos, properties=None) + + and for MQTT v5.0: + subscribe_callback(client, userdata, mid, reasonCodes, properties) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + mid: matches the mid variable returned from the corresponding + subscribe() call. + granted_qos: list of integers that give the QoS level the broker has + granted for each of the different subscription requests. + reasonCodes: the MQTT v5.0 reason codes received from the broker for each + subscription. A list of ReasonCodes instances. + properties: the MQTT v5.0 properties received from the broker. A + list of Properties class instances. + """ + with self._callback_mutex: + self._on_subscribe = func + + @property + def on_message(self): + """If implemented, called when a message has been received on a topic + that the client subscribes to. + + This callback will be called for every message received. Use + message_callback_add() to define multiple callbacks that will be called + for specific topic filters.""" + return self._on_message + + @on_message.setter + def on_message(self, func): + """ Define the message received callback implementation. + + Expected signature is: + on_message_callback(client, userdata, message) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + message: an instance of MQTTMessage. + This is a class with members topic, payload, qos, retain. + """ + with self._callback_mutex: + self._on_message = func + + @property + def on_publish(self): + """If implemented, called when a message that was to be sent using the + publish() call has completed transmission to the broker. + + For messages with QoS levels 1 and 2, this means that the appropriate + handshakes have completed. For QoS 0, this simply means that the message + has left the client. + This callback is important because even if the publish() call returns + success, it does not always mean that the message has been sent.""" + return self._on_publish + + @on_publish.setter + def on_publish(self, func): + """ Define the published message callback implementation. + + Expected signature is: + on_publish_callback(client, userdata, mid) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + mid: matches the mid variable returned from the corresponding + publish() call, to allow outgoing messages to be tracked. + """ + with self._callback_mutex: + self._on_publish = func + + @property + def on_unsubscribe(self): + """If implemented, called when the broker responds to an unsubscribe + request.""" + return self._on_unsubscribe + + @on_unsubscribe.setter + def on_unsubscribe(self, func): + """ Define the unsubscribe callback implementation. + + Expected signature for MQTT v3.1.1 and v3.1 is: + unsubscribe_callback(client, userdata, mid) + + and for MQTT v5.0: + unsubscribe_callback(client, userdata, mid, properties, reasonCodes) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + mid: matches the mid variable returned from the corresponding + unsubscribe() call. + properties: the MQTT v5.0 properties received from the broker. A + list of Properties class instances. + reasonCodes: the MQTT v5.0 reason codes received from the broker for each + unsubscribe topic. A list of ReasonCodes instances + """ + with self._callback_mutex: + self._on_unsubscribe = func + + @property + def on_disconnect(self): + """If implemented, called when the client disconnects from the broker. + """ + return self._on_disconnect + + @on_disconnect.setter + def on_disconnect(self, func): + """ Define the disconnect callback implementation. + + Expected signature for MQTT v3.1.1 and v3.1 is: + disconnect_callback(client, userdata, rc) + + and for MQTT v5.0: + disconnect_callback(client, userdata, reasonCode, properties) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + rc: the disconnection result + The rc parameter indicates the disconnection state. If + MQTT_ERR_SUCCESS (0), the callback was called in response to + a disconnect() call. If any other value the disconnection + was unexpected, such as might be caused by a network error. + """ + with self._callback_mutex: + self._on_disconnect = func + + @property + def on_socket_open(self): + """If implemented, called just after the socket was opend.""" + return self._on_socket_open + + @on_socket_open.setter + def on_socket_open(self, func): + """Define the socket_open callback implementation. + + This should be used to register the socket to an external event loop for reading. + + Expected signature is: + socket_open_callback(client, userdata, socket) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + sock: the socket which was just opened. + """ + with self._callback_mutex: + self._on_socket_open = func + + def _call_socket_open(self): + """Call the socket_open callback with the just-opened socket""" + with self._callback_mutex: + if self.on_socket_open: + with self._in_callback_mutex: + try: + self.on_socket_open(self, self._userdata, self._sock) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_socket_open: %s', err) + if not self.suppress_exceptions: + raise + + @property + def on_socket_close(self): + """If implemented, called just before the socket is closed.""" + return self._on_socket_close + + @on_socket_close.setter + def on_socket_close(self, func): + """Define the socket_close callback implementation. + + This should be used to unregister the socket from an external event loop for reading. + + Expected signature is: + socket_close_callback(client, userdata, socket) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + sock: the socket which is about to be closed. + """ + with self._callback_mutex: + self._on_socket_close = func + + def _call_socket_close(self, sock): + """Call the socket_close callback with the about-to-be-closed socket""" + with self._callback_mutex: + if self.on_socket_close: + with self._in_callback_mutex: + try: + self.on_socket_close(self, self._userdata, sock) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_socket_close: %s', err) + if not self.suppress_exceptions: + raise + + @property + def on_socket_register_write(self): + """If implemented, called when the socket needs writing but can't.""" + return self._on_socket_register_write + + @on_socket_register_write.setter + def on_socket_register_write(self, func): + """Define the socket_register_write callback implementation. + + This should be used to register the socket with an external event loop for writing. + + Expected signature is: + socket_register_write_callback(client, userdata, socket) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + sock: the socket which should be registered for writing + """ + with self._callback_mutex: + self._on_socket_register_write = func + + def _call_socket_register_write(self): + """Call the socket_register_write callback with the unwritable socket""" + if not self._sock or self._registered_write: + return + self._registered_write = True + with self._callback_mutex: + if self.on_socket_register_write: + try: + self.on_socket_register_write( + self, self._userdata, self._sock) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_socket_register_write: %s', err) + if not self.suppress_exceptions: + raise + + @property + def on_socket_unregister_write(self): + """If implemented, called when the socket doesn't need writing anymore.""" + return self._on_socket_unregister_write + + @on_socket_unregister_write.setter + def on_socket_unregister_write(self, func): + """Define the socket_unregister_write callback implementation. + + This should be used to unregister the socket from an external event loop for writing. + + Expected signature is: + socket_unregister_write_callback(client, userdata, socket) + + client: the client instance for this callback + userdata: the private user data as set in Client() or userdata_set() + sock: the socket which should be unregistered for writing + """ + with self._callback_mutex: + self._on_socket_unregister_write = func + + def _call_socket_unregister_write(self, sock=None): + """Call the socket_unregister_write callback with the writable socket""" + sock = sock or self._sock + if not sock or not self._registered_write: + return + self._registered_write = False + + with self._callback_mutex: + if self.on_socket_unregister_write: + try: + self.on_socket_unregister_write(self, self._userdata, sock) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_socket_unregister_write: %s', err) + if not self.suppress_exceptions: + raise + + def message_callback_add(self, sub, callback): + """Register a message callback for a specific topic. + Messages that match 'sub' will be passed to 'callback'. Any + non-matching messages will be passed to the default on_message + callback. + + Call multiple times with different 'sub' to define multiple topic + specific callbacks. + + Topic specific callbacks may be removed with + message_callback_remove().""" + if callback is None or sub is None: + raise ValueError("sub and callback must both be defined.") + + with self._callback_mutex: + self._on_message_filtered[sub] = callback + + def message_callback_remove(self, sub): + """Remove a message callback previously registered with + message_callback_add().""" + if sub is None: + raise ValueError("sub must defined.") + + with self._callback_mutex: + try: + del self._on_message_filtered[sub] + except KeyError: # no such subscription + pass + + # ============================================================ + # Private functions + # ============================================================ + + def _loop_rc_handle(self, rc, properties=None): + if rc: + self._sock_close() + + if self._state == mqtt_cs_disconnecting: + rc = MQTT_ERR_SUCCESS + + self._do_on_disconnect(rc, properties) + + return rc + + def _packet_read(self): + # This gets called if pselect() indicates that there is network data + # available - ie. at least one byte. What we do depends on what data we + # already have. + # If we've not got a command, attempt to read one and save it. This should + # always work because it's only a single byte. + # Then try to read the remaining length. This may fail because it is may + # be more than one byte - will need to save data pending next read if it + # does fail. + # Then try to read the remaining payload, where 'payload' here means the + # combined variable header and actual payload. This is the most likely to + # fail due to longer length, so save current data and current position. + # After all data is read, send to _mqtt_handle_packet() to deal with. + # Finally, free the memory and reset everything to starting conditions. + if self._in_packet['command'] == 0: + try: + command = self._sock_recv(1) + except WouldBlockError: + return MQTT_ERR_AGAIN + except socket.error as err: + self._easy_log( + MQTT_LOG_ERR, 'failed to receive on socket: %s', err) + return 1 + else: + if len(command) == 0: + return 1 + command, = struct.unpack("!B", command) + self._in_packet['command'] = command + + if self._in_packet['have_remaining'] == 0: + # Read remaining + # Algorithm for decoding taken from pseudo code at + # http://publib.boulder.ibm.com/infocenter/wmbhelp/v6r0m0/topic/com.ibm.etools.mft.doc/ac10870_.htm + while True: + try: + byte = self._sock_recv(1) + except WouldBlockError: + return MQTT_ERR_AGAIN + except socket.error as err: + self._easy_log( + MQTT_LOG_ERR, 'failed to receive on socket: %s', err) + return 1 + else: + if len(byte) == 0: + return 1 + byte, = struct.unpack("!B", byte) + self._in_packet['remaining_count'].append(byte) + # Max 4 bytes length for remaining length as defined by protocol. + # Anything more likely means a broken/malicious client. + if len(self._in_packet['remaining_count']) > 4: + return MQTT_ERR_PROTOCOL + + self._in_packet['remaining_length'] += ( + byte & 127) * self._in_packet['remaining_mult'] + self._in_packet['remaining_mult'] = self._in_packet['remaining_mult'] * 128 + + if (byte & 128) == 0: + break + + self._in_packet['have_remaining'] = 1 + self._in_packet['to_process'] = self._in_packet['remaining_length'] + + while self._in_packet['to_process'] > 0: + try: + data = self._sock_recv(self._in_packet['to_process']) + except WouldBlockError: + return MQTT_ERR_AGAIN + except socket.error as err: + self._easy_log( + MQTT_LOG_ERR, 'failed to receive on socket: %s', err) + return 1 + else: + if len(data) == 0: + return 1 + self._in_packet['to_process'] -= len(data) + self._in_packet['packet'] += data + + # All data for this packet is read. + self._in_packet['pos'] = 0 + rc = self._packet_handle() + + # Free data and reset values + self._in_packet = { + 'command': 0, + 'have_remaining': 0, + 'remaining_count': [], + 'remaining_mult': 1, + 'remaining_length': 0, + 'packet': b"", + 'to_process': 0, + 'pos': 0} + + with self._msgtime_mutex: + self._last_msg_in = time_func() + return rc + + def _packet_write(self): + self._current_out_packet_mutex.acquire() + + while self._current_out_packet: + packet = self._current_out_packet + + try: + write_length = self._sock_send( + packet['packet'][packet['pos']:]) + except (AttributeError, ValueError): + self._current_out_packet_mutex.release() + return MQTT_ERR_SUCCESS + except WouldBlockError: + self._current_out_packet_mutex.release() + return MQTT_ERR_AGAIN + except socket.error as err: + self._current_out_packet_mutex.release() + self._easy_log( + MQTT_LOG_ERR, 'failed to receive on socket: %s', err) + return 1 + + if write_length > 0: + packet['to_process'] -= write_length + packet['pos'] += write_length + + if packet['to_process'] == 0: + if (packet['command'] & 0xF0) == PUBLISH and packet['qos'] == 0: + with self._callback_mutex: + if self.on_publish: + with self._in_callback_mutex: + try: + self.on_publish( + self, self._userdata, packet['mid']) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_publish: %s', err) + if not self.suppress_exceptions: + raise + + packet['info']._set_as_published() + + if (packet['command'] & 0xF0) == DISCONNECT: + self._current_out_packet_mutex.release() + + with self._msgtime_mutex: + self._last_msg_out = time_func() + + self._do_on_disconnect(0) + + self._sock_close() + return MQTT_ERR_SUCCESS + + with self._out_packet_mutex: + if len(self._out_packet) > 0: + self._current_out_packet = self._out_packet.popleft() + else: + self._current_out_packet = None + else: + break + + self._current_out_packet_mutex.release() + + with self._msgtime_mutex: + self._last_msg_out = time_func() + + return MQTT_ERR_SUCCESS + + def _easy_log(self, level, fmt, *args): + if self.on_log is not None: + buf = fmt % args + try: + self.on_log(self, self._userdata, level, buf) + except Exception: + # Can't _easy_log this, as we'll recurse until we break + pass # self._logger will pick this up, so we're fine + if self._logger is not None: + level_std = LOGGING_LEVEL[level] + self._logger.log(level_std, fmt, *args) + + def _check_keepalive(self): + if self._keepalive == 0: + return MQTT_ERR_SUCCESS + + now = time_func() + + with self._msgtime_mutex: + last_msg_out = self._last_msg_out + last_msg_in = self._last_msg_in + + if self._sock is not None and (now - last_msg_out >= self._keepalive or now - last_msg_in >= self._keepalive): + if self._state == mqtt_cs_connected and self._ping_t == 0: + self._send_pingreq() + with self._msgtime_mutex: + self._last_msg_out = now + self._last_msg_in = now + else: + self._sock_close() + + if self._state == mqtt_cs_disconnecting: + rc = MQTT_ERR_SUCCESS + else: + rc = 1 + + self._do_on_disconnect(rc) + + def _mid_generate(self): + with self._mid_generate_mutex: + self._last_mid += 1 + if self._last_mid == 65536: + self._last_mid = 1 + return self._last_mid + + @staticmethod + def _topic_wildcard_len_check(topic): + # Search for + or # in a topic. Return MQTT_ERR_INVAL if found. + # Also returns MQTT_ERR_INVAL if the topic string is too long. + # Returns MQTT_ERR_SUCCESS if everything is fine. + if b'+' in topic or b'#' in topic or len(topic) > 65535: + return MQTT_ERR_INVAL + else: + return MQTT_ERR_SUCCESS + + @staticmethod + def _filter_wildcard_len_check(sub): + if (len(sub) == 0 or len(sub) > 65535 + or any(b'+' in p or b'#' in p for p in sub.split(b'/') if len(p) > 1) + or b'#/' in sub): + return MQTT_ERR_INVAL + else: + return MQTT_ERR_SUCCESS + + def _send_pingreq(self): + self._easy_log(MQTT_LOG_DEBUG, "Sending PINGREQ") + rc = self._send_simple_command(PINGREQ) + if rc == MQTT_ERR_SUCCESS: + self._ping_t = time_func() + return rc + + def _send_pingresp(self): + self._easy_log(MQTT_LOG_DEBUG, "Sending PINGRESP") + return self._send_simple_command(PINGRESP) + + def _send_puback(self, mid): + self._easy_log(MQTT_LOG_DEBUG, "Sending PUBACK (Mid: %d)", mid) + return self._send_command_with_mid(PUBACK, mid, False) + + def _send_pubcomp(self, mid): + self._easy_log(MQTT_LOG_DEBUG, "Sending PUBCOMP (Mid: %d)", mid) + return self._send_command_with_mid(PUBCOMP, mid, False) + + def _pack_remaining_length(self, packet, remaining_length): + remaining_bytes = [] + while True: + byte = remaining_length % 128 + remaining_length = remaining_length // 128 + # If there are more digits to encode, set the top bit of this digit + if remaining_length > 0: + byte |= 0x80 + + remaining_bytes.append(byte) + packet.append(byte) + if remaining_length == 0: + # FIXME - this doesn't deal with incorrectly large payloads + return packet + + def _pack_str16(self, packet, data): + if isinstance(data, unicode): + data = data.encode('utf-8') + packet.extend(struct.pack("!H", len(data))) + packet.extend(data) + + def _send_publish(self, mid, topic, payload=b'', qos=0, retain=False, dup=False, info=None, properties=None): + # we assume that topic and payload are already properly encoded + assert not isinstance(topic, unicode) and not isinstance( + payload, unicode) and payload is not None + + if self._sock is None: + return MQTT_ERR_NO_CONN + + command = PUBLISH | ((dup & 0x1) << 3) | (qos << 1) | retain + packet = bytearray() + packet.append(command) + + payloadlen = len(payload) + remaining_length = 2 + len(topic) + payloadlen + + if payloadlen == 0: + if self._protocol == MQTTv5: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending PUBLISH (d%d, q%d, r%d, m%d), '%s', properties=%s (NULL payload)", + dup, qos, retain, mid, topic, properties + ) + else: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending PUBLISH (d%d, q%d, r%d, m%d), '%s' (NULL payload)", + dup, qos, retain, mid, topic + ) + else: + if self._protocol == MQTTv5: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending PUBLISH (d%d, q%d, r%d, m%d), '%s', properties=%s, ... (%d bytes)", + dup, qos, retain, mid, topic, properties, payloadlen + ) + else: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending PUBLISH (d%d, q%d, r%d, m%d), '%s', ... (%d bytes)", + dup, qos, retain, mid, topic, payloadlen + ) + + if qos > 0: + # For message id + remaining_length += 2 + + if self._protocol == MQTTv5: + if properties == None: + packed_properties = b'\x00' + else: + packed_properties = properties.pack() + remaining_length += len(packed_properties) + + self._pack_remaining_length(packet, remaining_length) + self._pack_str16(packet, topic) + + if qos > 0: + # For message id + packet.extend(struct.pack("!H", mid)) + + if self._protocol == MQTTv5: + packet.extend(packed_properties) + + packet.extend(payload) + + return self._packet_queue(PUBLISH, packet, mid, qos, info) + + def _send_pubrec(self, mid): + self._easy_log(MQTT_LOG_DEBUG, "Sending PUBREC (Mid: %d)", mid) + return self._send_command_with_mid(PUBREC, mid, False) + + def _send_pubrel(self, mid): + self._easy_log(MQTT_LOG_DEBUG, "Sending PUBREL (Mid: %d)", mid) + return self._send_command_with_mid(PUBREL | 2, mid, False) + + def _send_command_with_mid(self, command, mid, dup): + # For PUBACK, PUBCOMP, PUBREC, and PUBREL + if dup: + command |= 0x8 + + remaining_length = 2 + packet = struct.pack('!BBH', command, remaining_length, mid) + return self._packet_queue(command, packet, mid, 1) + + def _send_simple_command(self, command): + # For DISCONNECT, PINGREQ and PINGRESP + remaining_length = 0 + packet = struct.pack('!BB', command, remaining_length) + return self._packet_queue(command, packet, 0, 0) + + def _send_connect(self, keepalive): + proto_ver = self._protocol + # hard-coded UTF-8 encoded string + protocol = b"MQTT" if proto_ver >= MQTTv311 else b"MQIsdp" + + remaining_length = 2 + len(protocol) + 1 + \ + 1 + 2 + 2 + len(self._client_id) + + connect_flags = 0 + if self._protocol == MQTTv5: + if self._clean_start == True: + connect_flags |= 0x02 + elif self._clean_start == MQTT_CLEAN_START_FIRST_ONLY and self._mqttv5_first_connect: + connect_flags |= 0x02 + elif self._clean_session: + connect_flags |= 0x02 + + if self._will: + remaining_length += 2 + \ + len(self._will_topic) + 2 + len(self._will_payload) + connect_flags |= 0x04 | ((self._will_qos & 0x03) << 3) | ( + (self._will_retain & 0x01) << 5) + + if self._username is not None: + remaining_length += 2 + len(self._username) + connect_flags |= 0x80 + if self._password is not None: + connect_flags |= 0x40 + remaining_length += 2 + len(self._password) + + if self._protocol == MQTTv5: + if self._connect_properties == None: + packed_connect_properties = b'\x00' + else: + packed_connect_properties = self._connect_properties.pack() + remaining_length += len(packed_connect_properties) + if self._will: + if self._will_properties == None: + packed_will_properties = b'\x00' + else: + packed_will_properties = self._will_properties.pack() + remaining_length += len(packed_will_properties) + + command = CONNECT + packet = bytearray() + packet.append(command) + + # as per the mosquitto broker, if the MSB of this version is set + # to 1, then it treats the connection as a bridge + if self._client_mode == MQTT_BRIDGE: + proto_ver |= 0x80 + + self._pack_remaining_length(packet, remaining_length) + packet.extend(struct.pack("!H" + str(len(protocol)) + "sBBH", len(protocol), protocol, proto_ver, connect_flags, + keepalive)) + + if self._protocol == MQTTv5: + packet += packed_connect_properties + + self._pack_str16(packet, self._client_id) + + if self._will: + if self._protocol == MQTTv5: + packet += packed_will_properties + self._pack_str16(packet, self._will_topic) + self._pack_str16(packet, self._will_payload) + + if self._username is not None: + self._pack_str16(packet, self._username) + + if self._password is not None: + self._pack_str16(packet, self._password) + + self._keepalive = keepalive + if self._protocol == MQTTv5: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending CONNECT (u%d, p%d, wr%d, wq%d, wf%d, c%d, k%d) client_id=%s properties=%s", + (connect_flags & 0x80) >> 7, + (connect_flags & 0x40) >> 6, + (connect_flags & 0x20) >> 5, + (connect_flags & 0x18) >> 3, + (connect_flags & 0x4) >> 2, + (connect_flags & 0x2) >> 1, + keepalive, + self._client_id, + self._connect_properties + ) + else: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending CONNECT (u%d, p%d, wr%d, wq%d, wf%d, c%d, k%d) client_id=%s", + (connect_flags & 0x80) >> 7, + (connect_flags & 0x40) >> 6, + (connect_flags & 0x20) >> 5, + (connect_flags & 0x18) >> 3, + (connect_flags & 0x4) >> 2, + (connect_flags & 0x2) >> 1, + keepalive, + self._client_id + ) + return self._packet_queue(command, packet, 0, 0) + + def _send_disconnect(self, reasoncode=None, properties=None): + if self._protocol == MQTTv5: + self._easy_log(MQTT_LOG_DEBUG, "Sending DISCONNECT reasonCode=%s properties=%s", + reasoncode, + properties + ) + else: + self._easy_log(MQTT_LOG_DEBUG, "Sending DISCONNECT") + + remaining_length = 0 + + command = DISCONNECT + packet = bytearray() + packet.append(command) + + if self._protocol == MQTTv5: + if properties != None or reasoncode != None: + if reasoncode == None: + reasoncode = ReasonCodes(DISCONNECT >> 4, identifier=0) + remaining_length += 1 + if properties != None: + packed_props = properties.pack() + remaining_length += len(packed_props) + + self._pack_remaining_length(packet, remaining_length) + + if self._protocol == MQTTv5: + if reasoncode != None: + packet += reasoncode.pack() + if properties != None: + packet += packed_props + + return self._packet_queue(command, packet, 0, 0) + + def _send_subscribe(self, dup, topics, properties=None): + remaining_length = 2 + if self._protocol == MQTTv5: + if properties == None: + packed_subscribe_properties = b'\x00' + else: + packed_subscribe_properties = properties.pack() + remaining_length += len(packed_subscribe_properties) + for t, _ in topics: + remaining_length += 2 + len(t) + 1 + + command = SUBSCRIBE | (dup << 3) | 0x2 + packet = bytearray() + packet.append(command) + self._pack_remaining_length(packet, remaining_length) + local_mid = self._mid_generate() + packet.extend(struct.pack("!H", local_mid)) + + if self._protocol == MQTTv5: + packet += packed_subscribe_properties + + for t, q in topics: + self._pack_str16(packet, t) + if self._protocol == MQTTv5: + packet += q.pack() + else: + packet.append(q) + + self._easy_log( + MQTT_LOG_DEBUG, + "Sending SUBSCRIBE (d%d, m%d) %s", + dup, + local_mid, + topics, + ) + return (self._packet_queue(command, packet, local_mid, 1), local_mid) + + def _send_unsubscribe(self, dup, topics, properties=None): + remaining_length = 2 + if self._protocol == MQTTv5: + if properties == None: + packed_unsubscribe_properties = b'\x00' + else: + packed_unsubscribe_properties = properties.pack() + remaining_length += len(packed_unsubscribe_properties) + for t in topics: + remaining_length += 2 + len(t) + + command = UNSUBSCRIBE | (dup << 3) | 0x2 + packet = bytearray() + packet.append(command) + self._pack_remaining_length(packet, remaining_length) + local_mid = self._mid_generate() + packet.extend(struct.pack("!H", local_mid)) + + if self._protocol == MQTTv5: + packet += packed_unsubscribe_properties + + for t in topics: + self._pack_str16(packet, t) + + # topics_repr = ", ".join("'"+topic.decode('utf8')+"'" for topic in topics) + if self._protocol == MQTTv5: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending UNSUBSCRIBE (d%d, m%d) %s %s", + dup, + local_mid, + properties, + topics, + ) + else: + self._easy_log( + MQTT_LOG_DEBUG, + "Sending UNSUBSCRIBE (d%d, m%d) %s", + dup, + local_mid, + topics, + ) + return (self._packet_queue(command, packet, local_mid, 1), local_mid) + + def _message_retry_check_actual(self, messages, mutex): + with mutex: + now = time_func() + for m in messages.values(): + if m.timestamp + self._message_retry < now: + if m.state == mqtt_ms_wait_for_puback or m.state == mqtt_ms_wait_for_pubrec: + m.timestamp = now + m.dup = True + self._send_publish( + m.mid, + m.topic.encode('utf-8'), + m.payload, + m.qos, + m.retain, + m.dup, + properties=m.properties, + ) + elif m.state == mqtt_ms_wait_for_pubrel: + m.timestamp = now + self._send_pubrec(m.mid) + elif m.state == mqtt_ms_wait_for_pubcomp: + m.timestamp = now + self._send_pubrel(m.mid) + + def _message_retry_check(self): + self._message_retry_check_actual( + self._out_messages, self._out_message_mutex) + self._message_retry_check_actual( + self._in_messages, self._in_message_mutex) + + def _check_clean_session(self): + if self._protocol == MQTTv5: + if self._clean_start == MQTT_CLEAN_START_FIRST_ONLY: + return self._mqttv5_first_connect + else: + return self._clean_start + else: + return self._clean_session + + def _messages_reconnect_reset_out(self): + with self._out_message_mutex: + self._inflight_messages = 0 + for m in self._out_messages.values(): + m.timestamp = 0 + if self._max_inflight_messages == 0 or self._inflight_messages < self._max_inflight_messages: + if m.qos == 0: + m.state = mqtt_ms_publish + elif m.qos == 1: + # self._inflight_messages = self._inflight_messages + 1 + if m.state == mqtt_ms_wait_for_puback: + m.dup = True + m.state = mqtt_ms_publish + elif m.qos == 2: + # self._inflight_messages = self._inflight_messages + 1 + if self._check_clean_session(): + if m.state != mqtt_ms_publish: + m.dup = True + m.state = mqtt_ms_publish + else: + if m.state == mqtt_ms_wait_for_pubcomp: + m.state = mqtt_ms_resend_pubrel + else: + if m.state == mqtt_ms_wait_for_pubrec: + m.dup = True + m.state = mqtt_ms_publish + else: + m.state = mqtt_ms_queued + + def _messages_reconnect_reset_in(self): + with self._in_message_mutex: + if self._check_clean_session(): + self._in_messages = collections.OrderedDict() + return + for m in self._in_messages.values(): + m.timestamp = 0 + if m.qos != 2: + self._in_messages.pop(m.mid) + else: + # Preserve current state + pass + + def _messages_reconnect_reset(self): + self._messages_reconnect_reset_out() + self._messages_reconnect_reset_in() + + def _packet_queue(self, command, packet, mid, qos, info=None): + mpkt = { + 'command': command, + 'mid': mid, + 'qos': qos, + 'pos': 0, + 'to_process': len(packet), + 'packet': packet, + 'info': info} + + with self._out_packet_mutex: + self._out_packet.append(mpkt) + if self._current_out_packet_mutex.acquire(False): + if self._current_out_packet is None and len(self._out_packet) > 0: + self._current_out_packet = self._out_packet.popleft() + self._current_out_packet_mutex.release() + + # Write a single byte to sockpairW (connected to sockpairR) to break + # out of select() if in threaded mode. + try: + self._sockpairW.send(sockpair_data) + except socket.error as err: + if err.errno != EAGAIN: + raise + + if self._thread is None: + if self._in_callback_mutex.acquire(False): + self._in_callback_mutex.release() + return self.loop_write() + + self._call_socket_register_write() + + return MQTT_ERR_SUCCESS + + def _packet_handle(self): + cmd = self._in_packet['command'] & 0xF0 + if cmd == PINGREQ: + return self._handle_pingreq() + elif cmd == PINGRESP: + return self._handle_pingresp() + elif cmd == PUBACK: + return self._handle_pubackcomp("PUBACK") + elif cmd == PUBCOMP: + return self._handle_pubackcomp("PUBCOMP") + elif cmd == PUBLISH: + return self._handle_publish() + elif cmd == PUBREC: + return self._handle_pubrec() + elif cmd == PUBREL: + return self._handle_pubrel() + elif cmd == CONNACK: + return self._handle_connack() + elif cmd == SUBACK: + return self._handle_suback() + elif cmd == UNSUBACK: + return self._handle_unsuback() + elif cmd == DISCONNECT and self._protocol == MQTTv5: # only allowed in MQTT 5.0 + return self._handle_disconnect() + else: + # If we don't recognise the command, return an error straight away. + self._easy_log(MQTT_LOG_ERR, "Error: Unrecognised command %s", cmd) + return MQTT_ERR_PROTOCOL + + def _handle_pingreq(self): + if self._in_packet['remaining_length'] != 0: + return MQTT_ERR_PROTOCOL + + self._easy_log(MQTT_LOG_DEBUG, "Received PINGREQ") + return self._send_pingresp() + + def _handle_pingresp(self): + if self._in_packet['remaining_length'] != 0: + return MQTT_ERR_PROTOCOL + + # No longer waiting for a PINGRESP. + self._ping_t = 0 + self._easy_log(MQTT_LOG_DEBUG, "Received PINGRESP") + return MQTT_ERR_SUCCESS + + def _handle_connack(self): + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] < 2: + return MQTT_ERR_PROTOCOL + elif self._in_packet['remaining_length'] != 2: + return MQTT_ERR_PROTOCOL + + if self._protocol == MQTTv5: + (flags, result) = struct.unpack( + "!BB", self._in_packet['packet'][:2]) + reason = ReasonCodes(CONNACK >> 4, identifier=result) + properties = Properties(CONNACK >> 4) + properties.unpack(self._in_packet['packet'][2:]) + else: + (flags, result) = struct.unpack("!BB", self._in_packet['packet']) + if self._protocol == MQTTv311: + if result == CONNACK_REFUSED_PROTOCOL_VERSION: + self._easy_log( + MQTT_LOG_DEBUG, + "Received CONNACK (%s, %s), attempting downgrade to MQTT v3.1.", + flags, result + ) + # Downgrade to MQTT v3.1 + self._protocol = MQTTv31 + return self.reconnect() + elif (result == CONNACK_REFUSED_IDENTIFIER_REJECTED + and self._client_id == b''): + self._easy_log( + MQTT_LOG_DEBUG, + "Received CONNACK (%s, %s), attempting to use non-empty CID", + flags, result, + ) + self._client_id = base62(uuid.uuid4().int, padding=22) + return self.reconnect() + + if result == 0: + self._state = mqtt_cs_connected + self._reconnect_delay = None + + if self._protocol == MQTTv5: + self._easy_log( + MQTT_LOG_DEBUG, "Received CONNACK (%s, %s) properties=%s", flags, reason, properties) + else: + self._easy_log( + MQTT_LOG_DEBUG, "Received CONNACK (%s, %s)", flags, result) + + # it won't be the first successful connect any more + self._mqttv5_first_connect = False + + with self._callback_mutex: + if self.on_connect: + flags_dict = {} + flags_dict['session present'] = flags & 0x01 + with self._in_callback_mutex: + try: + if self._protocol == MQTTv5: + self.on_connect(self, self._userdata, + flags_dict, reason, properties) + else: + self.on_connect( + self, self._userdata, flags_dict, result) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_connect: %s', err) + if not self.suppress_exceptions: + raise + + if result == 0: + rc = 0 + with self._out_message_mutex: + for m in self._out_messages.values(): + m.timestamp = time_func() + if m.state == mqtt_ms_queued: + self.loop_write() # Process outgoing messages that have just been queued up + return MQTT_ERR_SUCCESS + + if m.qos == 0: + with self._in_callback_mutex: # Don't call loop_write after _send_publish() + rc = self._send_publish( + m.mid, + m.topic.encode('utf-8'), + m.payload, + m.qos, + m.retain, + m.dup, + properties=m.properties + ) + if rc != 0: + return rc + elif m.qos == 1: + if m.state == mqtt_ms_publish: + self._inflight_messages += 1 + m.state = mqtt_ms_wait_for_puback + with self._in_callback_mutex: # Don't call loop_write after _send_publish() + rc = self._send_publish( + m.mid, + m.topic.encode('utf-8'), + m.payload, + m.qos, + m.retain, + m.dup, + properties=m.properties + ) + if rc != 0: + return rc + elif m.qos == 2: + if m.state == mqtt_ms_publish: + self._inflight_messages += 1 + m.state = mqtt_ms_wait_for_pubrec + with self._in_callback_mutex: # Don't call loop_write after _send_publish() + rc = self._send_publish( + m.mid, + m.topic.encode('utf-8'), + m.payload, + m.qos, + m.retain, + m.dup, + properties=m.properties + ) + if rc != 0: + return rc + elif m.state == mqtt_ms_resend_pubrel: + self._inflight_messages += 1 + m.state = mqtt_ms_wait_for_pubcomp + with self._in_callback_mutex: # Don't call loop_write after _send_publish() + rc = self._send_pubrel(m.mid) + if rc != 0: + return rc + self.loop_write() # Process outgoing messages that have just been queued up + + return rc + elif result > 0 and result < 6: + return MQTT_ERR_CONN_REFUSED + else: + return MQTT_ERR_PROTOCOL + + def _handle_disconnect(self): + packet_type = DISCONNECT >> 4 + reasonCode = properties = None + if self._in_packet['remaining_length'] > 2: + reasonCode = ReasonCodes(packet_type) + reasonCode.unpack(self._in_packet['packet']) + if self._in_packet['remaining_length'] > 3: + properties = Properties(packet_type) + props, props_len = properties.unpack( + self._in_packet['packet'][1:]) + self._easy_log(MQTT_LOG_DEBUG, "Received DISCONNECT %s %s", + reasonCode, + properties + ) + + self._loop_rc_handle(reasonCode, properties) + + return MQTT_ERR_SUCCESS + + def _handle_suback(self): + self._easy_log(MQTT_LOG_DEBUG, "Received SUBACK") + pack_format = "!H" + str(len(self._in_packet['packet']) - 2) + 's' + (mid, packet) = struct.unpack(pack_format, self._in_packet['packet']) + + if self._protocol == MQTTv5: + properties = Properties(SUBACK >> 4) + props, props_len = properties.unpack(packet) + reasoncodes = [] + for c in packet[props_len:]: + if sys.version_info[0] < 3: + c = ord(c) + reasoncodes.append(ReasonCodes(SUBACK >> 4, identifier=c)) + else: + pack_format = "!" + "B" * len(packet) + granted_qos = struct.unpack(pack_format, packet) + + with self._callback_mutex: + if self.on_subscribe: + with self._in_callback_mutex: # Don't call loop_write after _send_publish() + try: + if self._protocol == MQTTv5: + self.on_subscribe( + self, self._userdata, mid, reasoncodes, properties) + else: + self.on_subscribe( + self, self._userdata, mid, granted_qos) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_subscribe: %s', err) + if not self.suppress_exceptions: + raise + + return MQTT_ERR_SUCCESS + + def _handle_publish(self): + rc = 0 + + header = self._in_packet['command'] + message = MQTTMessage() + message.dup = (header & 0x08) >> 3 + message.qos = (header & 0x06) >> 1 + message.retain = (header & 0x01) + + pack_format = "!H" + str(len(self._in_packet['packet']) - 2) + 's' + (slen, packet) = struct.unpack(pack_format, self._in_packet['packet']) + pack_format = '!' + str(slen) + 's' + str(len(packet) - slen) + 's' + (topic, packet) = struct.unpack(pack_format, packet) + + if self._protocol != MQTTv5 and len(topic) == 0: + return MQTT_ERR_PROTOCOL + + # Handle topics with invalid UTF-8 + # This replaces an invalid topic with a message and the hex + # representation of the topic for logging. When the user attempts to + # access message.topic in the callback, an exception will be raised. + try: + print_topic = topic.decode('utf-8') + except UnicodeDecodeError: + print_topic = "TOPIC WITH INVALID UTF-8: " + str(topic) + + message.topic = topic + + if message.qos > 0: + pack_format = "!H" + str(len(packet) - 2) + 's' + (message.mid, packet) = struct.unpack(pack_format, packet) + + if self._protocol == MQTTv5: + message.properties = Properties(PUBLISH >> 4) + props, props_len = message.properties.unpack(packet) + packet = packet[props_len:] + + message.payload = packet + + if self._protocol == MQTTv5: + self._easy_log( + MQTT_LOG_DEBUG, + "Received PUBLISH (d%d, q%d, r%d, m%d), '%s', properties=%s, ... (%d bytes)", + message.dup, message.qos, message.retain, message.mid, + print_topic, message.properties, len(message.payload) + ) + else: + self._easy_log( + MQTT_LOG_DEBUG, + "Received PUBLISH (d%d, q%d, r%d, m%d), '%s', ... (%d bytes)", + message.dup, message.qos, message.retain, message.mid, + print_topic, len(message.payload) + ) + + message.timestamp = time_func() + if message.qos == 0: + self._handle_on_message(message) + return MQTT_ERR_SUCCESS + elif message.qos == 1: + rc = self._send_puback(message.mid) + self._handle_on_message(message) + return rc + elif message.qos == 2: + rc = self._send_pubrec(message.mid) + message.state = mqtt_ms_wait_for_pubrel + with self._in_message_mutex: + self._in_messages[message.mid] = message + return rc + else: + return MQTT_ERR_PROTOCOL + + def _handle_pubrel(self): + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] < 2: + return MQTT_ERR_PROTOCOL + elif self._in_packet['remaining_length'] != 2: + return MQTT_ERR_PROTOCOL + + mid, = struct.unpack("!H", self._in_packet['packet']) + self._easy_log(MQTT_LOG_DEBUG, "Received PUBREL (Mid: %d)", mid) + + with self._in_message_mutex: + if mid in self._in_messages: + # Only pass the message on if we have removed it from the queue - this + # prevents multiple callbacks for the same message. + message = self._in_messages.pop(mid) + self._handle_on_message(message) + self._inflight_messages -= 1 + if self._max_inflight_messages > 0: + with self._out_message_mutex: + rc = self._update_inflight() + if rc != MQTT_ERR_SUCCESS: + return rc + + # FIXME: this should only be done if the message is known + # If unknown it's a protocol error and we should close the connection. + # But since we don't have (on disk) persistence for the session, it + # is possible that we must known about this message. + # Choose to acknwoledge this messsage (and thus losing a message) but + # avoid hanging. See #284. + return self._send_pubcomp(mid) + + def _update_inflight(self): + # Dont lock message_mutex here + for m in self._out_messages.values(): + if self._inflight_messages < self._max_inflight_messages: + if m.qos > 0 and m.state == mqtt_ms_queued: + self._inflight_messages += 1 + if m.qos == 1: + m.state = mqtt_ms_wait_for_puback + elif m.qos == 2: + m.state = mqtt_ms_wait_for_pubrec + rc = self._send_publish( + m.mid, + m.topic.encode('utf-8'), + m.payload, + m.qos, + m.retain, + m.dup, + properties=m.properties, + ) + if rc != 0: + return rc + else: + return MQTT_ERR_SUCCESS + return MQTT_ERR_SUCCESS + + def _handle_pubrec(self): + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] < 2: + return MQTT_ERR_PROTOCOL + elif self._in_packet['remaining_length'] != 2: + return MQTT_ERR_PROTOCOL + + mid, = struct.unpack("!H", self._in_packet['packet'][:2]) + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] > 2: + reasonCode = ReasonCodes(PUBREC >> 4) + reasonCode.unpack(self._in_packet['packet'][2:]) + if self._in_packet['remaining_length'] > 3: + properties = Properties(PUBREC >> 4) + props, props_len = properties.unpack( + self._in_packet['packet'][3:]) + self._easy_log(MQTT_LOG_DEBUG, "Received PUBREC (Mid: %d)", mid) + + with self._out_message_mutex: + if mid in self._out_messages: + msg = self._out_messages[mid] + msg.state = mqtt_ms_wait_for_pubcomp + msg.timestamp = time_func() + return self._send_pubrel(mid) + + return MQTT_ERR_SUCCESS + + def _handle_unsuback(self): + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] < 4: + return MQTT_ERR_PROTOCOL + elif self._in_packet['remaining_length'] != 2: + return MQTT_ERR_PROTOCOL + + mid, = struct.unpack("!H", self._in_packet['packet'][:2]) + if self._protocol == MQTTv5: + packet = self._in_packet['packet'][2:] + properties = Properties(UNSUBACK >> 4) + props, props_len = properties.unpack(packet) + reasoncodes = [] + for c in packet[props_len:]: + if sys.version_info[0] < 3: + c = ord(c) + reasoncodes.append(ReasonCodes(UNSUBACK >> 4, identifier=c)) + if len(reasoncodes) == 1: + reasoncodes = reasoncodes[0] + + self._easy_log(MQTT_LOG_DEBUG, "Received UNSUBACK (Mid: %d)", mid) + with self._callback_mutex: + if self.on_unsubscribe: + with self._in_callback_mutex: + try: + if self._protocol == MQTTv5: + self.on_unsubscribe( + self, self._userdata, mid, properties, reasoncodes) + else: + self.on_unsubscribe(self, self._userdata, mid) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_unsubscribe: %s', err) + if not self.suppress_exceptions: + raise + return MQTT_ERR_SUCCESS + + def _do_on_disconnect(self, rc, properties=None): + with self._callback_mutex: + if self.on_disconnect: + with self._in_callback_mutex: + try: + if properties: + self.on_disconnect( + self, self._userdata, rc, properties) + else: + self.on_disconnect(self, self._userdata, rc) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_disconnect: %s', err) + if not self.suppress_exceptions: + raise + + def _do_on_publish(self, mid): + with self._callback_mutex: + if self.on_publish: + with self._in_callback_mutex: + try: + self.on_publish(self, self._userdata, mid) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_publish: %s', err) + if not self.suppress_exceptions: + raise + + msg = self._out_messages.pop(mid) + msg.info._set_as_published() + if msg.qos > 0: + self._inflight_messages -= 1 + if self._max_inflight_messages > 0: + rc = self._update_inflight() + if rc != MQTT_ERR_SUCCESS: + return rc + return MQTT_ERR_SUCCESS + + def _handle_pubackcomp(self, cmd): + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] < 2: + return MQTT_ERR_PROTOCOL + elif self._in_packet['remaining_length'] != 2: + return MQTT_ERR_PROTOCOL + + packet_type = PUBACK if cmd == "PUBACK" else PUBCOMP + packet_type = packet_type >> 4 + mid, = struct.unpack("!H", self._in_packet['packet'][:2]) + if self._protocol == MQTTv5: + if self._in_packet['remaining_length'] > 2: + reasonCode = ReasonCodes(packet_type) + reasonCode.unpack(self._in_packet['packet'][2:]) + if self._in_packet['remaining_length'] > 3: + properties = Properties(packet_type) + props, props_len = properties.unpack( + self._in_packet['packet'][3:]) + self._easy_log(MQTT_LOG_DEBUG, "Received %s (Mid: %d)", cmd, mid) + + with self._out_message_mutex: + if mid in self._out_messages: + # Only inform the client the message has been sent once. + rc = self._do_on_publish(mid) + return rc + + return MQTT_ERR_SUCCESS + + def _handle_on_message(self, message): + matched = False + with self._callback_mutex: + try: + topic = message.topic + except UnicodeDecodeError: + topic = None + + if topic is not None: + for callback in self._on_message_filtered.iter_match(message.topic): + with self._in_callback_mutex: + try: + callback(self, self._userdata, message) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, + 'Caught exception in user defined callback function %s: %s', + callback.__name__, + err + ) + if not self.suppress_exceptions: + raise + matched = True + + if matched == False and self.on_message: + with self._in_callback_mutex: + try: + self.on_message(self, self._userdata, message) + except Exception as err: + self._easy_log( + MQTT_LOG_ERR, 'Caught exception in on_message: %s', err) + if not self.suppress_exceptions: + raise + + def _thread_main(self): + self.loop_forever(retry_first_connection=True) + + def _reconnect_wait(self): + # See reconnect_delay_set for details + now = time_func() + with self._reconnect_delay_mutex: + if self._reconnect_delay is None: + self._reconnect_delay = self._reconnect_min_delay + else: + self._reconnect_delay = min( + self._reconnect_delay * 2, + self._reconnect_max_delay, + ) + + target_time = now + self._reconnect_delay + + remaining = target_time - now + while (self._state != mqtt_cs_disconnecting + and not self._thread_terminate + and remaining > 0): + + time.sleep(min(remaining, 1)) + remaining = target_time - time_func() + + @staticmethod + def _proxy_is_valid(p): + def check(t, a): + return (socks is not None and + t in set([socks.HTTP, socks.SOCKS4, socks.SOCKS5]) and a) + + if isinstance(p, dict): + return check(p.get("proxy_type"), p.get("proxy_addr")) + elif isinstance(p, (list, tuple)): + return len(p) == 6 and check(p[0], p[1]) + else: + return False + + def _get_proxy(self): + if socks is None: + return None + + # First, check if the user explicitly passed us a proxy to use + if self._proxy_is_valid(self._proxy): + return self._proxy + + # Next, check for an mqtt_proxy environment variable as long as the host + # we're trying to connect to isn't listed under the no_proxy environment + # variable (matches built-in module urllib's behavior) + if not (hasattr(urllib_dot_request, "proxy_bypass") and + urllib_dot_request.proxy_bypass(self._host)): + env_proxies = urllib_dot_request.getproxies() + if "mqtt" in env_proxies: + parts = urllib_dot_parse.urlparse(env_proxies["mqtt"]) + if parts.scheme == "http": + proxy = { + "proxy_type": socks.HTTP, + "proxy_addr": parts.hostname, + "proxy_port": parts.port + } + return proxy + elif parts.scheme == "socks": + proxy = { + "proxy_type": socks.SOCKS5, + "proxy_addr": parts.hostname, + "proxy_port": parts.port + } + return proxy + + # Finally, check if the user has monkeypatched the PySocks library with + # a default proxy + socks_default = socks.get_default_proxy() + if self._proxy_is_valid(socks_default): + proxy_keys = ("proxy_type", "proxy_addr", "proxy_port", + "proxy_rdns", "proxy_username", "proxy_password") + return dict(zip(proxy_keys, socks_default)) + + # If we didn't find a proxy through any of the above methods, return + # None to indicate that the connection should be handled normally + return None + + def _create_socket_connection(self): + proxy = self._get_proxy() + addr = (self._host, self._port) + source = (self._bind_address, self._bind_port) + + + if sys.version_info < (2, 7) or (3, 0) < sys.version_info < (3, 2): + # Have to short-circuit here because of unsupported source_address + # param in earlier Python versions. + return socket.create_connection(addr, timeout=self._keepalive) + + if proxy: + return socks.create_connection(addr, source_address=source, timeout=self._keepalive, **proxy) + else: + return socket.create_connection(addr, source_address=source, timeout=self._keepalive) + + +# Compatibility class for easy porting from mosquitto.py. +class Mosquitto(Client): + def __init__(self, client_id="", clean_session=True, userdata=None): + super(Mosquitto, self).__init__(client_id, clean_session, userdata) + + +class WebsocketWrapper(object): + OPCODE_CONTINUATION = 0x0 + OPCODE_TEXT = 0x1 + OPCODE_BINARY = 0x2 + OPCODE_CONNCLOSE = 0x8 + OPCODE_PING = 0x9 + OPCODE_PONG = 0xa + + def __init__(self, socket, host, port, is_ssl, path, extra_headers): + + self.connected = False + + self._ssl = is_ssl + self._host = host + self._port = port + self._socket = socket + self._path = path + + self._sendbuffer = bytearray() + self._readbuffer = bytearray() + + self._requested_size = 0 + self._payload_head = 0 + self._readbuffer_head = 0 + + self._do_handshake(extra_headers) + + def __del__(self): + + self._sendbuffer = None + self._readbuffer = None + + def _do_handshake(self, extra_headers): + + sec_websocket_key = uuid.uuid4().bytes + sec_websocket_key = base64.b64encode(sec_websocket_key) + + websocket_headers = { + "Host": "{self._host:s}:{self._port:d}".format(self=self), + "Upgrade": "websocket", + "Connection": "Upgrade", + "Origin": "https://{self._host:s}:{self._port:d}".format(self=self), + "Sec-WebSocket-Key": sec_websocket_key.decode("utf8"), + "Sec-Websocket-Version": "13", + "Sec-Websocket-Protocol": "mqtt", + } + + # This is checked in ws_set_options so it will either be None, a + # dictionary, or a callable + if isinstance(extra_headers, dict): + websocket_headers.update(extra_headers) + elif callable(extra_headers): + websocket_headers = extra_headers(websocket_headers) + + header = "\r\n".join([ + "GET {self._path} HTTP/1.1".format(self=self), + "\r\n".join("{}: {}".format(i, j) + for i, j in websocket_headers.items()), + "\r\n", + ]).encode("utf8") + + self._socket.send(header) + + has_secret = False + has_upgrade = False + + while True: + # read HTTP response header as lines + byte = self._socket.recv(1) + + self._readbuffer.extend(byte) + + # line end + if byte == b"\n": + if len(self._readbuffer) > 2: + # check upgrade + if b"connection" in str(self._readbuffer).lower().encode('utf-8'): + if b"upgrade" not in str(self._readbuffer).lower().encode('utf-8'): + raise WebsocketConnectionError( + "WebSocket handshake error, connection not upgraded") + else: + has_upgrade = True + + # check key hash + if b"sec-websocket-accept" in str(self._readbuffer).lower().encode('utf-8'): + GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + server_hash = self._readbuffer.decode( + 'utf-8').split(": ", 1)[1] + server_hash = server_hash.strip().encode('utf-8') + + client_hash = sec_websocket_key.decode('utf-8') + GUID + client_hash = hashlib.sha1(client_hash.encode('utf-8')) + client_hash = base64.b64encode(client_hash.digest()) + + if server_hash != client_hash: + raise WebsocketConnectionError( + "WebSocket handshake error, invalid secret key") + else: + has_secret = True + else: + # ending linebreak + break + + # reset linebuffer + self._readbuffer = bytearray() + + # connection reset + elif not byte: + raise WebsocketConnectionError("WebSocket handshake error") + + if not has_upgrade or not has_secret: + raise WebsocketConnectionError("WebSocket handshake error") + + self._readbuffer = bytearray() + self.connected = True + + def _create_frame(self, opcode, data, do_masking=1): + + header = bytearray() + length = len(data) + + mask_key = bytearray(os.urandom(4)) + mask_flag = do_masking + + # 1 << 7 is the final flag, we don't send continuated data + header.append(1 << 7 | opcode) + + if length < 126: + header.append(mask_flag << 7 | length) + + elif length < 65536: + header.append(mask_flag << 7 | 126) + header += struct.pack("!H", length) + + elif length < 0x8000000000000001: + header.append(mask_flag << 7 | 127) + header += struct.pack("!Q", length) + + else: + raise ValueError("Maximum payload size is 2^63") + + if mask_flag == 1: + for index in range(length): + data[index] ^= mask_key[index % 4] + data = mask_key + data + + return header + data + + def _buffered_read(self, length): + + # try to recv and strore needed bytes + wanted_bytes = length - (len(self._readbuffer) - self._readbuffer_head) + if wanted_bytes > 0: + + data = self._socket.recv(wanted_bytes) + + if not data: + raise socket.error(errno.ECONNABORTED, 0) + else: + self._readbuffer.extend(data) + + if len(data) < wanted_bytes: + raise socket.error(EAGAIN, 0) + + self._readbuffer_head += length + return self._readbuffer[self._readbuffer_head - length:self._readbuffer_head] + + def _recv_impl(self, length): + + # try to decode websocket payload part from data + try: + + self._readbuffer_head = 0 + + result = None + + chunk_startindex = self._payload_head + chunk_endindex = self._payload_head + length + + header1 = self._buffered_read(1) + header2 = self._buffered_read(1) + + opcode = (header1[0] & 0x0f) + maskbit = (header2[0] & 0x80) == 0x80 + lengthbits = (header2[0] & 0x7f) + payload_length = lengthbits + mask_key = None + + # read length + if lengthbits == 0x7e: + + value = self._buffered_read(2) + payload_length, = struct.unpack("!H", value) + + elif lengthbits == 0x7f: + + value = self._buffered_read(8) + payload_length, = struct.unpack("!Q", value) + + # read mask + if maskbit: + mask_key = self._buffered_read(4) + + # if frame payload is shorter than the requested data, read only the possible part + readindex = chunk_endindex + if payload_length < readindex: + readindex = payload_length + + if readindex > 0: + # get payload chunk + payload = self._buffered_read(readindex) + + # unmask only the needed part + if maskbit: + for index in range(chunk_startindex, readindex): + payload[index] ^= mask_key[index % 4] + + result = payload[chunk_startindex:readindex] + self._payload_head = readindex + else: + payload = bytearray() + + # check if full frame arrived and reset readbuffer and payloadhead if needed + if readindex == payload_length: + self._readbuffer = bytearray() + self._payload_head = 0 + + # respond to non-binary opcodes, their arrival is not guaranteed beacause of non-blocking sockets + if opcode == WebsocketWrapper.OPCODE_CONNCLOSE: + frame = self._create_frame( + WebsocketWrapper.OPCODE_CONNCLOSE, payload, 0) + self._socket.send(frame) + + if opcode == WebsocketWrapper.OPCODE_PING: + frame = self._create_frame( + WebsocketWrapper.OPCODE_PONG, payload, 0) + self._socket.send(frame) + + if opcode == WebsocketWrapper.OPCODE_BINARY and payload_length > 0: + return result + else: + raise socket.error(EAGAIN, 0) + + except socket.error as err: + + if err.errno == errno.ECONNABORTED: + self.connected = False + return b'' + else: + # no more data + raise + + def _send_impl(self, data): + + # if previous frame was sent successfully + if len(self._sendbuffer) == 0: + # create websocket frame + frame = self._create_frame( + WebsocketWrapper.OPCODE_BINARY, bytearray(data)) + self._sendbuffer.extend(frame) + self._requested_size = len(data) + + # try to write out as much as possible + length = self._socket.send(self._sendbuffer) + + self._sendbuffer = self._sendbuffer[length:] + + if len(self._sendbuffer) == 0: + # buffer sent out completely, return with payload's size + return self._requested_size + else: + # couldn't send whole data, request the same data again with 0 as sent length + return 0 + + def recv(self, length): + return self._recv_impl(length) + + def read(self, length): + return self._recv_impl(length) + + def send(self, data): + return self._send_impl(data) + + def write(self, data): + return self._send_impl(data) + + def close(self): + self._socket.close() + + def fileno(self): + return self._socket.fileno() + + def pending(self): + # Fix for bug #131: a SSL socket may still have data available + # for reading without select() being aware of it. + if self._ssl: + return self._socket.pending() + else: + # normal socket rely only on select() + return 0 + + def setblocking(self, flag): + self._socket.setblocking(flag) diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/matcher.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/matcher.py new file mode 100644 index 0000000..7fc966a --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/matcher.py @@ -0,0 +1,78 @@ +class MQTTMatcher(object): + """Intended to manage topic filters including wildcards. + + Internally, MQTTMatcher use a prefix tree (trie) to store + values associated with filters, and has an iter_match() + method to iterate efficiently over all filters that match + some topic name.""" + + class Node(object): + __slots__ = '_children', '_content' + + def __init__(self): + self._children = {} + self._content = None + + def __init__(self): + self._root = self.Node() + + def __setitem__(self, key, value): + """Add a topic filter :key to the prefix tree + and associate it to :value""" + node = self._root + for sym in key.split('/'): + node = node._children.setdefault(sym, self.Node()) + node._content = value + + def __getitem__(self, key): + """Retrieve the value associated with some topic filter :key""" + try: + node = self._root + for sym in key.split('/'): + node = node._children[sym] + if node._content is None: + raise KeyError(key) + return node._content + except KeyError: + raise KeyError(key) + + def __delitem__(self, key): + """Delete the value associated with some topic filter :key""" + lst = [] + try: + parent, node = None, self._root + for k in key.split('/'): + parent, node = node, node._children[k] + lst.append((parent, k, node)) + # TODO + node._content = None + except KeyError: + raise KeyError(key) + else: # cleanup + for parent, k, node in reversed(lst): + if node._children or node._content is not None: + break + del parent._children[k] + + def iter_match(self, topic): + """Return an iterator on all values associated with filters + that match the :topic""" + lst = topic.split('/') + normal = not topic.startswith('$') + def rec(node, i=0): + if i == len(lst): + if node._content is not None: + yield node._content + else: + part = lst[i] + if part in node._children: + for content in rec(node._children[part], i + 1): + yield content + if '+' in node._children and (normal or i > 0): + for content in rec(node._children['+'], i + 1): + yield content + if '#' in node._children and (normal or i > 0): + content = node._children['#']._content + if content is not None: + yield content + return rec(self._root) diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/packettypes.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/packettypes.py new file mode 100644 index 0000000..7eb4069 --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/packettypes.py @@ -0,0 +1,43 @@ +""" +******************************************************************* + Copyright (c) 2017, 2019 IBM Corp. + + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + and Eclipse Distribution License v1.0 which accompany this distribution. + + The Eclipse Public License is available at + http://www.eclipse.org/legal/epl-v10.html + and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + + Contributors: + Ian Craggs - initial implementation and/or documentation +******************************************************************* +""" + + +class PacketTypes: + + """ + Packet types class. Includes the AUTH packet for MQTT v5.0. + + Holds constants for each packet type such as PacketTypes.PUBLISH + and packet name strings: PacketTypes.Names[PacketTypes.PUBLISH]. + + """ + + indexes = range(1, 16) + + # Packet types + CONNECT, CONNACK, PUBLISH, PUBACK, PUBREC, PUBREL, \ + PUBCOMP, SUBSCRIBE, SUBACK, UNSUBSCRIBE, UNSUBACK, \ + PINGREQ, PINGRESP, DISCONNECT, AUTH = indexes + + # Dummy packet type for properties use - will delay only applies to will + WILLMESSAGE = 99 + + Names = [ "reserved", \ + "Connect", "Connack", "Publish", "Puback", "Pubrec", "Pubrel", \ + "Pubcomp", "Subscribe", "Suback", "Unsubscribe", "Unsuback", \ + "Pingreq", "Pingresp", "Disconnect", "Auth"] diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/properties.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/properties.py new file mode 100644 index 0000000..99f654a --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/properties.py @@ -0,0 +1,409 @@ +""" +******************************************************************* + Copyright (c) 2017, 2019 IBM Corp. + + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + and Eclipse Distribution License v1.0 which accompany this distribution. + + The Eclipse Public License is available at + http://www.eclipse.org/legal/epl-v10.html + and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + + Contributors: + Ian Craggs - initial implementation and/or documentation +******************************************************************* +""" + +import sys, struct + +from .packettypes import PacketTypes + + +class MQTTException(Exception): + pass + + +class MalformedPacket(MQTTException): + pass + + +def writeInt16(length): + # serialize a 16 bit integer to network format + return bytearray(struct.pack("!H", length)) + + +def readInt16(buf): + # deserialize a 16 bit integer from network format + return struct.unpack("!H", buf[:2])[0] + + +def writeInt32(length): + # serialize a 32 bit integer to network format + return bytearray(struct.pack("!L", length)) + + +def readInt32(buf): + # deserialize a 32 bit integer from network format + return struct.unpack("!L", buf[:4])[0] + + +def writeUTF(data): + # data could be a string, or bytes. If string, encode into bytes with utf-8 + if sys.version_info[0] < 3: + data = bytearray(data, 'utf-8') + else: + data = data if type(data) == type(b"") else bytes(data, "utf-8") + return writeInt16(len(data)) + data + + +def readUTF(buffer, maxlen): + if maxlen >= 2: + length = readInt16(buffer) + else: + raise MalformedPacket("Not enough data to read string length") + maxlen -= 2 + if length > maxlen: + raise MalformedPacket("Length delimited string too long") + buf = buffer[2:2+length].decode("utf-8") + # look for chars which are invalid for MQTT + for c in buf: # look for D800-DFFF in the UTF string + ord_c = ord(c) + if ord_c >= 0xD800 and ord_c <= 0xDFFF: + raise MalformedPacket("[MQTT-1.5.4-1] D800-DFFF found in UTF-8 data") + if ord_c == 0x00: # look for null in the UTF string + raise MalformedPacket("[MQTT-1.5.4-2] Null found in UTF-8 data") + if ord_c == 0xFEFF: + raise MalformedPacket("[MQTT-1.5.4-3] U+FEFF in UTF-8 data") + return buf, length+2 + + +def writeBytes(buffer): + return writeInt16(len(buffer)) + buffer + + +def readBytes(buffer): + length = readInt16(buffer) + return buffer[2:2+length], length+2 + + +class VariableByteIntegers: # Variable Byte Integer + """ + MQTT variable byte integer helper class. Used + in several places in MQTT v5.0 properties. + + """ + + @staticmethod + def encode(x): + """ + Convert an integer 0 <= x <= 268435455 into multi-byte format. + Returns the buffer convered from the integer. + """ + assert 0 <= x <= 268435455 + buffer = b'' + while 1: + digit = x % 128 + x //= 128 + if x > 0: + digit |= 0x80 + if sys.version_info[0] >= 3: + buffer += bytes([digit]) + else: + buffer += bytes(chr(digit)) + if x == 0: + break + return buffer + + @staticmethod + def decode(buffer): + """ + Get the value of a multi-byte integer from a buffer + Return the value, and the number of bytes used. + + [MQTT-1.5.5-1] the encoded value MUST use the minimum number of bytes necessary to represent the value + """ + multiplier = 1 + value = 0 + bytes = 0 + while 1: + bytes += 1 + digit = buffer[0] + buffer = buffer[1:] + value += (digit & 127) * multiplier + if digit & 128 == 0: + break + multiplier *= 128 + return (value, bytes) + + +class Properties(object): + """MQTT v5.0 properties class. + + See Properties.names for a list of accepted property names along with their numeric values. + + See Properties.properties for the data type of each property. + + Example of use: + + publish_properties = Properties(PacketTypes.PUBLISH) + publish_properties.UserProperty = ("a", "2") + publish_properties.UserProperty = ("c", "3") + + First the object is created with packet type as argument, no properties will be present at + this point. Then properties are added as attributes, the name of which is the string property + name without the spaces. + + """ + + def __init__(self, packetType): + self.packetType = packetType + self.types = ["Byte", "Two Byte Integer", "Four Byte Integer", "Variable Byte Integer", + "Binary Data", "UTF-8 Encoded String", "UTF-8 String Pair"] + + self.names = { + "Payload Format Indicator": 1, + "Message Expiry Interval": 2, + "Content Type": 3, + "Response Topic": 8, + "Correlation Data": 9, + "Subscription Identifier": 11, + "Session Expiry Interval": 17, + "Assigned Client Identifier": 18, + "Server Keep Alive": 19, + "Authentication Method": 21, + "Authentication Data": 22, + "Request Problem Information": 23, + "Will Delay Interval": 24, + "Request Response Information": 25, + "Response Information": 26, + "Server Reference": 28, + "Reason String": 31, + "Receive Maximum": 33, + "Topic Alias Maximum": 34, + "Topic Alias": 35, + "Maximum QoS": 36, + "Retain Available": 37, + "User Property": 38, + "Maximum Packet Size": 39, + "Wildcard Subscription Available": 40, + "Subscription Identifier Available": 41, + "Shared Subscription Available": 42 + } + + self.properties = { + # id: type, packets + # payload format indicator + 1: (self.types.index("Byte"), [PacketTypes.PUBLISH, PacketTypes.WILLMESSAGE]), + 2: (self.types.index("Four Byte Integer"), [PacketTypes.PUBLISH, PacketTypes.WILLMESSAGE]), + 3: (self.types.index("UTF-8 Encoded String"), [PacketTypes.PUBLISH, PacketTypes.WILLMESSAGE]), + 8: (self.types.index("UTF-8 Encoded String"), [PacketTypes.PUBLISH, PacketTypes.WILLMESSAGE]), + 9: (self.types.index("Binary Data"), [PacketTypes.PUBLISH, PacketTypes.WILLMESSAGE]), + 11: (self.types.index("Variable Byte Integer"), + [PacketTypes.PUBLISH, PacketTypes.SUBSCRIBE]), + 17: (self.types.index("Four Byte Integer"), + [PacketTypes.CONNECT, PacketTypes.CONNACK, PacketTypes.DISCONNECT]), + 18: (self.types.index("UTF-8 Encoded String"), [PacketTypes.CONNACK]), + 19: (self.types.index("Two Byte Integer"), [PacketTypes.CONNACK]), + 21: (self.types.index("UTF-8 Encoded String"), + [PacketTypes.CONNECT, PacketTypes.CONNACK, PacketTypes.AUTH]), + 22: (self.types.index("Binary Data"), + [PacketTypes.CONNECT, PacketTypes.CONNACK, PacketTypes.AUTH]), + 23: (self.types.index("Byte"), + [PacketTypes.CONNECT]), + 24: (self.types.index("Four Byte Integer"), [PacketTypes.WILLMESSAGE]), + 25: (self.types.index("Byte"), [PacketTypes.CONNECT]), + 26: (self.types.index("UTF-8 Encoded String"), [PacketTypes.CONNACK]), + 28: (self.types.index("UTF-8 Encoded String"), + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]), + 31: (self.types.index("UTF-8 Encoded String"), + [PacketTypes.CONNACK, PacketTypes.PUBACK, PacketTypes.PUBREC, + PacketTypes.PUBREL, PacketTypes.PUBCOMP, PacketTypes.SUBACK, + PacketTypes.UNSUBACK, PacketTypes.DISCONNECT, PacketTypes.AUTH]), + 33: (self.types.index("Two Byte Integer"), + [PacketTypes.CONNECT, PacketTypes.CONNACK]), + 34: (self.types.index("Two Byte Integer"), + [PacketTypes.CONNECT, PacketTypes.CONNACK]), + 35: (self.types.index("Two Byte Integer"), [PacketTypes.PUBLISH]), + 36: (self.types.index("Byte"), [PacketTypes.CONNACK]), + 37: (self.types.index("Byte"), [PacketTypes.CONNACK]), + 38: (self.types.index("UTF-8 String Pair"), + [PacketTypes.CONNECT, PacketTypes.CONNACK, + PacketTypes.PUBLISH, PacketTypes.PUBACK, + PacketTypes.PUBREC, PacketTypes.PUBREL, PacketTypes.PUBCOMP, + PacketTypes.SUBSCRIBE, PacketTypes.SUBACK, + PacketTypes.UNSUBSCRIBE, PacketTypes.UNSUBACK, + PacketTypes.DISCONNECT, PacketTypes.AUTH, PacketTypes.WILLMESSAGE]), + 39: (self.types.index("Four Byte Integer"), + [PacketTypes.CONNECT, PacketTypes.CONNACK]), + 40: (self.types.index("Byte"), [PacketTypes.CONNACK]), + 41: (self.types.index("Byte"), [PacketTypes.CONNACK]), + 42: (self.types.index("Byte"), [PacketTypes.CONNACK]), + } + + def allowsMultiple(self, compressedName): + return self.getIdentFromName(compressedName) in [11, 38] + + def getIdentFromName(self, compressedName): + # return the identifier corresponding to the property name + result = -1 + for name in self.names.keys(): + if compressedName == name.replace(' ', ''): + result = self.names[name] + break + return result + + def __setattr__(self, name, value): + name = name.replace(' ', '') + privateVars = ["packetType", "types", "names", "properties"] + if name in privateVars: + object.__setattr__(self, name, value) + else: + # the name could have spaces in, or not. Remove spaces before assignment + if name not in [aname.replace(' ', '') for aname in self.names.keys()]: + raise MQTTException( + "Property name must be one of "+str(self.names.keys())) + # check that this attribute applies to the packet type + if self.packetType not in self.properties[self.getIdentFromName(name)][1]: + raise MQTTException("Property %s does not apply to packet type %s" + % (name, PacketTypes.Names[self.packetType])) + if self.allowsMultiple(name): + if type(value) != type([]): + value = [value] + if hasattr(self, name): + value = object.__getattribute__(self, name) + value + object.__setattr__(self, name, value) + + def __str__(self): + buffer = "[" + first = True + for name in self.names.keys(): + compressedName = name.replace(' ', '') + if hasattr(self, compressedName): + if not first: + buffer += ", " + buffer += compressedName + " : " + \ + str(getattr(self, compressedName)) + first = False + buffer += "]" + return buffer + + def json(self): + data = {} + for name in self.names.keys(): + compressedName = name.replace(' ', '') + if hasattr(self, compressedName): + data[compressedName] = getattr(self, compressedName) + return data + + def isEmpty(self): + rc = True + for name in self.names.keys(): + compressedName = name.replace(' ', '') + if hasattr(self, compressedName): + rc = False + break + return rc + + def clear(self): + for name in self.names.keys(): + compressedName = name.replace(' ', '') + if hasattr(self, compressedName): + delattr(self, compressedName) + + def writeProperty(self, identifier, type, value): + buffer = b"" + buffer += VariableByteIntegers.encode(identifier) # identifier + if type == self.types.index("Byte"): # value + if sys.version_info[0] < 3: + buffer += chr(value) + else: + buffer += bytes([value]) + elif type == self.types.index("Two Byte Integer"): + buffer += writeInt16(value) + elif type == self.types.index("Four Byte Integer"): + buffer += writeInt32(value) + elif type == self.types.index("Variable Byte Integer"): + buffer += VariableByteIntegers.encode(value) + elif type == self.types.index("Binary Data"): + buffer += writeBytes(value) + elif type == self.types.index("UTF-8 Encoded String"): + buffer += writeUTF(value) + elif type == self.types.index("UTF-8 String Pair"): + buffer += writeUTF(value[0]) + writeUTF(value[1]) + return buffer + + def pack(self): + # serialize properties into buffer for sending over network + buffer = b"" + for name in self.names.keys(): + compressedName = name.replace(' ', '') + if hasattr(self, compressedName): + identifier = self.getIdentFromName(compressedName) + attr_type = self.properties[identifier][0] + if self.allowsMultiple(compressedName): + for prop in getattr(self, compressedName): + buffer += self.writeProperty(identifier, + attr_type, prop) + else: + buffer += self.writeProperty(identifier, attr_type, + getattr(self, compressedName)) + return VariableByteIntegers.encode(len(buffer)) + buffer + + def readProperty(self, buffer, type, propslen): + if type == self.types.index("Byte"): + value = buffer[0] + valuelen = 1 + elif type == self.types.index("Two Byte Integer"): + value = readInt16(buffer) + valuelen = 2 + elif type == self.types.index("Four Byte Integer"): + value = readInt32(buffer) + valuelen = 4 + elif type == self.types.index("Variable Byte Integer"): + value, valuelen = VariableByteIntegers.decode(buffer) + elif type == self.types.index("Binary Data"): + value, valuelen = readBytes(buffer) + elif type == self.types.index("UTF-8 Encoded String"): + value, valuelen = readUTF(buffer, propslen) + elif type == self.types.index("UTF-8 String Pair"): + value, valuelen = readUTF(buffer, propslen) + buffer = buffer[valuelen:] # strip the bytes used by the value + value1, valuelen1 = readUTF(buffer, propslen - valuelen) + value = (value, value1) + valuelen += valuelen1 + return value, valuelen + + def getNameFromIdent(self, identifier): + rc = None + for name in self.names: + if self.names[name] == identifier: + rc = name + return rc + + def unpack(self, buffer): + if sys.version_info[0] < 3: + buffer = bytearray(buffer) + self.clear() + # deserialize properties into attributes from buffer received from network + propslen, VBIlen = VariableByteIntegers.decode(buffer) + buffer = buffer[VBIlen:] # strip the bytes used by the VBI + propslenleft = propslen + while propslenleft > 0: # properties length is 0 if there are none + identifier, VBIlen = VariableByteIntegers.decode( + buffer) # property identifier + buffer = buffer[VBIlen:] # strip the bytes used by the VBI + propslenleft -= VBIlen + attr_type = self.properties[identifier][0] + value, valuelen = self.readProperty( + buffer, attr_type, propslenleft) + buffer = buffer[valuelen:] # strip the bytes used by the value + propslenleft -= valuelen + propname = self.getNameFromIdent(identifier) + compressedName = propname.replace(' ', '') + if not self.allowsMultiple(compressedName) and hasattr(self, compressedName): + raise MQTTException( + "Property '%s' must not exist more than once" % property) + setattr(self, propname, value) + return self, propslen + VBIlen diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/publish.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/publish.py new file mode 100644 index 0000000..dcb34ff --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/publish.py @@ -0,0 +1,232 @@ +# Copyright (c) 2014 Roger Light +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# and Eclipse Distribution License v1.0 which accompany this distribution. +# +# The Eclipse Public License is available at +# http://www.eclipse.org/legal/epl-v10.html +# and the Eclipse Distribution License is available at +# http://www.eclipse.org/org/documents/edl-v10.php. +# +# Contributors: +# Roger Light - initial API and implementation + +""" +This module provides some helper functions to allow straightforward publishing +of messages in a one-shot manner. In other words, they are useful for the +situation where you have a single/multiple messages you want to publish to a +broker, then disconnect and nothing else is required. +""" +from __future__ import absolute_import + +import collections +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable + +from . import client as paho +from .. import mqtt + +def _do_publish(client): + """Internal function""" + + message = client._userdata.popleft() + + if isinstance(message, dict): + client.publish(**message) + elif isinstance(message, (tuple, list)): + client.publish(*message) + else: + raise TypeError('message must be a dict, tuple, or list') + + +def _on_connect(client, userdata, flags, rc): + """Internal callback""" + #pylint: disable=invalid-name, unused-argument + + if rc == 0: + if len(userdata) > 0: + _do_publish(client) + else: + raise mqtt.MQTTException(paho.connack_string(rc)) + + +def _on_publish(client, userdata, mid): + """Internal callback""" + #pylint: disable=unused-argument + + if len(userdata) == 0: + client.disconnect() + else: + _do_publish(client) + + +def multiple(msgs, hostname="localhost", port=1883, client_id="", keepalive=60, + will=None, auth=None, tls=None, protocol=paho.MQTTv311, + transport="tcp", proxy_args=None): + """Publish multiple messages to a broker, then disconnect cleanly. + + This function creates an MQTT client, connects to a broker and publishes a + list of messages. Once the messages have been delivered, it disconnects + cleanly from the broker. + + msgs : a list of messages to publish. Each message is either a dict or a + tuple. + + If a dict, only the topic must be present. Default values will be + used for any missing arguments. The dict must be of the form: + + msg = {'topic':"", 'payload':"", 'qos':, + 'retain':} + topic must be present and may not be empty. + If payload is "", None or not present then a zero length payload + will be published. + If qos is not present, the default of 0 is used. + If retain is not present, the default of False is used. + + If a tuple, then it must be of the form: + ("", "", qos, retain) + + hostname : a string containing the address of the broker to connect to. + Defaults to localhost. + + port : the port to connect to the broker on. Defaults to 1883. + + client_id : the MQTT client id to use. If "" or None, the Paho library will + generate a client id automatically. + + keepalive : the keepalive timeout value for the client. Defaults to 60 + seconds. + + will : a dict containing will parameters for the client: will = {'topic': + "", 'payload':", 'qos':, 'retain':}. + Topic is required, all other parameters are optional and will + default to None, 0 and False respectively. + Defaults to None, which indicates no will should be used. + + auth : a dict containing authentication parameters for the client: + auth = {'username':"", 'password':""} + Username is required, password is optional and will default to None + if not provided. + Defaults to None, which indicates no authentication is to be used. + + tls : a dict containing TLS configuration parameters for the client: + dict = {'ca_certs':"", 'certfile':"", + 'keyfile':"", 'tls_version':"", + 'ciphers':", 'insecure':""} + ca_certs is required, all other parameters are optional and will + default to None if not provided, which results in the client using + the default behaviour - see the paho.mqtt.client documentation. + Alternatively, tls input can be an SSLContext object, which will be + processed using the tls_set_context method. + Defaults to None, which indicates that TLS should not be used. + + transport : set to "tcp" to use the default setting of transport which is + raw TCP. Set to "websockets" to use WebSockets as the transport. + proxy_args: a dictionary that will be given to the client. + """ + + if not isinstance(msgs, Iterable): + raise TypeError('msgs must be an iterable') + + client = paho.Client(client_id=client_id, userdata=collections.deque(msgs), + protocol=protocol, transport=transport) + + client.on_publish = _on_publish + client.on_connect = _on_connect + + if proxy_args is not None: + client.proxy_set(**proxy_args) + + if auth: + username = auth.get('username') + if username: + password = auth.get('password') + client.username_pw_set(username, password) + else: + raise KeyError("The 'username' key was not found, this is " + "required for auth") + + if will is not None: + client.will_set(**will) + + if tls is not None: + if isinstance(tls, dict): + insecure = tls.pop('insecure', False) + client.tls_set(**tls) + if insecure: + # Must be set *after* the `client.tls_set()` call since it sets + # up the SSL context that `client.tls_insecure_set` alters. + client.tls_insecure_set(insecure) + else: + # Assume input is SSLContext object + client.tls_set_context(tls) + + client.connect(hostname, port, keepalive) + client.loop_forever() + + +def single(topic, payload=None, qos=0, retain=False, hostname="localhost", + port=1883, client_id="", keepalive=60, will=None, auth=None, + tls=None, protocol=paho.MQTTv311, transport="tcp", proxy_args=None): + """Publish a single message to a broker, then disconnect cleanly. + + This function creates an MQTT client, connects to a broker and publishes a + single message. Once the message has been delivered, it disconnects cleanly + from the broker. + + topic : the only required argument must be the topic string to which the + payload will be published. + + payload : the payload to be published. If "" or None, a zero length payload + will be published. + + qos : the qos to use when publishing, default to 0. + + retain : set the message to be retained (True) or not (False). + + hostname : a string containing the address of the broker to connect to. + Defaults to localhost. + + port : the port to connect to the broker on. Defaults to 1883. + + client_id : the MQTT client id to use. If "" or None, the Paho library will + generate a client id automatically. + + keepalive : the keepalive timeout value for the client. Defaults to 60 + seconds. + + will : a dict containing will parameters for the client: will = {'topic': + "", 'payload':", 'qos':, 'retain':}. + Topic is required, all other parameters are optional and will + default to None, 0 and False respectively. + Defaults to None, which indicates no will should be used. + + auth : a dict containing authentication parameters for the client: + auth = {'username':"", 'password':""} + Username is required, password is optional and will default to None + if not provided. + Defaults to None, which indicates no authentication is to be used. + + tls : a dict containing TLS configuration parameters for the client: + dict = {'ca_certs':"", 'certfile':"", + 'keyfile':"", 'tls_version':"", + 'ciphers':", 'insecure':""} + ca_certs is required, all other parameters are optional and will + default to None if not provided, which results in the client using + the default behaviour - see the paho.mqtt.client documentation. + Defaults to None, which indicates that TLS should not be used. + Alternatively, tls input can be an SSLContext object, which will be + processed using the tls_set_context method. + + transport : set to "tcp" to use the default setting of transport which is + raw TCP. Set to "websockets" to use WebSockets as the transport. + proxy_args: a dictionary that will be given to the client. + """ + + msg = {'topic':topic, 'payload':payload, 'qos':qos, 'retain':retain} + + multiple([msg], hostname, port, client_id, keepalive, will, auth, tls, + protocol, transport, proxy_args) diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/reasoncodes.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/reasoncodes.py new file mode 100644 index 0000000..12325bc --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/reasoncodes.py @@ -0,0 +1,191 @@ +""" +******************************************************************* + Copyright (c) 2017, 2019 IBM Corp. + + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + and Eclipse Distribution License v1.0 which accompany this distribution. + + The Eclipse Public License is available at + http://www.eclipse.org/legal/epl-v10.html + and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + + Contributors: + Ian Craggs - initial implementation and/or documentation +******************************************************************* +""" + +import sys +from .packettypes import PacketTypes + + +class ReasonCodes: + """MQTT version 5.0 reason codes class. + + See ReasonCodes.names for a list of possible numeric values along with their + names and the packets to which they apply. + + """ + + def __init__(self, packetType, aName="Success", identifier=-1): + """ + packetType: the type of the packet, such as PacketTypes.CONNECT that + this reason code will be used with. Some reason codes have different + names for the same identifier when used a different packet type. + + aName: the String name of the reason code to be created. Ignored + if the identifier is set. + + identifier: an integer value of the reason code to be created. + + """ + + self.packetType = packetType + self.names = { + 0: {"Success": [PacketTypes.CONNACK, PacketTypes.PUBACK, + PacketTypes.PUBREC, PacketTypes.PUBREL, PacketTypes.PUBCOMP, + PacketTypes.UNSUBACK, PacketTypes.AUTH], + "Normal disconnection": [PacketTypes.DISCONNECT], + "Granted QoS 0": [PacketTypes.SUBACK]}, + 1: {"Granted QoS 1": [PacketTypes.SUBACK]}, + 2: {"Granted QoS 2": [PacketTypes.SUBACK]}, + 4: {"Disconnect with will message": [PacketTypes.DISCONNECT]}, + 16: {"No matching subscribers": + [PacketTypes.PUBACK, PacketTypes.PUBREC]}, + 17: {"No subscription found": [PacketTypes.UNSUBACK]}, + 24: {"Continue authentication": [PacketTypes.AUTH]}, + 25: {"Re-authenticate": [PacketTypes.AUTH]}, + 128: {"Unspecified error": [PacketTypes.CONNACK, PacketTypes.PUBACK, + PacketTypes.PUBREC, PacketTypes.SUBACK, PacketTypes.UNSUBACK, + PacketTypes.DISCONNECT], }, + 129: {"Malformed packet": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 130: {"Protocol error": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 131: {"Implementation specific error": [PacketTypes.CONNACK, + PacketTypes.PUBACK, PacketTypes.PUBREC, PacketTypes.SUBACK, + PacketTypes.UNSUBACK, PacketTypes.DISCONNECT], }, + 132: {"Unsupported protocol version": [PacketTypes.CONNACK]}, + 133: {"Client identifier not valid": [PacketTypes.CONNACK]}, + 134: {"Bad user name or password": [PacketTypes.CONNACK]}, + 135: {"Not authorized": [PacketTypes.CONNACK, PacketTypes.PUBACK, + PacketTypes.PUBREC, PacketTypes.SUBACK, PacketTypes.UNSUBACK, + PacketTypes.DISCONNECT], }, + 136: {"Server unavailable": [PacketTypes.CONNACK]}, + 137: {"Server busy": [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 138: {"Banned": [PacketTypes.CONNACK]}, + 139: {"Server shutting down": [PacketTypes.DISCONNECT]}, + 140: {"Bad authentication method": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 141: {"Keep alive timeout": [PacketTypes.DISCONNECT]}, + 142: {"Session taken over": [PacketTypes.DISCONNECT]}, + 143: {"Topic filter invalid": + [PacketTypes.SUBACK, PacketTypes.UNSUBACK, PacketTypes.DISCONNECT]}, + 144: {"Topic name invalid": + [PacketTypes.CONNACK, PacketTypes.PUBACK, + PacketTypes.PUBREC, PacketTypes.DISCONNECT]}, + 145: {"Packet identifier in use": + [PacketTypes.PUBACK, PacketTypes.PUBREC, + PacketTypes.SUBACK, PacketTypes.UNSUBACK]}, + 146: {"Packet identifier not found": + [PacketTypes.PUBREL, PacketTypes.PUBCOMP]}, + 147: {"Receive maximum exceeded": [PacketTypes.DISCONNECT]}, + 148: {"Topic alias invalid": [PacketTypes.DISCONNECT]}, + 149: {"Packet too large": [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 150: {"Message rate too high": [PacketTypes.DISCONNECT]}, + 151: {"Quota exceeded": [PacketTypes.CONNACK, PacketTypes.PUBACK, + PacketTypes.PUBREC, PacketTypes.SUBACK, PacketTypes.DISCONNECT], }, + 152: {"Administrative action": [PacketTypes.DISCONNECT]}, + 153: {"Payload format invalid": + [PacketTypes.PUBACK, PacketTypes.PUBREC, PacketTypes.DISCONNECT]}, + 154: {"Retain not supported": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 155: {"QoS not supported": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 156: {"Use another server": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 157: {"Server moved": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 158: {"Shared subscription not supported": + [PacketTypes.SUBACK, PacketTypes.DISCONNECT]}, + 159: {"Connection rate exceeded": + [PacketTypes.CONNACK, PacketTypes.DISCONNECT]}, + 160: {"Maximum connect time": + [PacketTypes.DISCONNECT]}, + 161: {"Subscription identifiers not supported": + [PacketTypes.SUBACK, PacketTypes.DISCONNECT]}, + 162: {"Wildcard subscription not supported": + [PacketTypes.SUBACK, PacketTypes.DISCONNECT]}, + } + if identifier == -1: + if packetType == PacketTypes.DISCONNECT and aName == "Success": + aName = "Normal disconnection" + self.set(aName) + else: + self.value = identifier + self.getName() # check it's good + + def __getName__(self, packetType, identifier): + """ + Get the reason code string name for a specific identifier. + The name can vary by packet type for the same identifier, which + is why the packet type is also required. + + Used when displaying the reason code. + """ + assert identifier in self.names.keys(), identifier + names = self.names[identifier] + namelist = [name for name in names.keys() if packetType in names[name]] + assert len(namelist) == 1 + return namelist[0] + + def getId(self, name): + """ + Get the numeric id corresponding to a reason code name. + + Used when setting the reason code for a packetType + check that only valid codes for the packet are set. + """ + identifier = None + for code in self.names.keys(): + if name in self.names[code].keys(): + if self.packetType in self.names[code][name]: + identifier = code + break + assert identifier != None, name + return identifier + + def set(self, name): + self.value = self.getId(name) + + def unpack(self, buffer): + c = buffer[0] + if sys.version_info[0] < 3: + c = ord(c) + name = self.__getName__(self.packetType, c) + self.value = self.getId(name) + return 1 + + def getName(self): + """Returns the reason code name corresponding to the numeric value which is set. + """ + return self.__getName__(self.packetType, self.value) + + def __eq__(self, other): + if isinstance(other, int): + return self.value == other + if isinstance(other, str): + return self.value == str(self) + if isinstance(other, ReasonCodes): + return self.value == other.value + return False + + def __str__(self): + return self.getName() + + def json(self): + return self.getName() + + def pack(self): + return bytearray([self.value]) \ No newline at end of file diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/subscribe.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/subscribe.py new file mode 100644 index 0000000..8900a6b --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/subscribe.py @@ -0,0 +1,266 @@ +# Copyright (c) 2016 Roger Light +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# and Eclipse Distribution License v1.0 which accompany this distribution. +# +# The Eclipse Public License is available at +# http://www.eclipse.org/legal/epl-v10.html +# and the Eclipse Distribution License is available at +# http://www.eclipse.org/org/documents/edl-v10.php. +# +# Contributors: +# Roger Light - initial API and implementation + +""" +This module provides some helper functions to allow straightforward subscribing +to topics and retrieving messages. The two functions are simple(), which +returns one or messages matching a set of topics, and callback() which allows +you to pass a callback for processing of messages. +""" +from __future__ import absolute_import + +from . import client as paho +from .. import mqtt + +def _on_connect(client, userdata, flags, rc): + """Internal callback""" + if rc != 0: + raise mqtt.MQTTException(paho.connack_string(rc)) + + if isinstance(userdata['topics'], list): + for topic in userdata['topics']: + client.subscribe(topic, userdata['qos']) + else: + client.subscribe(userdata['topics'], userdata['qos']) + + +def _on_message_callback(client, userdata, message): + """Internal callback""" + userdata['callback'](client, userdata['userdata'], message) + + +def _on_message_simple(client, userdata, message): + """Internal callback""" + + if userdata['msg_count'] == 0: + return + + # Don't process stale retained messages if 'retained' was false + if message.retain and not userdata['retained']: + return + + userdata['msg_count'] = userdata['msg_count'] - 1 + + if userdata['messages'] is None and userdata['msg_count'] == 0: + userdata['messages'] = message + client.disconnect() + return + + userdata['messages'].append(message) + if userdata['msg_count'] == 0: + client.disconnect() + + +def callback(callback, topics, qos=0, userdata=None, hostname="localhost", + port=1883, client_id="", keepalive=60, will=None, auth=None, + tls=None, protocol=paho.MQTTv311, transport="tcp", + clean_session=True, proxy_args=None): + """Subscribe to a list of topics and process them in a callback function. + + This function creates an MQTT client, connects to a broker and subscribes + to a list of topics. Incoming messages are processed by the user provided + callback. This is a blocking function and will never return. + + callback : function of the form "on_message(client, userdata, message)" for + processing the messages received. + + topics : either a string containing a single topic to subscribe to, or a + list of topics to subscribe to. + + qos : the qos to use when subscribing. This is applied to all topics. + + userdata : passed to the callback + + hostname : a string containing the address of the broker to connect to. + Defaults to localhost. + + port : the port to connect to the broker on. Defaults to 1883. + + client_id : the MQTT client id to use. If "" or None, the Paho library will + generate a client id automatically. + + keepalive : the keepalive timeout value for the client. Defaults to 60 + seconds. + + will : a dict containing will parameters for the client: will = {'topic': + "", 'payload':", 'qos':, 'retain':}. + Topic is required, all other parameters are optional and will + default to None, 0 and False respectively. + Defaults to None, which indicates no will should be used. + + auth : a dict containing authentication parameters for the client: + auth = {'username':"", 'password':""} + Username is required, password is optional and will default to None + if not provided. + Defaults to None, which indicates no authentication is to be used. + + tls : a dict containing TLS configuration parameters for the client: + dict = {'ca_certs':"", 'certfile':"", + 'keyfile':"", 'tls_version':"", + 'ciphers':", 'insecure':""} + ca_certs is required, all other parameters are optional and will + default to None if not provided, which results in the client using + the default behaviour - see the paho.mqtt.client documentation. + Alternatively, tls input can be an SSLContext object, which will be + processed using the tls_set_context method. + Defaults to None, which indicates that TLS should not be used. + + transport : set to "tcp" to use the default setting of transport which is + raw TCP. Set to "websockets" to use WebSockets as the transport. + + clean_session : a boolean that determines the client type. If True, + the broker will remove all information about this client + when it disconnects. If False, the client is a persistent + client and subscription information and queued messages + will be retained when the client disconnects. + Defaults to True. + + proxy_args: a dictionary that will be given to the client. + """ + + if qos < 0 or qos > 2: + raise ValueError('qos must be in the range 0-2') + + callback_userdata = { + 'callback':callback, + 'topics':topics, + 'qos':qos, + 'userdata':userdata} + + client = paho.Client(client_id=client_id, userdata=callback_userdata, + protocol=protocol, transport=transport, + clean_session=clean_session) + client.on_message = _on_message_callback + client.on_connect = _on_connect + + if proxy_args is not None: + client.proxy_set(**proxy_args) + + if auth: + username = auth.get('username') + if username: + password = auth.get('password') + client.username_pw_set(username, password) + else: + raise KeyError("The 'username' key was not found, this is " + "required for auth") + + if will is not None: + client.will_set(**will) + + if tls is not None: + if isinstance(tls, dict): + insecure = tls.pop('insecure', False) + client.tls_set(**tls) + if insecure: + # Must be set *after* the `client.tls_set()` call since it sets + # up the SSL context that `client.tls_insecure_set` alters. + client.tls_insecure_set(insecure) + else: + # Assume input is SSLContext object + client.tls_set_context(tls) + + client.connect(hostname, port, keepalive) + client.loop_forever() + + +def simple(topics, qos=0, msg_count=1, retained=True, hostname="localhost", + port=1883, client_id="", keepalive=60, will=None, auth=None, + tls=None, protocol=paho.MQTTv311, transport="tcp", + clean_session=True, proxy_args=None): + """Subscribe to a list of topics and return msg_count messages. + + This function creates an MQTT client, connects to a broker and subscribes + to a list of topics. Once "msg_count" messages have been received, it + disconnects cleanly from the broker and returns the messages. + + topics : either a string containing a single topic to subscribe to, or a + list of topics to subscribe to. + + qos : the qos to use when subscribing. This is applied to all topics. + + msg_count : the number of messages to retrieve from the broker. + if msg_count == 1 then a single MQTTMessage will be returned. + if msg_count > 1 then a list of MQTTMessages will be returned. + + retained : If set to True, retained messages will be processed the same as + non-retained messages. If set to False, retained messages will + be ignored. This means that with retained=False and msg_count=1, + the function will return the first message received that does + not have the retained flag set. + + hostname : a string containing the address of the broker to connect to. + Defaults to localhost. + + port : the port to connect to the broker on. Defaults to 1883. + + client_id : the MQTT client id to use. If "" or None, the Paho library will + generate a client id automatically. + + keepalive : the keepalive timeout value for the client. Defaults to 60 + seconds. + + will : a dict containing will parameters for the client: will = {'topic': + "", 'payload':", 'qos':, 'retain':}. + Topic is required, all other parameters are optional and will + default to None, 0 and False respectively. + Defaults to None, which indicates no will should be used. + + auth : a dict containing authentication parameters for the client: + auth = {'username':"", 'password':""} + Username is required, password is optional and will default to None + if not provided. + Defaults to None, which indicates no authentication is to be used. + + tls : a dict containing TLS configuration parameters for the client: + dict = {'ca_certs':"", 'certfile':"", + 'keyfile':"", 'tls_version':"", + 'ciphers':", 'insecure':""} + ca_certs is required, all other parameters are optional and will + default to None if not provided, which results in the client using + the default behaviour - see the paho.mqtt.client documentation. + Alternatively, tls input can be an SSLContext object, which will be + processed using the tls_set_context method. + Defaults to None, which indicates that TLS should not be used. + + transport : set to "tcp" to use the default setting of transport which is + raw TCP. Set to "websockets" to use WebSockets as the transport. + + clean_session : a boolean that determines the client type. If True, + the broker will remove all information about this client + when it disconnects. If False, the client is a persistent + client and subscription information and queued messages + will be retained when the client disconnects. + Defaults to True. + + proxy_args: a dictionary that will be given to the client. + """ + + if msg_count < 1: + raise ValueError('msg_count must be > 0') + + # Set ourselves up to return a single message if msg_count == 1, or a list + # if > 1. + if msg_count == 1: + messages = None + else: + messages = [] + + userdata = {'retained':retained, 'msg_count':msg_count, 'messages':messages} + + callback(_on_message_simple, topics, qos, userdata, hostname, port, + client_id, keepalive, will, auth, tls, protocol, transport, + clean_session, proxy_args) + + return userdata['messages'] diff --git a/lambda/sns-to-mqtt/vendor/paho/mqtt/subscribeoptions.py b/lambda/sns-to-mqtt/vendor/paho/mqtt/subscribeoptions.py new file mode 100644 index 0000000..f55e90a --- /dev/null +++ b/lambda/sns-to-mqtt/vendor/paho/mqtt/subscribeoptions.py @@ -0,0 +1,110 @@ +""" +******************************************************************* + Copyright (c) 2017, 2019 IBM Corp. + + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + and Eclipse Distribution License v1.0 which accompany this distribution. + + The Eclipse Public License is available at + http://www.eclipse.org/legal/epl-v10.html + and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + + Contributors: + Ian Craggs - initial implementation and/or documentation +******************************************************************* +""" + +import sys + + +class MQTTException(Exception): + pass + + +class SubscribeOptions(object): + """The MQTT v5.0 subscribe options class. + + The options are: + qos: As in MQTT v3.1.1. + noLocal: True or False. If set to True, the subscriber will not receive its own publications. + retainAsPublished: True or False. If set to True, the retain flag on received publications will be as set + by the publisher. + retainHandling: RETAIN_SEND_ON_SUBSCRIBE, RETAIN_SEND_IF_NEW_SUB or RETAIN_DO_NOT_SEND + Controls when the broker should send retained messages: + - RETAIN_SEND_ON_SUBSCRIBE: on any successful subscribe request + - RETAIN_SEND_IF_NEW_SUB: only if the subscribe request is new + - RETAIN_DO_NOT_SEND: never send retained messages + """ + + # retain handling options + RETAIN_SEND_ON_SUBSCRIBE, RETAIN_SEND_IF_NEW_SUB, RETAIN_DO_NOT_SEND = range( + 0, 3) + + def __init__(self, qos=0, noLocal=False, retainAsPublished=False, retainHandling=RETAIN_SEND_ON_SUBSCRIBE): + """ + qos: 0, 1 or 2. 0 is the default. + noLocal: True or False. False is the default and corresponds to MQTT v3.1.1 behavior. + retainAsPublished: True or False. False is the default and corresponds to MQTT v3.1.1 behavior. + retainHandling: RETAIN_SEND_ON_SUBSCRIBE, RETAIN_SEND_IF_NEW_SUB or RETAIN_DO_NOT_SEND + RETAIN_SEND_ON_SUBSCRIBE is the default and corresponds to MQTT v3.1.1 behavior. + """ + object.__setattr__(self, "names", + ["QoS", "noLocal", "retainAsPublished", "retainHandling"]) + self.QoS = qos # bits 0,1 + self.noLocal = noLocal # bit 2 + self.retainAsPublished = retainAsPublished # bit 3 + self.retainHandling = retainHandling # bits 4 and 5: 0, 1 or 2 + assert self.QoS in [0, 1, 2] + assert self.retainHandling in [ + 0, 1, 2], "Retain handling should be 0, 1 or 2" + + def __setattr__(self, name, value): + if name not in self.names: + raise MQTTException( + name + " Attribute name must be one of "+str(self.names)) + object.__setattr__(self, name, value) + + def pack(self): + assert self.QoS in [0, 1, 2] + assert self.retainHandling in [ + 0, 1, 2], "Retain handling should be 0, 1 or 2" + noLocal = 1 if self.noLocal else 0 + retainAsPublished = 1 if self.retainAsPublished else 0 + data = [(self.retainHandling << 4) | (retainAsPublished << 3) | + (noLocal << 2) | self.QoS] + if sys.version_info[0] >= 3: + buffer = bytes(data) + else: + buffer = bytearray(data) + return buffer + + def unpack(self, buffer): + b0 = buffer[0] + self.retainHandling = ((b0 >> 4) & 0x03) + self.retainAsPublished = True if ((b0 >> 3) & 0x01) == 1 else False + self.noLocal = True if ((b0 >> 2) & 0x01) == 1 else False + self.QoS = (b0 & 0x03) + assert self.retainHandling in [ + 0, 1, 2], "Retain handling should be 0, 1 or 2, not %d" % self.retainHandling + assert self.QoS in [ + 0, 1, 2], "QoS should be 0, 1 or 2, not %d" % self.QoS + return 1 + + def __repr__(self): + return str(self) + + def __str__(self): + return "{QoS="+str(self.QoS)+", noLocal="+str(self.noLocal) +\ + ", retainAsPublished="+str(self.retainAsPublished) +\ + ", retainHandling="+str(self.retainHandling)+"}" + + def json(self): + data = { + "QoS": self.QoS, + "noLocal": self.noLocal, + "retainAsPublished": self.retainAsPublished, + "retainHandling": self.retainHandling, + } + return data diff --git a/sonde-api-to-iot-core/lambda_function.py b/lambda/sonde-api-to-iot-core/__init__.py similarity index 83% rename from sonde-api-to-iot-core/lambda_function.py rename to lambda/sonde-api-to-iot-core/__init__.py index da7bcc3..fd98fb3 100644 --- a/sonde-api-to-iot-core/lambda_function.py +++ b/lambda/sonde-api-to-iot-core/__init__.py @@ -1,10 +1,8 @@ -import sys import json import boto3 import zlib import base64 import datetime -import functools from email.utils import parsedate import os import re @@ -367,45 +365,3 @@ def lambda_handler(event, context): else: return {"statusCode": 200, "body": "^v^ telm logged"} -if __name__ == "__main__": - payload = { - "version": "2.0", - "routeKey": "PUT /sondes/telemetry", - "rawPath": "/sondes/telemetry", - "rawQueryString": "", - "headers": { - "accept": "*/*", - "accept-encoding": "gzip, deflate", - "content-encoding": "gzip", - "content-length": "2135", - "content-type": "application/json", - "host": "api.v2.sondehub.org", - "user-agent": "autorx-1.4.1-beta4", - "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", - "x-forwarded-for": "103.107.130.22", - "x-forwarded-port": "443", - "x-forwarded-proto": "https", - "date": "Sun, 31 Jan 2021 00:21:45 GMT", - }, - "requestContext": { - "accountId": "143841941773", - "apiId": "r03szwwq41", - "domainName": "api.v2.sondehub.org", - "domainPrefix": "api", - "http": { - "method": "PUT", - "path": "/sondes/telemetry", - "protocol": "HTTP/1.1", - "sourceIp": "103.107.130.22", - "userAgent": "autorx-1.4.1-beta4", - }, - "requestId": "Z_NJvh0RoAMEJaw=", - "routeKey": "PUT /sondes/telemetry", - "stage": "$default", - "time": "31/Jan/2021:00:10:25 +0000", - "timeEpoch": 1612051825409, - }, - "body": "H4sIAFsEMmEAA+XaS2/jNhAA4Pv+CiNnh+X74fOiLYr2UvTUojC0MbMRYEuuLDtdFP3vHWlFKiapmMAaEFAtkMOOKVkjfiGHZP74sFr9Az+r1cOpfm5fi8Zuq+JgHzarh6bYlfWprnZ2W5zbetv8/bAOml5scyrrqmtNkEACGnz42uR83NfFzjbbp2K/P5Wf+zYffyY//fjrI+nbXTcrqtZWVdHf6Tu+ei0udnWoq/pY76372rY82G1jn2x5sbuuJcWUPGLziNlvWG4E3giOhDBaqt/dNbuitd116eYM4f6fb34oqvNz8dSeG9t0l/zQFK/+678c+9t8/P4X/yLOn95EsfFx25TFvk9GY40Zx+6T5+br2yWMMGYIFmz4YF+0EOYKaUYUd8H+3RKMtGGaDMFi37UkUmvEqXs2ezhCUCE5BC52v71AhCLC34ReIAT9pIfQi4U+rj5DUENDd7NT0Z4gZIb/firatr+MuTu9nA/lrmy/QPSR+MTsX2dbPXVBjikS2F1/qpo+Yf9sx/pUtgMbn7BL0r2oUUafruAC4v+u/2deBeKSSZzrlc/vlae8imuvtO9KE3g1GBl+2ytLeFWRV6UQY7leb9qEb522KZxNsyybCmwaTmSmTTm/TZmyqa9tdl3JsQptSsQjmyKyyWObOLapCSL6bjYZItM29doltCCbZIMZvCcqlcixCc3p7DYlTdjUKmFTBDYVjE5c3rYpTGCTI8oimwpn2KS5NumkTEjNpbMsmfAM0NtaZ8qcf0aXqRndJGD6qtTDVMiwECaPYZIAph4JXg2avuG3w2TvTOhm7fJZlkyY4qDQNlm1Zg95dpkiJZMkaGoW0KQYMXKbJglpCkRjmhKqdHM3mgSpaZpk7RJals1uKauFzJ3P1fw2VcqmiG2KUKZAMkOmkIFMPRafbwZNjPDNVXv+bO6fIyFzWAWJZbk0SDAmNcl0qed3qVMuE3Xm2PtephrVTcqEbqGBTIFimJIgKu4I850hcyg0BV7S8pxsYB4RHLPcQhPe0dw0FY5p0gRMGg6ZjCA/jL4D04QLIIlMos5Uo+B7bBxNLs7p2qWzLJfdLglXima6JPO7JAmXOFFmChaugBgfdyG9TBbIpIiJSGZi20gBpptl5l1k4qHMhISWZZMjLnH2EojMfzykEsdDFMuETT+oeZsaaZ1hk0U2ZcImLIzU3WxOrs271Fw6y5IJi3NFlOSZMudfnKvE4pzixL6R8CWpk8kZ0tHBZSyThIUmLIFIJFOYsTj4dpnYb6wmbA4bR5DQsmxC5yiuOM60Of9BkEocBFGSqjVVZFMmNo5imyIcNRXyo/LVjK7veUg5uQrqknMJLcsmjAnKYJa5cUTm3zhSwcaR7ruPJWzq8CCIG/hFvG3ThIeUciwErv7gQ93P5vSMDh+5dN6X+ed/QRi1DYwkAAA=", - "isBase64Encoded": True, - } - print(lambda_handler(payload, {})) diff --git a/lambda/sonde-api-to-iot-core/__main__.py b/lambda/sonde-api-to-iot-core/__main__.py new file mode 100644 index 0000000..8fcfd0e --- /dev/null +++ b/lambda/sonde-api-to-iot-core/__main__.py @@ -0,0 +1,42 @@ +from . import * +payload = { + "version": "2.0", + "routeKey": "PUT /sondes/telemetry", + "rawPath": "/sondes/telemetry", + "rawQueryString": "", + "headers": { + "accept": "*/*", + "accept-encoding": "gzip, deflate", + "content-encoding": "gzip", + "content-length": "2135", + "content-type": "application/json", + "host": "api.v2.sondehub.org", + "user-agent": "autorx-1.4.1-beta4", + "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", + "x-forwarded-for": "103.107.130.22", + "x-forwarded-port": "443", + "x-forwarded-proto": "https", + "date": "Sun, 31 Jan 2021 00:21:45 GMT", + }, + "requestContext": { + "accountId": "143841941773", + "apiId": "r03szwwq41", + "domainName": "api.v2.sondehub.org", + "domainPrefix": "api", + "http": { + "method": "PUT", + "path": "/sondes/telemetry", + "protocol": "HTTP/1.1", + "sourceIp": "103.107.130.22", + "userAgent": "autorx-1.4.1-beta4", + }, + "requestId": "Z_NJvh0RoAMEJaw=", + "routeKey": "PUT /sondes/telemetry", + "stage": "$default", + "time": "31/Jan/2021:00:10:25 +0000", + "timeEpoch": 1612051825409, + }, + "body": "H4sIAFsEMmEAA+XaS2/jNhAA4Pv+CiNnh+X74fOiLYr2UvTUojC0MbMRYEuuLDtdFP3vHWlFKiapmMAaEFAtkMOOKVkjfiGHZP74sFr9Az+r1cOpfm5fi8Zuq+JgHzarh6bYlfWprnZ2W5zbetv8/bAOml5scyrrqmtNkEACGnz42uR83NfFzjbbp2K/P5Wf+zYffyY//fjrI+nbXTcrqtZWVdHf6Tu+ei0udnWoq/pY76372rY82G1jn2x5sbuuJcWUPGLziNlvWG4E3giOhDBaqt/dNbuitd116eYM4f6fb34oqvNz8dSeG9t0l/zQFK/+678c+9t8/P4X/yLOn95EsfFx25TFvk9GY40Zx+6T5+br2yWMMGYIFmz4YF+0EOYKaUYUd8H+3RKMtGGaDMFi37UkUmvEqXs2ezhCUCE5BC52v71AhCLC34ReIAT9pIfQi4U+rj5DUENDd7NT0Z4gZIb/firatr+MuTu9nA/lrmy/QPSR+MTsX2dbPXVBjikS2F1/qpo+Yf9sx/pUtgMbn7BL0r2oUUafruAC4v+u/2deBeKSSZzrlc/vlae8imuvtO9KE3g1GBl+2ytLeFWRV6UQY7leb9qEb522KZxNsyybCmwaTmSmTTm/TZmyqa9tdl3JsQptSsQjmyKyyWObOLapCSL6bjYZItM29doltCCbZIMZvCcqlcixCc3p7DYlTdjUKmFTBDYVjE5c3rYpTGCTI8oimwpn2KS5NumkTEjNpbMsmfAM0NtaZ8qcf0aXqRndJGD6qtTDVMiwECaPYZIAph4JXg2avuG3w2TvTOhm7fJZlkyY4qDQNlm1Zg95dpkiJZMkaGoW0KQYMXKbJglpCkRjmhKqdHM3mgSpaZpk7RJals1uKauFzJ3P1fw2VcqmiG2KUKZAMkOmkIFMPRafbwZNjPDNVXv+bO6fIyFzWAWJZbk0SDAmNcl0qed3qVMuE3Xm2PtephrVTcqEbqGBTIFimJIgKu4I850hcyg0BV7S8pxsYB4RHLPcQhPe0dw0FY5p0gRMGg6ZjCA/jL4D04QLIIlMos5Uo+B7bBxNLs7p2qWzLJfdLglXima6JPO7JAmXOFFmChaugBgfdyG9TBbIpIiJSGZi20gBpptl5l1k4qHMhISWZZMjLnH2EojMfzykEsdDFMuETT+oeZsaaZ1hk0U2ZcImLIzU3WxOrs271Fw6y5IJi3NFlOSZMudfnKvE4pzixL6R8CWpk8kZ0tHBZSyThIUmLIFIJFOYsTj4dpnYb6wmbA4bR5DQsmxC5yiuOM60Of9BkEocBFGSqjVVZFMmNo5imyIcNRXyo/LVjK7veUg5uQrqknMJLcsmjAnKYJa5cUTm3zhSwcaR7ruPJWzq8CCIG/hFvG3ThIeUciwErv7gQ93P5vSMDh+5dN6X+ed/QRi1DYwkAAA=", + "isBase64Encoded": True, +} +print(lambda_handler(payload, {})) diff --git a/sqs-to-elk/lambda_function.py b/lambda/sqs-to-elk/__init__.py similarity index 54% rename from sqs-to-elk/lambda_function.py rename to lambda/sqs-to-elk/__init__.py index d6bd4e8..07422bf 100644 --- a/sqs-to-elk/lambda_function.py +++ b/lambda/sqs-to-elk/__init__.py @@ -1,33 +1,6 @@ import json -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth -import boto3 -import botocore.credentials -import os -from io import BytesIO -import gzip -null = None -HOST = os.getenv("ES") -http_session = URLLib3Session() +import es -def es_request(payload, path, method): - # get aws creds - session = boto3.Session() - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(payload.encode('utf-8')) - payload = compressed.getvalue() - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://{HOST}/{path}", data=payload, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - - r = http_session.send(request.prepare()) - if r.status_code != 200: - raise RuntimeError - return json.loads(r.text) def lambda_handler(event, context): @@ -52,7 +25,7 @@ def lambda_handler(event, context): body += "{\"index\":{}}\n" + json.dumps(payload) + "\n" body += "\n" - result = es_request(body, f"telm-{index}/_doc/_bulk", "POST") + result = es.request(body, f"telm-{index}/_doc/_bulk", "POST") if 'errors' in result and result['errors'] == True: error_types = [x['index']['error']['type'] for x in result['items'] if 'error' in x['index']] # get all the error types error_types = [a for a in error_types if a != 'mapper_parsing_exception'] # filter out mapper failures since they will never succeed diff --git a/lambda/sqs-to-elk/__main__.py b/lambda/sqs-to-elk/__main__.py new file mode 100644 index 0000000..cffae6e --- /dev/null +++ b/lambda/sqs-to-elk/__main__.py @@ -0,0 +1,15 @@ +from . import * + +lambda_handler( +{"Records": [ {"body": json.dumps({ + "Type" : "Notification", + "MessageId" : "ac3ff6e4-dc9b-5429-9867-acdec910f1d3", + "TopicArn" : "arn:aws:sns:us-east-1:143841941773:sonde-telem", + "Message" : "[{\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:08.275935Z\", \"datetime\": \"2021-12-20T05:23:24.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2795, \"lat\": 49.36924, \"lon\": 7.49886, \"alt\": 13526.89728, \"temp\": -67.1, \"humidity\": 16.1, \"pressure\": 149.8, \"vel_v\": 2.84469, \"vel_h\": 22.2337, \"heading\": 134.28404, \"sats\": 9, \"batt\": 2.8, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 10.3, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36924,7.49886\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:09.181181Z\", \"datetime\": \"2021-12-20T05:23:25.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2796, \"lat\": 49.36909, \"lon\": 7.49908, \"alt\": 13532.5104, \"temp\": -67.0, \"humidity\": 15.5, \"pressure\": 149.64, \"vel_v\": 8.40014, \"vel_h\": 23.5398, \"heading\": 137.35364, \"sats\": 8, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 10.1, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36909,7.49908\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:10.186732Z\", \"datetime\": \"2021-12-20T05:23:26.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2797, \"lat\": 49.36894, \"lon\": 7.4993, \"alt\": 13538.89727, \"temp\": -67.0, \"humidity\": 15.3, \"pressure\": 149.5, \"vel_v\": 3.88558, \"vel_h\": 23.33546, \"heading\": 137.92239, \"sats\": 9, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 9.9, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36894,7.4993\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:11.192482Z\", \"datetime\": \"2021-12-20T05:23:27.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2798, \"lat\": 49.36878, \"lon\": 7.49952, \"alt\": 13544.98645, \"temp\": -66.9, \"humidity\": 15.1, \"pressure\": 149.29, \"vel_v\": 8.19858, \"vel_h\": 23.61324, \"heading\": 137.41834, \"sats\": 7, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 9.9, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36878,7.49952\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:12.298679Z\", \"datetime\": \"2021-12-20T05:23:28.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2799, \"lat\": 49.36863, \"lon\": 7.49974, \"alt\": 13551.28573, \"temp\": -67.0, \"humidity\": 15.1, \"pressure\": 149.2, \"vel_v\": 4.18856, \"vel_h\": 23.79089, \"heading\": 136.0238, \"sats\": 9, \"batt\": 2.8, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 10.2, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36863,7.49974\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:13.203452Z\", \"datetime\": \"2021-12-20T05:23:29.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2800, \"lat\": 49.36848, \"lon\": 7.49997, \"alt\": 13557.41325, \"temp\": -66.9, \"humidity\": 15.1, \"pressure\": 149.09, \"vel_v\": 9.42486, \"vel_h\": 22.75925, \"heading\": 131.82185, \"sats\": 9, \"batt\": 2.8, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 10.2, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36848,7.49997\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:14.208788Z\", \"datetime\": \"2021-12-20T05:23:30.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2801, \"lat\": 49.36835, \"lon\": 7.5002, \"alt\": 13564.25548, \"temp\": -66.9, \"humidity\": 14.9, \"pressure\": 148.94, \"vel_v\": 5.08188, \"vel_h\": 21.98384, \"heading\": 131.45089, \"sats\": 9, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 9.9, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36835,7.5002\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:15.214374Z\", \"datetime\": \"2021-12-20T05:23:31.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2802, \"lat\": 49.36822, \"lon\": 7.50043, \"alt\": 13568.25246, \"temp\": -66.9, \"humidity\": 15.0, \"pressure\": 148.83, \"vel_v\": 4.3075, \"vel_h\": 21.67209, \"heading\": 129.45889, \"sats\": 9, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 12.3, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36822,7.50043\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:16.168994Z\", \"datetime\": \"2021-12-20T05:23:32.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2803, \"lat\": 49.3681, \"lon\": 7.50066, \"alt\": 13573.57916, \"temp\": -66.8, \"humidity\": 14.6, \"pressure\": 148.63, \"vel_v\": 5.77128, \"vel_h\": 21.90847, \"heading\": 127.22131, \"sats\": 9, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 12.3, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.3681,7.50066\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:17.175154Z\", \"datetime\": \"2021-12-20T05:23:33.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2804, \"lat\": 49.36798, \"lon\": 7.50091, \"alt\": 13577.87674, \"temp\": -66.8, \"humidity\": 14.5, \"pressure\": 148.56, \"vel_v\": 2.98149, \"vel_h\": 22.32659, \"heading\": 127.16235, \"sats\": 9, \"batt\": 2.8, \"frequency\": 402.7, \"burst_timer\": 15902, \"snr\": 10.1, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36798,7.50091\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:18.281927Z\", \"datetime\": \"2021-12-20T05:23:34.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2805, \"lat\": 49.36786, \"lon\": 7.50115, \"alt\": 13584.40235, \"temp\": -66.7, \"humidity\": 14.2, \"pressure\": 148.36, \"vel_v\": 9.96909, \"vel_h\": 22.43142, \"heading\": 128.7627, \"sats\": 8, \"batt\": 2.8, \"frequency\": 402.7, \"burst_timer\": 15851, \"snr\": 9.6, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36786,7.50115\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:20.192606Z\", \"datetime\": \"2021-12-20T05:23:36.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2807, \"lat\": 49.3676, \"lon\": 7.50162, \"alt\": 13597.72634, \"temp\": -66.7, \"humidity\": 14.4, \"pressure\": 148.11, \"vel_v\": 9.48174, \"vel_h\": 23.08522, \"heading\": 130.91454, \"sats\": 9, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15851, \"snr\": 8.2, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.3676,7.50162\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:21.198260Z\", \"datetime\": \"2021-12-20T05:23:37.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2808, \"lat\": 49.36746, \"lon\": 7.50187, \"alt\": 13604.03948, \"temp\": -66.7, \"humidity\": 14.3, \"pressure\": 147.96, \"vel_v\": 3.59212, \"vel_h\": 23.53213, \"heading\": 130.71552, \"sats\": 9, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15851, \"snr\": 8.8, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36746,7.50187\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}, {\"software_name\": \"radiosonde_auto_rx\", \"software_version\": \"1.5.4\", \"uploader_callsign\": \"BARN5\", \"uploader_position\": \"48.950958,6.135996\", \"uploader_antenna\": \"1/4 wave monopole\", \"time_received\": \"2021-12-20T05:23:22.204030Z\", \"datetime\": \"2021-12-20T05:23:38.001000Z\", \"manufacturer\": \"Vaisala\", \"type\": \"RS41\", \"serial\": \"T1310890\", \"subtype\": \"RS41-SGP\", \"frame\": 2809, \"lat\": 49.36733, \"lon\": 7.50211, \"alt\": 13608.1672, \"temp\": -66.7, \"humidity\": 14.4, \"pressure\": 147.89, \"vel_v\": 4.90639, \"vel_h\": 23.74467, \"heading\": 129.53831, \"sats\": 8, \"batt\": 2.7, \"frequency\": 402.7, \"burst_timer\": 15851, \"snr\": 9.5, \"user-agent\": \"Amazon CloudFront\", \"position\": \"49.36733,7.50211\", \"upload_time_delta\": -1.463, \"uploader_alt\": 185.0}]", + "Timestamp" : "2021-12-20T05:23:23.523Z", + "SignatureVersion" : "1", + "Signature" : "WHop4RM+0qm/Q/fyvSZ1Zsnu7NXNnQjLqp5UsJy0LXCy51heeHPdg4pOOuOECit/lcmHBXF2w8r2mhqg/3CstP5BClGUNnBFEZ3mUlz9ia1y3LZvBbyl8ls1/fJg39ooULXXfe1I+2NgQD580rV34KMcF6LDuguHmdb++yTLupRvZc9AKlrpY+VYvALE5ZwQTjqZ6+Y1HSqQXvuYj6PTWxJics68ddRv1ddMWS2jXp3N7cLPNbqW0CiiNBNewlMlIXFoGa/liSjai3Ix3YW83HFcFsAE+s4moHhQHx3xmTlmaRhzNwZreF7dzzuv9tZ2zrx51nXqG+0+rf32sQ6MZQ==", + "SigningCertURL" : "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-7ff5318490ec183fbaddaa2a969abfda.pem", + "UnsubscribeURL" : "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:143841941773:sonde-telem:1a52ac41-6e17-43da-bfb6-114577c94ca6" +})}]} +,{}) \ No newline at end of file diff --git a/lambda/station-api-to-iot-core/__init__.py b/lambda/station-api-to-iot-core/__init__.py new file mode 100644 index 0000000..6b7aa52 --- /dev/null +++ b/lambda/station-api-to-iot-core/__init__.py @@ -0,0 +1,56 @@ +import json +import zlib +import base64 +import datetime +from email.utils import parsedate + + +import es + +def lambda_handler(event, context): + if "isBase64Encoded" in event and event["isBase64Encoded"] == True: + event["body"] = base64.b64decode(event["body"]) + if ( + "content-encoding" in event["headers"] + and event["headers"]["content-encoding"] == "gzip" + ): + event["body"] = zlib.decompress(event["body"], 16 + zlib.MAX_WBITS) + time_delta = None + if "date" in event["headers"]: + try: + time_delta_header = event["headers"]["date"] + time_delta = ( + datetime.datetime(*parsedate(time_delta_header)[:7]) + - datetime.datetime.utcfromtimestamp(event["requestContext"]["timeEpoch"]/1000) + ).total_seconds() + except: + pass + try: + payload = json.loads(event["body"]) + except: + return {"statusCode": 400, "body": "JSON decode issue"} + print(payload) + if "user-agent" in event["headers"]: + event["time_server"] = datetime.datetime.now().isoformat() + payload["user-agent"] = event["headers"]["user-agent"] + if time_delta: + payload["upload_time_delta"] = time_delta + + payload.pop("uploader_contact_email", None) + + # clean up None reports + + if "uploader_position" in payload and None == payload["uploader_position"] or None in payload["uploader_position"]: + payload.pop("uploader_position", None) + + if "uploader_position" in payload: + (payload["uploader_alt"], payload["uploader_position_elk"]) = ( + payload["uploader_position"][2], + f"{payload['uploader_position'][0]},{payload['uploader_position'][1]}", + ) + index = datetime.datetime.utcnow().strftime("listeners-%Y-%m") + payload["ts"] = datetime.datetime.utcnow().isoformat() + + es.request(json.dumps(payload),f"{index}/_doc","POST") + + return {"statusCode": 200, "body": "^v^ telm logged"} \ No newline at end of file diff --git a/lambda/station-api-to-iot-core/__main__.py b/lambda/station-api-to-iot-core/__main__.py new file mode 100644 index 0000000..74cd825 --- /dev/null +++ b/lambda/station-api-to-iot-core/__main__.py @@ -0,0 +1,43 @@ +from . import * +payload = { + "version": "2.0", + "routeKey": "PUT /sondes/telemetry", + "rawPath": "/sondes/telemetry", + "rawQueryString": "", + "headers": { + "accept": "*/*", + "accept-encoding": "gzip, deflate", + "content-length": "2135", + "content-type": "application/json", + "host": "api.v2.sondehub.org", + "user-agent": "autorx-1.4.1-beta4", + "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", + "x-forwarded-for": "103.107.130.22", + "x-forwarded-port": "443", + "x-forwarded-proto": "https", + "date": "Sun, 31 Jan 2021 00:21:45 GMT", + }, + "requestContext": { + "accountId": "143841941773", + "apiId": "r03szwwq41", + "domainName": "api.v2.sondehub.org", + "domainPrefix": "api", + "http": { + "method": "PUT", + "path": "/sondes/telemetry", + "protocol": "HTTP/1.1", + "sourceIp": "103.107.130.22", + "userAgent": "autorx-1.4.1-beta4", + }, + "requestId": "Z_NJvh0RoAMEJaw=", + "routeKey": "PUT /sondes/telemetry", + "stage": "$default", + "time": "31/Jan/2021:00:10:25 +0000", + "timeEpoch": 1612051825409, + }, + "body": """ + {"software_name": "radiosonde_auto_rx", "software_version": "1.5.8-beta2", "uploader_callsign": "LZ3DJ-18", "uploader_position": [null,null,null], "uploader_antenna": "Dipole", "uploader_contact_email": "none@none.com", "mobile": false} + """, + "isBase64Encoded": False, +} +print(lambda_handler(payload, {})) \ No newline at end of file diff --git a/lambda/tawhiri-updater/__init__.py b/lambda/tawhiri-updater/__init__.py new file mode 100644 index 0000000..316c4cd --- /dev/null +++ b/lambda/tawhiri-updater/__init__.py @@ -0,0 +1,21 @@ +import boto3 +import json +import re + +MATCH_OBJECT = re.compile(r"^gfs.\d{8}/\d{2}/atmos/gfs.t\d{2}z.pgrb2.0p50.f192$") +BUCKET = 'noaa-gfs-bdp-pds' +SERVICE_NAME="tawhiri" +CLUSTER_NAME="Tawhiri" +ecs = boto3.client('ecs', region_name="us-east-1") + +def handler(event, context): + for record in event["Records"]: + message = json.loads(record["Sns"]["Message"]) + for inner_record in message['Records']: + if "ObjectCreated" in inner_record['eventName']: + if inner_record['s3']['bucket']['name'] == BUCKET: + print(inner_record['s3']['object']['key']) + if MATCH_OBJECT.match(inner_record['s3']['object']['key']): + print(f"Found new GFS - updating service {inner_record['s3']['object']['key']}") + ecs.update_service(cluster=CLUSTER_NAME, service=SERVICE_NAME, forceNewDeployment=True) + diff --git a/tawhiri-updater/index.py b/lambda/tawhiri-updater/__main__.py similarity index 70% rename from tawhiri-updater/index.py rename to lambda/tawhiri-updater/__main__.py index 926c481..9f31ed5 100644 --- a/tawhiri-updater/index.py +++ b/lambda/tawhiri-updater/__main__.py @@ -1,26 +1,5 @@ -import boto3 -import json -import re - -MATCH_OBJECT = re.compile(r"^gfs.\d{8}/\d{2}/atmos/gfs.t\d{2}z.pgrb2.0p50.f192$") -BUCKET = 'noaa-gfs-bdp-pds' -SERVICE_NAME="tawhiri" -CLUSTER_NAME="Tawhiri" -ecs = boto3.client('ecs', region_name="us-east-1") - -def handler(event, context): - for record in event["Records"]: - message = json.loads(record["Sns"]["Message"]) - for inner_record in message['Records']: - if "ObjectCreated" in inner_record['eventName']: - if inner_record['s3']['bucket']['name'] == BUCKET: - print(inner_record['s3']['object']['key']) - if MATCH_OBJECT.match(inner_record['s3']['object']['key']): - print(f"Found new GFS - updating service {inner_record['s3']['object']['key']}") - ecs.update_service(cluster=CLUSTER_NAME, service=SERVICE_NAME, forceNewDeployment=True) - -if __name__ == "__main__": - handler( +from . import * +handler( { "Records": [ { diff --git a/main.tf b/main.tf index 689031a..f22abf5 100644 --- a/main.tf +++ b/main.tf @@ -123,4 +123,10 @@ resource "aws_acm_certificate" "CertificateManagerCertificate_root" { "*.sondehub.org" ] validation_method = "DNS" +} + +data "archive_file" "lambda" { + type = "zip" + source_dir = "lambda/" + output_path = "${path.module}/build/lambda.zip" } \ No newline at end of file diff --git a/predictor.tf b/predictor.tf index 67aa94a..cd94110 100644 --- a/predictor.tf +++ b/predictor.tf @@ -1,9 +1,3 @@ -data "archive_file" "predict_updater" { - type = "zip" - source_file = "predict_updater/lambda_function.py" - output_path = "${path.module}/build/predict_updater.zip" -} - resource "aws_iam_role" "predict_updater" { path = "/service-role/" name = "predict-updater" @@ -68,9 +62,9 @@ EOF resource "aws_lambda_function" "predict_updater" { function_name = "predict_updater" - handler = "lambda_function.predict" - filename = "${path.module}/build/predict_updater.zip" - source_code_hash = data.archive_file.predict_updater.output_base64sha256 + handler = "predict_updater.predict" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 1024 role = aws_iam_role.predict_updater.arn @@ -142,23 +136,11 @@ resource "aws_apigatewayv2_integration" "reverse_predictions" { payload_format_version = "2.0" } -data "archive_file" "predictions" { - type = "zip" - source_file = "predict/lambda_function.py" - output_path = "${path.module}/build/predictions.zip" -} - -data "archive_file" "reverse_predictions" { - type = "zip" - source_file = "reverse-predict/lambda_function.py" - output_path = "${path.module}/build/reverse-predict.zip" -} - resource "aws_lambda_function" "predictions" { function_name = "predictions" - handler = "lambda_function.predict" - filename = "${path.module}/build/predictions.zip" - source_code_hash = data.archive_file.predictions.output_base64sha256 + handler = "predict.predict" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.basic_lambda_role.arn @@ -181,9 +163,9 @@ resource "aws_lambda_permission" "predictions" { resource "aws_lambda_function" "reverse_predictions" { function_name = "reverse-predictions" - handler = "lambda_function.predict" - filename = "${path.module}/build/reverse-predict.zip" - source_code_hash = data.archive_file.reverse_predictions.output_base64sha256 + handler = "reverse-predict.predict" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.basic_lambda_role.arn @@ -674,17 +656,11 @@ EOF role = aws_iam_role.predictor_update_trigger_lambda.name } -data "archive_file" "predictor_update_trigger_lambda" { - type = "zip" - source_file = "tawhiri-updater/index.py" - output_path = "${path.module}/build/tawhiri-updater.zip" -} - resource "aws_lambda_function" "predictor_update_trigger_lambda" { function_name = "tawhiri-updater" - handler = "index.handler" - filename = "${path.module}/build/tawhiri-updater.zip" - source_code_hash = data.archive_file.predictor_update_trigger_lambda.output_base64sha256 + handler = "tawhiri-updater.handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.predictor_update_trigger_lambda.arn diff --git a/query.tf b/query.tf index a4603fa..7fdd09f 100644 --- a/query.tf +++ b/query.tf @@ -1,25 +1,9 @@ - -data "archive_file" "query" { - type = "zip" - source_file = "query/lambda_function.py" - output_path = "${path.module}/build/query.zip" -} - - - - - - - - - - resource "aws_lambda_function" "get_sondes" { function_name = "query" - handler = "lambda_function.get_sondes" - filename = "${path.module}/build/query.zip" - source_code_hash = data.archive_file.query.output_base64sha256 + handler = "query.get_sondes" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 256 role = aws_iam_role.basic_lambda_role.arn @@ -40,9 +24,9 @@ resource "aws_lambda_function" "get_sondes" { resource "aws_lambda_function" "get_telem" { function_name = "get_telem" - handler = "lambda_function.get_telem" - filename = "${path.module}/build/query.zip" - source_code_hash = data.archive_file.query.output_base64sha256 + handler = "query.get_telem" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 256 role = aws_iam_role.basic_lambda_role.arn @@ -58,9 +42,9 @@ resource "aws_lambda_function" "get_telem" { resource "aws_lambda_function" "get_sites" { function_name = "get_sites" - handler = "lambda_function.get_sites" - filename = "${path.module}/build/query.zip" - source_code_hash = data.archive_file.query.output_base64sha256 + handler = "query.get_sites" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 256 role = aws_iam_role.basic_lambda_role.arn @@ -76,9 +60,9 @@ resource "aws_lambda_function" "get_sites" { resource "aws_lambda_function" "get_listener_telemetry" { function_name = "get_listener_telemetry" - handler = "lambda_function.get_listener_telemetry" - filename = "${path.module}/build/query.zip" - source_code_hash = data.archive_file.query.output_base64sha256 + handler = "query.get_listener_telemetry" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 256 role = aws_iam_role.basic_lambda_role.arn diff --git a/recovered.tf b/recovered.tf index aa99f5e..bd82f33 100644 --- a/recovered.tf +++ b/recovered.tf @@ -1,8 +1,3 @@ -data "archive_file" "recovered" { - type = "zip" - source_file = "recovered/lambda_function.py" - output_path = "${path.module}/build/recovered.zip" -} resource "aws_iam_role" "recovered" { path = "/service-role/" @@ -58,9 +53,9 @@ EOF resource "aws_lambda_function" "recovered_get" { function_name = "recovered_get" - handler = "lambda_function.get" - filename = "${path.module}/build/recovered.zip" - source_code_hash = data.archive_file.recovered.output_base64sha256 + handler = "recovered.get" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.recovered.arn @@ -77,9 +72,9 @@ resource "aws_lambda_function" "recovered_get" { resource "aws_lambda_function" "recovered_stats" { function_name = "recovered_stats" - handler = "lambda_function.stats" - filename = "${path.module}/build/recovered.zip" - source_code_hash = data.archive_file.recovered.output_base64sha256 + handler = "recovered.stats" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.recovered.arn @@ -96,9 +91,9 @@ resource "aws_lambda_function" "recovered_stats" { resource "aws_lambda_function" "recovered_put" { function_name = "recovered_put" - handler = "lambda_function.put" - filename = "${path.module}/build/recovered.zip" - source_code_hash = data.archive_file.recovered.output_base64sha256 + handler = "recovered.put" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.recovered.arn diff --git a/sign-websocket/lambda_function.py b/sign-websocket/lambda_function.py deleted file mode 100644 index 86aaf1a..0000000 --- a/sign-websocket/lambda_function.py +++ /dev/null @@ -1,13 +0,0 @@ -import boto3 -import time -import uuid -import urllib.parse -import hmac, datetime, hashlib -import os - -def lambda_handler(event, context): - - return {"statusCode": 200, "body": "wss://ws-reader.v2.sondehub.org/"} - -if __name__ == "__main__": - print(lambda_handler({}, {})) \ No newline at end of file diff --git a/sonde-to-s3/README.md b/sonde-to-s3/README.md deleted file mode 100644 index ed8c512..0000000 --- a/sonde-to-s3/README.md +++ /dev/null @@ -1 +0,0 @@ -Lambda function which reads off SQS queue and processes sonde data to S3. \ No newline at end of file diff --git a/sonde-to-s3/lambda_function.py b/sonde-to-s3/lambda_function.py deleted file mode 100644 index 1d80679..0000000 --- a/sonde-to-s3/lambda_function.py +++ /dev/null @@ -1,106 +0,0 @@ -import sys -sys.path.append("vendor/lib/python3.9/site-packages") -import json -import boto3 -import os -import uuid -import hashlib -import asyncio -import aioboto3 - -BUCKET = "sondehub-open-data" - -def values_to_hash(payload): - fields = [ - "type", - "serial", - "frame", - "datetime", - "lat", - "lon", - "alt", - "subtype", - "temp", - "humidity", - "pressure", - "vel_h", - "vel_v", - "heading", - "sats", - "batt", - "burst_timer", - "xdata" - ] - output = "" - for field in fields: - if field in payload: - output += str(payload[field]) - return output - -def set_connection_header(request, operation_name, **kwargs): - request.headers['Connection'] = 'keep-alive' - -async def upload(event, context): - async with aioboto3.client("s3") as s3: - s3.meta.events.register('request-created.s3', set_connection_header) - tasks = [] - payloads = {} - for record in event['Records']: - sns_message = json.loads(record["body"]) - if type(json.loads(sns_message["Message"])) == dict: - incoming_payloads = [json.loads(sns_message["Message"])] - else: - incoming_payloads = json.loads(sns_message["Message"]) - for payload in incoming_payloads: - - body = json.dumps(payload) - id = str(uuid.uuid4()) - hash = hashlib.sha256(values_to_hash(payload).encode("utf-8")).hexdigest() - - filenames = [ - f"date/{payload['datetime']}-{payload['serial']}-{id}.json", - f"serial/{payload['serial']}/{payload['datetime']}-{id}.json", - f"serial-hashed/{payload['serial']}/{payload['datetime']}-{hash}.json" - ] - - for filename in filenames: - tasks.append(s3.put_object( - ACL="public-read", - Bucket=BUCKET, - Body=body, - Key=filename - )) - await asyncio.gather(*tasks) - - -def lambda_handler(event, context): - asyncio.run(upload(event, context)) - - -# test event -########### -if __name__ == "__main__": - demo_event = { - "Records": [ - { - "messageId": "262d4090-e23b-4907-b677-3c94334dc899", - "receiptHandle": "AQEBL1FXHS4m+Om59KZH9ayxC5VBqDEDh6DgXUZuBhV2uQJS312bhOTpLvptuCCIWaeLkfHU+7NajqV2kTVhnz5lehE/zfQ8OU1jqqm+cHxyul99MxA7K7+C+ww2Ri9KSbgaAgqvZzcLbwpW8rP0MNhrBcIQAE5Pz1urfTZKx1RVnv/XQHbR2ARPwocOzk2yEexa0y2f7FedS4F10gju8Ypp0Zr4DSRb1zUkES3QJGiSJakaO1QJT5npRySjAd0CUSPXw7IDTejolfGkItQG5eMRx0enELTUDv8LPsHJkr7ha3DHNfbvxTtdk406nWFn8U8DW515emp7+Y+AD469OnceIMdVC62GHwrpMkedXzLEH0C8TOXHQ+WuRkhR1dauwKqO", - "body": "{\n \"Type\" : \"Notification\",\n \"MessageId\" : \"65147554-e06d-5324-a87d-2da107fea807\",\n \"TopicArn\" : \"arn:aws:sns:us-east-1:143841941773:sonde-telem\",\n \"Message\" : \"{\\\"software_name\\\":\\\"radiosonde_auto_rx\\\",\\\"software_version\\\":\\\"1.5.1\\\",\\\"uploader_callsign\\\":\\\"BIOWL1\\\",\\\"uploader_position\\\":\\\"52.014417,8.47351\\\",\\\"uploader_antenna\\\":\\\"SirioCX395\\\",\\\"time_received\\\":\\\"2021-04-18T07:52:37.196266Z\\\",\\\"datetime\\\":\\\"2021-04-18T07:52:53.001000Z\\\",\\\"manufacturer\\\":\\\"Vaisala\\\",\\\"type\\\":\\\"RS41\\\",\\\"serial\\\":\\\"meowmeowtest\\\",\\\"subtype\\\":\\\"RS41-SGP\\\",\\\"frame\\\":12781,\\\"lat\\\":50.65064,\\\"lon\\\":6.60805,\\\"alt\\\":2954.44289,\\\"temp\\\":-9.3,\\\"humidity\\\":75.4,\\\"pressure\\\":709.79,\\\"vel_v\\\":-2.85326,\\\"vel_h\\\":8.53055,\\\"heading\\\":236.0122,\\\"sats\\\":9,\\\"batt\\\":2.7,\\\"frequency\\\":405.3,\\\"burst_timer\\\":25423,\\\"snr\\\":12.5,\\\"user-agent\\\":\\\"Amazon CloudFront\\\",\\\"position\\\":\\\"50.65064,6.60805\\\",\\\"upload_time_delta\\\":-0.713689,\\\"uploader_alt\\\":340}\",\n \"Timestamp\" : \"2021-04-18T07:52:51.776Z\",\n \"SignatureVersion\" : \"1\",\n \"Signature\" : \"qXuYwDAGPYYLjKXfDtF69AWKDEhhz9MXlqxO2nBwJ/dgOqNSUZtDPqOYSuge3jVCoTSRY5qGw38gg2G+JnEbJd8SVvp9GRsFre8MKWu8T0obq3rj8S0YAh7dTqi4EILIMmi2KziasCDQlrVuZvCSgPnC+hYF3GByI626QW6m3a4E2igclvbE+O6x6qvVDKwmf/eh+8LRiH1PCrEckiXthnr+qOCiTcstyZoOqMOShJBun9k0DK07+Yf1tYDPSHnqZSIaOvAMSjIKKXfGCkel3SWieO7Zgk7xQuo9Z1bcV8Miu4uEvge4G9HKU3S41zaVcQjYvEhQLxxgd1x3HxXImA==\",\n \"SigningCertURL\" : \"https://sns.us-east-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem\",\n \"UnsubscribeURL\" : \"https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:143841941773:sonde-telem:1a52ac41-6e17-43da-bfb6-114577c94ca6\"\n}", - "attributes": { - "ApproximateReceiveCount": "2", - "SentTimestamp": "1618732371814", - "SenderId": "AIDAIT2UOQQY3AUEKVGXU", - "ApproximateFirstReceiveTimestamp": "1618732640317" - }, - "messageAttributes": {}, - "md5OfMessageAttributes": None, - "md5OfBody": "a0191fc5ea3705340c088e457c31095b", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:143841941773:to-elk", - "awsRegion": "us-east-1" - } - ] - } - lambda_handler(demo_event, {}) - - diff --git a/sqs_to_elk.tf b/sqs_to_elk.tf index 341b4df..49fc711 100644 --- a/sqs_to_elk.tf +++ b/sqs_to_elk.tf @@ -1,9 +1,3 @@ -data "archive_file" "sqs_to_elk" { - type = "zip" - source_file = "sqs-to-elk/lambda_function.py" - output_path = "${path.module}/build/sqs-to-elk.zip" -} - resource "aws_iam_role" "sqs_to_elk" { path = "/service-role/" name = "sqs-to-elk" @@ -62,9 +56,9 @@ EOF resource "aws_lambda_function" "sqs_to_elk" { function_name = "sqs-to-elk" - handler = "lambda_function.lambda_handler" - filename = "${path.module}/build/sqs-to-elk.zip" - source_code_hash = data.archive_file.sqs_to_elk.output_base64sha256 + handler = "sqs-to-elk.lambda_handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.sqs_to_elk.arn diff --git a/station-api-to-iot-core/lambda_function.py b/station-api-to-iot-core/lambda_function.py deleted file mode 100644 index 1c18e51..0000000 --- a/station-api-to-iot-core/lambda_function.py +++ /dev/null @@ -1,147 +0,0 @@ -import json -import boto3 -import zlib -import base64 -import datetime -import functools -import uuid -import threading -from email.utils import parsedate -from botocore.awsrequest import AWSRequest -from botocore.endpoint import URLLib3Session -from botocore.auth import SigV4Auth -import boto3 -import botocore.credentials - -import os -from io import BytesIO -import gzip - -from multiprocessing import Process -http_session = URLLib3Session() - - -def mirror(path,params): - session = boto3.Session() - headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com", "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - r = http_session.send(request.prepare()) - - -HOST = os.getenv("ES") - -def lambda_handler(event, context): - if "isBase64Encoded" in event and event["isBase64Encoded"] == True: - event["body"] = base64.b64decode(event["body"]) - if ( - "content-encoding" in event["headers"] - and event["headers"]["content-encoding"] == "gzip" - ): - event["body"] = zlib.decompress(event["body"], 16 + zlib.MAX_WBITS) - time_delta = None - if "date" in event["headers"]: - try: - time_delta_header = event["headers"]["date"] - time_delta = ( - datetime.datetime(*parsedate(time_delta_header)[:7]) - - datetime.datetime.utcfromtimestamp(event["requestContext"]["timeEpoch"]/1000) - ).total_seconds() - except: - pass - try: - payload = json.loads(event["body"]) - except: - return {"statusCode": 400, "body": "JSON decode issue"} - print(payload) - if "user-agent" in event["headers"]: - event["time_server"] = datetime.datetime.now().isoformat() - payload["user-agent"] = event["headers"]["user-agent"] - if time_delta: - payload["upload_time_delta"] = time_delta - - payload.pop("uploader_contact_email", None) - - # clean up None reports - - if "uploader_position" in payload and None == payload["uploader_position"]: - payload.pop("uploader_position", None) - - if "uploader_position" in payload: - (payload["uploader_alt"], payload["uploader_position_elk"]) = ( - payload["uploader_position"][2], - f"{payload['uploader_position'][0]},{payload['uploader_position'][1]}", - ) - index = datetime.datetime.utcnow().strftime("listeners-%Y-%m") - payload["ts"] = datetime.datetime.utcnow().isoformat() - - es_request(json.dumps(payload),f"{index}/_doc","POST") - - return {"statusCode": 200, "body": "^v^ telm logged"} - - - -def es_request(payload, path, method): - # get aws creds - session = boto3.Session() - compressed = BytesIO() - with gzip.GzipFile(fileobj=compressed, mode='w') as f: - f.write(payload.encode('utf-8')) - payload = compressed.getvalue() - headers = {"Host": HOST, "Content-Type": "application/json", "Content-Encoding":"gzip"} - request = AWSRequest( - method="POST", url=f"https://{HOST}/{path}", data=payload, headers=headers - ) - SigV4Auth(boto3.Session().get_credentials(), "es", "us-east-1").add_auth(request) - #p = Process(target=mirror, args=(path,payload)).start() - session = URLLib3Session() - r = session.send(request.prepare()) - if r.status_code != 200 and r.status_code != 201: - raise RuntimeError - return json.loads(r.text) - -if __name__ == "__main__": - payload = { - "version": "2.0", - "routeKey": "PUT /sondes/telemetry", - "rawPath": "/sondes/telemetry", - "rawQueryString": "", - "headers": { - "accept": "*/*", - "accept-encoding": "gzip, deflate", - "content-length": "2135", - "content-type": "application/json", - "host": "api.v2.sondehub.org", - "user-agent": "autorx-1.4.1-beta4", - "x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a", - "x-forwarded-for": "103.107.130.22", - "x-forwarded-port": "443", - "x-forwarded-proto": "https", - "date": "Sun, 31 Jan 2021 00:21:45 GMT", - }, - "requestContext": { - "accountId": "143841941773", - "apiId": "r03szwwq41", - "domainName": "api.v2.sondehub.org", - "domainPrefix": "api", - "http": { - "method": "PUT", - "path": "/sondes/telemetry", - "protocol": "HTTP/1.1", - "sourceIp": "103.107.130.22", - "userAgent": "autorx-1.4.1-beta4", - }, - "requestId": "Z_NJvh0RoAMEJaw=", - "routeKey": "PUT /sondes/telemetry", - "stage": "$default", - "time": "31/Jan/2021:00:10:25 +0000", - "timeEpoch": 1612051825409, - }, - "body": """ - {"software_name": "radiosonde_auto_rx", "software_version": "1.5.8-beta2", "uploader_callsign": "LZ3DJ-18", "uploader_position": null, "uploader_antenna": "Dipole", "uploader_contact_email": "none@none.com", "mobile": false} - """, - "isBase64Encoded": False, - } - print(lambda_handler(payload, {})) \ No newline at end of file diff --git a/websockets.tf b/websockets.tf index 5d9285e..c14dc8b 100644 --- a/websockets.tf +++ b/websockets.tf @@ -23,17 +23,11 @@ EOF max_session_duration = 3600 } -data "archive_file" "sign_socket" { - type = "zip" - source_file = "sign-websocket/lambda_function.py" - output_path = "${path.module}/build/sign_socket.zip" -} - resource "aws_lambda_function" "sign_socket" { function_name = "sign-websocket" - handler = "lambda_function.lambda_handler" - filename = "${path.module}/build/sign_socket.zip" - source_code_hash = data.archive_file.sign_socket.output_base64sha256 + handler = "sign-websocket.lambda_handler" + filename = data.archive_file.lambda.output_path + source_code_hash = data.archive_file.lambda.output_base64sha256 publish = true memory_size = 128 role = aws_iam_role.sign_socket.arn