mirror of
https://github.com/projecthorus/sondehub-infra.git
synced 2024-12-30 09:58:54 +00:00
484 lines
14 KiB
Python
484 lines
14 KiB
Python
from multiprocessing import Process
|
|
import json
|
|
import boto3
|
|
import botocore.credentials
|
|
from botocore.awsrequest import AWSRequest
|
|
from botocore.endpoint import URLLib3Session
|
|
from botocore.auth import SigV4Auth
|
|
|
|
import zlib
|
|
import base64
|
|
from datetime import datetime, timedelta
|
|
import os
|
|
from io import BytesIO
|
|
import gzip
|
|
|
|
HOST = os.getenv("ES")
|
|
http_session = URLLib3Session()
|
|
|
|
# get aws creds
|
|
aws_session = boto3.Session()
|
|
|
|
|
|
def mirror(path, params):
|
|
headers = {"Host": "search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com",
|
|
"Content-Type": "application/json", "Content-Encoding": "gzip"}
|
|
request = AWSRequest(
|
|
method="POST", url=f"https://search-sondes-v2-hiwdpmnjbuckpbwfhhx65mweee.us-east-1.es.amazonaws.com/{path}", data=params, headers=headers
|
|
)
|
|
SigV4Auth(aws_session.get_credentials(),
|
|
"es", "us-east-1").add_auth(request)
|
|
session = URLLib3Session()
|
|
r = session.send(request.prepare())
|
|
|
|
|
|
def es_request(payload, path, method):
|
|
|
|
params = json.dumps(payload)
|
|
compressed = BytesIO()
|
|
with gzip.GzipFile(fileobj=compressed, mode='w') as f:
|
|
f.write(params.encode('utf-8'))
|
|
params = compressed.getvalue()
|
|
|
|
headers = {"Host": HOST, "Content-Type": "application/json",
|
|
"Content-Encoding": "gzip"}
|
|
|
|
request = AWSRequest(
|
|
method="POST", url=f"https://{HOST}/{path}", data=params, headers=headers
|
|
)
|
|
SigV4Auth(aws_session.get_credentials(),
|
|
"es", "us-east-1").add_auth(request)
|
|
#p = Process(target=mirror, args=(path,params)).start()
|
|
r = http_session.send(request.prepare())
|
|
return json.loads(r.text)
|
|
|
|
|
|
def getSonde(serial):
|
|
query = {
|
|
"aggs": {
|
|
"1": {
|
|
"top_hits": {
|
|
"docvalue_fields": [
|
|
{
|
|
"field": "position"
|
|
},
|
|
{
|
|
"field": "alt"
|
|
}
|
|
],
|
|
"_source": "position",
|
|
"size": 1,
|
|
"sort": [
|
|
{
|
|
"datetime": {
|
|
"order": "desc"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"query": {
|
|
"bool": {
|
|
"filter": [
|
|
{
|
|
"match_phrase": {
|
|
"serial.keyword": serial
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
results = es_request(query, "telm-*/_search", "POST")
|
|
return results["aggregations"]["1"]["hits"]["hits"]
|
|
|
|
|
|
def getRecovered(serial):
|
|
query = {
|
|
"aggs": {
|
|
"1": {
|
|
"top_hits": {
|
|
"docvalue_fields": [
|
|
{
|
|
"field": "recovered_by.keyword"
|
|
}
|
|
],
|
|
"size": 1,
|
|
"sort": [
|
|
{
|
|
"datetime": {
|
|
"order": "desc"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"query": {
|
|
"bool": {
|
|
"filter": [
|
|
{
|
|
"match_phrase": {
|
|
"serial.keyword": serial
|
|
}
|
|
},
|
|
{
|
|
"match_phrase": {
|
|
"recovered": True
|
|
}
|
|
},
|
|
]
|
|
}
|
|
}
|
|
}
|
|
results = es_request(query, "recovered*/_search", "POST")
|
|
return results["aggregations"]["1"]["hits"]["hits"]
|
|
|
|
|
|
def put(event, context):
|
|
if "isBase64Encoded" in event and event["isBase64Encoded"] == True:
|
|
event["body"] = base64.b64decode(event["body"])
|
|
if (
|
|
"content-encoding" in event["headers"]
|
|
and event["headers"]["content-encoding"] == "gzip"
|
|
):
|
|
event["body"] = zlib.decompress(event["body"], 16 + zlib.MAX_WBITS)
|
|
time_delta = None
|
|
if "date" in event["headers"]:
|
|
try:
|
|
time_delta_header = event["headers"]["date"]
|
|
time_delta = (
|
|
datetime(*parsedate(time_delta_header)[:7])
|
|
- datetime.utcnow()
|
|
).total_seconds()
|
|
except:
|
|
pass
|
|
|
|
recovered = json.loads(event["body"])
|
|
|
|
if "datetime" not in recovered:
|
|
recovered["datetime"] = datetime.now().isoformat()
|
|
|
|
sonde_last_data = getSonde(recovered["serial"])
|
|
|
|
if recovered["serial"] == "":
|
|
return {"statusCode": 400, "body": json.dumps({"message": "serial cannot be empty"})}
|
|
|
|
if len(sonde_last_data) == 0:
|
|
return {"statusCode": 400, "body": json.dumps({"message": "serial not found in db"})}
|
|
|
|
already_recovered = getRecovered(recovered["serial"])
|
|
if len(already_recovered) != 0:
|
|
recovered_by = already_recovered[0]['fields']['recovered_by.keyword'][0]
|
|
return {"statusCode": 400, "body": json.dumps({"message": f"Already recovered by {recovered_by}"})}
|
|
|
|
recovered['position'] = [recovered['lon'], recovered['lat']]
|
|
|
|
result = es_request(recovered, "recovered/_doc", "POST")
|
|
|
|
# add in elasticsearch extra position field
|
|
return {"statusCode": 200, "body": json.dumps({"message": "telm logged. Have a good day ^_^"})}
|
|
|
|
|
|
def get(event, context):
|
|
filters = []
|
|
should = []
|
|
last = 259200
|
|
serials = None
|
|
lat = None
|
|
lon = None
|
|
distance = None
|
|
|
|
# grab query parameters
|
|
if "queryStringParameters" in event:
|
|
if "last" in event["queryStringParameters"]:
|
|
last = int(event['queryStringParameters']['last'])
|
|
if "serial" in event["queryStringParameters"]:
|
|
serials = event['queryStringParameters']['serial'].split(",")
|
|
if "lat" in event["queryStringParameters"]:
|
|
lat = float(event["queryStringParameters"]['lat'])
|
|
if "lon" in event["queryStringParameters"]:
|
|
lon = float(event["queryStringParameters"]['lon'])
|
|
if "distance" in event["queryStringParameters"]:
|
|
distance = int(event["queryStringParameters"]['distance'])
|
|
|
|
if last != 0:
|
|
filters.append(
|
|
{
|
|
"range": {
|
|
"datetime": {
|
|
"gte": f"now-{last}s",
|
|
"lte": "now",
|
|
}
|
|
}
|
|
}
|
|
)
|
|
if serials:
|
|
for serial in serials:
|
|
should.append(
|
|
{
|
|
"match_phrase": {
|
|
"serial.keyword": serial
|
|
}
|
|
}
|
|
)
|
|
if lat and lon and distance:
|
|
filters.append(
|
|
{
|
|
"geo_distance": {
|
|
"distance": f"{distance}m",
|
|
"position": {
|
|
"lat": lat,
|
|
"lon": lon,
|
|
},
|
|
}
|
|
}
|
|
)
|
|
|
|
query = {
|
|
"query": {
|
|
"bool": {
|
|
"filter": filters,
|
|
"should": should,
|
|
}
|
|
},
|
|
"aggs": {
|
|
"2": {
|
|
"terms": {
|
|
"field": "serial.keyword",
|
|
"order": {
|
|
"2-orderAgg": "desc"
|
|
},
|
|
"size": 500
|
|
},
|
|
"aggs": {
|
|
"2-orderAgg": {
|
|
"max": {
|
|
"field": "datetime"
|
|
},
|
|
},
|
|
"1": {
|
|
"top_hits": {
|
|
"_source": True,
|
|
"size": 1,
|
|
"sort": [
|
|
{
|
|
"recovered": {
|
|
"order": "desc"
|
|
}
|
|
},
|
|
{
|
|
"datetime": {
|
|
"order": "desc"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if serials:
|
|
query["query"]["bool"]["minimum_should_match"] = 1
|
|
results = es_request(query, "recovered*/_search", "POST")
|
|
output = [x['1']['hits']['hits'][0]["_source"]
|
|
for x in results['aggregations']['2']['buckets']]
|
|
return {"statusCode": 200, "body": json.dumps(output)}
|
|
|
|
|
|
def stats(event, context):
|
|
filters = []
|
|
should = []
|
|
duration = 0
|
|
serials = None
|
|
lat = None
|
|
lon = None
|
|
distance = None
|
|
requested_time = None
|
|
|
|
# grab query parameters
|
|
if "queryStringParameters" in event:
|
|
if "duration" in event["queryStringParameters"]:
|
|
duration = int(event['queryStringParameters']['duration'])
|
|
if "lat" in event["queryStringParameters"]:
|
|
lat = float(event["queryStringParameters"]['lat'])
|
|
if "lon" in event["queryStringParameters"]:
|
|
lon = float(event["queryStringParameters"]['lon'])
|
|
if "distance" in event["queryStringParameters"]:
|
|
distance = int(event["queryStringParameters"]['distance'])
|
|
if "datetime" in event["queryStringParameters"]:
|
|
requested_time = datetime.fromisoformat(
|
|
event["queryStringParameters"]["datetime"].replace("Z", "+00:00")
|
|
)
|
|
if duration != 0:
|
|
if requested_time:
|
|
lt = requested_time + timedelta(0, 1)
|
|
gte = requested_time - timedelta(0, duration)
|
|
filters.append(
|
|
{
|
|
"range": {
|
|
"datetime": {"gte": gte.isoformat(), "lt": lt.isoformat()}
|
|
}
|
|
}
|
|
)
|
|
if lat and lon and distance:
|
|
filters.append(
|
|
{
|
|
"geo_distance": {
|
|
"distance": f"{distance}m",
|
|
"position": {
|
|
"lat": lat,
|
|
"lon": lon,
|
|
},
|
|
}
|
|
}
|
|
)
|
|
|
|
query = {
|
|
"query": {
|
|
"bool": {
|
|
"filter": filters,
|
|
"should": should,
|
|
}
|
|
},
|
|
"aggs": {
|
|
"chaser_count": {
|
|
"cardinality": {
|
|
"field": "recovered_by.keyword"
|
|
}
|
|
},
|
|
"breakdown": {
|
|
"terms": {
|
|
"field": "recovered",
|
|
"order": {
|
|
"counts": "desc"
|
|
},
|
|
"size": 5
|
|
},
|
|
"aggs": {
|
|
"counts": {
|
|
"cardinality": {
|
|
"field": "serial.keyword"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"top_recovered": {
|
|
"terms": {
|
|
"field": "recovered_by.keyword",
|
|
"order": {
|
|
"recovered_by": "desc"
|
|
},
|
|
"size": 5
|
|
},
|
|
"aggs": {
|
|
"recovered_by": {
|
|
"cardinality": {
|
|
"field": "serial.keyword"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"total_count": {
|
|
"cardinality": {
|
|
"field": "serial.keyword"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
results = es_request(query, "recovered*/_search", "POST")
|
|
|
|
output = {
|
|
"total": 0,
|
|
"recovered": 0,
|
|
"failed": 0,
|
|
"chaser_count": 0,
|
|
"top_chasers": {}
|
|
}
|
|
try:
|
|
output['total'] = results['aggregations']['total_count']['value']
|
|
except:
|
|
output['total'] = 0
|
|
stats = { x['key_as_string'] : x['counts']['value'] for x in results['aggregations']['breakdown']['buckets']}
|
|
try:
|
|
output['recovered'] = stats['true']
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
output['failed'] = stats['false']
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
output['chaser_count'] = results['aggregations']['chaser_count']['value']
|
|
except:
|
|
output['chaser_count'] = 0
|
|
|
|
try:
|
|
output['top_chasers'] = { x['key'] : x['recovered_by']['value'] for x in results['aggregations']['top_recovered']['buckets']}
|
|
except:
|
|
pass
|
|
|
|
return {"statusCode": 200, "body": json.dumps(output)}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
payload = {
|
|
"version": "2.0",
|
|
"routeKey": "PUT /recovered",
|
|
"rawPath": "/recovered",
|
|
"rawQueryString": "",
|
|
"queryStringParameters": {
|
|
"datetime": "2021-12-20T00:00",
|
|
"duration": 100000
|
|
},
|
|
"headers": {
|
|
"accept": "*/*",
|
|
"accept-encoding": "deflate",
|
|
"content-encoding": "",
|
|
"content-length": "2135",
|
|
"content-type": "application/json",
|
|
"host": "api.v2.sondehub.org",
|
|
"user-agent": "autorx-1.4.1-beta4",
|
|
"x-amzn-trace-id": "Root=1-6015f571-6aef2e73165042d53fcc317a",
|
|
"x-forwarded-for": "103.107.130.22",
|
|
"x-forwarded-port": "443",
|
|
"x-forwarded-proto": "https",
|
|
"date": "Sun, 31 Jan 2021 00:21:45 GMT",
|
|
},
|
|
"requestContext": {
|
|
"accountId": "143841941773",
|
|
"apiId": "r03szwwq41",
|
|
"domainName": "api.v2.sondehub.org",
|
|
"domainPrefix": "api",
|
|
"http": {
|
|
"method": "PUT",
|
|
"path": "/sondes/telemetry",
|
|
"protocol": "HTTP/1.1",
|
|
"sourceIp": "103.107.130.22",
|
|
"userAgent": "autorx-1.4.1-beta4",
|
|
},
|
|
"requestId": "Z_NJvh0RoAMEJaw=",
|
|
"routeKey": "PUT /sondes/telemetry",
|
|
"stage": "$default",
|
|
"time": "31/Jan/2021:00:10:25 +0000",
|
|
"timeEpoch": 1612051825409,
|
|
},
|
|
"body": json.dumps({
|
|
"datetime": "2021-06-06T01:10:07.629Z",
|
|
"serial": "string",
|
|
"lat": 0,
|
|
"lon": 0,
|
|
"alt": 0,
|
|
"recovered": True,
|
|
"recovered_by": "string",
|
|
"description": "string"
|
|
}),
|
|
"isBase64Encoded": False,
|
|
}
|
|
# print(put(payload, {}))
|
|
print(stats(payload, {}))
|