From 3d382abf06162d14760ed1ca53ee852f02bf7233 Mon Sep 17 00:00:00 2001 From: ReachableCEO Date: Thu, 10 Jul 2025 23:42:11 -0500 Subject: [PATCH] apple health data will soon be out in the world... --- inprep/applehealth-grafana/.gitignore | 129 ++++++++++++++++++ inprep/applehealth-grafana/Dockerfile | 6 + inprep/applehealth-grafana/LICENSE | 21 +++ inprep/applehealth-grafana/README.md | 1 + inprep/applehealth-grafana/app.py | 107 +++++++++++++++ inprep/applehealth-grafana/docker-compose.yml | 10 +- inprep/applehealth-grafana/requirements.txt | 3 + inprep/applehealth-grafana/sample-data.json | 37 +++++ inprep/cleanslate/docker-compose.yml | 2 +- .../atomichabits/docker-compose.yml | 2 +- .../easy-gate/easy-gate.json | 8 +- .../reactive-resume/docker-compose.yml | 2 +- 12 files changed, 320 insertions(+), 8 deletions(-) create mode 100755 inprep/applehealth-grafana/.gitignore create mode 100644 inprep/applehealth-grafana/Dockerfile create mode 100755 inprep/applehealth-grafana/LICENSE create mode 100755 inprep/applehealth-grafana/README.md create mode 100755 inprep/applehealth-grafana/app.py create mode 100755 inprep/applehealth-grafana/requirements.txt create mode 100755 inprep/applehealth-grafana/sample-data.json diff --git a/inprep/applehealth-grafana/.gitignore b/inprep/applehealth-grafana/.gitignore new file mode 100755 index 0000000..b6e4761 --- /dev/null +++ b/inprep/applehealth-grafana/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/inprep/applehealth-grafana/Dockerfile b/inprep/applehealth-grafana/Dockerfile new file mode 100644 index 0000000..1ffdfc5 --- /dev/null +++ b/inprep/applehealth-grafana/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.9 +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY . . +CMD ["python", "app.py"] \ No newline at end of file diff --git a/inprep/applehealth-grafana/LICENSE b/inprep/applehealth-grafana/LICENSE new file mode 100755 index 0000000..6bfda43 --- /dev/null +++ b/inprep/applehealth-grafana/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Ivaylo Pavlov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/inprep/applehealth-grafana/README.md b/inprep/applehealth-grafana/README.md new file mode 100755 index 0000000..a4973c3 --- /dev/null +++ b/inprep/applehealth-grafana/README.md @@ -0,0 +1 @@ +A step by step setup guide is available in this [blog post on Ivo's Blog](https://www.ivaylopavlov.com/charting-apple-healthkit-data-in-grafana/). diff --git a/inprep/applehealth-grafana/app.py b/inprep/applehealth-grafana/app.py new file mode 100755 index 0000000..024994c --- /dev/null +++ b/inprep/applehealth-grafana/app.py @@ -0,0 +1,107 @@ +import json +import sys +import socket +import logging +from datetime import datetime +from flask import request, Flask +from influxdb import InfluxDBClient +from geolib import geohash + +DATAPOINTS_CHUNK = 80000 + +logger = logging.getLogger("console-output") +logger.setLevel(logging.DEBUG) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) + +app = Flask(__name__) +app.debug = True + +client = InfluxDBClient(host='localhost', port=28086) +client.create_database('db') +client.switch_database('db') + +@app.route('/collect', methods=['POST', 'GET']) +def collect(): + logger.info(f"Request received") + + healthkit_data = None + transformed_data = [] + + try: + healthkit_data = json.loads(request.data) + except: + return "Invalid JSON Received", 400 + + try: + logger.info(f"Ingesting Metrics") + for metric in healthkit_data.get("data", {}).get("metrics", []): + number_fields = [] + string_fields = [] + for datapoint in metric["data"]: + metric_fields = set(datapoint.keys()) + metric_fields.remove("date") + for mfield in metric_fields: + if type(datapoint[mfield]) == int or type(datapoint[mfield]) == float: + number_fields.append(mfield) + else: + string_fields.append(mfield) + point = { + "measurement": metric["name"], + "time": datapoint["date"], + "tags": {str(nfield): str(datapoint[nfield]) for nfield in string_fields}, + "fields": {str(nfield): float(datapoint[nfield]) for nfield in number_fields} + } + transformed_data.append(point) + number_fields.clear() + string_fields.clear() + + logger.info(f"Data Transformation Complete") + logger.info(f"Number of data points to write: {len(transformed_data)}") + logger.info(f"DB Write Started") + + for i in range(0, len(transformed_data), DATAPOINTS_CHUNK): + logger.info(f"DB Writing chunk") + client.write_points(transformed_data[i:i + DATAPOINTS_CHUNK]) + + logger.info(f"DB Metrics Write Complete") + logger.info(f"Ingesting Workouts Routes") + + transformed_workout_data = [] + + for workout in healthkit_data.get("data", {}).get("workouts", []): + tags = { + "id": workout["name"] + "-" + workout["start"] + "-" + workout["end"] + } + for gps_point in workout["route"]: + point = { + "measurement": "workouts", + "time": gps_point["timestamp"], + "tags": tags, + "fields": { + "lat": gps_point["lat"], + "lng": gps_point["lon"], + "geohash": geohash.encode(gps_point["lat"], gps_point["lon"], 7), + } + } + transformed_workout_data.append(point) + + for i in range(0, len(transformed_workout_data), DATAPOINTS_CHUNK): + logger.info(f"DB Writing chunk") + client.write_points(transformed_workout_data[i:i + DATAPOINTS_CHUNK]) + + logger.info(f"Ingesting Workouts Complete") + except: + logger.exception("Caught Exception. See stacktrace for details.") + return "Server Error", 500 + + return "Success", 200 + +if __name__ == "__main__": + hostname = socket.gethostname() + ip_address = socket.gethostbyname(hostname) + logger.info(f"Local Network Endpoint: http://{ip_address}/collect") + app.run(host='0.0.0.0', port=5353) diff --git a/inprep/applehealth-grafana/docker-compose.yml b/inprep/applehealth-grafana/docker-compose.yml index 7523dcc..9991f93 100644 --- a/inprep/applehealth-grafana/docker-compose.yml +++ b/inprep/applehealth-grafana/docker-compose.yml @@ -1,9 +1,17 @@ services: + web: + build: . + ports: + - 2007:2007 + depends_on: + - influxdbV1 + volumes: + - .:/app influxdbV1: image: influxdb:1.8.4 container_name: reachableceo-health-data ports: - - "8086:8086" + - "28086:8086" volumes: - reachableceo-health-data:/var/lib/influxdb restart: unless-stopped diff --git a/inprep/applehealth-grafana/requirements.txt b/inprep/applehealth-grafana/requirements.txt new file mode 100755 index 0000000..29e4147 --- /dev/null +++ b/inprep/applehealth-grafana/requirements.txt @@ -0,0 +1,3 @@ +flask +influxdb +geolib diff --git a/inprep/applehealth-grafana/sample-data.json b/inprep/applehealth-grafana/sample-data.json new file mode 100755 index 0000000..23b23de --- /dev/null +++ b/inprep/applehealth-grafana/sample-data.json @@ -0,0 +1,37 @@ +{ + "data" : { + "workouts" : [], + "metrics" : [ + { + "data" : [ + { + "qty" : 34, + "date" : "2021-03-14 00:01:00 +0000" + }, + { + "date" : "2021-03-14 00:51:00 +0000", + "qty" : 25 + }, + { + "qty" : 8.2796763649358063, + "date" : "2021-03-14 01:13:00 +0000" + }, + { + "date" : "2021-03-14 01:14:00 +0000", + "qty" : 17.719295979506633 + }, + { + "date" : "2021-03-14 01:15:00 +0000", + "qty" : 17.719295979506636 + }, + { + "date" : "2021-03-14 01:16:00 +0000", + "qty" : 7.2817316760509208 + } + ], + "name" : "step_count", + "units" : "count" + } + ] + } +} diff --git a/inprep/cleanslate/docker-compose.yml b/inprep/cleanslate/docker-compose.yml index f67dd26..77ccc1e 100644 --- a/inprep/cleanslate/docker-compose.yml +++ b/inprep/cleanslate/docker-compose.yml @@ -78,7 +78,7 @@ services: restart: always container_name: cleanslate-client ports: - - '2010:3000' + - '2006:3000' depends_on: - cleanslate-database - cleanslate-graphql-server diff --git a/released-to-production/atomichabits/docker-compose.yml b/released-to-production/atomichabits/docker-compose.yml index 0f353a6..4749bcc 100644 --- a/released-to-production/atomichabits/docker-compose.yml +++ b/released-to-production/atomichabits/docker-compose.yml @@ -4,7 +4,7 @@ services: image: ghcr.io/majorpeter/atomic-tracker:latest container_name: reachableceo-atomichabits ports: - - "2008:8080" + - "2005:8080" volumes: - tsys-atomichabits:/config restart: always diff --git a/released-to-production/easy-gate/easy-gate.json b/released-to-production/easy-gate/easy-gate.json index 716a0c7..5dfe452 100644 --- a/released-to-production/easy-gate/easy-gate.json +++ b/released-to-production/easy-gate/easy-gate.json @@ -38,22 +38,22 @@ { "name": "Reactive Resume", "category": "", - "url": "http://CharlesDevServer.knel.net:2007", + "url": "http://CharlesDevServer.knel.net:2003", }, { "name": "Atomic Habits", "category": "", - "url": "http://CharlesDevServer.knel.net:2008" + "url": "http://CharlesDevServer.knel.net:2005" }, { "name": "Cleanslate", "category": "", - "url": "http://CharlesDevServer.knel.net:2010" + "url": "http://CharlesDevServer.knel.net:2006" }, { "name": "Apple Health Exporter", "category": "", - "url": "http://CharlesDevServer.knel.net:4000", + "url": "http://CharlesDevServer.knel.net:2007", }, ], "notes": [ diff --git a/released-to-production/reactive-resume/docker-compose.yml b/released-to-production/reactive-resume/docker-compose.yml index e79b68f..06163d7 100644 --- a/released-to-production/reactive-resume/docker-compose.yml +++ b/released-to-production/reactive-resume/docker-compose.yml @@ -52,7 +52,7 @@ services: container_name: tsys-reactiveresume-app restart: unless-stopped ports: - - "2007:3000" + - "2003:3000" depends_on: - reactiveresume-postgres - reactiveresume-minio