apple health data will soon be out in the world...

inprep/applehealth-grafana/.gitignore

@@ -0,0 +1,129 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

inprep/applehealth-grafana/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.9
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY . .
+CMD ["python", "app.py"]

inprep/applehealth-grafana/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Ivaylo Pavlov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

inprep/applehealth-grafana/README.md

@@ -0,0 +1 @@
+A step by step setup guide is available in this [blog post on Ivo's Blog](https://www.ivaylopavlov.com/charting-apple-healthkit-data-in-grafana/).

inprep/applehealth-grafana/app.py

@@ -0,0 +1,107 @@
+import json
+import sys
+import socket
+import logging
+from datetime import datetime
+from flask import request, Flask
+from influxdb import InfluxDBClient
+from geolib import geohash
+
+DATAPOINTS_CHUNK = 80000
+
+logger = logging.getLogger("console-output")
+logger.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+app = Flask(__name__)
+app.debug = True
+
+client = InfluxDBClient(host='localhost', port=28086)
+client.create_database('db')
+client.switch_database('db')
+
+@app.route('/collect', methods=['POST', 'GET'])
+def collect():
+    logger.info(f"Request received")
+    
+    healthkit_data = None
+    transformed_data = []
+
+    try:
+        healthkit_data = json.loads(request.data)
+    except:
+        return "Invalid JSON Received", 400
+    
+    try:
+        logger.info(f"Ingesting Metrics")
+        for metric in healthkit_data.get("data", {}).get("metrics", []):
+            number_fields = []
+            string_fields = []
+            for datapoint in metric["data"]:
+                metric_fields = set(datapoint.keys())
+                metric_fields.remove("date")
+                for mfield in metric_fields:
+                    if type(datapoint[mfield]) == int or type(datapoint[mfield]) == float:
+                        number_fields.append(mfield)
+                    else:
+                        string_fields.append(mfield)
+                point = {
+                    "measurement": metric["name"],
+                    "time": datapoint["date"],
+                    "tags": {str(nfield): str(datapoint[nfield]) for nfield in string_fields},
+                    "fields": {str(nfield): float(datapoint[nfield]) for nfield in number_fields}
+                }
+                transformed_data.append(point)
+                number_fields.clear()
+                string_fields.clear()
+
+        logger.info(f"Data Transformation Complete")
+        logger.info(f"Number of data points to write: {len(transformed_data)}")
+        logger.info(f"DB Write Started")
+
+        for i in range(0, len(transformed_data), DATAPOINTS_CHUNK):
+            logger.info(f"DB Writing chunk")
+            client.write_points(transformed_data[i:i + DATAPOINTS_CHUNK])
+        
+        logger.info(f"DB Metrics Write Complete")
+        logger.info(f"Ingesting Workouts Routes")
+
+        transformed_workout_data = []
+        
+        for workout in healthkit_data.get("data", {}).get("workouts", []):
+            tags = {
+                "id": workout["name"] + "-" + workout["start"] + "-" + workout["end"]
+            }
+            for gps_point in workout["route"]:
+                point = {
+                    "measurement": "workouts",
+                    "time": gps_point["timestamp"],
+                    "tags": tags,
+                    "fields": {
+                        "lat": gps_point["lat"],
+                        "lng": gps_point["lon"],
+                        "geohash": geohash.encode(gps_point["lat"], gps_point["lon"], 7),
+                    }
+                }
+                transformed_workout_data.append(point)
+
+            for i in range(0, len(transformed_workout_data), DATAPOINTS_CHUNK):
+                logger.info(f"DB Writing chunk")
+                client.write_points(transformed_workout_data[i:i + DATAPOINTS_CHUNK])
+        
+        logger.info(f"Ingesting Workouts Complete")
+    except:
+        logger.exception("Caught Exception. See stacktrace for details.")
+        return "Server Error", 500
+
+    return "Success", 200
+
+if __name__ == "__main__":
+    hostname = socket.gethostname()
+    ip_address = socket.gethostbyname(hostname)
+    logger.info(f"Local Network Endpoint: http://{ip_address}/collect")
+    app.run(host='0.0.0.0', port=5353)

inprep/applehealth-grafana/docker-compose.yml

@@ -1,9 +1,17 @@
 services:
+  web:
+     build: .
+     ports: 
+      - 2007:2007
+     depends_on: 
+      - influxdbV1
+     volumes:
+       - .:/app
   influxdbV1:
     image: influxdb:1.8.4
     container_name: reachableceo-health-data
     ports:
-      - "8086:8086"
+      - "28086:8086"
     volumes:
       - reachableceo-health-data:/var/lib/influxdb
     restart: unless-stopped

inprep/applehealth-grafana/requirements.txt

@@ -0,0 +1,3 @@
+flask
+influxdb
+geolib

inprep/applehealth-grafana/sample-data.json

@@ -0,0 +1,37 @@
+{
+  "data" : {
+    "workouts" : [],
+    "metrics" : [
+      {
+        "data" : [
+          {
+            "qty" : 34,
+            "date" : "2021-03-14 00:01:00 +0000"
+          },
+          {
+            "date" : "2021-03-14 00:51:00 +0000",
+            "qty" : 25
+          },
+          {
+            "qty" : 8.2796763649358063,
+            "date" : "2021-03-14 01:13:00 +0000"
+          },
+          {
+            "date" : "2021-03-14 01:14:00 +0000",
+            "qty" : 17.719295979506633
+          },
+          {
+            "date" : "2021-03-14 01:15:00 +0000",
+            "qty" : 17.719295979506636
+          },
+          {
+            "date" : "2021-03-14 01:16:00 +0000",
+            "qty" : 7.2817316760509208
+          }
+        ],
+        "name" : "step_count",
+        "units" : "count"
+      }
+    ]
+  }
+}

inprep/cleanslate/docker-compose.yml

@@ -78,7 +78,7 @@ services:
     restart: always
     container_name: cleanslate-client
     ports:
-      - '2010:3000'
+      - '2006:3000'
     depends_on:
       - cleanslate-database
       - cleanslate-graphql-server

released-to-production/atomichabits/docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: ghcr.io/majorpeter/atomic-tracker:latest
     container_name: reachableceo-atomichabits
     ports:
-      - "2008:8080"
+      - "2005:8080"
     volumes:
       - tsys-atomichabits:/config
     restart: always

released-to-production/easy-gate/easy-gate.json

@@ -38,22 +38,22 @@
         {
             "name": "Reactive Resume",
             "category": "",
-            "url": "http://CharlesDevServer.knel.net:2007",
+            "url": "http://CharlesDevServer.knel.net:2003",
         },
         {
             "name": "Atomic Habits",
             "category": "",
-            "url": "http://CharlesDevServer.knel.net:2008"
+            "url": "http://CharlesDevServer.knel.net:2005"
         },
         {
             "name": "Cleanslate",
             "category": "",
-            "url": "http://CharlesDevServer.knel.net:2010"
+            "url": "http://CharlesDevServer.knel.net:2006"
         },
         {
             "name": "Apple Health Exporter",
             "category": "",
-            "url": "http://CharlesDevServer.knel.net:4000",
+            "url": "http://CharlesDevServer.knel.net:2007",
         },
     ],
     "notes": [

released-to-production/reactive-resume/docker-compose.yml

@@ -52,7 +52,7 @@ services:
     container_name: tsys-reactiveresume-app
     restart: unless-stopped
     ports:
-      - "2007:3000"
+      - "2003:3000"
     depends_on:
       - reactiveresume-postgres
       - reactiveresume-minio