PFVCluster/scripts/kuma-fleet-sync.py

#!/usr/bin/env python3
"""kuma-fleet-sync.py — Uptime Kuma ICMP coverage sync [#435]

Compares every Linux tailnet peer + static network gear against Uptime
Kuma ping monitors, reports coverage gaps, and (with --add) creates the
missing monitors over the socket.io API.

Env (from ~/.creds/uptime-kuma.env):
  UPTIME_KUMA_URL, UPTIME_KUMA_TOKEN (long-lived login token),
  UPTIME_KUMA_API_KEY (unused, kept for compatibility)

Notes:
  - status.knownelement.com is behind a proxy that breaks engine.io
    polling pushes, so the websocket transport is forced.
  - Monitor payloads must include `conditions: []` and
    `notificationIDList` or this Kuma build rejects the insert.
  - Static gear (switches/router/APs/PDU) is appended manually below;
    keep in sync with LibreNMS + Technitium knel.net zone.

Usage:
  python3 kuma-fleet-sync.py            # gap report only
  python3 kuma-fleet-sync.py --add      # create missing monitors
"""
import argparse
import os
import socketio
import subprocess
import sys
import time

STATIC_GEAR = [
    "pfv-r5-core-01.knel.net",
    "pfv-r3-tor-mgmt-01.knel.net",
    "pfv-r3-tor-stor-01.knel.net",
    "pfv-r2-tor-01.knel.net",
    "pfv-r1-tor-top.knel.net",
    "pfv-r6-mgmt-01.knel.net",
    "pfv-rrinfra-rtr.knel.net",
    "ap-tablemount.knel.net",
    "ap-wallmount.knel.net",
    "pfv-garage-pdu-1.knel.net",
    "pfv-stor1.knel.net",
    "pfv-printer.knel.net",
    "pfv-consrv.knel.net",
    "pfv-tsys6-oob.knel.net",
    "pfv-tsys7-oob.knel.net",
    "stl-canon-scanner-artroom.knel.net",
    "brother-label-printer.knel.net",
    "dell-openmanage-enterprise.knel.net",
    "netbird.knel.net",
    "tsys-cloudron.knel.net",
]
PAUSED_BY_DEFAULT = {
    "stlp-3dscanner.knel.net",
    "dell-openmanage-enterprise.knel.net",
}


def fleet_targets():
    # Host-gathered peer list (tailscale CLI lives on the workstation, not in
    # the container this script runs in): FLEET_TARGETS=newline-separated FQDNs.
    env = os.environ.get("FLEET_TARGETS")
    if env:
        hosts = {h.strip() for h in env.splitlines() if h.strip()}
    else:
        out = subprocess.run(
            ["tailscale", "status", "--json"], capture_output=True, text=True, check=True
        ).stdout
        import json

        peers = json.loads(out)["Peer"]
        hosts = {
            p["DNSName"].split(".")[0] + ".knel.net"
            for p in peers
            if p.get("OS") == "linux" and p.get("DNSName")
        }
    return hosts | set(STATIC_GEAR)


def kuma_connect():
    sio = socketio.Client(reconnection=False)
    state = {}

    @sio.on("info")
    def _info(_d):
        pass

    @sio.on("monitorList")
    def _ml(d):
        state["ml"] = d

    sio.connect(
        os.environ["UPTIME_KUMA_URL"], transports=["websocket"], wait_timeout=30
    )
    r = sio.call("loginByToken", os.environ["UPTIME_KUMA_TOKEN"], timeout=30)
    if not (isinstance(r, dict) and r.get("ok")):
        raise SystemExit(f"login failed: {r}")
    sio.emit("monitorList")
    for _ in range(20):
        if "ml" in state:
            break
        time.sleep(1)
    return sio, state.get("ml", {})


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--add", action="store_true", help="create missing monitors")
    ap.add_argument(
        "--group-id",
        type=int,
        default=0,
        help="parent group id for new monitors (REQUIRED — root placement "
        "is banned per founder ruling 2026-09-02; use groups)",
    )
    args = ap.parse_args()
    if args.add and not args.group_id:
        raise SystemExit("--group-id is required with --add (no root monitors)")

    targets = fleet_targets()
    sio, mons = kuma_connect()
    covered = {str(m.get("hostname", "")) for m in mons.values()}
    missing = sorted(targets - covered)

    print(f"fleet targets: {len(targets)}  kuma monitors: {len(mons)}  covered: {len(targets & covered)}")
    if not missing:
        print("ICMP coverage: COMPLETE")
        sio.disconnect()
        return
    print("missing:")
    for host in missing:
        print(f"  {host}{'  (would add PAUSED)' if host in PAUSED_BY_DEFAULT else ''}")

    if args.add:
        for host in missing:
            payload = {
                "type": "ping",
                "name": host.split(".")[0],
                "hostname": host,
                "interval": 60,
                "retryInterval": 60,
                "resendInterval": 0,
                "maxretries": 2,
                "notificationIDList": {"2": True, "3": True},  # app webhooks (Pushover retired 2026-09-03)
                "upsideDown": False,
                "description": "ICMP up/down (kuma-fleet-sync)",
                "httpBodyEncoding": "json",
                "accepted_statuscodes": ["200-299"],
                "conditions": [],
                "active": host not in PAUSED_BY_DEFAULT,
                "packetSize": 56,
                "parent": args.group_id,
            }
            r = sio.call("add", payload, timeout=30)
            ok = isinstance(r, dict) and r.get("ok")
            print(f"  ADD {host}: {'id=' + str(r.get('monitorID')) if ok else r}")
    sio.disconnect()


if __name__ == "__main__":
    main()