#!/usr/bin/env bash
# =============================================================================
# setup-netinfra.sh
# -----------------------------------------------------------------------------
# Replicate pfv-netboot's network services (Pi-hole, Technitium DNS, NTP)
# onto pfv-netinfra-01 and pfv-netinfra-02.
#
# DESIGN
#   * pfv-netboot is REFERENCE ONLY -- this script NEVER mutates it. All reads
#     from it are via `ssh localuser@pfv-netboot 'sudo ...'` (read-only cmds).
#   * The targets cannot SSH to pfv-netboot directly, so config tarballs are
#     relayed through this workstation:
#         ssh netboot 'sudo tar -cf - ...'  |  ssh target 'sudo tar -xf - ...'
#   * Services are deployed under /home/localuser/services/<svc>/ on each
#     target so localuser can manage them (mirrors netboot's localuser-owned
#     pihole data dir). `sudo docker` is used since localuser is not in the
#     docker group (same as on netboot).
#
# SERVICES
#   pihole      pihole/pihole:latest   :53 tcp/udp  :10002->80  :10003->443
#   ntp (chrony) dockurr/chrony        <tailscale-ip>:123:123/udp
#   technitium  technitium/dns-server  :5300->53 tcp/udp  :5380  :53443
#     (Technitium DNS is remapped off :53 to avoid clashing with Pi-hole.
#      The knel.net authoritative zone + Tailscale reverse zones are preserved
#      verbatim from netboot's orphaned dns_tsys-dns-config volume.)
#
# USAGE
#   ./setup-netinfra.sh                # deploy to BOTH nodes
#   ./setup-netinfra.sh pfv-netinfra-01   # deploy to one node
#   ./setup-netinfra.sh pfv-netinfra-01 verify   # verify only
# =============================================================================

set -euo pipefail

NETBOOT="localuser@pfv-netboot"
SVC_ROOT="/home/localuser/services"

log()  { printf '\n\033[1;36m[%s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; }
warn() { printf '\n\033[1;33m[WARN %s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; }

# Per-node parameters. (LAN iface is auto-detected at deploy time as a fallback.)
declare -A NODE_TSIP=(
  [pfv-netinfra-01]="100.70.181.72"
  [pfv-netinfra-02]="100.93.194.82"
)

on_node() { ssh -o StrictHostKeyChecking=no "localuser@$1" "$2"; }

#------------------------------------------------------------------------------
# Verify-only mode
#------------------------------------------------------------------------------
verify_node() {
  local node="$1" tsip="${NODE_TSIP[$1]}"
  log "VERIFY $node (tailscale $tsip)"
  on_node "$node" "bash -s" <<EOF
set +e
echo "### containers ###"
sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' 2>&1
echo
echo "### Pi-hole DNS (dig @127.0.0.1:53 pi.hole) ###"
dig +time=3 +tries=1 +short @127.0.0.1 -p 53 pi.hole 2>&1
echo "### Pi-hole -> Technitium (dig @53 knel.net SOA) ###"
dig +time=3 +tries=1 +short @127.0.0.1 -p 53 knel.net SOA 2>&1
echo "### Pi-hole -> Technitium (dig @53 pfv-netboot.knel.net A) ###"
dig +time=3 +tries=1 +short @127.0.0.1 -p 53 pfv-netboot.knel.net A 2>&1
echo "### Pi-hole web (curl :10002) ###"
curl -sk -o /dev/null -w 'http=%{http_code}\n' http://127.0.0.1:10002/admin/ 2>&1
echo
echo "### Technitium DNS (dig @127.0.0.1:5300 knel.net SOA) ###"
dig +time=3 +tries=1 @127.0.0.1 -p 5300 knel.net SOA +short 2>&1
echo "### Technitium web (curl :5380) ###"
curl -sk -o /dev/null -w 'http=%{http_code}\n' http://127.0.0.1:5380/ 2>&1
echo
echo "### NTP service ###"
HOST_NTP=""
for u in ntpsec ntp chrony openntpd; do
  systemctl is-active --quiet "\$u" 2>/dev/null && { HOST_NTP="\$u"; break; }
done
echo "host daemon: \${HOST_NTP:-none}"
if [ -n "\$HOST_NTP" ]; then
  ntpq -c "rv 0 leap,stratum,offset" 2>&1 | head -3
else
  echo "(no host NTP; chrony container:)"
  sudo docker exec tsys-ntp chronyc -n tracking 2>&1 | head -6
fi
EOF
}

#------------------------------------------------------------------------------
# Deploy to one node
#------------------------------------------------------------------------------
deploy_node() {
  local node="$1" tsip="${NODE_TSIP[$1]}"
  log "==== DEPLOY $node (tailscale $tsip) ===="

  # ---- 1. Prepare directories on the target -------------------------------
  log "$node: create service dirs"
  on_node "$node" "bash -s" <<EOF
set -e
sudo mkdir -p $SVC_ROOT/pihole $SVC_ROOT/ntp $SVC_ROOT/technitium
sudo chown -R localuser:localuser $SVC_ROOT
EOF

  # ---- 2. Write compose files (as localuser) ------------------------------
  log "$node: write docker-compose files"
  on_node "$node" "cat > $SVC_ROOT/pihole/docker-compose.yml" <<'YAML'
services:
  pihole:
    container_name: pihole
    image: pihole/pihole:latest
    hostname: pihole
    ports:
      - "53:53/tcp"
      - "53:53/udp"
      - "10002:80/tcp"
      - "10003:443/tcp"
    environment:
      TZ: 'America/Chicago'
      FTLCONF_webserver_api_password: 'REDACTED_PASSWORD'
      FTLCONF_dns_listeningMode: 'all'
    volumes:
      - './etc-pihole:/etc/pihole'
    cap_add:
      - SYS_NICE
    restart: always
    networks:
      - default
      - dnsnet
networks:
  dnsnet:
    external: true
YAML

  on_node "$node" "cat > $SVC_ROOT/ntp/docker-compose.yml" <<YAML
services:
  ntp:
    image: dockurr/chrony
    container_name: tsys-ntp
    environment:
      NTP_SERVERS: "pool.ntp.org"
    ports:
      - "$tsip:123:123/udp"
    restart: always
YAML

  on_node "$node" "cat > $SVC_ROOT/technitium/docker-compose.yml" <<'YAML'
services:
  technitium:
    image: technitium/dns-server
    container_name: tsys-dns
    ports:
      - "5300:53/tcp"
      - "5300:53/udp"
      - "5380:5380/tcp"
      - "53443:53443/tcp"
    volumes:
      - './config:/etc/dns'
    restart: always
    networks:
      default:
      dnsnet:
        ipv4_address: 10.53.0.53
networks:
  dnsnet:
    external: true
YAML

  # ---- 3. Relay Pi-hole config from netboot -> target ---------------------
  log "$node: copy Pi-hole /etc/pihole from netboot (excluding query logs)"
  if on_node "$node" "test -f $SVC_ROOT/pihole/etc-pihole/gravity.db"; then
    log "$node: Pi-hole config already present; skipping copy"
  else
    on_node "$node" "sudo rm -rf $SVC_ROOT/pihole/etc-pihole"
    ssh -o StrictHostKeyChecking=no "$NETBOOT" \
        "sudo tar -cf - -C /root/pihole --exclude='etc-pihole/pihole-FTL.db*' \
                                   --exclude='etc-pihole/listsCache' \
                                   --exclude='etc-pihole/gravity_backups' \
                                   --exclude='etc-pihole/config_backups' \
                                   etc-pihole" \
      | on_node "$node" "sudo tar -xf - -C $SVC_ROOT/pihole"
  fi

  # ---- 4. Relay Technitium config from netboot orphaned volume -----------
  log "$node: copy Technitium config from netboot (orphaned dns_tsys-dns-config volume)"
  if on_node "$node" "test -f $SVC_ROOT/technitium/config/dns.config"; then
    log "$node: Technitium config already present; skipping copy"
  else
    on_node "$node" "sudo rm -rf $SVC_ROOT/technitium/config"
    ssh -o StrictHostKeyChecking=no "$NETBOOT" \
        "sudo tar -cf - -C /var/lib/docker/volumes/dns_tsys-dns-config _data" \
      | on_node "$node" "sudo tar -xf - -C $SVC_ROOT/technitium && sudo mv $SVC_ROOT/technitium/_data $SVC_ROOT/technitium/config"
  fi

  # ---- 5. Adapt copied config: interface + repoint knel.net to local Technitium
  log "$node: adapt Pi-hole pihole.toml (interface + revServer -> local Technitium)"
  on_node "$node" "bash -s" <<'EOF'
set -e
IFACE=$(ip -o -4 route show to default 2>/dev/null | awk '{print $5; exit}')
IFACE=${IFACE:-ens18}
TOML=/home/localuser/services/pihole/etc-pihole/pihole.toml
if sudo test -f "$TOML"; then
  sudo sed -i "s|^  interface = .*|  interface = \"$IFACE\"  ### ADAPTED from eth0 on clone|" "$TOML"
  echo "set interface=$IFACE"
  # Repoint knel.net conditional forward from netboot's upstream (192.168.3.16)
  # to the LOCAL Technitium container at its fixed dnsnet IP 10.53.0.53.
  # Subnet 100.64.0.0/10 = Tailscale CGNAT range (covers all Tailscale reverse zones).
  if sudo grep -q 'revServers' "$TOML"; then
    sudo sed -i 's|"true,[0-9./]*,192\.168\.3\.16,knel\.net"|"true,100.64.0.0/10,10.53.0.53,knel.net"|' "$TOML"
    echo "revServer repointed to 10.53.0.53 (local Technitium)"
  else
    echo "(revServers not found; FTL will use defaults)"
  fi
else
  echo "(pihole.toml not present; FTL will create it on first run)"
fi
EOF

  # ---- 5b. Create shared Docker network for Pi-hole <-> Technitium ----------
  log "$node: create dnsnet shared Docker network (10.53.0.0/24)"
  on_node "$node" "sudo docker network create --subnet 10.53.0.0/24 dnsnet 2>/dev/null || true"

  # ---- 6. Pull images -----------------------------------------------------
  log "$node: docker compose pull (pihole, ntp, technitium)"
  on_node "$node" "bash -s" <<EOF
for c in pihole ntp technitium; do
  sudo docker compose -f $SVC_ROOT/\$c/docker-compose.yml pull || echo "(pull \$c failed, continuing)"
done
EOF

  # ---- 6a. Pi-hole up -----------------------------------------------------
  log "$node: bring up Pi-hole"
  on_node "$node" "sudo docker compose -f $SVC_ROOT/pihole/docker-compose.yml up -d"

  # ---- 6b. NTP -- only deploy the chrony container if nothing already ----
  # serves UDP/123 on the host. The targets already run a bare-metal ntpsec
  # daemon (stratum-2, synced) on 0.0.0.0:123 -- the SAME service family as
  # netboot's own bare-metal ntpsec. netboot additionally runs a chrony
  # container on its tailscale IP, but that only works there because ntpsec
  # there does not pre-bind the specific tailscale-IP socket. On these targets
  # ntpsec DOES bind the tailscale IP, so the container cannot claim it and is
  # redundant anyway. We therefore keep the host ntpsec as the NTP service.
  log "$node: NTP -- detect host NTP service"
  on_node "$node" "bash -s" <<'EOF'
set +e
HOST_NTP=""
for u in ntpsec ntp chrony openntpd; do
  if systemctl is-active --quiet "$u" 2>/dev/null; then HOST_NTP="$u"; break; fi
done
if [ -n "$HOST_NTP" ]; then
  echo "Host NTP daemon '$HOST_NTP' is active -- it serves NTP on all local"
  echo "addresses (incl. the Tailscale IP). This is the same daemon family as"
  echo "netboot's bare-metal ntpsec; the netboot chrony container is redundant"
  echo "here and CANNOT bind the Tailscale IP (the host daemon already owns it)."
  echo "-> Keeping host NTP. Removing any stale chrony container (tsys-ntp)."
  sudo docker rm -f tsys-ntp 2>/dev/null && echo "   (removed tsys-ntp)" || echo "   (no tsys-ntp to remove)"
  echo "   host peers:"; ntpq -pn 2>/dev/null | head -12 || true
else
  echo "No host NTP daemon active; starting chrony container."
  sudo docker compose -f /home/localuser/services/ntp/docker-compose.yml up -d
fi
EOF

  # ---- 6c. Technitium up --------------------------------------------------
  log "$node: bring up Technitium"
  on_node "$node" "sudo docker compose -f $SVC_ROOT/technitium/docker-compose.yml up -d"

  # ---- 7. Wait for Pi-hole health -----------------------------------------
  log "$node: wait for Pi-hole to become healthy"
  on_node "$node" "bash -s" <<'EOF'
for i in $(seq 1 30); do
  st=$(sudo docker inspect --format '{{.State.Health.Status}}' pihole 2>/dev/null || echo none)
  echo "  pihole health: $st"
  [ "$st" = "healthy" ] && break
  sleep 4
done
EOF

  # ---- 8. Technitium defensive fallback -----------------------------------
  # If the copied (binary) config from an older Technitium version makes the
  # new container crash, move it aside and let Technitium start fresh so the
  # service is at least up (admin reachable) rather than crash-looping.
  log "$node: check Technitium health (fallback to fresh config if crash)"
  on_node "$node" "bash -s" <<'EOF'
set +e
sleep 8
rst=$(sudo docker inspect --format '{{.RestartCount}}' tsys-dns 2>/dev/null || echo 0)
running=$(sudo docker inspect --format '{{.State.Running}}' tsys-dns 2>/dev/null || echo false)
if [ "$running" != "true" ] || [ "$rst" -ge 4 ]; then
  echo "Technitium unhealthy (running=$running restarts=$rst); quarantining copied config"
  sudo docker compose -f /home/localuser/services/technitium/docker-compose.yml stop
  sudo mv /home/localuser/services/technitium/config /home/localuser/services/technitium/config.quarantine.$(date +%s)
  sudo mkdir -p /home/localuser/services/technitium/config
  sudo docker compose -f /home/localuser/services/technitium/docker-compose.yml up -d
  echo "Technitium restarted with fresh config (old config saved as config.quarantine.*)"
else
  echo "Technitium OK (running=$running restarts=$rst)"
fi
EOF

  # ---- 9. Final status -----------------------------------------------------
  log "$node: final container status"
  on_node "$node" "sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'"
}

#------------------------------------------------------------------------------
# Main
#------------------------------------------------------------------------------
main() {
  local mode="${2:-deploy}"
  if [ "${1:-all}" = "all" ]; then
    targets=(pfv-netinfra-01 pfv-netinfra-02)
  else
    targets=("$1")
  fi
  for t in "${targets[@]}"; do
    : "${NODE_TSIP[$t]:?unknown node $t}"
    if [ "$mode" = "verify" ]; then verify_node "$t"; else deploy_node "$t"; fi
  done
  log "DONE"
}

main "$@"