#!/bin/bash
# install-utils.sh - installs useful observability packages on a host.
# These are all small, dependency-light, and read-only at runtime.
set -uo pipefail
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)

HOSTS=(pfv-tsys6 pfv-tsys7)

# Packages to install, with rationale
PKGS=(
    sysstat          # sar, iostat, mpstat, pidstat - the missing observability suite
    jq               # JSON parsing for pvesh/scripts
    numactl          # NUMA topology/controls for the dual-socket hosts
    nvme-cli         # NVMe health (for when NVMe shows up)
    tcpdump          # packet capture for network debugging
    mtr-tiny         # traceroute on steroids
    nstat            # kernel SNMP stats (already partly there)
    dnsutils         # dig, nslookup, host
    bmon             # bandwidth monitor ( curses, real-time)
)

for host in "${HOSTS[@]}"; do
    echo "================================================================"
    echo "[$host] installing observability packages"
    echo "================================================================"

    if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
        echo "  UNREACHABLE"
        continue
    fi

    # Check which are missing
    missing=""
    for pkg in "${PKGS[@]}"; do
        if ! ssh "${SSH_OPTS[@]}" "root@$host" "dpkg -s $pkg 2>/dev/null | grep -q 'Status: install ok installed'" 2>/dev/null; then
            missing="$missing $pkg"
        fi
    done

    if [ -z "$missing" ]; then
        echo "  All packages already installed."
        continue
    fi

    echo "  Installing:$missing"
    ssh "${SSH_OPTS[@]}" "root@$host" \
        "DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && \
         DEBIAN_FRONTEND=noninteractive apt-get install -y -qq$missing 2>&1 | tail -5"
    echo "  Done."
    echo ""
done

# Enable sysstat data collection (sar) — off by default on Debian
for host in "${HOSTS[@]}"; do
    echo "[$host] enabling sysstat/sar data collection..."
    ssh "${SSH_OPTS[@]}" "root@$host" \
        "sed -i 's/^ENABLED=\"false\"/ENABLED=\"true\"/' /etc/default/sysstat 2>/dev/null; \
         systemctl enable --now sysstat 2>&1 | tail -2; \
         grep ENABLED /etc/default/sysstat"
done