PFVCluster/perf/deploy-check.sh

#!/bin/bash
###############################################################################
# deploy-check.sh
#
# Deploys scripts/check.sh to each reachable Proxmox host, executes it
# read-only, and pulls the resulting log back to returned-logs/.
#
# EXPLICITLY SKIPS:
#   - pfv-tsys2   (off the air per user)
#   - pfv-tsys9   (off the air per user; also not in original inventory)
#
# Safety features:
#   - BatchMode=yes        : never hang on a password prompt
#   - ConnectTimeout=8     : fail fast on dead hosts
#   - per-host try/skip    : one bad host never aborts the run
#   - ServerAliveInterval  : detect hung connections
#   - read-only script     : check.sh modifies nothing on the target
###############################################################################
set -uo pipefail

SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "$SELF_DIR/.." && pwd)"
CHECK_SH="$SELF_DIR/scripts/check.sh"
LOG_DIR="$ROOT_DIR/returned-logs"
mkdir -p "$LOG_DIR"

# ONLY the hosts the user told us are alive.
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9)

# Common ssh options: non-interactive, fail-fast, no host-key prompt blocking.
SSH_OPTS=(-o BatchMode=yes
          -o ConnectTimeout=8
          -o ServerAliveInterval=10
          -o ServerAliveCountMax=3
          -o StrictHostKeyChecking=accept-new)

log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }

if [ ! -r "$CHECK_SH" ]; then
    echo "FATAL: $CHECK_SH not found" >&2
    exit 1
fi

# Sanity-check shellcheck clean before shipping (best-effort, non-blocking)
if command -v docker >/dev/null 2>&1; then
    log "pre-flight: shellcheck on check.sh"
    if ! docker run --rm -v "$SELF_DIR:/mnt" -w /mnt \
         koalaman/shellcheck:stable --severity=style --format=gcc scripts/check.sh \
         >"$LOG_DIR/_shellcheck.preflight.txt" 2>&1; then
        log "WARNING: shellcheck reported issues — see _shellcheck.preflight.txt"
        log "         aborting deploy to avoid shipping a broken script"
        exit 1
    fi
    log "pre-flight: shellcheck clean"
fi

summary_pass=()
summary_fail=()
declare -A HOST_PID        # host -> background pid
declare -A HOST_MARKER     # host -> per-host marker file

# Per-host worker — runs in background, one per host, all in parallel.
# Writes status into a marker file consumed by the parent.
worker() {
    local host="$1"
    local marker="$LOG_DIR/_marker.$host"
    : > "$marker"  # truncate
    echo "running" >> "$marker"

    local short=""
    if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
        echo "fail unreachable" >> "$marker"
        return
    fi
    if ! scp "${SSH_OPTS[@]}" "$CHECK_SH" "root@$host:/root/check.sh" >/dev/null 2>&1; then
        echo "fail scp-upload-failed" >> "$marker"
        return
    fi
    local remote_size
    remote_size=$(ssh "${SSH_OPTS[@]}" "root@$host" 'wc -c < /root/check.sh' 2>/dev/null || echo 0)
    if [ "${remote_size:-0}" -lt 1000 ]; then
        echo "fail upload-corrupt" >> "$marker"
        return
    fi
    local remote_stdout
    remote_stdout=$(ssh "${SSH_OPTS[@]}" "root@$host" \
        'chmod +x /root/check.sh && bash /root/check.sh' 2>&1)
    local rc=$?
    if [ "$rc" -ne 0 ]; then
        echo "fail check-exit-$rc" >> "$marker"
        # don't return - still try to pull whatever log got produced
    fi
    short=$(printf '%s\n' "$remote_stdout" | grep -oE 'Wrote: /root/[a-zA-Z0-9_-]+\.log' | head -n1 | awk '{print $2}')
    if [ -z "$short" ]; then
        short=$(ssh "${SSH_OPTS[@]}" "root@$host" 'echo "/root/$(hostname -s).log"' 2>/dev/null)
    fi
    if [ -z "$short" ]; then
        echo "fail no-log-path" >> "$marker"
        return
    fi
    if ! scp "${SSH_OPTS[@]}" "root@$host:$short" "$LOG_DIR/" >/dev/null 2>&1; then
        echo "fail scp-download-failed" >> "$marker"
        return
    fi
    local local_name local_path
    local_name="$(basename "$short")"
    local_path="$LOG_DIR/$local_name"
    if [ ! -s "$local_path" ]; then
        echo "fail local-empty" >> "$marker"
        return
    fi
    echo "ok $local_name $(wc -c < "$local_path") $(wc -l < "$local_path")" >> "$marker"
}

# ---- launch all workers in parallel --------------------------------------
log "launching ${#HOSTS[@]} hosts in parallel..."
for host in "${HOSTS[@]}"; do
    rm -f "$LOG_DIR/_marker.$host"
    worker "$host" &
    HOST_PID[$host]=$!
    HOST_MARKER[$host]="$LOG_DIR/_marker.$host"
    log "  launched $host (pid ${HOST_PID[$host]})"
done

# ---- wait for all, with periodic progress --------------------------------
remaining=("${HOSTS[@]}")
while [ "${#remaining[@]}" -gt 0 ]; do
    sleep 10
    new_remaining=()
    for host in "${remaining[@]}"; do
        if ! kill -0 "${HOST_PID[$host]}" 2>/dev/null; then
            # process finished
            wait "${HOST_PID[$host]}" 2>/dev/null || true
            marker="${HOST_MARKER[$host]}"
            if [ -r "$marker" ]; then
                status_line="$(tail -n1 "$marker")"
                log "[$host] done: $status_line"
                case "$status_line" in
                    ok*)     summary_pass+=("$host:$status_line") ;;
                    fail*)   summary_fail+=("$host:$status_line") ;;
                    *)       summary_fail+=("$host:unknown") ;;
                esac
            else
                log "[$host] done but marker missing"
                summary_fail+=("$host:no-marker")
            fi
        else
            new_remaining+=("$host")
        fi
    done
    if [ "${#new_remaining[@]}" -gt 0 ]; then
        remaining=("${new_remaining[@]}")
    else
        remaining=()
    fi
    if [ "${#remaining[@]}" -gt 0 ]; then
        log "still running: ${remaining[*]} (${#remaining[@]} hosts)"
    fi
done

# Final summary
log "============================================================"
log "DEPLOY SUMMARY"
log "============================================================"
log "Passed (${#summary_pass[@]}):"
for p in "${summary_pass[@]:-}"; do [ -n "$p" ] && log "  ✓ $p"; done
log "Failed (${#summary_fail[@]}):"
for f in "${summary_fail[@]:-}"; do [ -n "$f" ] && log "  ✗ $f"; done
log ""
log "Contents of $LOG_DIR:"
ls -la "$LOG_DIR"

# Clean up marker files
rm -f "$LOG_DIR"/_marker.* 2>/dev/null