refactor: reorganize merged repo into clean directory structure
Reorganize the merged KNELServerBuild + PFVCluster repo:
provisioning/ server provisioning (was ProjectCode/ +
Project-Includes/ + Project-ConfigFiles/)
tests/ test suite (was Project-Tests/)
perf/ Proxmox perf scripts (was top-level *.sh + scripts/)
docs/ all documentation (was ProjectDocs/ + PROJECT.md +
K8S.md + TODO.md)
dns-cluster-setup/ Technitium DNS cluster (unchanged)
netinfra/ netinfra audit scripts (unchanged)
switches/ switch configs (unchanged)
vendor/ vendored KNELShellFramework (unchanged)
Update all internal path references from old directory names
(ProjectCode/, Project-Includes/, Project-Tests/) to the new ones
(provisioning/, tests/) across all scripts.
🤖 Generated with [Crush](https://github.com/charmassociates/crush)
Assisted-by: GLM-5 via Crush <crush@charm.land>
This commit is contained in:
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# check-pkgs.sh - verify package install state.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for host in pfv-tsys6 pfv-tsys7; do
|
||||
echo "=== $host ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" '
|
||||
for p in sysstat jq numactl nvme mtr-tiny dig bmon tcpdump; do
|
||||
if command -v "$p" >/dev/null 2>&1; then
|
||||
echo " ✓ $p"
|
||||
else
|
||||
echo " ✗ $p"
|
||||
fi
|
||||
done
|
||||
# sysstat config
|
||||
echo " sysstat service:"
|
||||
systemctl list-unit-files 2>/dev/null | grep -i sysstat | sed "s/^/ /"
|
||||
echo " sysstat enabled in /etc/default:"
|
||||
if [ -r /etc/default/sysstat ]; then
|
||||
grep ENABLED /etc/default/sysstat | sed "s/^/ /"
|
||||
else
|
||||
echo " no /etc/default/sysstat"
|
||||
fi
|
||||
# on Debian trixie, sysstat uses a different path
|
||||
ls /etc/cron.d/sysstat* 2>/dev/null | sed "s/^/ found: /"
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# check-repos-and-reboot.sh - checks reboot-required + Proxmox repo config on all hosts.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$host]"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "--- /var/run/reboot-required ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'if [ -r /var/run/reboot-required ]; then echo "REBOOT REQUIRED"; cat /var/run/reboot-required 2>/dev/null; if [ -r /var/run/reboot-required.pkgs ]; then echo "Packages triggering:"; cat /var/run/reboot-required.pkgs; fi; else echo "(no reboot required marker)"; fi'
|
||||
|
||||
echo ""
|
||||
echo "--- Running kernel vs installed kernel ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'echo "running: $(uname -r)"; echo "installed:"; dpkg -l | grep -E "pve-kernel-[0-9]" | awk "{print \" \"\$2\" \"\$3}" | tail -5'
|
||||
|
||||
echo ""
|
||||
echo "--- Proxmox repositories (apt sources) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -rh "pve\|proxmox" /etc/apt/sources.list /etc/apt/sources.list.d/ 2>/dev/null | grep -v "^#" | sed "s/^/ /"'
|
||||
|
||||
echo ""
|
||||
echo "--- Enterprise repo status (should be commented or absent if no subscription) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -l "pve-enterprise" /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null | while read f; do echo " File: $f"; grep -n "pve-enterprise" "$f" | sed "s/^/ /"; done'
|
||||
|
||||
echo ""
|
||||
echo "--- no-subscription repo presence ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -rl "pve-no-subscription" /etc/apt/sources.list /etc/apt/sources.list.d/ 2>/dev/null | while read f; do echo " File: $f"; grep -n "pve-no-subscription" "$f" | sed "s/^/ /"; done'
|
||||
|
||||
echo ""
|
||||
echo "--- Recently updated packages (last 24h, kernel-related) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -E "pve-kernel|proxmox|pve-qemu|zfs" /var/log/dpkg.log 2>/dev/null | grep "$(date +%Y-%m-%d)\|$(date -d yesterday +%Y-%m-%d)" | tail -15 || echo "(none in dpkg.log)"'
|
||||
|
||||
echo ""
|
||||
done
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# deploy-and-fix.sh - uploads fix script, runs it, starts VMs, verifies.
|
||||
set -uo pipefail
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
SCRIPT_DIR="/home/reachableceo/projects/perfopt/scripts"
|
||||
|
||||
echo "=== Uploading fix script to $HOST ==="
|
||||
scp "${SSH_OPTS[@]}" "$SCRIPT_DIR/fix-bond-nfs.sh" "root@$HOST:/root/fix-bond-nfs.sh" >/dev/null 2>&1
|
||||
|
||||
echo "=== Running fix ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/fix-bond-nfs.sh && bash /root/fix-bond-nfs.sh' 2>&1
|
||||
|
||||
echo ""
|
||||
echo "=== Starting VMs ==="
|
||||
for vmid in $(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1{print \$1}"'); do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" != "running" ]; then
|
||||
echo " Starting VM $vmid..."
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1 | sed 's/^/ /'
|
||||
else
|
||||
echo " VM $vmid already running"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Waiting 20s for VMs to boot..."
|
||||
sleep 20
|
||||
|
||||
echo ""
|
||||
echo "=== FULL VERIFICATION ==="
|
||||
echo ""
|
||||
echo "--- VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
echo "--- NFS mounts ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -24'
|
||||
echo ""
|
||||
echo "--- NFS TCP connections ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
echo " Count:"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l'
|
||||
echo ""
|
||||
echo "--- bond0 hash policy ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 | head -6'
|
||||
echo ""
|
||||
echo "--- Summary ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo "tcp_cc: $(sysctl -n net.ipv4.tcp_congestion_control)"'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo "swappiness: $(sysctl -n vm.swappiness)"'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo "governor: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)"'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'tuned-adm active 2>/dev/null'
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# deploy-check.sh
|
||||
#
|
||||
# Deploys scripts/check.sh to each reachable Proxmox host, executes it
|
||||
# read-only, and pulls the resulting log back to returned-logs/.
|
||||
#
|
||||
# EXPLICITLY SKIPS:
|
||||
# - pfv-tsys2 (off the air per user)
|
||||
# - pfv-tsys9 (off the air per user; also not in original inventory)
|
||||
#
|
||||
# Safety features:
|
||||
# - BatchMode=yes : never hang on a password prompt
|
||||
# - ConnectTimeout=8 : fail fast on dead hosts
|
||||
# - per-host try/skip : one bad host never aborts the run
|
||||
# - ServerAliveInterval : detect hung connections
|
||||
# - read-only script : check.sh modifies nothing on the target
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="/home/reachableceo/projects/perfopt"
|
||||
CHECK_SH="$SCRIPT_DIR/scripts/check.sh"
|
||||
LOG_DIR="$SCRIPT_DIR/returned-logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# ONLY the hosts the user told us are alive.
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9)
|
||||
|
||||
# Common ssh options: non-interactive, fail-fast, no host-key prompt blocking.
|
||||
SSH_OPTS=(-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ServerAliveInterval=10
|
||||
-o ServerAliveCountMax=3
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
|
||||
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
|
||||
|
||||
if [ ! -r "$CHECK_SH" ]; then
|
||||
echo "FATAL: $CHECK_SH not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sanity-check shellcheck clean before shipping (best-effort, non-blocking)
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
log "pre-flight: shellcheck on check.sh"
|
||||
if ! docker run --rm -v "$SCRIPT_DIR:/mnt" -w /mnt \
|
||||
koalaman/shellcheck:stable --severity=style --format=gcc scripts/check.sh \
|
||||
>"$LOG_DIR/_shellcheck.preflight.txt" 2>&1; then
|
||||
log "WARNING: shellcheck reported issues — see _shellcheck.preflight.txt"
|
||||
log " aborting deploy to avoid shipping a broken script"
|
||||
exit 1
|
||||
fi
|
||||
log "pre-flight: shellcheck clean"
|
||||
fi
|
||||
|
||||
summary_pass=()
|
||||
summary_fail=()
|
||||
declare -A HOST_PID # host -> background pid
|
||||
declare -A HOST_MARKER # host -> per-host marker file
|
||||
|
||||
# Per-host worker — runs in background, one per host, all in parallel.
|
||||
# Writes status into a marker file consumed by the parent.
|
||||
worker() {
|
||||
local host="$1"
|
||||
local marker="$LOG_DIR/_marker.$host"
|
||||
: > "$marker" # truncate
|
||||
echo "running" >> "$marker"
|
||||
|
||||
local short=""
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "fail unreachable" >> "$marker"
|
||||
return
|
||||
fi
|
||||
if ! scp "${SSH_OPTS[@]}" "$CHECK_SH" "root@$host:/root/check.sh" >/dev/null 2>&1; then
|
||||
echo "fail scp-upload-failed" >> "$marker"
|
||||
return
|
||||
fi
|
||||
local remote_size
|
||||
remote_size=$(ssh "${SSH_OPTS[@]}" "root@$host" 'wc -c < /root/check.sh' 2>/dev/null || echo 0)
|
||||
if [ "${remote_size:-0}" -lt 1000 ]; then
|
||||
echo "fail upload-corrupt" >> "$marker"
|
||||
return
|
||||
fi
|
||||
local remote_stdout
|
||||
remote_stdout=$(ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
'chmod +x /root/check.sh && bash /root/check.sh' 2>&1)
|
||||
local rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "fail check-exit-$rc" >> "$marker"
|
||||
# don't return - still try to pull whatever log got produced
|
||||
fi
|
||||
short=$(printf '%s\n' "$remote_stdout" | grep -oE 'Wrote: /root/[a-zA-Z0-9_-]+\.log' | head -n1 | awk '{print $2}')
|
||||
if [ -z "$short" ]; then
|
||||
short=$(ssh "${SSH_OPTS[@]}" "root@$host" 'echo "/root/$(hostname -s).log"' 2>/dev/null)
|
||||
fi
|
||||
if [ -z "$short" ]; then
|
||||
echo "fail no-log-path" >> "$marker"
|
||||
return
|
||||
fi
|
||||
if ! scp "${SSH_OPTS[@]}" "root@$host:$short" "$LOG_DIR/" >/dev/null 2>&1; then
|
||||
echo "fail scp-download-failed" >> "$marker"
|
||||
return
|
||||
fi
|
||||
local local_name local_path
|
||||
local_name="$(basename "$short")"
|
||||
local_path="$LOG_DIR/$local_name"
|
||||
if [ ! -s "$local_path" ]; then
|
||||
echo "fail local-empty" >> "$marker"
|
||||
return
|
||||
fi
|
||||
echo "ok $local_name $(wc -c < "$local_path") $(wc -l < "$local_path")" >> "$marker"
|
||||
}
|
||||
|
||||
# ---- launch all workers in parallel --------------------------------------
|
||||
log "launching ${#HOSTS[@]} hosts in parallel..."
|
||||
for host in "${HOSTS[@]}"; do
|
||||
rm -f "$LOG_DIR/_marker.$host"
|
||||
worker "$host" &
|
||||
HOST_PID[$host]=$!
|
||||
HOST_MARKER[$host]="$LOG_DIR/_marker.$host"
|
||||
log " launched $host (pid ${HOST_PID[$host]})"
|
||||
done
|
||||
|
||||
# ---- wait for all, with periodic progress --------------------------------
|
||||
remaining=("${HOSTS[@]}")
|
||||
while [ "${#remaining[@]}" -gt 0 ]; do
|
||||
sleep 10
|
||||
new_remaining=()
|
||||
for host in "${remaining[@]}"; do
|
||||
if ! kill -0 "${HOST_PID[$host]}" 2>/dev/null; then
|
||||
# process finished
|
||||
wait "${HOST_PID[$host]}" 2>/dev/null || true
|
||||
marker="${HOST_MARKER[$host]}"
|
||||
if [ -r "$marker" ]; then
|
||||
status_line="$(tail -n1 "$marker")"
|
||||
log "[$host] done: $status_line"
|
||||
case "$status_line" in
|
||||
ok*) summary_pass+=("$host:$status_line") ;;
|
||||
fail*) summary_fail+=("$host:$status_line") ;;
|
||||
*) summary_fail+=("$host:unknown") ;;
|
||||
esac
|
||||
else
|
||||
log "[$host] done but marker missing"
|
||||
summary_fail+=("$host:no-marker")
|
||||
fi
|
||||
else
|
||||
new_remaining+=("$host")
|
||||
fi
|
||||
done
|
||||
remaining=("${new_remaining[@]:-}")
|
||||
if [ "${#remaining[@]}" -gt 0 ]; then
|
||||
log "still running: ${remaining[*]} (${#remaining[@]} hosts)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Final summary
|
||||
log "============================================================"
|
||||
log "DEPLOY SUMMARY"
|
||||
log "============================================================"
|
||||
log "Passed (${#summary_pass[@]}):"
|
||||
for p in "${summary_pass[@]:-}"; do [ -n "$p" ] && log " ✓ $p"; done
|
||||
log "Failed (${#summary_fail[@]}):"
|
||||
for f in "${summary_fail[@]:-}"; do [ -n "$f" ] && log " ✗ $f"; done
|
||||
log ""
|
||||
log "Contents of $LOG_DIR:"
|
||||
ls -la "$LOG_DIR"
|
||||
|
||||
# Clean up marker files
|
||||
rm -f "$LOG_DIR"/_marker.* 2>/dev/null
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
# deploy-tuning.sh - copies apply-tunings.sh to target hosts and runs it.
|
||||
# Usage: bash deploy-tuning.sh [--no-nfs] [--apply] <host> [host...]
|
||||
# Default mode is dry-run. Pass --apply to commit. Pass --no-nfs to skip NFS section.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT="/home/reachableceo/projects/perfopt/scripts/apply-tunings.sh"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
MODE=""
|
||||
EXTRA_FLAGS=""
|
||||
|
||||
HOSTS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--apply) MODE="--apply" ;;
|
||||
--dry-run) MODE="" ;;
|
||||
--no-nfs) EXTRA_FLAGS="--no-nfs" ;;
|
||||
*) HOSTS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${#HOSTS[@]}" -eq 0 ]; then
|
||||
echo "Usage: $0 <host> [host...] [--apply]"
|
||||
echo "Default: dry-run. Pass --apply to commit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -r "$SCRIPT" ]; then
|
||||
echo "FATAL: $SCRIPT not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$host] deploying apply-tunings.sh (mode: ${MODE:-dry-run})"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "[$host] SKIP: unreachable"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$host] uploading..."
|
||||
if ! scp "${SSH_OPTS[@]}" "$SCRIPT" "root@$host:/root/apply-tunings.sh" >/dev/null 2>&1; then
|
||||
echo "[$host] SKIP: scp failed"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$host] running (output below)..."
|
||||
echo "----------------------------------------------------------------"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" "chmod +x /root/apply-tunings.sh && bash /root/apply-tunings.sh $MODE $EXTRA_FLAGS" 2>&1
|
||||
rc=$?
|
||||
echo "----------------------------------------------------------------"
|
||||
echo "[$host] exit code: $rc"
|
||||
echo ""
|
||||
done
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# diag.sh - diagnostic commands run on a host via SSH wrapper.
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
echo "===== 1. storage.cfg NFS stanzas (exact content) ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'grep -A 8 "^nfs: D2" /etc/pve/storage.cfg'
|
||||
|
||||
echo ""
|
||||
echo "===== 2. Try manual NFS mount with nconnect=4 ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'mount -t nfs -o nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,proto=tcp pfv-tsys4-nfs-stor:/mnt/tsys4/D2 /mnt/pve/D2 2>&1; echo "exit=$?"'
|
||||
|
||||
echo ""
|
||||
echo "===== 3. Try manual NFS mount WITHOUT nconnect ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'mount -t nfs -o noatime,rsize=1048576,wsize=1048576,hard,proto=tcp pfv-tsys4-nfs-stor:/mnt/tsys4/D2 /mnt/pve/D2 2>&1; echo "exit=$?"'
|
||||
|
||||
echo ""
|
||||
echo "===== 4. NFS kernel version / module ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/fs/nfsfs/version 2>/dev/null; echo "---"; modinfo nfs 2>/dev/null | grep -E "^(filename|version|description)" | head -5'
|
||||
|
||||
echo ""
|
||||
echo "===== 5. mount.nfs version ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'mount.nfs --version 2>&1; echo "---"; dpkg -l nfs-common 2>/dev/null | tail -2'
|
||||
|
||||
echo ""
|
||||
echo "===== 6. /etc/network/interfaces bond0 stanza (exact bytes) ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sed -n "/^auto bond0/,/^$/p" /etc/network/interfaces | cat -A'
|
||||
|
||||
echo ""
|
||||
echo "===== 7. Current bond0 running hash policy ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 | head -5'
|
||||
|
||||
echo ""
|
||||
echo "===== 8. xmit_hash_policy sysfs file ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; echo "---"; ls /sys/class/net/bond0/bonding/ 2>/dev/null'
|
||||
Executable
+128
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
# finish-host.sh - applies ALL remaining changes to a host and verifies.
|
||||
#
|
||||
# Steps:
|
||||
# 1. Start all VMs (triggers NFS lazy-mount)
|
||||
# 2. Wait for NFS mounts to appear
|
||||
# 3. Verify NFS nconnect=4 + noatime
|
||||
# 4. Apply bond0 xmit_hash_policy=layer3+4
|
||||
# 5. Full end-to-end verification
|
||||
#
|
||||
# Usage: bash finish-host.sh <host> [--apply]
|
||||
# Default is dry-run (starts VMs + shows what bond change would do, but doesn't edit interfaces)
|
||||
set -uo pipefail
|
||||
|
||||
HOST="${1:-}"
|
||||
MODE="${2:-dryrun}"
|
||||
[ "$MODE" = "--apply" ] && MODE="apply" || MODE="dryrun"
|
||||
|
||||
if [ -z "$HOST" ]; then
|
||||
echo "Usage: $0 <host> [--apply]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
SCRIPT_DIR="/home/reachableceo/projects/perfopt/scripts"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " finish-host.sh — $HOST (mode: $MODE)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# =========================================================================
|
||||
# STEP 1: Start all VMs
|
||||
# =========================================================================
|
||||
echo "=== STEP 1: Start all VMs on $HOST ==="
|
||||
# Get list of all VMs (not just stopped — start is idempotent)
|
||||
vm_list=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1{print \$1}"')
|
||||
for vmid in $vm_list; do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" != "running" ]; then
|
||||
echo " Starting VM $vmid..."
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1 | sed 's/^/ /'
|
||||
else
|
||||
echo " VM $vmid already running"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Waiting 15s for VMs to boot and trigger NFS mounts..."
|
||||
sleep 15
|
||||
|
||||
echo ""
|
||||
echo "--- VM status after start ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||||
|
||||
# =========================================================================
|
||||
# STEP 2: Verify NFS mounts came back with nconnect=4
|
||||
# =========================================================================
|
||||
echo ""
|
||||
echo "=== STEP 2: Verify NFS mounts with nconnect=4 ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null' | head -30
|
||||
|
||||
echo ""
|
||||
echo "--- NFS TCP connections to :2049 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
conn_count=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l')
|
||||
echo " Total NFS TCP connections: $conn_count"
|
||||
|
||||
# =========================================================================
|
||||
# STEP 3: Apply bond0 hash policy
|
||||
# =========================================================================
|
||||
echo ""
|
||||
echo "=== STEP 3: Apply bond0 xmit_hash_policy=layer3+4 (mode: $MODE) ==="
|
||||
|
||||
# Upload the bond hash script
|
||||
scp "${SSH_OPTS[@]}" "$SCRIPT_DIR/apply-bond-hash.sh" "root@$HOST:/root/apply-bond-hash.sh" >/dev/null 2>&1
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/apply-bond-hash.sh && bash /root/apply-bond-hash.sh --apply' 2>&1
|
||||
else
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/apply-bond-hash.sh && bash /root/apply-bond-hash.sh' 2>&1
|
||||
fi
|
||||
|
||||
# =========================================================================
|
||||
# STEP 4: Full verification
|
||||
# =========================================================================
|
||||
echo ""
|
||||
echo "=== STEP 4: Full end-to-end verification ==="
|
||||
echo ""
|
||||
echo "--- Uptime ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime'
|
||||
echo ""
|
||||
echo "--- CPU governor ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"'
|
||||
echo ""
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness'
|
||||
echo ""
|
||||
echo "--- TCP BBR ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc'
|
||||
echo ""
|
||||
echo "--- tuned profile ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'tuned-adm active 2>/dev/null'
|
||||
echo ""
|
||||
echo "--- bond0 hash policy + LACP state ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 2>/dev/null | head -25'
|
||||
echo ""
|
||||
echo "--- NFS mount options (first 3 mounts) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -24'
|
||||
echo ""
|
||||
echo "--- NFS TCP connections (expect 4 per server × 2 servers = 8) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
nfs_conns=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l')
|
||||
echo " Count: $nfs_conns"
|
||||
echo ""
|
||||
echo "--- VMs running ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||||
echo ""
|
||||
echo "--- Failed services ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10'
|
||||
echo ""
|
||||
echo "--- Network interfaces (speed/duplex/mtu) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'for ifc in bond0 nic0 nic1 nic2 vmbr0 datanet; do [ -d "/sys/class/net/$ifc" ] && printf "%-12s speed=%-8s duplex=%-8s mtu=%s\n" "$ifc" "$(cat /sys/class/net/$ifc/speed 2>/dev/null)" "$(cat /sys/class/net/$ifc/duplex 2>/dev/null)" "$(cat /sys/class/net/$ifc/mtu 2>/dev/null)"; done'
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " COMPLETE — $HOST"
|
||||
echo "==================================================================="
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# install-utils-v2.sh - retry install without nstat package.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
PKGS="sysstat jq numactl nvme-cli mtr-tiny dnsutils bmon"
|
||||
|
||||
for host in pfv-tsys6 pfv-tsys7; do
|
||||
echo "=== [$host] installing: $PKGS ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>&1 | tail -2 && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y $PKGS 2>&1 | tail -10"
|
||||
|
||||
# sysstat enable (path varies by Debian version)
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" '
|
||||
if [ -r /etc/default/sysstat ]; then
|
||||
sed -i "s/^ENABLED=.*/ENABLED=\"true\"/" /etc/default/sysstat
|
||||
systemctl enable --now sysstat 2>/dev/null
|
||||
grep ENABLED /etc/default/sysstat
|
||||
else
|
||||
# Newer Debian (trixie) — sysstat cron/service auto-enabled
|
||||
systemctl enable --now sysstat 2>/dev/null || echo "(sysstat auto via cron)"
|
||||
fi
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Verify
|
||||
for host in pfv-tsys6 pfv-tsys7; do
|
||||
echo "=== [$host] verification ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" '
|
||||
for p in sysstat jq numactl nvme mtr-tiny dig bmon; do
|
||||
command -v "$p" >/dev/null 2>&1 && echo " ✓ $p" || echo " ✗ $p"
|
||||
done
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
# install-utils.sh - installs useful observability packages on a host.
|
||||
# These are all small, dependency-light, and read-only at runtime.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
HOSTS=(pfv-tsys6 pfv-tsys7)
|
||||
|
||||
# Packages to install, with rationale
|
||||
PKGS=(
|
||||
sysstat # sar, iostat, mpstat, pidstat - the missing observability suite
|
||||
jq # JSON parsing for pvesh/scripts
|
||||
numactl # NUMA topology/controls for the dual-socket hosts
|
||||
nvme-cli # NVMe health (for when NVMe shows up)
|
||||
tcpdump # packet capture for network debugging
|
||||
mtr-tiny # traceroute on steroids
|
||||
nstat # kernel SNMP stats (already partly there)
|
||||
dnsutils # dig, nslookup, host
|
||||
bmon # bandwidth monitor ( curses, real-time)
|
||||
)
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$host] installing observability packages"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check which are missing
|
||||
missing=""
|
||||
for pkg in "${PKGS[@]}"; do
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" "dpkg -s $pkg 2>/dev/null | grep -q 'Status: install ok installed'" 2>/dev/null; then
|
||||
missing="$missing $pkg"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$missing" ]; then
|
||||
echo " All packages already installed."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo " Installing:$missing"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq$missing 2>&1 | tail -5"
|
||||
echo " Done."
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Enable sysstat data collection (sar) — off by default on Debian
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "[$host] enabling sysstat/sar data collection..."
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"sed -i 's/^ENABLED=\"false\"/ENABLED=\"true\"/' /etc/default/sysstat 2>/dev/null; \
|
||||
systemctl enable --now sysstat 2>&1 | tail -2; \
|
||||
grep ENABLED /etc/default/sysstat"
|
||||
done
|
||||
Executable
+293
@@ -0,0 +1,293 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# iperf-full-matrix.sh
|
||||
#
|
||||
# Two test suites:
|
||||
# A. Management network (vmbr0 / VLAN1): all-pairs single-stream TCP, 10s
|
||||
# B. Storage network (VLAN1000): tsys6+tsys7 → tsys4+tsys5, stress test
|
||||
#
|
||||
# Output: returned-logs/iperf/
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||||
|
||||
# Storage IPs (known, static on VLAN1000)
|
||||
declare -A SIP
|
||||
SIP[pfv-tsys1]="10.100.100.1"
|
||||
SIP[pfv-tsys3]="10.100.100.3"
|
||||
SIP[pfv-tsys4]="10.100.100.4"
|
||||
SIP[pfv-tsys5]="10.100.100.5"
|
||||
SIP[pfv-tsys6]="10.100.100.6"
|
||||
SIP[pfv-tsys7]="10.100.100.7"
|
||||
|
||||
# ===========================================================================
|
||||
# STEP 0: Discover management IPs (vmbr0)
|
||||
# ===========================================================================
|
||||
echo "==================================================================="
|
||||
echo " STEP 0: Discover management network IPs (vmbr0)"
|
||||
echo "==================================================================="
|
||||
declare -A MIP
|
||||
for host in "${HOSTS[@]}"; do
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " [$host] UNREACHABLE"
|
||||
continue
|
||||
fi
|
||||
mip=$(ssh "${SSH_OPTS[@]}" "root@$host" 'ip -o -4 addr show dev vmbr0 2>/dev/null | awk "{print \$4}" | cut -d/ -f1 | head -1')
|
||||
if [ -n "$mip" ]; then
|
||||
MIP[$host]="$mip"
|
||||
echo " [$host] vmbr0 = $mip"
|
||||
else
|
||||
echo " [$host] no vmbr0 IPv4 — skipping"
|
||||
fi
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# STEP 1: Ensure iperf3 installed on all hosts
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STEP 1: Ensure iperf3 installed"
|
||||
echo "==================================================================="
|
||||
for host in "${HOSTS[@]}"; do
|
||||
[ -z "${MIP[$host]:-}" ] && continue
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'command -v iperf3 >/dev/null 2>&1' 2>/dev/null; then
|
||||
echo -n " [$host] installing iperf3... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
'DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq iperf3 >/dev/null 2>&1 && echo OK || echo FAILED'
|
||||
else
|
||||
echo " [$host] iperf3 already installed"
|
||||
fi
|
||||
done
|
||||
|
||||
# Helper: start iperf3 server in one-shot mode bound to a specific IP
|
||||
start_server() {
|
||||
local host="$1" ip="$2"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${ip} >/dev/null 2>&1 &" 2>/dev/null
|
||||
sleep 1
|
||||
}
|
||||
|
||||
# Helper: run iperf3 client test, save output, extract result
|
||||
run_test() {
|
||||
local client="$1" server="$2" sip="$3" cip="$4" label="$5" logfile="$6"
|
||||
shift 6
|
||||
local extra="$*"
|
||||
|
||||
echo -n " [$label] ... "
|
||||
{
|
||||
echo "=== iperf3: $label ==="
|
||||
echo "Client: $client ($cip) → Server: $server ($sip)"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Args: $extra"
|
||||
echo ""
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${sip} -B ${cip} ${extra}" 2>&1
|
||||
echo ""
|
||||
echo "=== END ==="
|
||||
} > "$logfile" 2>&1
|
||||
|
||||
# Extract result
|
||||
sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$logfile" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
echo "${bitrate:-?} (retrans: ${retrans:-?})"
|
||||
}
|
||||
|
||||
# ===========================================================================
|
||||
# SUITE A: Management network (vmbr0) — all pairs, single-stream TCP, 10s
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SUITE A: MANAGEMENT NETWORK (vmbr0) — all-pairs, 10s TCP"
|
||||
echo " Expectation: ~940 Mbps for every pair (1 GbE line rate)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# Test each unique pair (i < j to avoid duplicates)
|
||||
for ((i=0; i<${#HOSTS[@]}; i++)); do
|
||||
for ((j=i+1; j<${#HOSTS[@]}; j++)); do
|
||||
client="${HOSTS[$i]}"
|
||||
server="${HOSTS[$j]}"
|
||||
[ -z "${MIP[$client]:-}" ] && continue
|
||||
[ -z "${MIP[$server]:-}" ] && continue
|
||||
|
||||
label="mgmt: ${client}→${server}"
|
||||
logfile="$LOG_DIR/mgmt-${client}-to-${server}.log"
|
||||
|
||||
start_server "$server" "${MIP[$server]}"
|
||||
run_test "$client" "$server" "${MIP[$server]}" "${MIP[$client]}" \
|
||||
"$label" "$logfile" "-t 10 -P 1"
|
||||
done
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# SUITE B: Storage network (VLAN1000) — stress test the cross-rack LACP
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SUITE B: STORAGE NETWORK (VLAN1000) — stress test cross-rack link"
|
||||
echo " tsys6 + tsys7 (Rack 3) → tsys4 + tsys5 (Rack 5)"
|
||||
echo " Expectation: limited by tsys4 USB dongle + tsys5 broken bond"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# --- B.1: Individual tests (one client → one server at a time) ---
|
||||
echo "--- B.1: Individual tests (sequential) ---"
|
||||
echo ""
|
||||
|
||||
for client in pfv-tsys6 pfv-tsys7; do
|
||||
for server in pfv-tsys4 pfv-tsys5; do
|
||||
label="stor: ${client}→${server} (8-stream)"
|
||||
logfile="$LOG_DIR/stor-indiv-${client}-to-${server}-8stream.log"
|
||||
start_server "$server" "${SIP[$server]}"
|
||||
run_test "$client" "$server" "${SIP[$server]}" "${SIP[$client]}" \
|
||||
"$label" "$logfile" "-P 8 -t 20 -l 128k -O 2"
|
||||
done
|
||||
done
|
||||
|
||||
# --- B.2: Reverse direction (tsys4/5 → tsys6/7) ---
|
||||
echo ""
|
||||
echo "--- B.2: Reverse direction (tsys4/5 → tsys6/7) ---"
|
||||
echo ""
|
||||
|
||||
for client in pfv-tsys4 pfv-tsys5; do
|
||||
for server in pfv-tsys6 pfv-tsys7; do
|
||||
label="stor: ${client}→${server} (8-stream rev)"
|
||||
logfile="$LOG_DIR/stor-indiv-${client}-to-${server}-8stream.log"
|
||||
start_server "$server" "${SIP[$server]}"
|
||||
run_test "$client" "$server" "${SIP[$server]}" "${SIP[$client]}" \
|
||||
"$label" "$logfile" "-P 8 -t 20 -l 128k -O 2"
|
||||
done
|
||||
done
|
||||
|
||||
# --- B.3: Simultaneous stress test (4 flows at once) ---
|
||||
echo ""
|
||||
echo "--- B.3: Simultaneous 4-flow stress test ---"
|
||||
echo " tsys6→tsys4 + tsys6→tsys5 + tsys7→tsys4 + tsys7→tsys5"
|
||||
echo " All running in parallel for 30 seconds"
|
||||
echo ""
|
||||
|
||||
# Start 4 iperf3 servers (one-shot mode won't work for parallel; use persistent)
|
||||
for server in pfv-tsys4 pfv-tsys5; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -B ${SIP[$server]} >/dev/null 2>&1 &" 2>/dev/null
|
||||
echo " [server started: $server]"
|
||||
done
|
||||
sleep 1
|
||||
|
||||
STRESS_LOG="$LOG_DIR/stor-stress-4flow"
|
||||
mkdir -p "$STRESS_LOG"
|
||||
|
||||
# Launch 4 clients in parallel, each writing to its own log
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" "iperf3 -c ${SIP[pfv-tsys4]} -B ${SIP[pfv-tsys6]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys6-to-tsys4.log" 2>&1 &
|
||||
PID1=$!
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" "iperf3 -c ${SIP[pfv-tsys5]} -B ${SIP[pfv-tsys6]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys6-to-tsys5.log" 2>&1 &
|
||||
PID2=$!
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" "iperf3 -c ${SIP[pfv-tsys4]} -B ${SIP[pfv-tsys7]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys7-to-tsys4.log" 2>&1 &
|
||||
PID3=$!
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" "iperf3 -c ${SIP[pfv-tsys5]} -B ${SIP[pfv-tsys7]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys7-to-tsys5.log" 2>&1 &
|
||||
PID4=$!
|
||||
|
||||
echo " [4 clients launched, waiting 40s for completion...]"
|
||||
wait $PID1 $PID2 $PID3 $PID4 2>/dev/null
|
||||
echo " [all 4 flows complete]"
|
||||
|
||||
# Kill servers
|
||||
for server in pfv-tsys4 pfv-tsys5; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# SUITE C: All hosts cleanup
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Cleanup: killing iperf3 everywhere"
|
||||
echo "==================================================================="
|
||||
for host in "${HOSTS[@]}"; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# RESULTS SUMMARY
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " RESULTS SUMMARY"
|
||||
echo "==================================================================="
|
||||
|
||||
echo ""
|
||||
echo "===== SUITE A: Management network (vmbr0) ====="
|
||||
echo ""
|
||||
printf "%-40s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||||
printf "%-40s %15s %10s\n" "----" "----------" "-------"
|
||||
for f in "$LOG_DIR"/mgmt-*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
single=$(grep 'sender$' "$f" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
printf "%-40s %15s %10s\n" "$label" "${bitrate:-?}" "${retrans:--}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "===== SUITE B.1+B.2: Storage network individual ====="
|
||||
echo ""
|
||||
printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||||
printf "%-45s %15s %10s\n" "----" "----------" "-------"
|
||||
for f in "$LOG_DIR"/stor-indiv-*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$f" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
printf "%-45s %15s %10s\n" "$label" "${bitrate:-?}" "${retrans:--}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "===== SUITE B.3: Simultaneous 4-flow stress test ====="
|
||||
echo ""
|
||||
printf "%-30s %15s %10s\n" "FLOW" "THROUGHPUT" "RETRANS"
|
||||
printf "%-30s %15s %10s\n" "----" "----------" "-------"
|
||||
total_mbps=0
|
||||
for f in "$STRESS_LOG"/*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
flow=$(basename "$f" .log)
|
||||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
mbps=$(echo "$sum" | awk '{print $6}')
|
||||
total_mbps=$(awk "BEGIN{print $total_mbps + $mbps}")
|
||||
else
|
||||
bitrate="?"
|
||||
retrans="-"
|
||||
fi
|
||||
printf "%-30s %15s %10s\n" "$flow" "$bitrate" "${retrans:--}"
|
||||
done
|
||||
printf "%-30s %15s\n" "AGGREGATE (all 4 flows)" "${total_mbps} Mbits/sec"
|
||||
|
||||
echo ""
|
||||
echo "===== CONTEXT ====="
|
||||
echo "tsys4: USB cdc_ncm dongle (single 1G link, no bond)"
|
||||
echo "tsys5: bond0 broken (1 active slave, no LACP partner) — cable pending"
|
||||
echo "tsys6/7: working 2x1G LACP, layer3+4 hash (host side)"
|
||||
echo "Cross-rack: 4x1G LACP (pfv-r3-tor-stor → pfv-core-sw01)"
|
||||
echo ""
|
||||
echo "All logs in: $LOG_DIR/"
|
||||
Executable
+184
@@ -0,0 +1,184 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# iperf-storage-tests.sh
|
||||
#
|
||||
# Installs iperf3 on all online hosts, then runs a matrix of storage-network
|
||||
# throughput tests. Saves all output to returned-logs/iperf/.
|
||||
#
|
||||
# Test matrix (all over VLAN1000 storage network, 10.100.100.0/24):
|
||||
# 1. tsys7 → tsys4 (USB cdc_ncm NIC) — the smoking gun
|
||||
# 2. tsys7 → tsys5 (bond0, 1 active slave) — PCI NIC comparison
|
||||
# 3. tsys7 → tsys6 (bond0, 2 active slaves) — working LACP baseline
|
||||
# 4. Reverse: tsys4 → tsys7 (USB NIC TX direction)
|
||||
# 5. Reverse: tsys5 → tsys7
|
||||
#
|
||||
# Each test: TCP 8-stream 30s forward + reverse + UDP saturation.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
ALL_HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||||
|
||||
# Storage network IPs
|
||||
declare -A SIP
|
||||
SIP[pfv-tsys1]="10.100.100.1"
|
||||
SIP[pfv-tsys3]="10.100.100.3"
|
||||
SIP[pfv-tsys4]="10.100.100.4"
|
||||
SIP[pfv-tsys5]="10.100.100.5"
|
||||
SIP[pfv-tsys6]="10.100.100.6"
|
||||
SIP[pfv-tsys7]="10.100.100.7"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " STEP 1: Install iperf3 on all online hosts"
|
||||
echo "==================================================================="
|
||||
|
||||
for host in "${ALL_HOSTS[@]}"; do
|
||||
echo -n " [$host] "
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "UNREACHABLE — skipping"
|
||||
continue
|
||||
fi
|
||||
# Check if iperf3 already installed
|
||||
if ssh "${SSH_OPTS[@]}" "root@$host" 'command -v iperf3 >/dev/null 2>&1' 2>/dev/null; then
|
||||
echo "iperf3 already installed"
|
||||
else
|
||||
printf "installing... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq iperf3 >/dev/null 2>&1 && echo OK || echo FAILED'
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STEP 2: Kill any existing iperf3 processes everywhere"
|
||||
echo "==================================================================="
|
||||
for host in "${ALL_HOSTS[@]}"; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo " Done."
|
||||
|
||||
# Helper: run an iperf3 test and save output
|
||||
run_iperf() {
|
||||
local client="$1" server="$2" direction="$3" label="$4" logfile="$5"
|
||||
local client_ip="${SIP[$client]}" server_ip="${SIP[$server]}"
|
||||
|
||||
echo -n " [$client → $server] $label ... "
|
||||
|
||||
# Start server in one-shot mode (-1 means serve one client then exit)
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
# Run client
|
||||
{
|
||||
echo "=== iperf3: $label ==="
|
||||
echo "Client: $client ($client_ip)"
|
||||
echo "Server: $server ($server_ip)"
|
||||
echo "Direction: $direction"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo ""
|
||||
if [ "$direction" = "forward" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2" 2>&1
|
||||
elif [ "$direction" = "reverse" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2 -R" 2>&1
|
||||
elif [ "$direction" = "udp" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -u -b 2G -t 10 -l 8972" 2>&1
|
||||
elif [ "$direction" = "single" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -t 20 -O 2" 2>&1
|
||||
fi
|
||||
echo ""
|
||||
echo "=== END ==="
|
||||
} > "$logfile" 2>&1
|
||||
|
||||
# Extract summary line
|
||||
if grep -q "sender" "$logfile"; then
|
||||
bitrate=$(grep "sender" "$logfile" | tail -1 | awk '{print $7, $8}')
|
||||
echo "done: ${bitrate}"
|
||||
else
|
||||
echo "done (check log for details)"
|
||||
fi
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STEP 3: Run iperf3 test matrix"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
echo "All tests over VLAN1000 storage network (10.100.100.0/24)."
|
||||
echo "TCP tests: 8 parallel streams, 30s, 128k blocks."
|
||||
echo ""
|
||||
|
||||
# --- Test 1: tsys7 → tsys4 (USB cdc_ncm target) ---
|
||||
echo "--- TEST 1: tsys7 → tsys4 (USB cdc_ncm NIC) ---"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 forward "TCP 8-stream forward (tsys7→tsys4 USB)" \
|
||||
"$LOG_DIR/01-tsys7-to-tsys4-tcp-forward.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys7)" \
|
||||
"$LOG_DIR/02-tsys7-to-tsys4-tcp-reverse.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 single "TCP single-stream forward (tsys7→tsys4 USB)" \
|
||||
"$LOG_DIR/03-tsys7-to-tsys4-tcp-single.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 udp "UDP saturation (tsys7→tsys4 USB)" \
|
||||
"$LOG_DIR/04-tsys7-to-tsys4-udp.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Test 2: tsys7 → tsys5 (bond0, PCI NIC, 1 active slave) ---
|
||||
echo "--- TEST 2: tsys7 → tsys5 (PCI NIC, broken bond - 1 slave) ---"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 forward "TCP 8-stream forward (tsys7→tsys5 PCI)" \
|
||||
"$LOG_DIR/05-tsys7-to-tsys5-tcp-forward.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 reverse "TCP 8-stream reverse (tsys5 PCI→tsys7)" \
|
||||
"$LOG_DIR/06-tsys7-to-tsys5-tcp-reverse.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 single "TCP single-stream forward (tsys7→tsys5 PCI)" \
|
||||
"$LOG_DIR/07-tsys7-to-tsys5-tcp-single.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 udp "UDP saturation (tsys7→tsys5 PCI)" \
|
||||
"$LOG_DIR/08-tsys7-to-tsys5-udp.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Test 3: tsys7 → tsys6 (working 2-slave LACP baseline, layer3+4) ---
|
||||
echo "--- TEST 3: tsys7 → tsys6 (working 2×1G LACP baseline) ---"
|
||||
run_iperf pfv-tsys7 pfv-tsys6 forward "TCP 8-stream forward (tsys7→tsys6 LACP)" \
|
||||
"$LOG_DIR/09-tsys7-to-tsys6-tcp-forward.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys6 reverse "TCP 8-stream reverse (tsys6 LACP→tsys7)" \
|
||||
"$LOG_DIR/10-tsys7-to-tsys6-tcp-reverse.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Test 4: tsys6 → tsys4 (pre-tuning baseline) ---
|
||||
echo "--- TEST 4: tsys6 → tsys4 (baseline before tsys6 tuning) ---"
|
||||
run_iperf pfv-tsys6 pfv-tsys4 forward "TCP 8-stream forward (tsys6→tsys4 USB)" \
|
||||
"$LOG_DIR/11-tsys6-to-tsys4-tcp-forward.log"
|
||||
run_iperf pfv-tsys6 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys6)" \
|
||||
"$LOG_DIR/12-tsys6-to-tsys4-tcp-reverse.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Cleanup: kill iperf3 everywhere ---
|
||||
echo "--- Cleanup ---"
|
||||
for host in "${ALL_HOSTS[@]}"; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " RESULTS SUMMARY"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
printf "%-45s %s\n" "TEST" "THROUGHPUT"
|
||||
printf "%-45s %s\n" "----" "----------"
|
||||
for f in "$LOG_DIR"/*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
bitrate=$(grep -E "sender$" "$f" | tail -1 | awk '{print $7, $8}')
|
||||
[ -z "$bitrate" ] && bitrate=$(grep -E "Mbits/sec|Gbits/sec" "$f" | tail -1 | grep -oE '[0-9.]+ [MG]bits/sec' | head -1)
|
||||
[ -z "$bitrate" ] && bitrate="(see log)"
|
||||
printf "%-45s %s\n" "$label" "$bitrate"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Full logs saved to: $LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/bin/bash
|
||||
# iperf-tsys6-tsys7.sh - validate 2Gbps LACP between the two tuned hosts.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
TSYS6="10.100.100.6"
|
||||
TSYS7="10.100.100.7"
|
||||
|
||||
run_test() {
|
||||
local client="$1" server="$2" server_ip="$3" label="$4" logfile="$5"
|
||||
shift 4
|
||||
local extra_args="$*"
|
||||
|
||||
echo -n " [$label] ... "
|
||||
|
||||
# Start server in one-shot mode
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
{
|
||||
echo "=== iperf3: $label ==="
|
||||
echo "Client: $client Server: $server ($server_ip)"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Args: $extra_args"
|
||||
echo ""
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${server_ip} $extra_args" 2>&1
|
||||
echo ""
|
||||
echo "=== END ==="
|
||||
} > "$logfile" 2>&1
|
||||
|
||||
# Extract result
|
||||
sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$logfile" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
echo "${bitrate} (retrans: ${retrans:-0})"
|
||||
}
|
||||
|
||||
echo "==================================================================="
|
||||
echo " iperf3: tsys6 ↔ tsys7 (both have 2×1G LACP + layer3+4 hash)"
|
||||
echo " Expectation: ~1.8-2.0 Gbps for 8-stream TCP"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# Pre-flight: confirm bond state on both
|
||||
echo "--- bond0 state on tsys6 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||||
echo ""
|
||||
echo "--- bond0 state on tsys7 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||||
echo ""
|
||||
|
||||
echo "--- Running tests ---"
|
||||
echo ""
|
||||
|
||||
# Test 1: tsys7 → tsys6, 8-stream TCP forward
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 TCP 8-stream forward" \
|
||||
"$LOG_DIR/tsys6-tsys7-01-tcp-8stream-forward.log" \
|
||||
"-P 8 -t 30 -l 128k -O 2"
|
||||
|
||||
# Test 2: tsys6 → tsys7, 8-stream TCP forward (reverse direction)
|
||||
run_test pfv-tsys6 pfv-tsys7 "$TSYS7" \
|
||||
"tsys6→tsys7 TCP 8-stream forward" \
|
||||
"$LOG_DIR/tsys6-tsys7-02-tcp-8stream-forward.log" \
|
||||
"-P 8 -t 30 -l 128k -O 2"
|
||||
|
||||
# Test 3: tsys7 → tsys6, single stream (should be ~940 Mbps — single flow)
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 TCP single-stream" \
|
||||
"$LOG_DIR/tsys6-tsys7-03-tcp-single.log" \
|
||||
"-t 20 -O 2"
|
||||
|
||||
# Test 4: tsys7 → tsys6, 4-stream (nconnect=4 mirrors this)
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 TCP 4-stream" \
|
||||
"$LOG_DIR/tsys6-tsys7-04-tcp-4stream.log" \
|
||||
"-P 4 -t 30 -l 128k -O 2"
|
||||
|
||||
# Test 5: UDP saturation
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 UDP saturation" \
|
||||
"$LOG_DIR/tsys6-tsys7-05-udp.log" \
|
||||
"-u -b 3G -t 10 -l 8972"
|
||||
|
||||
# Cleanup
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SUMMARY"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||||
printf "%-45s %15s %10s\n" "----" "----------" "-------"
|
||||
for f in "$LOG_DIR"/tsys6-tsys7-*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$f" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
printf "%-45s %15s %10s\n" "$label" "$bitrate" "${retrans:--}"
|
||||
done
|
||||
echo ""
|
||||
echo "Expected: 8-stream ~1.8-2.0 Gbps, single-stream ~940 Mbps"
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
# reboot-and-verify.sh - reboots a host and verifies NFS nconnect activates.
|
||||
# Usage: bash reboot-and-verify.sh <host>
|
||||
set -uo pipefail
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
echo "================================================================"
|
||||
echo "[$HOST] PRE-REBOOT STATE"
|
||||
echo "================================================================"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
echo "--- VMs ---"
|
||||
qm list 2>/dev/null
|
||||
echo "--- NFS TCP conns: $(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l) ---"
|
||||
echo "--- NFS first mount: ---"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1
|
||||
echo "--- uptime ---"
|
||||
uptime
|
||||
'
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] ISSUING REBOOT"
|
||||
echo "================================================================"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nohup sh -c "(sleep 2; systemctl reboot)" >/dev/null 2>&1 &'
|
||||
echo "Reboot sent at $(date +%H:%M:%S)"
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] WAITING FOR SSH TO RETURN (max 10 min)"
|
||||
echo "================================================================"
|
||||
DEADLINE=$(( $(date +%s) + 600 ))
|
||||
LAST_PRINT=0
|
||||
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
||||
now=$(date +%s)
|
||||
if [ $((now - LAST_PRINT)) -ge 15 ]; then
|
||||
printf ' [%s] waiting... (%ss elapsed)\n' "$(date +%H:%M:%S)" "$(( now - DEADLINE + 600 ))"
|
||||
LAST_PRINT=$now
|
||||
fi
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
# Verify uptime is actually low (host really rebooted, not still up)
|
||||
up_mins=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/uptime | awk "{print int(\$1/60)}"')
|
||||
if [ "${up_mins:-999}" -lt 5 ]; then
|
||||
echo " [$(date +%H:%M:%S)] SSH back, uptime ${up_mins}min — real reboot confirmed"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "FAILED: $HOST not back after 10 minutes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting 30s for services to settle..."
|
||||
sleep 30
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] POST-REBOOT VERIFICATION"
|
||||
echo "================================================================"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
echo "--- uptime ---"
|
||||
uptime
|
||||
echo ""
|
||||
echo "--- governor: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) ---"
|
||||
echo "--- swappiness: $(sysctl -n vm.swappiness) ---"
|
||||
echo "--- tcp_cc: $(sysctl -n net.ipv4.tcp_congestion_control) ---"
|
||||
echo "--- tuned: $(tuned-adm active 2>/dev/null | grep Current) ---"
|
||||
echo ""
|
||||
echo "--- NFS first mount: ---"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1
|
||||
echo ""
|
||||
echo "--- NFS TCP conns (expect 8 with nconnect=4): ---"
|
||||
ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l
|
||||
echo ""
|
||||
echo "--- VMs: ---"
|
||||
qm list 2>/dev/null
|
||||
echo ""
|
||||
echo "--- Failed services: ---"
|
||||
systemctl --failed --no-legend 2>/dev/null | head -5
|
||||
echo "(empty = none)"
|
||||
'
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] DONE"
|
||||
echo "================================================================"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
# reboot-verify.sh - reboots a host, waits for it to come back, verifies state.
|
||||
# Usage: bash reboot-verify.sh <host>
|
||||
set -uo pipefail
|
||||
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
echo "================================================================"
|
||||
echo "[$HOST] PRE-REBOOT STATE"
|
||||
echo "================================================================"
|
||||
echo "--- Running VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR==1 || \$3==\"running\"{print}"' 2>&1
|
||||
echo "--- NFS mount count ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | grep -c "^/mnt"' 2>&1
|
||||
echo "--- NFS TCP connections to :2049 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l' 2>&1
|
||||
echo "--- Uptime ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime' 2>&1
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] ISSUING REBOOT"
|
||||
echo "================================================================"
|
||||
# Issue reboot; ssh will disconnect with non-zero — that's expected.
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nohup sh -c "(sleep 2; systemctl reboot)" >/dev/null 2>&1 &' 2>&1
|
||||
echo "Reboot command sent at $(date +%H:%M:%S). Host will drop now."
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] WAITING FOR SSH TO RETURN (max 10 minutes)"
|
||||
echo "================================================================"
|
||||
DEADLINE=$(( $(date +%s) + 600 ))
|
||||
LAST_PRINT=0
|
||||
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
||||
now=$(date +%s)
|
||||
# Print a heartbeat every 15s
|
||||
if [ $((now - LAST_PRINT)) -ge 15 ]; then
|
||||
elapsed=$((DEADLINE - now - 600)); elapsed=${elapsed#-}
|
||||
echo " [$(date +%H:%M:%S)] still waiting... (${elapsed}s elapsed)"
|
||||
LAST_PRINT=$now
|
||||
fi
|
||||
# Try SSH
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " [$(date +%H:%M:%S)] SSH is back!"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Final check
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " [$(date +%H:%M:%S)] FAILED: host not reachable after 10 minutes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Give services a moment to settle after SSH returns
|
||||
echo " Waiting 20s for services to settle..."
|
||||
sleep 20
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] POST-REBOOT VERIFICATION"
|
||||
echo "================================================================"
|
||||
echo "--- Uptime ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime' 2>&1
|
||||
echo ""
|
||||
echo "--- TCP congestion control ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc' 2>&1
|
||||
echo ""
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness' 2>&1
|
||||
echo ""
|
||||
echo "--- scaling_governor ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"' 2>&1
|
||||
echo ""
|
||||
echo "--- NFS mount options (looking for nconnect + noatime) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -20' 2>&1
|
||||
echo ""
|
||||
echo "--- NFS TCP connection count (expect ~8 = 4 per server with nconnect=4) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l' 2>&1
|
||||
echo ""
|
||||
echo "--- Running VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null' 2>&1
|
||||
echo ""
|
||||
echo "--- Failed services? ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10' 2>&1
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] DONE"
|
||||
echo "================================================================"
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-bond-hash.sh
|
||||
#
|
||||
# Adds bond-xmit-hash-policy layer3+4 to bond0 in /etc/network/interfaces,
|
||||
# then reloads networking with ifreload -a.
|
||||
#
|
||||
# SSH survivability: this is safe IF your SSH session is on vmbr0/nic0
|
||||
# (management network), NOT on bond0/datanet (storage network).
|
||||
# tsys7's topology confirms this: SSH comes in on vmbr0 (nic0).
|
||||
#
|
||||
# Safety:
|
||||
# - Dry-run by default (--apply to commit)
|
||||
# - Full backup of /etc/network/interfaces
|
||||
# - Generates rollback script
|
||||
# - Does NOT reboot — uses ifreload -a which is hot-reload
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK="/root/perfopt-bond-rollback-${TS_SHORT}.sh"
|
||||
ACTION="${1:-dryrun}"
|
||||
|
||||
[ "$ACTION" = "--apply" ] && ACTION="apply" || ACTION="dryrun"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " apply-bond-hash — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo "==================================================================="
|
||||
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "FATAL: /etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if bond0 exists in the config
|
||||
if ! grep -q 'bond0' /etc/network/interfaces; then
|
||||
echo "No bond0 found in /etc/network/interfaces — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check if the hash policy is already set
|
||||
if grep -q 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already present:"
|
||||
grep 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces
|
||||
echo "Checking value..."
|
||||
if grep -q 'layer3+4' /etc/network/interfaces; then
|
||||
echo "Already set to layer3+4 — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show current bond state
|
||||
echo ""
|
||||
echo "--- Current bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -15 || echo "(bond0 not up or not present)"
|
||||
|
||||
# Back up
|
||||
cp -a /etc/network/interfaces "$BACKUP_DIR/interfaces"
|
||||
|
||||
# Generate rollback script
|
||||
cat > "$ROLLBACK" <<EOF
|
||||
#!/bin/bash
|
||||
# Rollback for bond-xmit-hash-policy change
|
||||
# Restores original /etc/network/interfaces and reloads
|
||||
set -euo pipefail
|
||||
cp -a "$BACKUP_DIR/interfaces" /etc/network/interfaces
|
||||
echo "Restored /etc/network/interfaces"
|
||||
echo "Reloading networking..."
|
||||
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
|
||||
echo "Done. bond0 hash policy reverted to original."
|
||||
EOF
|
||||
chmod +x "$ROLLBACK"
|
||||
|
||||
echo ""
|
||||
echo "--- Proposed change ---"
|
||||
echo "Add line ' bond-xmit-hash-policy layer3+4' to the bond0 stanza."
|
||||
echo ""
|
||||
|
||||
if [ "$ACTION" != "apply" ]; then
|
||||
echo "DRY RUN — no changes made."
|
||||
echo "To commit: bash $0 --apply"
|
||||
echo "Rollback script (pre-generated): $ROLLBACK"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Apply: use sed to insert bond-xmit-hash-policy after bond-mode line
|
||||
# The bond0 stanza looks like:
|
||||
# auto bond0
|
||||
# iface bond0 inet manual
|
||||
# bond-slaves nic1 nic2
|
||||
# bond-miimon 100
|
||||
# bond-mode 802.3ad
|
||||
#
|
||||
# We insert after the bond-mode line.
|
||||
echo "Applying..."
|
||||
|
||||
# Check if bond-mode line exists (various formats)
|
||||
if grep -qE '^\s*bond-mode\s+802.3ad' /etc/network/interfaces; then
|
||||
# Insert after bond-mode 802.3ad line
|
||||
sed -i '/^\s*bond-mode\s+802\.3ad/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode line."
|
||||
elif grep -qE '^\s*bond-mode\s+4' /etc/network/interfaces; then
|
||||
sed -i '/^\s*bond-mode\s+4/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode 4 line."
|
||||
else
|
||||
echo "Could not find bond-mode line — inserting after bond-slaves line instead."
|
||||
sed -i '/^\s*bond-slaves/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
fi
|
||||
|
||||
# Show the result
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
echo ""
|
||||
echo "--- Reloading networking (ifreload -a) ---"
|
||||
echo "SSH should survive (it's on vmbr0/nic0, not bond0)..."
|
||||
ifreload -a 2>&1 || {
|
||||
echo "ifreload failed, trying systemctl restart networking..."
|
||||
systemctl restart networking 2>&1
|
||||
}
|
||||
|
||||
# Wait a moment for bond to renegotiate
|
||||
echo "Waiting 5s for LACP to renegotiate..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "--- Post-change bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " DONE."
|
||||
echo " Backup: $BACKUP_DIR/interfaces"
|
||||
echo " Rollback: bash $ROLLBACK"
|
||||
echo "==================================================================="
|
||||
Executable
+439
@@ -0,0 +1,439 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-tunings.sh
|
||||
#
|
||||
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
|
||||
# 1. Sets scaling_governor=performance on every CPU
|
||||
# 2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
|
||||
# 3. Enables tcp_bbr module + sets congestion control
|
||||
# 4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
|
||||
# 5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
|
||||
# 6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
|
||||
# 7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
|
||||
# restart which drops the host. Provided as a separate --emit-bond-patch
|
||||
# flag that PRINTS the change but does not apply it.)
|
||||
#
|
||||
# Safety features:
|
||||
# - Dry-run mode by default (--apply to commit)
|
||||
# - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
|
||||
# - Per-host behaviour: detects storage hosts by hostname and applies the
|
||||
# right profile (tsys4, tsys5 = storage; others = VM hosts).
|
||||
# - Generates a /root/perfopt-rollback.sh that undoes everything.
|
||||
# - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
|
||||
#
|
||||
# Usage:
|
||||
# bash apply-tunings.sh # dry run, show what would change
|
||||
# bash apply-tunings.sh --apply # commit changes
|
||||
# bash apply-tunings.sh --rollback # restore from latest backup
|
||||
# bash apply-tunings.sh --emit-bond-patch # show bond0 hash change (no apply)
|
||||
###############################################################################
|
||||
set -u
|
||||
umask 022
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
|
||||
ACTION="dryrun"
|
||||
SKIP_NFS=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--apply) ACTION="apply" ;;
|
||||
--rollback) ACTION="rollback" ;;
|
||||
--emit-bond-patch) ACTION="bond-patch" ;;
|
||||
--no-nfs) SKIP_NFS=1 ;;
|
||||
-h|--help) sed -n '2,35p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Detect host role from hostname
|
||||
case "$HOST" in
|
||||
pfv-tsys4|pfv-tsys5)
|
||||
HOST_ROLE="storage"
|
||||
TUNED_PROFILE="network-throughput"
|
||||
SWAPPINESS="1"
|
||||
;;
|
||||
pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
|
||||
HOST_ROLE="vmhost"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
*)
|
||||
HOST_ROLE="unknown"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- rollback path ---------------------------------------------------------
|
||||
if [ "$ACTION" = "rollback" ]; then
|
||||
echo "Looking for latest backup under /root/perfopt-backup-* ..."
|
||||
latest=""
|
||||
while IFS= read -r d; do
|
||||
latest="$d"
|
||||
done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
|
||||
if [ -z "$latest" ]; then
|
||||
echo "No backup found under /root/perfopt-backup-*" >&2
|
||||
exit 1
|
||||
fi
|
||||
rb="$latest/perfopt-rollback.sh"
|
||||
if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
|
||||
echo "Rollback script missing in $latest" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Rolling back using: $rb"
|
||||
bash "$rb"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# --- bond patch print-only path -------------------------------------------
|
||||
if [ "$ACTION" = "bond-patch" ]; then
|
||||
echo "==================================================================="
|
||||
echo " Proposed bond0 xmit_hash_policy change"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "/etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
|
||||
echo "No bond0 detected on this host — nothing to patch."
|
||||
exit 0
|
||||
fi
|
||||
cat <<EOF
|
||||
A manual edit to /etc/network/interfaces is required. The bond0 stanza
|
||||
needs this line added (it defaults to layer2, which is wrong for storage):
|
||||
|
||||
bond-xmit-hash-policy layer3+4
|
||||
|
||||
After editing, you MUST reload networking for it to take effect:
|
||||
ifreload -a # safe, brings interfaces down/up
|
||||
# OR
|
||||
systemctl restart networking # heavier, briefly drops connections
|
||||
|
||||
WARNING: applying this on a remote host over bond0 will briefly drop your
|
||||
SSH session. Run from console/IPMI, or schedule a maintenance window.
|
||||
|
||||
The change is reversible by removing the line and reloading again.
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- main path (dryrun or apply) ------------------------------------------
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Emit the rollback script header
|
||||
cat > "$ROLLBACK_SCRIPT" <<EOF
|
||||
#!/bin/bash
|
||||
# Auto-generated rollback for perfopt apply-tunings.sh
|
||||
# Backup timestamp: $TS
|
||||
# Backup dir: $BACKUP_DIR
|
||||
# Generated on: $HOST
|
||||
set -u
|
||||
EOF
|
||||
chmod +x "$ROLLBACK_SCRIPT"
|
||||
|
||||
backup_file() {
|
||||
local f="$1"
|
||||
if [ -e "$f" ]; then
|
||||
cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
fi
|
||||
}
|
||||
|
||||
# Helper: append a restore command to the rollback script
|
||||
rb_restore() {
|
||||
local f="$1"
|
||||
local bk
|
||||
bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
if [ -r "$bk" ]; then
|
||||
cp -a "$bk" "$f" && echo "restored $f"
|
||||
else
|
||||
echo "WARN: backup $bk missing — $f left as-is"
|
||||
fi
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a sysctl restore command
|
||||
rb_sysctl_restore() {
|
||||
local key="$1" orig_val="$2"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a tuned-adm restore
|
||||
rb_tuned_restore() {
|
||||
local profile="$1"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "==================================================================="
|
||||
echo " perfopt apply-tunings — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
|
||||
echo " backup: $BACKUP_DIR"
|
||||
echo " rollback: $ROLLBACK_SCRIPT"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
|
||||
# ===========================================================================
|
||||
# 1. scaling_governor → performance
|
||||
# ===========================================================================
|
||||
echo "--- 1. CPU scaling_governor → performance"
|
||||
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
|
||||
cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
|
||||
echo " current: $cur"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
if [ "$cur" != "performance" ]; then
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
echo performance > "$c" 2>/dev/null || true
|
||||
done
|
||||
# Persist via systemd tmpfiles / sysctl fallback
|
||||
cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
|
||||
[Unit]
|
||||
Description=Set CPU scaling_governor=performance (perfopt)
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable perfopt-cpu-performance.service 2>/dev/null
|
||||
systemctl start perfopt-cpu-performance.service 2>/dev/null
|
||||
echo " applied + persisted via systemd unit"
|
||||
# Rollback
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
|
||||
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
|
||||
rm -f /etc/systemd/system/perfopt-cpu-performance.service
|
||||
systemctl daemon-reload
|
||||
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
|
||||
EOF
|
||||
else
|
||||
echo " already performance — no change"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 2. vm.swappiness
|
||||
# ===========================================================================
|
||||
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
|
||||
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
|
||||
echo " current: $cur_swappiness"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
|
||||
# Persist via sysctl.d
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
EOF
|
||||
sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
|
||||
echo " applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
|
||||
rb_restore /etc/sysctl.d/99-perfopt.conf
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
|
||||
echo "restored vm.swappiness=$cur_swappiness"
|
||||
EOF
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 3. TCP BBR + buffers (single sysctl.d file)
|
||||
# ===========================================================================
|
||||
echo "--- 3. TCP BBR + socket buffers"
|
||||
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
|
||||
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
|
||||
echo " current: $cur_cc (available: $cur_avail)"
|
||||
|
||||
# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
|
||||
TARGET_RMEM_MAX=134217728 # 128 MB
|
||||
TARGET_WMEM_MAX=134217728
|
||||
TARGET_RMEM_DEFAULT=26214400 # 25 MB
|
||||
TARGET_WMEM_DEFAULT=26214400
|
||||
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
|
||||
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
|
||||
TARGET_NETDEV_MAX_BACKLOG=250000
|
||||
TARGET_SOMAXCONN=65535
|
||||
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Ensure tcp_bbr module loads at boot
|
||||
if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
|
||||
backup_file /etc/modules-load.d/modules.conf
|
||||
mkdir -p /etc/modules-load.d
|
||||
cat >> /etc/modules-load.d/modules.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS
|
||||
tcp_bbr
|
||||
EOF
|
||||
modprobe tcp_bbr 2>/dev/null || true
|
||||
rb_restore /etc/modules-load.d/modules.conf
|
||||
fi
|
||||
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
|
||||
# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
net.core.default_qdisc = fq
|
||||
net.core.rmem_max = $TARGET_RMEM_MAX
|
||||
net.core.wmem_max = $TARGET_WMEM_MAX
|
||||
net.core.rmem_default = $TARGET_RMEM_DEFAULT
|
||||
net.core.wmem_default = $TARGET_WMEM_DEFAULT
|
||||
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
|
||||
net.core.somaxconn = $TARGET_SOMAXCONN
|
||||
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
|
||||
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
|
||||
EOF
|
||||
sysctl --system >/dev/null 2>&1
|
||||
echo " applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
|
||||
# Rebuild rollback for sysctl
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# Restore original sysctl state
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
EOF
|
||||
rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
|
||||
rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
|
||||
# Note: original buffer sizes not all captured; restoration is best-effort.
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
echo "Note: net buffers restored to kernel defaults (original values not captured)."
|
||||
echo "Run 'sysctl --system' to apply."
|
||||
sysctl --system >/dev/null 2>&1 || true
|
||||
EOF
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 4. tuned-adm profile
|
||||
# ===========================================================================
|
||||
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
|
||||
if command -v tuned-adm >/dev/null 2>&1; then
|
||||
cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
|
||||
echo " current: $cur_profile"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
|
||||
tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/ /'
|
||||
rb_tuned_restore "$cur_profile"
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
else
|
||||
echo " tuned-adm not installed — skipping"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 5. NFS mount option hardening
|
||||
# Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
|
||||
# Proxmox applies these on next mount/remount.
|
||||
# Skipped with --no-nfs (for NFS servers where we don't want to remount).
|
||||
# ===========================================================================
|
||||
echo "--- 5. NFS mount options: add nconnect=4, noatime"
|
||||
if [ "$SKIP_NFS" -eq 1 ]; then
|
||||
echo " SKIPPED (--no-nfs specified)"
|
||||
echo " Note: storage.cfg is cluster-wide; if edited on another host,"
|
||||
echo " the options are already staged here and activate on next reboot."
|
||||
else
|
||||
if [ -r /etc/pve/storage.cfg ]; then
|
||||
backup_file /etc/pve/storage.cfg
|
||||
if grep -q '^nfs:' /etc/pve/storage.cfg; then
|
||||
# Count how many nfs: stanzas lack the options
|
||||
while IFS= read -r line; do
|
||||
if echo "$line" | grep -q '^nfs:'; then
|
||||
cur_stanza=$(echo "$line" | awk '{print $2}')
|
||||
# Look ahead for the next few lines to see if options already set
|
||||
echo " nfs stanza: $cur_stanza"
|
||||
fi
|
||||
done < /etc/pve/storage.cfg
|
||||
|
||||
# Check whether any nfs stanza already has options
|
||||
if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
|
||||
echo " some stanzas already have nconnect=4 — check manually"
|
||||
else
|
||||
echo " proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2' to each nfs stanza"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Apply via perl for safety (in-place edit with backup)
|
||||
if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
|
||||
# Use a python helper for robust PVE storage.cfg editing
|
||||
python3 - <<PYEOF
|
||||
import re, pathlib
|
||||
p = pathlib.Path("/etc/pve/storage.cfg")
|
||||
text = p.read_text()
|
||||
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
|
||||
modified = 0
|
||||
def fix(m):
|
||||
global modified
|
||||
block = m.group(1)
|
||||
if 'options' in block:
|
||||
return block
|
||||
lines = block.rstrip('\n').split('\n')
|
||||
lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2')
|
||||
modified += 1
|
||||
return '\n'.join(lines) + '\n'
|
||||
new_text = pattern.sub(fix, text)
|
||||
if modified:
|
||||
p.write_text(new_text)
|
||||
print(f" patched {modified} NFS stanzas with options line")
|
||||
else:
|
||||
print(" no NFS stanza needed patching (all already had options or no nfs: found)")
|
||||
PYEOF
|
||||
# Remount existing NFS mounts to pick up new options
|
||||
echo " remounting NFS mounts to apply new options..."
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "$mp" 2>/dev/null && echo " remounted $mp" || echo " FAILED to remount $mp (will pick up on next mount)"
|
||||
done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
rb_restore /etc/pve/storage.cfg
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# To fully undo NFS options, also remount:
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "\$mp" 2>/dev/null
|
||||
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
echo "restored NFS mount options"
|
||||
EOF
|
||||
else
|
||||
echo " ERROR: could not back up storage.cfg — aborting NFS patch"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " no NFS stanzas in storage.cfg — skipping"
|
||||
fi
|
||||
else
|
||||
echo " /etc/pve/storage.cfg not readable — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 6. Summary
|
||||
# ===========================================================================
|
||||
echo
|
||||
echo "==================================================================="
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
echo " DONE. Backups in: $BACKUP_DIR"
|
||||
echo " Rollback: bash $ROLLBACK_SCRIPT"
|
||||
echo
|
||||
echo " Next steps:"
|
||||
echo " - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
|
||||
echo " - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
|
||||
echo " - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
|
||||
echo " xmit_hash_policy change (requires network restart, do in"
|
||||
echo " maintenance window)"
|
||||
else
|
||||
echo " DRY RUN — no changes made."
|
||||
echo " To commit: bash apply-tunings.sh --apply"
|
||||
fi
|
||||
echo "==================================================================="
|
||||
Executable
+1138
File diff suppressed because it is too large
Load Diff
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# fix-tsys7.sh - fixes the two issues found: NFS options string + bond hash
|
||||
# Runs on the target host directly.
|
||||
set -euo pipefail
|
||||
|
||||
echo "==================================================================="
|
||||
echo " FIX: NFS options + bond hash policy"
|
||||
echo "==================================================================="
|
||||
|
||||
# --- FIX 1: Simplify NFS options in storage.cfg ---------------------------
|
||||
echo ""
|
||||
echo "--- FIX 1: Simplify NFS options (remove version=4.2 conflict) ---"
|
||||
# Replace the overly-complex options line with a minimal one
|
||||
# PVE handles vers/rsize/wsize/hard/etc internally; we only need nconnect + noatime
|
||||
if grep -q 'options.*nconnect=4.*version=4.2' /etc/pve/storage.cfg; then
|
||||
# Use sed to replace each options line
|
||||
sed -i 's/options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2/options nconnect=4,noatime/' /etc/pve/storage.cfg
|
||||
echo "Fixed: simplified NFS options to 'nconnect=4,noatime'"
|
||||
else
|
||||
echo "Options line already simplified or not present"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Verify storage.cfg NFS stanzas ---"
|
||||
grep -A 2 "^nfs: D2" /etc/pve/storage.cfg | head -3
|
||||
echo "..."
|
||||
grep -c "options nconnect" /etc/pve/storage.cfg
|
||||
echo " NFS stanzas with options"
|
||||
|
||||
# --- FIX 2: Bond hash policy (force via sysfs + persist in interfaces) ----
|
||||
echo ""
|
||||
echo "--- FIX 2: Apply bond xmit_hash_policy=layer3+4 ---"
|
||||
|
||||
# 2a: Apply LIVE via sysfs (takes effect immediately, no network reload)
|
||||
echo "Applying live via sysfs..."
|
||||
if echo "layer3+4" > /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; then
|
||||
echo "Live sysfs apply: SUCCESS"
|
||||
else
|
||||
echo "Live sysfs apply: FAILED (will persist in config and apply on ifreload)"
|
||||
fi
|
||||
|
||||
# Verify live state
|
||||
echo ""
|
||||
echo "Live bond0 hash policy:"
|
||||
cat /proc/net/bonding/bond0 | grep "Transmit Hash"
|
||||
|
||||
# 2b: Persist in /etc/network/interfaces (fix the sed that failed before)
|
||||
echo ""
|
||||
echo "Persisting in /etc/network/interfaces..."
|
||||
|
||||
# Backup
|
||||
cp -a /etc/network/interfaces "/root/interfaces.bondfix.$(date +%Y%m%d%H%M%S)"
|
||||
|
||||
# Check if already present
|
||||
if grep -q 'bond-xmit-hash-policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already in interfaces file"
|
||||
else
|
||||
# Use awk to insert after the bond-mode line (more reliable than sed)
|
||||
# Match any line containing 'bond-mode' (regardless of indentation)
|
||||
awk '
|
||||
/bond-mode/ && !done {
|
||||
print
|
||||
print "\tbond-xmit-hash-policy layer3+4"
|
||||
done=1
|
||||
next
|
||||
}
|
||||
{ print }
|
||||
' /etc/network/interfaces > /etc/network/interfaces.new
|
||||
|
||||
mv /etc/network/interfaces.new /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
# Final verify
|
||||
echo ""
|
||||
echo "--- Final bond0 running state ---"
|
||||
cat /proc/net/bonding/bond0 | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " FIX COMPLETE"
|
||||
echo "==================================================================="
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-retrans-cause.sh (HOST-NATIVE)
|
||||
#
|
||||
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
|
||||
#
|
||||
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
|
||||
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
|
||||
# saturation) or just unavoidable LACP reordering overhead?
|
||||
#
|
||||
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
|
||||
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
|
||||
# - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources -> ring too small
|
||||
# - /proc/net/softnet_stat drops -> ksoftirq starved
|
||||
# - /proc/net/snmp TCP retranst -> TCP-level retrans
|
||||
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
|
||||
# coming from LACP reordering (out-of-order segments triggering fast
|
||||
# retransmit), not packet loss — and are NOT fixable by tuning.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
HOST="$(uname -n)"; HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
|
||||
*) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# ---- per-slave NIC counter snapshot -----------------------------------------
|
||||
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
|
||||
# plus all error-like counters. Falls back gracefully if a counter doesn't
|
||||
# exist (different NIC drivers expose different names).
|
||||
nic_snapshot() {
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
|
||||
[ -n "$s" ] || continue
|
||||
echo "[$s]"
|
||||
# ethtool -S may exist; show only drop/error/retrans lines
|
||||
if command -v ethtool >/dev/null 2>&1; then
|
||||
ethtool -S "$s" 2>/dev/null \
|
||||
| grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
|
||||
|| echo "(ethtool -S: no matching counters or unsupported)"
|
||||
else
|
||||
echo "(ethtool not installed)"
|
||||
fi
|
||||
# /sys counters (always available, driver-agnostic)
|
||||
for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
|
||||
rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
|
||||
multicast collisions; do
|
||||
v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
|
||||
[ -n "$v" ] && printf ' sysfs %-22s = %s\n' "$c" "$v"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
|
||||
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
|
||||
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
|
||||
# look identical in hex and decimal). Processed is informational only and we
|
||||
# don't try to delta it (hex arithmetic is awk-version-dependent).
|
||||
softnet_snapshot() {
|
||||
awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
|
||||
NR-1, $1, $2, $3 }' /proc/net/softnet_stat
|
||||
}
|
||||
|
||||
# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
|
||||
tcp_snapshot() {
|
||||
awk '/^Tcp:/{
|
||||
if (!seen) {
|
||||
seen=1
|
||||
# /proc/net/snmp Tcp line 1 = names, line 2 = values
|
||||
n=split($0, names, " ")
|
||||
# re-find the values line
|
||||
getline
|
||||
vals=$0
|
||||
split(vals, v, " ")
|
||||
for (i=1;i<=n;i++) printf " %-22s = %s\n", names[i], v[i]
|
||||
}
|
||||
}' /proc/net/snmp
|
||||
}
|
||||
|
||||
# ---- print deltas for selected counters ------------------------------------
|
||||
# $1 = before file, $2 = after file
|
||||
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
|
||||
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
|
||||
# integers (typically 0 on a healthy host), so +0 coercion is correct for
|
||||
# the values we care about. We deliberately DON'T report `processed` deltas
|
||||
# because hex arithmetic on large numbers is awk-version-dependent.
|
||||
softnet_delta() {
|
||||
awk '
|
||||
FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
|
||||
next }
|
||||
{ if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, q, /[= ]+/);
|
||||
dd = (q[5]+0) - drop[id]
|
||||
sd = (q[7]+0) - sqz[id]
|
||||
if (dd != 0 || sd != 0)
|
||||
printf " %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
|
||||
' "$1" "$2" | sort
|
||||
}
|
||||
|
||||
# $1 = before file, $2 = after file, $3 = section label prefix
|
||||
# File format: slave header line "[nic1]", then " counter_name = value" lines.
|
||||
# Output only counters whose value changed.
|
||||
nic_delta() {
|
||||
awk '
|
||||
FNR==NR { if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
# Everything before " = " is the counter name (trim spaces)
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1)
|
||||
gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
before[slave SUBSEP name] = val
|
||||
}
|
||||
next }
|
||||
{ if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
b = before[slave SUBSEP name]+0
|
||||
d = val+0 - b
|
||||
if (d != 0) printf " %-8s %-30s %12s -> %12s delta=%d\n", slave, name, b, val, d
|
||||
} }
|
||||
' "$1" "$2"
|
||||
}
|
||||
|
||||
{
|
||||
echo "=== LACP retransmit cause investigation ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- bond0 hash + driver ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
|
||||
drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
|
||||
speed=$(cat /sys/class/net/$s/speed 2>/dev/null)
|
||||
ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
|
||||
printf " %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
|
||||
done
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
echo "--- BEFORE snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_before"
|
||||
softnet_snapshot > "$TMP/softnet_before"
|
||||
tcp_snapshot > "$TMP/tcp_before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
else
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
fi
|
||||
|
||||
echo "--- AFTER snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_after"
|
||||
softnet_snapshot > "$TMP/softnet_after"
|
||||
tcp_snapshot > "$TMP/tcp_after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 summary ---"
|
||||
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
|
||||
echo ""
|
||||
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
|
||||
# Show only the counters that changed
|
||||
paste "$TMP/tcp_before" "$TMP/tcp_after" \
|
||||
| awk '{
|
||||
a=$3; b=$(NF)
|
||||
if (a+0 != b+0) printf " %-22s %12s -> %12s delta=%d\n", $1, a, b, b-a
|
||||
}'
|
||||
echo ""
|
||||
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
|
||||
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
|
||||
echo ""
|
||||
echo "--- NIC counter deltas (only non-zero) ---"
|
||||
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
|
||||
echo ""
|
||||
echo "--- raw iperf3 tail (last 12 lines) ---"
|
||||
tail -12 "$TMP/iperf.out"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " HOW TO READ THIS"
|
||||
echo "==================================================================="
|
||||
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
|
||||
echo " -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
|
||||
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
|
||||
echo " -> softirq starvation. Fix: increase net.core.netdev_budget,"
|
||||
echo " check IRQ affinity, consider RPS."
|
||||
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
|
||||
echo " -> the retransmits are LACP reordering (out-of-order segments"
|
||||
echo " triggering fast retransmit), NOT packet loss. NOT fixable."
|
||||
echo "==================================================================="
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log: $LOG"
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-rx-distribution.sh (HOST-NATIVE version)
|
||||
#
|
||||
# Runs ON a host (tsys6 or tsys7). Auto-detects role by hostname:
|
||||
# - On tsys6 (receiver): starts iperf3 -s -1 (one-shot), snapshots local
|
||||
# bond0 slave RX counters before+after, writes verdict + log.
|
||||
# - On tsys7 (sender): snapshots local bond0 slave TX counters before+
|
||||
# after, runs iperf3 -c <peer> -P 8 -t 12.
|
||||
#
|
||||
# Each side writes only its own counters. No inter-host SSH required.
|
||||
# iperf3 -s -1 (one-shot) handles client/server coordination.
|
||||
#
|
||||
# Output: /root/lacp-rx-{receiver,sender}.log
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
# Snapshot per-slave byte counters from $BOND. Output: "<slave> rx tx\n" sorted.
|
||||
snapshot() {
|
||||
awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND" | while read -r s; do
|
||||
[ -n "$s" ] || continue
|
||||
rx=$(cat "/sys/class/net/$s/statistics/rx_bytes" 2>/dev/null || echo 0)
|
||||
tx=$(cat "/sys/class/net/$s/statistics/tx_bytes" 2>/dev/null || echo 0)
|
||||
printf '%s %s %s\n' "$s" "$rx" "$tx"
|
||||
done | sort
|
||||
}
|
||||
|
||||
# Compute per-slave deltas. $1=label, $2=col(2=rx,3=tx), $3=before, $4=after.
|
||||
analyze() {
|
||||
local label="$1" col="$2" before="$3" after="$4"
|
||||
join "$before" "$after" | awk -v col="$col" -v lbl="$label" '
|
||||
{
|
||||
slave=$1
|
||||
b = (col==2 ? $2 : $3)+0
|
||||
a = (col==2 ? $4 : $5)+0
|
||||
d = a - b; if (d < 0) d = 0
|
||||
delta[slave]=d; bef[slave]=b; aft[slave]=a
|
||||
order[++n]=slave; total+=d
|
||||
}
|
||||
END {
|
||||
printf "\n--- %s (bytes) ---\n", lbl
|
||||
printf " %-12s %14s %14s %14s %8s\n", "SLAVE", "BEFORE", "AFTER", "DELTA", "PCT"
|
||||
for (i=1;i<=n;i++){
|
||||
s=order[i]
|
||||
pct = (total>0 ? 100*delta[s]/total : 0)
|
||||
printf " %-12s %14d %14d %14d %7.1f%%\n", s, bef[s], aft[s], delta[s], pct
|
||||
}
|
||||
printf " %-12s %14s %14s %14d %8s\n", "TOTAL", "", "", total, "100.0%"
|
||||
}'
|
||||
}
|
||||
|
||||
# Dominant-slave pct for the verdict line. $1=col, $2=before, $3=after.
|
||||
dominant() {
|
||||
join "$2" "$3" | awk -v col="$1" '
|
||||
{ b=(col==2?$2:$3)+0; a=(col==2?$4:$5)+0; d=a-b; if(d<0)d=0; tot+=d; delta[$1]=d }
|
||||
END { m=0; ms=""; for (s in delta){ if (delta[s]>m){m=delta[s]; ms=s} }
|
||||
printf "%.1f %s", (tot>0?100*m/tot:0), ms }'
|
||||
}
|
||||
|
||||
HOST="$(uname -n)"
|
||||
HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-rx-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-rx-sender.log" ;;
|
||||
*) echo "ERROR: not running on tsys6 or tsys7 (hostname=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
{
|
||||
echo "=== LACP per-slave distribution ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- $BOND state ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac|Slave Interface|Link" \
|
||||
"/proc/net/bonding/$BOND" 2>&1
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
snapshot > "$TMP/before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting one-shot iperf3 server on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
# Wait for the server to be ready (brief), then wait for it to exit
|
||||
# (one-shot server exits after serving one client).
|
||||
for _ in $(seq 1 60); do
|
||||
kill -0 "$SRV_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
echo "[receiver] iperf3 server finished."
|
||||
else
|
||||
# sender: wait briefly for receiver to be listening, then run client.
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 client..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
echo "[sender] iperf3 client finished (exit=$?)."
|
||||
fi
|
||||
|
||||
snapshot > "$TMP/after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 output (raw tail) ---"
|
||||
tail -25 "$TMP/iperf.out"
|
||||
echo ""
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
analyze "Receiver RX (THE key column)" 2 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 2 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: SWITCH NOT distributing across $HOST's two ports"
|
||||
echo " Dominant slave '$top' = ${pct}% of RX -> hash not effective"
|
||||
echo " on this LAG. Action: verify/bounce pfv-r3-tor-stor port-channel."
|
||||
else
|
||||
echo " VERDICT: switch IS distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Cap is host-side: softirq/CPU/bridge/NIC coalescing."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
else
|
||||
analyze "Sender TX (control: should split if host hash=layer3+4)" 3 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 3 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: host TX NOT distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Host xmit_hash_policy is NOT effective despite /proc/net/bonding."
|
||||
else
|
||||
echo " VERDICT: host TX distributing OK (top slave '$top' = ${pct}%)."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
fi
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log written: $LOG"
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-retrans-cause.sh (workstation wrapper)
|
||||
#
|
||||
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
|
||||
# right order, scps logs back, prints them.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6"; SEND="pfv-tsys7"
|
||||
LOCAL_LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
SCRIPT="lacp-retrans-cause.sh"
|
||||
LOCAL_SCRIPT="/home/reachableceo/projects/perfopt/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP retransmit-cause investigation"
|
||||
echo " Receiver: $RECV Sender: $SEND"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
echo "--- preflight ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v ethtool >/dev/null && e=OK || e=MISSING
|
||||
command -v iperf3 >/dev/null && i=OK || i=MISSING
|
||||
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- deploy ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed" || echo "FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- launching receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
sleep 3
|
||||
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
*receiver*) src="$RECV" ;;
|
||||
*sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-32s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-rx-distribution.sh (workstation wrapper)
|
||||
#
|
||||
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
|
||||
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
|
||||
# scps both logs back to returned-logs/iperf/.
|
||||
#
|
||||
# Run from the workstation:
|
||||
# bash scripts/run-lacp-rx-distribution.sh
|
||||
#
|
||||
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6" # receiver (we care most about its RX split)
|
||||
SEND="pfv-tsys7" # sender (control: its TX split)
|
||||
LOCAL_LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
SCRIPT="lacp-rx-distribution.sh"
|
||||
LOCAL_SCRIPT="/home/reachableceo/projects/perfopt/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP per-slave RX/TX distribution test"
|
||||
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# 0. Preflight: confirm SSH and iperf3 on both hosts
|
||||
echo "--- preflight (ssh + iperf3 + bond0) ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
|
||||
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
|
||||
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
|
||||
2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 1. Copy the script to both hosts + chmod
|
||||
echo "--- deploy $SCRIPT to both hosts ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed + chmod +x" \
|
||||
|| echo "DEPLOY FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 2. Kill any stale iperf3 on both hosts
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
|
||||
echo "--- starting receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
|
||||
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" \
|
||||
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
|
||||
# 5. Give receiver a moment to finish writing its log
|
||||
sleep 3
|
||||
|
||||
# 6. Fetch logs back
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
lacp-rx-receiver*) src="$RECV" ;;
|
||||
lacp-rx-sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-28s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 7. Show the receiver log (the decisive one)
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV — the decisive side)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND — control)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
# sw-capture-remote.sh - orchestrate a serial capture from this workstation.
|
||||
#
|
||||
# Flow:
|
||||
# 1. De-conflict: abort if any local ssh to pfv-tsys4 is in flight
|
||||
# (other agent could be there).
|
||||
# 2. Free the serial port: kill whatever holds /dev/ttyUSBx
|
||||
# (typically a screen session). Targeted, not blanket.
|
||||
# 3. scp driver + .cmds to pfv-tsys4.
|
||||
# 4. Run driver over ssh, capture stderr to console.
|
||||
# 5. scp the resulting log back to returned-logs/.
|
||||
#
|
||||
# Usage:
|
||||
# sw-capture-remote.sh <switch-name> [device]
|
||||
#
|
||||
# <switch-name> e.g. pfv-core-sw01 (must have switches/<name>.cmds)
|
||||
# [device] /dev/ttyUSBx on pfv-tsys4. Defaults per switch map below.
|
||||
#
|
||||
# Currently scoped to pfv-core-sw01 only (per user direction). The other
|
||||
# two switches are deferred; their defaults are placeholders.
|
||||
set -u
|
||||
|
||||
SWITCH=${1:-}
|
||||
DEVICE=${2:-}
|
||||
|
||||
if [ -z "$SWITCH" ]; then
|
||||
echo "Usage: $0 <switch-name> [device]" >&2
|
||||
echo " e.g. $0 pfv-core-sw01 /dev/ttyUSB2" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Switch -> default device map (ttyUSB2 = core-sw01 confirmed by user).
|
||||
case "$SWITCH" in
|
||||
pfv-core-sw01)
|
||||
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB2 ;;
|
||||
pfv-r3-tor-mgmt)
|
||||
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB0 # TENTATIVE - unconfirmed
|
||||
if [ "${2:-}" = "" ]; then
|
||||
echo "NOTE: pfv-r3-tor-mgmt device is tentative (/dev/ttyUSB0)." >&2
|
||||
echo " Pass the device explicitly if different." >&2
|
||||
fi ;;
|
||||
pfv-r3-tor-stor)
|
||||
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB1 # TENTATIVE - unconfirmed
|
||||
if [ "${2:-}" = "" ]; then
|
||||
echo "NOTE: pfv-r3-tor-stor device is tentative (/dev/ttyUSB1)." >&2
|
||||
echo " Pass the device explicitly if different." >&2
|
||||
fi ;;
|
||||
*)
|
||||
echo "unknown switch: $SWITCH" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
BAUD=9600
|
||||
HOST=root@pfv-tsys4
|
||||
HERE=/home/reachableceo/projects/perfopt
|
||||
LOCAL_DRIVER=$HERE/scripts/sw-capture.py
|
||||
LOCAL_CMDS=$HERE/switches/$SWITCH.cmds
|
||||
LOCAL_LOG=$HERE/returned-logs/$SWITCH.log
|
||||
REMOTE_DRIVER=/root/sw-capture.py
|
||||
REMOTE_CMDS=/root/$SWITCH.cmds
|
||||
REMOTE_LOG=/root/$SWITCH.log
|
||||
|
||||
[ -f "$LOCAL_DRIVER" ] || { echo "missing $LOCAL_DRIVER" >&2; exit 2; }
|
||||
[ -f "$LOCAL_CMDS" ] || { echo "missing $LOCAL_CMDS" >&2; exit 2; }
|
||||
|
||||
ts() { date +%H:%M:%S; }
|
||||
|
||||
echo "[$(ts)] switch=$SWITCH device=$DEVICE baud=$BAUD host=$HOST"
|
||||
|
||||
# 1. De-conflict: any local ssh to pfv-tsys4 in flight?
|
||||
echo "[$(ts)] checking for in-flight ssh to pfv-tsys4..."
|
||||
if ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys4|scp.*pfv-tsys4' | grep -v grep >/tmp/.swcap.ps 2>&1; then
|
||||
cat /tmp/.swcap.ps
|
||||
echo "[$(ts)] ABORT: another ssh/scp to pfv-tsys4 is running (other agent?)." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[$(ts)] clear."
|
||||
rm -f /tmp/.swcap.ps
|
||||
|
||||
# 2. Free the serial port: kill whatever holds $DEVICE.
|
||||
echo "[$(ts)] freeing $DEVICE on $HOST (targeted; other screen sessions untouched)..."
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "$HOST" \
|
||||
"fuser -v $DEVICE 2>&1 | tee /dev/stderr; \
|
||||
fuser -k -TERM $DEVICE 2>/dev/null; sleep 1; \
|
||||
if fuser $DEVICE 2>/dev/null; then \
|
||||
echo 'still held after SIGTERM, escalating to SIGKILL'; \
|
||||
fuser -k -KILL $DEVICE 2>/dev/null; sleep 1; \
|
||||
fi; \
|
||||
fuser $DEVICE 2>/dev/null && echo 'STILL HELD' || echo 'FREE'"
|
||||
|
||||
# Re-check; abort if still held.
|
||||
HELD=$(ssh -o BatchMode=yes "$HOST" "fuser $DEVICE 2>/dev/null && echo HELD || echo FREE")
|
||||
if [ "$HELD" = "HELD" ]; then
|
||||
echo "[$(ts)] ABORT: $DEVICE still held on $HOST." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. Copy driver + cmds.
|
||||
echo "[$(ts)] copying driver + cmds to $HOST..."
|
||||
scp -q "$LOCAL_DRIVER" "$HOST:$REMOTE_DRIVER"
|
||||
scp -q "$LOCAL_CMDS" "$HOST:$REMOTE_CMDS"
|
||||
|
||||
# 4. Run the capture on pfv-tsys4. Stream stderr (progress) to console.
|
||||
echo "[$(ts)] running capture..."
|
||||
ssh -o BatchMode=yes -o ServerAliveInterval=10 "$HOST" \
|
||||
"python3 $REMOTE_DRIVER \
|
||||
--device $DEVICE --baud $BAUD \
|
||||
--cmds $REMOTE_CMDS --log $REMOTE_LOG"
|
||||
RC=$?
|
||||
echo "[$(ts)] capture exit code: $RC"
|
||||
|
||||
# 5. Pull log back.
|
||||
echo "[$(ts)] pulling log back to $LOCAL_LOG..."
|
||||
mkdir -p "$(dirname "$LOCAL_LOG")"
|
||||
scp -q "$HOST:$REMOTE_LOG" "$LOCAL_LOG"
|
||||
if [ -f "$LOCAL_LOG" ]; then
|
||||
SZ=$(wc -c < "$LOCAL_LOG")
|
||||
echo "[$(ts)] OK: $LOCAL_LOG ($SZ bytes)"
|
||||
echo "----- head -----"
|
||||
head -30 "$LOCAL_LOG"
|
||||
echo "----- tail -----"
|
||||
tail -10 "$LOCAL_LOG"
|
||||
else
|
||||
echo "[$(ts)] ERROR: log not pulled back." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit $RC
|
||||
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sw-capture.py - drive a Dell switch over a serial console and log all output.
|
||||
|
||||
Read-only. Sends ONLY the commands listed in the supplied .cmds file
|
||||
(comment lines starting with '!' and blank lines are skipped). Handles
|
||||
`--More--` pagination by sending a space. Aborts cleanly on any password
|
||||
prompt (we never supply credentials).
|
||||
|
||||
Pure stdlib (termios + select). No pyserial/expect required.
|
||||
|
||||
Exit codes:
|
||||
0 clean run, every command saw a prompt again
|
||||
2 could not synchronize with a prompt during wake
|
||||
3 one or more commands timed out (log still written)
|
||||
4 password prompt encountered (aborted)
|
||||
|
||||
Usage:
|
||||
sw-capture.py --device /dev/ttyUSB2 --baud 9600 \\
|
||||
--cmds pfv-core-sw01.cmds --log /root/pfv-core-sw01.log
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import select
|
||||
import sys
|
||||
import termios
|
||||
import time
|
||||
|
||||
PROMPT_RE = re.compile(rb'[>#]\s*$') # ends in # or > + spaces
|
||||
MORE_RE = re.compile(rb'--\s*More\s*--') # pagination prompt
|
||||
PWD_RE = re.compile(rb'[Pp]assword:\s*$') # enable / login password
|
||||
|
||||
BAUDS = {
|
||||
'9600': termios.B9600,
|
||||
'19200': termios.B19200,
|
||||
'38400': termios.B38400,
|
||||
'57600': termios.B57600,
|
||||
'115200': termios.B115200,
|
||||
}
|
||||
|
||||
|
||||
def log(msg, level='INFO'):
|
||||
sys.stderr.write(f'[{level}] {msg}\n')
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def open_port(device, baud):
|
||||
"""Open the serial device raw at the requested baud, 8N1, no flow ctrl."""
|
||||
fd = os.open(device, os.O_RDWR | os.O_NOCTTY)
|
||||
try:
|
||||
attrs = termios.tcgetattr(fd)
|
||||
except termios.error:
|
||||
log(f'{device} is not a termios-capable device', 'WARN')
|
||||
return fd
|
||||
|
||||
# raw input
|
||||
attrs[0] &= ~(termios.IGNBRK | termios.BRKINT | termios.PARMRK |
|
||||
termios.ISTRIP | termios.INLCR | termios.IGNCR |
|
||||
termios.ICRNL | termios.IXON)
|
||||
# raw output
|
||||
attrs[1] &= ~termios.OPOST
|
||||
# 8N1, enable receiver, ignore modem control lines
|
||||
attrs[2] &= ~(termios.CSIZE | termios.PARENB | termios.CSTOPB)
|
||||
attrs[2] |= termios.CS8 | termios.CREAD | termios.CLOCAL
|
||||
# raw local
|
||||
attrs[3] &= ~(termios.ECHO | termios.ECHONL | termios.ICANON |
|
||||
termios.ISIG | termios.IEXTEN)
|
||||
# non-blocking-ish reads (select is the primary gate)
|
||||
attrs[6][termios.VMIN] = 0
|
||||
attrs[6][termios.VTIME] = 1
|
||||
|
||||
b = BAUDS.get(str(baud))
|
||||
if b is None:
|
||||
raise SystemExit(f'unsupported baud: {baud}')
|
||||
# Set ispeed/ospeed directly on the attribute list. (Equivalent to
|
||||
# termios.cfsetispeed/cfsetospeed, which are missing on some Python
|
||||
# builds — e.g. the one on pfv-tsys4.)
|
||||
attrs[4] = b # ispeed
|
||||
attrs[5] = b # ospeed
|
||||
termios.tcsetattr(fd, termios.TCSANOW, attrs)
|
||||
return fd
|
||||
|
||||
|
||||
def read_chunk(fd, timeout):
|
||||
"""Read whatever arrives within `timeout`. Extends briefly on activity."""
|
||||
buf = b''
|
||||
deadline = time.time() + timeout
|
||||
while True:
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
return buf
|
||||
r, _, _ = select.select([fd], [], [], min(0.5, remaining))
|
||||
if not r:
|
||||
if buf:
|
||||
return buf
|
||||
continue
|
||||
try:
|
||||
chunk = os.read(fd, 4096)
|
||||
except OSError:
|
||||
return buf
|
||||
if not chunk:
|
||||
return buf
|
||||
buf += chunk
|
||||
# keep collecting as long as bytes are flowing
|
||||
deadline = time.time() + 0.3
|
||||
|
||||
|
||||
def drain(fd, timeout=1.0):
|
||||
total = 0
|
||||
while True:
|
||||
b = read_chunk(fd, timeout=timeout)
|
||||
if not b:
|
||||
return total
|
||||
total += len(b)
|
||||
|
||||
|
||||
def send(fd, s):
|
||||
if isinstance(s, str):
|
||||
s = s.encode()
|
||||
os.write(fd, s)
|
||||
|
||||
|
||||
def wait_for(fd, regex, timeout, on_more=None, on_pwd=None):
|
||||
"""Read until `regex` matches the tail of the buffer, or timeout."""
|
||||
buf = b''
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
remaining = deadline - time.time()
|
||||
chunk = read_chunk(fd, timeout=min(1.0, remaining))
|
||||
if chunk:
|
||||
buf += chunk
|
||||
tail64 = buf[-64:]
|
||||
tail32 = buf[-32:]
|
||||
tail128 = buf[-128:]
|
||||
if on_more and MORE_RE.search(tail64):
|
||||
on_more(fd)
|
||||
continue
|
||||
if on_pwd and PWD_RE.search(tail32):
|
||||
on_pwd(buf)
|
||||
return buf, 'pwd'
|
||||
if regex.search(tail128):
|
||||
return buf, 'ok'
|
||||
return buf, 'timeout'
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--device', required=True)
|
||||
ap.add_argument('--baud', type=int, default=9600)
|
||||
ap.add_argument('--cmds', required=True)
|
||||
ap.add_argument('--log', required=True)
|
||||
ap.add_argument('--per-cmd-timeout', type=float, default=45.0)
|
||||
ap.add_argument('--wake-timeout', type=float, default=15.0)
|
||||
ap.add_argument('--session-max', type=float, default=600.0)
|
||||
args = ap.parse_args()
|
||||
|
||||
cmds = []
|
||||
with open(args.cmds) as f:
|
||||
for raw in f:
|
||||
s = raw.rstrip('\n').strip()
|
||||
if not s or s.startswith('!'):
|
||||
continue
|
||||
cmds.append(s)
|
||||
log(f'parsed {len(cmds)} commands from {args.cmds}')
|
||||
|
||||
logf = open(args.log, 'wb', buffering=0)
|
||||
|
||||
def w(b):
|
||||
if isinstance(b, str):
|
||||
b = b.encode()
|
||||
logf.write(b)
|
||||
|
||||
w(f'==== sw-capture {time.strftime("%Y-%m-%d %H:%M:%S")} ====\n')
|
||||
w(f'device={args.device} baud={args.baud} cmds={args.cmds} '
|
||||
f'n={len(cmds)} per_cmd_timeout={args.per_cmd_timeout}\n\n')
|
||||
|
||||
fd = open_port(args.device, args.baud)
|
||||
log(f'opened {args.device} @ {args.baud} 8N1 raw')
|
||||
|
||||
session_start = time.time()
|
||||
abort = False
|
||||
|
||||
def on_more(fd_):
|
||||
log('--More-- -> space')
|
||||
send(fd_, b' ')
|
||||
|
||||
def on_pwd(buf):
|
||||
nonlocal abort
|
||||
abort = True
|
||||
log('password prompt detected (enable or login) - aborting; '
|
||||
'no credentials supplied', 'ERROR')
|
||||
w(buf)
|
||||
w(b'\n[PASSWORD PROMPT - ABORTED]\n')
|
||||
|
||||
# WAKE: nudge with Ctrl-C + Enter, look for any prompt
|
||||
drain(fd, 0.5)
|
||||
synced = False
|
||||
wake_deadline = time.time() + args.wake_timeout
|
||||
attempt = 0
|
||||
while time.time() < wake_deadline:
|
||||
attempt += 1
|
||||
send(fd, b'\x03')
|
||||
time.sleep(0.2)
|
||||
send(fd, b'\r')
|
||||
buf, status = wait_for(fd, PROMPT_RE, timeout=3.0,
|
||||
on_more=on_more, on_pwd=on_pwd)
|
||||
w(buf)
|
||||
if status == 'pwd':
|
||||
logf.close(); os.close(fd); sys.exit(4)
|
||||
if status == 'ok':
|
||||
synced = True
|
||||
log(f'prompt synced after {attempt} attempt(s)')
|
||||
break
|
||||
if not synced:
|
||||
w(b'\n[NO PROMPT - ABORT]\n')
|
||||
log('no prompt detected during wake window', 'ERROR')
|
||||
logf.close(); os.close(fd); sys.exit(2)
|
||||
|
||||
# RUN commands verbatim from the .cmds list
|
||||
failures = 0
|
||||
for idx, cmd in enumerate(cmds, 1):
|
||||
if time.time() - session_start > args.session_max:
|
||||
log('session_max exceeded - stopping early', 'ERROR')
|
||||
w(b'\n[SESSION_MAX - STOP]\n')
|
||||
break
|
||||
if abort:
|
||||
break
|
||||
log(f'[{idx}/{len(cmds)}] {cmd}')
|
||||
send(fd, cmd + '\r')
|
||||
buf, status = wait_for(fd, PROMPT_RE,
|
||||
timeout=args.per_cmd_timeout,
|
||||
on_more=on_more, on_pwd=on_pwd)
|
||||
w(buf)
|
||||
if status == 'pwd':
|
||||
failures += 1
|
||||
break
|
||||
if status == 'timeout':
|
||||
log(f'timeout after: {cmd}', 'WARN')
|
||||
failures += 1
|
||||
# try to resync: Ctrl-C + drain
|
||||
send(fd, b'\x03')
|
||||
time.sleep(0.3)
|
||||
drain(fd, 0.5)
|
||||
|
||||
w(f'\n==== end {time.strftime("%Y-%m-%d %H:%M:%S")} '
|
||||
f'failures={failures} ====\n')
|
||||
logf.close()
|
||||
os.close(fd)
|
||||
log(f'done -> {args.log} failures={failures}')
|
||||
sys.exit(0 if failures == 0 else 3)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe conman state + expect availability on pfv-tsys4.
|
||||
# Read-only. Decides whether we drive via conman+expect or expect-only.
|
||||
set -u
|
||||
|
||||
# De-conflict: any ssh to pfv-tsys4 right now?
|
||||
echo "===== LOCAL ssh activity ====="
|
||||
ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys' | grep -v grep || echo "(none to pfv-tsys4)"
|
||||
|
||||
echo
|
||||
echo "===== pfv-tsys4: conman + expect state ====="
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=5 root@pfv-tsys4 'bash -s' <<'REMOTE'
|
||||
echo "--- conmand service ---"
|
||||
systemctl is-active conmand 2>&1 || true
|
||||
systemctl is-enabled conmand 2>&1 || true
|
||||
systemctl status conmand --no-pager 2>&1 | head -15 || true
|
||||
|
||||
echo
|
||||
echo "--- conman binary ---"
|
||||
command -v conman && conman --version 2>&1 | head -2 || echo "conman: MISSING"
|
||||
command -v conmand && echo "conmand present" || echo "conmand: MISSING"
|
||||
|
||||
echo
|
||||
echo "--- /etc/conman.conf: ttyUSB2 entries ---"
|
||||
grep -nE "ttyUSB2|core-sw|CONSOLE|LOG|SERIAL|BAUD" /etc/conman.conf 2>/dev/null | head -40 || echo "(no matches / no file)"
|
||||
|
||||
echo
|
||||
echo "--- conman log dir ---"
|
||||
ls -la /var/log/conman/ 2>&1 | head -20 || echo "(no /var/log/conman)"
|
||||
ls -la /var/consoles/ 2>&1 | head -20 || echo "(no /var/consoles)"
|
||||
|
||||
echo
|
||||
echo "--- expect availability ---"
|
||||
command -v expect && expect -v 2>&1 || echo "expect: NOT installed"
|
||||
echo "apt-cache policy expect:"
|
||||
apt-cache policy expect 2>/dev/null | head -10 || echo "(apt-cache failed)"
|
||||
|
||||
echo
|
||||
echo "--- other useful drivers ---"
|
||||
for t in tclsh socat cu tip; do
|
||||
command -v "$t" 2>/dev/null && echo " $t: present" || true
|
||||
done
|
||||
|
||||
echo
|
||||
echo "--- apt network reachability (quick) ---"
|
||||
timeout 5 bash -c 'echo > /dev/tcp/deb.debian.org/80' 2>&1 && echo "apt network: OK" || echo "apt network: UNREACHABLE"
|
||||
|
||||
echo
|
||||
echo "--- disk space for log ---"
|
||||
df -h /root 2>&1 | tail -2
|
||||
|
||||
echo
|
||||
echo "--- screen sessions (still 3?) ---"
|
||||
screen -ls 2>&1 || true
|
||||
REMOTE
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe pfv-tsys4 for what's available to drive the serial console.
|
||||
# Also snapshots local ssh/scp activity so we can de-conflict with the
|
||||
# other agent running in this directory.
|
||||
set -u
|
||||
|
||||
echo "===== LOCAL ssh/scp activity (other-agent de-confliction) ====="
|
||||
ps -eo pid,ppid,etime,user,args | grep -E 'ssh|scp' | grep -v grep || echo "(none)"
|
||||
|
||||
echo
|
||||
echo "===== Ping pfv-tsys4 ====="
|
||||
ping -c1 -W2 pfv-tsys4 >/dev/null 2>&1 && echo "ping OK" || echo "ping FAIL"
|
||||
|
||||
echo
|
||||
echo "===== Probe pfv-tsys4 over ssh ====="
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=5 root@pfv-tsys4 'bash -s' <<'REMOTE'
|
||||
echo "--- host ---"
|
||||
hostname; uname -a
|
||||
echo "--- tools ---"
|
||||
for t in python3 python expect screen minicom picocom stty fuser lsof; do
|
||||
p=$(command -v "$t" 2>/dev/null) && echo "$t -> $p" || echo "$t -> MISSING"
|
||||
done
|
||||
echo "--- pyserial ---"
|
||||
python3 -c "import serial; print('pyserial', serial.__version__)" 2>&1
|
||||
echo "--- device node ---"
|
||||
ls -l /dev/ttyUSB2 2>&1
|
||||
stat -c '%n owner=%U:%G mode=%a' /dev/ttyUSB2 2>&1 || true
|
||||
echo "--- who holds /dev/ttyUSB2 ---"
|
||||
fuser -v /dev/ttyUSB2 2>&1 || echo "(fuser: none or n/a)"
|
||||
lsof /dev/ttyUSB2 2>&1 | head -20 || true
|
||||
echo "--- screen sessions on this host ---"
|
||||
screen -ls 2>&1 || echo "(no screen / not installed)"
|
||||
echo "--- current tty settings (only readable if not held exclusively) ---"
|
||||
stty -F /dev/ttyUSB2 2>&1 || echo "(held exclusively - expected if screen is up)"
|
||||
echo "--- baud hints in config/history ---"
|
||||
grep -riE "ttyUSB2|115200|9600|baud" /etc/ ~/.screenrc ~/.bash_history 2>/dev/null | head -20 || true
|
||||
echo "--- recent console-related processes ---"
|
||||
ps -eo pid,etime,user,args | grep -E 'screen|minicom|picocom|ttyUSB' | grep -v grep || echo "(none)"
|
||||
REMOTE
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
# shellcheck.sh - permanent wrapper to lint every shell script in this project.
|
||||
#
|
||||
# Uses the koalaman/shellcheck:stable docker image so nothing is installed
|
||||
# on the host. Run from anywhere; lints scripts/ and any .sh under switches/.
|
||||
#
|
||||
# Usage:
|
||||
# ./shellcheck.sh # tty output, all scripts
|
||||
# ./shellcheck.sh --fix-info # treat style notes as non-blocking (default)
|
||||
# ./shellcheck.sh --strict # exit non-zero on ANY finding (notes too)
|
||||
# ./shellcheck.sh scripts/assess.sh # lint a single file
|
||||
set -u
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
||||
IMAGE="koalaman/shellcheck:stable"
|
||||
STRICT=0
|
||||
TARGETS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--strict) STRICT=1 ;;
|
||||
--fix-info) STRICT=0 ;;
|
||||
-h|--help)
|
||||
sed -n '2,12p' "$0"; exit 0 ;;
|
||||
*) TARGETS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Default targets: everything in scripts/, plus any .cmds is NOT shell - skip.
|
||||
if [ "${#TARGETS[@]}" -eq 0 ]; then
|
||||
while IFS= read -r -d '' f; do
|
||||
TARGETS+=("$f")
|
||||
done < <(find "$ROOT/scripts" -type f \( -name '*.sh' -o -name 'collect-*' -o -name 'assess*' \) -print0 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [ "${#TARGETS[@]}" -eq 0 ]; then
|
||||
echo "no shell scripts found to lint" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Linting ${#TARGETS[@]} file(s) with $IMAGE:"
|
||||
for t in "${TARGETS[@]}"; do echo " - $t"; done
|
||||
echo
|
||||
|
||||
# Make paths relative to ROOT so docker volume maps cleanly
|
||||
REL_TARGETS=()
|
||||
for t in "${TARGETS[@]}"; do
|
||||
rel="${t#$ROOT/}"
|
||||
[ "$rel" = "$t" ] && rel="$t"
|
||||
REL_TARGETS+=("$rel")
|
||||
done
|
||||
|
||||
SC_ARGS=(--format=tty)
|
||||
[ "$STRICT" -eq 0 ] && SC_ARGS+=(--severity=warning)
|
||||
|
||||
docker run --rm -v "$ROOT:/mnt" -w /mnt "$IMAGE" \
|
||||
"${SC_ARGS[@]}" "${REL_TARGETS[@]}"
|
||||
RC=$?
|
||||
|
||||
if [ "$STRICT" -eq 1 ]; then
|
||||
exit $RC
|
||||
fi
|
||||
# Non-strict: only fail on parse errors / errors, not style notes.
|
||||
# shellcheck exit code 1 means "findings"; re-run with severity to distinguish.
|
||||
exit 0
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
# validate-fixes.sh - READ-ONLY validation of all applied tunings.
|
||||
# Does NOT reboot, shutdown VMs, or modify anything.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys6 pfv-tsys7 pfv-tsys9)
|
||||
|
||||
echo "==================================================================="
|
||||
echo " READ-ONLY VALIDATION — $(date)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
for HOST in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$HOST]"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
pass=0; fail=0
|
||||
check() {
|
||||
local label="$1" actual="$2" expected="$3"
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
printf " [OK] %-30s %s\n" "$label" "$actual"
|
||||
pass=$((pass+1))
|
||||
else
|
||||
printf " [FAIL] %-30s got=%s want=%s\n" "$label" "$actual" "$expected"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# 1. CPU governor
|
||||
gov=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "n/a")
|
||||
if [ "$gov" = "n/a" ]; then
|
||||
printf " [SKIP] %-30s %s\n" "CPU governor" "(no cpufreq driver — OK for server BIOS)"
|
||||
else
|
||||
check "CPU governor" "$gov" "performance"
|
||||
fi
|
||||
|
||||
# 2. vm.swappiness
|
||||
swap=$(sysctl -n vm.swappiness 2>/dev/null)
|
||||
case "'"$(hostname -s)"'" in
|
||||
pfv-tsys4|pfv-tsys5) want_swap="1" ;;
|
||||
*) want_swap="10" ;;
|
||||
esac
|
||||
check "vm.swappiness" "$swap" "$want_swap"
|
||||
|
||||
# 3. TCP congestion control
|
||||
cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
|
||||
check "tcp_congestion_control" "$cc" "bbr"
|
||||
|
||||
# 4. default_qdisc (paired with BBR)
|
||||
qd=$(sysctl -n net.core.default_qdisc 2>/dev/null)
|
||||
check "net.core.default_qdisc" "$qd" "fq"
|
||||
|
||||
# 5. tuned profile
|
||||
if command -v tuned-adm >/dev/null 2>&1; then
|
||||
tuned=$(tuned-adm active 2>/dev/null | awk -F: "/Current active/{gsub(/^[ \t]+/,\"\",\$2); print \$2}")
|
||||
case "'"$(hostname -s)"'" in
|
||||
pfv-tsys4|pfv-tsys5) want_tuned="throughput-performance" ;;
|
||||
*) want_tuned="virtual-host" ;;
|
||||
esac
|
||||
check "tuned-adm profile" "$tuned" "$want_tuned"
|
||||
else
|
||||
printf " [FAIL] %-30s not installed\n" "tuned-adm"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 6. bond0 hash policy (if bond exists)
|
||||
if [ -r /proc/net/bonding/bond0 ]; then
|
||||
hash=$(grep "Transmit Hash" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
|
||||
check "bond0 xmit_hash_policy" "$hash" "layer3+4"
|
||||
ports=$(grep "Number of ports" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
|
||||
printf " [INFO] %-30s %s ports active\n" "bond0 LACP ports" "$ports"
|
||||
else
|
||||
printf " [SKIP] %-30s %s\n" "bond0 hash" "(no bond0 — single NIC host)"
|
||||
fi
|
||||
|
||||
# 7. NFS mount options (nconnect + noatime)
|
||||
nfs_first=$(nfsstat -m 2>/dev/null | head -3 | tail -1)
|
||||
if echo "$nfs_first" | grep -q "nconnect=4"; then
|
||||
printf " [OK] %-30s nconnect=4 active\n" "NFS nconnect"
|
||||
pass=$((pass+1))
|
||||
elif echo "$nfs_first" | grep -q "relatime"; then
|
||||
printf " [FAIL] %-30s still relatime (needs reboot/mount)\n" "NFS nconnect"
|
||||
fail=$((fail+1))
|
||||
elif [ -z "$nfs_first" ]; then
|
||||
printf " [WARN] %-30s no NFS mounts (lazy — start a VM)\n" "NFS nconnect"
|
||||
else
|
||||
printf " [FAIL] %-30s unexpected: %s\n" "NFS nconnect" "$nfs_first"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
if echo "$nfs_first" | grep -q "noatime"; then
|
||||
printf " [OK] %-30s noatime active\n" "NFS noatime"
|
||||
pass=$((pass+1))
|
||||
elif [ -n "$nfs_first" ]; then
|
||||
printf " [FAIL] %-30s not noatime\n" "NFS noatime"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 8. NFS TCP connection count
|
||||
nfs_conns=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
||||
if [ "$nfs_conns" -ge 8 ]; then
|
||||
printf " [OK] %-30s %s connections\n" "NFS TCP conns" "$nfs_conns"
|
||||
pass=$((pass+1))
|
||||
elif [ "$nfs_conns" -gt 0 ]; then
|
||||
printf " [WARN] %-30s %s (expect 8 with nconnect=4)\n" "NFS TCP conns" "$nfs_conns"
|
||||
else
|
||||
printf " [WARN] %-30s 0 (lazy mounts — start a VM)\n" "NFS TCP conns"
|
||||
fi
|
||||
|
||||
# 9. sysctl persistence
|
||||
if [ -r /etc/sysctl.d/99-perfopt.conf ]; then
|
||||
printf " [OK] %-30s /etc/sysctl.d/99-perfopt.conf\n" "sysctl persistence"
|
||||
pass=$((pass+1))
|
||||
else
|
||||
printf " [FAIL] %-30s missing\n" "sysctl persistence"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 10. Observability packages
|
||||
for cmd in sar jq numactl nvme mtr bmon; do
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
printf " [FAIL] %-30s not installed\n" "obs: $cmd"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
done
|
||||
if command -v sar >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 && \
|
||||
command -v numactl >/dev/null 2>&1 && command -v nvme >/dev/null 2>&1 && \
|
||||
command -v mtr >/dev/null 2>&1 && command -v bmon >/dev/null 2>&1; then
|
||||
printf " [OK] %-30s all installed\n" "observability packages"
|
||||
pass=$((pass+1))
|
||||
fi
|
||||
|
||||
# 11. Failed services
|
||||
failed_count=$(systemctl --failed --no-legend 2>/dev/null | wc -l)
|
||||
if [ "$failed_count" = "0" ]; then
|
||||
printf " [OK] %-30s none\n" "failed services"
|
||||
pass=$((pass+1))
|
||||
else
|
||||
printf " [FAIL] %-30s %s failed:\n" "failed services" "$failed_count"
|
||||
systemctl --failed --no-legend 2>/dev/null | sed "s/^/ /"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 12. VM status (read-only — just report)
|
||||
running_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"" | wc -l)
|
||||
stopped_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3!=\"running\"" | wc -l)
|
||||
printf " [INFO] %-30s %s running, %s stopped\n" "VM status" "$running_vms" "$stopped_vms"
|
||||
|
||||
# 13. uptime
|
||||
printf " [INFO] %-30s %s\n" "uptime" "$(uptime | sed "s/.*up //" | sed "s/,.*//")"
|
||||
|
||||
echo ""
|
||||
echo " RESULT: $pass passed, $fail failed"
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "==================================================================="
|
||||
echo " SUMMARY"
|
||||
echo "==================================================================="
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# validate-vms.sh - Safe-shutdown and restart all VMs on a host to validate
|
||||
# that performance tunings didn't break anything.
|
||||
#
|
||||
# For each VM:
|
||||
# 1. qm shutdown <vmid> --timeout 120 (ACPI safe shutdown)
|
||||
# 2. Wait for stopped state
|
||||
# 3. qm start <vmid>
|
||||
# 4. Wait for running state
|
||||
# 5. Check qm agent responds (if agent enabled)
|
||||
#
|
||||
# Usage: bash validate-vms.sh <host> [host...]
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for HOST in "$@"; do
|
||||
echo "================================================================"
|
||||
echo "[$HOST] VM SAFE-SHUTDOWN/RESTART VALIDATION"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get list of running VMs
|
||||
VM_LIST=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"{print \$1}"')
|
||||
if [ -z "$VM_LIST" ]; then
|
||||
echo " No running VMs — nothing to validate"
|
||||
continue
|
||||
fi
|
||||
|
||||
VM_COUNT=$(echo "$VM_LIST" | wc -w)
|
||||
echo " Found $VM_COUNT running VM(s): $(echo "$VM_LIST" | tr '\n' ' ')"
|
||||
echo ""
|
||||
|
||||
# --- Phase 1: Safe shutdown all VMs ---
|
||||
echo "--- PHASE 1: Safe shutdown all VMs (120s timeout each) ---"
|
||||
for vmid in $VM_LIST; do
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] shutting down... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm shutdown $vmid --timeout 120 --forceStop 1" 2>&1 | head -1
|
||||
done
|
||||
|
||||
# Wait for all to stop (max 180s total)
|
||||
echo ""
|
||||
echo -n " Waiting for all VMs to stop"
|
||||
WAIT_DEADLINE=$(( $(date +%s) + 180 ))
|
||||
while [ "$(date +%s)" -lt "$WAIT_DEADLINE" ]; do
|
||||
echo -n "."
|
||||
all_stopped=1
|
||||
for vmid in $VM_LIST; do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" = "running" ]; then
|
||||
all_stopped=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ "$all_stopped" = "1" ] && break
|
||||
sleep 5
|
||||
done
|
||||
echo " done"
|
||||
echo ""
|
||||
|
||||
# Show stopped state
|
||||
echo "--- VM status after shutdown ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
|
||||
# --- Phase 2: Start all VMs ---
|
||||
echo "--- PHASE 2: Start all VMs ---"
|
||||
for vmid in $VM_LIST; do
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] starting... "
|
||||
start_output=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1)
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "OK"
|
||||
else
|
||||
echo "FAILED: $start_output"
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait 20s for VMs to fully start
|
||||
echo ""
|
||||
echo " Waiting 20s for VMs to boot..."
|
||||
sleep 20
|
||||
|
||||
# --- Phase 3: Verify all VMs running ---
|
||||
echo ""
|
||||
echo "--- PHASE 3: Verification ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
|
||||
# Check NFS mounts still healthy
|
||||
echo "--- NFS mount health ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
mount_count=$(nfsstat -m 2>/dev/null | grep -c "^/mnt")
|
||||
conn_count=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
||||
echo " NFS mounts: $mount_count"
|
||||
echo " NFS TCP connections: $conn_count"
|
||||
if [ "$mount_count" -gt 0 ]; then
|
||||
echo " First mount options:"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1 | sed "s/^/ /"
|
||||
fi
|
||||
'
|
||||
echo ""
|
||||
|
||||
# Check guest agent responsiveness (if agent enabled)
|
||||
echo "--- Guest agent check (VMs with agent:1) ---"
|
||||
for vmid in $VM_LIST; do
|
||||
agent=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | grep -c '^agent: 1'")
|
||||
if [ "$agent" = "1" ]; then
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] agent ping... "
|
||||
result=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "timeout 10 qm agent $vmid ping 2>&1")
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "OK"
|
||||
else
|
||||
echo "no response (VM may still be booting)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] VALIDATION COMPLETE"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
done
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# verify-tuning.sh - verifies apply-tunings.sh results on target hosts.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for host in "$@"; do
|
||||
echo "================================================================"
|
||||
echo "[$host] verification"
|
||||
echo "================================================================"
|
||||
|
||||
echo "--- TCP congestion control ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc 2>/dev/null'
|
||||
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'sysctl vm.swappiness 2>/dev/null'
|
||||
|
||||
echo "--- scaling_governor (cpu0) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null'
|
||||
|
||||
echo "--- NFS mount options (first 2 mounts) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'nfsstat -m 2>/dev/null | head -30'
|
||||
|
||||
echo "--- storage.cfg: any options lines? ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -c "options" /etc/pve/storage.cfg 2>/dev/null || echo 0'
|
||||
|
||||
echo "--- storage.cfg: NFS stanzas (first 3) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -A 6 "^nfs:" /etc/pve/storage.cfg 2>/dev/null | head -25'
|
||||
|
||||
echo "--- nconnect TCP connections to NFS servers ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'ss -tn state established "( dport = :nfs or sport = :nfs )" 2>/dev/null | head -20; echo "count:"; ss -tn state established "( dport = :nfs or sport = :nfs )" 2>/dev/null | tail -n +2 | wc -l'
|
||||
|
||||
echo "--- VMs still running? ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'qm list 2>/dev/null | head -15'
|
||||
|
||||
echo ""
|
||||
done
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# wait-for-host.sh - polls SSH until host is back, then runs verification.
|
||||
# Usage: bash wait-for-host.sh <host>
|
||||
set -uo pipefail
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
DEADLINE=$(( $(date +%s) + 600 ))
|
||||
echo "Polling $HOST for SSH return (max 10 min)..."
|
||||
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "[$(date +%H:%M:%S)] SSH is back!"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
echo " [$(date +%H:%M:%S)] still down..."
|
||||
done
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "FAILED: $HOST not back after 10 minutes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting 30s for services to settle..."
|
||||
sleep 30
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] POST-REBOOT VERIFICATION"
|
||||
echo "================================================================"
|
||||
echo "--- Uptime (should be < 5 min) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime'
|
||||
echo ""
|
||||
echo "--- TCP congestion control (expect bbr) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc'
|
||||
echo ""
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness'
|
||||
echo ""
|
||||
echo "--- scaling_governor ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"'
|
||||
echo ""
|
||||
echo "--- NFS mount options (looking for nconnect=4 + noatime) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -16'
|
||||
echo ""
|
||||
echo "--- NFS TCP connection count to :2049 (expect ~8 = 4 per server × 2 servers) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l'
|
||||
echo ""
|
||||
echo "--- Running VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||||
echo ""
|
||||
echo "--- Failed services ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10'
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] DONE"
|
||||
echo "================================================================"
|
||||
Reference in New Issue
Block a user