KNEL/perf
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: initial import from KNEL/PFVCluster@041d311 [#769]

Browse Source 
Split per O&M lane work order. Full history: KNEL/PFVCluster.
https://projects.knownelement.com/issues/769#note-4152
This commit is contained in:
mrcharles
2026-09-03 21:39:35 -05:00
commit 7a899b8ac6
67 changed files with 7166 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/add-datanet-nics.sh
+96
View File
@@ -0,0 +1,96 @@
#!/bin/bash
###############################################################################
# add-datanet-nics.sh
#
# Adds a second virtio NIC (net1) to all k8s + ultix VMs, bridged to the
# storage/datanet network (VLAN 1000). Records the assigned IP in phpIPAM.
#
# Runs from the workstation — uses tests/remote.sh to reach each hypervisor.
# Hot-adds the NIC (no VM downtime). Guest-side IP config must be done
# separately (the guest OS needs the IP configured on the new interface).
#
# Usage:
# bash add-datanet-nics.sh # add NICs (hot-add, no reboot)
# bash add-datanet-nics.sh --status # show current state only
#
# IP allocations (phpIPAM VPTechOps, subnet 10.100.100.0/24):
# .10 = cnode1 (102/tsys1)
# .11 = cnode2 (705/tsys7)
# .12 = cnode3 (603/tsys6)
# .13 = wnode-tsys3 (313/tsys3)
# .14 = wnode-tsys5 (500/tsys5)
# .15 = wnode-tsys6 (601/tsys6)
# .16 = wnode-tsys7 (701/tsys7)
# .17 = wnode-tsys9 (905/tsys9)
# .18 = ultix-streaming (5111/tsys5)
# .19 = ultix-offstage (5112/tsys5)
#
# Related: Redmine [#396]
###############################################################################
set -euo pipefail
# VM definitions: VMID HOST BRIDGE IP HOSTNAME
VMS=(
"102 tsys1 datanet 10.100.100.10 pfv-k8s-cnode1"
"705 tsys7 datanet 10.100.100.11 pfv-k8s-cnode2"
"603 tsys6 storagenet 10.100.100.12 pfv-k8s-cnode3"
"313 tsys3 datanet 10.100.100.13 pfv-k8s-wnode-tsys3"
"500 tsys5 datanet 10.100.100.14 pfv-k8s-wnode-tsys5"
"601 tsys6 storagenet 10.100.100.15 pfv-k8s-wnode-tsys6"
"701 tsys7 datanet 10.100.100.16 pfv-k8s-wnode-tsys7"
"905 tsys9 datanet 10.100.100.17 pfv-k8s-wnode-tsys9"
"5111 tsys5 datanet 10.100.100.18 ultix-streaming"
"5112 tsys5 datanet 10.100.100.19 ultix-offstage"
)
ACTION="${1:-add}"
echo "==================================================================="
echo " add-datanet-nics — [#396]"
echo " mode: ${ACTION}"
echo "==================================================================="
echo ""
for entry in "${VMS[@]}"; do
read -r vmid host bridge ip hostname <<< "$entry"
prox_host="pfv-${host}"
echo "--- ${hostname} (VM ${vmid} on ${prox_host}) ---"
if [ "$ACTION" = "--status" ]; then
# Show current NIC state
PROX_HOST="$prox_host" bash tests/remote.sh prox \
"qm config ${vmid} 2>/dev/null | grep -E '^net|^name'" 2>&1
echo ""
continue
fi
# Check if net1 already exists
existing=$(PROX_HOST="$prox_host" bash tests/remote.sh prox \
"qm config ${vmid} 2>/dev/null | grep '^net1'" 2>&1 || true)
if [ -n "$existing" ]; then
echo " net1 already exists: ${existing}"
echo " Skipping."
echo ""
continue
fi
# Hot-add net1 bridged to the storage network
echo " Adding net1 (bridge=${bridge}, IP=${ip})..."
if PROX_HOST="$prox_host" bash tests/remote.sh prox \
"qm set ${vmid} -net1 virtio,bridge=${bridge}" 2>&1; then
echo " NIC added. Verify with: qm config ${vmid} | grep net1"
else
echo " FAILED — check error above"
fi
echo ""
done
if [ "$ACTION" != "--status" ]; then
echo "==================================================================="
echo " NICs added. Guest-side IP config still needed."
echo " Each guest needs the IP configured on the new interface."
echo " IPs are allocated in phpIPAM (VPTechOps / 10.100.100.0/24)."
echo "==================================================================="
fi
scripts/apply-bond-hash.sh
Executable
+141
View File
@@ -0,0 +1,141 @@
#!/bin/bash
###############################################################################
# apply-bond-hash.sh
#
# Adds bond-xmit-hash-policy layer3+4 to bond0 in /etc/network/interfaces,
# then reloads networking with ifreload -a.
#
# SSH survivability: this is safe IF your SSH session is on vmbr0/nic0
# (management network), NOT on bond0/datanet (storage network).
# tsys7's topology confirms this: SSH comes in on vmbr0 (nic0).
#
# Safety:
# - Dry-run by default (--apply to commit)
# - Full backup of /etc/network/interfaces
# - Generates rollback script
# - Does NOT reboot — uses ifreload -a which is hot-reload
###############################################################################
set -euo pipefail
HOST="$(hostname -s)"
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
ROLLBACK="/root/perfopt-bond-rollback-${TS_SHORT}.sh"
ACTION="${1:-dryrun}"
[ "$ACTION" = "--apply" ] && ACTION="apply" || ACTION="dryrun"
mkdir -p "$BACKUP_DIR"
echo "==================================================================="
echo " apply-bond-hash — $HOST"
echo " mode: $ACTION"
echo "==================================================================="
if [ ! -r /etc/network/interfaces ]; then
echo "FATAL: /etc/network/interfaces not readable"
exit 1
fi
# Check if bond0 exists in the config
if ! grep -q 'bond0' /etc/network/interfaces; then
echo "No bond0 found in /etc/network/interfaces — nothing to do."
exit 0
fi
# Check if the hash policy is already set
if grep -q 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces; then
echo "bond-xmit-hash-policy already present:"
grep 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces
echo "Checking value..."
if grep -q 'layer3+4' /etc/network/interfaces; then
echo "Already set to layer3+4 — nothing to do."
exit 0
fi
fi
# Show current bond state
echo ""
echo "--- Current bond0 state ---"
cat /proc/net/bonding/bond0 2>/dev/null | head -15 || echo "(bond0 not up or not present)"
# Back up
cp -a /etc/network/interfaces "$BACKUP_DIR/interfaces"
# Generate rollback script
cat > "$ROLLBACK" <<EOF
#!/bin/bash
# Rollback for bond-xmit-hash-policy change
# Restores original /etc/network/interfaces and reloads
set -euo pipefail
cp -a "$BACKUP_DIR/interfaces" /etc/network/interfaces
echo "Restored /etc/network/interfaces"
echo "Reloading networking..."
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
echo "Done. bond0 hash policy reverted to original."
EOF
chmod +x "$ROLLBACK"
echo ""
echo "--- Proposed change ---"
echo "Add line ' bond-xmit-hash-policy layer3+4' to the bond0 stanza."
echo ""
if [ "$ACTION" != "apply" ]; then
echo "DRY RUN — no changes made."
echo "To commit: bash $0 --apply"
echo "Rollback script (pre-generated): $ROLLBACK"
exit 0
fi
# Apply: use sed to insert bond-xmit-hash-policy after bond-mode line
# The bond0 stanza looks like:
# auto bond0
# iface bond0 inet manual
# bond-slaves nic1 nic2
# bond-miimon 100
# bond-mode 802.3ad
#
# We insert after the bond-mode line.
echo "Applying..."
# Check if bond-mode line exists (various formats)
if grep -qE '^\s*bond-mode\s+802.3ad' /etc/network/interfaces; then
# Insert after bond-mode 802.3ad line
sed -i '/^\s*bond-mode\s+802\.3ad/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode line."
elif grep -qE '^\s*bond-mode\s+4' /etc/network/interfaces; then
sed -i '/^\s*bond-mode\s+4/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode 4 line."
else
echo "Could not find bond-mode line — inserting after bond-slaves line instead."
sed -i '/^\s*bond-slaves/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
fi
# Show the result
echo ""
echo "--- Updated bond0 stanza ---"
awk '/^auto bond0/,/^$/' /etc/network/interfaces
echo ""
echo "--- Reloading networking (ifreload -a) ---"
echo "SSH should survive (it's on vmbr0/nic0, not bond0)..."
ifreload -a 2>&1 || {
echo "ifreload failed, trying systemctl restart networking..."
systemctl restart networking 2>&1
}
# Wait a moment for bond to renegotiate
echo "Waiting 5s for LACP to renegotiate..."
sleep 5
echo ""
echo "--- Post-change bond0 state ---"
cat /proc/net/bonding/bond0 2>/dev/null | head -20
echo ""
echo "==================================================================="
echo " DONE."
echo " Backup: $BACKUP_DIR/interfaces"
echo " Rollback: bash $ROLLBACK"
echo "==================================================================="
scripts/apply-tunings.sh
Executable
+439
View File
@@ -0,0 +1,439 @@
#!/bin/bash
###############################################################################
# apply-tunings.sh
#
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
# 1. Sets scaling_governor=performance on every CPU
# 2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
# 3. Enables tcp_bbr module + sets congestion control
# 4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
# 5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
# 6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
# 7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
# restart which drops the host. Provided as a separate --emit-bond-patch
# flag that PRINTS the change but does not apply it.)
#
# Safety features:
# - Dry-run mode by default (--apply to commit)
# - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
# - Per-host behaviour: detects storage hosts by hostname and applies the
# right profile (tsys4, tsys5 = storage; others = VM hosts).
# - Generates a /root/perfopt-rollback.sh that undoes everything.
# - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
#
# Usage:
# bash apply-tunings.sh # dry run, show what would change
# bash apply-tunings.sh --apply # commit changes
# bash apply-tunings.sh --rollback # restore from latest backup
# bash apply-tunings.sh --emit-bond-patch # show bond0 hash change (no apply)
###############################################################################
set -u
umask 022
HOST="$(hostname -s)"
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
ACTION="dryrun"
SKIP_NFS=0
while [ $# -gt 0 ]; do
case "$1" in
--apply) ACTION="apply" ;;
--rollback) ACTION="rollback" ;;
--emit-bond-patch) ACTION="bond-patch" ;;
--no-nfs) SKIP_NFS=1 ;;
-h|--help) sed -n '2,35p' "$0"; exit 0 ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac
shift
done
# Detect host role from hostname
case "$HOST" in
pfv-tsys4|pfv-tsys5)
HOST_ROLE="storage"
TUNED_PROFILE="network-throughput"
SWAPPINESS="1"
;;
pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
HOST_ROLE="vmhost"
TUNED_PROFILE="virtual-host"
SWAPPINESS="10"
;;
*)
HOST_ROLE="unknown"
TUNED_PROFILE="virtual-host"
SWAPPINESS="10"
;;
esac
# --- rollback path ---------------------------------------------------------
if [ "$ACTION" = "rollback" ]; then
echo "Looking for latest backup under /root/perfopt-backup-* ..."
latest=""
while IFS= read -r d; do
latest="$d"
done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
if [ -z "$latest" ]; then
echo "No backup found under /root/perfopt-backup-*" >&2
exit 1
fi
rb="$latest/perfopt-rollback.sh"
if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
echo "Rollback script missing in $latest" >&2
exit 1
fi
echo "Rolling back using: $rb"
bash "$rb"
exit $?
fi
# --- bond patch print-only path -------------------------------------------
if [ "$ACTION" = "bond-patch" ]; then
echo "==================================================================="
echo " Proposed bond0 xmit_hash_policy change"
echo "==================================================================="
echo
if [ ! -r /etc/network/interfaces ]; then
echo "/etc/network/interfaces not readable"
exit 1
fi
if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
echo "No bond0 detected on this host — nothing to patch."
exit 0
fi
cat <<EOF
A manual edit to /etc/network/interfaces is required. The bond0 stanza
needs this line added (it defaults to layer2, which is wrong for storage):
bond-xmit-hash-policy layer3+4
After editing, you MUST reload networking for it to take effect:
ifreload -a # safe, brings interfaces down/up
# OR
systemctl restart networking # heavier, briefly drops connections
WARNING: applying this on a remote host over bond0 will briefly drop your
SSH session. Run from console/IPMI, or schedule a maintenance window.
The change is reversible by removing the line and reloading again.
EOF
exit 0
fi
# --- main path (dryrun or apply) ------------------------------------------
mkdir -p "$BACKUP_DIR"
# Emit the rollback script header
cat > "$ROLLBACK_SCRIPT" <<EOF
#!/bin/bash
# Auto-generated rollback for perfopt apply-tunings.sh
# Backup timestamp: $TS
# Backup dir: $BACKUP_DIR
# Generated on: $HOST
set -u
EOF
chmod +x "$ROLLBACK_SCRIPT"
backup_file() {
local f="$1"
if [ -e "$f" ]; then
cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
fi
}
# Helper: append a restore command to the rollback script
rb_restore() {
local f="$1"
local bk
bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
cat >> "$ROLLBACK_SCRIPT" <<EOF
if [ -r "$bk" ]; then
cp -a "$bk" "$f" && echo "restored $f"
else
echo "WARN: backup $bk missing — $f left as-is"
fi
EOF
}
# Helper: append a sysctl restore command
rb_sysctl_restore() {
local key="$1" orig_val="$2"
cat >> "$ROLLBACK_SCRIPT" <<EOF
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
EOF
}
# Helper: append a tuned-adm restore
rb_tuned_restore() {
local profile="$1"
cat >> "$ROLLBACK_SCRIPT" <<EOF
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
EOF
}
echo "==================================================================="
echo " perfopt apply-tunings — $HOST"
echo " mode: $ACTION"
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
echo " backup: $BACKUP_DIR"
echo " rollback: $ROLLBACK_SCRIPT"
echo "==================================================================="
echo
# ===========================================================================
# 1. scaling_governor → performance
# ===========================================================================
echo "--- 1. CPU scaling_governor → performance"
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
echo " current: $cur"
if [ "$ACTION" = "apply" ]; then
if [ "$cur" != "performance" ]; then
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
echo performance > "$c" 2>/dev/null || true
done
# Persist via systemd tmpfiles / sysctl fallback
cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
[Unit]
Description=Set CPU scaling_governor=performance (perfopt)
After=multi-user.target
[Service]
Type=oneshot
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable perfopt-cpu-performance.service 2>/dev/null
systemctl start perfopt-cpu-performance.service 2>/dev/null
echo " applied + persisted via systemd unit"
# Rollback
cat >> "$ROLLBACK_SCRIPT" <<EOF
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
rm -f /etc/systemd/system/perfopt-cpu-performance.service
systemctl daemon-reload
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
EOF
else
echo " already performance — no change"
fi
fi
else
echo " cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
fi
# ===========================================================================
# 2. vm.swappiness
# ===========================================================================
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
echo " current: $cur_swappiness"
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
# Persist via sysctl.d
backup_file /etc/sysctl.d/99-perfopt.conf
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
vm.swappiness = $SWAPPINESS
EOF
sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
echo " applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
rb_restore /etc/sysctl.d/99-perfopt.conf
cat >> "$ROLLBACK_SCRIPT" <<EOF
rm -f /etc/sysctl.d/99-perfopt.conf
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
echo "restored vm.swappiness=$cur_swappiness"
EOF
else
echo " no change"
fi
# ===========================================================================
# 3. TCP BBR + buffers (single sysctl.d file)
# ===========================================================================
echo "--- 3. TCP BBR + socket buffers"
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
echo " current: $cur_cc (available: $cur_avail)"
# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
TARGET_RMEM_MAX=134217728 # 128 MB
TARGET_WMEM_MAX=134217728
TARGET_RMEM_DEFAULT=26214400 # 25 MB
TARGET_WMEM_DEFAULT=26214400
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
TARGET_NETDEV_MAX_BACKLOG=250000
TARGET_SOMAXCONN=65535
if [ "$ACTION" = "apply" ]; then
# Ensure tcp_bbr module loads at boot
if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
backup_file /etc/modules-load.d/modules.conf
mkdir -p /etc/modules-load.d
cat >> /etc/modules-load.d/modules.conf <<EOF
# perfopt apply-tunings.sh — $TS
tcp_bbr
EOF
modprobe tcp_bbr 2>/dev/null || true
rb_restore /etc/modules-load.d/modules.conf
fi
backup_file /etc/sysctl.d/99-perfopt.conf
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
vm.swappiness = $SWAPPINESS
# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
net.ipv4.tcp_congestion_control = bbr
net.core.default_qdisc = fq
net.core.rmem_max = $TARGET_RMEM_MAX
net.core.wmem_max = $TARGET_WMEM_MAX
net.core.rmem_default = $TARGET_RMEM_DEFAULT
net.core.wmem_default = $TARGET_WMEM_DEFAULT
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
net.core.somaxconn = $TARGET_SOMAXCONN
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
EOF
sysctl --system >/dev/null 2>&1
echo " applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
# Rebuild rollback for sysctl
cat >> "$ROLLBACK_SCRIPT" <<EOF
# Restore original sysctl state
rm -f /etc/sysctl.d/99-perfopt.conf
EOF
rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
# Note: original buffer sizes not all captured; restoration is best-effort.
cat >> "$ROLLBACK_SCRIPT" <<EOF
echo "Note: net buffers restored to kernel defaults (original values not captured)."
echo "Run 'sysctl --system' to apply."
sysctl --system >/dev/null 2>&1 || true
EOF
fi
# ===========================================================================
# 4. tuned-adm profile
# ===========================================================================
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
if command -v tuned-adm >/dev/null 2>&1; then
cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
echo " current: $cur_profile"
if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/ /'
rb_tuned_restore "$cur_profile"
else
echo " no change"
fi
else
echo " tuned-adm not installed — skipping"
fi
# ===========================================================================
# 5. NFS mount option hardening
# Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
# Proxmox applies these on next mount/remount.
# Skipped with --no-nfs (for NFS servers where we don't want to remount).
# ===========================================================================
echo "--- 5. NFS mount options: add nconnect=4, noatime"
if [ "$SKIP_NFS" -eq 1 ]; then
echo " SKIPPED (--no-nfs specified)"
echo " Note: storage.cfg is cluster-wide; if edited on another host,"
echo " the options are already staged here and activate on next reboot."
else
if [ -r /etc/pve/storage.cfg ]; then
backup_file /etc/pve/storage.cfg
if grep -q '^nfs:' /etc/pve/storage.cfg; then
# Count how many nfs: stanzas lack the options
while IFS= read -r line; do
if echo "$line" | grep -q '^nfs:'; then
cur_stanza=$(echo "$line" | awk '{print $2}')
# Look ahead for the next few lines to see if options already set
echo " nfs stanza: $cur_stanza"
fi
done < /etc/pve/storage.cfg
# Check whether any nfs stanza already has options
if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
echo " some stanzas already have nconnect=4 — check manually"
else
echo " proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2' to each nfs stanza"
if [ "$ACTION" = "apply" ]; then
# Apply via perl for safety (in-place edit with backup)
if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
# Use a python helper for robust PVE storage.cfg editing
python3 - <<PYEOF
import re, pathlib
p = pathlib.Path("/etc/pve/storage.cfg")
text = p.read_text()
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
modified = 0
def fix(m):
global modified
block = m.group(1)
if 'options' in block:
return block
lines = block.rstrip('\n').split('\n')
lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2')
modified += 1
return '\n'.join(lines) + '\n'
new_text = pattern.sub(fix, text)
if modified:
p.write_text(new_text)
print(f" patched {modified} NFS stanzas with options line")
else:
print(" no NFS stanza needed patching (all already had options or no nfs: found)")
PYEOF
# Remount existing NFS mounts to pick up new options
echo " remounting NFS mounts to apply new options..."
while IFS= read -r mp; do
mount -o remount "$mp" 2>/dev/null && echo " remounted $mp" || echo " FAILED to remount $mp (will pick up on next mount)"
done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
rb_restore /etc/pve/storage.cfg
cat >> "$ROLLBACK_SCRIPT" <<EOF
# To fully undo NFS options, also remount:
while IFS= read -r mp; do
mount -o remount "\$mp" 2>/dev/null
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
echo "restored NFS mount options"
EOF
else
echo " ERROR: could not back up storage.cfg — aborting NFS patch"
fi
fi
fi
else
echo " no NFS stanzas in storage.cfg — skipping"
fi
else
echo " /etc/pve/storage.cfg not readable — skipping"
fi
fi
# ===========================================================================
# 6. Summary
# ===========================================================================
echo
echo "==================================================================="
if [ "$ACTION" = "apply" ]; then
echo " DONE. Backups in: $BACKUP_DIR"
echo " Rollback: bash $ROLLBACK_SCRIPT"
echo
echo " Next steps:"
echo " - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
echo " - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
echo " - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
echo " xmit_hash_policy change (requires network restart, do in"
echo " maintenance window)"
else
echo " DRY RUN — no changes made."
echo " To commit: bash apply-tunings.sh --apply"
fi
echo "==================================================================="
scripts/audit-guest-io.sh
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/bash
# Audit VM disk cache from inside guests — confirm writeback visible + I/O perf
set -uo pipefail
source "$(cd "$(dirname "$0")/../../.." && pwd)/k8s/env.sh"
echo "=== CNODES (via Tailscale) ==="
for ip in "${ALL_CNODES[@]}"; do
echo "--- $ip ---"
# shellcheck disable=SC2016 # heredoc-style remote command uses $() on the remote side
cn "$ip" '
echo " write_cache: $(cat /sys/block/sda/queue/write_cache 2>/dev/null)"
echo " scheduler: $(cat /sys/block/sda/queue/scheduler 2>/dev/null)"
echo " fsync (5x 1KB):"
for i in 1 2 3 4 5; do
t0=$(date +%s%N)
dd if=/dev/zero of=/tmp/.ft bs=1k count=1 conv=fsync 2>/dev/null
t1=$(date +%s%N)
echo -n " $(( (t1-t0)/1000000 )) ms"
done
echo
rm -f /tmp/.ft
' 2>&1 || echo " UNREACHABLE"
done
echo ""
echo "=== DNS INFRA VMs ==="
for host in netinfra01 netinfra02; do
echo "--- $host ---"
# shellcheck disable=SC2016 # heredoc-style remote command uses $() on the remote side
netinfra/dns-cluster-setup/remote-dns.sh "$host-root" '
echo " write_cache: $(cat /sys/block/sda/queue/write_cache 2>/dev/null)"
echo " fsync (3x 1KB):"
for i in 1 2 3; do
t0=$(date +%s%N)
dd if=/dev/zero of=/tmp/.ft bs=1k count=1 conv=fsync 2>/dev/null
t1=$(date +%s%N)
echo -n " $(( (t1-t0)/1000000 )) ms"
done
echo
rm -f /tmp/.ft
' 2>&1 || echo " UNREACHABLE"
done
scripts/audit-vm-disks.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/bash
# Audit all VM disk configs on a Proxmox host
set -uo pipefail
HOST="$1"
PROX_HOST="$HOST" bash tests/remote.sh prox '
qm list 2>/dev/null | tail -n +2 | while read -r line; do
vmid=$(echo "$line" | awk "{print \$1}")
name=$(echo "$line" | awk "{print \$2}")
status=$(echo "$line" | awk "{print \$3}")
echo "VMID=$vmid NAME=$name STATUS=$status"
qm config "$vmid" 2>/dev/null | grep -E "^(scsi|virtio|ide)[0-9]+:" | sed "s/^/ /"
echo ""
done
'
scripts/blacklist-zfs.sh
+55
View File
@@ -0,0 +1,55 @@
#!/bin/bash
# blacklist-zfs.sh — ZFS is unused on the tsys Proxmox hosts (zero pools):
# keep the module and its service family out of RAM and out of boot. [#737]
# Run ON a Proxmox host: tests/remote.sh prox-file proxmox/perf/scripts/blacklist-zfs.sh
# Reversible: rm /etc/modprobe.d/blacklist-zfs.conf && systemctl enable zfs.target zfs-zed ...
set -u
log() { echo "[zfs-blacklist] $*"; }
# Guard: never touch a host whose root filesystem is ZFS.
root_fs=$(findmnt -n -o FSTYPE / 2>/dev/null)
case "$root_fs" in
*zfs*) log "ABORT: root fs is '$root_fs' — this host needs ZFS at boot"; exit 1 ;;
esac
conf=/etc/modprobe.d/blacklist-zfs.conf
if [ ! -f "$conf" ]; then
{
echo "# ZFS unused on this host (no pools) — keep module out of RAM and boot [#737]"
echo "blacklist zfs"
} > "$conf"
log "wrote $conf"
else
log "blacklist already present"
fi
# Stop + disable the zfs service family (zed holds the module open; the
# import/mount/share units would fail or reload the module every boot).
systemctl disable --now \
zfs-zed.service zfs-import-cache.service zfs-import-scan.service \
zfs-import.target zfs-mount.service zfs-share.service \
zfs-volume-wait.service zfs-volumes.target zfs.target >/dev/null 2>&1
log "zfs service family disabled"
if lsmod | grep -q '^zfs'; then
if modprobe -r zfs 2>/dev/null; then
log "module unloaded (ARC reclaimed)"
else
log "WARNING: module still in use — will stay away after next reboot anyway"
fi
else
log "module not loaded"
fi
# Bake the blacklist into the initramfs so early boot cannot load it either.
if command -v update-initramfs >/dev/null 2>&1; then
if update-initramfs -u -k all >/dev/null 2>&1; then
log "initramfs updated"
else
log "WARNING: update-initramfs failed — blacklist applies from main boot only"
fi
fi
state=$(systemctl is-enabled zfs-zed 2>/dev/null)
log "done: root_fs=${root_fs:-?} zfs_zed=${state:-n/a} module=$(lsmod | grep -c '^zfs')"
scripts/check-rules.sh
Executable
+274
View File
@@ -0,0 +1,274 @@
#!/usr/bin/env bash
# check-rules.sh — project rule audit engine.
#
# Usage:
# bash scripts/check-rules.sh # full audit (verbose, includes slow checks)
# bash scripts/check-rules.sh --fast # fast audit (quiet, skips slow checks) — for pre-commit
# bash scripts/check-rules.sh --quiet # full audit, only prints failures
#
# Exit code: 0 = all rules pass (warnings are non-fatal), 1 = one or more FAILED.
#
# This is a generalized version of the rules engine proven in the
# RCEO-PersonalAssistant project. Add project-specific checks by appending
# `check "<desc>" "<pass|warn|fail>"` calls below.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
# shellcheck disable=SC1091
source "$HERE/lib/common.sh"
REPO_ROOT="$(repo_root)"
cd "$REPO_ROOT"
# --- argument parsing ---
RULE_FAST=false
RULE_VERBOSE=true
for arg in "$@"; do
case "$arg" in
--fast) RULE_FAST=true; RULE_VERBOSE=false ;;
--quiet) RULE_VERBOSE=false ;;
*) die "check-rules.sh: unknown argument '$arg'" ;;
esac
done
export RULE_FAST RULE_VERBOSE
init_counters
$RULE_VERBOSE && echo "=== Project Rule Audit ==="
TODAY="$(date +%Y-%m-%d)"
# ----------------------------------------------------------------------------
# 1. Shellcheck — every .sh/.bash must pass (zero warnings, incl. info-level).
# Runs in Docker so the host stays clean (no native shellcheck required).
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Shell scripts (shellcheck)"
mapfile -d '' SH_FILES < <(find . -path ./.git -prune -o -path ./.tmp -prune -o -path ./.crush -prune -o -path ./vendor -prune -o -path ./oam/librenms-agent/agent -prune -o -path ./archive -prune -o -path ./node_modules -prune -o \( -name '*.sh' -o -name '*.bash' \) -print0 2>/dev/null)
if [ "${#SH_FILES[@]}" -gt 0 ]; then
if have shellcheck; then
if shellcheck "${SH_FILES[@]}" >/dev/null 2>&1; then
check "All shell scripts pass shellcheck (host)" "pass"
else
check "shellcheck reports violations — run: shellcheck <file>" "fail"
fi
elif have docker; then
MNT_FILES=()
for f in "${SH_FILES[@]}"; do MNT_FILES+=("/mnt/${f#./}"); done
if docker run --rm -v "$REPO_ROOT:/mnt" koalaman/shellcheck:stable "${MNT_FILES[@]}" >/dev/null 2>&1; then
check "All shell scripts pass shellcheck (docker)" "pass"
else
check "shellcheck (docker) reports violations" "fail"
fi
else
check "No shellcheck or docker available to lint scripts" "warn"
fi
else
check "No shell scripts to lint" "pass"
fi
# ----------------------------------------------------------------------------
# 2. Docker image pinning — no ':latest' tags in compose or Dockerfiles.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Docker image pinning"
if grep -rqE '(image:|FROM).*:latest' --include='docker-compose*.y*ml' --include='Dockerfile*' . 2>/dev/null; then
check "No ':latest' image tags (pin everything)" "fail"
else
check "No ':latest' image tags" "pass"
fi
# ----------------------------------------------------------------------------
# 2b. Container naming — every service in a docker-compose file MUST set an
# explicit container_name (never rely on Docker's default <dir>_<n>).
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Container naming"
COMPOSE_FILES="$(find . -path ./.git -prune -o \( -name 'docker-compose*.yml' -o -name 'docker-compose*.yaml' -o -name 'compose.yml' -o -name 'compose.yaml' \) -print 2>/dev/null || true)"
if [ -n "$COMPOSE_FILES" ]; then
BAD=0
while IFS= read -r cf; do
[ -n "$cf" ] || continue
# Count top-level service keys (2-space indent under services:) and
# compare against the number of container_name: declarations.
svc_count=$(awk '/^services:/{f=1;next} f&&/^[^[:space:]]/{f=0} f&&/^[[:space:]]{2}[[:alnum:]_-]+:[[:space:]]*$/{c++} END{print c+0}' "$cf")
cn_count=$(grep -cE '^[[:space:]]*container_name:' "$cf" 2>/dev/null || echo 0)
if [ "${svc_count:-0}" -gt 0 ] && [ "$cn_count" -lt "$svc_count" ]; then
BAD=$((BAD + 1))
fi
done <<EOF
$COMPOSE_FILES
EOF
if [ "$BAD" -eq 0 ]; then
check "All compose services set container_name" "pass"
else
check "$BAD compose file(s) with services missing container_name" "fail"
fi
else
check "No compose files (container-name check skipped)" "pass"
fi
# ----------------------------------------------------------------------------
# 3. Required-files manifest — the files every project using this template owns.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Required files"
REQUIRED_FILES="AGENTS.md STATUS.md .env.example scripts/check-rules.sh scripts/setup-hooks.sh"
REQUIRED_FILES="$REQUIRED_FILES ${PROJECT_REQUIRED_FILES:-}"
for f in $REQUIRED_FILES; do
if [ -f "$f" ]; then check "$f exists" "pass"; else check "$f MISSING" "fail"; fi
done
if compgen -G 'questions-v*.md' > /dev/null; then
latest_q="$(find . -maxdepth 1 -name 'questions-v*.md' -printf '%f\n' | sort -V | tail -1)"
check "questions-v*.md exists ($latest_q)" "pass"
else
check "questions-v*.md MISSING" "fail"
fi
# ----------------------------------------------------------------------------
# 4. Doc freshness — STATUS.md touched today.
# Warning (not failure): staleness is a signal, not a break.
# Redmine is the system of record for work; Discourse for docs. STATUS.md is
# a scratchpad only — see BASELINE-PROMPT.md §3, §8.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Doc freshness"
if [ -f STATUS.md ]; then
STATUS_DATE="$(grep -oE 'Last updated: [0-9]{4}-[0-9]{2}-[0-9]{2}' STATUS.md | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' || echo unknown)"
if [ "$STATUS_DATE" = "$TODAY" ]; then
check "STATUS.md updated today ($STATUS_DATE)" "pass"
else
check "STATUS.md is stale (last: $STATUS_DATE, today: $TODAY) — update it" "warn"
fi
else
check "STATUS.md MISSING" "fail"
fi
# ----------------------------------------------------------------------------
# 4b. Discourse pointer-header policy (MANDATORY).
# Discourse is the system of record for documentation. In-repo .md files are
# stubs that point to a Discourse topic URL. Operational files exempt.
# Override exemptions via PROJECT_DOC_EXEMPT (space-separated globs of
# basenames) and the Discourse host via PROJECT_DISCOURSE_HOST.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Discourse pointer-header"
DISCOURSE_HOST="${PROJECT_DISCOURSE_HOST:-community.turnsys.com}"
DOC_EXEMPT="${PROJECT_DOC_EXEMPT:-AGENTS.md STATUS.md WORKING.md README.md ADOPTING.md LICENSE .env.example questions-v*.md BASELINE-PROMPT.md PATTERNS.md}"
POINTER_MISSING=0
while IFS= read -r -d '' f; do
base="$(basename "$f")"
exempt=false
for pat in $DOC_EXEMPT; do
# shellcheck disable=SC2254
case "$base" in $pat) exempt=true; break ;; esac
done
[ "$exempt" = true ] && continue
if ! grep -qF "$DISCOURSE_HOST" "$f" 2>/dev/null; then
if [ "$POINTER_MISSING" -eq 0 ]; then
$RULE_VERBOSE && printf ' %s\n' "Missing $DISCOURSE_HOST URL in:"
fi
POINTER_MISSING=$((POINTER_MISSING + 1))
$RULE_VERBOSE && printf ' %s\n' "$f"
fi
done < <(find . -path ./.git -prune -o -path ./.crush -prune -o -path ./.tmp -prune -o -path ./vendor -prune -o -path ./archive -prune -o -name '*.md' -print0 2>/dev/null)
if [ "$POINTER_MISSING" -eq 0 ]; then
check "All non-exempt .md cite Discourse ($DISCOURSE_HOST)" "pass"
else
check "$POINTER_MISSING .md file(s) missing Discourse pointer (see BASELINE-PROMPT.md §3)" "fail"
fi
# ----------------------------------------------------------------------------
# 5. Git state — uncommitted changes are a warning (the pre-push hook hardens
# this where it matters).
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Git state"
if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
if git diff --quiet && git diff --cached --quiet; then
check "Working tree clean" "pass"
else
check "Uncommitted changes present" "warn"
fi
else
check "Not a git repo (git checks skipped)" "pass"
fi
# ----------------------------------------------------------------------------
# 6. Hooks installed — self-check that git hooks were set up.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Git hooks"
if [ -f .git/hooks/pre-commit ]; then
check "pre-commit hook installed" "pass"
else
check "pre-commit NOT installed (run: bash scripts/setup-hooks.sh)" "warn"
fi
if [ -f .git/hooks/pre-push ]; then
check "pre-push hook installed" "pass"
else
check "pre-push NOT installed (run: bash scripts/setup-hooks.sh)" "warn"
fi
# ----------------------------------------------------------------------------
# 7. WORKING.md completion — no unchecked tasks may remain at commit time.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Task completion"
if [ -f WORKING.md ]; then
UNCHECKED="$(grep -cF -- '- [ ]' WORKING.md || true)"
if [ "$UNCHECKED" -eq 0 ]; then
check "WORKING.md has no unchecked tasks" "pass"
else
check "WORKING.md has ${UNCHECKED} unchecked task(s) — finish them before committing" "fail"
fi
else
check "WORKING.md absent (no active task tracker)" "pass"
fi
# ----------------------------------------------------------------------------
# 8. CNW markers — empty `CNW:` markers flag unresolved questions for the human.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Unresolved questions"
EMPTY_CNW="$(grep -rn 'CNW:$' . --include='*.md' 2>/dev/null | head -20 || true)"
if [ -z "$EMPTY_CNW" ]; then
check "No empty CNW: markers (unresolved questions)" "pass"
else
CNW_COUNT="$(printf '%s\n' "$EMPTY_CNW" | grep -c . || true)"
check "${CNW_COUNT} unresolved CNW: marker(s) — needs user input" "warn"
fi
# ----------------------------------------------------------------------------
# 9. Hygiene — merge-conflict markers and trailing whitespace must never land.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "File hygiene"
if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
# archive/ excluded: legacy files are preserved verbatim and may contain
# decorative `====` banners that false-positive as conflict markers.
CONFLICT="$(git diff --cached --name-only --diff-filter=ACM 2>/dev/null | grep -v '^archive/' | xargs -r grep -lE '^(<{7}( \S+)?|=======|>{7}( \S+)?)$' 2>/dev/null || true)"
if [ -z "$CONFLICT" ]; then check "No merge-conflict markers staged" "pass"; else check "Merge-conflict markers staged: $CONFLICT" "fail"; fi
fi
# ----------------------------------------------------------------------------
# 10. (slow, skipped in --fast) Project test suite via scripts/test.sh.
# ----------------------------------------------------------------------------
if [ "$RULE_FAST" = false ] && [ -x scripts/test.sh ]; then
$RULE_VERBOSE && log_step "Test suite (scripts/test.sh)"
if bash scripts/test.sh >/dev/null 2>&1; then
check "scripts/test.sh passes" "pass"
else
check "scripts/test.sh FAILS" "fail"
fi
fi
# ----------------------------------------------------------------------------
# 11. Access-channel policy — qemu guest-agent must NEVER be used as an
# execution or key-delivery channel. SSH (sshd) is the ONLY approved
# remote access path; all commands must be logged through standard
# auth/audit infrastructure. ITAR/CMMC environment — non-negotiable.
# Allowed: installing/checking qemu-guest-agent for Proxmox state
# visibility. Forbidden: `qm guest exec` + any `vm-guest` wrapper.
# Scans CODE only (not .md) so docs may describe the ban.
# ----------------------------------------------------------------------------
$RULE_VERBOSE && log_step "Access-channel policy (no guest-agent exec)"
GA_HITS="$(grep -rnE 'qm guest exec|vm-guest|_vm_guest' \
--include='*.sh' --include='*.bash' --include='*.py' \
. 2>/dev/null | grep -vE 'scripts/check-rules\.sh|/archive/' || true)"
if [ -z "$GA_HITS" ]; then
check "No guest-agent exec / vm-guest access patterns in code" "pass"
else
GA_COUNT="$(printf '%s\n' "$GA_HITS" | grep -c . || true)"
$RULE_VERBOSE && printf '%s\n' "$GA_HITS" | sed 's/^/ /'
check "${GA_COUNT} guest-agent exec / vm-guest reference(s) — SSH-only access policy (AGENTS.md)" "fail"
fi
print_summary_and_exit
scripts/check.sh
Executable
+1138
View File
File diff suppressed because it is too large Load Diff
scripts/conman-console.py
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""
conman-console.py — Drive a serial console via a conman server, read-only.
Connects to a conmand server (RFC 127-style multiplexer) over the network,
opens a named console, sends the commands from a .cmds file, captures all
output, and disconnects. Requires no expect/tcl — pure stdlib PTY.
This replaces the old sw-capture-remote.sh workflow that killed the serial
device holder (conflicting with conman/ser2net). Instead, it talks to conman
over TCP, which multiplexes safely with other sessions.
All endpoints are configurable via environment variables so this works on
any network with a conman server:
CONMAN_SERVER conman server host:port (default: via CONSOLE_HOST)
CONSOLE console name to open (required)
CMDS_FILE file of commands to send (required)
TIMEOUT overall timeout in seconds (default: 45)
CMD_DELAY seconds between commands (default: 3)
WAKE_DELAY seconds after connect (default: 2)
Usage:
CONMAN_SERVER=console-host:7890 \\
python3 conman-console.py --console pfv-r5-core-01 --cmds switches/pfv-r5-core-01.cmds
Lines starting with '!' or '#' in the cmds file are comments (skipped).
Blank lines are skipped. The conman escape sequence (&.) is sent automatically
to disconnect. A password prompt aborts immediately (we never send creds).
Exit codes:
0 clean run
1 usage / setup error
2 could not connect to conman server
3 timeout (partial output still printed)
4 password prompt encountered (aborted)
"""
import argparse
import os
import pty
import re
import select
import sys
import time
PWD_RE = re.compile(rb"[Pp]assword:\s*$")
MORE_RE = re.compile(rb"--\s*[Mm]ore\s*--|[Mm]ore:\s*<space>")
def main():
ap = argparse.ArgumentParser(
description="Drive a conman console session read-only via PTY")
ap.add_argument("--console", required=True,
help="console name (e.g. pfv-r5-core-01)")
ap.add_argument("--cmds", required=True,
help="command file (one command per line; !/# = comment)")
ap.add_argument("--server",
default=os.environ.get("CONMAN_SERVER", ""),
help="conman server host:port (env: CONMAN_SERVER)")
ap.add_argument("--timeout", type=int,
default=int(os.environ.get("TIMEOUT", "45")),
help="overall timeout seconds (env: TIMEOUT)")
ap.add_argument("--cmd-delay", type=float,
default=float(os.environ.get("CMD_DELAY", "3")),
help="seconds between commands (env: CMD_DELAY)")
ap.add_argument("--wake-delay", type=float,
default=float(os.environ.get("WAKE_DELAY", "2")),
help="seconds after connect before first command (env: WAKE_DELAY)")
args = ap.parse_args()
if not args.server:
sys.stderr.write("ERROR: --server or CONMAN_SERVER env required\n")
return 1
with open(args.cmds) as f:
cmds = [l.strip() for l in f
if l.strip() and not l.strip().startswith(("!", "#"))]
pid, fd = pty.fork()
if pid == 0:
try:
os.execvp("conman",
["conman", "-d", args.server, "-f", args.console])
except OSError as e:
sys.stderr.write(f"ERROR: cannot exec conman: {e}\n")
os._exit(2)
os._exit(2)
output = b""
cmd_queue = list(cmds)
sent_disconnect = False
start = time.time()
last_action = 0.0
phase = "connect"
while time.time() - start < args.timeout:
ready, _, _ = select.select([fd], [], [], 0.5)
if ready:
try:
data = os.read(fd, 8192)
except OSError:
break
if not data:
break
output += data
if PWD_RE.search(output.split(b"\n")[-1] if output else b""):
sys.stderr.write("[ABORT] password prompt detected — "
"never sending credentials\n")
os.write(fd, b"&.\n")
break
# Handle pagination: send space to continue
if MORE_RE.search(output[-200:] if output else b""):
os.write(fd, b" ")
time.sleep(0.5)
elapsed = time.time() - start
gap = elapsed - last_action
if phase == "connect" and gap >= args.wake_delay:
os.write(fd, b"\n")
phase = "send"
last_action = elapsed
elif phase == "send" and gap >= args.cmd_delay:
if cmd_queue:
cmd = cmd_queue.pop(0)
os.write(fd, (cmd + "\n").encode())
last_action = elapsed
else:
phase = "drain"
last_action = elapsed
elif phase == "drain" and gap >= args.cmd_delay:
os.write(fd, b"&.\n")
sent_disconnect = True
phase = "done"
last_action = elapsed
elif phase == "done" and gap >= 2:
break
if not sent_disconnect:
try:
os.write(fd, b"&.\n")
except OSError:
pass
try:
os.close(fd)
except OSError:
pass
try:
os.waitpid(pid, 0)
except ChildProcessError:
pass
sys.stdout.buffer.write(output)
sys.stdout.flush()
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/deploy-guest-perfpack.sh
+84
View File
@@ -0,0 +1,84 @@
#!/bin/bash
# deploy-guest-perfpack.sh — guest-side perf pack (ticket #737)
# Run INSIDE a guest VM: bash tests/remote.sh vm-file proxmox/perf/scripts/deploy-guest-perfpack.sh
# Idempotent. Applies: sysctl tuning (swappiness/dirty/net buffers/slow-start),
# fq + bbr (when the kernel module exists), THP->madvise (skipped when tuned
# manages THP, e.g. k8s cnodes on network-latency), fstrim.timer.
# Pairs with the host-side qm flags (iothread/ssd=1/discard=on).
set -u
log() { echo "[perfpack] $*"; }
# --- 1. sysctl profile ---
bbr=0
if modprobe tcp_bbr 2>/dev/null && grep -qw bbr /proc/sys/net/ipv4/tcp_available_congestion_control 2>/dev/null; then
bbr=1
fi
conf=/etc/sysctl.d/99-perfopt.conf
{
echo "# perf pack #737 (idempotent redeploy overwrites)"
echo "vm.swappiness = 10"
echo "vm.dirty_ratio = 10"
echo "vm.dirty_background_ratio = 3"
echo "net.core.rmem_max = 4194304"
echo "net.core.wmem_max = 4194304"
echo "net.core.rmem_default = 262144"
echo "net.core.wmem_default = 262144"
echo "net.ipv4.tcp_slow_start_after_idle = 0"
echo "net.core.default_qdisc = fq"
if [ "$bbr" = 1 ]; then
echo "net.ipv4.tcp_congestion_control = bbr"
fi
} > "$conf"
sysctl --system >/dev/null 2>&1
log "sysctl applied ($conf): swappiness=$(sysctl -n vm.swappiness 2>/dev/null) cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null) qdisc=$(sysctl -n net.core.default_qdisc 2>/dev/null) bbr=$bbr"
# --- 2. THP -> madvise (live now + persisted via oneshot unit) ---
tuned_active=0
if command -v tuned-adm >/dev/null 2>&1; then
tuned-adm active 2>/dev/null | grep -qvi 'no current\|inactive\|none' && tuned_active=1
fi
if [ "$tuned_active" = 1 ]; then
log "THP: tuned profile active ($(tuned-adm active 2>/dev/null | awk '{print $NF}')) — leaving THP to tuned"
else
thp=/sys/kernel/mm/transparent_hugepage/enabled
cur=$(tr '[]' ' ' < "$thp" 2>/dev/null | awk '{print $NF}')
if [ "$cur" != "madvise" ]; then
if echo madvise > "$thp" 2>/dev/null; then
log "THP: ${cur:-unknown} -> madvise (live)"
else
log "THP: WARNING live set failed (check $thp permissions)"
fi
else
log "THP: already madvise"
fi
unit=/etc/systemd/system/thp-madvise.service
if [ ! -f "$unit" ]; then
{
echo "[Unit]"
echo "Description=Set transparent hugepages to madvise (perf pack #737)"
echo "DefaultDependencies=no"
echo "After=local-fs.target"
echo "Before=sysinit.target"
echo
echo "[Service]"
echo "Type=oneshot"
echo "ExecStart=/bin/sh -c 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'"
echo
echo "[Install]"
echo "WantedBy=sysinit.target"
} > "$unit"
systemctl daemon-reload
systemctl enable thp-madvise.service >/dev/null 2>&1
log "THP: persistence unit installed + enabled"
fi
fi
# --- 3. fstrim (pairs with host-side discard=on) ---
if systemctl enable --now fstrim.timer >/dev/null 2>&1; then
log "fstrim.timer enabled"
else
log "fstrim.timer: not available on this guest"
fi
log "done"
scripts/deploy-tuned-guests.sh
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/bash
# Deploy tuned inside VMs over SSH (sshd is the only approved access channel —
# see AGENTS.md "Access-channel policy: SSH only"). Formerly used the
# qemu guest-agent channel; converted to SSH now that all VMs have key + sudo.
set -uo pipefail
# Profile mapping: most VMs get throughput-performance, k8s/network-sensitive get network-latency
PROFILE="${1:-throughput-performance}"
REMOTE_SH="${REMOTE_SH:-$(cd "$(dirname "$0")/../../.." && pwd)/tests/remote.sh}"
VM_USER="${VM_USER:-localuser}"
deploy_vm() {
local host="$1" vmid="$2" name="$3"
# name is the Tailscale hostname (SSH target); vmid retained for reference.
echo -n " VMID $vmid ($name) on $host: "
local result
result=$(VM_IP="$name" VM_USER="$VM_USER" bash "$REMOTE_SH" vmroot \
"DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>/dev/null; \
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq tuned 2>/dev/null; \
tuned-adm profile $PROFILE 2>/dev/null; \
systemctl enable --now tuned 2>/dev/null; \
tuned-adm active 2>/dev/null" </dev/null 2>&1)
if echo "$result" | grep -q 'Current active'; then
echo "$result" | grep -o 'Current active.*' | head -1
elif echo "$result" | grep -qi 'permission denied\|no route\|timed out'; then
echo "SSH FAILED (no key/no sudo) — run bootstrap-all.sh first"
else
echo "INSTALL FAILED (apt issue or no network)"
fi
}
echo "============================================"
echo " Deploying tuned ($PROFILE) to VMs via SSH"
echo "============================================"
echo ""
# tsys1 VMs
echo "--- pfv-tsys1 ---"
deploy_vm pfv-tsys1 100 pfv-bms
deploy_vm pfv-tsys1 101 tsys-ca
deploy_vm pfv-tsys1 102 pfv-k8s-cnode1
deploy_vm pfv-tsys1 104 tsys-librenms
deploy_vm pfv-tsys1 105 tsys-proxmox-datacenter
deploy_vm pfv-tsys1 108 tsys-ucs-01
# tsys3 VMs
echo "--- pfv-tsys3 ---"
deploy_vm pfv-tsys3 313 pfv-k8s-wnode-tsys3
# tsys6 VMs
echo "--- pfv-tsys6 ---"
deploy_vm pfv-tsys6 600 tsys-awx
deploy_vm pfv-tsys6 601 pfv-k8s-wnode-tsys6
deploy_vm pfv-tsys6 602 pfv-rr-middleware-02
deploy_vm pfv-tsys6 603 pfv-k8s-cnode3
deploy_vm pfv-tsys6 604 tsys-proxmox-mailgw-01
# tsys7 VMs
echo "--- pfv-tsys7 ---"
deploy_vm pfv-tsys7 701 pfv-k8s-wnode-tsys7
deploy_vm pfv-tsys7 702 hfnoc-uisp
deploy_vm pfv-tsys7 703 pfv-rr-middleware-01
deploy_vm pfv-tsys7 705 pfv-k8s-cnode2
deploy_vm pfv-tsys7 706 kali-rd
deploy_vm pfv-tsys7 707 tsys-siem
deploy_vm pfv-tsys7 708 kali-tsys
deploy_vm pfv-tsys7 709 tsys-voip
deploy_vm pfv-tsys7 711 tsys-proxmox-mailgw-02
# tsys9 VMs
echo "--- pfv-tsys9 ---"
deploy_vm pfv-tsys9 902 tsys-ucs-02
deploy_vm pfv-tsys9 905 pfv-k8s-wnode-tsys9
echo ""
echo "============================================"
echo " Deployment complete."
echo " NOT deployed (do-not-reboot VMs):"
echo " ultix-streaming, ultix-offstage,"
echo " pfv-netinfra-01, pfv-netinfra-02"
echo "============================================"
scripts/docker-run.sh
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# docker-run.sh — canonical ephemeral-container wrapper.
#
# Keeps the host clean: every build/test/generation runs inside a pinned image.
# Ensures output files are owned by the invoking user (not root).
#
# Usage:
# docker-run.sh <image> [command...]
# Runs <command> in <image> with the repo mounted at /data, cwd /data.
# With no command, drops into the image's default entrypoint.
# docker-run.sh --shell <image>
# Interactive shell inside the container (for debugging).
#
# Examples:
# docker-run.sh python:3.12-slim python3 -m pytest
# docker-run.sh pandoc/extra report.md -o report.pdf
# docker-run.sh --shell node:20
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
# shellcheck disable=SC1091
source "$HERE/lib/common.sh"
SHELL_MODE=false
case "${1:-}" in
--shell) SHELL_MODE=true; shift ;;
-h|--help)
sed -n '2,18p' "$0"; exit 0 ;;
esac
[ "$#" -ge 1 ] || { sed -n '2,18p' "$0"; exit 1; }
if [ "$SHELL_MODE" = true ]; then
# ${SHELL:-sh} must expand inside the container, not in this outer shell.
# shellcheck disable=SC2016
docker_run "$1" sh -c 'exec "${SHELL:-sh}"'
else
docker_run "$@"
fi
scripts/fix-bond-nfs.sh
Executable
+86
View File
@@ -0,0 +1,86 @@
#!/bin/bash
# fix-tsys7.sh - fixes the two issues found: NFS options string + bond hash
# Runs on the target host directly.
set -euo pipefail
echo "==================================================================="
echo " FIX: NFS options + bond hash policy"
echo "==================================================================="
# --- FIX 1: Simplify NFS options in storage.cfg ---------------------------
echo ""
echo "--- FIX 1: Simplify NFS options (remove version=4.2 conflict) ---"
# Replace the overly-complex options line with a minimal one
# PVE handles vers/rsize/wsize/hard/etc internally; we only need nconnect + noatime
if grep -q 'options.*nconnect=4.*version=4.2' /etc/pve/storage.cfg; then
# Use sed to replace each options line
sed -i 's/options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2/options nconnect=4,noatime/' /etc/pve/storage.cfg
echo "Fixed: simplified NFS options to 'nconnect=4,noatime'"
else
echo "Options line already simplified or not present"
fi
echo ""
echo "--- Verify storage.cfg NFS stanzas ---"
grep -A 2 "^nfs: D2" /etc/pve/storage.cfg | head -3
echo "..."
grep -c "options nconnect" /etc/pve/storage.cfg
echo " NFS stanzas with options"
# --- FIX 2: Bond hash policy (force via sysfs + persist in interfaces) ----
echo ""
echo "--- FIX 2: Apply bond xmit_hash_policy=layer3+4 ---"
# 2a: Apply LIVE via sysfs (takes effect immediately, no network reload)
echo "Applying live via sysfs..."
if echo "layer3+4" > /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; then
echo "Live sysfs apply: SUCCESS"
else
echo "Live sysfs apply: FAILED (will persist in config and apply on ifreload)"
fi
# Verify live state
echo ""
echo "Live bond0 hash policy:"
cat /proc/net/bonding/bond0 | grep "Transmit Hash"
# 2b: Persist in /etc/network/interfaces (fix the sed that failed before)
echo ""
echo "Persisting in /etc/network/interfaces..."
# Backup
cp -a /etc/network/interfaces "/root/interfaces.bondfix.$(date +%Y%m%d%H%M%S)"
# Check if already present
if grep -q 'bond-xmit-hash-policy' /etc/network/interfaces; then
echo "bond-xmit-hash-policy already in interfaces file"
else
# Use awk to insert after the bond-mode line (more reliable than sed)
# Match any line containing 'bond-mode' (regardless of indentation)
awk '
/bond-mode/ && !done {
print
print "\tbond-xmit-hash-policy layer3+4"
done=1
next
}
{ print }
' /etc/network/interfaces > /etc/network/interfaces.new
mv /etc/network/interfaces.new /etc/network/interfaces
echo "Inserted bond-xmit-hash-policy layer3+4"
fi
echo ""
echo "--- Updated bond0 stanza ---"
awk '/^auto bond0/,/^$/' /etc/network/interfaces
# Final verify
echo ""
echo "--- Final bond0 running state ---"
cat /proc/net/bonding/bond0 | head -20
echo ""
echo "==================================================================="
echo " FIX COMPLETE"
echo "==================================================================="
scripts/fix-e1000e-offload.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/bin/bash
###############################################################################
# fix-e1000e-offload.sh — Disable offload on e1000e NICs to prevent hangs
#
# The Intel e1000e driver has a known bug where TSO/GSO/GRO offload causes
# "hardware unit hang" resets on certain Intel NICs (I217-LM, I219-LM, 82571EB).
# This script disables offload features on all e1000e interfaces and installs
# a systemd service to persist across reboots.
#
# References:
# https://forum.proxmox.com/threads/e1000-driver-hang.58284/
# https://serverfault.com/questions/616485
#
# Run on each Proxmox host:
# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file proxmox/perf/scripts/fix-e1000e-offload.sh
###############################################################################
set -euo pipefail
echo "=== e1000e Offload Fix on $(hostname) ==="
# Find all e1000e physical NICs (skip bridges, bonds, virtual interfaces)
AFFECTED_NICS=()
for nic_path in /sys/class/net/*; do
nic=$(basename "$nic_path")
[ "$nic" = "lo" ] && continue
# Skip bridges, bonds, virtual interfaces
[ -d "${nic_path}/bridge" ] && continue
[ -d "${nic_path}/bonding" ] && continue
case "$nic" in
tap*|veth*|fwpr*|fwln*|vmbr*|datanet*|storagenet*|tailscale*) continue ;;
esac
driver=$(ethtool -i "$nic" 2>/dev/null | awk '/^driver:/{print $2}')
if [ "$driver" = "e1000e" ]; then
AFFECTED_NICS+=("$nic")
echo " Found e1000e NIC: $nic"
fi
done
if [ "${#AFFECTED_NICS[@]}" -eq 0 ]; then
echo " No e1000e NICs found. Nothing to do."
exit 0
fi
# Apply fix live
echo ""
echo "--- Disabling offload features ---"
for nic in "${AFFECTED_NICS[@]}"; do
echo " $nic:"
ethtool -K "$nic" tso off gro off gso off tx off rx off 2>&1 | sed 's/^/ /' || true
tso_state=$(ethtool -k "$nic" 2>/dev/null | awk '/tcp-segmentation-offload/{print $2}' | head -1)
echo " tso=$tso_state"
done
# Install systemd service for persistence
echo ""
echo "--- Installing systemd service ---"
{
echo "[Unit]"
echo "Description=Disable offload on e1000e NICs (prevent hardware unit hang)"
echo "After=network.target"
echo "Wants=network.target"
echo ""
echo "[Service]"
echo "Type=oneshot"
echo "RemainAfterExit=yes"
for nic in "${AFFECTED_NICS[@]}"; do
echo "ExecStart=/sbin/ethtool -K $nic tso off gro off gso off tx off rx off"
done
echo ""
echo "[Install]"
echo "WantedBy=multi-user.target"
} > /etc/systemd/system/fix-e1000e-offload.service
systemctl daemon-reload
systemctl enable fix-e1000e-offload.service
echo " Service installed and enabled (fix-e1000e-offload.service)"
echo ""
echo "=== Done. Affected NICs: ${AFFECTED_NICS[*]} ==="
scripts/fix-tsys4-storage-bond.sh
+283
View File
@@ -0,0 +1,283 @@
#!/bin/bash
###############################################################################
# fix-tsys4-storage-bond.sh
#
# Reconfigures pfv-tsys4 storage network bond from broken 802.3ad (LACP) to
# active-backup mode. USB NICs cannot participate in LACP — the driver does
# not report speed/duplex to the kernel (ethtool shows "Speed: Unknown!"),
# so the bonding driver never transmits LACP PDUs. active-backup requires no
# switch-side LACP and works correctly with a single port.
#
# Run ON pfv-tsys4. Designed for maintenance-window execution.
#
# Safety:
# - Dry-run by default (--apply to commit)
# - Full backup of /etc/network/interfaces
# - 5 health checks with automatic rollback on failure
# - SSH survives (management on vmbr0/tailscale0, not bond0/datanet)
# - Pre-generated rollback script for manual recovery
#
# Switch side (core-sw01): NO changes needed. g31 is already a standalone
# access port in VLAN 1000 — correct for active-backup (no LACP required).
# g32/ch2 cleanup (dead NIC) is left for separate maintenance.
#
# Usage:
# bash fix-tsys4-storage-bond.sh # dry-run (show changes only)
# bash fix-tsys4-storage-bond.sh --apply # commit with auto-rollback
#
# Related: Redmine [#394] BUG 4
###############################################################################
set -euo pipefail
STORAGE_PEER="10.100.100.6"
INTERFACES="/etc/network/interfaces"
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="/root/tsys4-bondfix-backup-${TS_SHORT}"
ROLLBACK="/root/tsys4-bondfix-rollback-${TS_SHORT}.sh"
ACTION="${1:-dryrun}"
[ "${ACTION}" = "--apply" ] && ACTION="apply" || ACTION="dryrun"
echo "==================================================================="
echo " fix-tsys4-storage-bond — $(hostname -s)"
echo " mode: ${ACTION}"
echo " time: $(date)"
echo "==================================================================="
echo ""
# -------------------------------------------------------------------------
# Pre-flight checks
# -------------------------------------------------------------------------
echo "--- Pre-flight checks ---"
# Must be pfv-tsys4
HOSTNAME_S="$(hostname -s)"
if [ "${HOSTNAME_S}" != "pfv-tsys4" ]; then
echo "FATAL: This script targets pfv-tsys4 (this host: ${HOSTNAME_S})"
exit 1
fi
echo " Host: OK (pfv-tsys4)"
# bond0 must exist
if [ ! -d /sys/class/net/bond0 ]; then
echo "FATAL: bond0 not found — no bond to fix"
exit 1
fi
echo " bond0: present"
# ifreload must be available
if ! command -v ifreload >/dev/null 2>&1; then
echo "FATAL: ifreload not found (need ifupdown2)"
exit 1
fi
echo " ifreload: available"
# SSH must NOT be on bond0/datanet (check incoming route)
SSH_SRC="$(echo "${SSH_CLIENT:-}" | awk '{print $1}')"
if [ -n "${SSH_SRC}" ]; then
SSH_IFACE="$(ip route get "${SSH_SRC}" 2>/dev/null | grep -oP 'dev \K\S+' || echo "unknown")"
echo " SSH ingress: ${SSH_IFACE}"
if echo "${SSH_IFACE}" | grep -qE 'bond0|datanet'; then
echo "FATAL: SSH is on storage network — cannot safely reload."
echo " Use physical console (pfv-tsys4) to run this script."
exit 1
fi
else
echo " SSH ingress: (local/console session — OK)"
fi
echo " SSH safety: OK (not on storage network)"
echo ""
# -------------------------------------------------------------------------
# Show current state
# -------------------------------------------------------------------------
echo "--- Current bond0 state ---"
grep -E "Bonding Mode|Transmit Hash|MII Status|Number of ports" /proc/net/bonding/bond0
echo ""
grep -E "Slave Interface|MII Status|Speed" /proc/net/bonding/bond0
echo ""
echo "--- Current bond0 stanza in /etc/network/interfaces ---"
awk '/^auto bond0/,/^$/' "${INTERFACES}"
echo ""
# -------------------------------------------------------------------------
# Baseline connectivity
# -------------------------------------------------------------------------
echo "--- Baseline connectivity ---"
echo -n " Ping ${STORAGE_PEER}: "
if ping -c 1 -W 2 "${STORAGE_PEER}" >/dev/null 2>&1; then
echo "OK"
else
echo "UNREACHABLE (baseline already broken — proceed with caution)"
fi
echo -n " NFS server: "
systemctl is-active nfs-server 2>/dev/null || echo "(not active)"
echo ""
# -------------------------------------------------------------------------
# Backup
# -------------------------------------------------------------------------
mkdir -p "${BACKUP_DIR}"
cp -a "${INTERFACES}" "${BACKUP_DIR}/interfaces"
echo "Backup: ${BACKUP_DIR}/interfaces"
# Generate rollback script
cat > "${ROLLBACK}" <<ROLLBACKEOF
#!/bin/bash
set -euo pipefail
echo "Rolling back tsys4 bond config..."
cp -a "${BACKUP_DIR}/interfaces" "${INTERFACES}"
echo "Restored ${INTERFACES}"
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
sleep 3
echo "Post-rollback bond0 state:"
grep -E "Bonding Mode|MII Status|Slave Interface|Speed" /proc/net/bonding/bond0 2>/dev/null
echo "Rollback complete."
ROLLBACKEOF
chmod +x "${ROLLBACK}"
echo "Rollback: ${ROLLBACK}"
echo ""
# -------------------------------------------------------------------------
# Describe the change
# -------------------------------------------------------------------------
echo "==================================================================="
echo " PROPOSED CHANGE"
echo "==================================================================="
echo " 1. bond-mode: 802.3ad → active-backup"
echo " 2. Remove bond-xmit-hash-policy (unused in active-backup)"
echo " 3. bond-slaves: keep enx8cae4ccda926 (active NIC only)"
echo ""
echo " Rationale: USB NICs cannot do LACP. ethtool reports Speed: Unknown,"
echo " so the bonding driver never sends LACP PDUs (verified via tcpdump:"
echo " 0 LACP PDUs in 65s on both slave and bond master). active-backup"
echo " needs no LACP and works with the single working port."
echo ""
echo " Switch: NO changes needed. g31 is standalone access VLAN 1000."
echo ""
if [ "${ACTION}" != "apply" ]; then
echo "==================================================================="
echo " DRY RUN — no changes made."
echo " Commit: bash \$0 --apply"
echo "==================================================================="
exit 0
fi
# =========================================================================
# APPLY
# =========================================================================
echo "==================================================================="
echo " APPLYING — auto-rollback on health check failure"
echo "==================================================================="
echo ""
# 1. Change bond-mode
echo "Changing bond-mode 802.3ad → active-backup..."
sed -i 's/bond-mode 802\.3ad/bond-mode active-backup/' "${INTERFACES}"
# 2. Remove bond-xmit-hash-policy (not used by active-backup)
echo "Removing bond-xmit-hash-policy..."
sed -i '/bond-xmit-hash-policy/d' "${INTERFACES}"
# Show updated stanza
echo ""
echo "--- Updated bond0 stanza ---"
awk '/^auto bond0/,/^$/' "${INTERFACES}"
echo ""
# 3. Reload networking
echo "--- Reloading networking (ifreload -a) ---"
echo "SSH safe on vmbr0/tailscale0 (not bond0)..."
ifreload -a 2>&1 || echo "WARNING: ifreload returned non-zero — checking state..."
# Wait for bond to settle
echo "Waiting 5s for bond to settle..."
sleep 5
# =========================================================================
# HEALTH CHECKS
# =========================================================================
echo ""
echo "==================================================================="
echo " HEALTH CHECKS (5 tests)"
echo "==================================================================="
HEALTH_OK=true
# Check 1: bond0 is UP
echo -n " [1/5] bond0 MII up: "
if grep -q "MII Status: up" /proc/net/bonding/bond0 2>/dev/null; then
echo "PASS"
else
echo "FAIL"
HEALTH_OK=false
fi
# Check 2: Active slave assigned
echo -n " [2/5] Active slave: "
ACTIVE_SLAVE="$(cat /sys/class/net/bond0/bonding/active_slave 2>/dev/null || echo "")"
if [ -n "${ACTIVE_SLAVE}" ]; then
echo "PASS (${ACTIVE_SLAVE})"
else
echo "FAIL (no active slave)"
HEALTH_OK=false
fi
# Check 3: datanet bridge UP
echo -n " [3/5] datanet bridge up: "
if ip link show datanet 2>/dev/null | grep -q "state UP"; then
echo "PASS"
else
echo "FAIL"
HEALTH_OK=false
fi
# Check 4: Storage network IP present on datanet
echo -n " [4/5] Storage IP (10.100.100.4): "
if ip addr show datanet 2>/dev/null | grep -q "10.100.100.4"; then
echo "PASS"
else
echo "FAIL"
HEALTH_OK=false
fi
# Check 5: Storage peer reachable
echo -n " [5/5] Ping ${STORAGE_PEER}: "
if ping -c 3 -W 2 "${STORAGE_PEER}" >/dev/null 2>&1; then
echo "PASS"
else
echo "FAIL"
HEALTH_OK=false
fi
# =========================================================================
# COMMIT OR ROLLBACK
# =========================================================================
echo ""
if [ "${HEALTH_OK}" = "true" ]; then
echo "==================================================================="
echo " SUCCESS — all 5 health checks passed"
echo "==================================================================="
echo ""
echo "--- Final bond0 state ---"
grep -E "Bonding Mode|MII Status|Slave Interface|Speed|Active" /proc/net/bonding/bond0
echo ""
echo "Backup: ${BACKUP_DIR}/interfaces"
echo "Rollback: ${ROLLBACK}"
echo ""
echo "Config change is LIVE but not yet reboot-tested."
echo "Verify NFS clients are healthy before next maintenance window."
exit 0
else
echo "==================================================================="
echo " HEALTH CHECK FAILED — auto-rolling back"
echo "==================================================================="
bash "${ROLLBACK}"
echo ""
echo "Auto-rollback complete."
echo "Original config restored. Manual investigation needed."
exit 1
fi
scripts/garden.sh
Executable
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env bash
# garden.sh — the gardening loop.
#
# Reports doc sprawl and files that violate the "Discourse is the system of
# record for documentation; gitea .md files are stubs" policy. Run via
# `bash scripts/garden.sh`. Findings are WARNINGS (advisory); fix them at a natural break.
#
# What it checks:
# 1. Markdown sprawl: count of .md files per directory (top-10 by count).
# 2. Oversized .md files (default >300 lines) that don't cite a Discourse URL
# — candidates to migrate to Discourse, leaving a stub.
# 3. .md files with no Discourse link at all (informational; exempt: the
# operational files in EXEMPT_FILES).
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
# shellcheck disable=SC1091
source "$HERE/lib/common.sh"
REPO_ROOT="$(repo_root)"
cd "$REPO_ROOT"
SIZE_LIMIT="${GARDEN_MD_LINE_LIMIT:-300}"
# Operational files that legitimately live next to code, not in Discourse.
EXEMPT_FILES="${GARDEN_EXEMPT:-AGENTS.md STATUS.md WORKING.md questions-v.*.md PATTERNS.md BASELINE-PROMPT.md README.md}"
log_step "Gardening report for $REPO_ROOT"
# --- 1. sprawl by directory -------------------------------------------------
log_info "Markdown file count by directory (top 10):"
find . -path ./.git -prune -o -name '*.md' -print 2>/dev/null \
| sed 's|/[^/]*$||' | sort | uniq -c | sort -rn | head -10 | sed 's/^/ /'
# --- 2. oversized .md without a Discourse link ------------------------------
log_info "Oversized .md (>${SIZE_LIMIT} lines) lacking a Discourse URL — migrate candidates:"
OVERSIZED=0
while IFS= read -r -d '' f; do
# skip exempt files (glob match against basename and relative path)
exempt=false
base=$(basename "$f")
rel=${f#./}
for pat in $EXEMPT_FILES; do
# shellcheck disable=SC2254 # glob match is intentional
case "$base" in $pat) exempt=true; break ;; esac
# shellcheck disable=SC2254
case "$rel" in $pat) exempt=true; break ;; esac
done
[ "$exempt" = true ] && continue
lines=$(wc -l < "$f" 2>/dev/null || echo 0)
if [ "$lines" -gt "$SIZE_LIMIT" ]; then
if ! grep -qiE 'community\.turnsys\.com|discourse' "$f" 2>/dev/null; then
printf ' %-60s %s lines\n' "$f" "$lines"
OVERSIZED=$((OVERSIZED + 1))
fi
fi
done < <(find . -path ./.git -prune -o -name '*.md' -print0 2>/dev/null)
[ "$OVERSIZED" -eq 0 ] && echo " (none)"
# --- 3. summary -------------------------------------------------------------
log_step "Gardening summary"
echo " Oversized non-Discourse .md files: $OVERSIZED"
if [ "$OVERSIZED" -eq 0 ]; then
log_ok "no migration candidates"
else
log_warn "$OVERSIZED file(s) to migrate to Discourse"
fi
scripts/identify-hosts.sh
Executable
+34
View File
@@ -0,0 +1,34 @@
#!/bin/bash
# identify-hosts.sh — identify all live hosts on a network via DNS/SNMP/SSH
# Usage: bash identify-hosts.sh <hosts-file-or-subnet>
# bash identify-hosts.sh 192.168.0.0/22
# bash identify-hosts.sh /tmp/live-hosts.txt
set -u
COMMUNITY="${SNMP_COMMUNITY:-kn3lmgmt}"
if [ -f "${1:-}" ]; then
HOSTS=$(cat "$1")
elif [ -n "${1:-}" ]; then
HOSTS=$(nmap -sn -n "$1" 2>/dev/null | grep 'Nmap scan report' | awk '{print $5}')
else
echo "Usage: $0 <hosts-file-or-subnet>"
exit 1
fi
printf "%-16s %-30s %-25s %-20s\n" "IP" "DNS_NAME" "SNMP_SYSDESCR" "SSH_BANNER"
printf "%-16s %-30s %-25s %-20s\n" "--" "-------" "-------------" "----------"
for ip in $HOSTS; do
# DNS reverse
rdns=$(dig +short -x "$ip" 2>/dev/null | head -1 | sed 's/\.$//')
[ -z "$rdns" ] && rdns="(no-ptr)"
# SNMP sysDescr (first 25 chars)
snmp=$(timeout 3 snmpget -Oqv -v2c -c "$COMMUNITY" "$ip" 1.3.6.1.2.1.1.1.0 2>/dev/null | head -1 | tr -d '"' | cut -c1-25)
[ -z "$snmp" ] && snmp="-"
# SSH banner (quick)
ssh_banner=$(timeout 3 bash -c "echo > /dev/tcp/$ip/22" 2>/dev/null && echo "ssh:open" || echo "-")
printf "%-16s %-30s %-25s %-20s\n" "$ip" "$rdns" "$snmp" "$ssh_banner"
done
scripts/lacp-retrans-cause.sh
+219
View File
@@ -0,0 +1,219 @@
#!/usr/bin/env bash
###############################################################################
# lacp-retrans-cause.sh (HOST-NATIVE)
#
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
#
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
# saturation) or just unavoidable LACP reordering overhead?
#
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
# - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources -> ring too small
# - /proc/net/softnet_stat drops -> ksoftirq starved
# - /proc/net/snmp TCP retranst -> TCP-level retrans
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
# coming from LACP reordering (out-of-order segments triggering fast
# retransmit), not packet loss — and are NOT fixable by tuning.
###############################################################################
set -uo pipefail
DURATION="${LACP_TEST_SECS:-12}"
STREAMS="${LACP_STREAMS:-8}"
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
LOG_DIR="/root"
BOND="bond0"
HOST="$(uname -n)"; HOST="${HOST%%.*}"
case "$HOST" in
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
*) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
esac
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT
# ---- per-slave NIC counter snapshot -----------------------------------------
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
# plus all error-like counters. Falls back gracefully if a counter doesn't
# exist (different NIC drivers expose different names).
nic_snapshot() {
# shellcheck disable=SC2013 # interface names contain no spaces; word-splitting is safe here
for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
[ -n "$s" ] || continue
echo "[$s]"
# ethtool -S may exist; show only drop/error/retrans lines
if command -v ethtool >/dev/null 2>&1; then
ethtool -S "$s" 2>/dev/null \
| grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
|| echo "(ethtool -S: no matching counters or unsupported)"
else
echo "(ethtool not installed)"
fi
# /sys counters (always available, driver-agnostic)
for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
multicast collisions; do
v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
[ -n "$v" ] && printf ' sysfs %-22s = %s\n' "$c" "$v"
done
done
}
# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
# look identical in hex and decimal). Processed is informational only and we
# don't try to delta it (hex arithmetic is awk-version-dependent).
softnet_snapshot() {
awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
NR-1, $1, $2, $3 }' /proc/net/softnet_stat
}
# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
tcp_snapshot() {
awk '/^Tcp:/{
if (!seen) {
seen=1
# /proc/net/snmp Tcp line 1 = names, line 2 = values
n=split($0, names, " ")
# re-find the values line
getline
vals=$0
split(vals, v, " ")
for (i=1;i<=n;i++) printf " %-22s = %s\n", names[i], v[i]
}
}' /proc/net/snmp
}
# ---- print deltas for selected counters ------------------------------------
# $1 = before file, $2 = after file
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
# integers (typically 0 on a healthy host), so +0 coercion is correct for
# the values we care about. We deliberately DON'T report `processed` deltas
# because hex arithmetic on large numbers is awk-version-dependent.
softnet_delta() {
awk '
FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
next }
{ if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
split($0, q, /[= ]+/);
dd = (q[5]+0) - drop[id]
sd = (q[7]+0) - sqz[id]
if (dd != 0 || sd != 0)
printf " %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
' "$1" "$2" | sort
}
# $1 = before file, $2 = after file, $3 = section label prefix
# File format: slave header line "[nic1]", then " counter_name = value" lines.
# Output only counters whose value changed.
nic_delta() {
awk '
FNR==NR { if ($0 ~ /^\[/) slave=$0
else if ($0 ~ /=/) {
# Everything before " = " is the counter name (trim spaces)
pos = index($0, "=")
name = substr($0, 1, pos-1)
gsub(/^ +| +$/, "", name)
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
before[slave SUBSEP name] = val
}
next }
{ if ($0 ~ /^\[/) slave=$0
else if ($0 ~ /=/) {
pos = index($0, "=")
name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
b = before[slave SUBSEP name]+0
d = val+0 - b
if (d != 0) printf " %-8s %-30s %12s -> %12s delta=%d\n", slave, name, b, val, d
} }
' "$1" "$2"
}
{
echo "=== LACP retransmit cause investigation ($ROLE) ==="
echo "Host: $HOST Role: $ROLE"
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Bond: $BOND"
echo ""
echo "--- bond0 hash + driver ---"
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
# shellcheck disable=SC2013 # interface names contain no spaces; word-splitting is safe here
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
speed=$(cat "/sys/class/net/$s/speed" 2>/dev/null)
ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
printf " %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
done
echo ""
} | tee "$LOG"
echo "--- BEFORE snapshots ---"
nic_snapshot > "$TMP/nic_before"
softnet_snapshot > "$TMP/softnet_before"
tcp_snapshot > "$TMP/tcp_before"
if [ "$ROLE" = "receiver" ]; then
echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
pkill -x iperf3 2>/dev/null; sleep 0.5
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
SRV_PID=$!
for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
wait "$SRV_PID" 2>/dev/null
else
echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
sleep 3
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
> "$TMP/iperf.out" 2>&1
fi
echo "--- AFTER snapshots ---"
nic_snapshot > "$TMP/nic_after"
softnet_snapshot > "$TMP/softnet_after"
tcp_snapshot > "$TMP/tcp_after"
{
echo ""
echo "--- iperf3 summary ---"
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
echo ""
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
# Show only the counters that changed
paste "$TMP/tcp_before" "$TMP/tcp_after" \
| awk '{
a=$3; b=$(NF)
if (a+0 != b+0) printf " %-22s %12s -> %12s delta=%d\n", $1, a, b, b-a
}'
echo ""
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
echo ""
echo "--- NIC counter deltas (only non-zero) ---"
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
echo ""
echo "--- raw iperf3 tail (last 12 lines) ---"
tail -12 "$TMP/iperf.out"
echo ""
echo "==================================================================="
echo " HOW TO READ THIS"
echo "==================================================================="
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
echo " -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
echo " -> softirq starvation. Fix: increase net.core.netdev_budget,"
echo " check IRQ affinity, consider RPS."
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
echo " -> the retransmits are LACP reordering (out-of-order segments"
echo " triggering fast retransmit), NOT packet loss. NOT fixable."
echo "==================================================================="
} | tee -a "$LOG"
echo ""
echo "Log: $LOG"
scripts/lacp-rx-distribution.sh
Executable
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env bash
###############################################################################
# lacp-rx-distribution.sh (HOST-NATIVE version)
#
# Runs ON a host (tsys6 or tsys7). Auto-detects role by hostname:
# - On tsys6 (receiver): starts iperf3 -s -1 (one-shot), snapshots local
# bond0 slave RX counters before+after, writes verdict + log.
# - On tsys7 (sender): snapshots local bond0 slave TX counters before+
# after, runs iperf3 -c <peer> -P 8 -t 12.
#
# Each side writes only its own counters. No inter-host SSH required.
# iperf3 -s -1 (one-shot) handles client/server coordination.
#
# Output: /root/lacp-rx-{receiver,sender}.log
###############################################################################
set -uo pipefail
DURATION="${LACP_TEST_SECS:-12}"
STREAMS="${LACP_STREAMS:-8}"
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
LOG_DIR="/root"
BOND="bond0"
# Snapshot per-slave byte counters from $BOND. Output: "<slave> rx tx\n" sorted.
snapshot() {
awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND" | while read -r s; do
[ -n "$s" ] || continue
rx=$(cat "/sys/class/net/$s/statistics/rx_bytes" 2>/dev/null || echo 0)
tx=$(cat "/sys/class/net/$s/statistics/tx_bytes" 2>/dev/null || echo 0)
printf '%s %s %s\n' "$s" "$rx" "$tx"
done | sort
}
# Compute per-slave deltas. $1=label, $2=col(2=rx,3=tx), $3=before, $4=after.
analyze() {
local label="$1" col="$2" before="$3" after="$4"
join "$before" "$after" | awk -v col="$col" -v lbl="$label" '
{
slave=$1
b = (col==2 ? $2 : $3)+0
a = (col==2 ? $4 : $5)+0
d = a - b; if (d < 0) d = 0
delta[slave]=d; bef[slave]=b; aft[slave]=a
order[++n]=slave; total+=d
}
END {
printf "\n--- %s (bytes) ---\n", lbl
printf " %-12s %14s %14s %14s %8s\n", "SLAVE", "BEFORE", "AFTER", "DELTA", "PCT"
for (i=1;i<=n;i++){
s=order[i]
pct = (total>0 ? 100*delta[s]/total : 0)
printf " %-12s %14d %14d %14d %7.1f%%\n", s, bef[s], aft[s], delta[s], pct
}
printf " %-12s %14s %14s %14d %8s\n", "TOTAL", "", "", total, "100.0%"
}'
}
# Dominant-slave pct for the verdict line. $1=col, $2=before, $3=after.
dominant() {
join "$2" "$3" | awk -v col="$1" '
{ b=(col==2?$2:$3)+0; a=(col==2?$4:$5)+0; d=a-b; if(d<0)d=0; tot+=d; delta[$1]=d }
END { m=0; ms=""; for (s in delta){ if (delta[s]>m){m=delta[s]; ms=s} }
printf "%.1f %s", (tot>0?100*m/tot:0), ms }'
}
HOST="$(uname -n)"
HOST="${HOST%%.*}"
case "$HOST" in
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-rx-receiver.log" ;;
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-rx-sender.log" ;;
*) echo "ERROR: not running on tsys6 or tsys7 (hostname=$HOST)"; exit 2 ;;
esac
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT
{
echo "=== LACP per-slave distribution ($ROLE) ==="
echo "Host: $HOST Role: $ROLE"
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Bond: $BOND"
echo ""
echo "--- $BOND state ---"
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac|Slave Interface|Link" \
"/proc/net/bonding/$BOND" 2>&1
echo ""
} | tee "$LOG"
snapshot > "$TMP/before"
if [ "$ROLE" = "receiver" ]; then
echo "[receiver] starting one-shot iperf3 server on $PEER_RX_IP ..."
pkill -x iperf3 2>/dev/null; sleep 0.5
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
SRV_PID=$!
# Wait for the server to be ready (brief), then wait for it to exit
# (one-shot server exits after serving one client).
for _ in $(seq 1 60); do
kill -0 "$SRV_PID" 2>/dev/null || break
sleep 1
done
wait "$SRV_PID" 2>/dev/null
echo "[receiver] iperf3 server finished."
else
# sender: wait briefly for receiver to be listening, then run client.
echo "[sender] waiting 3s for receiver to listen, then running iperf3 client..."
sleep 3
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
> "$TMP/iperf.out" 2>&1
echo "[sender] iperf3 client finished (exit=$?)."
fi
snapshot > "$TMP/after"
{
echo ""
echo "--- iperf3 output (raw tail) ---"
tail -25 "$TMP/iperf.out"
echo ""
if [ "$ROLE" = "receiver" ]; then
analyze "Receiver RX (THE key column)" 2 "$TMP/before" "$TMP/after"
dom="$(dominant 2 "$TMP/before" "$TMP/after")"
pct=${dom%% *}; top=${dom##* }
echo ""
echo "==================================================================="
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
echo " VERDICT: SWITCH NOT distributing across $HOST's two ports"
echo " Dominant slave '$top' = ${pct}% of RX -> hash not effective"
echo " on this LAG. Action: verify/bounce pfv-r3-tor-stor port-channel."
else
echo " VERDICT: switch IS distributing (top slave '$top' = ${pct}%)."
echo " Cap is host-side: softirq/CPU/bridge/NIC coalescing."
fi
echo "==================================================================="
else
analyze "Sender TX (control: should split if host hash=layer3+4)" 3 "$TMP/before" "$TMP/after"
dom="$(dominant 3 "$TMP/before" "$TMP/after")"
pct=${dom%% *}; top=${dom##* }
echo ""
echo "==================================================================="
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
echo " VERDICT: host TX NOT distributing (top slave '$top' = ${pct}%)."
echo " Host xmit_hash_policy is NOT effective despite /proc/net/bonding."
else
echo " VERDICT: host TX distributing OK (top slave '$top' = ${pct}%)."
fi
echo "==================================================================="
fi
} | tee -a "$LOG"
echo ""
echo "Log written: $LOG"
scripts/lib/common.sh
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# lib/common.sh — shared helpers for shell scripts and hooks in this repo.
#
# Source it from any script:
# #!/usr/bin/env bash
# set -euo pipefail
# HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# # shellcheck source=lib/common.sh
# source "$HERE/lib/common.sh" # or the appropriate relative path
#
# This library exists to drive a known cross-project inconsistency to zero:
# every repo used to re-paste the ANSI color block, redefine log_* helpers,
# pick one of three incompatible shebangs, and roll its own docker wrapper.
# Import this once instead.
# Do NOT set -euo pipefail here unconditionally — some callers (git hooks)
# source this file and rely on controlling their own shell options. We only
# guarantee the functions below are defined.
###############################################################################
# Config — override via environment before sourcing if needed
###############################################################################
: "${TEMPLATE_ROOT:=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
export TEMPLATE_ROOT
###############################################################################
# ANSI colors (defined once, used everywhere)
###############################################################################
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
BLUE='\033[0;34m'; BOLD='\033[1m'; NC='\033[0m'
else
RED=''; GREEN=''; YELLOW=''; BLUE=''; BOLD=''; NC=''
fi
export RED GREEN YELLOW BLUE BOLD NC
###############################################################################
# Logging
###############################################################################
log_info() { printf "${BLUE}›${NC} %s\n" "$*"; }
log_ok() { printf "${GREEN}✓${NC} %s\n" "$*"; }
log_warn() { printf "${YELLOW}⚠${NC} %s\n" "$*" >&2; }
log_error() { printf "${RED}✗${NC} %s\n" "$*" >&2; }
log_step() { printf "\n${BOLD}== %s ==${NC}\n" "$*"; }
die() { log_error "$*"; exit 1; }
###############################################################################
# Predicates
###############################################################################
# have <cmd> — return 0 if <cmd> is on PATH
have() { command -v "$1" >/dev/null 2>&1; }
###############################################################################
# Path helpers
###############################################################################
repo_root() {
# Prefer git's notion of the repo root, fall back to $TEMPLATE_ROOT, then pwd.
if git rev-parse --show-toplevel >/dev/null 2>&1; then
git rev-parse --show-toplevel
else
printf '%s\n' "${TEMPLATE_ROOT:-$(pwd)}"
fi
}
###############################################################################
# Privilege helpers
###############################################################################
# as_root — run the remaining args as root via sudo, or directly if already root.
as_root() {
if [ "$(id -u)" -eq 0 ]; then "$@"; else sudo "$@"; fi
}
###############################################################################
# Docker wrapper
###############################################################################
# docker_run <image> <args...>
# Ephemeral container, host-uid ownership, repo mounted at /data, cwd /data.
# Drives the "host stays clean; everything runs in containers" policy and
# ensures output files are owned by the invoking user, not root.
docker_run() {
[ "$#" -ge 1 ] || die "docker_run: image required"
local image="$1"; shift
have docker || die "docker not found on PATH"
local root
root="$(repo_root)"
docker run --rm \
--user "$(id -u):$(id -g)" \
-e HOME=/tmp \
-v "$root:/data" \
-w /data \
"$image" "$@"
}
###############################################################################
# Rule-audit accumulator (used by scripts/check-rules.sh)
# Globals read/written: RULE_PASS RULE_WARN RULE_FAIL
###############################################################################
init_counters() { RULE_PASS=0; RULE_WARN=0; RULE_FAIL=0; }
# check <description> <pass|warn|fail>
check() {
local desc="$1" result="$2"
case "$result" in
pass)
RULE_PASS=$((RULE_PASS + 1))
if [ "${RULE_VERBOSE:-true}" = true ]; then printf " ${GREEN}PASS${NC} %s\n" "$desc"; fi
;;
warn)
RULE_WARN=$((RULE_WARN + 1))
if [ "${RULE_VERBOSE:-true}" = true ]; then printf " ${YELLOW}WARN${NC} %s\n" "$desc"; fi
;;
fail)
RULE_FAIL=$((RULE_FAIL + 1))
printf " ${RED}FAIL${NC} %s\n" "$desc"
;;
*)
die "check(): invalid result '$result' (use pass|warn|fail)"
;;
esac
}
# print_summary_and_exit
print_summary_and_exit() {
if [ "${RULE_VERBOSE:-true}" = true ]; then
printf "\n=== Summary ===\n PASS: %s\n WARN: %s\n FAIL: %s\n\n" \
"$RULE_PASS" "$RULE_WARN" "$RULE_FAIL"
fi
if [ "$RULE_FAIL" -gt 0 ]; then
if [ "${RULE_VERBOSE:-true}" = true ]; then
printf "AUDIT FAILED — %s rule(s) violated.\n" "$RULE_FAIL"
fi
exit 1
fi
if [ "${RULE_VERBOSE:-true}" = true ]; then printf "AUDIT PASSED.\n"; fi
exit 0
}
scripts/perf-matrix.sh
Executable
+242
View File
@@ -0,0 +1,242 @@
#!/bin/bash
###############################################################################
# perf-matrix.sh — Any-to-any performance testing across the PFVCluster
#
# Tests three network planes:
# A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network
# B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x)
# C. Storage I/O: dd read/write to NFS mounts
#
# Prerequisites:
# - iperf3 installed on all hosts (systemd service: iperf3-server)
# - iperf3 installed inside guest VMs
# - SSH key + passwordless sudo on all guest VMs (remote.sh; sshd is the
# only approved access channel — see AGENTS.md)
#
# Usage:
# bash perf-matrix.sh # run all tests
# bash perf-matrix.sh datanet # host-to-host datanet only
# bash perf-matrix.sh guests # guest-to-guest datanet only
# bash perf-matrix.sh storage # NFS I/O only
#
# Environment:
# REMOTE_SH path to tests/remote.sh (auto-detected)
###############################################################################
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}"
LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs"
mkdir -p "$LOG_DIR"
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
# --- Host datanet IPs (VLAN 1000) ---
declare -A DATANET_IP
DATANET_IP[pfv-tsys1]="10.100.100.1"
DATANET_IP[pfv-tsys3]="10.100.100.3"
DATANET_IP[pfv-tsys4]="10.100.100.4"
DATANET_IP[pfv-tsys5]="10.100.100.5"
DATANET_IP[pfv-tsys6]="10.100.100.6"
DATANET_IP[pfv-tsys7]="10.100.100.7"
DATANET_IP[pfv-tsys9]="10.100.100.9"
HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9"
# --- k8s/ultix VM targets (SSH for control; datanet_ip for iperf traffic) ---
# Format: prox_host:vmid:datanet_ip:ssh_host:label
GUEST_TARGETS="
pfv-tsys1:102:10.100.100.10:pfv-k8s-cnode1:cnode1
pfv-tsys7:705:10.100.100.11:pfv-k8s-cnode2:cnode2
pfv-tsys6:603:10.100.100.12:pfv-k8s-cnode3:cnode3
pfv-tsys3:313:10.100.100.13:pfv-k8s-wnode-tsys3:wnode-tsys3
pfv-tsys5:500:10.100.100.14:pfv-k8s-wnode-tsys5:wnode-tsys5
pfv-tsys6:601:10.100.100.15:pfv-k8s-wnode-tsys6:wnode-tsys6
pfv-tsys7:701:10.100.100.16:pfv-k8s-wnode-tsys7:wnode-tsys7
pfv-tsys9:905:10.100.100.17:pfv-k8s-wnode-tsys9:wnode-tsys9
pfv-tsys5:5111:10.100.100.18:ultix-streaming:ultix-streaming
pfv-tsys5:5112:10.100.100.19:ultix-offstage:ultix-offstage
"
DURATION="${DURATION:-3}" # seconds per iperf3 test
STREAMS="${STREAMS:-4}" # parallel streams
# ============================================================================
# Helpers
# ============================================================================
start_iperf_servers() {
echo "--- Starting iperf3 servers on all hosts ---"
for h in $HOSTS; do
PROX_HOST="$h" bash "$REMOTE_SH" prox \
'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \
>/dev/null 2>&1 &
done
wait
echo " All servers started."
}
stop_iperf_servers() {
echo "--- Stopping iperf3 servers on all hosts ---"
for h in $HOSTS; do
PROX_HOST="$h" bash "$REMOTE_SH" prox \
'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \
>/dev/null 2>&1 &
done
wait
echo " All servers stopped."
}
# ============================================================================
# A. Host-to-host datanet matrix
# ============================================================================
test_datanet() {
echo ""
echo "==================================================================="
echo " DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix"
echo " ${STREAMS} parallel streams, ${DURATION}s per test"
echo "==================================================================="
start_iperf_servers
local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv"
echo "host_from,host_to,mbps" > "$outfile"
for client in $HOSTS; do
for server in $HOSTS; do
[ "$client" = "$server" ] && continue
local sip="${DATANET_IP[$server]}"
local result
result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \
"iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
| awk '/SUM.*receiver/{printf "%.0f", $6}')
if [ -n "$result" ]; then
printf " %-14s → %-14s : %s Mbps\n" "$client" "$server" "$result"
echo "$client,$server,$result" >> "$outfile"
else
printf " %-14s → %-14s : FAIL\n" "$client" "$server"
echo "$client,$server,FAIL" >> "$outfile"
fi
done
done
stop_iperf_servers
echo ""
echo " Results saved: $outfile"
}
# ============================================================================
# B. Guest-to-guest datanet
# ============================================================================
test_guests() {
echo ""
echo "==================================================================="
echo " DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)"
echo " ${STREAMS} parallel streams, ${DURATION}s per test"
echo "==================================================================="
# Start iperf3 server on the first guest (cnode1)
local server_entry
server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1)
local s_ip s_ssh s_label
s_ip=$(echo "$server_entry" | cut -d: -f3)
s_ssh=$(echo "$server_entry" | cut -d: -f4)
s_label=$(echo "$server_entry" | cut -d: -f5)
echo " Starting iperf3 server on $s_label ($s_ip)..."
VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \
'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1
sleep 1
local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv"
echo "guest_from,guest_to,mbps" > "$outfile"
while read -r entry; do
[ -z "$entry" ] && continue
local c_ip c_ssh c_label
c_ip=$(echo "$entry" | cut -d: -f3)
c_ssh=$(echo "$entry" | cut -d: -f4)
c_label=$(echo "$entry" | cut -d: -f5)
[ "$c_ip" = "$s_ip" ] && continue
local result
result=$(VM_IP="$c_ssh" bash "$REMOTE_SH" vmroot \
"iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
| awk '/SUM.*receiver/{printf "%.0f", $6}')
if [ -n "$result" ]; then
printf " %-18s → %-18s : %s Mbps\n" "$c_label" "$s_label" "$result"
echo "$c_label,$s_label,$result" >> "$outfile"
else
printf " %-18s → %-18s : FAIL\n" "$c_label" "$s_label"
echo "$c_label,$s_label,FAIL" >> "$outfile"
fi
done <<< "$GUEST_TARGETS"
# Cleanup
VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \
'pkill iperf3' >/dev/null 2>&1
echo ""
echo " Results saved: $outfile"
}
# ============================================================================
# C. Storage I/O (NFS read/write)
# ============================================================================
test_storage() {
echo ""
echo "==================================================================="
echo " STORAGE I/O — NFS Read/Write (100MB dd)"
echo "==================================================================="
local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv"
echo "host,mount,write_mbps,read_mbps" > "$outfile"
local ddscript="/tmp/perf-dd-$$.sh"
cat > "$ddscript" <<'DDSCRIPT'
#!/bin/bash
nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc)
for m in $nfs; do
tf="$m/.perf-$$"
w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
rm -f "$tf" 2>/dev/null
echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s"
done
DDSCRIPT
for h in $HOSTS; do
echo ""
echo " --- $h ---"
PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do
[ -n "$line" ] && echo " $line"
done
done
rm -f "$ddscript"
echo ""
echo " Results saved: $outfile"
}
# ============================================================================
# Main
# ============================================================================
MODE="${1:-all}"
case "$MODE" in
datanet|a) test_datanet ;;
guests|b) test_guests ;;
storage|c) test_storage ;;
all|"") test_datanet; test_guests; test_storage ;;
*)
echo "Usage: $0 [datanet|guests|storage|all]"
echo ""
echo " datanet — host-to-host bandwidth matrix over VLAN 1000"
echo " guests — guest-to-guest (k8s/ultix VMs over VLAN 1000)"
echo " storage — NFS read/write I/O"
echo " all — run all three (default)"
exit 1
;;
esac
echo ""
echo "==================================================================="
echo " Perf testing complete. Logs in: $LOG_DIR/"
echo "==================================================================="
scripts/pre-commit
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# pre-commit — fast rule audit (< 1s typical).
# Hot-path bypass: commits that ONLY touch STATUS.md / WORKING.md skip the
# audit so frequent status/task commits stay frictionless.
set -euo pipefail
REPO_ROOT="$(git rev-parse --show-toplevel)"
cd "$REPO_ROOT"
CHANGED="$(git diff --cached --name-only)"
HOT_PATHS="$(printf '%s\n' "$CHANGED" | grep -vE '^(STATUS.md|WORKING.md)$' || true)"
if [ -z "$HOT_PATHS" ]; then
echo "hot-path files only (STATUS/WORKING) — skipping rule audit"
exit 0
fi
if ! bash scripts/check-rules.sh --fast; then
echo ""
echo "pre-commit audit FAILED. Fix the violations above before committing."
echo "Full audit: bash scripts/check-rules.sh"
echo "Bypass: git commit --no-verify (emergencies only)"
exit 1
fi
exit 0
scripts/pre-push
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# pre-push — full rule audit + clean-working-tree gate before pushing.
# Installed via: bash scripts/setup-hooks.sh
#
# Combines two proven policies observed across projects:
# - KNEL-AIMiddleware: block push if the working tree is dirty.
# - RCEO-PersonalAssistant: block push if the full test suite fails.
set -euo pipefail
REPO_ROOT="$(git rev-parse --show-toplevel)"
cd "$REPO_ROOT"
echo "pre-push: running full rule audit..."
# Full audit (non-fast): runs the slow test suite via `make test` if present.
if ! bash scripts/check-rules.sh --quiet; then
echo ""
echo "pre-push audit FAILED. Push blocked."
echo "Re-run with output: bash scripts/check-rules.sh"
echo "Bypass: git push --no-verify (emergencies only)"
exit 1
fi
echo "pre-push: all rules and tests passed."
exit 0
scripts/probe-drift.sh
+79
View File
@@ -0,0 +1,79 @@
#!/bin/bash
# Read-only fleet drift probe — gathers package/tooling/config state for
# consistency comparison across hosts. Writes only stdout.
set -u
echo "===== DRIFT PROBE: $(hostname -s) $(date -u +%FT%TZ) ====="
echo
echo "##### OS / kernel / PVE #####"
pveversion 2>&1
cat /etc/debian_version 2>&1
echo
echo "##### Installed key packages (versions) #####"
dpkg-query -W -f='${Package}\t${Version}\n' 2>/dev/null | grep -iE 'lldpd|lldpad|smartmontools|nfs-common|nfs-kernel-server|iperf3|tcpdump|htop|rsyslog|qemu-guest-agent|snmpd|snmp|net-tools|ethtool|sysstat|ioping|fio|nvme-cli|conman|ser2net|nut-server|nut-client|tuned-adm|tuned' 2>/dev/null | sort
echo
echo "##### lldpcli present? #####"
command -v lldpcli >/dev/null 2>&1 && lldpcli -v 2>&1 | head -1 || echo "lldpcli: NOT INSTALLED"
echo
echo "##### lldpd service #####"
systemctl is-active lldpd 2>&1 || true
systemctl is-enabled lldpd 2>&1 || true
echo
echo "##### smartmontools service #####"
systemctl is-active smartd 2>&1 || echo "smartd: inactive/disabled"
systemctl is-enabled smartd 2>&1 || true
echo
echo "##### snmpd service #####"
systemctl is-active snmpd 2>&1 || echo "snmpd: not running"
systemctl is-enabled snmpd 2>&1 || true
echo
echo "##### rsyslog #####"
systemctl is-active rsyslog 2>&1 || echo "rsyslog: not running"
echo
echo "##### tuned profile #####"
tuned-adm active 2>&1 | head -2 || echo "tuned: not available"
echo
echo "##### sshd config (key settings) #####"
grep -E '^(PermitRootLogin|PasswordAuthentication|PubkeyAuthentication|KbdInteractiveAuthentication|Port |PermitEmptyPasswords)' /etc/ssh/sshd_config 2>/dev/null || echo "(defaults)"
grep -E '^(PermitRootLogin|PasswordAuthentication|PubkeyAuthentication|KbdInteractiveAuthentication|Port |PermitEmptyPasswords)' /etc/ssh/sshd_config.d/*.conf 2>/dev/null || true
echo
echo "##### authorized_keys (root) — count + key types #####"
if [ -f /root/.ssh/authorized_keys ]; then
echo " total keys: $(wc -l < /root/.ssh/authorized_keys)"
awk '{print " "$1}' /root/.ssh/authorized_keys | sort | uniq -c
else
echo " NO authorized_keys for root"
fi
echo
echo "##### ssh host key fingerprints #####"
for f in /etc/ssh/ssh_host_*_key.pub; do
[ -f "$f" ] && ssh-keygen -lf "$f" 2>/dev/null
done
echo
echo "##### sysctl tuning (key values) #####"
echo " net.core.rmem_max=$(cat /proc/sys/net/core/rmem_max 2>/dev/null)"
echo " net.core.wmem_max=$(cat /proc/sys/net/core/wmem_max 2>/dev/null)"
echo " net.ipv4.tcp_congestion_control=$(cat /proc/sys/net/ipv4/tcp_congestion_control 2>/dev/null)"
echo " net.ipv4.tcp_rmem=$(cat /proc/sys/net/ipv4/tcp_rmem 2>/dev/null)"
echo " net.ipv4.tcp_wmem=$(cat /proc/sys/net/ipv4/tcp_wmem 2>/dev/null)"
echo " net.ipv4.tcp_max_syn_backlog=$(cat /proc/sys/net/ipv4/tcp_max_syn_backlog 2>/dev/null)"
echo " net.core.netdev_max_backlog=$(cat /proc/sys/net/core/netdev_max_backlog 2>/dev/null)"
echo " vm.swappiness=$(cat /proc/sys/vm/swappiness 2>/dev/null)"
echo
echo "##### CPU governor #####"
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "no cpufreq driver"
echo
echo "##### beszel agent? #####"
systemctl is-active beszel-hub 2>/dev/null || systemctl is-active beszel-agent 2>/dev/null || echo "beszel: not installed"
echo
echo "##### /root/bin or custom scripts present? #####"
# shellcheck disable=SC2012 # diagnostic listing, not for processing
find /root/bin/ -maxdepth 1 -type f 2>/dev/null | head -10 || echo "(no /root/bin)"
find /root/ -maxdepth 1 -name '*.sh' -type f 2>/dev/null | head -10 || echo "(no /root/*.sh)"
echo
echo "##### mount options (noatime check on root) #####"
mount | grep ' / ' | head -1
echo
echo "##### NFS server threads (if NFS server) #####"
grep -E '^[[:space:]]*RPCNFSDCOUNT' /etc/default/nfs-kernel-server 2>/dev/null || grep -E 'RPCNFSDCOUNT' /etc/default/nfs-kernel-server 2>/dev/null || echo "(default, not a NFS server or default threads)"
echo
echo "===== END $(hostname -s) ====="
scripts/probe-network.sh
+82
View File
@@ -0,0 +1,82 @@
#!/bin/bash
###############################################################################
# probe-network.sh
#
# READ-ONLY network + NFS ground-truth probe. Writes only stdout.
# Run on any Proxmox host to inventory its NICs, bonds, LLDP neighbors,
# NFS client mounts (including nconnect), ethtool link state, and error
# counters. No hardcoded values — fully portable.
#
# Usage (via tests/remote.sh):
# PROX_HOST=pfv-tsys6 bash tests/remote.sh prox-file perf/scripts/probe-network.sh
###############################################################################
set -u
echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) ====="
echo
echo "##### ip -br link #####"
ip -br link 2>&1
echo
echo "##### ip -br addr #####"
ip -br addr 2>&1
echo
echo "##### /etc/network/interfaces #####"
cat /etc/network/interfaces 2>&1
echo
echo "##### bond0 state (if present) #####"
if [ -r /proc/net/bonding/bond0 ]; then
cat /proc/net/bonding/bond0 2>&1
else
echo "(no bond0)"
fi
echo
echo "##### ethtool per physical NIC #####"
for nic in /sys/class/net/*; do
nic=$(basename "$nic")
case "$nic" in lo|bond*|br*|venet*|veth*|docker*|tap*|vnet*|fw*) continue;; esac
echo "--- ethtool $nic ---"
ethtool "$nic" 2>&1 | grep -iE 'Speed|Duplex|Port|Link|Supported link modes|Advertising|Auto-neg|Settings' || echo "(ethtool failed for $nic)"
done
echo
echo "##### lldpcli (if installed) #####"
if command -v lldpcli >/dev/null 2>&1; then
echo "--- lldpcli show neighbors ---"
lldpcli show neighbors 2>&1
echo
echo "--- lldpcli show interfaces ---"
lldpcli show interfaces 2>&1
echo
echo "--- lldpcli show chassis ---"
lldpcli show chassis 2>&1
else
echo "(lldpcli not installed)"
fi
echo
echo "##### lldpd / lldpad service #####"
systemctl is-active lldpd 2>&1 || true
systemctl is-enabled lldpd 2>&1 || true
echo
echo "##### NFS mounts (mount | grep nfs) #####"
mount | grep -i nfs 2>&1 || echo "(no nfs mounts)"
echo
echo "##### mount nconnect detail (nfsstat -m) #####"
nfsstat -m 2>&1
echo
echo "##### storage.cfg NFS stanzas (options) #####"
grep -A3 '^nfs:' /etc/pve/storage.cfg 2>&1
echo
echo "##### ip route #####"
ip route 2>&1
echo
echo "##### ethtool -S bond slaves (key counters) #####"
if [ -r /proc/net/bonding/bond0 ]; then
# shellcheck disable=SC2013 # intentional: extract NIC names from bonding info
for nic in $(grep -oE 'eth[0-9]+|en[psx][a-z0-9]+' /proc/net/bonding/bond0 2>/dev/null | sort -u); do
echo "--- ethtool -S $nic (errors) ---"
ethtool -S "$nic" 2>/dev/null | grep -iE 'error|drop|discard|crc|pause|miss' || echo "(no error counters)"
done
fi
echo
echo "##### ip neigh (ARP table, reachable/stale) #####"
ip neigh show 2>&1 | grep -vE ' FAILED|INCOMPLETE' | sort -t. -k4 -n
echo
echo "===== END $(hostname -s) ====="
scripts/probe-storage.sh
+75
View File
@@ -0,0 +1,75 @@
#!/bin/bash
###############################################################################
# probe-storage.sh
#
# READ-ONLY storage + disk ground-truth probe. Writes only stdout.
# Run on any Proxmox host (or any Linux NFS server) to inventory its physical
# disks, mounts, exports, SMART health, and Proxmox storage config.
#
# Portable: no hardcoded values. Uses only standard CLI tools + smartmontools.
#
# Usage (via tests/remote.sh):
# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file perf/scripts/probe-storage.sh
#
# Or directly on a host:
# bash probe-storage.sh > storage-audit.txt
###############################################################################
set -u
echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) ====="
echo
echo "##### lsblk (tree, with model/serial/size/type) #####"
lsblk -o NAME,MAJ:MIN,SIZE,TYPE,MOUNTPOINT,MODEL,SERIAL,STATE,ROTA,TRAN,REV 2>&1
echo
echo "##### block devices by-id #####"
for dev in /dev/disk/by-id/*; do
[ -L "$dev" ] || continue
case "$(basename "$dev")" in *part[0-9]*) continue;; esac
ls -l "$dev"
done 2>&1
echo
echo "##### nvme list (if any) #####"
command -v nvme >/dev/null 2>&1 && nvme list 2>&1 || echo "(no nvme-cli or no nvme devices)"
echo
echo "##### blkid #####"
blkid 2>&1
echo
echo "##### mounted filesystems #####"
findmnt -o TARGET,SOURCE,FSTYPE,OPTIONS 2>&1
echo
echo "##### /etc/fstab #####"
cat /etc/fstab 2>&1
echo
echo "##### /etc/exports (+ exports.d) #####"
cat /etc/exports 2>&1
for f in /etc/exports.d/*.exports; do [ -f "$f" ] && echo "--- $f ---" && cat "$f"; done 2>&1
echo
echo "##### df -h (all mounts) #####"
df -h 2>&1
echo
echo "##### smartctl -a per block device #####"
command -v smartctl >/dev/null 2>&1 || echo "(smartctl not installed)"
for d in /dev/sd? /dev/nvme?n1; do
[ -b "$d" ] || continue
echo "----- smartctl -a $d -----"
smartctl -a "$d" 2>&1 | grep -iE 'Device Model|Model Number|Serial|Firmware|User Capacity|Rotation Rate|Form Factor|SATA Version|NVMe|SMART overall|Reallocated|Pending|Uncorrect|Power On|Temperature|Media and Data Integrity' || true
done
echo
echo "##### /etc/pve/storage.cfg #####"
cat /etc/pve/storage.cfg 2>&1
echo
echo "##### pvesm status #####"
pvesm status 2>&1
echo
echo "##### pvesm list per store #####"
for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3>0 {print $1}'); do
echo "--- pvesm list $s ---"
pvesm list "$s" 2>&1 | head -40
done
echo
echo "##### zpool status (if any) #####"
command -v zpool >/dev/null 2>&1 && zpool status 2>&1 || echo "(no zfs)"
echo
echo "##### lvm: pvs/vgs/lvs #####"
command -v pvs >/dev/null 2>&1 && { pvs 2>&1; echo; vgs 2>&1; echo; lvs 2>&1; } || echo "(no lvm tools)"
echo
echo "===== END $(hostname -s) ====="
scripts/probe-vm-dns.sh
+44
View File
@@ -0,0 +1,44 @@
#!/bin/bash
# Read-only DNS consistency checker for Proxmox VMs.
# Checks that every running VM's name resolves to the IP it's actually using,
# and that reverse DNS (PTR) matches the VM name. Mismatches are reported as bugs.
set -u
echo "===== VM DNS CONSISTENCY CHECK: $(hostname -s) $(date -u +%FT%TZ) ====="
echo
echo "##### Running VMs with IPs (via guest-agent or ARP) #####"
for vmid in $(qm list 2>/dev/null | awk 'NR>1 && $4=="running" {print $1}'); do
name=$(qm config "$vmid" 2>/dev/null | awk '/^name:/{print $2}')
# Get the VM's net0 config to determine the bridge
net0=$(qm config "$vmid" 2>/dev/null | awk '/^net0:/{print $0}')
echo ""
echo "--- VMID $vmid: $name ---"
echo " net0: ${net0//net0: /}"
# Try to get IP via guest-agent
if qm config "$vmid" 2>/dev/null | grep -q 'agent:.*enabled=1\|^agent: 1'; then
guest_ips=$(qm guest cmd "$vmid" network-get-interfaces 2>/dev/null)
if [ -n "$guest_ips" ]; then
echo " guest-agent IPs:"
echo "$guest_ips" | python3 -c "
import sys, json
try:
data = json.load(sys.stdin)
for iface in data:
name = iface.get('name','?')
for addr in iface.get('ip-addresses',[]):
ip = addr.get('ip-address','')
typ = addr.get('ip-address-type','')
if typ == 'ipv4' and not ip.startswith('127.'):
print(f' {name}: {ip}')
except: pass
" 2>/dev/null
else
echo " guest-agent: no response"
fi
else
echo " guest-agent: not enabled"
fi
done
echo
echo "===== END $(hostname -s) ====="
scripts/run-lacp-retrans-cause.sh
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env bash
###############################################################################
# run-lacp-retrans-cause.sh (workstation wrapper)
#
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
# right order, scps logs back, prints them.
###############################################################################
set -uo pipefail
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
-o StrictHostKeyChecking=accept-new)
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
RECV="pfv-tsys6"; SEND="pfv-tsys7"
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
SCRIPT="lacp-retrans-cause.sh"
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
REMOTE_SCRIPT="/root/${SCRIPT}"
mkdir -p "$LOCAL_LOG_DIR"
echo "==================================================================="
echo " LACP retransmit-cause investigation"
echo " Receiver: $RECV Sender: $SEND"
echo "==================================================================="
echo ""
echo "--- preflight ---"
for h in "$RECV" "$SEND"; do
printf ' %-12s ' "$h"
ssh "${SSH[@]}" "root@$h" \
'command -v ethtool >/dev/null && e=OK || e=MISSING
command -v iperf3 >/dev/null && i=OK || i=MISSING
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
done
echo ""
echo "--- deploy ---"
for h in "$RECV" "$SEND"; do
printf ' %-12s ' "$h"
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
&& echo "deployed" || echo "FAILED"
done
echo ""
echo "--- cleanup stale iperf3 ---"
for h in "$RECV" "$SEND"; do
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
done
echo ""
echo "--- launching receiver on $RECV (background) ---"
ssh "${SSH[@]}" "root@$RECV" \
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
sleep 5
echo ""
echo "--- running sender on $SEND (foreground, ~20s) ---"
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
echo ""
sleep 3
echo "--- fetching logs ---"
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
src=""
case "$f" in
*receiver*) src="$RECV" ;;
*sender*) src="$SEND" ;;
esac
printf ' %-32s <- %s : ' "$f" "$src"
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
else
echo "MISSING"
fi
done
echo ""
echo "==================================================================="
echo " RECEIVER LOG ($RECV)"
echo "==================================================================="
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
echo ""
echo "==================================================================="
echo " SENDER LOG ($SEND)"
echo "==================================================================="
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
echo ""
echo "==================================================================="
echo " Local copies in: $LOCAL_LOG_DIR/"
echo "==================================================================="
scripts/run-lacp-rx-distribution.sh
Executable
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
###############################################################################
# run-lacp-rx-distribution.sh (workstation wrapper)
#
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
# scps both logs back to returned-logs/iperf/.
#
# Run from the workstation:
# bash scripts/run-lacp-rx-distribution.sh
#
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
###############################################################################
set -uo pipefail
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
-o StrictHostKeyChecking=accept-new)
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
RECV="pfv-tsys6" # receiver (we care most about its RX split)
SEND="pfv-tsys7" # sender (control: its TX split)
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
SCRIPT="lacp-rx-distribution.sh"
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
REMOTE_SCRIPT="/root/${SCRIPT}"
mkdir -p "$LOCAL_LOG_DIR"
echo "==================================================================="
echo " LACP per-slave RX/TX distribution test"
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
echo "==================================================================="
echo ""
# 0. Preflight: confirm SSH and iperf3 on both hosts
echo "--- preflight (ssh + iperf3 + bond0) ---"
for h in "$RECV" "$SEND"; do
printf ' %-12s ' "$h"
ssh "${SSH[@]}" "root@$h" \
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
2>&1 | head -1
done
echo ""
# 1. Copy the script to both hosts + chmod
echo "--- deploy $SCRIPT to both hosts ---"
for h in "$RECV" "$SEND"; do
printf ' %-12s ' "$h"
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
&& echo "deployed + chmod +x" \
|| echo "DEPLOY FAILED"
done
echo ""
# 2. Kill any stale iperf3 on both hosts
echo "--- cleanup stale iperf3 ---"
for h in "$RECV" "$SEND"; do
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
done
echo ""
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
echo "--- starting receiver on $RECV (background) ---"
ssh "${SSH[@]}" "root@$RECV" \
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
sleep 5
echo ""
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
echo "--- running sender on $SEND (foreground, ~20s) ---"
ssh "${SSH[@]}" "root@$SEND" \
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
echo ""
# 5. Give receiver a moment to finish writing its log
sleep 3
# 6. Fetch logs back
echo "--- fetching logs ---"
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
src=""
case "$f" in
lacp-rx-receiver*) src="$RECV" ;;
lacp-rx-sender*) src="$SEND" ;;
esac
printf ' %-28s <- %s : ' "$f" "$src"
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
else
echo "MISSING"
fi
done
echo ""
# 7. Show the receiver log (the decisive one)
echo "==================================================================="
echo " RECEIVER LOG ($RECV — the decisive side)"
echo "==================================================================="
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
echo ""
echo "==================================================================="
echo " SENDER LOG ($SEND — control)"
echo "==================================================================="
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
echo ""
echo "==================================================================="
echo " Local copies in: $LOCAL_LOG_DIR/"
echo "==================================================================="
scripts/setup-hooks.sh
Executable
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
# setup-hooks.sh — install this repo's git hooks.
#
# Mechanism: copy scripts/pre-commit and scripts/pre-push into .git/hooks/ and
# make them executable. This is the most portable pattern (works on any clone,
# no `git config core.hooksPath` mutation, survives config resets, idempotent).
#
# Run once after cloning: bash scripts/setup-hooks.sh
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
# shellcheck disable=SC1091
source "$HERE/lib/common.sh"
REPO_ROOT="$(repo_root)"
cd "$REPO_ROOT"
[ -d .git ] || die "no .git directory here — run this from a git checkout"
HOOKS_DIR=".git/hooks"
HOOK_NAMES="pre-commit pre-push"
log_step "Installing git hooks"
for name in $HOOK_NAMES; do
src="scripts/$name"
dst="$HOOKS_DIR/$name"
[ -f "$src" ] || die "source hook not found: $src"
cp "$src" "$dst"
chmod +x "$dst"
log_ok "installed $dst"
done
cat <<EOF
Git hooks installed. The following now run automatically:
pre-commit fast rule audit (shellcheck, image pinning, container naming,
required files, doc freshness, Discourse pointers, WORKING.md
completion, hygiene).
Hot-path bypass for STATUS.md / WORKING.md.
pre-push full rule audit (includes scripts/test.sh) + clean-working-tree gate.
Bypass either with \`git commit --no-verify\` / \`git push --no-verify\`
(emergencies only).
EOF
scripts/snmp-switch-audit.py
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""
snmp-switch-audit.py — READ-ONLY switch inventory via SNMP.
Gathers interface status/speed/errors, LLDP neighbor topology, port-channel
(LAG) membership, and VLAN membership from any SNMPv2c-capable switch.
Designed for Dell/Radlan (Neyland) and standard IF/LLDP/Q-BRIDGE MIB switches,
but works on any SNMP-manageable device.
All parameters configurable via env vars or CLI flags so this works on any
network:
SNMP_COMMUNITY SNMPv2c community string (env, default: public)
SWITCH_IPS space-separated switch IPs (env, or pass as args)
OUTPUT_DIR where to write per-switch (env, default: returned-logs/snmp)
Usage:
SNMP_COMMUNITY=kn3lmgmt SWITCH_IPS="192.168.0.9 192.168.0.12" \\
python3 snmp-switch-audit.py
# or pass IPs as positional args:
SNMP_COMMUNITY=kn3lmgmt python3 snmp-switch-audit.py 192.168.0.9 192.168.0.12
Requires: pysnmp (pip install pysnmp) or net-snmp utils (snmpwalk) on PATH.
Outputs: per-switch JSON + human-readable text in OUTPUT_DIR.
Read-only: sends only SNMP GET/GETNEXT/GETBULK. Never SETs anything.
"""
import argparse
import json
import os
import re
import subprocess
import sys
COMMUNITY = os.environ.get("SNMP_COMMUNITY", "public")
OUTPUT_DIR = os.environ.get("OUTPUT_DIR",
os.path.join(os.path.dirname(__file__), "..",
"..", "returned-logs", "snmp"))
# OID constants
OID_SYSDESCR = "1.3.6.1.2.1.1.1.0"
OID_SYSNAME = "1.3.6.1.2.1.1.5.0"
OID_IF_NAME = "1.3.6.1.2.1.31.1.1.1.1"
OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5"
OID_IF_OPER = "1.3.6.1.2.1.2.2.1.8"
OID_IF_INERR = "1.3.6.1.2.1.2.2.1.14"
OID_IF_OUTERR = "1.3.6.1.2.1.2.2.1.20"
OID_IF_INOCT = "1.3.6.1.2.1.31.1.1.1.6"
OID_IF_OUTOCT = "1.3.6.1.2.1.31.1.1.1.10"
OID_LACP_LAG = "1.2.840.10006.300.43.1.1.1.1"
OID_LLDP_REM_PORT = "1.0.8802.1.1.2.1.4.1.1.7"
OID_LLDP_REM_SYSNAME = "1.0.8802.1.1.2.1.4.1.1.9"
OID_LLDP_REM_CHASSIS = "1.0.8802.1.1.2.1.4.1.1.6"
OID_LLDP_REM_LOCALPORT = "1.0.8802.1.1.2.1.4.1.1.3"
OID_QBRIDGE_VLAN = "1.3.6.1.2.1.17.7.1.4.3.1.1"
def snmpget(ip, oid):
"""Single SNMP GET, returns string value or None."""
try:
r = subprocess.run(
["snmpget", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
capture_output=True, text=True, timeout=10)
if r.returncode == 0 and r.stdout.strip():
return r.stdout.strip().strip('"')
except (subprocess.TimeoutExpired, FileNotFoundError):
pass
return None
def snmpwalk(ip, oid):
"""SNMP BULKWALK, returns dict of ifIndex -> value."""
try:
r = subprocess.run(
["snmpbulkwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
capture_output=True, text=True, timeout=30)
if r.returncode != 0:
r = subprocess.run(
["snmpwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
capture_output=True, text=True, timeout=30)
except (subprocess.TimeoutExpired, FileNotFoundError):
return {}
result = {}
for i, line in enumerate(r.stdout.strip().split("\n"), 1):
line = line.strip().strip('"')
if line:
result[i] = line
return result
def walk_indexed(ip, oid):
"""SNMP walk preserving OID index. Returns dict: index_str -> value."""
try:
r = subprocess.run(
["snmpbulkwalk", "-v2c", "-c", COMMUNITY, ip, oid],
capture_output=True, text=True, timeout=30)
if r.returncode != 0:
r = subprocess.run(
["snmpwalk", "-v2c", "-c", COMMUNITY, ip, oid],
capture_output=True, text=True, timeout=30)
except (subprocess.TimeoutExpired, FileNotFoundError):
return {}
result = {}
for line in r.stdout.strip().split("\n"):
m = re.search(r'(\d+)\s*=\s*(.+)', line)
if not m:
m = re.search(r'\.(\d+)\s*=\s*(.+)', line)
if m:
idx = m.group(1).split(".")[-1]
val = m.group(2).strip()
val = re.sub(r'^(INTEGER: |STRING: |Hex-STRING: |Gauge32: |Counter32: |Counter64: )', '', val)
result[idx] = val
return result
def audit_switch(ip):
"""Gather all data for one switch."""
data = {"ip": ip}
data["sysDescr"] = snmpget(ip, OID_SYSDESCR)
data["sysName"] = snmpget(ip, OID_SYSNAME)
if not data["sysDescr"]:
return data
names = snmpwalk(ip, OID_IF_NAME)
speeds = snmpwalk(ip, OID_IF_SPEED)
oper = snmpwalk(ip, OID_IF_OPER)
inerr = snmpwalk(ip, OID_IF_INERR)
outerr = snmpwalk(ip, OID_IF_OUTERR)
interfaces = []
for idx in sorted(names.keys()):
if idx not in names:
continue
speed_raw = speeds.get(idx, "0")
try:
speed_mbps = int(re.sub(r'\D', '', str(speed_raw))) // 1000000
except (ValueError, TypeError):
speed_mbps = 0
is_up = str(oper.get(idx, "0")).strip() == "1"
interfaces.append({
"ifIndex": idx,
"name": names[idx],
"speedMbps": speed_mbps,
"up": is_up,
"inErrors": inerr.get(idx, "0"),
"outErrors": outerr.get(idx, "0"),
})
data["interfaces"] = interfaces
# LLDP neighbors
rem_ports = walk_indexed(ip, OID_LLDP_REM_PORT)
rem_sysnames = walk_indexed(ip, OID_LLDP_REM_SYSNAME)
rem_chassis = walk_indexed(ip, OID_LLDP_REM_CHASSIS)
rem_local = walk_indexed(ip, OID_LLDP_REM_LOCALPORT)
lldp = []
for idx in rem_ports:
lldp.append({
"localPort": rem_local.get(idx, "?"),
"remotePort": rem_ports[idx],
"remoteSysName": rem_sysnames.get(idx, ""),
"remoteChassis": rem_chassis.get(idx, ""),
})
data["lldpNeighbors"] = lldp
# LACP LAG table
lag_data = walk_indexed(ip, OID_LACP_LAG)
data["lagTable"] = lag_data
# VLAN membership
vlan_data = walk_indexed(ip, OID_QBRIDGE_VLAN)
data["vlans"] = vlan_data
return data
def print_switch(data):
"""Human-readable summary."""
print(f"\n{'='*60}")
print(f" {data.get('sysName', data['ip'])} ({data['ip']})")
print(f" {data.get('sysDescr', '?')}")
print(f"{'='*60}")
print(f"\n Active ports (UP only):")
print(f" {'Port':<12} {'Speed':>10} {'InErrors':>10} {'OutErrors':>10}")
print(f" {'-'*12} {'-'*10} {'-'*10} {'-'*10}")
for iface in data.get("interfaces", []):
if iface["up"]:
print(f" {iface['name']:<12} {iface['speedMbps']:>8}Mb "
f"{iface['inErrors']:>10} {iface['outErrors']:>10}")
err_ports = [i for i in data.get("interfaces", [])
if i["up"] and (int(i["inErrors"] or 0) > 0
or int(i["outErrors"] or 0) > 0)]
if err_ports:
print(f"\n *** PORTS WITH ERRORS ***")
for p in err_ports:
print(f" {p['name']}: inErr={p['inErrors']} outErr={p['outErrors']}")
if data.get("lldpNeighbors"):
print(f"\n LLDP neighbors:")
for n in data["lldpNeighbors"]:
sysname = n.get("remoteSysName", "") or "(unknown)"
print(f" local={n['localPort']:<6} remote={n['remotePort']:<20} {sysname}")
if data.get("lagTable"):
print(f"\n LACP/LAG table entries: {len(data['lagTable'])}")
def main():
ap = argparse.ArgumentParser(
description="READ-ONLY SNMP switch audit (portable, config-driven)")
ap.add_argument("switches", nargs="*",
help="switch IPs (env: SWITCH_IPS)")
ap.add_argument("--community", default=COMMUNITY,
help=f"SNMPv2c community (env: SNMP_COMMUNITY, default: {COMMUNITY})")
ap.add_argument("--output", default=OUTPUT_DIR,
help=f"output dir (env: OUTPUT_DIR)")
args = ap.parse_args()
community = args.community
ips = args.switches
if not ips:
env_ips = os.environ.get("SWITCH_IPS", "")
ips = env_ips.split()
if not ips:
ap.error("no switch IPs provided (pass as args or set SWITCH_IPS)")
os.makedirs(args.output, exist_ok=True)
all_data = []
for ip in ips:
globals()["COMMUNITY"] = community
data = audit_switch(ip.strip())
all_data.append(data)
print_switch(data)
outpath = os.path.join(args.output, f"switch-{ip}.json")
with open(outpath, "w") as f:
json.dump(data, f, indent=2)
print(f"\n -> {outpath}")
combined = os.path.join(args.output, "switches-all.json")
with open(combined, "w") as f:
json.dump(all_data, f, indent=2)
print(f"\n Combined: {combined}")
return 0
if __name__ == "__main__":
sys.exit(main())