mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
440 lines
17 KiB
Bash
Executable File
440 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
###############################################################################
|
|
# apply-tunings.sh
|
|
#
|
|
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
|
|
# 1. Sets scaling_governor=performance on every CPU
|
|
# 2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
|
|
# 3. Enables tcp_bbr module + sets congestion control
|
|
# 4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
|
|
# 5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
|
|
# 6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
|
|
# 7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
|
|
# restart which drops the host. Provided as a separate --emit-bond-patch
|
|
# flag that PRINTS the change but does not apply it.)
|
|
#
|
|
# Safety features:
|
|
# - Dry-run mode by default (--apply to commit)
|
|
# - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
|
|
# - Per-host behaviour: detects storage hosts by hostname and applies the
|
|
# right profile (tsys4, tsys5 = storage; others = VM hosts).
|
|
# - Generates a /root/perfopt-rollback.sh that undoes everything.
|
|
# - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
|
|
#
|
|
# Usage:
|
|
# bash apply-tunings.sh # dry run, show what would change
|
|
# bash apply-tunings.sh --apply # commit changes
|
|
# bash apply-tunings.sh --rollback # restore from latest backup
|
|
# bash apply-tunings.sh --emit-bond-patch # show bond0 hash change (no apply)
|
|
###############################################################################
|
|
set -u
|
|
umask 022
|
|
|
|
HOST="$(hostname -s)"
|
|
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
|
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
|
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
|
|
ACTION="dryrun"
|
|
SKIP_NFS=0
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--apply) ACTION="apply" ;;
|
|
--rollback) ACTION="rollback" ;;
|
|
--emit-bond-patch) ACTION="bond-patch" ;;
|
|
--no-nfs) SKIP_NFS=1 ;;
|
|
-h|--help) sed -n '2,35p' "$0"; exit 0 ;;
|
|
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# Detect host role from hostname
|
|
case "$HOST" in
|
|
pfv-tsys4|pfv-tsys5)
|
|
HOST_ROLE="storage"
|
|
TUNED_PROFILE="network-throughput"
|
|
SWAPPINESS="1"
|
|
;;
|
|
pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
|
|
HOST_ROLE="vmhost"
|
|
TUNED_PROFILE="virtual-host"
|
|
SWAPPINESS="10"
|
|
;;
|
|
*)
|
|
HOST_ROLE="unknown"
|
|
TUNED_PROFILE="virtual-host"
|
|
SWAPPINESS="10"
|
|
;;
|
|
esac
|
|
|
|
# --- rollback path ---------------------------------------------------------
|
|
if [ "$ACTION" = "rollback" ]; then
|
|
echo "Looking for latest backup under /root/perfopt-backup-* ..."
|
|
latest=""
|
|
while IFS= read -r d; do
|
|
latest="$d"
|
|
done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
|
|
if [ -z "$latest" ]; then
|
|
echo "No backup found under /root/perfopt-backup-*" >&2
|
|
exit 1
|
|
fi
|
|
rb="$latest/perfopt-rollback.sh"
|
|
if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
|
|
echo "Rollback script missing in $latest" >&2
|
|
exit 1
|
|
fi
|
|
echo "Rolling back using: $rb"
|
|
bash "$rb"
|
|
exit $?
|
|
fi
|
|
|
|
# --- bond patch print-only path -------------------------------------------
|
|
if [ "$ACTION" = "bond-patch" ]; then
|
|
echo "==================================================================="
|
|
echo " Proposed bond0 xmit_hash_policy change"
|
|
echo "==================================================================="
|
|
echo
|
|
if [ ! -r /etc/network/interfaces ]; then
|
|
echo "/etc/network/interfaces not readable"
|
|
exit 1
|
|
fi
|
|
if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
|
|
echo "No bond0 detected on this host — nothing to patch."
|
|
exit 0
|
|
fi
|
|
cat <<EOF
|
|
A manual edit to /etc/network/interfaces is required. The bond0 stanza
|
|
needs this line added (it defaults to layer2, which is wrong for storage):
|
|
|
|
bond-xmit-hash-policy layer3+4
|
|
|
|
After editing, you MUST reload networking for it to take effect:
|
|
ifreload -a # safe, brings interfaces down/up
|
|
# OR
|
|
systemctl restart networking # heavier, briefly drops connections
|
|
|
|
WARNING: applying this on a remote host over bond0 will briefly drop your
|
|
SSH session. Run from console/IPMI, or schedule a maintenance window.
|
|
|
|
The change is reversible by removing the line and reloading again.
|
|
EOF
|
|
exit 0
|
|
fi
|
|
|
|
# --- main path (dryrun or apply) ------------------------------------------
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
# Emit the rollback script header
|
|
cat > "$ROLLBACK_SCRIPT" <<EOF
|
|
#!/bin/bash
|
|
# Auto-generated rollback for perfopt apply-tunings.sh
|
|
# Backup timestamp: $TS
|
|
# Backup dir: $BACKUP_DIR
|
|
# Generated on: $HOST
|
|
set -u
|
|
EOF
|
|
chmod +x "$ROLLBACK_SCRIPT"
|
|
|
|
backup_file() {
|
|
local f="$1"
|
|
if [ -e "$f" ]; then
|
|
cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
|
fi
|
|
}
|
|
|
|
# Helper: append a restore command to the rollback script
|
|
rb_restore() {
|
|
local f="$1"
|
|
local bk
|
|
bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
if [ -r "$bk" ]; then
|
|
cp -a "$bk" "$f" && echo "restored $f"
|
|
else
|
|
echo "WARN: backup $bk missing — $f left as-is"
|
|
fi
|
|
EOF
|
|
}
|
|
|
|
# Helper: append a sysctl restore command
|
|
rb_sysctl_restore() {
|
|
local key="$1" orig_val="$2"
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
|
|
EOF
|
|
}
|
|
|
|
# Helper: append a tuned-adm restore
|
|
rb_tuned_restore() {
|
|
local profile="$1"
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
|
|
EOF
|
|
}
|
|
|
|
echo "==================================================================="
|
|
echo " perfopt apply-tunings — $HOST"
|
|
echo " mode: $ACTION"
|
|
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
|
|
echo " backup: $BACKUP_DIR"
|
|
echo " rollback: $ROLLBACK_SCRIPT"
|
|
echo "==================================================================="
|
|
echo
|
|
|
|
# ===========================================================================
|
|
# 1. scaling_governor → performance
|
|
# ===========================================================================
|
|
echo "--- 1. CPU scaling_governor → performance"
|
|
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
|
|
cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
|
|
echo " current: $cur"
|
|
if [ "$ACTION" = "apply" ]; then
|
|
if [ "$cur" != "performance" ]; then
|
|
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
|
echo performance > "$c" 2>/dev/null || true
|
|
done
|
|
# Persist via systemd tmpfiles / sysctl fallback
|
|
cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
|
|
[Unit]
|
|
Description=Set CPU scaling_governor=performance (perfopt)
|
|
After=multi-user.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
|
|
RemainAfterExit=yes
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
systemctl daemon-reload
|
|
systemctl enable perfopt-cpu-performance.service 2>/dev/null
|
|
systemctl start perfopt-cpu-performance.service 2>/dev/null
|
|
echo " applied + persisted via systemd unit"
|
|
# Rollback
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
|
|
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
|
|
rm -f /etc/systemd/system/perfopt-cpu-performance.service
|
|
systemctl daemon-reload
|
|
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
|
|
EOF
|
|
else
|
|
echo " already performance — no change"
|
|
fi
|
|
fi
|
|
else
|
|
echo " cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 2. vm.swappiness
|
|
# ===========================================================================
|
|
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
|
|
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
|
|
echo " current: $cur_swappiness"
|
|
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
|
|
# Persist via sysctl.d
|
|
backup_file /etc/sysctl.d/99-perfopt.conf
|
|
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
|
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
|
vm.swappiness = $SWAPPINESS
|
|
EOF
|
|
sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
|
|
echo " applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
|
|
rb_restore /etc/sysctl.d/99-perfopt.conf
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
rm -f /etc/sysctl.d/99-perfopt.conf
|
|
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
|
|
echo "restored vm.swappiness=$cur_swappiness"
|
|
EOF
|
|
else
|
|
echo " no change"
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 3. TCP BBR + buffers (single sysctl.d file)
|
|
# ===========================================================================
|
|
echo "--- 3. TCP BBR + socket buffers"
|
|
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
|
|
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
|
|
echo " current: $cur_cc (available: $cur_avail)"
|
|
|
|
# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
|
|
TARGET_RMEM_MAX=134217728 # 128 MB
|
|
TARGET_WMEM_MAX=134217728
|
|
TARGET_RMEM_DEFAULT=26214400 # 25 MB
|
|
TARGET_WMEM_DEFAULT=26214400
|
|
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
|
|
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
|
|
TARGET_NETDEV_MAX_BACKLOG=250000
|
|
TARGET_SOMAXCONN=65535
|
|
|
|
if [ "$ACTION" = "apply" ]; then
|
|
# Ensure tcp_bbr module loads at boot
|
|
if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
|
|
backup_file /etc/modules-load.d/modules.conf
|
|
mkdir -p /etc/modules-load.d
|
|
cat >> /etc/modules-load.d/modules.conf <<EOF
|
|
# perfopt apply-tunings.sh — $TS
|
|
tcp_bbr
|
|
EOF
|
|
modprobe tcp_bbr 2>/dev/null || true
|
|
rb_restore /etc/modules-load.d/modules.conf
|
|
fi
|
|
|
|
backup_file /etc/sysctl.d/99-perfopt.conf
|
|
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
|
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
|
vm.swappiness = $SWAPPINESS
|
|
|
|
# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
|
|
net.ipv4.tcp_congestion_control = bbr
|
|
net.core.default_qdisc = fq
|
|
net.core.rmem_max = $TARGET_RMEM_MAX
|
|
net.core.wmem_max = $TARGET_WMEM_MAX
|
|
net.core.rmem_default = $TARGET_RMEM_DEFAULT
|
|
net.core.wmem_default = $TARGET_WMEM_DEFAULT
|
|
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
|
|
net.core.somaxconn = $TARGET_SOMAXCONN
|
|
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
|
|
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
|
|
EOF
|
|
sysctl --system >/dev/null 2>&1
|
|
echo " applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
|
|
# Rebuild rollback for sysctl
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
# Restore original sysctl state
|
|
rm -f /etc/sysctl.d/99-perfopt.conf
|
|
EOF
|
|
rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
|
|
rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
|
|
# Note: original buffer sizes not all captured; restoration is best-effort.
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
echo "Note: net buffers restored to kernel defaults (original values not captured)."
|
|
echo "Run 'sysctl --system' to apply."
|
|
sysctl --system >/dev/null 2>&1 || true
|
|
EOF
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 4. tuned-adm profile
|
|
# ===========================================================================
|
|
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
|
|
if command -v tuned-adm >/dev/null 2>&1; then
|
|
cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
|
|
echo " current: $cur_profile"
|
|
if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
|
|
tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/ /'
|
|
rb_tuned_restore "$cur_profile"
|
|
else
|
|
echo " no change"
|
|
fi
|
|
else
|
|
echo " tuned-adm not installed — skipping"
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 5. NFS mount option hardening
|
|
# Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
|
|
# Proxmox applies these on next mount/remount.
|
|
# Skipped with --no-nfs (for NFS servers where we don't want to remount).
|
|
# ===========================================================================
|
|
echo "--- 5. NFS mount options: add nconnect=4, noatime"
|
|
if [ "$SKIP_NFS" -eq 1 ]; then
|
|
echo " SKIPPED (--no-nfs specified)"
|
|
echo " Note: storage.cfg is cluster-wide; if edited on another host,"
|
|
echo " the options are already staged here and activate on next reboot."
|
|
else
|
|
if [ -r /etc/pve/storage.cfg ]; then
|
|
backup_file /etc/pve/storage.cfg
|
|
if grep -q '^nfs:' /etc/pve/storage.cfg; then
|
|
# Count how many nfs: stanzas lack the options
|
|
while IFS= read -r line; do
|
|
if echo "$line" | grep -q '^nfs:'; then
|
|
cur_stanza=$(echo "$line" | awk '{print $2}')
|
|
# Look ahead for the next few lines to see if options already set
|
|
echo " nfs stanza: $cur_stanza"
|
|
fi
|
|
done < /etc/pve/storage.cfg
|
|
|
|
# Check whether any nfs stanza already has options
|
|
if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
|
|
echo " some stanzas already have nconnect=4 — check manually"
|
|
else
|
|
echo " proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2' to each nfs stanza"
|
|
if [ "$ACTION" = "apply" ]; then
|
|
# Apply via perl for safety (in-place edit with backup)
|
|
if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
|
|
# Use a python helper for robust PVE storage.cfg editing
|
|
python3 - <<PYEOF
|
|
import re, pathlib
|
|
p = pathlib.Path("/etc/pve/storage.cfg")
|
|
text = p.read_text()
|
|
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
|
|
modified = 0
|
|
def fix(m):
|
|
global modified
|
|
block = m.group(1)
|
|
if 'options' in block:
|
|
return block
|
|
lines = block.rstrip('\n').split('\n')
|
|
lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2')
|
|
modified += 1
|
|
return '\n'.join(lines) + '\n'
|
|
new_text = pattern.sub(fix, text)
|
|
if modified:
|
|
p.write_text(new_text)
|
|
print(f" patched {modified} NFS stanzas with options line")
|
|
else:
|
|
print(" no NFS stanza needed patching (all already had options or no nfs: found)")
|
|
PYEOF
|
|
# Remount existing NFS mounts to pick up new options
|
|
echo " remounting NFS mounts to apply new options..."
|
|
while IFS= read -r mp; do
|
|
mount -o remount "$mp" 2>/dev/null && echo " remounted $mp" || echo " FAILED to remount $mp (will pick up on next mount)"
|
|
done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
|
|
rb_restore /etc/pve/storage.cfg
|
|
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
|
# To fully undo NFS options, also remount:
|
|
while IFS= read -r mp; do
|
|
mount -o remount "\$mp" 2>/dev/null
|
|
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
|
|
echo "restored NFS mount options"
|
|
EOF
|
|
else
|
|
echo " ERROR: could not back up storage.cfg — aborting NFS patch"
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
echo " no NFS stanzas in storage.cfg — skipping"
|
|
fi
|
|
else
|
|
echo " /etc/pve/storage.cfg not readable — skipping"
|
|
fi
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 6. Summary
|
|
# ===========================================================================
|
|
echo
|
|
echo "==================================================================="
|
|
if [ "$ACTION" = "apply" ]; then
|
|
echo " DONE. Backups in: $BACKUP_DIR"
|
|
echo " Rollback: bash $ROLLBACK_SCRIPT"
|
|
echo
|
|
echo " Next steps:"
|
|
echo " - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
|
|
echo " - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
|
|
echo " - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
|
|
echo " xmit_hash_policy change (requires network restart, do in"
|
|
echo " maintenance window)"
|
|
else
|
|
echo " DRY RUN — no changes made."
|
|
echo " To commit: bash apply-tunings.sh --apply"
|
|
fi
|
|
echo "==================================================================="
|