mrcharles
74934b1095
REPORT.md gets a post-pass status banner + per-item STATUS tags + phase/QM checklist truth; RUNBOOK-TODAY amended for the Oct window replan (Friday is troubleshooting-only #609); all ~/optimize paths updated after the repo move to ~/projects/ultix. Adds the 22:00 night-flip watcher log and 6-remove-agent-stacks.sh (boot-time screen/crush relauncher teardown with backups to removed-agent-stacks/). 💘 Generated with Crush Assisted-by: Crush:glm-5.2 [#602]
96 lines
4.0 KiB
Bash
Executable File
96 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ONE-SHOT host pass v3 — does EVERYTHING except rebooting VM 5111.
|
|
# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/2-host-one-shot.sh
|
|
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/projects/ultix/2-host-one-shot.sh
|
|
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ... # optional
|
|
#
|
|
# 1. VM 5111's disk/net/onboot flags are written as PENDING config: they
|
|
# activate automatically at the VM's NEXT reboot (tonight, your last step
|
|
# today, Friday — whenever the VM naturally restarts). Nothing forces it.
|
|
# 2. CPU priority (live): VM 5111 = 9000, sectestbed/preprod fleet = 50.
|
|
# 3. Reboot wave: every OTHER running VM.
|
|
# Optional --with-vm-restart: also gracefully bounce VM 5111 at the end.
|
|
# THE HOST ITSELF IS NEVER REBOOTED (NFS server). NO DISK IS RESIZED
|
|
# (storage ops are manual-only: GROW-ROOT-RUNBOOK.md).
|
|
set -euo pipefail
|
|
GO=0; VMR=0; SKIPWAVE=0
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--go) GO=1 ;;
|
|
--with-vm-restart) VMR=1 ;;
|
|
--skip-wave) SKIPWAVE=1 ;;
|
|
esac
|
|
done
|
|
VM=5111
|
|
FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108"
|
|
run() { if [ "$GO" = 1 ]; then "$@"; else echo "DRY: $*"; fi; }
|
|
staget() { # pending-stage a qm set; tolerate refusal on a running VM
|
|
if [ "$GO" = 1 ]; then
|
|
qm set "$@" || echo " NOTE: not staged while running; use --with-vm-restart (or set while stopped) later"
|
|
else
|
|
echo "DRY(pending): qm set $*"
|
|
fi
|
|
}
|
|
|
|
echo "== 1. VM $VM flags as PENDING config (live at its next reboot) =="
|
|
staget "$VM" --onboot 1 --startup order=10,up=180
|
|
staget "$VM" -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
|
|
staget "$VM" -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
|
|
staget "$VM" -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
|
|
staget "$VM" --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
|
|
staget "$VM" --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
|
|
staget "$VM" --delete ide2
|
|
|
|
echo "== 2. CPU priority (live, reversible) =="
|
|
run qm set "$VM" --cpuunits 9000
|
|
for id in $FLEET; do run qm set "$id" --cpuunits 50; done
|
|
|
|
echo "== 3. reboot wave: every other running VM =="
|
|
if [ "$SKIPWAVE" = 1 ]; then
|
|
echo "skipped (--skip-wave)"
|
|
else
|
|
OTHERS=$(qm list | awk 'NR>1 && $3=="running" && $1!="'"$VM"'" {print $1}')
|
|
echo "targets: ${OTHERS//$'\n'/ }"
|
|
for id in $OTHERS; do
|
|
if [ "$GO" = 1 ]; then
|
|
qm reboot "$id" 2>/dev/null || { qm shutdown "$id" --timeout 120 || true; sleep 2; qm start "$id" 2>/dev/null || true; }
|
|
else
|
|
echo "DRY: reboot $id"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ "$GO" = 1 ]; then
|
|
echo; echo "== pending queue for VM $VM (activates at its next restart) =="
|
|
qm pending "$VM" 2>/dev/null || true
|
|
fi
|
|
|
|
LOG=/var/log/ukrrs-vm5111-bounce.log
|
|
if [ "$VMR" = 1 ]; then
|
|
echo "== graceful bounce of VM $VM (detached; flags applied while STOPPED) =="
|
|
qm status "$VM"
|
|
if [ "$GO" = 1 ]; then
|
|
setsid bash -c "
|
|
echo bounce-start \$(date -Is)
|
|
qm shutdown $VM --timeout 120
|
|
qm wait $VM --timeout 180 || true
|
|
qm set $VM --onboot 1 --startup order=10,up=180
|
|
qm set $VM -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
|
|
qm set $VM -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
|
|
qm set $VM -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
|
|
qm set $VM --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
|
|
qm set $VM --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
|
|
qm set $VM --delete ide2 || true
|
|
qm start $VM
|
|
echo bounce-done \$(date -Is)
|
|
" </dev/null >"$LOG" 2>&1 &
|
|
echo "dispatched. Downtime ~3-4 min; it comes back on its own."
|
|
echo "Watch from anywhere: ssh root@pfv-tsys5.knel.net tail -f $LOG"
|
|
else
|
|
echo "DRY: would detach-bounce $VM, applying flags while stopped, log $LOG"
|
|
fi
|
|
else
|
|
echo
|
|
echo "VM $VM was NOT rebooted. Flags sit pending and go live at its next restart."
|
|
fi
|