fix(k8s): time-box remote cmds; k3s kubectl in post-setup; codify VM tuning
- join-workers.sh: timeout wrappers (30s token / 60s pre-clean / 300s install) - post-setup.sh: bare kubectl/etcdctl do not exist on cnodes; use k3s kubectl and etcd-snapshot for health checks (taints silently failed before) - k8s/proxmox-tuning.sh: codifies 2026-09-01 VM tuning (ssd=1,discard=on, queues=2, cpuunits, startup order) — dry-run default, RUN=1 applies [#367][#368] https://projects.knownelement.com/issues/367
This commit is contained in:
+5
-5
@@ -21,7 +21,7 @@ echo "============================================"
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---"
|
||||
JOIN_TOKEN=$(cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token')
|
||||
JOIN_TOKEN=$(timeout 30 cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token')
|
||||
if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
|
||||
echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}"
|
||||
exit 1
|
||||
@@ -40,8 +40,8 @@ for i in "${!ALL_WNODES[@]}"; do
|
||||
echo ""
|
||||
echo "--- [2/3] Joining worker: $node_name ($node_ip) ---"
|
||||
|
||||
# Wipe any existing k3s first
|
||||
cn "$node_ip" '
|
||||
# Wipe any existing k3s first (time-boxed: an unreachable node must not hang the run)
|
||||
timeout 60 cn "$node_ip" '
|
||||
systemctl stop k3s-agent 2>/dev/null || true
|
||||
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then
|
||||
/usr/local/bin/k3s-agent-uninstall.sh
|
||||
@@ -53,9 +53,9 @@ for i in "${!ALL_WNODES[@]}"; do
|
||||
ip link delete flannel.1 2>/dev/null || true
|
||||
' 2>/dev/null || true
|
||||
|
||||
# Install as agent
|
||||
# Install as agent (300s deploy-class limit; a timed-out install is a failed node)
|
||||
# shellcheck disable=SC2087 # heredoc intentionally expands local config
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_AGENT
|
||||
timeout 300 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_AGENT || { echo " FATAL: $node_name install failed or timed out"; continue; }
|
||||
set -euo pipefail
|
||||
export INSTALL_K3S_VERSION="$K3S_VERSION"
|
||||
export K3S_URL="$SERVER_URL"
|
||||
|
||||
+8
-7
@@ -21,10 +21,11 @@ echo "============================================"
|
||||
echo ""
|
||||
echo "--- [1/3] Tainting control-plane nodes (NoSchedule) ---"
|
||||
for name in "${ALL_CNODE_NAMES[@]}"; do
|
||||
# Set the control-plane role label explicitly
|
||||
cn "$CNODE1_IP" "kubectl label node $name node-role.kubernetes.io/control-plane= --overwrite" 2>/dev/null || true
|
||||
# Set the control-plane role label explicitly.
|
||||
# k3s kubectl only — bare kubectl is not on the cnodes' root PATH.
|
||||
cn "$CNODE1_IP" "k3s kubectl label node $name node-role.kubernetes.io/control-plane= --overwrite" 2>/dev/null || true
|
||||
# Apply the taint (idempotent with --overwrite)
|
||||
cn "$CNODE1_IP" "kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null
|
||||
cn "$CNODE1_IP" "k3s kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null
|
||||
echo " $name tainted."
|
||||
done
|
||||
|
||||
@@ -96,10 +97,10 @@ echo "=== Node IPs (should be 100.x Tailscale) ==="
|
||||
kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>&1
|
||||
|
||||
echo ""
|
||||
echo "=== etcd members (should be 3) ==="
|
||||
cn "$CNODE1_IP" 'etcdctl --write-out=table endpoint status' 2>&1 \
|
||||
|| cn "$CNODE1_IP" 'k3s etcdctl endpoint status --write-out=table' 2>&1 \
|
||||
|| echo "(could not query etcd directly)"
|
||||
echo "=== API + etcd liveness ==="
|
||||
cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>&1 || echo "(API not ready)"
|
||||
# k3s ships no etcdctl; snapshots answering proves the etcd store is readable
|
||||
cn "$CNODE1_IP" 'k3s etcd-snapshot list 2>/dev/null | head -3' 2>&1
|
||||
|
||||
echo ""
|
||||
echo "=== System pods ==="
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# k8s/proxmox-tuning.sh — apply ultix-style Proxmox tuning to k8s VMs
|
||||
#
|
||||
# Codifies the config applied 2026-09-01 during the #367 rebuild:
|
||||
# - disk: ssd=1,discard=on (guest sees non-rotational; qcow2 trims)
|
||||
# - net: queues=2 on both NICs (multiqueue; guest activates via ethtool)
|
||||
# - cpu: cpuunits so etcd/control plane wins host CPU contention
|
||||
# - boot: onboot=1 + startup order (cnodes before wnodes)
|
||||
# - balloon was already 0 on every k8s VM (etcd memory predictability)
|
||||
#
|
||||
# Pending items (ssd/discard, queues) activate at the VM's next restart.
|
||||
# Reboot waves are SERIAL with a health gate between hops — never batch.
|
||||
#
|
||||
# DRY RUN by default. Apply with RUN=1. Run on each Proxmox host, or via
|
||||
# PROX_HOST=<host> bash tests/remote.sh prox 'bash -s' < k8s/proxmox-tuning.sh
|
||||
#
|
||||
set -uo pipefail
|
||||
RUN="${RUN:-0}"
|
||||
|
||||
# vmid|cpuunits|startup|scsi0-line|net0-line|net1-line
|
||||
# scsi0/net lines are the FULL desired config (Proxmox replaces wholesale).
|
||||
TUNINGS=(
|
||||
"102|4000|order=20,up=180|S2:102/vm-102-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:CB:97:10,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:47:13:08,bridge=datanet,queues=2"
|
||||
"705|4000|order=20,up=180|S1:705/vm-705-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:40:25:F8,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:AF:3D:89,bridge=datanet,queues=2"
|
||||
"603|4000|order=20,up=180|D1:603/vm-603-disk-0.qcow2,cache=writeback,format=qcow2,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:38:C0:58,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:1F:F1:E1,bridge=storagenet,queues=2"
|
||||
"313|2000|order=40,up=120|local-lvm:vm-313-disk-0,iothread=1,ssd=1,discard=on,size=300G|virtio=BC:24:11:EE:7E:7B,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:DF:C2:E9,bridge=datanet,queues=2"
|
||||
"601|2000|order=40,up=120|local-lvm:vm-601-disk-0,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:FA:6E:B5,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:39:7D:0B,bridge=storagenet,queues=2"
|
||||
"701|2000|order=40,up=120|S2:701/vm-701-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:30:B8:07,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:C8:0E:8A,bridge=datanet,queues=2"
|
||||
"905|2000|order=40,up=120|S2:905/vm-905-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:EE:2B:B6,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:9B:57:06,bridge=datanet,queues=2"
|
||||
)
|
||||
|
||||
apply() {
|
||||
local vmid="$1" cpuunits="$2" startup="$3" scsi0="$4" net0="$5" net1="$6"
|
||||
if [ "$RUN" = "1" ]; then
|
||||
qm set "$vmid" --cpuunits "$cpuunits" --startup "$startup" \
|
||||
--scsi0 "$scsi0" --net0 "$net0" --net1 "$net1"
|
||||
else
|
||||
echo "DRY: qm set $vmid --cpuunits $cpuunits --startup $startup --scsi0 $scsi0 --net0 $net0 --net1 $net1"
|
||||
fi
|
||||
}
|
||||
|
||||
for t in "${TUNINGS[@]}"; do
|
||||
IFS='|' read -r vmid cpuunits startup scsi0 net0 net1 <<< "$t"
|
||||
# Skip VMs that live on other hosts (qm set errors on unknown VMID)
|
||||
if ! qm status "$vmid" >/dev/null 2>&1; then
|
||||
echo "skip: VM $vmid not on this host"
|
||||
continue
|
||||
fi
|
||||
echo "== VM $vmid =="
|
||||
apply "$vmid" "$cpuunits" "$startup" "$scsi0" "$net0" "$net1"
|
||||
done
|
||||
|
||||
echo
|
||||
if [ "$RUN" = "1" ]; then
|
||||
for t in "${TUNINGS[@]}"; do
|
||||
vmid="${t%%|*}"
|
||||
qm status "$vmid" >/dev/null 2>&1 || continue
|
||||
echo "--- pending VM $vmid ---"
|
||||
qm pending "$vmid" | grep -E "^(new|cur) (scsi0|net0|net1|cpuunits|startup)" || true
|
||||
done
|
||||
fi
|
||||
Reference in New Issue
Block a user