From d3c5ae8beb2b571a59383c07768f65135fd9aac5 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Tue, 1 Sep 2026 19:32:51 -0500 Subject: [PATCH] fix(k8s): time-box remote cmds; k3s kubectl in post-setup; codify VM tuning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - join-workers.sh: timeout wrappers (30s token / 60s pre-clean / 300s install) - post-setup.sh: bare kubectl/etcdctl do not exist on cnodes; use k3s kubectl and etcd-snapshot for health checks (taints silently failed before) - k8s/proxmox-tuning.sh: codifies 2026-09-01 VM tuning (ssd=1,discard=on, queues=2, cpuunits, startup order) — dry-run default, RUN=1 applies [#367][#368] https://projects.knownelement.com/issues/367 --- k8s/join-workers.sh | 10 +++---- k8s/post-setup.sh | 15 ++++++----- k8s/proxmox-tuning.sh | 62 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 k8s/proxmox-tuning.sh diff --git a/k8s/join-workers.sh b/k8s/join-workers.sh index 5530726..b7d2344 100755 --- a/k8s/join-workers.sh +++ b/k8s/join-workers.sh @@ -21,7 +21,7 @@ echo "============================================" # ------------------------------------------------------- echo "" echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---" -JOIN_TOKEN=$(cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token') +JOIN_TOKEN=$(timeout 30 cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token') if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}" exit 1 @@ -40,8 +40,8 @@ for i in "${!ALL_WNODES[@]}"; do echo "" echo "--- [2/3] Joining worker: $node_name ($node_ip) ---" - # Wipe any existing k3s first - cn "$node_ip" ' + # Wipe any existing k3s first (time-boxed: an unreachable node must not hang the run) + timeout 60 cn "$node_ip" ' systemctl stop k3s-agent 2>/dev/null || true if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh @@ -53,9 +53,9 @@ for i in "${!ALL_WNODES[@]}"; do ip link delete flannel.1 2>/dev/null || true ' 2>/dev/null || true - # Install as agent + # Install as agent (300s deploy-class limit; a timed-out install is a failed node) # shellcheck disable=SC2087 # heredoc intentionally expands local config - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" </dev/null || true + # Set the control-plane role label explicitly. + # k3s kubectl only — bare kubectl is not on the cnodes' root PATH. + cn "$CNODE1_IP" "k3s kubectl label node $name node-role.kubernetes.io/control-plane= --overwrite" 2>/dev/null || true # Apply the taint (idempotent with --overwrite) - cn "$CNODE1_IP" "kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null + cn "$CNODE1_IP" "k3s kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null echo " $name tainted." done @@ -96,10 +97,10 @@ echo "=== Node IPs (should be 100.x Tailscale) ===" kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>&1 echo "" -echo "=== etcd members (should be 3) ===" -cn "$CNODE1_IP" 'etcdctl --write-out=table endpoint status' 2>&1 \ - || cn "$CNODE1_IP" 'k3s etcdctl endpoint status --write-out=table' 2>&1 \ - || echo "(could not query etcd directly)" +echo "=== API + etcd liveness ===" +cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>&1 || echo "(API not ready)" +# k3s ships no etcdctl; snapshots answering proves the etcd store is readable +cn "$CNODE1_IP" 'k3s etcd-snapshot list 2>/dev/null | head -3' 2>&1 echo "" echo "=== System pods ===" diff --git a/k8s/proxmox-tuning.sh b/k8s/proxmox-tuning.sh new file mode 100644 index 0000000..74d75cd --- /dev/null +++ b/k8s/proxmox-tuning.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# k8s/proxmox-tuning.sh — apply ultix-style Proxmox tuning to k8s VMs +# +# Codifies the config applied 2026-09-01 during the #367 rebuild: +# - disk: ssd=1,discard=on (guest sees non-rotational; qcow2 trims) +# - net: queues=2 on both NICs (multiqueue; guest activates via ethtool) +# - cpu: cpuunits so etcd/control plane wins host CPU contention +# - boot: onboot=1 + startup order (cnodes before wnodes) +# - balloon was already 0 on every k8s VM (etcd memory predictability) +# +# Pending items (ssd/discard, queues) activate at the VM's next restart. +# Reboot waves are SERIAL with a health gate between hops — never batch. +# +# DRY RUN by default. Apply with RUN=1. Run on each Proxmox host, or via +# PROX_HOST= bash tests/remote.sh prox 'bash -s' < k8s/proxmox-tuning.sh +# +set -uo pipefail +RUN="${RUN:-0}" + +# vmid|cpuunits|startup|scsi0-line|net0-line|net1-line +# scsi0/net lines are the FULL desired config (Proxmox replaces wholesale). +TUNINGS=( + "102|4000|order=20,up=180|S2:102/vm-102-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:CB:97:10,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:47:13:08,bridge=datanet,queues=2" + "705|4000|order=20,up=180|S1:705/vm-705-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:40:25:F8,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:AF:3D:89,bridge=datanet,queues=2" + "603|4000|order=20,up=180|D1:603/vm-603-disk-0.qcow2,cache=writeback,format=qcow2,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:38:C0:58,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:1F:F1:E1,bridge=storagenet,queues=2" + "313|2000|order=40,up=120|local-lvm:vm-313-disk-0,iothread=1,ssd=1,discard=on,size=300G|virtio=BC:24:11:EE:7E:7B,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:DF:C2:E9,bridge=datanet,queues=2" + "601|2000|order=40,up=120|local-lvm:vm-601-disk-0,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:FA:6E:B5,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:39:7D:0B,bridge=storagenet,queues=2" + "701|2000|order=40,up=120|S2:701/vm-701-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:30:B8:07,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:C8:0E:8A,bridge=datanet,queues=2" + "905|2000|order=40,up=120|S2:905/vm-905-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:EE:2B:B6,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:9B:57:06,bridge=datanet,queues=2" +) + +apply() { + local vmid="$1" cpuunits="$2" startup="$3" scsi0="$4" net0="$5" net1="$6" + if [ "$RUN" = "1" ]; then + qm set "$vmid" --cpuunits "$cpuunits" --startup "$startup" \ + --scsi0 "$scsi0" --net0 "$net0" --net1 "$net1" + else + echo "DRY: qm set $vmid --cpuunits $cpuunits --startup $startup --scsi0 $scsi0 --net0 $net0 --net1 $net1" + fi +} + +for t in "${TUNINGS[@]}"; do + IFS='|' read -r vmid cpuunits startup scsi0 net0 net1 <<< "$t" + # Skip VMs that live on other hosts (qm set errors on unknown VMID) + if ! qm status "$vmid" >/dev/null 2>&1; then + echo "skip: VM $vmid not on this host" + continue + fi + echo "== VM $vmid ==" + apply "$vmid" "$cpuunits" "$startup" "$scsi0" "$net0" "$net1" +done + +echo +if [ "$RUN" = "1" ]; then + for t in "${TUNINGS[@]}"; do + vmid="${t%%|*}" + qm status "$vmid" >/dev/null 2>&1 || continue + echo "--- pending VM $vmid ---" + qm pending "$vmid" | grep -E "^(new|cur) (scsi0|net0|net1|cpuunits|startup)" || true + done +fi