feat(k8s): add worker join script + 6 worker nodes to env config

- env.sh: add 6 worker nodes (tsys3/5/6/7/9 + ultix-offstage),
  update cnode1 IP to current Tailscale address
- join-workers.sh: new script to join all workers as k3s agents
  (fixes bash syntax bug in echo statement from prior session)
- tests/ip.sh: pass-through wrapper for ip command

[#367] [#368]

k8s/env.sh

@@ -4,9 +4,9 @@
 #
 # All cluster communication goes over Tailscale IPs. No LAN IPs, ever.
 
-# --- Nodes (Tailscale 100.x addresses) ---
+# --- Control plane nodes (Tailscale 100.x addresses) ---
 CNODE1_NAME="pfv-k8s-cnode1"
-CNODE1_IP="100.97.178.106"
+CNODE1_IP="100.125.134.53"
 
 CNODE2_NAME="pfv-k8s-cnode2"
 CNODE2_IP="100.109.34.72"
@@ -21,6 +21,32 @@ ALL_CNODE_NAMES=("$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME")
 BOOTSTRAP_IP="$CNODE1_IP"
 BOOTSTRAP_NAME="$CNODE1_NAME"
 
+# --- Worker nodes (Tailscale 100.x addresses) ---
+WNODE1_NAME="pfv-k8s-wnode-tsys3"
+WNODE1_IP="100.126.9.112"
+
+WNODE2_NAME="pfv-k8s-wnode-tsys5"
+WNODE2_IP="100.122.252.116"
+
+WNODE3_NAME="pfv-k8s-wnode-tsys6"
+WNODE3_IP="100.83.49.75"
+
+WNODE4_NAME="pfv-k8s-wnode-tsys7"
+WNODE4_IP="100.119.240.11"
+
+WNODE5_NAME="pfv-k8s-wnode-tsys9"
+WNODE5_IP="100.95.201.66"
+
+WNODE6_NAME="ultix-offstage"
+WNODE6_IP="100.70.119.59"
+
+# ultix-streaming: SSH key not yet deployed — join after setup
+# WNODE7_NAME="ultix-streaming"
+# WNODE7_IP="100.101.187.119"
+
+ALL_WNODES=("$WNODE1_IP" "$WNODE2_IP" "$WNODE3_IP" "$WNODE4_IP" "$WNODE5_IP" "$WNODE6_IP")
+ALL_WNODE_NAMES=("$WNODE1_NAME" "$WNODE2_NAME" "$WNODE3_NAME" "$WNODE4_NAME" "$WNODE5_NAME" "$WNODE6_NAME")
+
 # --- SSH ---
 SSH_USER="localuser"
 SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
@@ -44,5 +70,5 @@ cn() {
 # Helper: run a heredoc script on a node
 cn_file() {
   local ip="$1"
-  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s" 
+  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s"
 }

k8s/join-workers.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/bash
+#
+# k8s/join-workers.sh — join worker nodes to the k3s cluster
+#
+# Joins all worker nodes defined in env.sh as k3s agents. Worker nodes
+# run user workloads; control plane nodes are tainted NoSchedule.
+#
+set -uo pipefail
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=./env.sh
+source "$SCRIPT_DIR/env.sh"
+
+echo "============================================"
+echo "  Joining worker nodes to cluster"
+echo "  Server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)"
+echo "  Workers: ${#ALL_WNODE_NAMES[@]}"
+echo "============================================"
+
+# -------------------------------------------------------
+# 1. Fetch join token from bootstrap node
+# -------------------------------------------------------
+echo ""
+echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---"
+JOIN_TOKEN=$(cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token')
+if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
+  echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}"
+  exit 1
+fi
+echo "  Token OK (masked: ${JOIN_TOKEN:0:12}***)"
+
+SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"
+
+# -------------------------------------------------------
+# 2. Install k3s-agent on each worker
+# -------------------------------------------------------
+for i in "${!ALL_WNODES[@]}"; do
+  node_ip="${ALL_WNODES[$i]}"
+  node_name="${ALL_WNODE_NAMES[$i]}"
+
+  echo ""
+  echo "--- [2/3] Joining worker: $node_name ($node_ip) ---"
+
+  # Wipe any existing k3s first
+  cn "$node_ip" '
+    systemctl stop k3s-agent 2>/dev/null || true
+    if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then
+      /usr/local/bin/k3s-agent-uninstall.sh
+    fi
+    rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni
+    rm -f /etc/systemd/system/k3s-agent.service
+    systemctl daemon-reload
+    ip link delete cni0 2>/dev/null || true
+    ip link delete flannel.1 2>/dev/null || true
+  ' 2>/dev/null || true
+
+  # Install as agent
+  # shellcheck disable=SC2087 # heredoc intentionally expands local config
+  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_AGENT
+set -euo pipefail
+export INSTALL_K3S_VERSION="$K3S_VERSION"
+export K3S_URL="$SERVER_URL"
+export K3S_TOKEN="$JOIN_TOKEN"
+export K3S_NODE_NAME="$node_name"
+curl -sfL https://get.k3s.io | sh -s - agent \
+  --node-name=$node_name \
+  --node-ip=$node_ip
+REMOTE_AGENT
+
+  echo "  $node_name agent install submitted."
+done
+
+# -------------------------------------------------------
+# 3. Wait for all workers to appear Ready
+# -------------------------------------------------------
+TOTAL_NODES=$(( ${#ALL_CNODES[@]} + ${#ALL_WNODES[@]} ))
+echo ""
+echo "--- [3/3] Waiting for all $TOTAL_NODES nodes (${#ALL_CNODES[@]} cp + ${#ALL_WNODES[@]} workers) ---"
+for i in $(seq 1 60); do
+  READY_NODES=$(cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
+  if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then
+    echo "  All $TOTAL_NODES nodes Ready."
+    break
+  fi
+  echo "  ...waiting ($i/60, $READY_NODES/$TOTAL_NODES ready)"
+  sleep 10
+done
+
+echo ""
+echo "============================================"
+echo "  Node status:"
+echo "============================================"
+cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes -o wide'
+
+if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then
+  echo ""
+  echo "============================================"
+  echo "  All workers joined. Cluster fully operational."
+  echo "============================================"
+else
+  echo ""
+  echo "WARN: $READY_NODES/$TOTAL_NODES ready. Check failing nodes."
+  exit 1
+fi