From bad40feae6caab7542a5fb10acdaa483c926dc93 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Thu, 6 Aug 2026 13:11:43 -0500 Subject: [PATCH] feat(k8s): add worker join script + 6 worker nodes to env config - env.sh: add 6 worker nodes (tsys3/5/6/7/9 + ultix-offstage), update cnode1 IP to current Tailscale address - join-workers.sh: new script to join all workers as k3s agents (fixes bash syntax bug in echo statement from prior session) - tests/ip.sh: pass-through wrapper for ip command [#367] [#368] --- k8s/env.sh | 32 ++++++++++++-- k8s/join-workers.sh | 103 ++++++++++++++++++++++++++++++++++++++++++++ tests/ip.sh | 22 ++++++++++ 3 files changed, 154 insertions(+), 3 deletions(-) create mode 100755 k8s/join-workers.sh create mode 100755 tests/ip.sh diff --git a/k8s/env.sh b/k8s/env.sh index 8de10c8..e500dbe 100644 --- a/k8s/env.sh +++ b/k8s/env.sh @@ -4,9 +4,9 @@ # # All cluster communication goes over Tailscale IPs. No LAN IPs, ever. -# --- Nodes (Tailscale 100.x addresses) --- +# --- Control plane nodes (Tailscale 100.x addresses) --- CNODE1_NAME="pfv-k8s-cnode1" -CNODE1_IP="100.97.178.106" +CNODE1_IP="100.125.134.53" CNODE2_NAME="pfv-k8s-cnode2" CNODE2_IP="100.109.34.72" @@ -21,6 +21,32 @@ ALL_CNODE_NAMES=("$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME") BOOTSTRAP_IP="$CNODE1_IP" BOOTSTRAP_NAME="$CNODE1_NAME" +# --- Worker nodes (Tailscale 100.x addresses) --- +WNODE1_NAME="pfv-k8s-wnode-tsys3" +WNODE1_IP="100.126.9.112" + +WNODE2_NAME="pfv-k8s-wnode-tsys5" +WNODE2_IP="100.122.252.116" + +WNODE3_NAME="pfv-k8s-wnode-tsys6" +WNODE3_IP="100.83.49.75" + +WNODE4_NAME="pfv-k8s-wnode-tsys7" +WNODE4_IP="100.119.240.11" + +WNODE5_NAME="pfv-k8s-wnode-tsys9" +WNODE5_IP="100.95.201.66" + +WNODE6_NAME="ultix-offstage" +WNODE6_IP="100.70.119.59" + +# ultix-streaming: SSH key not yet deployed — join after setup +# WNODE7_NAME="ultix-streaming" +# WNODE7_IP="100.101.187.119" + +ALL_WNODES=("$WNODE1_IP" "$WNODE2_IP" "$WNODE3_IP" "$WNODE4_IP" "$WNODE5_IP" "$WNODE6_IP") +ALL_WNODE_NAMES=("$WNODE1_NAME" "$WNODE2_NAME" "$WNODE3_NAME" "$WNODE4_NAME" "$WNODE5_NAME" "$WNODE6_NAME") + # --- SSH --- SSH_USER="localuser" SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15) @@ -44,5 +70,5 @@ cn() { # Helper: run a heredoc script on a node cn_file() { local ip="$1" - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s" } diff --git a/k8s/join-workers.sh b/k8s/join-workers.sh new file mode 100755 index 0000000..5530726 --- /dev/null +++ b/k8s/join-workers.sh @@ -0,0 +1,103 @@ +#!/usr/bin/bash +# +# k8s/join-workers.sh — join worker nodes to the k3s cluster +# +# Joins all worker nodes defined in env.sh as k3s agents. Worker nodes +# run user workloads; control plane nodes are tainted NoSchedule. +# +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=./env.sh +source "$SCRIPT_DIR/env.sh" + +echo "============================================" +echo " Joining worker nodes to cluster" +echo " Server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)" +echo " Workers: ${#ALL_WNODE_NAMES[@]}" +echo "============================================" + +# ------------------------------------------------------- +# 1. Fetch join token from bootstrap node +# ------------------------------------------------------- +echo "" +echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---" +JOIN_TOKEN=$(cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token') +if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then + echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}" + exit 1 +fi +echo " Token OK (masked: ${JOIN_TOKEN:0:12}***)" + +SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}" + +# ------------------------------------------------------- +# 2. Install k3s-agent on each worker +# ------------------------------------------------------- +for i in "${!ALL_WNODES[@]}"; do + node_ip="${ALL_WNODES[$i]}" + node_name="${ALL_WNODE_NAMES[$i]}" + + echo "" + echo "--- [2/3] Joining worker: $node_name ($node_ip) ---" + + # Wipe any existing k3s first + cn "$node_ip" ' + systemctl stop k3s-agent 2>/dev/null || true + if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then + /usr/local/bin/k3s-agent-uninstall.sh + fi + rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni + rm -f /etc/systemd/system/k3s-agent.service + systemctl daemon-reload + ip link delete cni0 2>/dev/null || true + ip link delete flannel.1 2>/dev/null || true + ' 2>/dev/null || true + + # Install as agent + # shellcheck disable=SC2087 # heredoc intentionally expands local config + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" </dev/null | grep -c " Ready"' 2>/dev/null || echo 0) + if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then + echo " All $TOTAL_NODES nodes Ready." + break + fi + echo " ...waiting ($i/60, $READY_NODES/$TOTAL_NODES ready)" + sleep 10 +done + +echo "" +echo "============================================" +echo " Node status:" +echo "============================================" +cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes -o wide' + +if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then + echo "" + echo "============================================" + echo " All workers joined. Cluster fully operational." + echo "============================================" +else + echo "" + echo "WARN: $READY_NODES/$TOTAL_NODES ready. Check failing nodes." + exit 1 +fi diff --git a/tests/ip.sh b/tests/ip.sh new file mode 100755 index 0000000..842dcc4 --- /dev/null +++ b/tests/ip.sh @@ -0,0 +1,22 @@ +#!/usr/bin/bash +# +# tests/ip.sh — pass-through wrapper for the ip command. +# +# Usage: +# tests/ip.sh +# +# Examples: +# tests/ip.sh route get 192.168.3.252 +# tests/ip.sh -br addr +# tests/ip.sh link show +# +set -uo pipefail + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 " >&2 + echo " e.g. $0 route get 192.168.3.252" >&2 + echo " $0 -br addr" >&2 + exit 2 +fi + +exec /sbin/ip "$@"