feat(k8s): add worker join script + 6 worker nodes to env config
- env.sh: add 6 worker nodes (tsys3/5/6/7/9 + ultix-offstage), update cnode1 IP to current Tailscale address - join-workers.sh: new script to join all workers as k3s agents (fixes bash syntax bug in echo statement from prior session) - tests/ip.sh: pass-through wrapper for ip command [#367] [#368]
This commit is contained in:
+29
-3
@@ -4,9 +4,9 @@
|
||||
#
|
||||
# All cluster communication goes over Tailscale IPs. No LAN IPs, ever.
|
||||
|
||||
# --- Nodes (Tailscale 100.x addresses) ---
|
||||
# --- Control plane nodes (Tailscale 100.x addresses) ---
|
||||
CNODE1_NAME="pfv-k8s-cnode1"
|
||||
CNODE1_IP="100.97.178.106"
|
||||
CNODE1_IP="100.125.134.53"
|
||||
|
||||
CNODE2_NAME="pfv-k8s-cnode2"
|
||||
CNODE2_IP="100.109.34.72"
|
||||
@@ -21,6 +21,32 @@ ALL_CNODE_NAMES=("$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME")
|
||||
BOOTSTRAP_IP="$CNODE1_IP"
|
||||
BOOTSTRAP_NAME="$CNODE1_NAME"
|
||||
|
||||
# --- Worker nodes (Tailscale 100.x addresses) ---
|
||||
WNODE1_NAME="pfv-k8s-wnode-tsys3"
|
||||
WNODE1_IP="100.126.9.112"
|
||||
|
||||
WNODE2_NAME="pfv-k8s-wnode-tsys5"
|
||||
WNODE2_IP="100.122.252.116"
|
||||
|
||||
WNODE3_NAME="pfv-k8s-wnode-tsys6"
|
||||
WNODE3_IP="100.83.49.75"
|
||||
|
||||
WNODE4_NAME="pfv-k8s-wnode-tsys7"
|
||||
WNODE4_IP="100.119.240.11"
|
||||
|
||||
WNODE5_NAME="pfv-k8s-wnode-tsys9"
|
||||
WNODE5_IP="100.95.201.66"
|
||||
|
||||
WNODE6_NAME="ultix-offstage"
|
||||
WNODE6_IP="100.70.119.59"
|
||||
|
||||
# ultix-streaming: SSH key not yet deployed — join after setup
|
||||
# WNODE7_NAME="ultix-streaming"
|
||||
# WNODE7_IP="100.101.187.119"
|
||||
|
||||
ALL_WNODES=("$WNODE1_IP" "$WNODE2_IP" "$WNODE3_IP" "$WNODE4_IP" "$WNODE5_IP" "$WNODE6_IP")
|
||||
ALL_WNODE_NAMES=("$WNODE1_NAME" "$WNODE2_NAME" "$WNODE3_NAME" "$WNODE4_NAME" "$WNODE5_NAME" "$WNODE6_NAME")
|
||||
|
||||
# --- SSH ---
|
||||
SSH_USER="localuser"
|
||||
SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
|
||||
@@ -44,5 +70,5 @@ cn() {
|
||||
# Helper: run a heredoc script on a node
|
||||
cn_file() {
|
||||
local ip="$1"
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s"
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s"
|
||||
}
|
||||
|
||||
Executable
+103
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/bash
|
||||
#
|
||||
# k8s/join-workers.sh — join worker nodes to the k3s cluster
|
||||
#
|
||||
# Joins all worker nodes defined in env.sh as k3s agents. Worker nodes
|
||||
# run user workloads; control plane nodes are tainted NoSchedule.
|
||||
#
|
||||
set -uo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=./env.sh
|
||||
source "$SCRIPT_DIR/env.sh"
|
||||
|
||||
echo "============================================"
|
||||
echo " Joining worker nodes to cluster"
|
||||
echo " Server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)"
|
||||
echo " Workers: ${#ALL_WNODE_NAMES[@]}"
|
||||
echo "============================================"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 1. Fetch join token from bootstrap node
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---"
|
||||
JOIN_TOKEN=$(cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token')
|
||||
if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
|
||||
echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}"
|
||||
exit 1
|
||||
fi
|
||||
echo " Token OK (masked: ${JOIN_TOKEN:0:12}***)"
|
||||
|
||||
SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 2. Install k3s-agent on each worker
|
||||
# -------------------------------------------------------
|
||||
for i in "${!ALL_WNODES[@]}"; do
|
||||
node_ip="${ALL_WNODES[$i]}"
|
||||
node_name="${ALL_WNODE_NAMES[$i]}"
|
||||
|
||||
echo ""
|
||||
echo "--- [2/3] Joining worker: $node_name ($node_ip) ---"
|
||||
|
||||
# Wipe any existing k3s first
|
||||
cn "$node_ip" '
|
||||
systemctl stop k3s-agent 2>/dev/null || true
|
||||
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then
|
||||
/usr/local/bin/k3s-agent-uninstall.sh
|
||||
fi
|
||||
rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni
|
||||
rm -f /etc/systemd/system/k3s-agent.service
|
||||
systemctl daemon-reload
|
||||
ip link delete cni0 2>/dev/null || true
|
||||
ip link delete flannel.1 2>/dev/null || true
|
||||
' 2>/dev/null || true
|
||||
|
||||
# Install as agent
|
||||
# shellcheck disable=SC2087 # heredoc intentionally expands local config
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_AGENT
|
||||
set -euo pipefail
|
||||
export INSTALL_K3S_VERSION="$K3S_VERSION"
|
||||
export K3S_URL="$SERVER_URL"
|
||||
export K3S_TOKEN="$JOIN_TOKEN"
|
||||
export K3S_NODE_NAME="$node_name"
|
||||
curl -sfL https://get.k3s.io | sh -s - agent \
|
||||
--node-name=$node_name \
|
||||
--node-ip=$node_ip
|
||||
REMOTE_AGENT
|
||||
|
||||
echo " $node_name agent install submitted."
|
||||
done
|
||||
|
||||
# -------------------------------------------------------
|
||||
# 3. Wait for all workers to appear Ready
|
||||
# -------------------------------------------------------
|
||||
TOTAL_NODES=$(( ${#ALL_CNODES[@]} + ${#ALL_WNODES[@]} ))
|
||||
echo ""
|
||||
echo "--- [3/3] Waiting for all $TOTAL_NODES nodes (${#ALL_CNODES[@]} cp + ${#ALL_WNODES[@]} workers) ---"
|
||||
for i in $(seq 1 60); do
|
||||
READY_NODES=$(cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
|
||||
if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then
|
||||
echo " All $TOTAL_NODES nodes Ready."
|
||||
break
|
||||
fi
|
||||
echo " ...waiting ($i/60, $READY_NODES/$TOTAL_NODES ready)"
|
||||
sleep 10
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " Node status:"
|
||||
echo "============================================"
|
||||
cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes -o wide'
|
||||
|
||||
if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " All workers joined. Cluster fully operational."
|
||||
echo "============================================"
|
||||
else
|
||||
echo ""
|
||||
echo "WARN: $READY_NODES/$TOTAL_NODES ready. Check failing nodes."
|
||||
exit 1
|
||||
fi
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/bash
|
||||
#
|
||||
# tests/ip.sh — pass-through wrapper for the ip command.
|
||||
#
|
||||
# Usage:
|
||||
# tests/ip.sh <ip args...>
|
||||
#
|
||||
# Examples:
|
||||
# tests/ip.sh route get 192.168.3.252
|
||||
# tests/ip.sh -br addr
|
||||
# tests/ip.sh link show
|
||||
#
|
||||
set -uo pipefail
|
||||
|
||||
if [ "$#" -eq 0 ]; then
|
||||
echo "Usage: $0 <ip args...>" >&2
|
||||
echo " e.g. $0 route get 192.168.3.252" >&2
|
||||
echo " $0 -br addr" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
exec /sbin/ip "$@"
|
||||
Reference in New Issue
Block a user