feat(k8s): deploy 3-node k3s HA control plane over Tailscale
Bootstrap a regular (non-ITAR) k3s cluster on cnode1/2/3 with embedded
etcd. All cluster communication — node registration, API server, etcd
peering, flannel VXLAN — runs exclusively over Tailscale IPs. Zero LAN
addresses appear in node status or TLS certificates.
Scripts (k8s/):
- env.sh: shared config (Tailscale IPs, SSH opts, k3s version)
- wipe.sh: remove existing k3s from all cnodes
- install-cp.sh: full bootstrap (cnode1 --cluster-init, then cnode2/3 join)
- join-servers.sh: re-join cnode2/3 only (fixes broken join state)
- post-setup.sh: apply NoSchedule taints, fetch kubeconfig, verify
- verify.sh: 13-point health check (nodes, Tailscale IPs, taints, etcd,
CoreDNS, API server, workload isolation)
- probe-nodes.sh: SSH + Tailscale reachability check
All 3 cnodes are tainted control-plane:NoSchedule so no user workloads
can schedule on the control plane. 13/13 health checks pass.
Docs updated: k8s README TL;DR reflects k3s (not Talos) as the deployed
choice, with Talos preserved for the future ITAR cluster.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/bash
|
||||
#
|
||||
# k8s/install-cp.sh — bootstrap 3-node k3s HA control plane (embedded etcd)
|
||||
#
|
||||
# All traffic goes over Tailscale IPs. LAN addresses are never used for
|
||||
# cluster communication.
|
||||
#
|
||||
# Sequence:
|
||||
# 1. Install cnode1 with --cluster-init (creates new etcd cluster)
|
||||
# 2. Wait for cnode1 API to be ready
|
||||
# 3. Fetch join token from cnode1
|
||||
# 4. Install cnode2 + cnode3 as additional servers (join etcd quorum)
|
||||
# 5. Wait for all 3 etcd members to be healthy
|
||||
#
|
||||
set -uo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=./env.sh
|
||||
source "$SCRIPT_DIR/env.sh"
|
||||
|
||||
# Build the shared TLS-SAN flags
|
||||
tls_san_flags=""
|
||||
for san in "${TLS_SANS[@]}"; do
|
||||
tls_san_flags+=" --tls-san=$san"
|
||||
done
|
||||
|
||||
echo "============================================"
|
||||
echo " Bootstrapping k3s HA control plane"
|
||||
echo " k3s version: $K3S_VERSION"
|
||||
echo " Transport: Tailscale (wireguard)"
|
||||
echo "============================================"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Phase 1: Install bootstrap node (cnode1) with --cluster-init
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- [1/5] Installing bootstrap node: $CNODE1_NAME ($CNODE1_IP) ---"
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${CNODE1_IP}" "sudo -n bash -s" <<REMOTE_BOOT
|
||||
set -euo pipefail
|
||||
export INSTALL_K3S_VERSION="$K3S_VERSION"
|
||||
export KILLALL_MODE=process
|
||||
curl -sfL https://get.k3s.io | sh -s - server \
|
||||
--cluster-init \
|
||||
--node-name=$CNODE1_NAME \
|
||||
--node-ip=$CNODE1_IP \
|
||||
--advertise-address=$CNODE1_IP \
|
||||
$tls_san_flags \
|
||||
--flannel-backend=vxlan \
|
||||
--etcd-snapshot-schedule-cron='0 */6 * * *' \
|
||||
--egress-selector-mode=agent
|
||||
REMOTE_BOOT
|
||||
|
||||
echo " cnode1 install submitted."
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Phase 2: Wait for cnode1 API + etcd to be ready
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- [2/5] Waiting for cnode1 API + etcd ---"
|
||||
API_READY=false
|
||||
for i in $(seq 1 30); do
|
||||
if cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>/dev/null | grep -q "ok"; then
|
||||
API_READY=true
|
||||
break
|
||||
fi
|
||||
echo " ...waiting ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [ "$API_READY" = false ]; then
|
||||
echo "ERROR: cnode1 API did not become ready in 150s."
|
||||
echo "Checking service status:"
|
||||
cn "$CNODE1_IP" 'systemctl status k3s --no-pager | tail -20'
|
||||
exit 1
|
||||
fi
|
||||
echo " cnode1 API is ready."
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Phase 3: Fetch join token from cnode1
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- [3/5] Fetching join token from cnode1 ---"
|
||||
JOIN_TOKEN=$(cn "$CNODE1_IP" 'cat /var/lib/rancher/k3s/server/token')
|
||||
if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
|
||||
echo "ERROR: could not fetch token from cnode1."
|
||||
echo " Got: ${JOIN_TOKEN:0:40}"
|
||||
exit 1
|
||||
fi
|
||||
echo " Token fetched (masked: ${JOIN_TOKEN:0:12}***)"
|
||||
|
||||
SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Phase 4: Install cnode2 and cnode3 as additional servers
|
||||
# -------------------------------------------------------
|
||||
for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do
|
||||
# Derive node name from IP
|
||||
case "$node_ip" in
|
||||
"$CNODE2_IP") node_name="$CNODE2_NAME" ;;
|
||||
"$CNODE3_IP") node_name="$CNODE3_NAME" ;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "--- [4/5] Joining server: $node_name ($node_ip) ---"
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_JOIN
|
||||
set -euo pipefail
|
||||
export INSTALL_K3S_VERSION="$K3S_VERSION"
|
||||
export K3S_TOKEN="$JOIN_TOKEN"
|
||||
export KILLALL_MODE=process
|
||||
curl -sfL https://get.k3s.io | sh -s - server \
|
||||
--server=$SERVER_URL \
|
||||
--node-name=$node_name \
|
||||
--node-ip=$node_ip \
|
||||
--advertise-address=$node_ip \
|
||||
$tls_san_flags \
|
||||
--flannel-backend=vxlan \
|
||||
--egress-selector-mode=agent
|
||||
REMOTE_JOIN
|
||||
|
||||
echo " $node_name install submitted."
|
||||
done
|
||||
|
||||
# -------------------------------------------------------
|
||||
# Phase 5: Wait for all 3 etcd members + nodes Ready
|
||||
# -------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- [5/5] Waiting for all 3 nodes to join ---"
|
||||
for i in $(seq 1 30); do
|
||||
READY_NODES=$(cn "$CNODE1_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
|
||||
if [ "$READY_NODES" = "3" ]; then
|
||||
echo " All 3 nodes Ready."
|
||||
break
|
||||
fi
|
||||
echo " ...waiting ($i/30, $READY_NODES/3 ready)"
|
||||
sleep 10
|
||||
done
|
||||
|
||||
if [ "$READY_NODES" != "3" ]; then
|
||||
echo "WARN: Only $READY_NODES/3 nodes ready after 300s. Check manually."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " Control plane nodes:"
|
||||
echo "============================================"
|
||||
cn "$CNODE1_IP" 'k3s kubectl get nodes -o wide'
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " Bootstrap complete."
|
||||
echo " Run: bash $SCRIPT_DIR/post-setup.sh"
|
||||
echo "============================================"
|
||||
Reference in New Issue
Block a user