PFVCluster/k8s/install-cp.sh

#!/usr/bin/bash
#
# k8s/install-cp.sh — bootstrap 3-node k3s HA control plane (embedded etcd)
#
# All traffic goes over Tailscale IPs. LAN addresses are never used for
# cluster communication.
#
# Sequence:
#   1. Install cnode1 with --cluster-init (creates new etcd cluster)
#   2. Wait for cnode1 API to be ready
#   3. Fetch join token from cnode1
#   4. Install cnode2 + cnode3 as additional servers (join etcd quorum)
#   5. Wait for all 3 etcd members to be healthy
#
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=./env.sh
source "$SCRIPT_DIR/env.sh"

# Build the shared TLS-SAN flags
tls_san_flags=""
for san in "${TLS_SANS[@]}"; do
  tls_san_flags+=" --tls-san=$san"
done

echo "============================================"
echo "  Bootstrapping k3s HA control plane"
echo "  k3s version: $K3S_VERSION"
echo "  Transport:   Tailscale (wireguard)"
echo "============================================"

# -------------------------------------------------------
# Phase 1: Install bootstrap node (cnode1) with --cluster-init
# -------------------------------------------------------
echo ""
echo "--- [1/5] Installing bootstrap node: $CNODE1_NAME ($CNODE1_IP) ---"

ssh "${SSH_OPTS[@]}" "${SSH_USER}@${CNODE1_IP}" "sudo -n bash -s" <<REMOTE_BOOT
set -euo pipefail
export INSTALL_K3S_VERSION="$K3S_VERSION"
export KILLALL_MODE=process
curl -sfL https://get.k3s.io | sh -s - server \
  --cluster-init \
  --node-name=$CNODE1_NAME \
  --node-ip=$CNODE1_IP \
  --advertise-address=$CNODE1_IP \
  $tls_san_flags \
  --flannel-backend=vxlan \
  --etcd-snapshot-schedule-cron='0 */6 * * *' \
  --egress-selector-mode=agent
REMOTE_BOOT

echo "  cnode1 install submitted."

# -------------------------------------------------------
# Phase 2: Wait for cnode1 API + etcd to be ready
# -------------------------------------------------------
echo ""
echo "--- [2/5] Waiting for cnode1 API + etcd ---"
API_READY=false
for i in $(seq 1 30); do
  if cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>/dev/null | grep -q "ok"; then
    API_READY=true
    break
  fi
  echo "  ...waiting ($i/30)"
  sleep 5
done

if [ "$API_READY" = false ]; then
  echo "ERROR: cnode1 API did not become ready in 150s."
  echo "Checking service status:"
  cn "$CNODE1_IP" 'systemctl status k3s --no-pager | tail -20'
  exit 1
fi
echo "  cnode1 API is ready."

# -------------------------------------------------------
# Phase 3: Fetch join token from cnode1
# -------------------------------------------------------
echo ""
echo "--- [3/5] Fetching join token from cnode1 ---"
JOIN_TOKEN=$(cn "$CNODE1_IP" 'cat /var/lib/rancher/k3s/server/token')
if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
  echo "ERROR: could not fetch token from cnode1."
  echo "  Got: ${JOIN_TOKEN:0:40}"
  exit 1
fi
echo "  Token fetched (masked: ${JOIN_TOKEN:0:12}***)"

SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"

# -------------------------------------------------------
# Phase 4: Install cnode2 and cnode3 as additional servers
# -------------------------------------------------------
for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do
  # Derive node name from IP
  case "$node_ip" in
    "$CNODE2_IP") node_name="$CNODE2_NAME" ;;
    "$CNODE3_IP") node_name="$CNODE3_NAME" ;;
  esac

  echo ""
  echo "--- [4/5] Joining server: $node_name ($node_ip) ---"

  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_JOIN
set -euo pipefail
export INSTALL_K3S_VERSION="$K3S_VERSION"
export K3S_TOKEN="$JOIN_TOKEN"
export KILLALL_MODE=process
curl -sfL https://get.k3s.io | sh -s - server \
  --server=$SERVER_URL \
  --node-name=$node_name \
  --node-ip=$node_ip \
  --advertise-address=$node_ip \
  $tls_san_flags \
  --flannel-backend=vxlan \
  --egress-selector-mode=agent
REMOTE_JOIN

  echo "  $node_name install submitted."
done

# -------------------------------------------------------
# Phase 5: Wait for all 3 etcd members + nodes Ready
# -------------------------------------------------------
echo ""
echo "--- [5/5] Waiting for all 3 nodes to join ---"
for i in $(seq 1 30); do
  READY_NODES=$(cn "$CNODE1_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
  if [ "$READY_NODES" = "3" ]; then
    echo "  All 3 nodes Ready."
    break
  fi
  echo "  ...waiting ($i/30, $READY_NODES/3 ready)"
  sleep 10
done

if [ "$READY_NODES" != "3" ]; then
  echo "WARN: Only $READY_NODES/3 nodes ready after 300s. Check manually."
fi

echo ""
echo "============================================"
echo "  Control plane nodes:"
echo "============================================"
cn "$CNODE1_IP" 'k3s kubectl get nodes -o wide'

echo ""
echo "============================================"
echo "  Bootstrap complete."
echo "  Run: bash $SCRIPT_DIR/post-setup.sh"
echo "============================================"