mrcharles
d3c5ae8beb
- join-workers.sh: timeout wrappers (30s token / 60s pre-clean / 300s install) - post-setup.sh: bare kubectl/etcdctl do not exist on cnodes; use k3s kubectl and etcd-snapshot for health checks (taints silently failed before) - k8s/proxmox-tuning.sh: codifies 2026-09-01 VM tuning (ssd=1,discard=on, queues=2, cpuunits, startup order) — dry-run default, RUN=1 applies [#367][#368] https://projects.knownelement.com/issues/367
116 lines
4.3 KiB
Bash
116 lines
4.3 KiB
Bash
#!/usr/bin/bash
|
|
#
|
|
# k8s/post-setup.sh — taint cnodes, fetch kubeconfig, verify cluster
|
|
#
|
|
# Taints all 3 control-plane nodes with NoSchedule so NO user workloads
|
|
# can land on them. Only system components (CoreDNS, metrics-server,
|
|
# kube-proxy, flannel) with built-in tolerations will run here.
|
|
#
|
|
set -uo pipefail
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
# shellcheck source=./env.sh
|
|
source "$SCRIPT_DIR/env.sh"
|
|
|
|
echo "============================================"
|
|
echo " Post-setup: taints, kubeconfig, verify"
|
|
echo "============================================"
|
|
|
|
# -------------------------------------------------------
|
|
# 1. Taint all cnodes NoSchedule (no user workloads on control plane)
|
|
# -------------------------------------------------------
|
|
echo ""
|
|
echo "--- [1/3] Tainting control-plane nodes (NoSchedule) ---"
|
|
for name in "${ALL_CNODE_NAMES[@]}"; do
|
|
# Set the control-plane role label explicitly.
|
|
# k3s kubectl only — bare kubectl is not on the cnodes' root PATH.
|
|
cn "$CNODE1_IP" "k3s kubectl label node $name node-role.kubernetes.io/control-plane= --overwrite" 2>/dev/null || true
|
|
# Apply the taint (idempotent with --overwrite)
|
|
cn "$CNODE1_IP" "k3s kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null
|
|
echo " $name tainted."
|
|
done
|
|
|
|
# -------------------------------------------------------
|
|
# 2. Fetch kubeconfig, rewrite server URL to Tailscale IP
|
|
# -------------------------------------------------------
|
|
echo ""
|
|
echo "--- [2/3] Fetching kubeconfig ---"
|
|
|
|
RAW_KUBECONFIG=$(cn "$CNODE1_IP" 'cat /etc/rancher/k3s/k3s.yaml')
|
|
|
|
# Rewrite 127.0.0.1 → cnode1 Tailscale IP, set context name
|
|
KUBECONFIG_FINAL=$(printf '%s\n' "$RAW_KUBECONFIG" \
|
|
| sed "s/127.0.0.1/$CNODE1_IP/g" \
|
|
| sed 's/default/pfv-k8s/g')
|
|
|
|
KUBECONFIG_DIR="$HOME/.kube"
|
|
KUBECONFIG_FILE="$KUBECONFIG_DIR/config.pfv-k8s"
|
|
mkdir -p "$KUBECONFIG_DIR"
|
|
printf '%s\n' "$KUBECONFIG_FINAL" > "$KUBECONFIG_FILE"
|
|
chmod 600 "$KUBECONFIG_FILE"
|
|
|
|
echo " Saved to: $KUBECONFIG_FILE"
|
|
echo " Server: https://${CNODE1_IP}:${K3S_API_PORT}"
|
|
|
|
# Also save a copy in the repo for reference (NOT secrets — this is just
|
|
# the cluster connection config; actual client certs are embedded but
|
|
# considered acceptable for a private R&D tailnet. If ITAR workloads are
|
|
# added later, move to OIDC and remove this file.)
|
|
K8S_DIR="$SCRIPT_DIR"
|
|
printf '%s\n' "$KUBECONFIG_FINAL" > "$K8S_DIR/kubeconfig.yaml"
|
|
chmod 600 "$K8S_DIR/kubeconfig.yaml"
|
|
echo " Copy saved: $K8S_DIR/kubeconfig.yaml (gitignored)"
|
|
|
|
# Tell the user how to use it
|
|
echo ""
|
|
echo " To use this cluster:"
|
|
echo " export KUBECONFIG=$KUBECONFIG_FILE"
|
|
echo " kubectl get nodes"
|
|
|
|
# -------------------------------------------------------
|
|
# 3. Deploy tuned (network-latency profile) on all cnodes
|
|
# -------------------------------------------------------
|
|
echo ""
|
|
echo "--- [3/4] Deploying tuned (network-latency) on cnodes ---"
|
|
for ip in "${ALL_CNODES[@]}"; do
|
|
echo " $ip..."
|
|
cn "$ip" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>/dev/null; DEBIAN_FRONTEND=noninteractive apt-get install -y -qq tuned 2>/dev/null; tuned-adm profile network-latency; systemctl enable tuned; systemctl restart tuned; tuned-adm active' 2>&1 | tail -1
|
|
done
|
|
|
|
# -------------------------------------------------------
|
|
# 4. Verify cluster health
|
|
# -------------------------------------------------------
|
|
echo ""
|
|
echo "--- [4/4] Verifying cluster health ---"
|
|
|
|
export KUBECONFIG="$KUBECONFIG_FILE"
|
|
|
|
echo ""
|
|
echo "=== Nodes ==="
|
|
kubectl get nodes -o wide 2>&1
|
|
|
|
echo ""
|
|
echo "=== Node taints ==="
|
|
kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints 2>&1
|
|
|
|
echo ""
|
|
echo "=== Node IPs (should be 100.x Tailscale) ==="
|
|
kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>&1
|
|
|
|
echo ""
|
|
echo "=== API + etcd liveness ==="
|
|
cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>&1 || echo "(API not ready)"
|
|
# k3s ships no etcdctl; snapshots answering proves the etcd store is readable
|
|
cn "$CNODE1_IP" 'k3s etcd-snapshot list 2>/dev/null | head -3' 2>&1
|
|
|
|
echo ""
|
|
echo "=== System pods ==="
|
|
kubectl get pods -A 2>&1
|
|
|
|
echo ""
|
|
echo "============================================"
|
|
echo " Cluster is ready."
|
|
echo ""
|
|
echo " KUBECONFIG: $KUBECONFIG_FILE"
|
|
echo " Next: bash $SCRIPT_DIR/verify.sh"
|
|
echo "============================================"
|