diff --git a/awx/README.md b/awx/README.md index 2babde9..be74cc4 100644 --- a/awx/README.md +++ b/awx/README.md @@ -1,10 +1,5 @@ -# awx/README.md +# awx — SPLIT OUT -> **Documentation moved to Discourse — the canonical source of truth.** -> -> **Ansible AWX deployment on k3s** -> -> **Read it here:** https://community.turnsys.com/t/302 -> -> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context. -> Do not update content here — edit the Discourse wiki topic instead.* +AWX control plane body of work moved to [KNEL/awx](https://git.knownelement.com/KNEL/awx) +on 2026-09-05 (split ticket [#769](https://projects.knownelement.com/issues/769), +bootstrap ticket [#707](https://projects.knownelement.com/issues/707)). diff --git a/awx/awx-instance.yaml b/awx/awx-instance.yaml deleted file mode 100644 index d069c01..0000000 --- a/awx/awx-instance.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -# AWX namespace -apiVersion: v1 -kind: Namespace -metadata: - name: awx ---- -# Admin password secret — the password is 'REDACTED_PASSWORD' (fleet standard) -apiVersion: v1 -kind: Secret -metadata: - name: awx-admin-password - namespace: awx -type: Opaque -stringData: - password: REDACTED_PASSWORD ---- -# AWX Custom Resource — single instance, LoadBalancer service -apiVersion: awx.ansible.com/v1beta1 -kind: AWX -metadata: - name: tsys-awx - namespace: awx -spec: - service_type: LoadBalancer - ingress_type: none - - admin_user: admin - admin_password_secret: awx-admin-password - - # PostgreSQL — bundled, stored on local disk via PVC (k3s local-path) - postgres_storage_class: local-path - postgres_storage_requirements: - requests: - storage: 8Gi - postgres_resource_requirements: - requests: - memory: 1Gi - - # Resource limits — fit within 12 GB host RAM - web_resource_requirements: - requests: - memory: 1Gi - task_resource_requirements: - requests: - memory: 1Gi diff --git a/awx/deploy-awx.sh b/awx/deploy-awx.sh deleted file mode 100644 index 1010e84..0000000 --- a/awx/deploy-awx.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -############################################################################### -# deploy-awx.sh — Deploy AWX Operator + instance on k3s. -# -# Prerequisites: k3s must be installed and running (install-k3s.sh). -# Intended to run ON the target VM (tsys-awx.knel.net) as root or via sudo. -# -# Usage: sudo bash deploy-awx.sh -############################################################################### -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}" - -OPERATOR_VERSION="${OPERATOR_VERSION:-2.19.1}" - -echo "==========================================================" -echo " AWX Operator deployment — version ${OPERATOR_VERSION}" -echo "==========================================================" - -# --------------------------------------------------------------------------- -# Step 1: Create namespace -# --------------------------------------------------------------------------- -echo "" -echo "=== Step 1: Create namespace ===" -kubectl apply -f "${SCRIPT_DIR}/namespace.yaml" - -# --------------------------------------------------------------------------- -# Step 2: Deploy AWX Operator -# --------------------------------------------------------------------------- -echo "" -echo "=== Step 2: Deploy AWX Operator ${OPERATOR_VERSION} ===" - -# Clone the operator to get kustomize manifests -OPERATOR_DIR="/tmp/awx-operator-${OPERATOR_VERSION}" -rm -rf "${OPERATOR_DIR}" -git clone --branch "${OPERATOR_VERSION}" --depth 1 \ - "https://github.com/ansible/awx-operator.git" "${OPERATOR_DIR}" 2>&1 | tail -3 - -# The operator's default namespace is 'awx' — matches our setup -# Apply the operator via kustomize (config/default has the full manifest set) -kubectl apply -k "${OPERATOR_DIR}/config/default" 2>&1 || { - echo "kustomize apply failed, trying raw manifests..." - kubectl apply -f "https://raw.githubusercontent.com/ansible/awx-operator/${OPERATOR_VERSION}/deploy/awx-operator.yaml" -} - -# Fix kube-rbac-proxy image (gcr.io/kubebuilder/kube-rbac-proxy was removed; -# quay.io/brancz/kube-rbac-proxy is the maintained replacement) -echo "" -echo "=== Patching kube-rbac-proxy image ===" -kubectl set image deployment/awx-operator-controller-manager -n awx \ - kube-rbac-proxy=quay.io/brancz/kube-rbac-proxy:v0.15.0 2>&1 || true - -# Scale down any old replicasets that still reference the broken image -for rs in $(kubectl -n awx get rs -l control-plane=controller-manager -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do - img=$(kubectl -n awx get rs "${rs}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="kube-rbac-proxy")].image}' 2>/dev/null) - if [[ "${img}" == *"gcr.io/kubebuilder"* ]]; then - echo "Scaling down old RS ${rs} (has broken gcr.io image)" - kubectl -n awx scale rs "${rs}" --replicas=0 2>&1 - fi -done - -echo "" -echo "Waiting for AWX Operator deployment to be ready..." -kubectl -n awx wait --for=condition=Available deployment/awx-operator-controller-manager \ - --timeout=300s 2>&1 || { - echo "Operator not ready yet — checking status..." - kubectl -n awx get pods -} - -# --------------------------------------------------------------------------- -# Step 3: Deploy AWX instance -# --------------------------------------------------------------------------- -echo "" -echo "=== Step 3: Deploy AWX instance ===" -kubectl apply -f "${SCRIPT_DIR}/awx-instance.yaml" - -echo "" -echo "AWX instance created. Operator will now reconcile." -echo "This typically takes 5-10 minutes for the first deployment." -echo "" -echo "Monitor progress with:" -echo " kubectl -n awx get awx tsys-awx -o jsonpath='{.status.conditions}' | jq ." -echo " kubectl -n awx get pods -w" -echo " kubectl -n awx logs deployment/awx-operator-controller-manager -f" diff --git a/awx/install-k3s.sh b/awx/install-k3s.sh deleted file mode 100644 index 9f98423..0000000 --- a/awx/install-k3s.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -############################################################################### -# install-k3s.sh — Install k3s single-node on the tsys-awx VM. -# -# Intended to run ON the target VM (tsys-awx.knel.net) as root or via sudo. -# Installs k3s without Traefik (we use NodePort/LoadBalancer directly). -# -# Usage: sudo bash install-k3s.sh -############################################################################### -set -euo pipefail - -NODE_NAME="${NODE_NAME:-tsys-awx}" - -echo "==========================================================" -echo " k3s single-node install — ${NODE_NAME}" -echo "==========================================================" - -if command -v k3s >/dev/null 2>&1 && k3s kubectl get nodes >/dev/null 2>&1; then - echo "k3s already installed and running. Skipping." - k3s kubectl get nodes - exit 0 -fi - -echo "" -echo "=== Installing k3s (this takes 1-2 minutes) ===" -curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh - - -echo "" -echo "=== Waiting for k3s node to be Ready ===" -export KUBECONFIG=/etc/rancher/k3s/k3s.yaml -for i in $(seq 1 30); do - if k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; then - echo "Node is Ready!" - k3s kubectl get nodes - break - fi - echo " waiting... (${i}/30)" - sleep 5 -done - -echo "" -echo "=== k3s install complete ===" -echo "kubeconfig: /etc/rancher/k3s/k3s.yaml" -echo "kubectl: k3s kubectl (or set KUBECONFIG=/etc/rancher/k3s/k3s.yaml)" diff --git a/awx/namespace.yaml b/awx/namespace.yaml deleted file mode 100644 index b9fd05b..0000000 --- a/awx/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# AWX Operator namespace -apiVersion: v1 -kind: Namespace -metadata: - name: awx diff --git a/awx/verify-awx.sh b/awx/verify-awx.sh deleted file mode 100644 index 8c70b7b..0000000 --- a/awx/verify-awx.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -############################################################################### -# verify-awx.sh — Verify AWX deployment status and access. -# -# Intended to run ON the target VM (tsys-awx.knel.net). -# Usage: bash verify-awx.sh -############################################################################### -set -euo pipefail - -export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}" - -echo "==========================================================" -echo " AWX Deployment Verification — $(date)" -echo "==========================================================" - -echo "" -echo "=== 1. k3s node ===" -kubectl get nodes - -echo "" -echo "=== 2. AWX pods ===" -kubectl -n awx get pods - -echo "" -echo "=== 3. AWX CR status ===" -kubectl -n awx get awx tsys-awx -o jsonpath='{range .status.conditions[*]}{.type}: {.message}{"\n"}{end}' 2>/dev/null || echo "AWX CR not found" - -echo "" -echo "=== 4. Services ===" -kubectl -n awx get svc - -echo "" -echo "=== 5. LoadBalancer / NodePort access ===" -LB_IP=$(kubectl -n awx get svc tsys-awx-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "") -LB_HOST=$(kubectl -n awx get svc tsys-awx-service -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || echo "") -NODE_PORT=$(kubectl -n awx get svc tsys-awx-service -o jsonpath='{.spec.ports[0].nodePort}' 2>/dev/null || echo "") - -if [ -n "${LB_IP}" ]; then - echo "LoadBalancer IP: ${LB_IP}" - ACCESS_URL="http://${LB_IP}" -elif [ -n "${LB_HOST}" ]; then - echo "LoadBalancer hostname: ${LB_HOST}" - ACCESS_URL="http://${LB_HOST}" -elif [ -n "${NODE_PORT}" ]; then - echo "NodePort: ${NODE_PORT}" - ACCESS_URL="http://$(hostname -I | awk '{print $1}'):${NODE_PORT}" -else - echo "Service not ready yet" - ACCESS_URL="" -fi - -echo "" -echo "=== 6. Admin password ===" -ADMIN_PASS=$(kubectl -n awx get secret awx-admin-password -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "") -if [ -n "${ADMIN_PASS}" ]; then - echo "User: admin" - echo "Password: ${ADMIN_PASS}" -else - echo "Admin password secret not found" -fi - -echo "" -echo "=== 7. HTTP check ===" -if [ -n "${ACCESS_URL}" ]; then - echo "Testing ${ACCESS_URL}..." - HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "${ACCESS_URL}" 2>/dev/null || echo "failed") - echo "HTTP response: ${HTTP_CODE}" - if [ "${HTTP_CODE}" = "200" ] || [ "${HTTP_CODE}" = "302" ] || [ "${HTTP_CODE}" = "301" ]; then - echo "✓ AWX is accessible at ${ACCESS_URL}" - else - echo "✗ AWX not yet responding (HTTP ${HTTP_CODE})" - fi -fi - -echo "" -echo "==========================================================" -if [ -n "${ACCESS_URL}" ]; then - echo " AWX Access URL: ${ACCESS_URL}" -fi -echo "==========================================================" diff --git a/k8s/README.md b/k8s/README.md index d568cd5..6f37996 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -1,10 +1,5 @@ -# k8s/README.md +# k8s — SPLIT OUT -> **Documentation moved to Discourse — the canonical source of truth.** -> -> **k3s cluster setup scripts: wipe, bootstrap, taint, verify** -> -> **Read it here:** https://community.turnsys.com/t/305 -> -> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context. -> Do not update content here — edit the Discourse wiki topic instead.* +K8S platform body of work moved to [KNEL/k8s](https://git.knownelement.com/KNEL/k8s) +on 2026-09-05 (split ticket [#769](https://projects.knownelement.com/issues/769), +master ticket [#699](https://projects.knownelement.com/issues/699)). diff --git a/k8s/code/nvidia-device-plugin.yaml b/k8s/code/nvidia-device-plugin.yaml deleted file mode 100644 index ec9ea3f..0000000 --- a/k8s/code/nvidia-device-plugin.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# nvidia GPU device-plugin DaemonSet — pfv-k8s-wnode-tsys3 (Quadro M2000M 4G) -# GPU verified end-to-end 2026-09-04 (#763/#769): pod with nvidia.com/gpu:1 -# schedules + nvidia-smi works in-container (driver 550.163.01, CUDA 12.4). -# Prereq on the node: nvidia driver + nvidia-container-runtime (k3s). -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nvidia-device-plugin-daemonset - namespace: kube-system -spec: - selector: - matchLabels: - name: nvidia-device-plugin-ds - template: - metadata: - labels: - name: nvidia-device-plugin-ds - spec: - nodeSelector: - kubernetes.io/hostname: pfv-k8s-wnode-tsys3 - tolerations: - - key: CriticalAddonsOnly - operator: Exists - containers: - - name: nvidia-device-plugin-ctr - image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0 - args: ["--fail-on-init-error=false"] - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins diff --git a/k8s/docs/ARCHITECTURE.md b/k8s/docs/ARCHITECTURE.md deleted file mode 100644 index f2d3e7b..0000000 --- a/k8s/docs/ARCHITECTURE.md +++ /dev/null @@ -1,10 +0,0 @@ -# k8s/docs/ARCHITECTURE.md - -> **Documentation moved to Discourse — the canonical source of truth.** -> -> **k8s target architecture: control plane, network, identity, storage, DR** -> -> **Read it here:** https://community.turnsys.com/t/305 -> -> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context. -> Do not update content here — edit the Discourse wiki topic instead.* diff --git a/k8s/docs/DISTRO-DECISION.md b/k8s/docs/DISTRO-DECISION.md deleted file mode 100644 index 1f4fa73..0000000 --- a/k8s/docs/DISTRO-DECISION.md +++ /dev/null @@ -1,10 +0,0 @@ -# k8s/docs/DISTRO-DECISION.md - -> **Documentation moved to Discourse — the canonical source of truth.** -> -> **Talos vs k3s distro analysis and decision** -> -> **Read it here:** https://community.turnsys.com/t/305 -> -> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context. -> Do not update content here — edit the Discourse wiki topic instead.* diff --git a/k8s/docs/README.md b/k8s/docs/README.md deleted file mode 100644 index 4ffc414..0000000 --- a/k8s/docs/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# k8s/docs/README.md - -> **Documentation moved to Discourse — the canonical source of truth.** -> -> **k8s docs index + TL;DR** -> -> **Read it here:** https://community.turnsys.com/t/305 -> -> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context. -> Do not update content here — edit the Discourse wiki topic instead.* diff --git a/k8s/env.sh b/k8s/env.sh deleted file mode 100644 index 64fbeb2..0000000 --- a/k8s/env.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/bash -# shellcheck disable=SC2034 # sourced config file; variables are consumed by scripts that source this -# k8s/env.sh — shared config for all k8s scripts. Source this. -# -# All cluster communication goes over Tailscale IPs. No LAN IPs, ever. - -# --- Control plane nodes (Tailscale 100.x addresses) --- -CNODE1_NAME="pfv-k8s-cnode1" -CNODE1_IP="100.125.134.53" - -CNODE2_NAME="pfv-k8s-cnode2" -CNODE2_IP="100.109.34.72" - -CNODE3_NAME="pfv-k8s-cnode3" -CNODE3_IP="100.106.222.18" - -ALL_CNODES=("$CNODE1_IP" "$CNODE2_IP" "$CNODE3_IP") -ALL_CNODE_NAMES=("$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME") - -# Bootstrap node (first etcd member) -BOOTSTRAP_IP="$CNODE1_IP" -BOOTSTRAP_NAME="$CNODE1_NAME" - -# --- Worker nodes (Tailscale 100.x addresses) --- -# Roster (2026-09-02, #368): the wnode-tsys5 slot was REPLACED by -# ultix-streaming (founder ruling) — same box, joined directly. -WNODE1_NAME="pfv-k8s-wnode-tsys3" -WNODE1_IP="100.126.9.112" - -WNODE2_NAME="ultix-streaming" -WNODE2_IP="100.101.187.119" - -WNODE3_NAME="pfv-k8s-wnode-tsys6" -WNODE3_IP="100.83.49.75" - -WNODE4_NAME="pfv-k8s-wnode-tsys7" -WNODE4_IP="100.119.240.11" - -WNODE5_NAME="pfv-k8s-wnode-tsys9" -WNODE5_IP="100.95.201.66" - -# ultix-offstage: unreachable 2026-09-02 — join when it is back on the tailnet -# WNODE6_NAME="ultix-offstage" -# WNODE6_IP="100.70.119.59" - -ALL_WNODES=("$WNODE1_IP" "$WNODE2_IP" "$WNODE3_IP" "$WNODE4_IP" "$WNODE5_IP") -ALL_WNODE_NAMES=("$WNODE1_NAME" "$WNODE2_NAME" "$WNODE3_NAME" "$WNODE4_NAME" "$WNODE5_NAME") - -# --- SSH --- -SSH_USER="localuser" -SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15) - -# --- k3s --- -K3S_VERSION="v1.36.2+k3s1" -K3S_API_PORT="6443" - -# Shared TLS SANs (all cnodes + hostnames so certs are valid cluster-wide) -TLS_SANS=( - "$CNODE1_IP" "$CNODE2_IP" "$CNODE3_IP" - "$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME" -) - -# Helper: run a command on a node as root (via passwordless sudo) -cn() { - local ip="$1"; shift - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -c '$*'" 2>&1 -} - -# Helper: run a heredoc script on a node -cn_file() { - local ip="$1" - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s" -} diff --git a/k8s/install-cp.sh b/k8s/install-cp.sh deleted file mode 100644 index 4dfaaa0..0000000 --- a/k8s/install-cp.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/bash -# -# k8s/install-cp.sh — bootstrap 3-node k3s HA control plane (embedded etcd) -# -# All traffic goes over Tailscale IPs. LAN addresses are never used for -# cluster communication. -# -# Sequence: -# 1. Install cnode1 with --cluster-init (creates new etcd cluster) -# 2. Wait for cnode1 API to be ready -# 3. Fetch join token from cnode1 -# 4. Install cnode2 + cnode3 as additional servers (join etcd quorum) -# 5. Wait for all 3 etcd members to be healthy -# -set -uo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# shellcheck source=./env.sh -source "$SCRIPT_DIR/env.sh" - -# Build the shared TLS-SAN flags -tls_san_flags="" -for san in "${TLS_SANS[@]}"; do - tls_san_flags+=" --tls-san=$san" -done - -echo "============================================" -echo " Bootstrapping k3s HA control plane" -echo " k3s version: $K3S_VERSION" -echo " Transport: Tailscale (wireguard)" -echo "============================================" - -# ------------------------------------------------------- -# Phase 1: Install bootstrap node (cnode1) with --cluster-init -# ------------------------------------------------------- -echo "" -echo "--- [1/5] Installing bootstrap node: $CNODE1_NAME ($CNODE1_IP) ---" - -# shellcheck disable=SC2087 # heredoc intentionally expands local config (node IPs, k3s version) before sending to remote -ssh "${SSH_OPTS[@]}" "${SSH_USER}@${CNODE1_IP}" "sudo -n bash -s" </dev/null | grep -q "ok"; then - API_READY=true - break - fi - echo " ...waiting ($i/30)" - sleep 5 -done - -if [ "$API_READY" = false ]; then - echo "ERROR: cnode1 API did not become ready in 150s." - echo "Checking service status:" - cn "$CNODE1_IP" 'systemctl status k3s --no-pager | tail -20' - exit 1 -fi -echo " cnode1 API is ready." - -# ------------------------------------------------------- -# Phase 3: Fetch join token from cnode1 -# ------------------------------------------------------- -echo "" -echo "--- [3/5] Fetching join token from cnode1 ---" -JOIN_TOKEN=$(cn "$CNODE1_IP" 'cat /var/lib/rancher/k3s/server/token') -if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then - echo "ERROR: could not fetch token from cnode1." - echo " Got: ${JOIN_TOKEN:0:40}" - exit 1 -fi -echo " Token fetched (masked: ${JOIN_TOKEN:0:12}***)" - -SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}" - -# ------------------------------------------------------- -# Phase 4: Install cnode2 and cnode3 as additional servers -# ------------------------------------------------------- -for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do - # Derive node name from IP - case "$node_ip" in - "$CNODE2_IP") node_name="$CNODE2_NAME" ;; - "$CNODE3_IP") node_name="$CNODE3_NAME" ;; - esac - - echo "" - echo "--- [4/5] Joining server: $node_name ($node_ip) ---" - -# shellcheck disable=SC2087 # heredoc intentionally expands local config before sending to remote - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" </dev/null | grep -c " Ready"' 2>/dev/null || echo 0) - if [ "$READY_NODES" = "3" ]; then - echo " All 3 nodes Ready." - break - fi - echo " ...waiting ($i/30, $READY_NODES/3 ready)" - sleep 10 -done - -if [ "$READY_NODES" != "3" ]; then - echo "WARN: Only $READY_NODES/3 nodes ready after 300s. Check manually." -fi - -echo "" -echo "============================================" -echo " Control plane nodes:" -echo "============================================" -cn "$CNODE1_IP" 'k3s kubectl get nodes -o wide' - -echo "" -echo "============================================" -echo " Bootstrap complete." -echo " Run: bash $SCRIPT_DIR/post-setup.sh" -echo "============================================" diff --git a/k8s/join-servers.sh b/k8s/join-servers.sh deleted file mode 100644 index e9aaeb3..0000000 --- a/k8s/join-servers.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/bash -# -# k8s/join-servers.sh — re-join cnode2 + cnode3 to the running cnode1 cluster -# -# Fixes the token-path bug from the initial install-cp.sh run. cnode1 is -# already healthy; this script only touches cnode2 and cnode3: -# 1. Uninstalls the broken k3s on each -# 2. Fetches the CORRECT token from cnode1 -# 3. Re-installs both as HA server nodes joining the etcd cluster -# -set -uo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -source "$SCRIPT_DIR/env.sh" - -# Build the shared TLS-SAN flags -tls_san_flags="" -for san in "${TLS_SANS[@]}"; do - tls_san_flags+=" --tls-san=$san" -done - -SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}" - -echo "============================================" -echo " Re-joining cnode2 + cnode3 to cluster" -echo " Bootstrap server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)" -echo "============================================" - -# ------------------------------------------------------- -# 1. Fetch the CORRECT token from cnode1 -# ------------------------------------------------------- -echo "" -echo "--- [1/4] Fetching join token from cnode1 ---" -JOIN_TOKEN=$(cn "$CNODE1_IP" 'cat /var/lib/rancher/k3s/server/token') -if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then - echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}" - exit 1 -fi -echo " Token OK (masked: ${JOIN_TOKEN:0:12}***)" - -# ------------------------------------------------------- -# 2. Uninstall broken k3s from cnode2 + cnode3 -# ------------------------------------------------------- -for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do - echo "" - echo "--- [2/4] Wiping broken k3s on $node_ip ---" - cn "$node_ip" ' - systemctl stop k3s 2>/dev/null || true - if [ -x /usr/local/bin/k3s-uninstall.sh ]; then - /usr/local/bin/k3s-uninstall.sh - else - echo "no k3s to remove" - fi - rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni - rm -f /etc/systemd/system/k3s.service /etc/systemd/system/k3s.service.env - systemctl daemon-reload - ip link delete cni0 2>/dev/null || true - ip link delete flannel.1 2>/dev/null || true - echo "WIPE DONE" - ' | tail -3 -done - -# ------------------------------------------------------- -# 3. Install cnode2 + cnode3 with correct token -# ------------------------------------------------------- -for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do - case "$node_ip" in - "$CNODE2_IP") node_name="$CNODE2_NAME" ;; - "$CNODE3_IP") node_name="$CNODE3_NAME" ;; - esac - - echo "" - echo "--- [3/4] Joining server: $node_name ($node_ip) ---" - -# shellcheck disable=SC2087 # heredoc intentionally expands local config before sending to remote - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" </dev/null | grep -c " Ready"' 2>/dev/null || echo 0) - if [ "$READY_NODES" = "3" ]; then - echo " All 3 nodes Ready." - break - fi - echo " ...waiting ($i/30, $READY_NODES/3 ready)" - sleep 10 -done - -echo "" -cn "$CNODE1_IP" 'k3s kubectl get nodes -o wide' -echo "" -cn "$CNODE1_IP" 'k3s etcdctl member list 2>/dev/null' - -if [ "$READY_NODES" = "3" ]; then - echo "" - echo "============================================" - echo " All 3 nodes joined. HA control plane active." - echo " Run: bash $SCRIPT_DIR/post-setup.sh" - echo "============================================" -else - echo "" - echo "WARN: $READY_NODES/3 ready. Check journalctl -u k3s on the failing node." - exit 1 -fi diff --git a/k8s/join-workers.sh b/k8s/join-workers.sh deleted file mode 100755 index b7d2344..0000000 --- a/k8s/join-workers.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/bash -# -# k8s/join-workers.sh — join worker nodes to the k3s cluster -# -# Joins all worker nodes defined in env.sh as k3s agents. Worker nodes -# run user workloads; control plane nodes are tainted NoSchedule. -# -set -uo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# shellcheck source=./env.sh -source "$SCRIPT_DIR/env.sh" - -echo "============================================" -echo " Joining worker nodes to cluster" -echo " Server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)" -echo " Workers: ${#ALL_WNODE_NAMES[@]}" -echo "============================================" - -# ------------------------------------------------------- -# 1. Fetch join token from bootstrap node -# ------------------------------------------------------- -echo "" -echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---" -JOIN_TOKEN=$(timeout 30 cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token') -if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then - echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}" - exit 1 -fi -echo " Token OK (masked: ${JOIN_TOKEN:0:12}***)" - -SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}" - -# ------------------------------------------------------- -# 2. Install k3s-agent on each worker -# ------------------------------------------------------- -for i in "${!ALL_WNODES[@]}"; do - node_ip="${ALL_WNODES[$i]}" - node_name="${ALL_WNODE_NAMES[$i]}" - - echo "" - echo "--- [2/3] Joining worker: $node_name ($node_ip) ---" - - # Wipe any existing k3s first (time-boxed: an unreachable node must not hang the run) - timeout 60 cn "$node_ip" ' - systemctl stop k3s-agent 2>/dev/null || true - if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then - /usr/local/bin/k3s-agent-uninstall.sh - fi - rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni - rm -f /etc/systemd/system/k3s-agent.service - systemctl daemon-reload - ip link delete cni0 2>/dev/null || true - ip link delete flannel.1 2>/dev/null || true - ' 2>/dev/null || true - - # Install as agent (300s deploy-class limit; a timed-out install is a failed node) - # shellcheck disable=SC2087 # heredoc intentionally expands local config - timeout 300 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" </dev/null | grep -c " Ready"' 2>/dev/null || echo 0) - if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then - echo " All $TOTAL_NODES nodes Ready." - break - fi - echo " ...waiting ($i/60, $READY_NODES/$TOTAL_NODES ready)" - sleep 10 -done - -echo "" -echo "============================================" -echo " Node status:" -echo "============================================" -cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes -o wide' - -if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then - echo "" - echo "============================================" - echo " All workers joined. Cluster fully operational." - echo "============================================" -else - echo "" - echo "WARN: $READY_NODES/$TOTAL_NODES ready. Check failing nodes." - exit 1 -fi diff --git a/k8s/post-setup.sh b/k8s/post-setup.sh deleted file mode 100644 index 5ac4884..0000000 --- a/k8s/post-setup.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/bash -# -# k8s/post-setup.sh — taint cnodes, fetch kubeconfig, verify cluster -# -# Taints all 3 control-plane nodes with NoSchedule so NO user workloads -# can land on them. Only system components (CoreDNS, metrics-server, -# kube-proxy, flannel) with built-in tolerations will run here. -# -set -uo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# shellcheck source=./env.sh -source "$SCRIPT_DIR/env.sh" - -echo "============================================" -echo " Post-setup: taints, kubeconfig, verify" -echo "============================================" - -# ------------------------------------------------------- -# 1. Taint all cnodes NoSchedule (no user workloads on control plane) -# ------------------------------------------------------- -echo "" -echo "--- [1/3] Tainting control-plane nodes (NoSchedule) ---" -for name in "${ALL_CNODE_NAMES[@]}"; do - # Set the control-plane role label explicitly. - # k3s kubectl only — bare kubectl is not on the cnodes' root PATH. - cn "$CNODE1_IP" "k3s kubectl label node $name node-role.kubernetes.io/control-plane= --overwrite" 2>/dev/null || true - # Apply the taint (idempotent with --overwrite) - cn "$CNODE1_IP" "k3s kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null - echo " $name tainted." -done - -# ------------------------------------------------------- -# 2. Fetch kubeconfig, rewrite server URL to Tailscale IP -# ------------------------------------------------------- -echo "" -echo "--- [2/3] Fetching kubeconfig ---" - -RAW_KUBECONFIG=$(cn "$CNODE1_IP" 'cat /etc/rancher/k3s/k3s.yaml') - -# Rewrite 127.0.0.1 → cnode1 Tailscale IP, set context name -KUBECONFIG_FINAL=$(printf '%s\n' "$RAW_KUBECONFIG" \ - | sed "s/127.0.0.1/$CNODE1_IP/g" \ - | sed 's/default/pfv-k8s/g') - -KUBECONFIG_DIR="$HOME/.kube" -KUBECONFIG_FILE="$KUBECONFIG_DIR/config.pfv-k8s" -mkdir -p "$KUBECONFIG_DIR" -printf '%s\n' "$KUBECONFIG_FINAL" > "$KUBECONFIG_FILE" -chmod 600 "$KUBECONFIG_FILE" - -echo " Saved to: $KUBECONFIG_FILE" -echo " Server: https://${CNODE1_IP}:${K3S_API_PORT}" - -# Also save a copy in the repo for reference (NOT secrets — this is just -# the cluster connection config; actual client certs are embedded but -# considered acceptable for a private R&D tailnet. If ITAR workloads are -# added later, move to OIDC and remove this file.) -K8S_DIR="$SCRIPT_DIR" -printf '%s\n' "$KUBECONFIG_FINAL" > "$K8S_DIR/kubeconfig.yaml" -chmod 600 "$K8S_DIR/kubeconfig.yaml" -echo " Copy saved: $K8S_DIR/kubeconfig.yaml (gitignored)" - -# Tell the user how to use it -echo "" -echo " To use this cluster:" -echo " export KUBECONFIG=$KUBECONFIG_FILE" -echo " kubectl get nodes" - -# ------------------------------------------------------- -# 3. Deploy tuned (network-latency profile) on all cnodes -# ------------------------------------------------------- -echo "" -echo "--- [3/4] Deploying tuned (network-latency) on cnodes ---" -for ip in "${ALL_CNODES[@]}"; do - echo " $ip..." - cn "$ip" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>/dev/null; DEBIAN_FRONTEND=noninteractive apt-get install -y -qq tuned 2>/dev/null; tuned-adm profile network-latency; systemctl enable tuned; systemctl restart tuned; tuned-adm active' 2>&1 | tail -1 -done - -# ------------------------------------------------------- -# 4. Verify cluster health -# ------------------------------------------------------- -echo "" -echo "--- [4/4] Verifying cluster health ---" - -export KUBECONFIG="$KUBECONFIG_FILE" - -echo "" -echo "=== Nodes ===" -kubectl get nodes -o wide 2>&1 - -echo "" -echo "=== Node taints ===" -kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints 2>&1 - -echo "" -echo "=== Node IPs (should be 100.x Tailscale) ===" -kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>&1 - -echo "" -echo "=== API + etcd liveness ===" -cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>&1 || echo "(API not ready)" -# k3s ships no etcdctl; snapshots answering proves the etcd store is readable -cn "$CNODE1_IP" 'k3s etcd-snapshot list 2>/dev/null | head -3' 2>&1 - -echo "" -echo "=== System pods ===" -kubectl get pods -A 2>&1 - -echo "" -echo "============================================" -echo " Cluster is ready." -echo "" -echo " KUBECONFIG: $KUBECONFIG_FILE" -echo " Next: bash $SCRIPT_DIR/verify.sh" -echo "============================================" diff --git a/k8s/probe-nodes.sh b/k8s/probe-nodes.sh deleted file mode 100644 index bdb4ce8..0000000 --- a/k8s/probe-nodes.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/bash -# k8s/probe-nodes.sh — verify SSH + Tailscale reachability on all cnodes -set -uo pipefail -source "$(cd "$(dirname "$0")" && pwd)/env.sh" - -for ip in "${ALL_CNODES[@]}"; do - echo "=== $ip ===" - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" \ - 'echo "host=$(hostname)"; \ - echo "tailscale_ip=$(tailscale ip -4)"; \ - echo "k3s=$(systemctl is-active k3s 2>/dev/null)"; \ - echo "kernel=$(uname -r)"; \ - free -h | head -2; \ - df -h / | tail -1' 2>&1 || echo "FAILED to reach $ip" - echo -done diff --git a/k8s/proxmox-tuning.sh b/k8s/proxmox-tuning.sh deleted file mode 100644 index def820d..0000000 --- a/k8s/proxmox-tuning.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# -# k8s/proxmox-tuning.sh — apply ultix-style Proxmox tuning to k8s VMs -# -# Codifies the config applied 2026-09-01 during the #367 rebuild: -# - disk: ssd=1,discard=on (guest sees non-rotational; qcow2 trims) -# - net: queues=2 on both NICs (multiqueue; guest activates via ethtool) -# - cpu: cpuunits so etcd/control plane wins host CPU contention -# - boot: onboot=1 + startup order (cnodes before wnodes) -# - balloon was already 0 on every k8s VM (etcd memory predictability) -# -# Pending items (ssd/discard, queues) activate at the VM's next restart. -# Reboot waves are SERIAL with a health gate between hops — never batch. -# -# DRY RUN by default. Apply with RUN=1. Run on each Proxmox host, or via -# PROX_HOST= bash tests/remote.sh prox 'bash -s' < k8s/proxmox-tuning.sh -# -set -uo pipefail -RUN="${RUN:-0}" - -# vmid|cpuunits|startup|scsi0-line|net0-line|net1-line -# scsi0/net lines are the FULL desired config (Proxmox replaces wholesale). -TUNINGS=( - "102|4000|order=20,up=180|TS5-SSD:102/vm-102-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:CB:97:10,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:47:13:08,bridge=datanet,queues=2" - "705|4000|order=20,up=180|TS5-SSD:705/vm-705-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:40:25:F8,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:AF:3D:89,bridge=datanet,queues=2" - "603|4000|order=20,up=180|TS5-SSD:603/vm-603-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:38:C0:58,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:1F:F1:E1,bridge=storagenet,queues=2" - "313|2000|order=40,up=120|local-lvm:vm-313-disk-0,iothread=1,ssd=1,discard=on,size=300G|virtio=BC:24:11:EE:7E:7B,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:DF:C2:E9,bridge=datanet,queues=2" - "601|2000|order=40,up=120|local-lvm:vm-601-disk-0,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:FA:6E:B5,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:39:7D:0B,bridge=storagenet,queues=2" - "701|2000|order=40,up=120|S2:701/vm-701-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:30:B8:07,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:C8:0E:8A,bridge=datanet,queues=2" - "905|2000|order=40,up=120|S2:905/vm-905-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:EE:2B:B6,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:9B:57:06,bridge=datanet,queues=2" -) - -apply() { - local vmid="$1" cpuunits="$2" startup="$3" scsi0="$4" net0="$5" net1="$6" - if [ "$RUN" = "1" ]; then - qm set "$vmid" --cpuunits "$cpuunits" --startup "$startup" \ - --scsi0 "$scsi0" --net0 "$net0" --net1 "$net1" - else - echo "DRY: qm set $vmid --cpuunits $cpuunits --startup $startup --scsi0 $scsi0 --net0 $net0 --net1 $net1" - fi -} - -for t in "${TUNINGS[@]}"; do - IFS='|' read -r vmid cpuunits startup scsi0 net0 net1 <<< "$t" - # Skip VMs that live on other hosts (qm set errors on unknown VMID) - if ! qm status "$vmid" >/dev/null 2>&1; then - echo "skip: VM $vmid not on this host" - continue - fi - echo "== VM $vmid ==" - apply "$vmid" "$cpuunits" "$startup" "$scsi0" "$net0" "$net1" -done - -echo -if [ "$RUN" = "1" ]; then - for t in "${TUNINGS[@]}"; do - vmid="${t%%|*}" - qm status "$vmid" >/dev/null 2>&1 || continue - echo "--- pending VM $vmid ---" - qm pending "$vmid" | grep -E "^(new|cur) (scsi0|net0|net1|cpuunits|startup)" || true - done -fi diff --git a/k8s/verify.sh b/k8s/verify.sh deleted file mode 100644 index 183eb9c..0000000 --- a/k8s/verify.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/bash -# -# k8s/verify.sh — health check for the pfv-k8s control plane -# -set -uo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# shellcheck source=./env.sh -source "$SCRIPT_DIR/env.sh" - -export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config.pfv-k8s}" - -PASS=0 -FAIL=0 -ok() { echo " [PASS] $1"; PASS=$((PASS+1)); } -fail() { echo " [FAIL] $1"; FAIL=$((FAIL+1)); } - -echo "============================================" -echo " pfv-k8s Control Plane Health Check" -echo "============================================" - -# 1. All 3 nodes Ready -echo "" -echo "--- Nodes Ready ---" -READY=$(kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo 0) -if [ "$READY" = "3" ]; then ok "All 3 nodes Ready"; else fail "Expected 3 Ready nodes, got $READY"; fi - -kubectl get nodes -o wide 2>&1 | sed 's/^/ /' - -# 2. Nodes use Tailscale IPs -echo "" -echo "--- Tailscale IPs ---" -for name in "${ALL_CNODE_NAMES[@]}"; do - IP=$(kubectl get node "$name" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null) - case "$IP" in - 100.*) ok "$name uses Tailscale IP ($IP)" ;; - *) fail "$name uses non-Tailscale IP ($IP)" ;; - esac -done - -# 3. Taints applied (NoSchedule on all cnodes) -echo "" -echo "--- Control-plane taints ---" -for name in "${ALL_CNODE_NAMES[@]}"; do - TAINT=$(kubectl get node "$name" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null) - if echo "$TAINT" | grep -q "control-plane"; then - ok "$name has control-plane taint" - else - fail "$name missing control-plane taint" - fi -done - -# 4. etcd members = 3 (k3s v1.36 embeds etcdctl; verify via node roles + API) -echo "" -echo "--- etcd quorum ---" -# All 3 nodes must have the etcd role label -ETCD_NODES=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.node-role\.kubernetes\.io/etcd}{"\n"}{end}' 2>/dev/null | grep -c "true" || echo 0) -if [ "$ETCD_NODES" = "3" ]; then ok "3 nodes have etcd role (embedded HA etcd)"; else fail "Only $ETCD_NODES/3 nodes have etcd role"; fi - -# Verify etcd is the backing store via the API (if etcd is down, this fails) -LEASE_COUNT=$(kubectl get leases -A --no-headers 2>/dev/null | wc -l) -if [ "$LEASE_COUNT" -gt "0" ]; then - ok "etcd backing store active ($LEASE_COUNT leases found)" -else - fail "No leases found — etcd may not be accepting writes" -fi - -# Check etcd leader via metrics on cnode1 -LEADER=$(cn "$CNODE1_IP" 'ETCDCTL_API=3 /var/lib/rancher/k3s/data/current/bin/etcdctl \ - --endpoints=https://127.0.0.1:2379 \ - --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ - --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \ - --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \ - endpoint status 2>/dev/null' 2>/dev/null) -if [ -n "$LEADER" ]; then - ok "etcd endpoint reachable ($LEADER)" -else - # etcdctl not on disk in k3s v1.36; rely on node roles + leases above - ok "etcd health confirmed via 3 node roles + active leases (etcdctl not standalone in k3s v1.36)" -fi - -# 5. CoreDNS running -echo "" -echo "--- System components ---" -COREDNS=$(kubectl get pods -n kube-system -l k8s-app=kube-dns --no-headers 2>/dev/null | grep -c "Running" || echo 0) -if [ "$COREDNS" -ge "1" ]; then ok "CoreDNS running"; else fail "CoreDNS not running"; fi - -# 6. API server reachable over Tailscale -echo "" -echo "--- API server (Tailscale) ---" -if kubectl get --raw=/readyz 2>/dev/null | grep -q "ok"; then - ok "API server healthy over Tailscale" -else - fail "API server not reachable" -fi - -# 7. No user workloads on cnodes -echo "" -echo "--- Workload isolation ---" -USER_PODS=$(kubectl get pods -A --field-selector spec.nodeName="${CNODE1_NAME}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | wc -w) -# Subtract system pods -SYSTEM_PODS=$(kubectl get pods -A --field-selector spec.nodeName="${CNODE1_NAME}" -l k8s-app --no-headers 2>/dev/null | wc -l) -if [ "$((USER_PODS - SYSTEM_PODS))" -le 0 ]; then ok "Only system pods on cnodes (expected)"; else fail "Unexpected pods on $CNODE1_NAME"; fi - -echo "" -echo "============================================" -echo " Results: $PASS passed, $FAIL failed" -if [ "$FAIL" -gt 0 ]; then exit 1; fi -echo " All checks passed." -echo "============================================" diff --git a/k8s/wipe.sh b/k8s/wipe.sh deleted file mode 100644 index 917a1c8..0000000 --- a/k8s/wipe.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/bash -# -# k8s/wipe.sh — remove existing k3s from all cnodes -# -# The current install on cnode1/2 uses LAN IP (192.168.3.x). We need to -# rebuild with Tailscale IPs. This cluster has only system pods (16h old, -# no user workloads), so a clean wipe is safe. -# -set -uo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# shellcheck source=./env.sh -source "$SCRIPT_DIR/env.sh" - -echo "============================================" -echo " Wiping existing k3s from all cnodes" -echo "============================================" - -for ip in "${ALL_CNODES[@]}"; do - echo "" - echo "--- $ip ---" - cn "$ip" ' - # Stop services first - systemctl stop k3s 2>/dev/null || true - systemctl stop k3s-agent 2>/dev/null || true - - # Server uninstall (also removes agents) - if [ -x /usr/local/bin/k3s-uninstall.sh ]; then - /usr/local/bin/k3s-uninstall.sh - elif [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then - /usr/local/bin/k3s-agent-uninstall.sh - else - echo "no k3s installed on this node" - fi - - # Clean up residual state - rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni - rm -f /etc/systemd/system/k3s.service /etc/systemd/system/k3s-agent.service - systemctl daemon-reload - - # Clean network interfaces left by k3s/flannel - ip link delete cni0 2>/dev/null || true - ip link delete flannel.1 2>/dev/null || true - ip link delete kube-ipvs0 2>/dev/null || true - - echo "WIPE DONE" - ' -done - -echo "" -echo "============================================" -echo " Wipe complete. Ready for bootstrap." -echo "============================================"