[#769] stub k8s/awx — split to KNEL/k8s + KNEL/awx

https://projects.knownelement.com/issues/769

awx/README.md

@@ -1,10 +1,5 @@
-# awx/README.md
+# awx — SPLIT OUT
 
-> **Documentation moved to Discourse — the canonical source of truth.**
+AWX control plane body of work moved to [KNEL/awx](https://git.knownelement.com/KNEL/awx)
->
+on 2026-09-05 (split ticket [#769](https://projects.knownelement.com/issues/769),
-> **Ansible AWX deployment on k3s**
+bootstrap ticket [#707](https://projects.knownelement.com/issues/707)).
->
-> **Read it here:** https://community.turnsys.com/t/302
->
-> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
-> Do not update content here — edit the Discourse wiki topic instead.*

awx/awx-instance.yaml

@@ -1,46 +0,0 @@
----
-# AWX namespace
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: awx
----
-# Admin password secret — the password is 'REDACTED_PASSWORD' (fleet standard)
-apiVersion: v1
-kind: Secret
-metadata:
-  name: awx-admin-password
-  namespace: awx
-type: Opaque
-stringData:
-  password: REDACTED_PASSWORD
----
-# AWX Custom Resource — single instance, LoadBalancer service
-apiVersion: awx.ansible.com/v1beta1
-kind: AWX
-metadata:
-  name: tsys-awx
-  namespace: awx
-spec:
-  service_type: LoadBalancer
-  ingress_type: none
-
-  admin_user: admin
-  admin_password_secret: awx-admin-password
-
-  # PostgreSQL — bundled, stored on local disk via PVC (k3s local-path)
-  postgres_storage_class: local-path
-  postgres_storage_requirements:
-    requests:
-      storage: 8Gi
-  postgres_resource_requirements:
-    requests:
-      memory: 1Gi
-
-  # Resource limits — fit within 12 GB host RAM
-  web_resource_requirements:
-    requests:
-      memory: 1Gi
-  task_resource_requirements:
-    requests:
-      memory: 1Gi

awx/deploy-awx.sh

@@ -1,85 +0,0 @@
-#!/usr/bin/env bash
-###############################################################################
-# deploy-awx.sh — Deploy AWX Operator + instance on k3s.
-#
-# Prerequisites: k3s must be installed and running (install-k3s.sh).
-# Intended to run ON the target VM (tsys-awx.knel.net) as root or via sudo.
-#
-# Usage:  sudo bash deploy-awx.sh
-###############################################################################
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}"
-
-OPERATOR_VERSION="${OPERATOR_VERSION:-2.19.1}"
-
-echo "=========================================================="
-echo " AWX Operator deployment — version ${OPERATOR_VERSION}"
-echo "=========================================================="
-
-# ---------------------------------------------------------------------------
-# Step 1: Create namespace
-# ---------------------------------------------------------------------------
-echo ""
-echo "=== Step 1: Create namespace ==="
-kubectl apply -f "${SCRIPT_DIR}/namespace.yaml"
-
-# ---------------------------------------------------------------------------
-# Step 2: Deploy AWX Operator
-# ---------------------------------------------------------------------------
-echo ""
-echo "=== Step 2: Deploy AWX Operator ${OPERATOR_VERSION} ==="
-
-# Clone the operator to get kustomize manifests
-OPERATOR_DIR="/tmp/awx-operator-${OPERATOR_VERSION}"
-rm -rf "${OPERATOR_DIR}"
-git clone --branch "${OPERATOR_VERSION}" --depth 1 \
-    "https://github.com/ansible/awx-operator.git" "${OPERATOR_DIR}" 2>&1 | tail -3
-
-# The operator's default namespace is 'awx' — matches our setup
-# Apply the operator via kustomize (config/default has the full manifest set)
-kubectl apply -k "${OPERATOR_DIR}/config/default" 2>&1 || {
-    echo "kustomize apply failed, trying raw manifests..."
-    kubectl apply -f "https://raw.githubusercontent.com/ansible/awx-operator/${OPERATOR_VERSION}/deploy/awx-operator.yaml"
-}
-
-# Fix kube-rbac-proxy image (gcr.io/kubebuilder/kube-rbac-proxy was removed;
-# quay.io/brancz/kube-rbac-proxy is the maintained replacement)
-echo ""
-echo "=== Patching kube-rbac-proxy image ==="
-kubectl set image deployment/awx-operator-controller-manager -n awx \
-    kube-rbac-proxy=quay.io/brancz/kube-rbac-proxy:v0.15.0 2>&1 || true
-
-# Scale down any old replicasets that still reference the broken image
-for rs in $(kubectl -n awx get rs -l control-plane=controller-manager -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do
-    img=$(kubectl -n awx get rs "${rs}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="kube-rbac-proxy")].image}' 2>/dev/null)
-    if [[ "${img}" == *"gcr.io/kubebuilder"* ]]; then
-        echo "Scaling down old RS ${rs} (has broken gcr.io image)"
-        kubectl -n awx scale rs "${rs}" --replicas=0 2>&1
-    fi
-done
-
-echo ""
-echo "Waiting for AWX Operator deployment to be ready..."
-kubectl -n awx wait --for=condition=Available deployment/awx-operator-controller-manager \
-    --timeout=300s 2>&1 || {
-    echo "Operator not ready yet — checking status..."
-    kubectl -n awx get pods
-}
-
-# ---------------------------------------------------------------------------
-# Step 3: Deploy AWX instance
-# ---------------------------------------------------------------------------
-echo ""
-echo "=== Step 3: Deploy AWX instance ==="
-kubectl apply -f "${SCRIPT_DIR}/awx-instance.yaml"
-
-echo ""
-echo "AWX instance created. Operator will now reconcile."
-echo "This typically takes 5-10 minutes for the first deployment."
-echo ""
-echo "Monitor progress with:"
-echo "  kubectl -n awx get awx tsys-awx -o jsonpath='{.status.conditions}' | jq ."
-echo "  kubectl -n awx get pods -w"
-echo "  kubectl -n awx logs deployment/awx-operator-controller-manager -f"

awx/install-k3s.sh

@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-###############################################################################
-# install-k3s.sh — Install k3s single-node on the tsys-awx VM.
-#
-# Intended to run ON the target VM (tsys-awx.knel.net) as root or via sudo.
-# Installs k3s without Traefik (we use NodePort/LoadBalancer directly).
-#
-# Usage:  sudo bash install-k3s.sh
-###############################################################################
-set -euo pipefail
-
-NODE_NAME="${NODE_NAME:-tsys-awx}"
-
-echo "=========================================================="
-echo " k3s single-node install — ${NODE_NAME}"
-echo "=========================================================="
-
-if command -v k3s >/dev/null 2>&1 && k3s kubectl get nodes >/dev/null 2>&1; then
-    echo "k3s already installed and running. Skipping."
-    k3s kubectl get nodes
-    exit 0
-fi
-
-echo ""
-echo "=== Installing k3s (this takes 1-2 minutes) ==="
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh -
-
-echo ""
-echo "=== Waiting for k3s node to be Ready ==="
-export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
-for i in $(seq 1 30); do
-    if k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; then
-        echo "Node is Ready!"
-        k3s kubectl get nodes
-        break
-    fi
-    echo "  waiting... (${i}/30)"
-    sleep 5
-done
-
-echo ""
-echo "=== k3s install complete ==="
-echo "kubeconfig: /etc/rancher/k3s/k3s.yaml"
-echo "kubectl:    k3s kubectl  (or set KUBECONFIG=/etc/rancher/k3s/k3s.yaml)"

awx/namespace.yaml

@@ -1,6 +0,0 @@
----
-# AWX Operator namespace
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: awx

awx/verify-awx.sh

@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-###############################################################################
-# verify-awx.sh — Verify AWX deployment status and access.
-#
-# Intended to run ON the target VM (tsys-awx.knel.net).
-# Usage:  bash verify-awx.sh
-###############################################################################
-set -euo pipefail
-
-export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}"
-
-echo "=========================================================="
-echo " AWX Deployment Verification — $(date)"
-echo "=========================================================="
-
-echo ""
-echo "=== 1. k3s node ==="
-kubectl get nodes
-
-echo ""
-echo "=== 2. AWX pods ==="
-kubectl -n awx get pods
-
-echo ""
-echo "=== 3. AWX CR status ==="
-kubectl -n awx get awx tsys-awx -o jsonpath='{range .status.conditions[*]}{.type}: {.message}{"\n"}{end}' 2>/dev/null || echo "AWX CR not found"
-
-echo ""
-echo "=== 4. Services ==="
-kubectl -n awx get svc
-
-echo ""
-echo "=== 5. LoadBalancer / NodePort access ==="
-LB_IP=$(kubectl -n awx get svc tsys-awx-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
-LB_HOST=$(kubectl -n awx get svc tsys-awx-service -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || echo "")
-NODE_PORT=$(kubectl -n awx get svc tsys-awx-service -o jsonpath='{.spec.ports[0].nodePort}' 2>/dev/null || echo "")
-
-if [ -n "${LB_IP}" ]; then
-    echo "LoadBalancer IP: ${LB_IP}"
-    ACCESS_URL="http://${LB_IP}"
-elif [ -n "${LB_HOST}" ]; then
-    echo "LoadBalancer hostname: ${LB_HOST}"
-    ACCESS_URL="http://${LB_HOST}"
-elif [ -n "${NODE_PORT}" ]; then
-    echo "NodePort: ${NODE_PORT}"
-    ACCESS_URL="http://$(hostname -I | awk '{print $1}'):${NODE_PORT}"
-else
-    echo "Service not ready yet"
-    ACCESS_URL=""
-fi
-
-echo ""
-echo "=== 6. Admin password ==="
-ADMIN_PASS=$(kubectl -n awx get secret awx-admin-password -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "")
-if [ -n "${ADMIN_PASS}" ]; then
-    echo "User: admin"
-    echo "Password: ${ADMIN_PASS}"
-else
-    echo "Admin password secret not found"
-fi
-
-echo ""
-echo "=== 7. HTTP check ==="
-if [ -n "${ACCESS_URL}" ]; then
-    echo "Testing ${ACCESS_URL}..."
-    HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "${ACCESS_URL}" 2>/dev/null || echo "failed")
-    echo "HTTP response: ${HTTP_CODE}"
-    if [ "${HTTP_CODE}" = "200" ] || [ "${HTTP_CODE}" = "302" ] || [ "${HTTP_CODE}" = "301" ]; then
-        echo "✓ AWX is accessible at ${ACCESS_URL}"
-    else
-        echo "✗ AWX not yet responding (HTTP ${HTTP_CODE})"
-    fi
-fi
-
-echo ""
-echo "=========================================================="
-if [ -n "${ACCESS_URL}" ]; then
-    echo " AWX Access URL: ${ACCESS_URL}"
-fi
-echo "=========================================================="

k8s/README.md

@@ -1,10 +1,5 @@
-# k8s/README.md
+# k8s — SPLIT OUT
 
-> **Documentation moved to Discourse — the canonical source of truth.**
+K8S platform body of work moved to [KNEL/k8s](https://git.knownelement.com/KNEL/k8s)
->
+on 2026-09-05 (split ticket [#769](https://projects.knownelement.com/issues/769),
-> **k3s cluster setup scripts: wipe, bootstrap, taint, verify**
+master ticket [#699](https://projects.knownelement.com/issues/699)).
->
-> **Read it here:** https://community.turnsys.com/t/305
->
-> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
-> Do not update content here — edit the Discourse wiki topic instead.*

k8s/code/nvidia-device-plugin.yaml

@@ -1,34 +0,0 @@
-# nvidia GPU device-plugin DaemonSet — pfv-k8s-wnode-tsys3 (Quadro M2000M 4G)
-# GPU verified end-to-end 2026-09-04 (#763/#769): pod with nvidia.com/gpu:1
-# schedules + nvidia-smi works in-container (driver 550.163.01, CUDA 12.4).
-# Prereq on the node: nvidia driver + nvidia-container-runtime (k3s).
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: nvidia-device-plugin-daemonset
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: nvidia-device-plugin-ds
-  template:
-    metadata:
-      labels:
-        name: nvidia-device-plugin-ds
-    spec:
-      nodeSelector:
-        kubernetes.io/hostname: pfv-k8s-wnode-tsys3
-      tolerations:
-        - key: CriticalAddonsOnly
-          operator: Exists
-      containers:
-        - name: nvidia-device-plugin-ctr
-          image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0
-          args: ["--fail-on-init-error=false"]
-          volumeMounts:
-            - name: device-plugin
-              mountPath: /var/lib/kubelet/device-plugins
-      volumes:
-        - name: device-plugin
-          hostPath:
-            path: /var/lib/kubelet/device-plugins

k8s/docs/ARCHITECTURE.md

@@ -1,10 +0,0 @@
-# k8s/docs/ARCHITECTURE.md
-
-> **Documentation moved to Discourse — the canonical source of truth.**
->
-> **k8s target architecture: control plane, network, identity, storage, DR**
->
-> **Read it here:** https://community.turnsys.com/t/305
->
-> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
-> Do not update content here — edit the Discourse wiki topic instead.*

k8s/docs/DISTRO-DECISION.md

@@ -1,10 +0,0 @@
-# k8s/docs/DISTRO-DECISION.md
-
-> **Documentation moved to Discourse — the canonical source of truth.**
->
-> **Talos vs k3s distro analysis and decision**
->
-> **Read it here:** https://community.turnsys.com/t/305
->
-> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
-> Do not update content here — edit the Discourse wiki topic instead.*

k8s/docs/README.md

@@ -1,10 +0,0 @@
-# k8s/docs/README.md
-
-> **Documentation moved to Discourse — the canonical source of truth.**
->
-> **k8s docs index + TL;DR**
->
-> **Read it here:** https://community.turnsys.com/t/305
->
-> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
-> Do not update content here — edit the Discourse wiki topic instead.*

k8s/env.sh

@@ -1,73 +0,0 @@
-#!/usr/bin/bash
-# shellcheck disable=SC2034 # sourced config file; variables are consumed by scripts that source this
-# k8s/env.sh — shared config for all k8s scripts. Source this.
-#
-# All cluster communication goes over Tailscale IPs. No LAN IPs, ever.
-
-# --- Control plane nodes (Tailscale 100.x addresses) ---
-CNODE1_NAME="pfv-k8s-cnode1"
-CNODE1_IP="100.125.134.53"
-
-CNODE2_NAME="pfv-k8s-cnode2"
-CNODE2_IP="100.109.34.72"
-
-CNODE3_NAME="pfv-k8s-cnode3"
-CNODE3_IP="100.106.222.18"
-
-ALL_CNODES=("$CNODE1_IP" "$CNODE2_IP" "$CNODE3_IP")
-ALL_CNODE_NAMES=("$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME")
-
-# Bootstrap node (first etcd member)
-BOOTSTRAP_IP="$CNODE1_IP"
-BOOTSTRAP_NAME="$CNODE1_NAME"
-
-# --- Worker nodes (Tailscale 100.x addresses) ---
-# Roster (2026-09-02, #368): the wnode-tsys5 slot was REPLACED by
-# ultix-streaming (founder ruling) — same box, joined directly.
-WNODE1_NAME="pfv-k8s-wnode-tsys3"
-WNODE1_IP="100.126.9.112"
-
-WNODE2_NAME="ultix-streaming"
-WNODE2_IP="100.101.187.119"
-
-WNODE3_NAME="pfv-k8s-wnode-tsys6"
-WNODE3_IP="100.83.49.75"
-
-WNODE4_NAME="pfv-k8s-wnode-tsys7"
-WNODE4_IP="100.119.240.11"
-
-WNODE5_NAME="pfv-k8s-wnode-tsys9"
-WNODE5_IP="100.95.201.66"
-
-# ultix-offstage: unreachable 2026-09-02 — join when it is back on the tailnet
-# WNODE6_NAME="ultix-offstage"
-# WNODE6_IP="100.70.119.59"
-
-ALL_WNODES=("$WNODE1_IP" "$WNODE2_IP" "$WNODE3_IP" "$WNODE4_IP" "$WNODE5_IP")
-ALL_WNODE_NAMES=("$WNODE1_NAME" "$WNODE2_NAME" "$WNODE3_NAME" "$WNODE4_NAME" "$WNODE5_NAME")
-
-# --- SSH ---
-SSH_USER="localuser"
-SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
-
-# --- k3s ---
-K3S_VERSION="v1.36.2+k3s1"
-K3S_API_PORT="6443"
-
-# Shared TLS SANs (all cnodes + hostnames so certs are valid cluster-wide)
-TLS_SANS=(
-  "$CNODE1_IP" "$CNODE2_IP" "$CNODE3_IP"
-  "$CNODE1_NAME" "$CNODE2_NAME" "$CNODE3_NAME"
-)
-
-# Helper: run a command on a node as root (via passwordless sudo)
-cn() {
-  local ip="$1"; shift
-  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -c '$*'" 2>&1
-}
-
-# Helper: run a heredoc script on a node
-cn_file() {
-  local ip="$1"
-  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" "sudo -n bash -s"
-}

k8s/install-cp.sh

@@ -1,159 +0,0 @@
-#!/usr/bin/bash
-#
-# k8s/install-cp.sh — bootstrap 3-node k3s HA control plane (embedded etcd)
-#
-# All traffic goes over Tailscale IPs. LAN addresses are never used for
-# cluster communication.
-#
-# Sequence:
-#   1. Install cnode1 with --cluster-init (creates new etcd cluster)
-#   2. Wait for cnode1 API to be ready
-#   3. Fetch join token from cnode1
-#   4. Install cnode2 + cnode3 as additional servers (join etcd quorum)
-#   5. Wait for all 3 etcd members to be healthy
-#
-set -uo pipefail
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-# shellcheck source=./env.sh
-source "$SCRIPT_DIR/env.sh"
-
-# Build the shared TLS-SAN flags
-tls_san_flags=""
-for san in "${TLS_SANS[@]}"; do
-  tls_san_flags+=" --tls-san=$san"
-done
-
-echo "============================================"
-echo "  Bootstrapping k3s HA control plane"
-echo "  k3s version: $K3S_VERSION"
-echo "  Transport:   Tailscale (wireguard)"
-echo "============================================"
-
-# -------------------------------------------------------
-# Phase 1: Install bootstrap node (cnode1) with --cluster-init
-# -------------------------------------------------------
-echo ""
-echo "--- [1/5] Installing bootstrap node: $CNODE1_NAME ($CNODE1_IP) ---"
-
-# shellcheck disable=SC2087 # heredoc intentionally expands local config (node IPs, k3s version) before sending to remote
-ssh "${SSH_OPTS[@]}" "${SSH_USER}@${CNODE1_IP}" "sudo -n bash -s" <<REMOTE_BOOT
-set -euo pipefail
-export INSTALL_K3S_VERSION="$K3S_VERSION"
-export KILLALL_MODE=process
-curl -sfL https://get.k3s.io | sh -s - server \
-  --cluster-init \
-  --node-name=$CNODE1_NAME \
-  --node-ip=$CNODE1_IP \
-  --advertise-address=$CNODE1_IP \
-  $tls_san_flags \
-  --flannel-backend=vxlan \
-  --etcd-snapshot-schedule-cron='0 */6 * * *' \
-  --egress-selector-mode=agent \
-  --etcd-arg heartbeat-interval=1000 \
-  --etcd-arg election-timeout=5000
-REMOTE_BOOT
-
-echo "  cnode1 install submitted."
-
-# -------------------------------------------------------
-# Phase 2: Wait for cnode1 API + etcd to be ready
-# -------------------------------------------------------
-echo ""
-echo "--- [2/5] Waiting for cnode1 API + etcd ---"
-API_READY=false
-for i in $(seq 1 30); do
-  if cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>/dev/null | grep -q "ok"; then
-    API_READY=true
-    break
-  fi
-  echo "  ...waiting ($i/30)"
-  sleep 5
-done
-
-if [ "$API_READY" = false ]; then
-  echo "ERROR: cnode1 API did not become ready in 150s."
-  echo "Checking service status:"
-  cn "$CNODE1_IP" 'systemctl status k3s --no-pager | tail -20'
-  exit 1
-fi
-echo "  cnode1 API is ready."
-
-# -------------------------------------------------------
-# Phase 3: Fetch join token from cnode1
-# -------------------------------------------------------
-echo ""
-echo "--- [3/5] Fetching join token from cnode1 ---"
-JOIN_TOKEN=$(cn "$CNODE1_IP" 'cat /var/lib/rancher/k3s/server/token')
-if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
-  echo "ERROR: could not fetch token from cnode1."
-  echo "  Got: ${JOIN_TOKEN:0:40}"
-  exit 1
-fi
-echo "  Token fetched (masked: ${JOIN_TOKEN:0:12}***)"
-
-SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"
-
-# -------------------------------------------------------
-# Phase 4: Install cnode2 and cnode3 as additional servers
-# -------------------------------------------------------
-for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do
-  # Derive node name from IP
-  case "$node_ip" in
-    "$CNODE2_IP") node_name="$CNODE2_NAME" ;;
-    "$CNODE3_IP") node_name="$CNODE3_NAME" ;;
-  esac
-
-  echo ""
-  echo "--- [4/5] Joining server: $node_name ($node_ip) ---"
-
-# shellcheck disable=SC2087 # heredoc intentionally expands local config before sending to remote
-  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_JOIN
-set -euo pipefail
-export INSTALL_K3S_VERSION="$K3S_VERSION"
-export K3S_TOKEN="$JOIN_TOKEN"
-export KILLALL_MODE=process
-curl -sfL https://get.k3s.io | sh -s - server \
-  --server=$SERVER_URL \
-  --node-name=$node_name \
-  --node-ip=$node_ip \
-  --advertise-address=$node_ip \
-  $tls_san_flags \
-  --flannel-backend=vxlan \
-  --egress-selector-mode=agent \
-  --etcd-arg heartbeat-interval=1000 \
-  --etcd-arg election-timeout=5000
-REMOTE_JOIN
-
-  echo "  $node_name install submitted."
-done
-
-# -------------------------------------------------------
-# Phase 5: Wait for all 3 etcd members + nodes Ready
-# -------------------------------------------------------
-echo ""
-echo "--- [5/5] Waiting for all 3 nodes to join ---"
-for i in $(seq 1 30); do
-  READY_NODES=$(cn "$CNODE1_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
-  if [ "$READY_NODES" = "3" ]; then
-    echo "  All 3 nodes Ready."
-    break
-  fi
-  echo "  ...waiting ($i/30, $READY_NODES/3 ready)"
-  sleep 10
-done
-
-if [ "$READY_NODES" != "3" ]; then
-  echo "WARN: Only $READY_NODES/3 nodes ready after 300s. Check manually."
-fi
-
-echo ""
-echo "============================================"
-echo "  Control plane nodes:"
-echo "============================================"
-cn "$CNODE1_IP" 'k3s kubectl get nodes -o wide'
-
-echo ""
-echo "============================================"
-echo "  Bootstrap complete."
-echo "  Run: bash $SCRIPT_DIR/post-setup.sh"
-echo "============================================"

k8s/join-servers.sh

@@ -1,122 +0,0 @@
-#!/usr/bin/bash
-#
-# k8s/join-servers.sh — re-join cnode2 + cnode3 to the running cnode1 cluster
-#
-# Fixes the token-path bug from the initial install-cp.sh run. cnode1 is
-# already healthy; this script only touches cnode2 and cnode3:
-#   1. Uninstalls the broken k3s on each
-#   2. Fetches the CORRECT token from cnode1
-#   3. Re-installs both as HA server nodes joining the etcd cluster
-#
-set -uo pipefail
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-source "$SCRIPT_DIR/env.sh"
-
-# Build the shared TLS-SAN flags
-tls_san_flags=""
-for san in "${TLS_SANS[@]}"; do
-  tls_san_flags+=" --tls-san=$san"
-done
-
-SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"
-
-echo "============================================"
-echo "  Re-joining cnode2 + cnode3 to cluster"
-echo "  Bootstrap server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)"
-echo "============================================"
-
-# -------------------------------------------------------
-# 1. Fetch the CORRECT token from cnode1
-# -------------------------------------------------------
-echo ""
-echo "--- [1/4] Fetching join token from cnode1 ---"
-JOIN_TOKEN=$(cn "$CNODE1_IP" 'cat /var/lib/rancher/k3s/server/token')
-if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
-  echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}"
-  exit 1
-fi
-echo "  Token OK (masked: ${JOIN_TOKEN:0:12}***)"
-
-# -------------------------------------------------------
-# 2. Uninstall broken k3s from cnode2 + cnode3
-# -------------------------------------------------------
-for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do
-  echo ""
-  echo "--- [2/4] Wiping broken k3s on $node_ip ---"
-  cn "$node_ip" '
-    systemctl stop k3s 2>/dev/null || true
-    if [ -x /usr/local/bin/k3s-uninstall.sh ]; then
-      /usr/local/bin/k3s-uninstall.sh
-    else
-      echo "no k3s to remove"
-    fi
-    rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni
-    rm -f /etc/systemd/system/k3s.service /etc/systemd/system/k3s.service.env
-    systemctl daemon-reload
-    ip link delete cni0 2>/dev/null || true
-    ip link delete flannel.1 2>/dev/null || true
-    echo "WIPE DONE"
-  ' | tail -3
-done
-
-# -------------------------------------------------------
-# 3. Install cnode2 + cnode3 with correct token
-# -------------------------------------------------------
-for node_ip in "$CNODE2_IP" "$CNODE3_IP"; do
-  case "$node_ip" in
-    "$CNODE2_IP") node_name="$CNODE2_NAME" ;;
-    "$CNODE3_IP") node_name="$CNODE3_NAME" ;;
-  esac
-
-  echo ""
-  echo "--- [3/4] Joining server: $node_name ($node_ip) ---"
-
-# shellcheck disable=SC2087 # heredoc intentionally expands local config before sending to remote
-  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_JOIN
-set -euo pipefail
-export INSTALL_K3S_VERSION="$K3S_VERSION"
-export K3S_TOKEN="$JOIN_TOKEN"
-curl -sfL https://get.k3s.io | sh -s - server \
-  --server=$SERVER_URL \
-  --node-name=$node_name \
-  --node-ip=$node_ip \
-  --advertise-address=$node_ip \
-  $tls_san_flags \
-  --flannel-backend=vxlan \
-  --egress-selector-mode=agent
-REMOTE_JOIN
-
-  echo "  $node_name install submitted."
-done
-
-# -------------------------------------------------------
-# 4. Wait for all 3 nodes Ready
-# -------------------------------------------------------
-echo ""
-echo "--- [4/4] Waiting for all 3 nodes Ready ---"
-for i in $(seq 1 30); do
-  READY_NODES=$(cn "$CNODE1_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
-  if [ "$READY_NODES" = "3" ]; then
-    echo "  All 3 nodes Ready."
-    break
-  fi
-  echo "  ...waiting ($i/30, $READY_NODES/3 ready)"
-  sleep 10
-done
-
-echo ""
-cn "$CNODE1_IP" 'k3s kubectl get nodes -o wide'
-echo ""
-cn "$CNODE1_IP" 'k3s etcdctl member list 2>/dev/null'
-
-if [ "$READY_NODES" = "3" ]; then
-  echo ""
-  echo "============================================"
-  echo "  All 3 nodes joined. HA control plane active."
-  echo "  Run: bash $SCRIPT_DIR/post-setup.sh"
-  echo "============================================"
-else
-  echo ""
-  echo "WARN: $READY_NODES/3 ready. Check journalctl -u k3s on the failing node."
-  exit 1
-fi

k8s/join-workers.sh

@@ -1,103 +0,0 @@
-#!/usr/bin/bash
-#
-# k8s/join-workers.sh — join worker nodes to the k3s cluster
-#
-# Joins all worker nodes defined in env.sh as k3s agents. Worker nodes
-# run user workloads; control plane nodes are tainted NoSchedule.
-#
-set -uo pipefail
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-# shellcheck source=./env.sh
-source "$SCRIPT_DIR/env.sh"
-
-echo "============================================"
-echo "  Joining worker nodes to cluster"
-echo "  Server: $BOOTSTRAP_NAME ($BOOTSTRAP_IP)"
-echo "  Workers: ${#ALL_WNODE_NAMES[@]}"
-echo "============================================"
-
-# -------------------------------------------------------
-# 1. Fetch join token from bootstrap node
-# -------------------------------------------------------
-echo ""
-echo "--- [1/3] Fetching join token from $BOOTSTRAP_NAME ---"
-JOIN_TOKEN=$(timeout 30 cn "$BOOTSTRAP_IP" 'cat /var/lib/rancher/k3s/server/token')
-if [ -z "$JOIN_TOKEN" ] || [[ "$JOIN_TOKEN" == cat:* ]]; then
-  echo "FATAL: could not fetch token. Got: ${JOIN_TOKEN:0:40}"
-  exit 1
-fi
-echo "  Token OK (masked: ${JOIN_TOKEN:0:12}***)"
-
-SERVER_URL="https://${BOOTSTRAP_IP}:${K3S_API_PORT}"
-
-# -------------------------------------------------------
-# 2. Install k3s-agent on each worker
-# -------------------------------------------------------
-for i in "${!ALL_WNODES[@]}"; do
-  node_ip="${ALL_WNODES[$i]}"
-  node_name="${ALL_WNODE_NAMES[$i]}"
-
-  echo ""
-  echo "--- [2/3] Joining worker: $node_name ($node_ip) ---"
-
-  # Wipe any existing k3s first (time-boxed: an unreachable node must not hang the run)
-  timeout 60 cn "$node_ip" '
-    systemctl stop k3s-agent 2>/dev/null || true
-    if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then
-      /usr/local/bin/k3s-agent-uninstall.sh
-    fi
-    rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni
-    rm -f /etc/systemd/system/k3s-agent.service
-    systemctl daemon-reload
-    ip link delete cni0 2>/dev/null || true
-    ip link delete flannel.1 2>/dev/null || true
-  ' 2>/dev/null || true
-
-  # Install as agent (300s deploy-class limit; a timed-out install is a failed node)
-  # shellcheck disable=SC2087 # heredoc intentionally expands local config
-  timeout 300 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node_ip}" "sudo -n bash -s" <<REMOTE_AGENT || { echo "  FATAL: $node_name install failed or timed out"; continue; }
-set -euo pipefail
-export INSTALL_K3S_VERSION="$K3S_VERSION"
-export K3S_URL="$SERVER_URL"
-export K3S_TOKEN="$JOIN_TOKEN"
-export K3S_NODE_NAME="$node_name"
-curl -sfL https://get.k3s.io | sh -s - agent \
-  --node-name=$node_name \
-  --node-ip=$node_ip
-REMOTE_AGENT
-
-  echo "  $node_name agent install submitted."
-done
-
-# -------------------------------------------------------
-# 3. Wait for all workers to appear Ready
-# -------------------------------------------------------
-TOTAL_NODES=$(( ${#ALL_CNODES[@]} + ${#ALL_WNODES[@]} ))
-echo ""
-echo "--- [3/3] Waiting for all $TOTAL_NODES nodes (${#ALL_CNODES[@]} cp + ${#ALL_WNODES[@]} workers) ---"
-for i in $(seq 1 60); do
-  READY_NODES=$(cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready"' 2>/dev/null || echo 0)
-  if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then
-    echo "  All $TOTAL_NODES nodes Ready."
-    break
-  fi
-  echo "  ...waiting ($i/60, $READY_NODES/$TOTAL_NODES ready)"
-  sleep 10
-done
-
-echo ""
-echo "============================================"
-echo "  Node status:"
-echo "============================================"
-cn "$BOOTSTRAP_IP" 'k3s kubectl get nodes -o wide'
-
-if [ "$READY_NODES" -ge "$TOTAL_NODES" ]; then
-  echo ""
-  echo "============================================"
-  echo "  All workers joined. Cluster fully operational."
-  echo "============================================"
-else
-  echo ""
-  echo "WARN: $READY_NODES/$TOTAL_NODES ready. Check failing nodes."
-  exit 1
-fi

k8s/post-setup.sh

@@ -1,115 +0,0 @@
-#!/usr/bin/bash
-#
-# k8s/post-setup.sh — taint cnodes, fetch kubeconfig, verify cluster
-#
-# Taints all 3 control-plane nodes with NoSchedule so NO user workloads
-# can land on them. Only system components (CoreDNS, metrics-server,
-# kube-proxy, flannel) with built-in tolerations will run here.
-#
-set -uo pipefail
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-# shellcheck source=./env.sh
-source "$SCRIPT_DIR/env.sh"
-
-echo "============================================"
-echo "  Post-setup: taints, kubeconfig, verify"
-echo "============================================"
-
-# -------------------------------------------------------
-# 1. Taint all cnodes NoSchedule (no user workloads on control plane)
-# -------------------------------------------------------
-echo ""
-echo "--- [1/3] Tainting control-plane nodes (NoSchedule) ---"
-for name in "${ALL_CNODE_NAMES[@]}"; do
-  # Set the control-plane role label explicitly.
-  # k3s kubectl only — bare kubectl is not on the cnodes' root PATH.
-  cn "$CNODE1_IP" "k3s kubectl label node $name node-role.kubernetes.io/control-plane= --overwrite" 2>/dev/null || true
-  # Apply the taint (idempotent with --overwrite)
-  cn "$CNODE1_IP" "k3s kubectl taint node $name node-role.kubernetes.io/control-plane=true:NoSchedule --overwrite" 2>/dev/null
-  echo "  $name tainted."
-done
-
-# -------------------------------------------------------
-# 2. Fetch kubeconfig, rewrite server URL to Tailscale IP
-# -------------------------------------------------------
-echo ""
-echo "--- [2/3] Fetching kubeconfig ---"
-
-RAW_KUBECONFIG=$(cn "$CNODE1_IP" 'cat /etc/rancher/k3s/k3s.yaml')
-
-# Rewrite 127.0.0.1 → cnode1 Tailscale IP, set context name
-KUBECONFIG_FINAL=$(printf '%s\n' "$RAW_KUBECONFIG" \
-  | sed "s/127.0.0.1/$CNODE1_IP/g" \
-  | sed 's/default/pfv-k8s/g')
-
-KUBECONFIG_DIR="$HOME/.kube"
-KUBECONFIG_FILE="$KUBECONFIG_DIR/config.pfv-k8s"
-mkdir -p "$KUBECONFIG_DIR"
-printf '%s\n' "$KUBECONFIG_FINAL" > "$KUBECONFIG_FILE"
-chmod 600 "$KUBECONFIG_FILE"
-
-echo "  Saved to: $KUBECONFIG_FILE"
-echo "  Server:   https://${CNODE1_IP}:${K3S_API_PORT}"
-
-# Also save a copy in the repo for reference (NOT secrets — this is just
-# the cluster connection config; actual client certs are embedded but
-# considered acceptable for a private R&D tailnet. If ITAR workloads are
-# added later, move to OIDC and remove this file.)
-K8S_DIR="$SCRIPT_DIR"
-printf '%s\n' "$KUBECONFIG_FINAL" > "$K8S_DIR/kubeconfig.yaml"
-chmod 600 "$K8S_DIR/kubeconfig.yaml"
-echo "  Copy saved: $K8S_DIR/kubeconfig.yaml (gitignored)"
-
-# Tell the user how to use it
-echo ""
-echo "  To use this cluster:"
-echo "    export KUBECONFIG=$KUBECONFIG_FILE"
-echo "    kubectl get nodes"
-
-# -------------------------------------------------------
-# 3. Deploy tuned (network-latency profile) on all cnodes
-# -------------------------------------------------------
-echo ""
-echo "--- [3/4] Deploying tuned (network-latency) on cnodes ---"
-for ip in "${ALL_CNODES[@]}"; do
-  echo "  $ip..."
-  cn "$ip" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>/dev/null; DEBIAN_FRONTEND=noninteractive apt-get install -y -qq tuned 2>/dev/null; tuned-adm profile network-latency; systemctl enable tuned; systemctl restart tuned; tuned-adm active' 2>&1 | tail -1
-done
-
-# -------------------------------------------------------
-# 4. Verify cluster health
-# -------------------------------------------------------
-echo ""
-echo "--- [4/4] Verifying cluster health ---"
-
-export KUBECONFIG="$KUBECONFIG_FILE"
-
-echo ""
-echo "=== Nodes ==="
-kubectl get nodes -o wide 2>&1
-
-echo ""
-echo "=== Node taints ==="
-kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints 2>&1
-
-echo ""
-echo "=== Node IPs (should be 100.x Tailscale) ==="
-kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>&1
-
-echo ""
-echo "=== API + etcd liveness ==="
-cn "$CNODE1_IP" 'k3s kubectl get --raw=/readyz' 2>&1 || echo "(API not ready)"
-# k3s ships no etcdctl; snapshots answering proves the etcd store is readable
-cn "$CNODE1_IP" 'k3s etcd-snapshot list 2>/dev/null | head -3' 2>&1
-
-echo ""
-echo "=== System pods ==="
-kubectl get pods -A 2>&1
-
-echo ""
-echo "============================================"
-echo "  Cluster is ready."
-echo ""
-echo "  KUBECONFIG: $KUBECONFIG_FILE"
-echo "  Next: bash $SCRIPT_DIR/verify.sh"
-echo "============================================"

k8s/probe-nodes.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/bash
-# k8s/probe-nodes.sh — verify SSH + Tailscale reachability on all cnodes
-set -uo pipefail
-source "$(cd "$(dirname "$0")" && pwd)/env.sh"
-
-for ip in "${ALL_CNODES[@]}"; do
-  echo "=== $ip ==="
-  ssh "${SSH_OPTS[@]}" "${SSH_USER}@${ip}" \
-    'echo "host=$(hostname)"; \
-     echo "tailscale_ip=$(tailscale ip -4)"; \
-     echo "k3s=$(systemctl is-active k3s 2>/dev/null)"; \
-     echo "kernel=$(uname -r)"; \
-     free -h | head -2; \
-     df -h / | tail -1' 2>&1 || echo "FAILED to reach $ip"
-  echo
-done

k8s/proxmox-tuning.sh

@@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-#
-# k8s/proxmox-tuning.sh — apply ultix-style Proxmox tuning to k8s VMs
-#
-# Codifies the config applied 2026-09-01 during the #367 rebuild:
-#   - disk: ssd=1,discard=on (guest sees non-rotational; qcow2 trims)
-#   - net:  queues=2 on both NICs (multiqueue; guest activates via ethtool)
-#   - cpu:  cpuunits so etcd/control plane wins host CPU contention
-#   - boot: onboot=1 + startup order (cnodes before wnodes)
-#   - balloon was already 0 on every k8s VM (etcd memory predictability)
-#
-# Pending items (ssd/discard, queues) activate at the VM's next restart.
-# Reboot waves are SERIAL with a health gate between hops — never batch.
-#
-# DRY RUN by default. Apply with RUN=1. Run on each Proxmox host, or via
-#   PROX_HOST=<host> bash tests/remote.sh prox 'bash -s' < k8s/proxmox-tuning.sh
-#
-set -uo pipefail
-RUN="${RUN:-0}"
-
-# vmid|cpuunits|startup|scsi0-line|net0-line|net1-line
-# scsi0/net lines are the FULL desired config (Proxmox replaces wholesale).
-TUNINGS=(
-  "102|4000|order=20,up=180|TS5-SSD:102/vm-102-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:CB:97:10,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:47:13:08,bridge=datanet,queues=2"
-  "705|4000|order=20,up=180|TS5-SSD:705/vm-705-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:40:25:F8,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:AF:3D:89,bridge=datanet,queues=2"
-  "603|4000|order=20,up=180|TS5-SSD:603/vm-603-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:38:C0:58,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:1F:F1:E1,bridge=storagenet,queues=2"
-  "313|2000|order=40,up=120|local-lvm:vm-313-disk-0,iothread=1,ssd=1,discard=on,size=300G|virtio=BC:24:11:EE:7E:7B,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:DF:C2:E9,bridge=datanet,queues=2"
-  "601|2000|order=40,up=120|local-lvm:vm-601-disk-0,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:FA:6E:B5,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:39:7D:0B,bridge=storagenet,queues=2"
-  "701|2000|order=40,up=120|S2:701/vm-701-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:30:B8:07,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:C8:0E:8A,bridge=datanet,queues=2"
-  "905|2000|order=40,up=120|S2:905/vm-905-disk-0.qcow2,cache=writeback,iothread=1,ssd=1,discard=on,size=32G|virtio=BC:24:11:EE:2B:B6,bridge=vmbr0,firewall=1,queues=2|virtio=BC:24:11:9B:57:06,bridge=datanet,queues=2"
-)
-
-apply() {
-  local vmid="$1" cpuunits="$2" startup="$3" scsi0="$4" net0="$5" net1="$6"
-  if [ "$RUN" = "1" ]; then
-    qm set "$vmid" --cpuunits "$cpuunits" --startup "$startup" \
-      --scsi0 "$scsi0" --net0 "$net0" --net1 "$net1"
-  else
-    echo "DRY: qm set $vmid --cpuunits $cpuunits --startup $startup --scsi0 $scsi0 --net0 $net0 --net1 $net1"
-  fi
-}
-
-for t in "${TUNINGS[@]}"; do
-  IFS='|' read -r vmid cpuunits startup scsi0 net0 net1 <<< "$t"
-  # Skip VMs that live on other hosts (qm set errors on unknown VMID)
-  if ! qm status "$vmid" >/dev/null 2>&1; then
-    echo "skip: VM $vmid not on this host"
-    continue
-  fi
-  echo "== VM $vmid =="
-  apply "$vmid" "$cpuunits" "$startup" "$scsi0" "$net0" "$net1"
-done
-
-echo
-if [ "$RUN" = "1" ]; then
-  for t in "${TUNINGS[@]}"; do
-    vmid="${t%%|*}"
-    qm status "$vmid" >/dev/null 2>&1 || continue
-    echo "--- pending VM $vmid ---"
-    qm pending "$vmid" | grep -E "^(new|cur) (scsi0|net0|net1|cpuunits|startup)" || true
-  done
-fi

k8s/verify.sh

@@ -1,109 +0,0 @@
-#!/usr/bin/bash
-#
-# k8s/verify.sh — health check for the pfv-k8s control plane
-#
-set -uo pipefail
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-# shellcheck source=./env.sh
-source "$SCRIPT_DIR/env.sh"
-
-export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config.pfv-k8s}"
-
-PASS=0
-FAIL=0
-ok()   { echo "  [PASS] $1"; PASS=$((PASS+1)); }
-fail() { echo "  [FAIL] $1"; FAIL=$((FAIL+1)); }
-
-echo "============================================"
-echo "  pfv-k8s Control Plane Health Check"
-echo "============================================"
-
-# 1. All 3 nodes Ready
-echo ""
-echo "--- Nodes Ready ---"
-READY=$(kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo 0)
-if [ "$READY" = "3" ]; then ok "All 3 nodes Ready"; else fail "Expected 3 Ready nodes, got $READY"; fi
-
-kubectl get nodes -o wide 2>&1 | sed 's/^/  /'
-
-# 2. Nodes use Tailscale IPs
-echo ""
-echo "--- Tailscale IPs ---"
-for name in "${ALL_CNODE_NAMES[@]}"; do
-  IP=$(kubectl get node "$name" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null)
-  case "$IP" in
-    100.*) ok "$name uses Tailscale IP ($IP)" ;;
-    *)     fail "$name uses non-Tailscale IP ($IP)" ;;
-  esac
-done
-
-# 3. Taints applied (NoSchedule on all cnodes)
-echo ""
-echo "--- Control-plane taints ---"
-for name in "${ALL_CNODE_NAMES[@]}"; do
-  TAINT=$(kubectl get node "$name" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null)
-  if echo "$TAINT" | grep -q "control-plane"; then
-    ok "$name has control-plane taint"
-  else
-    fail "$name missing control-plane taint"
-  fi
-done
-
-# 4. etcd members = 3 (k3s v1.36 embeds etcdctl; verify via node roles + API)
-echo ""
-echo "--- etcd quorum ---"
-# All 3 nodes must have the etcd role label
-ETCD_NODES=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.node-role\.kubernetes\.io/etcd}{"\n"}{end}' 2>/dev/null | grep -c "true" || echo 0)
-if [ "$ETCD_NODES" = "3" ]; then ok "3 nodes have etcd role (embedded HA etcd)"; else fail "Only $ETCD_NODES/3 nodes have etcd role"; fi
-
-# Verify etcd is the backing store via the API (if etcd is down, this fails)
-LEASE_COUNT=$(kubectl get leases -A --no-headers 2>/dev/null | wc -l)
-if [ "$LEASE_COUNT" -gt "0" ]; then
-  ok "etcd backing store active ($LEASE_COUNT leases found)"
-else
-  fail "No leases found — etcd may not be accepting writes"
-fi
-
-# Check etcd leader via metrics on cnode1
-LEADER=$(cn "$CNODE1_IP" 'ETCDCTL_API=3 /var/lib/rancher/k3s/data/current/bin/etcdctl \
-  --endpoints=https://127.0.0.1:2379 \
-  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
-  --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
-  --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
-  endpoint status 2>/dev/null' 2>/dev/null)
-if [ -n "$LEADER" ]; then
-  ok "etcd endpoint reachable ($LEADER)"
-else
-  # etcdctl not on disk in k3s v1.36; rely on node roles + leases above
-  ok "etcd health confirmed via 3 node roles + active leases (etcdctl not standalone in k3s v1.36)"
-fi
-
-# 5. CoreDNS running
-echo ""
-echo "--- System components ---"
-COREDNS=$(kubectl get pods -n kube-system -l k8s-app=kube-dns --no-headers 2>/dev/null | grep -c "Running" || echo 0)
-if [ "$COREDNS" -ge "1" ]; then ok "CoreDNS running"; else fail "CoreDNS not running"; fi
-
-# 6. API server reachable over Tailscale
-echo ""
-echo "--- API server (Tailscale) ---"
-if kubectl get --raw=/readyz 2>/dev/null | grep -q "ok"; then
-  ok "API server healthy over Tailscale"
-else
-  fail "API server not reachable"
-fi
-
-# 7. No user workloads on cnodes
-echo ""
-echo "--- Workload isolation ---"
-USER_PODS=$(kubectl get pods -A --field-selector spec.nodeName="${CNODE1_NAME}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | wc -w)
-# Subtract system pods
-SYSTEM_PODS=$(kubectl get pods -A --field-selector spec.nodeName="${CNODE1_NAME}" -l k8s-app --no-headers 2>/dev/null | wc -l)
-if [ "$((USER_PODS - SYSTEM_PODS))" -le 0 ]; then ok "Only system pods on cnodes (expected)"; else fail "Unexpected pods on $CNODE1_NAME"; fi
-
-echo ""
-echo "============================================"
-echo "  Results: $PASS passed, $FAIL failed"
-if [ "$FAIL" -gt 0 ]; then exit 1; fi
-echo "  All checks passed."
-echo "============================================"

k8s/wipe.sh

@@ -1,52 +0,0 @@
-#!/usr/bin/bash
-#
-# k8s/wipe.sh — remove existing k3s from all cnodes
-#
-# The current install on cnode1/2 uses LAN IP (192.168.3.x). We need to
-# rebuild with Tailscale IPs. This cluster has only system pods (16h old,
-# no user workloads), so a clean wipe is safe.
-#
-set -uo pipefail
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-# shellcheck source=./env.sh
-source "$SCRIPT_DIR/env.sh"
-
-echo "============================================"
-echo "  Wiping existing k3s from all cnodes"
-echo "============================================"
-
-for ip in "${ALL_CNODES[@]}"; do
-  echo ""
-  echo "--- $ip ---"
-  cn "$ip" '
-    # Stop services first
-    systemctl stop k3s 2>/dev/null || true
-    systemctl stop k3s-agent 2>/dev/null || true
-
-    # Server uninstall (also removes agents)
-    if [ -x /usr/local/bin/k3s-uninstall.sh ]; then
-      /usr/local/bin/k3s-uninstall.sh
-    elif [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then
-      /usr/local/bin/k3s-agent-uninstall.sh
-    else
-      echo "no k3s installed on this node"
-    fi
-
-    # Clean up residual state
-    rm -rf /etc/rancher/k3s /var/lib/rancher/k3s /var/lib/kubelet /var/lib/cni
-    rm -f /etc/systemd/system/k3s.service /etc/systemd/system/k3s-agent.service
-    systemctl daemon-reload
-
-    # Clean network interfaces left by k3s/flannel
-    ip link delete cni0 2>/dev/null || true
-    ip link delete flannel.1 2>/dev/null || true
-    ip link delete kube-ipvs0 2>/dev/null || true
-
-    echo "WIPE DONE"
-  '
-done
-
-echo ""
-echo "============================================"
-echo "  Wipe complete. Ready for bootstrap."
-echo "============================================"