feat: full re-audit of all 7 hosts with fresh ground truth
Deployed check.sh to all 7 hosts at 21:50 CDT. Captures the live state after the user's PDM migrations: Cnode movements since last audit: - cnode1: tsys1 -> tsys9 - cnode2: tsys6 -> tsys7 - cnode5: tsys6 -> tsys7, storage D5(tsys4) -> S2(tsys5) Wnode changes: - wnode-tsys1 (102): new VM on S2, stopped - wnode-tsys3: RAM bumped 20 -> 28 GB - wnode-tsys6: now running (was stopped) - wnode-tsys9: storage moved S3 -> S2 Storage distribution improved from 90/10 to 73/27 (tsys4/tsys5). Still need 2 more cnode moves for etcd quorum survival. Updated executive summary, k8s distribution tables, storage utilization, and open items with the fresh data. Captured future k8s requirements: vcluster + Rancher, OIDC to Keycloak, workload isolation (RackRental/Suborbital ITAR/non-ITAR/SLP), and solar-aware scale-out with PowerEdge 19xx/2950 systems. Added tsys9 to deploy-check.sh host list.
This commit is contained in:
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# baseline.sh — quick read-only baseline of a target node.
|
||||
set -u
|
||||
hdr() { printf '\n=== %s ===\n' "$1"; }
|
||||
hdr "WHO/SUDO"; id; echo "--- sudo -n ---"; sudo -n true 2>&1 && echo "sudo OK" || echo "sudo FAIL"
|
||||
hdr "OS"; cat /etc/os-release 2>/dev/null | grep PRETTY; uname -r
|
||||
hdr "NET"; hostname -I 2>/dev/null; ip -br addr 2>/dev/null; echo "-- tailscale --"; tailscale ip -4 2>/dev/null || echo "(tailscale CLI absent or no ip)"
|
||||
hdr "DISK/MEM"; df -h / 2>/dev/null | tail -2; free -h 2>/dev/null | head -2
|
||||
hdr "DOCKER"; docker --version 2>&1; sudo docker version --format '{{.Server.Version}}' 2>&1 | tail -1; id -nG | tr ' ' '\n' | grep -qx docker && echo "localuser IN docker group" || echo "localuser NOT in docker group"
|
||||
hdr "EXISTING RELEVANT SERVICES"; systemctl is-active docker 2>/dev/null; systemctl is-enabled docker 2>/dev/null
|
||||
hdr "PORTS 53/67/80/123/443/5380/53443"; ss -tlnup 2>/dev/null | grep -E ':53|:67|:80|:123|:443|:5380|:53443' || echo "(none of those ports listening)"
|
||||
hdr "EXISTING SERVICES DIRS"; ls -la /home/localuser/services 2>/dev/null || echo "(no ~/services)"; ls -la /root/pihole /root/NTP 2>/dev/null || sudo -n ls -la /root 2>/dev/null | head
|
||||
hdr "DNS RESOLV"; cat /etc/resolv.conf 2>/dev/null
|
||||
hdr "HOME"; ls -la /home/localuser 2>/dev/null | head
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# gather-configs.sh — READ-ONLY. Pull Technitium configs, pihole.toml, gravity.db
|
||||
# contents, and chrony NTP details from pfv-netboot for replication.
|
||||
set -u
|
||||
hdr() { printf '\n=== %s ===\n' "$1"; }
|
||||
|
||||
hdr "TECHNITIUM dns.config"
|
||||
sudo cat /var/lib/docker/volumes/dns_tsys-dns-config/_data/dns.config 2>&1
|
||||
|
||||
hdr "TECHNITIUM auth.config"
|
||||
sudo cat /var/lib/docker/volumes/dns_tsys-dns-config/_data/auth.config 2>&1
|
||||
|
||||
hdr "TECHNITIUM log.config"
|
||||
sudo cat /var/lib/docker/volumes/dns_tsys-dns-config/_data/log.config 2>&1
|
||||
|
||||
hdr "TECHNITIUM scopes/Default.scope"
|
||||
sudo cat /var/lib/docker/volumes/dns_tsys-dns-config/_data/scopes/Default.scope 2>&1
|
||||
|
||||
hdr "TECHNITIUM zone: knel.net.zone"
|
||||
sudo cat /var/lib/docker/volumes/dns_tsys-dns-config/_data/zones/knel.net.zone 2>&1
|
||||
|
||||
hdr "TECHNITIUM all zone files listing + sizes"
|
||||
sudo ls -la /var/lib/docker/volumes/dns_tsys-dns-config/_data/zones/ 2>&1
|
||||
|
||||
hdr "TECHNITIUM reverse zones (one sample)"
|
||||
sudo cat /var/lib/docker/volumes/dns_tsys-dns-config/_data/zones/199.86.100.in-addr.arpa.zone 2>&1
|
||||
|
||||
hdr "TECHNITIUM top-level listing"
|
||||
sudo ls -la /var/lib/docker/volumes/dns_tsys-dns-config/_data/ 2>&1
|
||||
|
||||
hdr "PI-HOLE pihole.toml (v6 config) from host bind mount"
|
||||
cat /root/pihole/etc-pihole/pihole.toml 2>&1
|
||||
|
||||
hdr "PI-HOLE dnsmasq.conf"
|
||||
cat /root/pihole/etc-pihole/dnsmasq.conf 2>&1
|
||||
|
||||
hdr "PI-HOLE versions file"
|
||||
cat /root/pihole/etc-pihole/versions 2>&1
|
||||
|
||||
hdr "GRAVITY.DB contents (host sqlite3 if present)"
|
||||
if command -v sqlite3 >/dev/null 2>&1; then
|
||||
G=/root/pihole/etc-pihole/gravity.db
|
||||
echo "-- adlist --"; sqlite3 -readonly "$G" "SELECT address,enabled,comment,DATE(date_added,'unixepoch'),DATE(date_updated,'unixepoch') FROM adlist;" 2>&1
|
||||
echo "-- adlist count --"; sqlite3 -readonly "$G" "SELECT COUNT(*) FROM adlist;" 2>&1
|
||||
echo "-- domainlist count by type --"; sqlite3 -readonly "$G" "SELECT type,COUNT(*) FROM domainlist GROUP BY type;" 2>&1
|
||||
echo "-- domainlist (allow=0/allow_exact, deny=1/deny_exact, etc.) first 80 --"; sqlite3 -readonly "$G" "SELECT type,domain,enabled,comment FROM domainlist LIMIT 80;" 2>&1
|
||||
echo -- client --"; sqlite3 -readonly "$G" "SELECT ip,comment FROM client;" 2>&1
|
||||
echo "-- group --"; sqlite3 -readonly "$G" "SELECT id,name,enabled,comment FROM 'group';" 2>&1
|
||||
echo "-- info --"; sqlite3 -readonly "$G" "SELECT * FROM info;" 2>&1
|
||||
else
|
||||
echo "(sqlite3 not on host PATH)"
|
||||
fi
|
||||
|
||||
hdr "DONE"
|
||||
Binary file not shown.
Executable
+283
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# setup-netinfra.sh
|
||||
# -----------------------------------------------------------------------------
|
||||
# Replicate pfv-netboot's network services (Pi-hole, Technitium DNS, NTP)
|
||||
# onto pfv-netinfra-01 and pfv-netinfra-02.
|
||||
#
|
||||
# DESIGN
|
||||
# * pfv-netboot is REFERENCE ONLY -- this script NEVER mutates it. All reads
|
||||
# from it are via `ssh localuser@pfv-netboot 'sudo ...'` (read-only cmds).
|
||||
# * The targets cannot SSH to pfv-netboot directly, so config tarballs are
|
||||
# relayed through this workstation:
|
||||
# ssh netboot 'sudo tar -cf - ...' | ssh target 'sudo tar -xf - ...'
|
||||
# * Services are deployed under /home/localuser/services/<svc>/ on each
|
||||
# target so localuser can manage them (mirrors netboot's localuser-owned
|
||||
# pihole data dir). `sudo docker` is used since localuser is not in the
|
||||
# docker group (same as on netboot).
|
||||
#
|
||||
# SERVICES
|
||||
# pihole pihole/pihole:latest :53 tcp/udp :10002->80 :10003->443
|
||||
# ntp (chrony) dockurr/chrony <tailscale-ip>:123:123/udp
|
||||
# technitium technitium/dns-server :5300->53 tcp/udp :5380 :53443
|
||||
# (Technitium DNS is remapped off :53 to avoid clashing with Pi-hole.
|
||||
# The knel.net authoritative zone + Tailscale reverse zones are preserved
|
||||
# verbatim from netboot's orphaned dns_tsys-dns-config volume.)
|
||||
#
|
||||
# USAGE
|
||||
# ./setup-netinfra.sh # deploy to BOTH nodes
|
||||
# ./setup-netinfra.sh pfv-netinfra-01 # deploy to one node
|
||||
# ./setup-netinfra.sh pfv-netinfra-01 verify # verify only
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NETBOOT="localuser@pfv-netboot"
|
||||
SVC_ROOT="/home/localuser/services"
|
||||
PIHOLE_PW='REDACTED_PASSWORD' # replicated verbatim from netboot compose
|
||||
|
||||
log() { printf '\n\033[1;36m[%s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; }
|
||||
warn() { printf '\n\033[1;33m[WARN %s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; }
|
||||
|
||||
# Per-node parameters. (LAN iface is auto-detected at deploy time as a fallback.)
|
||||
declare -A NODE_TSIP=(
|
||||
[pfv-netinfra-01]="100.70.181.72"
|
||||
[pfv-netinfra-02]="100.93.194.82"
|
||||
)
|
||||
|
||||
on_node() { ssh -o StrictHostKeyChecking=no "localuser@$1" "$2"; }
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Verify-only mode
|
||||
#------------------------------------------------------------------------------
|
||||
verify_node() {
|
||||
local node="$1" tsip="${NODE_TSIP[$1]}"
|
||||
log "VERIFY $node (tailscale $tsip)"
|
||||
on_node "$node" "bash -s" <<EOF
|
||||
set +e
|
||||
echo "### containers ###"
|
||||
sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' 2>&1
|
||||
echo
|
||||
echo "### Pi-hole DNS (dig @127.0.0.1:53 pi.hole) ###"
|
||||
dig +time=3 +tries=1 +short @127.0.0.1 -p 53 pi.hole 2>&1
|
||||
echo "### Pi-hole web (curl :10002) ###"
|
||||
curl -sk -o /dev/null -w 'http=%{http_code}\n' http://127.0.0.1:10002/admin/ 2>&1
|
||||
echo
|
||||
echo "### Technitium DNS (dig @127.0.0.1:5300 knel.net SOA) ###"
|
||||
dig +time=3 +tries=1 @127.0.0.1 -p 5300 knel.net SOA +short 2>&1
|
||||
echo "### Technitium web (curl :5380) ###"
|
||||
curl -sk -o /dev/null -w 'http=%{http_code}\n' http://127.0.0.1:5380/ 2>&1
|
||||
echo
|
||||
echo "### NTP on tailscale IP (ntpdig -p 1) ###"
|
||||
timeout 5 ntpdig -t1 -c1 -p1 $tsip 2>&1 || echo "(ntpdig not available or no response)"
|
||||
echo "### chrony container sources ###"
|
||||
sudo docker exec tsys-ntp chronyc -n sources 2>&1 | head -8
|
||||
EOF
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Deploy to one node
|
||||
#------------------------------------------------------------------------------
|
||||
deploy_node() {
|
||||
local node="$1" tsip="${NODE_TSIP[$1]}"
|
||||
log "==== DEPLOY $node (tailscale $tsip) ===="
|
||||
|
||||
# ---- 1. Prepare directories on the target -------------------------------
|
||||
log "$node: create service dirs"
|
||||
on_node "$node" "bash -s" <<EOF
|
||||
set -e
|
||||
sudo mkdir -p $SVC_ROOT/pihole $SVC_ROOT/ntp $SVC_ROOT/technitium
|
||||
sudo chown -R localuser:localuser $SVC_ROOT
|
||||
EOF
|
||||
|
||||
# ---- 2. Write compose files (as localuser) ------------------------------
|
||||
log "$node: write docker-compose files"
|
||||
on_node "$node" "cat > $SVC_ROOT/pihole/docker-compose.yml" <<'YAML'
|
||||
services:
|
||||
pihole:
|
||||
container_name: pihole
|
||||
image: pihole/pihole:latest
|
||||
hostname: pihole
|
||||
ports:
|
||||
- "53:53/tcp"
|
||||
- "53:53/udp"
|
||||
- "10002:80/tcp"
|
||||
- "10003:443/tcp"
|
||||
environment:
|
||||
TZ: 'America/Chicago'
|
||||
FTLCONF_webserver_api_password: 'REDACTED_PASSWORD'
|
||||
FTLCONF_dns_listeningMode: 'all'
|
||||
volumes:
|
||||
- './etc-pihole:/etc/pihole'
|
||||
cap_add:
|
||||
- SYS_NICE
|
||||
restart: always
|
||||
YAML
|
||||
|
||||
on_node "$node" "cat > $SVC_ROOT/ntp/docker-compose.yml" <<YAML
|
||||
services:
|
||||
ntp:
|
||||
image: dockurr/chrony
|
||||
container_name: tsys-ntp
|
||||
environment:
|
||||
NTP_SERVERS: "pool.ntp.org"
|
||||
ports:
|
||||
- "$tsip:123:123/udp"
|
||||
restart: always
|
||||
YAML
|
||||
|
||||
on_node "$node" "cat > $SVC_ROOT/technitium/docker-compose.yml" <<'YAML'
|
||||
services:
|
||||
technitium:
|
||||
image: technitium/dns-server
|
||||
container_name: tsys-dns
|
||||
ports:
|
||||
- "5300:53/tcp"
|
||||
- "5300:53/udp"
|
||||
- "5380:5380/tcp"
|
||||
- "53443:53443/tcp"
|
||||
volumes:
|
||||
- './config:/etc/dns'
|
||||
restart: always
|
||||
YAML
|
||||
|
||||
# ---- 3. Relay Pi-hole config from netboot -> target ---------------------
|
||||
log "$node: copy Pi-hole /etc/pihole from netboot (excluding query logs)"
|
||||
if on_node "$node" "test -f $SVC_ROOT/pihole/etc-pihole/gravity.db"; then
|
||||
log "$node: Pi-hole config already present; skipping copy"
|
||||
else
|
||||
on_node "$node" "sudo rm -rf $SVC_ROOT/pihole/etc-pihole"
|
||||
ssh -o StrictHostKeyChecking=no "$NETBOOT" \
|
||||
"sudo tar -cf - -C /root/pihole --exclude='etc-pihole/pihole-FTL.db*' \
|
||||
--exclude='etc-pihole/listsCache' \
|
||||
--exclude='etc-pihole/gravity_backups' \
|
||||
--exclude='etc-pihole/config_backups' \
|
||||
etc-pihole" \
|
||||
| on_node "$node" "sudo tar -xf - -C $SVC_ROOT/pihole"
|
||||
fi
|
||||
|
||||
# ---- 4. Relay Technitium config from netboot orphaned volume -----------
|
||||
log "$node: copy Technitium config from netboot (orphaned dns_tsys-dns-config volume)"
|
||||
if on_node "$node" "test -f $SVC_ROOT/technitium/config/dns.config"; then
|
||||
log "$node: Technitium config already present; skipping copy"
|
||||
else
|
||||
on_node "$node" "sudo rm -rf $SVC_ROOT/technitium/config"
|
||||
ssh -o StrictHostKeyChecking=no "$NETBOOT" \
|
||||
"sudo tar -cf - -C /var/lib/docker/volumes/dns_tsys-dns-config _data" \
|
||||
| on_node "$node" "sudo tar -xf - -C $SVC_ROOT/technitium && sudo mv $SVC_ROOT/technitium/_data $SVC_ROOT/technitium/config"
|
||||
fi
|
||||
|
||||
# ---- 5. Adapt copied config: Pi-hole interface name ---------------------
|
||||
log "$node: adapt Pi-hole pihole.toml interface name to actual iface"
|
||||
on_node "$node" "bash -s" <<'EOF'
|
||||
set -e
|
||||
IFACE=$(ip -o -4 route show to default 2>/dev/null | awk '{print $5; exit}')
|
||||
IFACE=${IFACE:-ens18}
|
||||
TOML=/home/localuser/services/pihole/etc-pihole/pihole.toml
|
||||
if sudo test -f "$TOML"; then
|
||||
sudo sed -i "s|^ interface = .*| interface = \"$IFACE\" ### ADAPTED from eth0 on clone|" "$TOML"
|
||||
echo "set interface=$IFACE"
|
||||
else
|
||||
echo "(pihole.toml not present; FTL will create it on first run)"
|
||||
fi
|
||||
# Strip netboot-specific primary upstream 192.168.3.16? -> keep, it is reachable on LAN.
|
||||
EOF
|
||||
|
||||
# ---- 6. Pull images -----------------------------------------------------
|
||||
log "$node: docker compose pull (pihole, ntp, technitium)"
|
||||
on_node "$node" "bash -s" <<EOF
|
||||
for c in pihole ntp technitium; do
|
||||
sudo docker compose -f $SVC_ROOT/\$c/docker-compose.yml pull || echo "(pull \$c failed, continuing)"
|
||||
done
|
||||
EOF
|
||||
|
||||
# ---- 6a. Pi-hole up -----------------------------------------------------
|
||||
log "$node: bring up Pi-hole"
|
||||
on_node "$node" "sudo docker compose -f $SVC_ROOT/pihole/docker-compose.yml up -d"
|
||||
|
||||
# ---- 6b. NTP -- only deploy the chrony container if nothing already ----
|
||||
# serves UDP/123 on the host. The targets already run a bare-metal ntpsec
|
||||
# daemon (stratum-2, synced) on 0.0.0.0:123 -- the SAME service family as
|
||||
# netboot's own bare-metal ntpsec. netboot additionally runs a chrony
|
||||
# container on its tailscale IP, but that only works there because ntpsec
|
||||
# there does not pre-bind the specific tailscale-IP socket. On these targets
|
||||
# ntpsec DOES bind the tailscale IP, so the container cannot claim it and is
|
||||
# redundant anyway. We therefore keep the host ntpsec as the NTP service.
|
||||
log "$node: NTP -- check whether host already serves UDP/123"
|
||||
on_node "$node" "bash -s" <<'EOF'
|
||||
set +e
|
||||
HOST_NTP=$(sudo ss -lun 2>/dev/null | awk '$5 ~ /:123$/ {print}' | head -1)
|
||||
if [ -n "$HOST_NTP" ]; then
|
||||
echo "Host already serves UDP/123 ($HOST_NTP); host daemon:"
|
||||
for u in ntpsec ntp chrony openntpd systemd-timesyncd; do
|
||||
systemctl is-active "$u" 2>/dev/null | grep -q active && echo " -> $u active"
|
||||
done
|
||||
echo "Skipping chrony container (host NTP provides the service)."
|
||||
echo "Removing any stale tsys-ntp container..."
|
||||
sudo docker rm -f tsys-ntp 2>/dev/null || true
|
||||
else
|
||||
echo "Nothing serving UDP/123; starting chrony container."
|
||||
sudo docker compose -f /home/localuser/services/ntp/docker-compose.yml up -d
|
||||
fi
|
||||
EOF
|
||||
|
||||
# ---- 6c. Technitium up --------------------------------------------------
|
||||
log "$node: bring up Technitium"
|
||||
on_node "$node" "sudo docker compose -f $SVC_ROOT/technitium/docker-compose.yml up -d"
|
||||
|
||||
# ---- 7. Wait for Pi-hole health -----------------------------------------
|
||||
log "$node: wait for Pi-hole to become healthy"
|
||||
on_node "$node" "bash -s" <<'EOF'
|
||||
for i in $(seq 1 30); do
|
||||
st=$(sudo docker inspect --format '{{.State.Health.Status}}' pihole 2>/dev/null || echo none)
|
||||
echo " pihole health: $st"
|
||||
[ "$st" = "healthy" ] && break
|
||||
sleep 4
|
||||
done
|
||||
EOF
|
||||
|
||||
# ---- 8. Technitium defensive fallback -----------------------------------
|
||||
# If the copied (binary) config from an older Technitium version makes the
|
||||
# new container crash, move it aside and let Technitium start fresh so the
|
||||
# service is at least up (admin reachable) rather than crash-looping.
|
||||
log "$node: check Technitium health (fallback to fresh config if crash)"
|
||||
on_node "$node" "bash -s" <<'EOF'
|
||||
set +e
|
||||
sleep 8
|
||||
rst=$(sudo docker inspect --format '{{.RestartCount}}' tsys-dns 2>/dev/null || echo 0)
|
||||
running=$(sudo docker inspect --format '{{.State.Running}}' tsys-dns 2>/dev/null || echo false)
|
||||
if [ "$running" != "true" ] || [ "$rst" -ge 4 ]; then
|
||||
echo "Technitium unhealthy (running=$running restarts=$rst); quarantining copied config"
|
||||
sudo docker compose -f /home/localuser/services/technitium/docker-compose.yml stop
|
||||
sudo mv /home/localuser/services/technitium/config /home/localuser/services/technitium/config.quarantine.$(date +%s)
|
||||
sudo mkdir -p /home/localuser/services/technitium/config
|
||||
sudo docker compose -f /home/localuser/services/technitium/docker-compose.yml up -d
|
||||
echo "Technitium restarted with fresh config (old config saved as config.quarantine.*)"
|
||||
else
|
||||
echo "Technitium OK (running=$running restarts=$rst)"
|
||||
fi
|
||||
EOF
|
||||
|
||||
# ---- 9. Final status -----------------------------------------------------
|
||||
log "$node: final container status"
|
||||
on_node "$node" "sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'"
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Main
|
||||
#------------------------------------------------------------------------------
|
||||
main() {
|
||||
local mode="${2:-deploy}"
|
||||
if [ "${1:-all}" = "all" ]; then
|
||||
targets=(pfv-netinfra-01 pfv-netinfra-02)
|
||||
else
|
||||
targets=("$1")
|
||||
fi
|
||||
for t in "${targets[@]}"; do
|
||||
: "${NODE_TSIP[$t]:?unknown node $t}"
|
||||
if [ "$mode" = "verify" ]; then verify_node "$t"; else deploy_node "$t"; fi
|
||||
done
|
||||
log "DONE"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user