mrcharles
291a7ebc27
Deployed check.sh to all 7 hosts at 21:50 CDT. Captures the live state after the user's PDM migrations: Cnode movements since last audit: - cnode1: tsys1 -> tsys9 - cnode2: tsys6 -> tsys7 - cnode5: tsys6 -> tsys7, storage D5(tsys4) -> S2(tsys5) Wnode changes: - wnode-tsys1 (102): new VM on S2, stopped - wnode-tsys3: RAM bumped 20 -> 28 GB - wnode-tsys6: now running (was stopped) - wnode-tsys9: storage moved S3 -> S2 Storage distribution improved from 90/10 to 73/27 (tsys4/tsys5). Still need 2 more cnode moves for etcd quorum survival. Updated executive summary, k8s distribution tables, storage utilization, and open items with the fresh data. Captured future k8s requirements: vcluster + Rancher, OIDC to Keycloak, workload isolation (RackRental/Suborbital ITAR/non-ITAR/SLP), and solar-aware scale-out with PowerEdge 19xx/2950 systems. Added tsys9 to deploy-check.sh host list.
284 lines
12 KiB
Bash
Executable File
284 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# setup-netinfra.sh
|
|
# -----------------------------------------------------------------------------
|
|
# Replicate pfv-netboot's network services (Pi-hole, Technitium DNS, NTP)
|
|
# onto pfv-netinfra-01 and pfv-netinfra-02.
|
|
#
|
|
# DESIGN
|
|
# * pfv-netboot is REFERENCE ONLY -- this script NEVER mutates it. All reads
|
|
# from it are via `ssh localuser@pfv-netboot 'sudo ...'` (read-only cmds).
|
|
# * The targets cannot SSH to pfv-netboot directly, so config tarballs are
|
|
# relayed through this workstation:
|
|
# ssh netboot 'sudo tar -cf - ...' | ssh target 'sudo tar -xf - ...'
|
|
# * Services are deployed under /home/localuser/services/<svc>/ on each
|
|
# target so localuser can manage them (mirrors netboot's localuser-owned
|
|
# pihole data dir). `sudo docker` is used since localuser is not in the
|
|
# docker group (same as on netboot).
|
|
#
|
|
# SERVICES
|
|
# pihole pihole/pihole:latest :53 tcp/udp :10002->80 :10003->443
|
|
# ntp (chrony) dockurr/chrony <tailscale-ip>:123:123/udp
|
|
# technitium technitium/dns-server :5300->53 tcp/udp :5380 :53443
|
|
# (Technitium DNS is remapped off :53 to avoid clashing with Pi-hole.
|
|
# The knel.net authoritative zone + Tailscale reverse zones are preserved
|
|
# verbatim from netboot's orphaned dns_tsys-dns-config volume.)
|
|
#
|
|
# USAGE
|
|
# ./setup-netinfra.sh # deploy to BOTH nodes
|
|
# ./setup-netinfra.sh pfv-netinfra-01 # deploy to one node
|
|
# ./setup-netinfra.sh pfv-netinfra-01 verify # verify only
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
NETBOOT="localuser@pfv-netboot"
|
|
SVC_ROOT="/home/localuser/services"
|
|
PIHOLE_PW='REDACTED_PASSWORD' # replicated verbatim from netboot compose
|
|
|
|
log() { printf '\n\033[1;36m[%s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; }
|
|
warn() { printf '\n\033[1;33m[WARN %s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; }
|
|
|
|
# Per-node parameters. (LAN iface is auto-detected at deploy time as a fallback.)
|
|
declare -A NODE_TSIP=(
|
|
[pfv-netinfra-01]="100.70.181.72"
|
|
[pfv-netinfra-02]="100.93.194.82"
|
|
)
|
|
|
|
on_node() { ssh -o StrictHostKeyChecking=no "localuser@$1" "$2"; }
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Verify-only mode
|
|
#------------------------------------------------------------------------------
|
|
verify_node() {
|
|
local node="$1" tsip="${NODE_TSIP[$1]}"
|
|
log "VERIFY $node (tailscale $tsip)"
|
|
on_node "$node" "bash -s" <<EOF
|
|
set +e
|
|
echo "### containers ###"
|
|
sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' 2>&1
|
|
echo
|
|
echo "### Pi-hole DNS (dig @127.0.0.1:53 pi.hole) ###"
|
|
dig +time=3 +tries=1 +short @127.0.0.1 -p 53 pi.hole 2>&1
|
|
echo "### Pi-hole web (curl :10002) ###"
|
|
curl -sk -o /dev/null -w 'http=%{http_code}\n' http://127.0.0.1:10002/admin/ 2>&1
|
|
echo
|
|
echo "### Technitium DNS (dig @127.0.0.1:5300 knel.net SOA) ###"
|
|
dig +time=3 +tries=1 @127.0.0.1 -p 5300 knel.net SOA +short 2>&1
|
|
echo "### Technitium web (curl :5380) ###"
|
|
curl -sk -o /dev/null -w 'http=%{http_code}\n' http://127.0.0.1:5380/ 2>&1
|
|
echo
|
|
echo "### NTP on tailscale IP (ntpdig -p 1) ###"
|
|
timeout 5 ntpdig -t1 -c1 -p1 $tsip 2>&1 || echo "(ntpdig not available or no response)"
|
|
echo "### chrony container sources ###"
|
|
sudo docker exec tsys-ntp chronyc -n sources 2>&1 | head -8
|
|
EOF
|
|
}
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Deploy to one node
|
|
#------------------------------------------------------------------------------
|
|
deploy_node() {
|
|
local node="$1" tsip="${NODE_TSIP[$1]}"
|
|
log "==== DEPLOY $node (tailscale $tsip) ===="
|
|
|
|
# ---- 1. Prepare directories on the target -------------------------------
|
|
log "$node: create service dirs"
|
|
on_node "$node" "bash -s" <<EOF
|
|
set -e
|
|
sudo mkdir -p $SVC_ROOT/pihole $SVC_ROOT/ntp $SVC_ROOT/technitium
|
|
sudo chown -R localuser:localuser $SVC_ROOT
|
|
EOF
|
|
|
|
# ---- 2. Write compose files (as localuser) ------------------------------
|
|
log "$node: write docker-compose files"
|
|
on_node "$node" "cat > $SVC_ROOT/pihole/docker-compose.yml" <<'YAML'
|
|
services:
|
|
pihole:
|
|
container_name: pihole
|
|
image: pihole/pihole:latest
|
|
hostname: pihole
|
|
ports:
|
|
- "53:53/tcp"
|
|
- "53:53/udp"
|
|
- "10002:80/tcp"
|
|
- "10003:443/tcp"
|
|
environment:
|
|
TZ: 'America/Chicago'
|
|
FTLCONF_webserver_api_password: 'REDACTED_PASSWORD'
|
|
FTLCONF_dns_listeningMode: 'all'
|
|
volumes:
|
|
- './etc-pihole:/etc/pihole'
|
|
cap_add:
|
|
- SYS_NICE
|
|
restart: always
|
|
YAML
|
|
|
|
on_node "$node" "cat > $SVC_ROOT/ntp/docker-compose.yml" <<YAML
|
|
services:
|
|
ntp:
|
|
image: dockurr/chrony
|
|
container_name: tsys-ntp
|
|
environment:
|
|
NTP_SERVERS: "pool.ntp.org"
|
|
ports:
|
|
- "$tsip:123:123/udp"
|
|
restart: always
|
|
YAML
|
|
|
|
on_node "$node" "cat > $SVC_ROOT/technitium/docker-compose.yml" <<'YAML'
|
|
services:
|
|
technitium:
|
|
image: technitium/dns-server
|
|
container_name: tsys-dns
|
|
ports:
|
|
- "5300:53/tcp"
|
|
- "5300:53/udp"
|
|
- "5380:5380/tcp"
|
|
- "53443:53443/tcp"
|
|
volumes:
|
|
- './config:/etc/dns'
|
|
restart: always
|
|
YAML
|
|
|
|
# ---- 3. Relay Pi-hole config from netboot -> target ---------------------
|
|
log "$node: copy Pi-hole /etc/pihole from netboot (excluding query logs)"
|
|
if on_node "$node" "test -f $SVC_ROOT/pihole/etc-pihole/gravity.db"; then
|
|
log "$node: Pi-hole config already present; skipping copy"
|
|
else
|
|
on_node "$node" "sudo rm -rf $SVC_ROOT/pihole/etc-pihole"
|
|
ssh -o StrictHostKeyChecking=no "$NETBOOT" \
|
|
"sudo tar -cf - -C /root/pihole --exclude='etc-pihole/pihole-FTL.db*' \
|
|
--exclude='etc-pihole/listsCache' \
|
|
--exclude='etc-pihole/gravity_backups' \
|
|
--exclude='etc-pihole/config_backups' \
|
|
etc-pihole" \
|
|
| on_node "$node" "sudo tar -xf - -C $SVC_ROOT/pihole"
|
|
fi
|
|
|
|
# ---- 4. Relay Technitium config from netboot orphaned volume -----------
|
|
log "$node: copy Technitium config from netboot (orphaned dns_tsys-dns-config volume)"
|
|
if on_node "$node" "test -f $SVC_ROOT/technitium/config/dns.config"; then
|
|
log "$node: Technitium config already present; skipping copy"
|
|
else
|
|
on_node "$node" "sudo rm -rf $SVC_ROOT/technitium/config"
|
|
ssh -o StrictHostKeyChecking=no "$NETBOOT" \
|
|
"sudo tar -cf - -C /var/lib/docker/volumes/dns_tsys-dns-config _data" \
|
|
| on_node "$node" "sudo tar -xf - -C $SVC_ROOT/technitium && sudo mv $SVC_ROOT/technitium/_data $SVC_ROOT/technitium/config"
|
|
fi
|
|
|
|
# ---- 5. Adapt copied config: Pi-hole interface name ---------------------
|
|
log "$node: adapt Pi-hole pihole.toml interface name to actual iface"
|
|
on_node "$node" "bash -s" <<'EOF'
|
|
set -e
|
|
IFACE=$(ip -o -4 route show to default 2>/dev/null | awk '{print $5; exit}')
|
|
IFACE=${IFACE:-ens18}
|
|
TOML=/home/localuser/services/pihole/etc-pihole/pihole.toml
|
|
if sudo test -f "$TOML"; then
|
|
sudo sed -i "s|^ interface = .*| interface = \"$IFACE\" ### ADAPTED from eth0 on clone|" "$TOML"
|
|
echo "set interface=$IFACE"
|
|
else
|
|
echo "(pihole.toml not present; FTL will create it on first run)"
|
|
fi
|
|
# Strip netboot-specific primary upstream 192.168.3.16? -> keep, it is reachable on LAN.
|
|
EOF
|
|
|
|
# ---- 6. Pull images -----------------------------------------------------
|
|
log "$node: docker compose pull (pihole, ntp, technitium)"
|
|
on_node "$node" "bash -s" <<EOF
|
|
for c in pihole ntp technitium; do
|
|
sudo docker compose -f $SVC_ROOT/\$c/docker-compose.yml pull || echo "(pull \$c failed, continuing)"
|
|
done
|
|
EOF
|
|
|
|
# ---- 6a. Pi-hole up -----------------------------------------------------
|
|
log "$node: bring up Pi-hole"
|
|
on_node "$node" "sudo docker compose -f $SVC_ROOT/pihole/docker-compose.yml up -d"
|
|
|
|
# ---- 6b. NTP -- only deploy the chrony container if nothing already ----
|
|
# serves UDP/123 on the host. The targets already run a bare-metal ntpsec
|
|
# daemon (stratum-2, synced) on 0.0.0.0:123 -- the SAME service family as
|
|
# netboot's own bare-metal ntpsec. netboot additionally runs a chrony
|
|
# container on its tailscale IP, but that only works there because ntpsec
|
|
# there does not pre-bind the specific tailscale-IP socket. On these targets
|
|
# ntpsec DOES bind the tailscale IP, so the container cannot claim it and is
|
|
# redundant anyway. We therefore keep the host ntpsec as the NTP service.
|
|
log "$node: NTP -- check whether host already serves UDP/123"
|
|
on_node "$node" "bash -s" <<'EOF'
|
|
set +e
|
|
HOST_NTP=$(sudo ss -lun 2>/dev/null | awk '$5 ~ /:123$/ {print}' | head -1)
|
|
if [ -n "$HOST_NTP" ]; then
|
|
echo "Host already serves UDP/123 ($HOST_NTP); host daemon:"
|
|
for u in ntpsec ntp chrony openntpd systemd-timesyncd; do
|
|
systemctl is-active "$u" 2>/dev/null | grep -q active && echo " -> $u active"
|
|
done
|
|
echo "Skipping chrony container (host NTP provides the service)."
|
|
echo "Removing any stale tsys-ntp container..."
|
|
sudo docker rm -f tsys-ntp 2>/dev/null || true
|
|
else
|
|
echo "Nothing serving UDP/123; starting chrony container."
|
|
sudo docker compose -f /home/localuser/services/ntp/docker-compose.yml up -d
|
|
fi
|
|
EOF
|
|
|
|
# ---- 6c. Technitium up --------------------------------------------------
|
|
log "$node: bring up Technitium"
|
|
on_node "$node" "sudo docker compose -f $SVC_ROOT/technitium/docker-compose.yml up -d"
|
|
|
|
# ---- 7. Wait for Pi-hole health -----------------------------------------
|
|
log "$node: wait for Pi-hole to become healthy"
|
|
on_node "$node" "bash -s" <<'EOF'
|
|
for i in $(seq 1 30); do
|
|
st=$(sudo docker inspect --format '{{.State.Health.Status}}' pihole 2>/dev/null || echo none)
|
|
echo " pihole health: $st"
|
|
[ "$st" = "healthy" ] && break
|
|
sleep 4
|
|
done
|
|
EOF
|
|
|
|
# ---- 8. Technitium defensive fallback -----------------------------------
|
|
# If the copied (binary) config from an older Technitium version makes the
|
|
# new container crash, move it aside and let Technitium start fresh so the
|
|
# service is at least up (admin reachable) rather than crash-looping.
|
|
log "$node: check Technitium health (fallback to fresh config if crash)"
|
|
on_node "$node" "bash -s" <<'EOF'
|
|
set +e
|
|
sleep 8
|
|
rst=$(sudo docker inspect --format '{{.RestartCount}}' tsys-dns 2>/dev/null || echo 0)
|
|
running=$(sudo docker inspect --format '{{.State.Running}}' tsys-dns 2>/dev/null || echo false)
|
|
if [ "$running" != "true" ] || [ "$rst" -ge 4 ]; then
|
|
echo "Technitium unhealthy (running=$running restarts=$rst); quarantining copied config"
|
|
sudo docker compose -f /home/localuser/services/technitium/docker-compose.yml stop
|
|
sudo mv /home/localuser/services/technitium/config /home/localuser/services/technitium/config.quarantine.$(date +%s)
|
|
sudo mkdir -p /home/localuser/services/technitium/config
|
|
sudo docker compose -f /home/localuser/services/technitium/docker-compose.yml up -d
|
|
echo "Technitium restarted with fresh config (old config saved as config.quarantine.*)"
|
|
else
|
|
echo "Technitium OK (running=$running restarts=$rst)"
|
|
fi
|
|
EOF
|
|
|
|
# ---- 9. Final status -----------------------------------------------------
|
|
log "$node: final container status"
|
|
on_node "$node" "sudo docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'"
|
|
}
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Main
|
|
#------------------------------------------------------------------------------
|
|
main() {
|
|
local mode="${2:-deploy}"
|
|
if [ "${1:-all}" = "all" ]; then
|
|
targets=(pfv-netinfra-01 pfv-netinfra-02)
|
|
else
|
|
targets=("$1")
|
|
fi
|
|
for t in "${targets[@]}"; do
|
|
: "${NODE_TSIP[$t]:?unknown node $t}"
|
|
if [ "$mode" = "verify" ]; then verify_node "$t"; else deploy_node "$t"; fi
|
|
done
|
|
log "DONE"
|
|
}
|
|
|
|
main "$@"
|