#!/usr/bin/env bash
# 15-docker-to-data2.sh — move the Docker data-root from the NVMe root
# disk to the 400G SSD /data2 (online, reversible, drain-aware).
#
# WHY: the agent fleet's image/build-cache growth lands on / today.
# Growing sda1 online is NOT safely possible (swap partition blocks it);
# relocating docker to the empty SSD achieves the headroom goal with
# zero partition changes. Asked for by Charles 2026-09-01; agent wrote
# the script, human runs it (sudo; the agent has no passwordless sudo).
#
# WHAT IT DOES:
#   1. graceful-stops containers (30s drain; in-flight LLM turns retry)
#   2. rsyncs /var/lib/docker -> /data2/docker (hardlinks/preserve)
#   3. installs a systemd drop-in: --data-root=/data2/docker
#   4. restarts docker, verifies containers + images are all back
#   5. leaves /var/lib/docker.OLD-<date> for rollback (delete later)
#
# ROLLBACK: remove /etc/systemd/system/docker.service.d/data-root.conf,
# systemctl daemon-reload && systemctl restart docker.
#
# RUN WHEN: any time; expect ~60-120s of gateway/LSP/MCP downtime
# (everything comes back automatically; the LSP/MCP fleets restart
# unless-stopped, the crush session's next LLM turn retries).
set -euo pipefail
[ "$EUID" -eq 0 ] || exec sudo bash "$0"

OUT=/home/reachableceo/projects/ultix/15-docker-to-data2.out
exec > >(tee "$OUT") 2>&1
echo "== docker -> /data2 $(date -Is) =="

NEW=/data2/docker
OLD=/var/lib/docker

# Sanity gates.
df -h / /data2 | sed 's/^/  /'
[ "$(df --output=pcent /data2 | tail -1 | tr -dc '0-9')" -lt 10 ] \
    || { echo "REFUSING: /data2 is >=10% used, not the empty SSD we expect"; exit 1; }
mkdir -p "$NEW"

echo "-- 1/5 drain: graceful container stop (30s each)"
# stop in reverse compose-dependency order is overkill; docker stop
# default 10s is raised to 30 for in-flight work.
for c in $(docker ps -q); do timeout 45 docker stop -t 30 "$c" >/dev/null 2>&1 || true; done

echo "-- 2/5 rsync $OLD -> $NEW (this is the long step, minutes)"
rsync -aHAX --info=progress2 "$OLD"/ "$NEW"/

echo "-- 3/5 systemd drop-in"
mkdir -p /etc/systemd/system/docker.service.d
cat > /etc/systemd/system/docker.service.d/data-root.conf <<'EOF'
# Docker lives on the 400G SSD (/data2) since 2026-09-01 — keeps the
# NVMe root disk lean for the OS + repos. Rollback: delete this file,
# daemon-reload, restart docker (old tree kept as /var/lib/docker.OLD-*).
[Service]
ExecStart=
ExecStart=/usr/sbin/dockerd --data-root=/data2/docker -H fd:// --containerd=/run/containerd/containerd.sock
EOF
systemctl daemon-reload

echo "-- 4/5 restart docker + verify"
systemctl restart docker
sleep 5
docker info 2>/dev/null | grep -E 'Docker Root Dir|Server Version' | sed 's/^/  /'
[ "$(docker info --format '{{.DockerRootDir}}')" = "$NEW" ] || { echo "FAIL: root dir not $NEW"; exit 1; }
docker start $(docker ps -aq) >/dev/null 2>&1 || true
sleep 10
echo "  containers up: $(docker ps -q | wc -l) (expect the LSP/MCP/gateway fleet)"
docker ps --format '  {{.Names}}\t{{.Status}}' | head -25

echo "-- 5/5 stash the old tree"
mv "$OLD" "${OLD}.OLD-$(date +%Y%m%d-%H%M%S)"
echo "== done $(date -Is) — verify everything looks right, then later:"
echo "   rm -rf /var/lib/docker.OLD-*   # frees ~35G on the NVMe"
chown reachableceo:reachableceo "$OUT" 2>/dev/null || true