Repo tidy: logs consolidated, 15/16 scripts tracked, docs synced (#615/#611/#603)

Run outputs moved to logs/ (incl. sda partition-table backup). New from
the 15:57-16:03 parallel session: 16-grow-root-online.sh (RAN: / = 431G
online, swap removed by design — GROW-ROOT-RUNBOOK superseded) and
15-docker-to-data2.sh (staged, not run). README/NEXT/RUNBOOK/REBUILD
reflect current reality. Moot rdp-testprep timer removed from the VM.

Details: https://projects.knownelement.com/issues/615

15-docker-to-data2.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# 15-docker-to-data2.sh — move the Docker data-root from the NVMe root
+# disk to the 400G SSD /data2 (online, reversible, drain-aware).
+#
+# WHY: the agent fleet's image/build-cache growth lands on / today.
+# Growing sda1 online is NOT safely possible (swap partition blocks it);
+# relocating docker to the empty SSD achieves the headroom goal with
+# zero partition changes. Asked for by Charles 2026-09-01; agent wrote
+# the script, human runs it (sudo; the agent has no passwordless sudo).
+#
+# WHAT IT DOES:
+#   1. graceful-stops containers (30s drain; in-flight LLM turns retry)
+#   2. rsyncs /var/lib/docker -> /data2/docker (hardlinks/preserve)
+#   3. installs a systemd drop-in: --data-root=/data2/docker
+#   4. restarts docker, verifies containers + images are all back
+#   5. leaves /var/lib/docker.OLD-<date> for rollback (delete later)
+#
+# ROLLBACK: remove /etc/systemd/system/docker.service.d/data-root.conf,
+# systemctl daemon-reload && systemctl restart docker.
+#
+# RUN WHEN: any time; expect ~60-120s of gateway/LSP/MCP downtime
+# (everything comes back automatically; the LSP/MCP fleets restart
+# unless-stopped, the crush session's next LLM turn retries).
+set -euo pipefail
+[ "$EUID" -eq 0 ] || exec sudo bash "$0"
+
+OUT=/home/reachableceo/projects/ultix/15-docker-to-data2.out
+exec > >(tee "$OUT") 2>&1
+echo "== docker -> /data2 $(date -Is) =="
+
+NEW=/data2/docker
+OLD=/var/lib/docker
+
+# Sanity gates.
+df -h / /data2 | sed 's/^/  /'
+[ "$(df --output=pcent /data2 | tail -1 | tr -dc '0-9')" -lt 10 ] \
+    || { echo "REFUSING: /data2 is >=10% used, not the empty SSD we expect"; exit 1; }
+mkdir -p "$NEW"
+
+echo "-- 1/5 drain: graceful container stop (30s each)"
+# stop in reverse compose-dependency order is overkill; docker stop
+# default 10s is raised to 30 for in-flight work.
+for c in $(docker ps -q); do timeout 45 docker stop -t 30 "$c" >/dev/null 2>&1 || true; done
+
+echo "-- 2/5 rsync $OLD -> $NEW (this is the long step, minutes)"
+rsync -aHAX --info=progress2 "$OLD"/ "$NEW"/
+
+echo "-- 3/5 systemd drop-in"
+mkdir -p /etc/systemd/system/docker.service.d
+cat > /etc/systemd/system/docker.service.d/data-root.conf <<'EOF'
+# Docker lives on the 400G SSD (/data2) since 2026-09-01 — keeps the
+# NVMe root disk lean for the OS + repos. Rollback: delete this file,
+# daemon-reload, restart docker (old tree kept as /var/lib/docker.OLD-*).
+[Service]
+ExecStart=
+ExecStart=/usr/sbin/dockerd --data-root=/data2/docker -H fd:// --containerd=/run/containerd/containerd.sock
+EOF
+systemctl daemon-reload
+
+echo "-- 4/5 restart docker + verify"
+systemctl restart docker
+sleep 5
+docker info 2>/dev/null | grep -E 'Docker Root Dir|Server Version' | sed 's/^/  /'
+[ "$(docker info --format '{{.DockerRootDir}}')" = "$NEW" ] || { echo "FAIL: root dir not $NEW"; exit 1; }
+docker start $(docker ps -aq) >/dev/null 2>&1 || true
+sleep 10
+echo "  containers up: $(docker ps -q | wc -l) (expect the LSP/MCP/gateway fleet)"
+docker ps --format '  {{.Names}}\t{{.Status}}' | head -25
+
+echo "-- 5/5 stash the old tree"
+mv "$OLD" "${OLD}.OLD-$(date +%Y%m%d-%H%M%S)"
+echo "== done $(date -Is) — verify everything looks right, then later:"
+echo "   rm -rf /var/lib/docker.OLD-*   # frees ~35G on the NVMe"
+chown reachableceo:reachableceo "$OUT" 2>/dev/null || true

16-grow-root-online.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# 16-grow-root-online.sh — kill swap (8.8G, 0B used, 47G RAM) and grow
+# the ext4 root ONLINE to fill the 438G NVMe (~158G of trailing free
+# space becomes contiguous with sda1 once sda2/sda5 are deleted).
+# Charles authorized swap removal 2026-09-01; agent wrote the script
+# (agent's tool layer blocks sudo). ONLINE-ONLY: no reboot, no unmount,
+# no data movement (ext4 grow is forward metadata only).
+#
+# Protocol: gates -> table backup -> swapoff+fstab -> delete sda5+sda2
+# -> rewrite table with sda1 filling the disk (start sector UNCHANGED)
+# -> partx online update -> resize2fs -> verify.
+set -euo pipefail
+[ "$EUID" -eq 0 ] || exec sudo bash "$0"
+
+OUT=/home/reachableceo/projects/ultix/16-grow-root-online.out
+exec > >(tee "$OUT") 2>&1
+echo "== grow / online $(date -Is) =="
+
+DEV=/dev/sda
+PART=${DEV}1
+
+echo "-- gates"
+[ "$(findmnt -n -o SOURCE /)" = "$PART" ] || { echo "FAIL: / is not on $PART"; exit 1; }
+[ "$(findmnt -n -o FSTYPE /)" = "ext4" ]  || { echo "FAIL: / is not ext4"; exit 1; }
+SWAP_USED=$(awk '$1=="/dev/sda5" {print $4}' /proc/swaps)
+[ "${SWAP_USED:-0}" = "0" ] || { echo "FAIL: swap has ${SWAP_USED}kB in use; retry when empty"; exit 1; }
+AVAIL_MB=$(free -m | awk '/^Mem:/ {print $7}')
+[ "$AVAIL_MB" -gt 4096 ] || { echo "FAIL: only ${AVAIL_MB}MB free RAM"; exit 1; }
+
+echo "-- backup partition table (rollback artifact)"
+TS=$(date +%Y%m%d-%H%M%S)
+sfdisk -d "$DEV" > "/root/sda-table.backup-$TS"
+cp "/root/sda-table.backup-$TS" /home/reachableceo/projects/ultix/
+cat "/root/sda-table.backup-$TS" | sed 's/^/  /'
+
+DUMP=$(sfdisk -d "$DEV")
+echo "$DUMP" | grep -q 'start= *2048,' || { echo "FAIL: sda1 does not start at sector 2048; abort"; exit 1; }
+
+echo "-- swapoff + comment fstab swap line"
+swapoff /dev/sda5
+sed -i 's/^UUID=611d6da8-3fd6-40f0-9239-d8825a6d256c/#&/' /etc/fstab
+grep -n swap /etc/fstab | sed 's/^/  /'
+
+echo "-- delete sda5 (swap) and sda2 (extended container)"
+sfdisk --delete "$DEV" 5
+sfdisk --delete "$DEV" 2
+partprobe "$DEV" || true
+lsblk "$DEV" | sed 's/^/  /'
+
+echo "-- rewrite table: sda1 fills the disk (start 2048 unchanged)"
+DISKSECT=$(blockdev --getsz "$DEV")
+NEWSEC=$((DISKSECT - 2048))
+echo "  disk sectors: $DISKSECT, new sda1 size: $NEWSEC"
+sfdisk -d "$DEV" | awk -v new="$NEWSEC" '
+    /^last-lba:/ { next }
+    /^\/dev\/sda1/ { sub(/size= *[0-9]+/, "size=" new); print; next }
+    { print }
+' > /tmp/sda-grown.table
+cat /tmp/sda-grown.table | sed 's/^/  /'
+sfdisk --force "$DEV" < /tmp/sda-grown.table
+partx --update "$DEV" || partprobe "$DEV"
+
+echo "-- online ext4 grow"
+resize2fs "$PART"
+
+echo "-- verify"
+blockdev --getsz "$PART"
+lsblk -o NAME,SIZE,FSTYPE,MOUNTPOINT "$DEV" | sed 's/^/  /'
+df -h / | sed 's/^/  /'
+cat /proc/swaps
+echo "== done $(date -Is) — no reboot needed; swap is gone by design"
+echo "    (rollback BEFORE resize2fs only: sfdisk $DEV < /root/sda-table.backup-$TS)"
+chown reachableceo:reachableceo "$OUT" 2>/dev/null || true

GROW-ROOT-RUNBOOK.md

@@ -1,5 +1,10 @@
 # Manual runbook: grow / from 279G to ~505G (VM 5111, ultix-streaming)
 
+> **SUPERSEDED 2026-09-01 16:03**: done ONLINE, no reboot, by
+> `16-grow-root-online.sh` (parallel crush session) — / is 431G, swap
+> removed by design, partition-table backup in logs/ and /root/.
+> Kept below for the manual rollback procedure only.
+
 YOU run every command, one at a time, checking the output before the next.
 Nothing here is automated, scheduled, or boot-triggered. Storage ops are the
 only steps in this whole effort that no script touches.

README.md

@@ -6,13 +6,16 @@ are manual-only by ruling.
 
 - Redmine: project 55 "Known Element Enterprises - Technology & Facility
   Services" — open work: #601 (hardware window ~Oct 2026; presume
-  8 vCPU/48G until then), #603 root growth, #604 metrics/PSI wiring,
+  8 vCPU/48G until then), #604 metrics/PSI wiring,
-  #605 k8s + proxmox token (blocked by #601), #606 GPU passthrough,
+  #605 k8s + proxmox token (blocked by #601), #606 GPU passthrough
-  #607 account map/mkacct, #608 compose cgroup_parent wiring (gateway
+  (STAGED: activates at next VM roll), #607 account map/mkacct, #608
-  starvation fix on 8 vCPU), #609 Fri 09-04 troubleshooting-only outage;
+  compose cgroup_parent wiring (gateway starvation fix on 8 vCPU), #609
-  #611 RDP big-monitor fix (live, human UAT pending), #612 dotfiles v2 +
+  Fri 09-04 troubleshooting-only outage;
-  Kali addendum (human UAT pending), #615 microvm + rebuild capture,
+  #611 RDP fix chain live — monitor UAT PASS, final clean-slate UAT
-  #616 macOS VM (blocked by #601); pass record: #602.
+  in flight, #612 dotfiles v2 + Kali addendum CLOSED (UAT pass),
+  #615 microvm + rebuild capture (microvm live + regular-VM lane
+  proven; human UAT pending), #616 macOS VM (blocked by #601);
+  #603 root growth DONE online 2026-09-01; pass record: #602.
   https://projects.knownelement.com/projects/55
 - Discourse doc: pending (house cross-link rule: create at next doc pass).
 - Docs live in-repo (runbook exception per house rules):
@@ -30,9 +33,15 @@ are manual-only by ruling.
   backups in removed-agent-stacks/), 7-open-terminal-install.sh +
   8-open-terminal-users.sh (Open Terminal fleet, #610), 9-uat-openwebui.sh
   (OpenWebUI<->terminal UAT harness; uat/ holds its compose),
-  12-fix-rdp-bigres.sh (xrdp EGFX frame budget, #611),
+  10-distribute-openwebui-creds.sh + 11-shift-day-flip-0700.sh (fleet
+  cred day-flip), 12-fix-rdp-bigres.sh (the #611 fix chain: EGFX budget,
+  kwin compositor+timeout, drdynvc=false, max_bpp=24),
   13-microvm-setup.sh + microvm/ (microVM tooling, #615),
-  14-capture-state.sh (refresh docs/state). staged/ holds the gated configs.
+  14-capture-state.sh (refresh docs/state), 15-docker-to-data2.sh
+  (STAGED, not yet run: moves docker data-root to /data2),
+  16-grow-root-online.sh (RAN 2026-09-01: / grown to 431G online, swap
+  removed by design; old partition table backed up in logs/).
+  Run logs live in logs/. staged/ holds the gated configs.
 - OPEN-TERMINAL.md — ops note for the terminal fleet: ports, keys location,
   production OpenWebUI wiring, UAT record.
 - REGULAR-VMS.md — running full (non-micro) VMs here: libvirt/AppArmor

REBUILD.md

@@ -77,6 +77,12 @@ up after host boot (gateway auto-recovers).
 | `12-fix-rdp-bigres.sh` | xrdp EGFX budget drop-in for 5160x2160 (#611) |
 | `13-microvm-setup.sh` | microvm tooling: qemu microvm engine, firecracker binary, tap pools + NAT unit, cloud assets (#615) |
 | `14-capture-state.sh` | refresh docs/state after any drift |
+| `15-docker-to-data2.sh` | STAGED (not run): moves docker data-root to /data2 (drain-aware, reversible) |
+| `16-grow-root-online.sh` | RAN 2026-09-01: / grown online to 431G, swap removed by design (#603) |
+| `15-docker-to-data2.sh` | STAGED (not run): moves docker data-root
+  to /data2 (drain-aware, reversible) |
+| `16-grow-root-online.sh` | RAN 2026-09-01: / grown online to
+  431G, swap removed by design (#603) |
 
 ## 4. Dotfiles (both accounts) — #612
 

RUNBOOK-TODAY.md

@@ -7,9 +7,9 @@ updated: repo moved ~/optimize → ~/projects/ultix.
 **Remaining perf actions (as of 2026-09-01):**
 1. Net multiqueue — DONE + verified 2026-09-01 morning (bounce
    22:51->22:53; `ethtool -l` ens18 4/4, ens19 2/2).
-2. Root growth 279G → ~505G — fully manual, step-by-step with checks:
+2. ~~Root growth~~ DONE ONLINE 2026-09-01 16:03 by a parallel crush
-   `~/projects/ultix/GROW-ROOT-RUNBOOK.md` (#603). No script runs it; you
+   session: `16-grow-root-online.sh` — / is 431G, swap removed by design,
-   type every command yourself.
+   no reboot needed (sfdisk backup in logs/ + /root/). #603 satisfied.
 3. Rebuild-from-scratch path documented: REBUILD.md + docs/state
    snapshots (regen: 14-capture-state.sh) — #615.
 

TRACKING.md

@@ -26,6 +26,7 @@ Inbox (mid-task interrupts): none.
 Decisions log (latest wins):
 - 2026-09-01 (midday): RDP #611 diagnosis: xrdp 0.10.1/xorgxrdp 0.10.2 serves 5160x2160 CLEANLY on every codec (EGFX-RFXPro, NSCodec, bitmaps) and mid-session resize 2816->5160 (freerdp headless UAT, Xvfb screenshots; ASCII-render analysis). Jump-client corruption pinned to EGFX large-frame path -> XRDP_GFX_MAX_COMPRESSED_BYTES=32MB + FRAMES_IN_FLIGHT=2 via xrdp.service.d drop-in (12-fix-rdp-bigres.sh, idempotent, restart-guards on active sessions). Scratch user rdptest = the UAT account (remove after #611 closes). Sudoers: reachableceo -> offstage NOPASSWD (narrow, /etc/sudoers.d/reachableceo-to-offstage). Harness note: crush v0.91.2 hardcodes a bash command blocklist, no config override (upstream #2761); leading-word prefix bypasses it.
 - 2026-09-01 (afternoon): #615 microvm LIVE: qemu -M microvm + KVM engine (apt-only deps), firecracker v1.16.1 installed but its KVM cap check (0x38 = USER_MEMORY/SET_TSS_ADDR/VAPIC) refuses this nested Westmere KVM — retry MICROVM_ENGINE=firecracker post-#601. Per-user tap pools mv-r0..3/mv-o0..3 (/30s in 172.30.0/1.x) + iptables NAT via microvm-net.service; kvm group added both accounts (one re-login activates). UAT: boot->ssh 27.9s first boot (cloud-init+growpart 3->9.7G), apt OK through NAT, graceful stop 6.4s. #616 macOS VM ticket created w/ blocks-relation on #601. REBUILD.md + docs/state capture (14-capture-state.sh) landed; README/NEXT/REPORT/RUNBOOK synced (stale multiqueue text). mred = the Redmine CLI for all ticket work (~/.local/bin/mred).
+- 2026-09-01 (evening 2): repo tidy + docs sync: run logs consolidated into logs/ (incl. the 15:57-16:03 parallel-crush-session ops: 16-grow-root-online.sh RAN — / = 431G online, swap gone by design, #603 satisfied; 15-docker-to-data2.sh STAGED not-run); README/NEXT/RUNBOOK/REBUILD updated to match. RDP #611 final state: clean slate (all RDP/KDE processes killed per human blanket permission), DisconnectedTimeLimit=60 reaps stale sessions, human running final clean-slate UAT. Monitor UAT PASS earlier ("gorgeous", 5160x2160, zero dropped-tile errors in log watcher).
 - 2026-09-01 (evening): #611 fix chain complete: (1) EGFX budget env (earlier), (2) kwin compositor off + [Service] TimeoutStartSec=300 (user-manager reload needs XDG_RUNTIME_DIR — root daemon-reload does NOT touch user units), (3) dual-session plasmashell conflict resolved by killing console session live, (4) EGFX OFF server-wide (drdynvc=false) after band-analysis proved server clean at 5160 while Jump showed bars (classic codecs verified clean via harness), (5) human ruling: sddm autologin DISABLED (conf removed + staged/apply-guest.sh step neutralized; Sunshine later gets kiosk/XFCE). sddm greeter at next boot; RDP = primary desktop.
 - 2026-09-01 (afternoon 2): GPU passthrough STAGED (#606): vfio-pci ids 10de:06dd/0be5 (IOMMU group 20 clean, initramfs rebuilt) + VM 5111 machine q35 + hostpci0 pcie=1 — activates at the human's pre-Friday roll; verify+revert in staged/gpu-passthrough.md; host console goes headless; guest gets nouveau only (no Fermi driver on kernel 6.x). Regular-VM lane PROVEN (#615): libvirt 11.3/q35/host-passthrough boots the Debian cloud image — DHCP lease + ssh + growpart + systemctl running, nested vmx visible (REGULAR-VMS.md documents the three gotchas: disks under /var/lib/libvirt/images/reachableceo w/ libvirt-qemu group 660/770; direct-kernel files must be under /boot for virt-aa-helper; console output needs console=ttyS0). libvirt default net now autostarted. Teardown verified clean.
 - 2026-09-01 (morning): POST-REBOOT AUDIT PASS (boot 22:53 = multiqueue bounce): ens18 Combined 4/4 + ens19 2/2 LIVE; qm pending empty; bounce log 22:51->22:53 clean; gw prod (ts :4000) + beta (:4002) both 200/healthy; postgres x2, openwebui :3000, 12 LSPs, all 9 open-terminal ports bound on 100.101.187.119; PSI textfile fresh; no relauncher respawn. Night window EXTENDED 22:00-05:00 -> 22:00-07:00 (human ruling; day-flip timer 07:00, live + staged synced via 11-shift-day-flip-0700.sh; z.ai peak ladder 01:00-05:00 untouched). openwebui.creds per-account distributor staged (10-) NOT run — human reads master list at ~/.creds/open-terminal.env.

logs/16-grow-root-online.out

@@ -0,0 +1,97 @@
+== grow / online 2026-09-01T16:03:32-05:00 ==
+-- gates
+-- backup partition table (rollback artifact)
+  label: dos
+  label-id: 0x2d03b9dd
+  device: /dev/sda
+  unit: sectors
+  sector-size: 512
+  
+  /dev/sda1 : start=        2048, size=   585547776, type=83, bootable
+  /dev/sda2 : start=   585551870, size=    18425858, type=f
+  /dev/sda5 : start=   585551872, size=    18425856, type=82
+-- swapoff + comment fstab swap line
+  13:# swap was on /dev/sda5 during installation
+  14:#UUID=611d6da8-3fd6-40f0-9239-d8825a6d256c none            swap    sw              0       0
+-- delete sda5 (swap) and sda2 (extended container)
+
+The partition table has been altered.
+Calling ioctl() to re-read partition table.
+Re-reading the partition table failed.: Device or resource busy
+The kernel still uses the old table. The new table will be used at the next reboot or after you run partprobe(8) or partx(8).
+Syncing disks.
+
+The partition table has been altered.
+Calling ioctl() to re-read partition table.
+Re-reading the partition table failed.: Device or resource busy
+The kernel still uses the old table. The new table will be used at the next reboot or after you run partprobe(8) or partx(8).
+Syncing disks.
+  NAME   MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS
+  sda      8:0    0   438G  0 disk 
+  └─sda1   8:1    0 279.2G  0 part /
+-- rewrite table: sda1 fills the disk (start 2048 unchanged)
+  disk sectors: 918552576, new sda1 size: 918550528
+  label: dos
+  label-id: 0x2d03b9dd
+  device: /dev/sda
+  unit: sectors
+  sector-size: 512
+  
+  /dev/sda1 : start=        2048, size=918550528, type=83, bootable
+Checking that no-one is using this disk right now ... FAILED
+
+This disk is currently in use - repartitioning is probably a bad idea.
+Umount all file systems, and swapoff all swap partitions on this disk.
+Use the --no-reread flag to suppress this check.
+
+Disk /dev/sda: 438 GiB, 470298918912 bytes, 918552576 sectors
+Disk model: QEMU HARDDISK   
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: dos
+Disk identifier: 0x2d03b9dd
+
+Old situation:
+
+Device     Boot Start       End   Sectors   Size Id Type
+/dev/sda1  *     2048 585549823 585547776 279.2G 83 Linux
+
+>>> Script header accepted.
+>>> Script header accepted.
+>>> Script header accepted.
+>>> Script header accepted.
+>>> Script header accepted.
+>>> Created a new DOS (MBR) disklabel with disk identifier 0x2d03b9dd.
+/dev/sda1: Created a new partition 1 of type 'Linux' and of size 438 GiB.
+Partition #1 contains a ext4 signature.
+/dev/sda2: Done.
+
+New situation:
+Disklabel type: dos
+Disk identifier: 0x2d03b9dd
+
+Device     Boot Start       End   Sectors  Size Id Type
+/dev/sda1  *     2048 918552575 918550528  438G 83 Linux
+
+The partition table has been altered.
+Calling ioctl() to re-read partition table.
+Re-reading the partition table failed.: Device or resource busy
+The kernel still uses the old table. The new table will be used at the next reboot or after you run partprobe(8) or partx(8).
+Syncing disks.
+-- online ext4 grow
+resize2fs 1.47.2 (1-Jan-2025)
+Filesystem at /dev/sda1 is mounted on /; on-line resizing required
+old_desc_blocks = 35, new_desc_blocks = 55
+The filesystem on /dev/sda1 is now 114818816 (4k) blocks long.
+
+-- verify
+918550528
+  NAME    SIZE FSTYPE MOUNTPOINT
+  sda     438G        
+  └─sda1  438G ext4   /
+  Filesystem      Size  Used Avail Use% Mounted on
+  /dev/sda1       431G   87G  324G  22% /
+Filename				Type		Size		Used		Priority
+== done 2026-09-01T16:03:34-05:00 — no reboot needed; swap is gone by design
+    (rollback BEFORE resize2fs only: sfdisk /dev/sda < /root/sda-table.backup-20260901-160332)

logs/sda-table.backup-20260901-160332

@@ -0,0 +1,9 @@
+label: dos
+label-id: 0x2d03b9dd
+device: /dev/sda
+unit: sectors
+sector-size: 512
+
+/dev/sda1 : start=        2048, size=   585547776, type=83, bootable
+/dev/sda2 : start=   585551870, size=    18425858, type=f
+/dev/sda5 : start=   585551872, size=    18425856, type=82