ultix perf/ops workbench: complete 2026-08-31 optimization pass for VM 5111

Everything for the ultix-streaming (VM 5111, pfv-tsys5) performance pass: full report + host audit results, staged/gated configs, guest prep + host one-shot + post-reboot-fix + netcheck lifecycle scripts, grow-root manual runbook, rolling tracking HUD, questions v1, and the gateway boot-race hardening units. Applied and verified live 2026-08-31; open work is tracked in Redmine project 55 as #601-#607. [#602] 💘 Generated with Crush Assisted-by: Crush:glm-5.2
2026-08-31 16:46:13 -05:00
commit cba2a64d67
43 changed files with 3137 additions and 0 deletions
@@ -0,0 +1,34 @@
+# /etc/sysctl.d/60-ukrrs-vm.conf — ultix-streaming mixed-workload profile
+# Measured-before values and rationale: ~/optimize/REPORT.md §4.1.
+# Deliberately NOT touched: swappiness(60), overcommit(0), vfs_cache_pressure,
+# pid_max, somaxconn(4096), conntrack(262144), autogroup(1), page-cluster(3).
+
+# Predictable writeback under ETL bursts (bytes-based, RAM-size independent).
+# Night profile (ukrrs-daynight.sh) raises these to 2G/512M.
+vm.dirty_background_bytes = 268435456
+vm.dirty_bytes = 1073741824
+
+# Survive bursty reclaim when compiles+ETL hit at once. 768M post-upgrade.
+vm.min_free_kbytes = 393216
+
+# Postgres AIO / io_uring era; 9 harness daemons + LSPs watching many repos.
+fs.aio-max-nr = 1048576
+fs.inotify.max_user_watches = 1048576
+fs.inotify.max_user_instances = 512
+
+# 9 accounts x long-lived provider streams + tailscale + docker NAT.
+net.ipv4.ip_local_port_range = 10240 65535
+net.ipv4.tcp_tw_reuse = 1
+
+# LLM turns idle minutes between bursts on live sockets; large SSE/JSON.
+net.ipv4.tcp_slow_start_after_idle = 0
+net.core.rmem_max = 16777216
+net.core.wmem_max = 16777216
+net.ipv4.tcp_rmem = 4096 131072 16777216
+net.ipv4.tcp_wmem = 4096 65536 16777216
+net.core.netdev_max_backlog = 8192
+
+# BBR requires the module: staged modules-load.d/tcp_bbr.conf loads it.
+# If bbr is unavailable, comment the last two lines out (cubic is fine).
+net.ipv4.tcp_congestion_control = bbr
+net.core.default_qdisc = fq
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# UKRRS guest optimizer installer. Review ~/optimize/REPORT.md section 4 first.
+#   sudo RUN=1 ./apply-guest.sh <step>
+# RUN=1 required to mutate anything (default: plan only).
+# daemonjson and desktop additionally require CONFIRM=1 (one-time disruption).
+# Steps: sysctl modules thp slices psi prune nofile noatime daemonjson desktop
+#        daynight all
+set -euo pipefail
+RUN=${RUN:-0}
+CONFIRM=${CONFIRM:-0}
+DESKTOP_USER=${DESKTOP_USER:-reachableceo}
+here=$(cd "$(dirname "$0")" && pwd)
+
+say() { echo "[apply-guest] $*"; }
+die() { echo "[apply-guest] $*" >&2; exit 2; }
+need_root() { [ "$(id -u)" = 0 ] || die "needs root"; }
+gate() {
+  if [ "$RUN" != 1 ]; then say "DRY: would $* (rerun with RUN=1)"; return 1; fi
+  need_root; return 0
+}
+inst() { # src dst mode
+  install -m "$3" "$1" "$2" && say "installed $2"
+}
+
+step_sysctl() {
+  gate "install sysctl.d profile" || return 0
+  inst "$here/60-ukrrs-vm.conf" /etc/sysctl.d/60-ukrrs-vm.conf 0644
+  sysctl --system >/dev/null && say "sysctl applied (bbr lines need the module: step modules)"
+}
+step_modules() {
+  gate "install tcp_bbr module load" || return 0
+  inst "$here/modules-load.d/tcp_bbr.conf" /etc/modules-load.d/tcp_bbr.conf 0644
+  modprobe tcp_bbr 2>/dev/null || say "tcp_bbr not loaded now (will load at boot)"
+}
+step_thp() {
+  gate "install THP madvise unit" || return 0
+  inst "$here/systemd/ukrrs-thp-madvise.service" /etc/systemd/system/ukrrs-thp-madvise.service 0644
+  systemctl daemon-reload
+  systemctl enable --now ukrrs-thp-madvise.service
+  say "THP=$(cat /sys/kernel/mm/transparent_hugepage/enabled)"
+}
+step_slices() {
+  gate "install ukrrs slices" || return 0
+  for s in gateway pmo lsp batch rt; do
+    inst "$here/systemd/ukrrs-$s.slice" "/etc/systemd/system/ukrrs-$s.slice" 0644
+  done
+  systemctl daemon-reload
+  say "slices ready; containers opt in via cgroup_parent=ukrrs-<x>.slice"
+}
+step_psi() {
+  gate "install PSI textfile collector" || return 0
+  inst "$here/ukrrs-psi-textfile.sh" /usr/local/sbin/ukrrs-psi-textfile.sh 0755
+  mkdir -p /var/lib/node_exporter/textfile && chmod 755 /var/lib/node_exporter /var/lib/node_exporter/textfile
+  inst "$here/systemd/ukrrs-psi-textfile.service" /etc/systemd/system/ukrrs-psi-textfile.service 0644
+  inst "$here/systemd/ukrrs-psi-textfile.timer" /etc/systemd/system/ukrrs-psi-textfile.timer 0644
+  systemctl daemon-reload
+  systemctl enable --now ukrrs-psi-textfile.timer
+  say "pressure gauges: /var/lib/node_exporter/textfile/ukrrs_pressure.prom"
+}
+step_prune() {
+  gate "install builder-prune timer" || return 0
+  inst "$here/systemd/ukrrs-builder-prune.service" /etc/systemd/system/ukrrs-builder-prune.service 0644
+  inst "$here/systemd/ukrrs-builder-prune.timer" /etc/systemd/system/ukrrs-builder-prune.timer 0644
+  systemctl daemon-reload
+  systemctl enable --now ukrrs-builder-prune.timer
+}
+step_nofile() {
+  gate "raise DefaultLimitNOFILE" || return 0
+  mkdir -p /etc/systemd/system.conf.d
+  cat > /etc/systemd/system.conf.d/50-ukrrs.conf <<'EOF'
+[Manager]
+DefaultLimitNOFILE=65536:1048576
+EOF
+  systemctl daemon-reload
+  say "DefaultLimitNOFILE raised (new sessions)"
+}
+step_noatime() {
+  gate "add noatime to /, /data1, /data2" || return 0
+  cp /etc/fstab "/etc/fstab.bak.ukrrs.$(date +%s)"
+  for mnt in / /data1 /data2; do
+    opts=$(findmnt -n -o OPTIONS "$mnt") || continue
+    case ",$opts," in *,noatime,*) say "$mnt already noatime"; continue ;; esac
+    mount -o remount,noatime "$mnt"
+    awk -v m="$mnt" 'BEGIN{FS=OFS=" "} $2==m && $3=="ext4" { if ($4 !~ /(^|,)noatime(,|$)/) $4=$4",noatime" } 1' \
+      /etc/fstab > /etc/fstab.ukrrs.new && mv /etc/fstab.ukrrs.new /etc/fstab
+    say "$mnt: remounted noatime + fstab updated"
+  done
+}
+step_daemonjson() {
+  [ "$CONFIRM" = 1 ] || die "daemonjson needs CONFIRM=1: restarts docker ONCE (live-restore protects future restarts)"
+  gate "install daemon.json" || return 0
+  [ -f /etc/docker/daemon.json ] && cp /etc/docker/daemon.json "/etc/docker/daemon.json.bak.ukrrs.$(date +%s)"
+  inst "$here/docker/daemon.json" /etc/docker/daemon.json 0644
+  systemctl restart docker
+  say "docker restarted with new config; check: docker info | grep -E 'Live|Logging'"
+}
+step_desktop() {
+  [ "$CONFIRM" = 1 ] || die "desktop needs CONFIRM=1: enables sddm autologin for $DESKTOP_USER"
+  gate "configure sddm autologin+lock" || return 0
+  mkdir -p /etc/sddm.conf.d
+  cat > /etc/sddm.conf.d/50-ukrrs-autologin.conf <<EOF
+[Autologin]
+User=$DESKTOP_USER
+Session=plasma.desktop
+Relogin=true
+EOF
+  say "sddm autologin configured (UNLOCKED session; autolock deliberately off"
+  say "per 2026-08-31 ruling: instant Jump/iPad re-attach outranks lock)"
+}
+step_daynight() {
+  gate "install day/night profile timers" || return 0
+  inst "$here/ukrrs-daynight.sh" /usr/local/sbin/ukrrs-daynight.sh 0755
+  mkdir -p /etc/ukrrs
+  inst "$here/etc-ukrrs-daynight.conf" /etc/ukrrs/daynight.conf 0644
+  for u in nightprofile.service nightprofile.timer dayprofile.service dayprofile.timer; do
+    inst "$here/systemd/ukrrs-$u" "/etc/systemd/system/ukrrs-$u" 0644
+  done
+  systemctl daemon-reload
+  systemctl enable --now ukrrs-nightprofile.timer ukrrs-dayprofile.timer
+  say "night profile flips at 22:00, day at 05:00; test: ukrrs-daynight.sh night"
+}
+step_all() {
+  step_sysctl; step_modules; step_thp; step_slices; step_psi; step_prune
+  step_nofile; step_noatime; step_daynight
+  if [ "$CONFIRM" = 1 ]; then step_daemonjson; step_desktop
+  else say "skipped daemonjson + desktop (need CONFIRM=1)"; fi
+}
+
+step=${1:-}
+case "$step" in
+  sysctl|modules|thp|slices|psi|prune|nofile|noatime|daemonjson|desktop|daynight|all)
+    "step_$step" ;;
+  *) die "unknown step: $step" ;;
+esac
@@ -0,0 +1,18 @@
+{
+  "live-restore": true,
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "20m",
+    "max-file": "3"
+  },
+  "max-concurrent-downloads": 6,
+  "max-concurrent-uploads": 4,
+  "metrics-addr": "127.0.0.1:9323",
+  "experimental": true,
+  "default-address-pools": [
+    {
+      "base": "172.16.0.0/12",
+      "size": 24
+    }
+  ]
+}
@@ -0,0 +1,11 @@
+# /etc/ukrrs/daynight.conf — overrides for ukrrs-daynight.sh
+# Interim values (8 vCPU / 48G) are the script defaults; only set what differs.
+
+DATA2_DEV=/dev/sdb
+
+# Post-upgrade values (20 vCPU / 128G), uncomment after the VM resize:
+# BATCH_CPUS_DAY=0-15
+# BATCH_CPUS_NIGHT=0-17
+# BATCH_HIGH_DAY=24G
+# BATCH_HIGH_NIGHT=80G
+# Also edit ukrrs-rt.slice AllowedCPUs to 18-19 and account slices (mkacct.sh).
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# UKRRS: create one account lane = cgroup slice + matching user-slice drop-in.
+# usage: mkacct.sh <account> <uid> [mem_high] [mem_max] [allowed_cpus] [tier]
+#
+#   tier=agent (default): CPUWeight=75, pinned to a CPU pool — the 9 PMO/worker
+#     lanes. interim rec: 3G 4.5G 0-5   post-upgrade: 10G 12G 0-15
+#   tier=human: CPUWeight=600, NO cpu restriction, generous memory — the two
+#     human interactive accounts (KDE/CAD/EDA/video via xrdp live in these).
+#     interim rec: 12G 16G all   post-upgrade: 32G 40G all
+#
+# Examples (uids verified 2026-08-31):
+#   sudo RUN=1 ./mkacct.sh reachableceo 1001 12G 16G all human
+#   sudo RUN=1 ./mkacct.sh reachableceo-offstage 1010 12G 16G all human
+#   sudo RUN=1 ./mkacct.sh TSGBOD <uid> 3G 4.5G 0-5 agent
+#
+# RUN=1 installs (needs root); default prints the plan + snippets only.
+set -euo pipefail
+acct=${1:?account}; uid=${2:?uid}
+high=${3:-3G}; max=${4:-4.5G}; cpus=${5:-0-5}; tier=${6:-agent}
+RUN=${RUN:-0}
+
+case "$tier" in
+  human) weight=600; cpuline="" ;;
+  agent) weight=75;  cpuline="AllowedCPUs=$cpus" ;;
+  *) echo "tier must be human or agent" >&2; exit 2 ;;
+esac
+
+slice="/etc/systemd/system/ukrrs-acct-$acct.slice"
+userdrop="/etc/systemd/system/user-$uid.slice.d/50-ukrrs.conf"
+
+cat <<EOF
+plan ($tier tier):
+  $slice
+      CPUWeight=$weight ${cpuline:+$cpuline }MemoryHigh=$high MemoryMax=$max TasksMax=4096
+  $userdrop
+      CPUWeight=$weight MemoryHigh=$high
+
+compose snippet (this account's agent projects):
+  x-ukrrs-acct: &ukrrs_acct
+    cgroup_parent: ukrrs-acct-$acct.slice
+  services:
+    anything: { <<: *ukrrs_acct }
+
+dev.sh one-shot builders belong in the batch pool, not the account slice:
+  docker run --rm --cgroup-parent ukrrs-batch.slice --cpus 4 --memory 4g ...
+EOF
+
+if [ "$RUN" = 1 ]; then
+  [ "$(id -u)" = 0 ] || { echo "RUN=1 needs root" >&2; exit 1; }
+  cat > "$slice" <<EOF
+[Unit]
+Description=UKRRS account lane ($tier): $acct
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+CPUWeight=$weight
+${cpuline}
+MemoryHigh=$high
+MemoryMax=$max
+TasksMax=4096
+EOF
+  mkdir -p "$(dirname "$userdrop")"
+  cat > "$userdrop" <<EOF
+[Slice]
+CPUWeight=$weight
+MemoryHigh=$high
+EOF
+  systemctl daemon-reload
+  # push live: drop-ins alone don't re-apply to already-existing user slices
+  systemctl set-property "user-$uid.slice" CPUWeight="$weight" MemoryHigh="$high"
+  echo "installed + daemon-reload + live-applied ok: $acct ($tier)"
+fi
@@ -0,0 +1,2 @@
+# Load tcp_bbr at boot so sysctl.d/60-ukrrs-vm.conf can set bbr.
+tcp_bbr
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# UKRRS Proxmox control wrapper for harness/human use. HARD RULES baked in:
+#   - vmids listed in PROTECTED are never touched (list THIS VM first).
+#   - graceful shutdown only; no reset, no hard stop, no bare down (house rule).
+# Env:
+#   PVE_LOCAL=1                      run ON the pve host itself (pvesh direct)
+#   or PVE_HOST + PVE_TOKEN          remote, token format: user@realm!tokenid=secret
+#   PVE_NODE                         node name (required for vm ops)
+#   PROTECTED                        required, comma-separated vmids
+# Usage:
+#   proxmox-ctl.sh vms
+#   proxmox-ctl.sh shutdown <vmid>
+#   proxmox-ctl.sh start <vmid>
+#   proxmox-ctl.sh snapshot <vmid> <name>
+# Token setup (on the pve host, see REPORT.md appendix B):
+#   pveum user add ukrrs-infra@pam
+#   pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0
+#   pveum acl modify /pool/<preprod-pool> -user ukrrs-infra@pam -role PVEVMUser
+set -euo pipefail
+: "${PROTECTED:?set PROTECTED=vmid1,vmid2,... (this VM must be in the list)}"
+
+pvesh_() {
+  if [ "${PVE_LOCAL:-0}" = 1 ]; then
+    command pvesh "$@"
+  else
+    : "${PVE_HOST:?}" "${PVE_TOKEN:?}"
+    command pvesh --host "$PVE_HOST" --api-token "$PVE_TOKEN" "$@"
+  fi
+}
+
+guard() {
+  case ",$PROTECTED," in
+    *",$1,"*) echo "REFUSED: vmid $1 is PROTECTED" >&2; exit 3 ;;
+  esac
+}
+
+cmd=${1:-}; shift || true
+case "$cmd" in
+  vms)
+    pvesh_ get /cluster/resources --type vm ;;
+  shutdown)
+    [ $# = 1 ] || { echo "usage: $0 shutdown <vmid>" >&2; exit 2; }
+    guard "$1"; : "${PVE_NODE:?}"
+    pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/shutdown" --timeout 120 ;;
+  start)
+    [ $# = 1 ] || { echo "usage: $0 start <vmid>" >&2; exit 2; }
+    guard "$1"; : "${PVE_NODE:?}"
+    pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/start" ;;
+  snapshot)
+    [ $# = 2 ] || { echo "usage: $0 snapshot <vmid> <name>" >&2; exit 2; }
+    guard "$1"; : "${PVE_NODE:?}"
+    pvesh_ create "/nodes/$PVE_NODE/qemu/$1/snapshot" snapname="$2" ;;
+  *)
+    echo "usage: $0 vms|shutdown <vmid>|start <vmid>|snapshot <vmid> <name>" >&2
+    exit 2 ;;
+esac
@@ -0,0 +1,14 @@
+[Unit]
+Description=UKRRS batch pool: dev.sh builders, ETL, doc builds, background agents
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Day bias: lowest weight, restricted to pool A. The night profile timer
+# (ukrrs-nightprofile.service) raises weight/MemoryHigh and widens AllowedCPUs
+# via `systemctl set-property --runtime`; reboot lands back in day mode.
+# IOWeight deliberately absent: inert under the `none` I/O scheduler.
+CPUWeight=25
+AllowedCPUs=0-5
+MemoryHigh=12884901888
+MemoryMax=17179869184
+TasksMax=16384
@@ -0,0 +1,7 @@
+[Unit]
+Description=UKRRS: weekly docker build-cache prune (25G ceiling)
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/docker builder prune --force --keep-storage 25GB
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS: build-cache prune, Sunday night window
+
+[Timer]
+OnCalendar=Sun *-*-* 22:30:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,6 @@
+[Unit]
+Description=UKRRS: switch back to day profile (interactive bias)
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ukrrs-daynight.sh day
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS: day profile at 05:00
+
+[Timer]
+OnCalendar=*-*-* 05:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,12 @@
+[Unit]
+Description=ukrrs gateway ensure-up (tailscale port race + live-restore endpoint loss)
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/ukrrs-gateway-ensure.sh
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,12 @@
+[Unit]
+Description=UKRRS gateway stack: LLM traffic + accounting postgres (prod+beta)
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Latency-sensitive: all provider traffic + spend accounting. Weight beats any
+# single account even under total compile storm (see REPORT section 2 math).
+CPUWeight=900
+# Hard floor so accounts can never squeeze the accounting DB into reclaim.
+MemoryMin=2147483648
+MemoryHigh=12884901888
+TasksMax=infinity
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS LSP fleet + MCP connector tier (mostly idle, bursty reads)
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+CPUWeight=150
+MemoryHigh=6442450944
+MemoryMax=8589934592
+TasksMax=8192
@@ -0,0 +1,12 @@
+[Unit]
+Description=Enable virtio NIC multiqueue (host offers 4 queues on ens18, 2 on ens19)
+After=network-pre.target
+Before=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c '/usr/sbin/ethtool -L ens18 combined 4 || true; /usr/sbin/ethtool -L ens19 combined 2 || true; true'
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,6 @@
+[Unit]
+Description=UKRRS: switch to night profile (batch burn window 22:00-05:00)
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ukrrs-daynight.sh night
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS: night profile at 22:00
+
+[Timer]
+OnCalendar=*-*-* 22:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,11 @@
+[Unit]
+Description=UKRRS PMO dispatch loops (one per account): control plane
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Dispatcher must never starve behind worker storms: cheap but weighted above
+# batch and idle accounts. Night profile leaves this untouched.
+CPUWeight=200
+MemoryHigh=1073741824
+MemoryMax=1610612736
+TasksMax=512
@@ -0,0 +1,7 @@
+[Unit]
+Description=UKRRS: PSI textfile collector tick
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ukrrs-psi-textfile.sh
@@ -0,0 +1,10 @@
+[Unit]
+Description=UKRRS: PSI textfile collector (15s)
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=15
+AccuracySec=5
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,12 @@
+[Unit]
+Description=UKRRS realtime pool: SDR DSP, video encode, latency-hard work
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Exclusive-ish by exclusion: account+batch slices are pinned OFF these cores,
+# so anything placed here keeps headroom even under full compile storm.
+# Interim 8 vCPU: 6-7. Post-upgrade 20 vCPU: 18-19 (edit after resize).
+CPUWeight=10000
+AllowedCPUs=6-7
+MemoryHigh=2147483648
+MemoryMax=4294967296
@@ -0,0 +1,11 @@
+[Unit]
+Description=UKRRS: transparent hugepages to madvise (postgres-friendly)
+After=multi-user.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# UKRRS day/night resource profile (REPORT.md section 5.5).
+# Runtime-only changes (set-property --runtime + sysctl): a reboot always
+# lands safely in day mode (slice unit files carry the day defaults).
+set -euo pipefail
+mode=${1:?usage: ukrrs-daynight.sh day|night}
+CONF=/etc/ukrrs/daynight.conf
+[ -r "$CONF" ] && . "$CONF"
+
+: "${DATA2_DEV:=/dev/sdb}"
+: "${BATCH_WEIGHT_DAY:=25}";        : "${BATCH_WEIGHT_NIGHT:=400}"
+: "${BATCH_CPUS_DAY:=0-5}";         : "${BATCH_CPUS_NIGHT:=0-6}"
+: "${BATCH_HIGH_DAY:=12G}";         : "${BATCH_HIGH_NIGHT:=30G}"
+: "${GW_WEIGHT_DAY:=900}";          : "${GW_WEIGHT_NIGHT:=500}"
+: "${DATA2_RA_DAY:=256}";           : "${DATA2_RA_NIGHT:=2048}"
+: "${DIRTY_DAY:=1073741824}";       : "${DIRTY_NIGHT:=2147483648}"
+: "${DIRTY_BG_DAY:=268435456}";     : "${DIRTY_BG_NIGHT:=536870912}"
+
+log() { echo "[ukrrs-daynight] $*"; }
+setprop() { systemctl set-property --runtime "$@"; }
+setra() { blockdev --setra "$1" "$2" 2>/dev/null || log "readahead skip: $2"; }
+
+case "$mode" in
+night)
+  setprop ukrrs-batch.slice \
+    CPUWeight="$BATCH_WEIGHT_NIGHT" \
+    AllowedCPUs="$BATCH_CPUS_NIGHT" \
+    MemoryHigh="$BATCH_HIGH_NIGHT"
+  setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_NIGHT"
+  setra "$DATA2_RA_NIGHT" "$DATA2_DEV"; setra "$DATA2_RA_NIGHT" "${DATA2_DEV}1"
+  sysctl -q -w vm.dirty_bytes="$DIRTY_NIGHT" vm.dirty_background_bytes="$DIRTY_BG_NIGHT"
+  ;;
+day)
+  setprop ukrrs-batch.slice \
+    CPUWeight="$BATCH_WEIGHT_DAY" \
+    AllowedCPUs="$BATCH_CPUS_DAY" \
+    MemoryHigh="$BATCH_HIGH_DAY"
+  setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_DAY"
+  setra "$DATA2_RA_DAY" "$DATA2_DEV"; setra "$DATA2_RA_DAY" "${DATA2_DEV}1"
+  sysctl -q -w vm.dirty_bytes="$DIRTY_DAY" vm.dirty_background_bytes="$DIRTY_BG_DAY"
+  ;;
+*)
+  echo "unknown mode: $mode" >&2; exit 2 ;;
+esac
+log "profile $mode applied $(date -Is)"
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Boots are racy for the gateway containers (both root-caused 2026-08-31):
+#  - prod binds :4000/:9090 to the tailscale IP; if dockerd wins the race over
+#    tailscaled, the bind fails and docker NEVER retries failed starts;
+#  - live-restore can drop a container's network endpoint across a boot,
+#    leaving it started with the host resolver (gateway-db unresolvable,
+#    crash loop). A plain `docker start` does NOT reattach the endpoint.
+# Waits for the tailscale IP, then recreates any lane not actually serving.
+# Idempotent; recreating is safe at boot time (nothing in flight).
+set -u
+GW_DIR=/home/reachableceo/projects/ukrrs/gateway
+TS_IP=100.101.187.119
+
+for _ in $(seq 1 60); do
+  /usr/sbin/ip -o addr | grep -q "$TS_IP/" && break
+  sleep 2
+done
+
+ensure_lane() { # name host:port project compose_file
+  local name=$1 hostport=$2 project=$3 file=$4
+  if timeout 5 bash -c "</dev/tcp/$hostport" 2>/dev/null; then
+    echo "$name: serving on $hostport"
+    return 0
+  fi
+  echo "$name: NOT serving on $hostport; recreating"
+  docker compose -p "$project" -f "$GW_DIR/$file" up -d --no-deps --force-recreate gateway
+}
+
+ensure_lane prod  "$TS_IP:4000"    mopac-gateway       docker-compose.yml
+ensure_lane beta "127.0.0.1:4002" mopac-gateway-beta  compose.beta.yaml
+exit 0
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Per-slice + host PSI (pressure stall info) avg60 -> node_exporter textfile.
+# Consumed by the existing mopac-harness prometheus; gateway may also read it.
+set -u
+OUT_DIR=${UKRRS_TEXTFILE_DIR:-/var/lib/node_exporter/textfile}
+CGROOT=${UKRRS_CGROOT:-/sys/fs/cgroup}
+mkdir -p "$OUT_DIR"
+tmp=$(mktemp "$OUT_DIR/.ukrrs.XXXXXX")
+
+some60() { # file -> prints avg60 value or nothing
+  awk '$1=="some"{for(i=2;i<=NF;i++) if($i ~ /^avg60=/){sub("avg60=","",$i); print $i; exit}}' "$1" 2>/dev/null
+}
+
+emit() { # name cgroup_path
+  local res v
+  for res in cpu memory io; do
+    [ -r "$2/$res.pressure" ] || continue
+    v=$(some60 "$2/$res.pressure")
+    [ -n "${v:-}" ] && echo "ukrrs_slice_pressure_some60{slice=\"$1\",res=\"$res\"} $v"
+  done
+}
+
+echo "# HELP ukrrs_slice_pressure_some60 PSI some avg60 (percent) for ukrrs slices" >>"$tmp"
+echo "# TYPE ukrrs_slice_pressure_some60 gauge" >>"$tmp"
+for cg in "$CGROOT"/system.slice/ukrrs-*.slice; do
+  [ -d "$cg" ] && emit "${cg##*/}" "$cg"
+done
+[ -d "$CGROOT/user.slice" ] && emit "user.slice" "$CGROOT/user.slice"
+
+echo "# HELP ukrrs_host_pressure_some60 PSI some avg60 (percent) host-wide" >>"$tmp"
+echo "# TYPE ukrrs_host_pressure_some60 gauge" >>"$tmp"
+for res in cpu memory io; do
+  [ -r "/proc/pressure/$res" ] || continue
+  v=$(some60 "/proc/pressure/$res")
+  [ -n "${v:-}" ] && echo "ukrrs_host_pressure_some60{res=\"$res\"} $v"
+done
+
+mv "$tmp" "$OUT_DIR/ukrrs_pressure.prom"
+chmod 0644 "$OUT_DIR/ukrrs_pressure.prom"