feat: utilization census script for daily monitor automation [#731]

%-sorted usage-vs-limit view across all apps + 24h memcg-OOM evidence. Backs the founder-directed monitor/adjust-over-time loop. Meat: https://projects.knownelement.com/issues/731#note-3929
2026-09-02 20:04:42 -05:00
parent 1267e60e8e
commit 23d71761c8
1 changed files with 40 additions and 0 deletions
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# perf/memcensus-util.sh — live per-app memory utilization census (all apps,
+# %-sorted) + 24h memcg-OOM evidence. Read-only. [#685/#731]
+#
+# Run (from a checkout):
+#   VM_IP=my.knownelement.com VM_USER=root timeout 120 bash tests/remote.sh vm-file perf/memcensus-util.sh
+# Needs: docker + awk on the host. Pairs with the daily monitor automation
+# (founder directive 2026-09-02: monitor/adjust over time — bump >80% utilized
+# apps, never decrease, never touch bi/platform services).
+
+set -u
+
+TMP=$(mktemp -d) || exit 1
+trap 'rm -rf "$TMP"' EXIT
+
+docker stats --no-stream --format "{{.Name}} {{.MemUsage}}" > "$TMP/usage"
+docker ps --format "{{.Names}}" | while read -r n; do
+    lim=$(docker inspect "$n" --format '{{.HostConfig.Memory}}' 2>/dev/null)
+    fq=$(docker inspect "$n" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null \
+        | grep -m1 '^CLOUDRON_APP_DOMAIN=' | cut -d= -f2)
+    # shellcheck disable=SC2312  # pipe to env-print is the only way to read container env here
+    [ -n "$fq" ] && printf '%s|%s|%s\n' "$n" "$fq" "$lim"
+done > "$TMP/limits"
+
+awk 'NR==FNR {
+    gsub(/,/,"");
+    if ($2 ~ /G/) u[$1] = $2 * 1024; else u[$1] = $2 + 0;
+    next
+  }
+  {
+    n = split($0, f, "|")
+    lim = f[3] / 1048576; use = u[f[1]] + 0
+    if (lim >= 200 && use > 0)
+      printf "%5.1f%% | %7.0fMB used | %6.0fMB limit | %s\n", use/lim*100, use, lim, f[2]
+  }' "$TMP/usage" "$TMP/limits" | sort -rn | head -30
+
+echo "== memcg OOM events, last 24h =="
+journalctl --since "-24 hours" --no-pager 2>/dev/null \
+    | grep -i "memory cgroup out of memory" | tail -6
+echo "(count: $(journalctl --since '-24 hours' --no-pager 2>/dev/null | grep -ci 'memory cgroup out of memory'))"