#!/usr/bin/env bash
# perf/memcensus-util.sh — live per-app memory utilization census (all apps,
# %-sorted) + 24h memcg-OOM evidence. Read-only. [#685/#731]
#
# Run (from a checkout):
#   VM_IP=my.knownelement.com VM_USER=root timeout 120 bash tests/remote.sh vm-file perf/memcensus-util.sh
# Needs: docker + awk on the host. Pairs with the daily monitor automation
# (founder directive 2026-09-02: monitor/adjust over time — bump >80% utilized
# apps, never decrease, never touch bi/platform services).

set -u

TMP=$(mktemp -d) || exit 1
trap 'rm -rf "$TMP"' EXIT

docker stats --no-stream --format "{{.Name}} {{.MemUsage}}" > "$TMP/usage"
docker ps --format "{{.Names}}" | while read -r n; do
    lim=$(docker inspect "$n" --format '{{.HostConfig.Memory}}' 2>/dev/null)
    fq=$(docker inspect "$n" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null \
        | grep -m1 '^CLOUDRON_APP_DOMAIN=' | cut -d= -f2)
    # shellcheck disable=SC2312  # pipe to env-print is the only way to read container env here
    [ -n "$fq" ] && printf '%s|%s|%s\n' "$n" "$fq" "$lim"
done > "$TMP/limits"

awk 'NR==FNR {
    gsub(/,/,"");
    if ($2 ~ /G/) u[$1] = $2 * 1024; else u[$1] = $2 + 0;
    next
  }
  {
    n = split($0, f, "|")
    lim = f[3] / 1048576; use = u[f[1]] + 0
    if (lim >= 200 && use > 0)
      printf "%5.1f%% | %7.0fMB used | %6.0fMB limit | %s\n", use/lim*100, use, lim, f[2]
  }' "$TMP/usage" "$TMP/limits" | sort -rn | head -30

echo "== memcg OOM events, last 24h =="
journalctl --since "-24 hours" --no-pager 2>/dev/null \
    | grep -i "memory cgroup out of memory" | tail -6
echo "(count: $(journalctl --since '-24 hours' --no-pager 2>/dev/null | grep -ci 'memory cgroup out of memory'))"