#!/usr/bin/env bash
# perf/memcensus-util.sh — live per-app resource census: memory utilization
# (%-sorted), per-container CPU, host disk + docker space, and 24h
# memcg-OOM evidence. Read-only. [#685/#731/#757]
#
# Run (from a checkout):
#   VM_IP=my.knownelement.com VM_USER=root timeout 300 bash tests/remote.sh vm-file perf/memcensus-util.sh
# Needs: docker + awk on the host. Pairs with the daily monitor automation
# (ratified policy t/317: memory >80%/OOM auto-bump under standing
# authorization; CPU + disk are REPORT-ONLY — human-required class,
# founder directive 2026-09-03, #757).

set -u

TMP=$(mktemp -d) || exit 1
trap 'rm -rf "$TMP"' EXIT

docker stats --no-stream --format "{{.Name}} {{.CPUPerc}} {{.MemUsage}}" > "$TMP/stats"
docker ps --format "{{.Names}}" | while read -r n; do
    lim=$(docker inspect "$n" --format '{{.HostConfig.Memory}}' 2>/dev/null)
    fq=$(docker inspect "$n" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null \
        | grep -m1 '^CLOUDRON_APP_DOMAIN=' | cut -d= -f2)
    # shellcheck disable=SC2312  # pipe to env-print is the only way to read container env here
    [ -n "$fq" ] && printf '%s|%s|%s\n' "$n" "$fq" "$lim"
done > "$TMP/limits"

echo "== memory utilization (apps with >=200MB limit, %-sorted) =="
awk 'NR==FNR {
    gsub(/,/,"", $3);
    if ($3 ~ /G/) u[$1] = $3 * 1024; else u[$1] = $3 + 0;
    next
  }
  {
    n = split($0, f, "|")
    lim = f[3] / 1048576; use = u[f[1]] + 0
    if (lim >= 200 && use > 0)
      printf "%5.1f%% | %7.0fMB used | %6.0fMB limit | %s\n", use/lim*100, use, lim, f[2]
  }' "$TMP/stats" "$TMP/limits" | sort -rn | head -30

echo "== cpu per container (top 12, incl. platform; report-only) =="
awk 'NR==FNR { n = split($0, f, "|"); fq[f[1]] = f[2]; next }
  { sub(/%/, "", $2)
    name = ($1 in fq) ? fq[$1] : $1
    printf "%8.2f%% | %s\n", $2, name }' "$TMP/limits" "$TMP/stats" \
    | sort -rn | head -12

echo "== disk (report-only; >=80% used = human-required flag per t/317) =="
df -h / | awk 'NR==2 {
    printf "root fs: %s used of %s (%s used)\n", $3, $2, $5
    if ($5 + 0 >= 80) print "WARN: root fs >=80% used — human-required, propose only"
  }'
docker system df
echo "-- top app-data dirs (MB; du capped at 60s — partial ranking if capped) --"
timeout 60 du -sm /home/yellowtent/appsdata/* 2>/dev/null \
    | awk '{ sub(/.*appsdata\//, "", $2); print }' > "$TMP/du"
awk 'NR==FNR { n = split($0, f, "|"); fq[f[1]] = f[2]; next }
  { name = ($2 in fq) ? fq[$2] : $2
    printf "%8dMB | %s\n", $1, name }' "$TMP/limits" "$TMP/du" \
    | sort -rn | head -15

echo "== memcg OOM events, last 24h =="
journalctl --since "-24 hours" --no-pager 2>/dev/null \
    | grep -i "memory cgroup out of memory" | tail -6
echo "(count: $(journalctl --since '-24 hours' --no-pager 2>/dev/null | grep -ci 'memory cgroup out of memory'))"

echo "== platform health (report; inotify re-apply per #755 is the only auto-fix) =="
printf 'inotify limits: instances %s, watches %s\n' \
    "$(cat /proc/sys/fs/inotify/max_user_instances)" \
    "$(cat /proc/sys/fs/inotify/max_user_watches)"
inst=$(find /proc/[0-9]*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
watches=$(cat /proc/[0-9]*/fdinfo/* 2>/dev/null | grep -c '^inotify wd')
printf 'inotify in use: %s instances, %s watches\n' "$inst" "$watches"
for svc in docker containerd; do
    printf '%s up since: %s\n' "$svc" \
        "$(systemctl show -p ActiveEnterTimestamp --value "$svc" 2>/dev/null)"
done
printf 'dockerd err/warn lines 24h: %s\n' \
    "$(journalctl -u docker --since '-24 hours' --no-pager 2>/dev/null | grep -ciE 'error|fail')"
printf 'pg too-many-clients 24h: %s\n' \
    "$(docker logs postgresql --since 24h 2>&1 | grep -c 'too many clients')"