diff --git a/STATUS.md b/STATUS.md index dbf5f49..d2be30d 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,4 +1,6 @@ # STATUS.md > Operations status lives on Discourse: https://community.turnsys.com/t/297 -> This file is a pointer stub. Active ticket: [#769](https://projects.knownelement.com/issues/769) (repo split). +> This file is a pointer stub. + +- [#826](https://projects.knownelement.com/issues/826) — KNELPerf v2 program · monitoring fleet-wide 09-06 (CR 25), toolbox on 7/7 nodes, first baselines in `data/baselines/` · next: placement scorecard + day/night UAT diff --git a/data/baselines/pfv-tsys1/20260906-192237/MANIFEST b/data/baselines/pfv-tsys1/20260906-192237/MANIFEST new file mode 100644 index 0000000..ed511bf --- /dev/null +++ b/data/baselines/pfv-tsys1/20260906-192237/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys1 20260906-192237 diff --git a/data/baselines/pfv-tsys1/20260906-192237/bench-run.tsv b/data/baselines/pfv-tsys1/20260906-192237/bench-run.tsv new file mode 100644 index 0000000..ea64e5b --- /dev/null +++ b/data/baselines/pfv-tsys1/20260906-192237/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys1|meta|stamp +BENCH|pfv-tsys1|meta|path +BENCH|pfv-tsys1|meta|runtime +BENCH|pfv-tsys1|meta|size +BENCH|pfv-tsys1|meta|load +BENCH|pfv-tsys1|psi_io_some_avg10|15.30 +BENCH|pfv-tsys1|psi_cpu_some_avg10|30.66 +BENCH|pfv-tsys1|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys1|meta|bench_device +BENCH|pfv-tsys1|randread4k|iops=48.480608|bw_kbps=193|p99_ms=196.083712 +BENCH|pfv-tsys1|randwrite4k|iops=124.287571|bw_kbps=497|p99_ms=143.654912 +BENCH|pfv-tsys1|seqread1m|iops=53.994601|bw_kbps=55290|p99_ms=187.695104 +BENCH|pfv-tsys1|seqwrite1m|iops=59.444056|bw_kbps=60870|p99_ms=183.5008 +BENCH|pfv-tsys1|meta|load_end +BENCH|pfv-tsys1|psi_io_some_avg10_end|35.56 +BENCH|pfv-tsys1|meta|end diff --git a/data/baselines/pfv-tsys1/20260906-192237/bench.err b/data/baselines/pfv-tsys1/20260906-192237/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys1/20260906-192237/cpu-bench.err b/data/baselines/pfv-tsys1/20260906-192237/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys1/20260906-192237/cpu-bench.tsv b/data/baselines/pfv-tsys1/20260906-192237/cpu-bench.tsv new file mode 100644 index 0000000..fce6f9c --- /dev/null +++ b/data/baselines/pfv-tsys1/20260906-192237/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor performance - +nproc 8 - +openssl_aes-256-cbc_1t 581661.52k k/s +openssl_aes-256-cbc_8t 3891114.80k k/s +openssl_sha256_1t 427686.49k k/s +openssl_sha256_8t 1594156.65k k/s +sysbench_cpu absent - +psi_cpu_avg60 22.55 % +psi_memory_avg60 0.00 % +psi_io_avg60 16.97 % diff --git a/data/baselines/pfv-tsys3/20260906-192801/MANIFEST b/data/baselines/pfv-tsys3/20260906-192801/MANIFEST new file mode 100644 index 0000000..774502b --- /dev/null +++ b/data/baselines/pfv-tsys3/20260906-192801/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys3 20260906-192801 diff --git a/data/baselines/pfv-tsys3/20260906-192801/bench-run.tsv b/data/baselines/pfv-tsys3/20260906-192801/bench-run.tsv new file mode 100644 index 0000000..4fd8c69 --- /dev/null +++ b/data/baselines/pfv-tsys3/20260906-192801/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys3|meta|stamp +BENCH|pfv-tsys3|meta|path +BENCH|pfv-tsys3|meta|runtime +BENCH|pfv-tsys3|meta|size +BENCH|pfv-tsys3|meta|load +BENCH|pfv-tsys3|psi_io_some_avg10|0.00 +BENCH|pfv-tsys3|psi_cpu_some_avg10|4.45 +BENCH|pfv-tsys3|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys3|meta|bench_device +BENCH|pfv-tsys3|randread4k|iops=12026.998650|bw_kbps=48107|p99_ms=0.1152 +BENCH|pfv-tsys3|randwrite4k|iops=56757.450000|bw_kbps=227029|p99_ms=0.035072 +BENCH|pfv-tsys3|seqread1m|iops=1220.738963|bw_kbps=1250036|p99_ms=1.028096 +BENCH|pfv-tsys3|seqwrite1m|iops=1489.325534|bw_kbps=1525069|p99_ms=0.774144 +BENCH|pfv-tsys3|meta|load_end +BENCH|pfv-tsys3|psi_io_some_avg10_end|18.52 +BENCH|pfv-tsys3|meta|end diff --git a/data/baselines/pfv-tsys3/20260906-192801/bench.err b/data/baselines/pfv-tsys3/20260906-192801/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys3/20260906-192801/cpu-bench.err b/data/baselines/pfv-tsys3/20260906-192801/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys3/20260906-192801/cpu-bench.tsv b/data/baselines/pfv-tsys3/20260906-192801/cpu-bench.tsv new file mode 100644 index 0000000..be6b125 --- /dev/null +++ b/data/baselines/pfv-tsys3/20260906-192801/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor performance - +nproc 8 - +openssl_aes-256-cbc_1t 986305.32k k/s +openssl_aes-256-cbc_8t 7198685.59k k/s +openssl_sha256_1t 452144.28k k/s +openssl_sha256_8t 1821861.48k k/s +sysbench_cpu absent - +psi_cpu_avg60 3.24 % +psi_memory_avg60 0.00 % +psi_io_avg60 0.00 % diff --git a/data/baselines/pfv-tsys4/20260906-193846/MANIFEST b/data/baselines/pfv-tsys4/20260906-193846/MANIFEST new file mode 100644 index 0000000..23fe4a5 --- /dev/null +++ b/data/baselines/pfv-tsys4/20260906-193846/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys4 20260906-193846 diff --git a/data/baselines/pfv-tsys4/20260906-193846/bench-run.tsv b/data/baselines/pfv-tsys4/20260906-193846/bench-run.tsv new file mode 100644 index 0000000..37b406a --- /dev/null +++ b/data/baselines/pfv-tsys4/20260906-193846/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys4|meta|stamp +BENCH|pfv-tsys4|meta|path +BENCH|pfv-tsys4|meta|runtime +BENCH|pfv-tsys4|meta|size +BENCH|pfv-tsys4|meta|load +BENCH|pfv-tsys4|psi_io_some_avg10|59.62 +BENCH|pfv-tsys4|psi_cpu_some_avg10|4.54 +BENCH|pfv-tsys4|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys4|meta|bench_device +BENCH|pfv-tsys4|randread4k|iops=52.536866|bw_kbps=210|p99_ms=112.72192 +BENCH|pfv-tsys4|randwrite4k|iops=151.366748|bw_kbps=605|p99_ms=62.652416 +BENCH|pfv-tsys4|seqread1m|iops=41.277297|bw_kbps=42267|p99_ms=112.72192 +BENCH|pfv-tsys4|seqwrite1m|iops=50.805205|bw_kbps=52024|p99_ms=76.02176 +BENCH|pfv-tsys4|meta|load_end +BENCH|pfv-tsys4|psi_io_some_avg10_end|88.30 +BENCH|pfv-tsys4|meta|end diff --git a/data/baselines/pfv-tsys4/20260906-193846/bench.err b/data/baselines/pfv-tsys4/20260906-193846/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys4/20260906-193846/cpu-bench.err b/data/baselines/pfv-tsys4/20260906-193846/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys4/20260906-193846/cpu-bench.tsv b/data/baselines/pfv-tsys4/20260906-193846/cpu-bench.tsv new file mode 100644 index 0000000..78ab261 --- /dev/null +++ b/data/baselines/pfv-tsys4/20260906-193846/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor performance - +nproc 8 - +openssl_aes-256-cbc_1t 614414.75k k/s +openssl_aes-256-cbc_8t 4687442.74k k/s +openssl_sha256_1t 482344.63k k/s +openssl_sha256_8t 1882695.27k k/s +sysbench_cpu absent - +psi_cpu_avg60 3.15 % +psi_memory_avg60 0.00 % +psi_io_avg60 62.40 % diff --git a/data/baselines/pfv-tsys5/20260906-194410/MANIFEST b/data/baselines/pfv-tsys5/20260906-194410/MANIFEST new file mode 100644 index 0000000..c9b5bfd --- /dev/null +++ b/data/baselines/pfv-tsys5/20260906-194410/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys5 20260906-194410 diff --git a/data/baselines/pfv-tsys5/20260906-194410/bench-run.tsv b/data/baselines/pfv-tsys5/20260906-194410/bench-run.tsv new file mode 100644 index 0000000..dedc6a4 --- /dev/null +++ b/data/baselines/pfv-tsys5/20260906-194410/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys5|meta|stamp +BENCH|pfv-tsys5|meta|path +BENCH|pfv-tsys5|meta|runtime +BENCH|pfv-tsys5|meta|size +BENCH|pfv-tsys5|meta|load +BENCH|pfv-tsys5|psi_io_some_avg10|15.30 +BENCH|pfv-tsys5|psi_cpu_some_avg10|64.07 +BENCH|pfv-tsys5|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys5|meta|bench_device +BENCH|pfv-tsys5|randread4k|iops=126.893655|bw_kbps=507|p99_ms=27.131904 +BENCH|pfv-tsys5|randwrite4k|iops=352.932353|bw_kbps=1411|p99_ms=10.15808 +BENCH|pfv-tsys5|seqread1m|iops=99.965012|bw_kbps=102364|p99_ms=49.545216 +BENCH|pfv-tsys5|seqwrite1m|iops=114.277145|bw_kbps=117019|p99_ms=28.442624 +BENCH|pfv-tsys5|meta|load_end +BENCH|pfv-tsys5|psi_io_some_avg10_end|30.96 +BENCH|pfv-tsys5|meta|end diff --git a/data/baselines/pfv-tsys5/20260906-194410/bench.err b/data/baselines/pfv-tsys5/20260906-194410/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys5/20260906-194410/cpu-bench.err b/data/baselines/pfv-tsys5/20260906-194410/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys5/20260906-194410/cpu-bench.tsv b/data/baselines/pfv-tsys5/20260906-194410/cpu-bench.tsv new file mode 100644 index 0000000..8a178b2 --- /dev/null +++ b/data/baselines/pfv-tsys5/20260906-194410/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor none - +nproc 8 - +openssl_aes-256-cbc_1t 461596.26k k/s +openssl_aes-256-cbc_8t 1970275.94k k/s +openssl_sha256_1t 149760.92k k/s +openssl_sha256_8t 523591.15k k/s +sysbench_cpu absent - +psi_cpu_avg60 56.95 % +psi_memory_avg60 0.00 % +psi_io_avg60 10.86 % diff --git a/data/baselines/pfv-tsys6/20260906-194935/MANIFEST b/data/baselines/pfv-tsys6/20260906-194935/MANIFEST new file mode 100644 index 0000000..4429046 --- /dev/null +++ b/data/baselines/pfv-tsys6/20260906-194935/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys6 20260906-194935 diff --git a/data/baselines/pfv-tsys6/20260906-194935/bench-run.tsv b/data/baselines/pfv-tsys6/20260906-194935/bench-run.tsv new file mode 100644 index 0000000..0faa92d --- /dev/null +++ b/data/baselines/pfv-tsys6/20260906-194935/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys6|meta|stamp +BENCH|pfv-tsys6|meta|path +BENCH|pfv-tsys6|meta|runtime +BENCH|pfv-tsys6|meta|size +BENCH|pfv-tsys6|meta|load +BENCH|pfv-tsys6|psi_io_some_avg10|10.45 +BENCH|pfv-tsys6|psi_cpu_some_avg10|13.39 +BENCH|pfv-tsys6|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys6|meta|bench_device +BENCH|pfv-tsys6|randread4k|iops=90.654673|bw_kbps=362|p99_ms=18.743296 +BENCH|pfv-tsys6|randwrite4k|iops=1295.538575|bw_kbps=5182|p99_ms=1.368064 +BENCH|pfv-tsys6|seqread1m|iops=2.286282|bw_kbps=2341|p99_ms=767.557632 +BENCH|pfv-tsys6|seqwrite1m|iops=29.959555|bw_kbps=30678|p99_ms=45.8752 +BENCH|pfv-tsys6|meta|load_end +BENCH|pfv-tsys6|psi_io_some_avg10_end|43.01 +BENCH|pfv-tsys6|meta|end diff --git a/data/baselines/pfv-tsys6/20260906-194935/bench.err b/data/baselines/pfv-tsys6/20260906-194935/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys6/20260906-194935/cpu-bench.err b/data/baselines/pfv-tsys6/20260906-194935/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys6/20260906-194935/cpu-bench.tsv b/data/baselines/pfv-tsys6/20260906-194935/cpu-bench.tsv new file mode 100644 index 0000000..eb6a649 --- /dev/null +++ b/data/baselines/pfv-tsys6/20260906-194935/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor performance - +nproc 16 - +openssl_aes-256-cbc_1t 175503.96k k/s +openssl_aes-256-cbc_16t 1498416.74k k/s +openssl_sha256_1t 196991.77k k/s +openssl_sha256_16t 1409610.55k k/s +sysbench_cpu absent - +psi_cpu_avg60 9.59 % +psi_memory_avg60 0.00 % +psi_io_avg60 16.25 % diff --git a/data/baselines/pfv-tsys7/20260906-195458/MANIFEST b/data/baselines/pfv-tsys7/20260906-195458/MANIFEST new file mode 100644 index 0000000..89968fc --- /dev/null +++ b/data/baselines/pfv-tsys7/20260906-195458/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys7 20260906-195458 diff --git a/data/baselines/pfv-tsys7/20260906-195458/bench-run.tsv b/data/baselines/pfv-tsys7/20260906-195458/bench-run.tsv new file mode 100644 index 0000000..1bcbfaf --- /dev/null +++ b/data/baselines/pfv-tsys7/20260906-195458/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys7|meta|stamp +BENCH|pfv-tsys7|meta|path +BENCH|pfv-tsys7|meta|runtime +BENCH|pfv-tsys7|meta|size +BENCH|pfv-tsys7|meta|load +BENCH|pfv-tsys7|psi_io_some_avg10|8.77 +BENCH|pfv-tsys7|psi_cpu_some_avg10|12.66 +BENCH|pfv-tsys7|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys7|meta|bench_device +BENCH|pfv-tsys7|randread4k|iops=83.729068|bw_kbps=334|p99_ms=20.578304 +BENCH|pfv-tsys7|randwrite4k|iops=1384.680766|bw_kbps=5538|p99_ms=1.417216 +BENCH|pfv-tsys7|seqread1m|iops=1.846632|bw_kbps=1890|p99_ms=943.7184 +BENCH|pfv-tsys7|seqwrite1m|iops=36.526258|bw_kbps=37402|p99_ms=30.277632 +BENCH|pfv-tsys7|meta|load_end +BENCH|pfv-tsys7|psi_io_some_avg10_end|30.73 +BENCH|pfv-tsys7|meta|end diff --git a/data/baselines/pfv-tsys7/20260906-195458/bench.err b/data/baselines/pfv-tsys7/20260906-195458/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys7/20260906-195458/cpu-bench.err b/data/baselines/pfv-tsys7/20260906-195458/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys7/20260906-195458/cpu-bench.tsv b/data/baselines/pfv-tsys7/20260906-195458/cpu-bench.tsv new file mode 100644 index 0000000..17db8ec --- /dev/null +++ b/data/baselines/pfv-tsys7/20260906-195458/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor performance - +nproc 24 - +openssl_aes-256-cbc_1t 400782.52k k/s +openssl_aes-256-cbc_24t 9105252.35k k/s +openssl_sha256_1t 273898.45k k/s +openssl_sha256_24t 3317389.72k k/s +sysbench_cpu absent - +psi_cpu_avg60 8.84 % +psi_memory_avg60 0.00 % +psi_io_avg60 12.48 % diff --git a/data/baselines/pfv-tsys9/20260906-193323/MANIFEST b/data/baselines/pfv-tsys9/20260906-193323/MANIFEST new file mode 100644 index 0000000..1ad68c3 --- /dev/null +++ b/data/baselines/pfv-tsys9/20260906-193323/MANIFEST @@ -0,0 +1 @@ +baseline pfv-tsys9 20260906-193323 diff --git a/data/baselines/pfv-tsys9/20260906-193323/bench-run.tsv b/data/baselines/pfv-tsys9/20260906-193323/bench-run.tsv new file mode 100644 index 0000000..09549d8 --- /dev/null +++ b/data/baselines/pfv-tsys9/20260906-193323/bench-run.tsv @@ -0,0 +1,16 @@ +BENCH|pfv-tsys9|meta|stamp +BENCH|pfv-tsys9|meta|path +BENCH|pfv-tsys9|meta|runtime +BENCH|pfv-tsys9|meta|size +BENCH|pfv-tsys9|meta|load +BENCH|pfv-tsys9|psi_io_some_avg10|5.14 +BENCH|pfv-tsys9|psi_cpu_some_avg10|2.66 +BENCH|pfv-tsys9|psi_memory_some_avg10|0.00 +BENCH|pfv-tsys9|meta|bench_device +BENCH|pfv-tsys9|randread4k|iops=7766.011699|bw_kbps=31064|p99_ms=0.164864 +BENCH|pfv-tsys9|randwrite4k|iops=17438.828059|bw_kbps=69755|p99_ms=0.091648 +BENCH|pfv-tsys9|seqread1m|iops=267.346531|bw_kbps=273762|p99_ms=4.292608 +BENCH|pfv-tsys9|seqwrite1m|iops=388.511149|bw_kbps=397835|p99_ms=2.965504 +BENCH|pfv-tsys9|meta|load_end +BENCH|pfv-tsys9|psi_io_some_avg10_end|61.96 +BENCH|pfv-tsys9|meta|end diff --git a/data/baselines/pfv-tsys9/20260906-193323/bench.err b/data/baselines/pfv-tsys9/20260906-193323/bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys9/20260906-193323/cpu-bench.err b/data/baselines/pfv-tsys9/20260906-193323/cpu-bench.err new file mode 100644 index 0000000..e69de29 diff --git a/data/baselines/pfv-tsys9/20260906-193323/cpu-bench.tsv b/data/baselines/pfv-tsys9/20260906-193323/cpu-bench.tsv new file mode 100644 index 0000000..654a6cc --- /dev/null +++ b/data/baselines/pfv-tsys9/20260906-193323/cpu-bench.tsv @@ -0,0 +1,11 @@ +metric value unit +governor performance - +nproc 12 - +openssl_aes-256-cbc_1t 1199179.37k k/s +openssl_aes-256-cbc_12t 13546392.78k k/s +openssl_sha256_1t 558840.22k k/s +openssl_sha256_12t 3421896.70k k/s +sysbench_cpu absent - +psi_cpu_avg60 1.93 % +psi_memory_avg60 0.00 % +psi_io_avg60 9.72 % diff --git a/monitoring/README.md b/monitoring/README.md index ec46ad0..fa7ed09 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -46,16 +46,16 @@ virtual-fs bind sources — the wrapper mounts them on start). sysbench, fio, iperf3, stress-ng, perf, smartmontools, nvme-cli, hwloc… ```bash -knelerf-toolbox run 'iostat -x 1 5' # starts LXC if stopped -knelerf-toolbox run 'bpftrace -e "tracepoint:block:block_rq_issue { @[comm] = count(); }" -c "sleep 5"' -knelerf-toolbox run 'sar -A -f /host/var/log/sysstat/sa06' # host sar, offline -knelerf-toolbox run 'smartctl -d sat -H /dev/sda' # SAT devices need -d sat; NVMe doesn't -knelerf-toolbox shell # interactive -knelerf-toolbox stop # back to zero footprint +knelperf-toolbox run 'iostat -x 1 5' # starts LXC if stopped +knelperf-toolbox run 'bpftrace -e "tracepoint:block:block_rq_issue { @[comm] = count(); }" -c "sleep 5"' +knelperf-toolbox run 'sar -A -f /host/var/log/sysstat/sa06' # host sar, offline +knelperf-toolbox run 'smartctl -d sat -H /dev/sda' # SAT devices need -d sat; NVMe doesn't +knelperf-toolbox shell # interactive +knelperf-toolbox stop # back to zero footprint ``` Docker-capable hosts use the same manifest as an image: -`monitoring/toolbox/Dockerfile` → `git.knownelement.com/knel/knelperf-toolbox:v1`. +`git.knownelement.com/knel/knelperf-toolbox:v1` (digest `sha256:3bb22bdb4a69707aadd0043d5151f863634a3ce5229f63d12773d5e3f90e001c`, from `monitoring/toolbox/Dockerfile`). ## Analysis recipes