openwrt/target/linux/generic/backport-5.15/600-v5.18-page_pool-Add-allocation-stats.patch

166 lines
5.0 KiB
Diff
Raw Normal View History

From 8610037e8106b48c79cfe0afb92b2b2466e51c3d Mon Sep 17 00:00:00 2001
From: Joe Damato <jdamato@fastly.com>
Date: Tue, 1 Mar 2022 23:55:47 -0800
Subject: [PATCH] page_pool: Add allocation stats
Add per-pool statistics counters for the allocation path of a page pool.
These stats are incremented in softirq context, so no locking or per-cpu
variables are needed.
This code is disabled by default and a kernel config option is provided for
users who wish to enable them.
The statistics added are:
- fast: successful fast path allocations
- slow: slow path order-0 allocations
- slow_high_order: slow path high order allocations
- empty: ptr ring is empty, so a slow path allocation was forced.
- refill: an allocation which triggered a refill of the cache
- waive: pages obtained from the ptr ring that cannot be added to
the cache due to a NUMA mismatch.
Signed-off-by: Joe Damato <jdamato@fastly.com>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Reviewed-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
---
include/net/page_pool.h | 18 ++++++++++++++++++
net/Kconfig | 13 +++++++++++++
net/core/page_pool.c | 24 ++++++++++++++++++++----
3 files changed, 51 insertions(+), 4 deletions(-)
--- a/include/net/page_pool.h
+++ b/include/net/page_pool.h
@@ -82,6 +82,19 @@ struct page_pool_params {
unsigned int offset; /* DMA addr offset */
};
+#ifdef CONFIG_PAGE_POOL_STATS
+struct page_pool_alloc_stats {
+ u64 fast; /* fast path allocations */
+ u64 slow; /* slow-path order 0 allocations */
+ u64 slow_high_order; /* slow-path high order allocations */
+ u64 empty; /* failed refills due to empty ptr ring, forcing
+ * slow path allocation
+ */
+ u64 refill; /* allocations via successful refill */
+ u64 waive; /* failed refills due to numa zone mismatch */
+};
+#endif
+
struct page_pool {
struct page_pool_params p;
@@ -132,6 +145,11 @@ struct page_pool {
refcount_t user_cnt;
u64 destroy_cnt;
+
+#ifdef CONFIG_PAGE_POOL_STATS
+ /* these stats are incremented while in softirq context */
+ struct page_pool_alloc_stats alloc_stats;
+#endif
};
struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -434,6 +434,19 @@ config NET_DEVLINK
config PAGE_POOL
bool
+config PAGE_POOL_STATS
+ default n
+ bool "Page pool stats"
+ depends on PAGE_POOL
+ help
+ Enable page pool statistics to track page allocation and recycling
+ in page pools. This option incurs additional CPU cost in allocation
+ and recycle paths and additional memory cost to store the statistics.
+ These statistics are only available if this option is enabled and if
+ the driver using the page pool supports exporting this data.
+
+ If unsure, say N.
+
config FAILOVER
tristate "Generic failover module"
help
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
kernel: bump 5.15 to 5.15.115 Manually rebased: backport-5.15/603-v5.19-page_pool-Add-recycle-stats-to-page_pool_put_page_bu.patch pending-5.15/723-net-mt7531-ensure-all-MACs-are-powered-down-before-r.patch* Removed upstreamed: generic-backport/610-v6.3-net-page_pool-use-in_softirq-instead.patch[1] backport-5.15/705-12-v6.0-net-dsa-mt7530-rework-mt753-01-_setup.patch[2] backport-5.15/790-v6.4-0010-net-dsa-mt7530-split-off-common-parts-from-mt7531_se.patch[3] backport-5.15/703-10-v5.16-net-dsa-introduce-helpers-for-iterating-through-port.patch[4] All other patches automatically rebased. * Modified to define the variable i as suggested by DragonBluep in PR discussion. See: https://github.com/openwrt/openwrt/pull/12823#issuecomment-1578518576 1. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=3af319d5147454dc63665ef451229c674b538377 2. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=0753c1ef24194580f7165ae6e259b59a851392f2 3. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=5a7266feaa6d708fc6880a161786eaa884ef3c8e 4. https://github.com/gregkh/linux/commit/9902f91cf666124a6b50bbcf483b46ecb09ef408 Build system: x86_64 Build-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Run-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Signed-off-by: John Audia <therealgraysky@proton.me> (cherry picked from commit 5714660643e9170920be2abbb2297d9aac0b9533)
2023-06-06 14:45:47 +00:00
@@ -49,6 +49,13 @@ static void page_pool_producer_unlock(st
spin_unlock_bh(&pool->ring.producer_lock);
}
+#ifdef CONFIG_PAGE_POOL_STATS
+/* alloc_stat_inc is intended to be used in softirq context */
+#define alloc_stat_inc(pool, __stat) (pool->alloc_stats.__stat++)
+#else
+#define alloc_stat_inc(pool, __stat)
+#endif
+
static int page_pool_init(struct page_pool *pool,
const struct page_pool_params *params)
{
kernel: bump 5.15 to 5.15.115 Manually rebased: backport-5.15/603-v5.19-page_pool-Add-recycle-stats-to-page_pool_put_page_bu.patch pending-5.15/723-net-mt7531-ensure-all-MACs-are-powered-down-before-r.patch* Removed upstreamed: generic-backport/610-v6.3-net-page_pool-use-in_softirq-instead.patch[1] backport-5.15/705-12-v6.0-net-dsa-mt7530-rework-mt753-01-_setup.patch[2] backport-5.15/790-v6.4-0010-net-dsa-mt7530-split-off-common-parts-from-mt7531_se.patch[3] backport-5.15/703-10-v5.16-net-dsa-introduce-helpers-for-iterating-through-port.patch[4] All other patches automatically rebased. * Modified to define the variable i as suggested by DragonBluep in PR discussion. See: https://github.com/openwrt/openwrt/pull/12823#issuecomment-1578518576 1. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=3af319d5147454dc63665ef451229c674b538377 2. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=0753c1ef24194580f7165ae6e259b59a851392f2 3. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=5a7266feaa6d708fc6880a161786eaa884ef3c8e 4. https://github.com/gregkh/linux/commit/9902f91cf666124a6b50bbcf483b46ecb09ef408 Build system: x86_64 Build-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Run-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Signed-off-by: John Audia <therealgraysky@proton.me> (cherry picked from commit 5714660643e9170920be2abbb2297d9aac0b9533)
2023-06-06 14:45:47 +00:00
@@ -140,8 +147,10 @@ static struct page *page_pool_refill_all
int pref_nid; /* preferred NUMA node */
/* Quicker fallback, avoid locks when ring is empty */
- if (__ptr_ring_empty(r))
+ if (__ptr_ring_empty(r)) {
+ alloc_stat_inc(pool, empty);
return NULL;
+ }
/* Softirq guarantee CPU and thus NUMA node is stable. This,
* assumes CPU refilling driver RX-ring will also run RX-NAPI.
kernel: bump 5.15 to 5.15.115 Manually rebased: backport-5.15/603-v5.19-page_pool-Add-recycle-stats-to-page_pool_put_page_bu.patch pending-5.15/723-net-mt7531-ensure-all-MACs-are-powered-down-before-r.patch* Removed upstreamed: generic-backport/610-v6.3-net-page_pool-use-in_softirq-instead.patch[1] backport-5.15/705-12-v6.0-net-dsa-mt7530-rework-mt753-01-_setup.patch[2] backport-5.15/790-v6.4-0010-net-dsa-mt7530-split-off-common-parts-from-mt7531_se.patch[3] backport-5.15/703-10-v5.16-net-dsa-introduce-helpers-for-iterating-through-port.patch[4] All other patches automatically rebased. * Modified to define the variable i as suggested by DragonBluep in PR discussion. See: https://github.com/openwrt/openwrt/pull/12823#issuecomment-1578518576 1. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=3af319d5147454dc63665ef451229c674b538377 2. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=0753c1ef24194580f7165ae6e259b59a851392f2 3. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=5a7266feaa6d708fc6880a161786eaa884ef3c8e 4. https://github.com/gregkh/linux/commit/9902f91cf666124a6b50bbcf483b46ecb09ef408 Build system: x86_64 Build-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Run-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Signed-off-by: John Audia <therealgraysky@proton.me> (cherry picked from commit 5714660643e9170920be2abbb2297d9aac0b9533)
2023-06-06 14:45:47 +00:00
@@ -171,14 +180,17 @@ static struct page *page_pool_refill_all
* This limit stress on page buddy alloactor.
*/
page_pool_return_page(pool, page);
+ alloc_stat_inc(pool, waive);
page = NULL;
break;
}
} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
/* Return last page */
- if (likely(pool->alloc.count > 0))
+ if (likely(pool->alloc.count > 0)) {
page = pool->alloc.cache[--pool->alloc.count];
+ alloc_stat_inc(pool, refill);
+ }
spin_unlock(&r->consumer_lock);
return page;
kernel: bump 5.15 to 5.15.115 Manually rebased: backport-5.15/603-v5.19-page_pool-Add-recycle-stats-to-page_pool_put_page_bu.patch pending-5.15/723-net-mt7531-ensure-all-MACs-are-powered-down-before-r.patch* Removed upstreamed: generic-backport/610-v6.3-net-page_pool-use-in_softirq-instead.patch[1] backport-5.15/705-12-v6.0-net-dsa-mt7530-rework-mt753-01-_setup.patch[2] backport-5.15/790-v6.4-0010-net-dsa-mt7530-split-off-common-parts-from-mt7531_se.patch[3] backport-5.15/703-10-v5.16-net-dsa-introduce-helpers-for-iterating-through-port.patch[4] All other patches automatically rebased. * Modified to define the variable i as suggested by DragonBluep in PR discussion. See: https://github.com/openwrt/openwrt/pull/12823#issuecomment-1578518576 1. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=3af319d5147454dc63665ef451229c674b538377 2. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=0753c1ef24194580f7165ae6e259b59a851392f2 3. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=5a7266feaa6d708fc6880a161786eaa884ef3c8e 4. https://github.com/gregkh/linux/commit/9902f91cf666124a6b50bbcf483b46ecb09ef408 Build system: x86_64 Build-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Run-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Signed-off-by: John Audia <therealgraysky@proton.me> (cherry picked from commit 5714660643e9170920be2abbb2297d9aac0b9533)
2023-06-06 14:45:47 +00:00
@@ -193,6 +205,7 @@ static struct page *__page_pool_get_cach
if (likely(pool->alloc.count)) {
/* Fast-path */
page = pool->alloc.cache[--pool->alloc.count];
+ alloc_stat_inc(pool, fast);
} else {
page = page_pool_refill_alloc_cache(pool);
}
kernel: bump 5.15 to 5.15.115 Manually rebased: backport-5.15/603-v5.19-page_pool-Add-recycle-stats-to-page_pool_put_page_bu.patch pending-5.15/723-net-mt7531-ensure-all-MACs-are-powered-down-before-r.patch* Removed upstreamed: generic-backport/610-v6.3-net-page_pool-use-in_softirq-instead.patch[1] backport-5.15/705-12-v6.0-net-dsa-mt7530-rework-mt753-01-_setup.patch[2] backport-5.15/790-v6.4-0010-net-dsa-mt7530-split-off-common-parts-from-mt7531_se.patch[3] backport-5.15/703-10-v5.16-net-dsa-introduce-helpers-for-iterating-through-port.patch[4] All other patches automatically rebased. * Modified to define the variable i as suggested by DragonBluep in PR discussion. See: https://github.com/openwrt/openwrt/pull/12823#issuecomment-1578518576 1. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=3af319d5147454dc63665ef451229c674b538377 2. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=0753c1ef24194580f7165ae6e259b59a851392f2 3. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=5a7266feaa6d708fc6880a161786eaa884ef3c8e 4. https://github.com/gregkh/linux/commit/9902f91cf666124a6b50bbcf483b46ecb09ef408 Build system: x86_64 Build-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Run-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Signed-off-by: John Audia <therealgraysky@proton.me> (cherry picked from commit 5714660643e9170920be2abbb2297d9aac0b9533)
2023-06-06 14:45:47 +00:00
@@ -264,6 +277,7 @@ static struct page *__page_pool_alloc_pa
return NULL;
}
+ alloc_stat_inc(pool, slow_high_order);
page_pool_set_pp_info(pool, page);
/* Track how many pages are held 'in-flight' */
kernel: bump 5.15 to 5.15.115 Manually rebased: backport-5.15/603-v5.19-page_pool-Add-recycle-stats-to-page_pool_put_page_bu.patch pending-5.15/723-net-mt7531-ensure-all-MACs-are-powered-down-before-r.patch* Removed upstreamed: generic-backport/610-v6.3-net-page_pool-use-in_softirq-instead.patch[1] backport-5.15/705-12-v6.0-net-dsa-mt7530-rework-mt753-01-_setup.patch[2] backport-5.15/790-v6.4-0010-net-dsa-mt7530-split-off-common-parts-from-mt7531_se.patch[3] backport-5.15/703-10-v5.16-net-dsa-introduce-helpers-for-iterating-through-port.patch[4] All other patches automatically rebased. * Modified to define the variable i as suggested by DragonBluep in PR discussion. See: https://github.com/openwrt/openwrt/pull/12823#issuecomment-1578518576 1. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=3af319d5147454dc63665ef451229c674b538377 2. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=0753c1ef24194580f7165ae6e259b59a851392f2 3. https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v5.15.115&id=5a7266feaa6d708fc6880a161786eaa884ef3c8e 4. https://github.com/gregkh/linux/commit/9902f91cf666124a6b50bbcf483b46ecb09ef408 Build system: x86_64 Build-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Run-tested: bcm2711/RPi4B, ramips/tplink_archer-a6-v3, filogic/xiaomi_redmi-router-ax6000-ubootmod Signed-off-by: John Audia <therealgraysky@proton.me> (cherry picked from commit 5714660643e9170920be2abbb2297d9aac0b9533)
2023-06-06 14:45:47 +00:00
@@ -318,10 +332,12 @@ static struct page *__page_pool_alloc_pa
}
/* Return last page */
- if (likely(pool->alloc.count > 0))
+ if (likely(pool->alloc.count > 0)) {
page = pool->alloc.cache[--pool->alloc.count];
- else
+ alloc_stat_inc(pool, slow);
+ } else {
page = NULL;
+ }
/* When page just alloc'ed is should/must have refcnt 1. */
return page;