mirror of
https://github.com/openwrt/openwrt.git
synced 2024-12-23 23:42:43 +00:00
kernel: backport improved checksum function for ARM64
Improves network performance in some cases when checksum offload is not available Signed-off-by: Felix Fietkau <nbd@nbd.name>
This commit is contained in:
parent
21ab979c70
commit
63b6b10670
@ -0,0 +1,176 @@
|
||||
From: Robin Murphy <robin.murphy@arm.com>
|
||||
Date: Wed, 15 Jan 2020 16:42:39 +0000
|
||||
Subject: [PATCH] arm64: Implement optimised checksum routine
|
||||
|
||||
Apparently there exist certain workloads which rely heavily on software
|
||||
checksumming, for which the generic do_csum() implementation becomes a
|
||||
significant bottleneck. Therefore let's give arm64 its own optimised
|
||||
version - for ease of maintenance this foregoes assembly or intrisics,
|
||||
and is thus not actually arm64-specific, but does rely heavily on C
|
||||
idioms that translate well to the A64 ISA and the typical load/store
|
||||
capabilities of most ARMv8 CPU cores.
|
||||
|
||||
The resulting increase in checksum throughput scales nicely with buffer
|
||||
size, tending towards 4x for a small in-order core (Cortex-A53), and up
|
||||
to 6x or more for an aggressive big core (Ampere eMAG).
|
||||
|
||||
Reported-by: Lingyan Huang <huanglingyan2@huawei.com>
|
||||
Tested-by: Lingyan Huang <huanglingyan2@huawei.com>
|
||||
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
|
||||
Signed-off-by: Will Deacon <will@kernel.org>
|
||||
---
|
||||
create mode 100644 arch/arm64/lib/csum.c
|
||||
|
||||
--- a/arch/arm64/include/asm/checksum.h
|
||||
+++ b/arch/arm64/include/asm/checksum.h
|
||||
@@ -36,6 +36,9 @@ static inline __sum16 ip_fast_csum(const
|
||||
}
|
||||
#define ip_fast_csum ip_fast_csum
|
||||
|
||||
+extern unsigned int do_csum(const unsigned char *buff, int len);
|
||||
+#define do_csum do_csum
|
||||
+
|
||||
#include <asm-generic/checksum.h>
|
||||
|
||||
#endif /* __ASM_CHECKSUM_H */
|
||||
--- a/arch/arm64/lib/Makefile
|
||||
+++ b/arch/arm64/lib/Makefile
|
||||
@@ -1,9 +1,9 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
lib-y := clear_user.o delay.o copy_from_user.o \
|
||||
copy_to_user.o copy_in_user.o copy_page.o \
|
||||
- clear_page.o memchr.o memcpy.o memmove.o memset.o \
|
||||
- memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
|
||||
- strchr.o strrchr.o tishift.o
|
||||
+ clear_page.o csum.o memchr.o memcpy.o memmove.o \
|
||||
+ memset.o memcmp.o strcmp.o strncmp.o strlen.o \
|
||||
+ strnlen.o strchr.o strrchr.o tishift.o
|
||||
|
||||
ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
|
||||
obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
|
||||
--- /dev/null
|
||||
+++ b/arch/arm64/lib/csum.c
|
||||
@@ -0,0 +1,123 @@
|
||||
+// SPDX-License-Identifier: GPL-2.0-only
|
||||
+// Copyright (C) 2019-2020 Arm Ltd.
|
||||
+
|
||||
+#include <linux/compiler.h>
|
||||
+#include <linux/kasan-checks.h>
|
||||
+#include <linux/kernel.h>
|
||||
+
|
||||
+#include <net/checksum.h>
|
||||
+
|
||||
+/* Looks dumb, but generates nice-ish code */
|
||||
+static u64 accumulate(u64 sum, u64 data)
|
||||
+{
|
||||
+ __uint128_t tmp = (__uint128_t)sum + data;
|
||||
+ return tmp + (tmp >> 64);
|
||||
+}
|
||||
+
|
||||
+unsigned int do_csum(const unsigned char *buff, int len)
|
||||
+{
|
||||
+ unsigned int offset, shift, sum;
|
||||
+ const u64 *ptr;
|
||||
+ u64 data, sum64 = 0;
|
||||
+
|
||||
+ offset = (unsigned long)buff & 7;
|
||||
+ /*
|
||||
+ * This is to all intents and purposes safe, since rounding down cannot
|
||||
+ * result in a different page or cache line being accessed, and @buff
|
||||
+ * should absolutely not be pointing to anything read-sensitive. We do,
|
||||
+ * however, have to be careful not to piss off KASAN, which means using
|
||||
+ * unchecked reads to accommodate the head and tail, for which we'll
|
||||
+ * compensate with an explicit check up-front.
|
||||
+ */
|
||||
+ kasan_check_read(buff, len);
|
||||
+ ptr = (u64 *)(buff - offset);
|
||||
+ len = len + offset - 8;
|
||||
+
|
||||
+ /*
|
||||
+ * Head: zero out any excess leading bytes. Shifting back by the same
|
||||
+ * amount should be at least as fast as any other way of handling the
|
||||
+ * odd/even alignment, and means we can ignore it until the very end.
|
||||
+ */
|
||||
+ shift = offset * 8;
|
||||
+ data = READ_ONCE_NOCHECK(*ptr++);
|
||||
+#ifdef __LITTLE_ENDIAN
|
||||
+ data = (data >> shift) << shift;
|
||||
+#else
|
||||
+ data = (data << shift) >> shift;
|
||||
+#endif
|
||||
+
|
||||
+ /*
|
||||
+ * Body: straightforward aligned loads from here on (the paired loads
|
||||
+ * underlying the quadword type still only need dword alignment). The
|
||||
+ * main loop strictly excludes the tail, so the second loop will always
|
||||
+ * run at least once.
|
||||
+ */
|
||||
+ while (unlikely(len > 64)) {
|
||||
+ __uint128_t tmp1, tmp2, tmp3, tmp4;
|
||||
+
|
||||
+ tmp1 = READ_ONCE_NOCHECK(*(__uint128_t *)ptr);
|
||||
+ tmp2 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 2));
|
||||
+ tmp3 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 4));
|
||||
+ tmp4 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 6));
|
||||
+
|
||||
+ len -= 64;
|
||||
+ ptr += 8;
|
||||
+
|
||||
+ /* This is the "don't dump the carry flag into a GPR" idiom */
|
||||
+ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
|
||||
+ tmp2 += (tmp2 >> 64) | (tmp2 << 64);
|
||||
+ tmp3 += (tmp3 >> 64) | (tmp3 << 64);
|
||||
+ tmp4 += (tmp4 >> 64) | (tmp4 << 64);
|
||||
+ tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64);
|
||||
+ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
|
||||
+ tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64);
|
||||
+ tmp3 += (tmp3 >> 64) | (tmp3 << 64);
|
||||
+ tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64);
|
||||
+ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
|
||||
+ tmp1 = ((tmp1 >> 64) << 64) | sum64;
|
||||
+ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
|
||||
+ sum64 = tmp1 >> 64;
|
||||
+ }
|
||||
+ while (len > 8) {
|
||||
+ __uint128_t tmp;
|
||||
+
|
||||
+ sum64 = accumulate(sum64, data);
|
||||
+ tmp = READ_ONCE_NOCHECK(*(__uint128_t *)ptr);
|
||||
+
|
||||
+ len -= 16;
|
||||
+ ptr += 2;
|
||||
+
|
||||
+#ifdef __LITTLE_ENDIAN
|
||||
+ data = tmp >> 64;
|
||||
+ sum64 = accumulate(sum64, tmp);
|
||||
+#else
|
||||
+ data = tmp;
|
||||
+ sum64 = accumulate(sum64, tmp >> 64);
|
||||
+#endif
|
||||
+ }
|
||||
+ if (len > 0) {
|
||||
+ sum64 = accumulate(sum64, data);
|
||||
+ data = READ_ONCE_NOCHECK(*ptr);
|
||||
+ len -= 8;
|
||||
+ }
|
||||
+ /*
|
||||
+ * Tail: zero any over-read bytes similarly to the head, again
|
||||
+ * preserving odd/even alignment.
|
||||
+ */
|
||||
+ shift = len * -8;
|
||||
+#ifdef __LITTLE_ENDIAN
|
||||
+ data = (data << shift) >> shift;
|
||||
+#else
|
||||
+ data = (data >> shift) << shift;
|
||||
+#endif
|
||||
+ sum64 = accumulate(sum64, data);
|
||||
+
|
||||
+ /* Finally, folding */
|
||||
+ sum64 += (sum64 >> 32) | (sum64 << 32);
|
||||
+ sum = sum64 >> 32;
|
||||
+ sum += (sum >> 16) | (sum << 16);
|
||||
+ if (offset & 1)
|
||||
+ return (u16)swab32(sum);
|
||||
+
|
||||
+ return sum >> 16;
|
||||
+}
|
@ -0,0 +1,28 @@
|
||||
From: Robin Murphy <robin.murphy@arm.com>
|
||||
Date: Fri, 17 Jan 2020 15:48:39 +0000
|
||||
Subject: [PATCH] arm64: csum: Fix pathological zero-length calls
|
||||
|
||||
In validating the checksumming results of the new routine, I sadly
|
||||
neglected to test its not-checksumming results. Thus it slipped through
|
||||
that the one case where @buff is already dword-aligned and @len = 0
|
||||
manages to defeat the tail-masking logic and behave as if @len = 8.
|
||||
For a zero length it doesn't make much sense to deference @buff anyway,
|
||||
so just add an early return (which has essentially zero impact on
|
||||
performance).
|
||||
|
||||
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
|
||||
Signed-off-by: Will Deacon <will@kernel.org>
|
||||
---
|
||||
|
||||
--- a/arch/arm64/lib/csum.c
|
||||
+++ b/arch/arm64/lib/csum.c
|
||||
@@ -20,6 +20,9 @@ unsigned int do_csum(const unsigned char
|
||||
const u64 *ptr;
|
||||
u64 data, sum64 = 0;
|
||||
|
||||
+ if (unlikely(len == 0))
|
||||
+ return 0;
|
||||
+
|
||||
offset = (unsigned long)buff & 7;
|
||||
/*
|
||||
* This is to all intents and purposes safe, since rounding down cannot
|
Loading…
Reference in New Issue
Block a user