diff options
| author | riastradh <riastradh@NetBSD.org> | 2022-07-30 14:11:00 +0000 |
|---|---|---|
| committer | riastradh <riastradh@NetBSD.org> | 2022-07-30 14:11:00 +0000 |
| commit | 3df45b0a9efe74751ab7d591c23881fb23b8e3cc (patch) | |
| tree | 613ff0854b8d7944709011a1af7744a1c02de9b9 /common | |
| parent | 1dc084e6c367176d6b9a1452f1315ff6e78f7802 (diff) | |
x86: Eliminate mfence hotpatch for membar_sync.
The more-compatible LOCK ADD $0,-N(%rsp) turns out to be cheaper
than MFENCE anyway. Let's save some space and maintenance and rip
out the hotpatching for it.
Diffstat (limited to 'common')
| -rw-r--r-- | common/lib/libc/arch/i386/atomic/atomic.S | 29 | ||||
| -rw-r--r-- | common/lib/libc/arch/x86_64/atomic/atomic.S | 36 |
2 files changed, 26 insertions, 39 deletions
diff --git a/common/lib/libc/arch/i386/atomic/atomic.S b/common/lib/libc/arch/i386/atomic/atomic.S index 4a0513dddbf..df7b7d77862 100644 --- a/common/lib/libc/arch/i386/atomic/atomic.S +++ b/common/lib/libc/arch/i386/atomic/atomic.S @@ -1,4 +1,4 @@ -/* $NetBSD: atomic.S,v 1.35 2022/04/09 23:32:51 riastradh Exp $ */ +/* $NetBSD: atomic.S,v 1.36 2022/07/30 14:11:00 riastradh Exp $ */ /*- * Copyright (c) 2007 The NetBSD Foundation, Inc. @@ -46,11 +46,9 @@ #include "opt_xen.h" #include <machine/frameasm.h> #define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock -#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7); #define HOTPATCH_CAS_64 HOTPATCH(HP_NAME_CAS_64, 49); #else #define LOCK lock -#define HOTPATCH_SSE2_MFENCE /* nothing */ #define HOTPATCH_CAS_64 /* nothing */ #endif @@ -198,13 +196,22 @@ END(_membar_release) ENTRY(_membar_sync) /* - * MFENCE, or a serializing instruction like a locked addq, + * MFENCE, or a serializing instruction like a locked ADDL, * is necessary to order store-before-load. Every other * ordering -- load-before-anything, anything-before-store -- * is already guaranteed without explicit barriers. + * + * Empirically it turns out locked ADDL is cheaper than MFENCE, + * so we use that, with an offset below the return address on + * the stack to avoid a false dependency with RET. (It might + * even be better to use a much lower offset, say -128, to + * avoid false dependencies for subsequent callees of the + * caller.) + * + * https://pvk.ca/Blog/2014/10/19/performance-optimisation-~-writing-an-essay/ + * https://shipilev.net/blog/2014/on-the-fence-with-dependencies/ + * https://www.agner.org/optimize/instruction_tables.pdf */ - HOTPATCH_SSE2_MFENCE - /* 7 bytes of instructions */ LOCK addl $0, -4(%esp) ret @@ -406,13 +413,3 @@ STRONG_ALIAS(_membar_consumer,_membar_acquire) STRONG_ALIAS(_membar_producer,_membar_release) STRONG_ALIAS(_membar_enter,_membar_sync) STRONG_ALIAS(_membar_exit,_membar_release) - -#ifdef _HARDKERNEL - .section .rodata - -LABEL(sse2_mfence) - mfence - ret - nop; nop; nop; -LABEL(sse2_mfence_end) -#endif /* _HARDKERNEL */ diff --git a/common/lib/libc/arch/x86_64/atomic/atomic.S b/common/lib/libc/arch/x86_64/atomic/atomic.S index f73b3887cf8..213b4ab9f25 100644 --- a/common/lib/libc/arch/x86_64/atomic/atomic.S +++ b/common/lib/libc/arch/x86_64/atomic/atomic.S @@ -1,4 +1,4 @@ -/* $NetBSD: atomic.S,v 1.28 2022/04/09 23:32:52 riastradh Exp $ */ +/* $NetBSD: atomic.S,v 1.29 2022/07/30 14:11:00 riastradh Exp $ */ /*- * Copyright (c) 2007 The NetBSD Foundation, Inc. @@ -38,15 +38,6 @@ #define ALIAS(f, t) WEAK_ALIAS(f,t) #endif -#ifdef _HARDKERNEL -#include <machine/frameasm.h> -#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock -#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 8); -#else -#define LOCK lock -#define HOTPATCH_SSE2_MFENCE /* nothing */ -#endif - .text /* 32-bit */ @@ -273,13 +264,22 @@ END(_membar_release) ENTRY(_membar_sync) /* - * MFENCE, or a serializing instruction like a locked addq, + * MFENCE, or a serializing instruction like a locked ADDQ, * is necessary to order store-before-load. Every other * ordering -- load-before-anything, anything-before-store -- * is already guaranteed without explicit barriers. + * + * Empirically it turns out locked ADDQ is cheaper than MFENCE, + * so we use that, with an offset below the return address on + * the stack to avoid a false dependency with RET. (It might + * even be better to use a much lower offset, say -128, to + * avoid false dependencies for subsequent callees of the + * caller.) + * + * https://pvk.ca/Blog/2014/10/19/performance-optimisation-~-writing-an-essay/ + * https://shipilev.net/blog/2014/on-the-fence-with-dependencies/ + * https://www.agner.org/optimize/instruction_tables.pdf */ - HOTPATCH_SSE2_MFENCE - /* 8 bytes of instructions */ LOCK addq $0, -8(%rsp) ret @@ -429,13 +429,3 @@ STRONG_ALIAS(_membar_consumer,_membar_acquire) STRONG_ALIAS(_membar_producer,_membar_release) STRONG_ALIAS(_membar_enter,_membar_sync) STRONG_ALIAS(_membar_exit,_membar_release) - -#ifdef _HARDKERNEL - .section .rodata - -LABEL(sse2_mfence) - mfence - ret - nop; nop; nop; nop; -LABEL(sse2_mfence_end) -#endif /* _HARDKERNEL */ |
