summaryrefslogtreecommitdiff
path: root/common
diff options
context:
space:
mode:
authorriastradh <riastradh@NetBSD.org>2022-07-30 14:11:00 +0000
committerriastradh <riastradh@NetBSD.org>2022-07-30 14:11:00 +0000
commit3df45b0a9efe74751ab7d591c23881fb23b8e3cc (patch)
tree613ff0854b8d7944709011a1af7744a1c02de9b9 /common
parent1dc084e6c367176d6b9a1452f1315ff6e78f7802 (diff)
x86: Eliminate mfence hotpatch for membar_sync.
The more-compatible LOCK ADD $0,-N(%rsp) turns out to be cheaper than MFENCE anyway. Let's save some space and maintenance and rip out the hotpatching for it.
Diffstat (limited to 'common')
-rw-r--r--common/lib/libc/arch/i386/atomic/atomic.S29
-rw-r--r--common/lib/libc/arch/x86_64/atomic/atomic.S36
2 files changed, 26 insertions, 39 deletions
diff --git a/common/lib/libc/arch/i386/atomic/atomic.S b/common/lib/libc/arch/i386/atomic/atomic.S
index 4a0513dddbf..df7b7d77862 100644
--- a/common/lib/libc/arch/i386/atomic/atomic.S
+++ b/common/lib/libc/arch/i386/atomic/atomic.S
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.35 2022/04/09 23:32:51 riastradh Exp $ */
+/* $NetBSD: atomic.S,v 1.36 2022/07/30 14:11:00 riastradh Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -46,11 +46,9 @@
#include "opt_xen.h"
#include <machine/frameasm.h>
#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7);
#define HOTPATCH_CAS_64 HOTPATCH(HP_NAME_CAS_64, 49);
#else
#define LOCK lock
-#define HOTPATCH_SSE2_MFENCE /* nothing */
#define HOTPATCH_CAS_64 /* nothing */
#endif
@@ -198,13 +196,22 @@ END(_membar_release)
ENTRY(_membar_sync)
/*
- * MFENCE, or a serializing instruction like a locked addq,
+ * MFENCE, or a serializing instruction like a locked ADDL,
* is necessary to order store-before-load. Every other
* ordering -- load-before-anything, anything-before-store --
* is already guaranteed without explicit barriers.
+ *
+ * Empirically it turns out locked ADDL is cheaper than MFENCE,
+ * so we use that, with an offset below the return address on
+ * the stack to avoid a false dependency with RET. (It might
+ * even be better to use a much lower offset, say -128, to
+ * avoid false dependencies for subsequent callees of the
+ * caller.)
+ *
+ * https://pvk.ca/Blog/2014/10/19/performance-optimisation-~-writing-an-essay/
+ * https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
+ * https://www.agner.org/optimize/instruction_tables.pdf
*/
- HOTPATCH_SSE2_MFENCE
- /* 7 bytes of instructions */
LOCK
addl $0, -4(%esp)
ret
@@ -406,13 +413,3 @@ STRONG_ALIAS(_membar_consumer,_membar_acquire)
STRONG_ALIAS(_membar_producer,_membar_release)
STRONG_ALIAS(_membar_enter,_membar_sync)
STRONG_ALIAS(_membar_exit,_membar_release)
-
-#ifdef _HARDKERNEL
- .section .rodata
-
-LABEL(sse2_mfence)
- mfence
- ret
- nop; nop; nop;
-LABEL(sse2_mfence_end)
-#endif /* _HARDKERNEL */
diff --git a/common/lib/libc/arch/x86_64/atomic/atomic.S b/common/lib/libc/arch/x86_64/atomic/atomic.S
index f73b3887cf8..213b4ab9f25 100644
--- a/common/lib/libc/arch/x86_64/atomic/atomic.S
+++ b/common/lib/libc/arch/x86_64/atomic/atomic.S
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.28 2022/04/09 23:32:52 riastradh Exp $ */
+/* $NetBSD: atomic.S,v 1.29 2022/07/30 14:11:00 riastradh Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -38,15 +38,6 @@
#define ALIAS(f, t) WEAK_ALIAS(f,t)
#endif
-#ifdef _HARDKERNEL
-#include <machine/frameasm.h>
-#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 8);
-#else
-#define LOCK lock
-#define HOTPATCH_SSE2_MFENCE /* nothing */
-#endif
-
.text
/* 32-bit */
@@ -273,13 +264,22 @@ END(_membar_release)
ENTRY(_membar_sync)
/*
- * MFENCE, or a serializing instruction like a locked addq,
+ * MFENCE, or a serializing instruction like a locked ADDQ,
* is necessary to order store-before-load. Every other
* ordering -- load-before-anything, anything-before-store --
* is already guaranteed without explicit barriers.
+ *
+ * Empirically it turns out locked ADDQ is cheaper than MFENCE,
+ * so we use that, with an offset below the return address on
+ * the stack to avoid a false dependency with RET. (It might
+ * even be better to use a much lower offset, say -128, to
+ * avoid false dependencies for subsequent callees of the
+ * caller.)
+ *
+ * https://pvk.ca/Blog/2014/10/19/performance-optimisation-~-writing-an-essay/
+ * https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
+ * https://www.agner.org/optimize/instruction_tables.pdf
*/
- HOTPATCH_SSE2_MFENCE
- /* 8 bytes of instructions */
LOCK
addq $0, -8(%rsp)
ret
@@ -429,13 +429,3 @@ STRONG_ALIAS(_membar_consumer,_membar_acquire)
STRONG_ALIAS(_membar_producer,_membar_release)
STRONG_ALIAS(_membar_enter,_membar_sync)
STRONG_ALIAS(_membar_exit,_membar_release)
-
-#ifdef _HARDKERNEL
- .section .rodata
-
-LABEL(sse2_mfence)
- mfence
- ret
- nop; nop; nop; nop;
-LABEL(sse2_mfence_end)
-#endif /* _HARDKERNEL */