summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorad <ad@NetBSD.org>2007-11-13 17:20:08 +0000
committerad <ad@NetBSD.org>2007-11-13 17:20:08 +0000
commit66ac2ffaf293a1766211e270fcce0ba856f8f80e (patch)
tree14c1c20985051c159c5d7b74ce133f33a8efdd64
parent3c776c8aa6570d9365bd4dc5468590e6c51103bb (diff)
Mutexes:
- Play scrooge again and chop more cycles off acquire/release. - Spin while the lock holder is running on another CPU (adaptive mutexes). - Do non-atomic release. Threadreg: - Add the necessary hooks to use a thread register. - Add the code for i386, using %gs. - Leave i386 code disabled until xen and COMPAT_NETBSD32 have the changes.
-rw-r--r--lib/libpthread/Makefile7
-rw-r--r--lib/libpthread/TODO13
-rw-r--r--lib/libpthread/arch/i386/_context_u.S23
-rw-r--r--lib/libpthread/arch/i386/pthread_md.c16
-rw-r--r--lib/libpthread/arch/i386/pthread_md.h43
-rw-r--r--lib/libpthread/arch/x86_64/_context_u.S20
-rw-r--r--lib/libpthread/arch/x86_64/pthread_md.h28
-rw-r--r--lib/libpthread/pthread.c77
-rw-r--r--lib/libpthread/pthread_int.h20
-rw-r--r--lib/libpthread/pthread_misc.c28
-rw-r--r--lib/libpthread/pthread_mutex2.c195
-rw-r--r--lib/libpthread/pthread_rwlock2.c10
12 files changed, 322 insertions, 158 deletions
diff --git a/lib/libpthread/Makefile b/lib/libpthread/Makefile
index 412f4c1d0ac..d5c905ff83c 100644
--- a/lib/libpthread/Makefile
+++ b/lib/libpthread/Makefile
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.45 2007/10/13 20:36:43 jnemeth Exp $
+# $NetBSD: Makefile,v 1.46 2007/11/13 17:20:08 ad Exp $
#
WARNS= 4
@@ -84,6 +84,11 @@ pthread_lock.po: pthread_lock.o
${_MKTARGET_CREATE}
cp pthread_lock.o pthread_lock.po
+COPTS.pthread_mutex2.c+= -fomit-frame-pointer -falign-functions=32
+pthread_mutex2.po: pthread_mutex2.o
+ ${_MKTARGET_CREATE}
+ cp pthread_mutex2.o pthread_mutex2.po
+
COPTS.pthread.c += -Wno-stack-protector
_context_u.po: _context_u.o
diff --git a/lib/libpthread/TODO b/lib/libpthread/TODO
index 89915e9917b..dcff58bd0a1 100644
--- a/lib/libpthread/TODO
+++ b/lib/libpthread/TODO
@@ -1,4 +1,4 @@
-$NetBSD: TODO,v 1.11 2007/09/08 23:00:31 ad Exp $
+$NetBSD: TODO,v 1.12 2007/11/13 17:20:09 ad Exp $
Bugs to fix:
@@ -15,16 +15,5 @@ Interfaces/features to implement:
- Allow threads to change their stack size. This probably depends on the
above item.
-- Have a user/kernel shared page that:
-
- o Has a hint mechanism that gives us a clue about whether an LWP is
- currently running on another CPU. This could be used for adaptive locks,
- but would need to be cheap to do in-kernel.
-
- o Perhaps has a flag value that's reset when a detached LWP is into the
- kernel and lwp_exit1(), meaning that its stack can be reclaimed. Again,
- may or may not be worth it. Currently we test for this with _lwp_kill()
- which is very inefficient.
-
- Keep a pool of dead LWPs so that we do not have take the full hit of
_lwp_create() every time pthread_create() is called.
diff --git a/lib/libpthread/arch/i386/_context_u.S b/lib/libpthread/arch/i386/_context_u.S
index 3dac247c3bd..6720bd67402 100644
--- a/lib/libpthread/arch/i386/_context_u.S
+++ b/lib/libpthread/arch/i386/_context_u.S
@@ -1,4 +1,4 @@
-/* $NetBSD: _context_u.S,v 1.6 2007/11/13 15:57:14 ad Exp $ */
+/* $NetBSD: _context_u.S,v 1.7 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
@@ -101,26 +101,17 @@ STUB(_swapcontext_u_xmm)
movl 8(%esp), %ecx
SETC
-STUB(pthread__atomic_cas_ptr)
+STUB(pthread__atomic_swap_ptr)
movl 4(%esp), %ecx
movl 8(%esp), %eax
- movl 12(%esp), %edx
- movl (%eax), %eax
- lock
- cmpxchgl %edx, (%ecx)
- jnz 1f
- mov $1, %eax
- ret
-1:
- mov 8(%esp), %edx
- mov %eax, (%edx)
- mov $0, %eax
+ xchgl %eax, (%ecx)
ret
-STUB(pthread__atomic_swap_ptr)
+STUB(pthread__atomic_or_ulong)
movl 4(%esp), %ecx
movl 8(%esp), %eax
- xchgl %eax, (%ecx)
+ lock
+ orl %eax, (%ecx)
ret
STUB(pthread__membar_full)
@@ -130,7 +121,7 @@ STUB(pthread__membar_full)
STUB(pthread__membar_producer)
/* A store is enough */
- movl $0, -4(%esp)
+ addl $0, -4(%esp)
ret
STUB(pthread__membar_consumer)
diff --git a/lib/libpthread/arch/i386/pthread_md.c b/lib/libpthread/arch/i386/pthread_md.c
index 0584a3b9b24..7dfabf2e0de 100644
--- a/lib/libpthread/arch/i386/pthread_md.c
+++ b/lib/libpthread/arch/i386/pthread_md.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.c,v 1.3 2003/03/08 08:03:37 lukem Exp $ */
+/* $NetBSD: pthread_md.c,v 1.4 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -37,10 +37,12 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_md.c,v 1.3 2003/03/08 08:03:37 lukem Exp $");
+__RCSID("$NetBSD: pthread_md.c,v 1.4 2007/11/13 17:20:09 ad Exp $");
#include <sys/param.h>
#include <sys/sysctl.h>
+
+#include <machine/sysarch.h>
#include <machine/cpu.h>
#include "pthread.h"
@@ -54,7 +56,8 @@ int (*_md_swapcontext_u)(ucontext_t *, const ucontext_t *);
* Initialize the function pointers for get/setcontext to the
* old-fp-save (s87) or new-fxsave (xmm) FP routines.
*/
-void pthread__i386_init(void)
+void
+pthread__i386_init(void)
{
int mib[2];
size_t len;
@@ -77,3 +80,10 @@ void pthread__i386_init(void)
}
}
+
+void
+pthread__threadreg_set(pthread_t self)
+{
+
+ sysarch(I386_SET_GSBASE, &self);
+}
diff --git a/lib/libpthread/arch/i386/pthread_md.h b/lib/libpthread/arch/i386/pthread_md.h
index e22fdc0c3de..c99ae7f412e 100644
--- a/lib/libpthread/arch/i386/pthread_md.h
+++ b/lib/libpthread/arch/i386/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.10 2007/09/24 12:19:40 skrll Exp $ */
+/* $NetBSD: pthread_md.h,v 1.11 2007/11/13 17:20:10 ad Exp $ */
/*-
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
@@ -183,5 +183,46 @@ pthread__sp(void)
#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory")
#define PTHREAD__HAVE_ATOMIC
+/* #define PTHREAD__HAVE_THREADREG */
+
+void pthread__threadreg_set(pthread_t);
+
+static inline pthread_t
+pthread__threadreg_get(void)
+{
+ pthread_t self;
+
+ __asm volatile("movl %%gs:0, %0"
+ : "=r" (self)
+ :);
+
+ return self;
+}
+
+static inline void *
+pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new)
+{
+ volatile uintptr_t *cast = ptr;
+ void *ret;
+
+ __asm __volatile ("lock; cmpxchgl %2, %1"
+ : "=a" (ret), "=m" (*cast)
+ : "r" (new), "m" (*cast), "0" (old));
+
+ return ret;
+}
+
+static inline void *
+pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new)
+{
+ volatile uintptr_t *cast = ptr;
+ void *ret;
+
+ __asm __volatile ("cmpxchgl %2, %1"
+ : "=a" (ret), "=m" (*cast)
+ : "r" (new), "m" (*cast), "0" (old));
+
+ return ret;
+}
#endif /* _LIB_PTHREAD_I386_MD_H */
diff --git a/lib/libpthread/arch/x86_64/_context_u.S b/lib/libpthread/arch/x86_64/_context_u.S
index cdb5c4a502c..8719b4fff5e 100644
--- a/lib/libpthread/arch/x86_64/_context_u.S
+++ b/lib/libpthread/arch/x86_64/_context_u.S
@@ -1,4 +1,4 @@
-/* $NetBSD: _context_u.S,v 1.8 2007/11/13 15:57:14 ad Exp $ */
+/* $NetBSD: _context_u.S,v 1.9 2007/11/13 17:20:10 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@@ -138,23 +138,17 @@ STUB(_swapcontext_u)
movq %rsi, %rdi
SETC
-STUB(pthread__atomic_cas_ptr)
- movq (%rsi), %rax
- lock
- cmpxchgq %rdx, (%rdi)
- jnz 1f
- movq $1, %rax
- ret
-1:
- movq %rax, (%rsi)
- movq $0, %rax
- ret
-
STUB(pthread__atomic_swap_ptr)
movq %rsi, %rax
+ lock
xchgq %rax, (%rdi)
ret
+STUB(pthread__atomic_or_ulong)
+ lock
+ orq %rsi, (%rdi)
+ ret
+
STUB(pthread__membar_full)
lock
addq $0, -8(%rsp)
diff --git a/lib/libpthread/arch/x86_64/pthread_md.h b/lib/libpthread/arch/x86_64/pthread_md.h
index a3a79b847d4..7f04edfd85a 100644
--- a/lib/libpthread/arch/x86_64/pthread_md.h
+++ b/lib/libpthread/arch/x86_64/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.6 2007/09/24 12:19:40 skrll Exp $ */
+/* $NetBSD: pthread_md.h,v 1.7 2007/11/13 17:20:10 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@@ -106,4 +106,30 @@ pthread__sp(void)
#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory")
#define PTHREAD__HAVE_ATOMIC
+static inline void *
+pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new)
+{
+ volatile uintptr_t *cast = ptr;
+ void *ret;
+
+ __asm __volatile ("lock; cmpxchgq %2, %1"
+ : "=a" (ret), "=m" (*cast)
+ : "r" (new), "m" (*cast), "0" (old));
+
+ return ret;
+}
+
+static inline void *
+pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new)
+{
+ volatile uintptr_t *cast = ptr;
+ void *ret;
+
+ __asm __volatile ("cmpxchgq %2, %1"
+ : "=a" (ret), "=m" (*cast)
+ : "r" (new), "m" (*cast), "0" (old));
+
+ return ret;
+}
+
#endif /* _LIB_PTHREAD_X86_64_MD_H */
diff --git a/lib/libpthread/pthread.c b/lib/libpthread/pthread.c
index 4d2a2fdb087..78de6571e5d 100644
--- a/lib/libpthread/pthread.c
+++ b/lib/libpthread/pthread.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread.c,v 1.87 2007/11/13 15:57:10 ad Exp $ */
+/* $NetBSD: pthread.c,v 1.88 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
@@ -37,13 +37,14 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread.c,v 1.87 2007/11/13 15:57:10 ad Exp $");
+__RCSID("$NetBSD: pthread.c,v 1.88 2007/11/13 17:20:09 ad Exp $");
#define __EXPOSE_STACK 1
#include <sys/param.h>
#include <sys/mman.h>
#include <sys/sysctl.h>
+#include <sys/lwpctl.h>
#include <err.h>
#include <errno.h>
@@ -68,22 +69,24 @@ static int pthread__cmp(struct __pthread_st *, struct __pthread_st *);
RB_PROTOTYPE_STATIC(__pthread__alltree, __pthread_st, pt_alltree, pthread__cmp)
#endif
-static void pthread__create_tramp(void *(*)(void *), void *);
+static void pthread__create_tramp(pthread_t, void *(*)(void *), void *);
static void pthread__initthread(pthread_t);
static void pthread__scrubthread(pthread_t, char *, int);
static int pthread__stackid_setup(void *, size_t, pthread_t *);
static int pthread__stackalloc(pthread_t *);
static void pthread__initmain(pthread_t *);
+static void pthread__fork_callback(void);
void pthread__init(void);
int pthread__started;
-
pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
pthread_queue_t pthread__deadqueue;
pthread_queue_t pthread__allqueue;
static pthread_attr_t pthread_default_attr;
+static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
+static pthread_t pthread__first;
enum {
DIAGASSERT_ABORT = 1<<0,
@@ -191,6 +194,13 @@ pthread__init(void)
PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
RB_INSERT(__pthread__alltree, &pthread__alltree, first);
+ /*
+ * Don't need lwpctl if running on a uniprocessor.
+ * Just use the dummy block.
+ */
+ if (pthread__concurrency > 1)
+ (void)_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl);
+
/* Start subsystems */
PTHREAD_MD_INIT
pthread__debug_init();
@@ -218,14 +228,27 @@ pthread__init(void)
}
}
-
/* Tell libc that we're here and it should role-play accordingly. */
+ pthread__first = first;
+ pthread_atfork(NULL, NULL, pthread__fork_callback);
__isthreaded = 1;
}
static void
+pthread__fork_callback(void)
+{
+
+ /* lwpctl state is not copied across fork. */
+ if (pthread__concurrency > 1) {
+ (void)_lwp_ctl(LWPCTL_FEATURE_CURCPU,
+ &pthread__first->pt_lwpctl);
+ }
+}
+
+static void
pthread__child_callback(void)
{
+
/*
* Clean up data structures that a forked child process might
* trip over. Note that if threads have been created (causing
@@ -268,6 +291,8 @@ pthread__initthread(pthread_t t)
t->pt_signalled = 0;
t->pt_havespecific = 0;
t->pt_early = NULL;
+ t->pt_lwpctl = &pthread__dummy_lwpctl;
+ t->pt_blocking = 0;
memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
pthread_mutex_init(&t->pt_lock, NULL);
@@ -339,8 +364,10 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr,
pthread_mutex_unlock(&pthread__deadqueue_lock);
if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) {
/* Still running? */
- if (_lwp_kill(newthread->pt_lid, 0) == 0 ||
- errno != ESRCH) {
+ if (newthread->pt_lwpctl->lc_curcpu !=
+ LWPCTL_CPU_EXITED &&
+ (_lwp_kill(newthread->pt_lid, 0) == 0 ||
+ errno != ESRCH)) {
pthread_mutex_lock(
&pthread__deadqueue_lock);
PTQ_INSERT_TAIL(&pthread__deadqueue,
@@ -388,8 +415,8 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr,
* Create the new LWP.
*/
pthread__scrubthread(newthread, name, nattr.pta_flags);
- makecontext(&newthread->pt_uc, pthread__create_tramp, 2,
- startfunc, arg);
+ makecontext(&newthread->pt_uc, pthread__create_tramp, 3,
+ newthread, startfunc, arg);
flag = 0;
if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0)
@@ -413,11 +440,15 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr,
static void
-pthread__create_tramp(void *(*start)(void *), void *arg)
+pthread__create_tramp(pthread_t self, void *(*start)(void *), void *arg)
{
- pthread_t self;
void *retval;
+#ifdef PTHREAD__HAVE_THREADREG
+ /* Set up identity register. */
+ pthread__threadreg_set(self);
+#endif
+
/*
* Throw away some stack in a feeble attempt to reduce cache
* thrash. May help for SMT processors. XXX We should not
@@ -426,16 +457,22 @@ pthread__create_tramp(void *(*start)(void *), void *arg)
* that pt_lid may not be set by this point, but we don't
* care.
*/
- self = pthread__self();
(void)alloca(((unsigned)self->pt_lid & 7) << 8);
if (self->pt_name != NULL) {
pthread_mutex_lock(&self->pt_lock);
if (self->pt_name != NULL)
- (void)_lwp_setname(_lwp_self(), self->pt_name);
+ (void)_lwp_setname(0, self->pt_name);
pthread_mutex_unlock(&self->pt_lock);
}
+ /*
+ * Don't need lwpctl if running on a uniprocessor.
+ * Just use the dummy block.
+ */
+ if (pthread__concurrency > 1)
+ (void)_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl);
+
retval = (*start)(arg);
pthread_exit(retval);
@@ -947,6 +984,14 @@ pthread__park(pthread_t self, pthread_spin_t *lock,
/* Clear the willpark flag, since we're about to block. */
self->pt_willpark = 0;
+ /*
+ * For non-interlocked release of mutexes we need a store
+ * barrier before incrementing pt_blocking away from zero.
+ * This is provided by the caller (it will release an
+ * interlock, or do an explicit barrier).
+ */
+ self->pt_blocking++;
+
/*
* Kernels before 4.99.27 can't park and unpark in one step,
* so take care of it now if on an old kernel.
@@ -1034,6 +1079,7 @@ pthread__park(pthread_t self, pthread_spin_t *lock,
pthread__spinunlock(self, lock);
}
self->pt_early = NULL;
+ self->pt_blocking--;
return rv;
}
@@ -1260,6 +1306,11 @@ pthread__initmain(pthread_t *newt)
}
*newt = t;
+
+#ifdef PTHREAD__HAVE_THREADREG
+ /* Set up identity register. */
+ pthread__threadreg_set(t);
+#endif
}
static int
diff --git a/lib/libpthread/pthread_int.h b/lib/libpthread/pthread_int.h
index 77b7ee371d4..913d4a4a380 100644
--- a/lib/libpthread/pthread_int.h
+++ b/lib/libpthread/pthread_int.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_int.h,v 1.60 2007/11/13 15:57:11 ad Exp $ */
+/* $NetBSD: pthread_int.h,v 1.61 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
@@ -130,6 +130,8 @@ struct __pthread_st {
* on other CPUs will access this data frequently.
*/
int pt_dummy1 __aligned(128);
+ struct lwpctl *pt_lwpctl; /* Kernel/user comms area */
+ volatile int pt_blocking; /* Blocking in userspace */
volatile int pt_rwlocked; /* Handed rwlock successfully */
volatile int pt_sleeponq; /* On a sleep queue */
volatile int pt_signalled; /* Received pthread_cond_signal() */
@@ -252,18 +254,16 @@ int pthread__find(pthread_t) PTHREAD_HIDE;
_INITCONTEXT_U_MD(ucp) \
} while (/*CONSTCOND*/0)
-#ifdef PTHREAD_MACHINE_HAS_ID_REGISTER
-#define pthread__id(reg) (reg)
-#else
/* Stack location of pointer to a particular thread */
#define pthread__id(sp) \
((pthread_t) (((vaddr_t)(sp)) & pthread__threadmask))
-#define pthread__id_reg() pthread__sp()
+#ifdef PTHREAD__HAVE_THREADREG
+#define pthread__self() pthread__threadreg_get()
+#else
+#define pthread__self() (pthread__id(pthread__sp()))
#endif
-#define pthread__self() (pthread__id(pthread__id_reg()))
-
#define pthread__abort() \
pthread__assertfunc(__FILE__, __LINE__, __func__, "unreachable")
@@ -286,14 +286,14 @@ void pthread__errorfunc(const char *, int, const char *, const char *)
PTHREAD_HIDE;
char *pthread__getenv(const char *) PTHREAD_HIDE;
-int pthread__atomic_cas_ptr(volatile void *, void **, void *) PTHREAD_HIDE;
-void *pthread__atomic_swap_ptr(volatile void *, void *) PTHREAD_HIDE;
+void *pthread__atomic_cas_ptr(volatile void *, const void *, const void *) PTHREAD_HIDE;
+void *pthread__atomic_swap_ptr(volatile void *, const void *) PTHREAD_HIDE;
+void pthread__atomic_or_ulong(volatile unsigned long *, unsigned long) PTHREAD_HIDE;
void pthread__membar_full(void) PTHREAD_HIDE;
void pthread__membar_producer(void) PTHREAD_HIDE;
void pthread__membar_consumer(void) PTHREAD_HIDE;
int pthread__mutex_deferwake(pthread_t, pthread_mutex_t *) PTHREAD_HIDE;
-int pthread__mutex_catchup(pthread_mutex_t *) PTHREAD_HIDE;
#ifndef pthread__smt_pause
#define pthread__smt_pause() /* nothing */
diff --git a/lib/libpthread/pthread_misc.c b/lib/libpthread/pthread_misc.c
index 62089aa7664..47b01eaeb07 100644
--- a/lib/libpthread/pthread_misc.c
+++ b/lib/libpthread/pthread_misc.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_misc.c,v 1.2 2007/08/16 13:54:17 ad Exp $ */
+/* $NetBSD: pthread_misc.c,v 1.3 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_misc.c,v 1.2 2007/08/16 13:54:17 ad Exp $");
+__RCSID("$NetBSD: pthread_misc.c,v 1.3 2007/11/13 17:20:09 ad Exp $");
#include <sys/types.h>
#include <sys/signal.h>
@@ -49,13 +49,16 @@ __RCSID("$NetBSD: pthread_misc.c,v 1.2 2007/08/16 13:54:17 ad Exp $");
#include "pthread.h"
#include "pthread_int.h"
+int pthread__sched_yield(void);
+
int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
int _sys_nanosleep(const struct timespec *, struct timespec *);
+int _sys_sched_yield(void);
__strong_alias(_nanosleep, nanosleep)
__strong_alias(__libc_thr_sigsetmask,pthread_sigmask)
__strong_alias(__sigprocmask14,pthread_sigmask)
-__strong_alias(__libc_thr_yield,_sys_sched_yield)
+__strong_alias(__libc_thr_yield,pthread__sched_yield)
/*ARGSUSED*/
int
@@ -109,3 +112,22 @@ nanosleep(const struct timespec *rqtp, struct timespec *rmtp)
*/
return _sys_nanosleep(rqtp, rmtp);
}
+
+int
+pthread__sched_yield(void)
+{
+ pthread_t self;
+ int error;
+
+ self = pthread__self();
+
+#ifdef PTHREAD__HAVE_ATOMIC
+ /* Memory barrier for unlocked mutex release. */
+ pthread__membar_producer();
+#endif
+ self->pt_blocking++;
+ error = _sys_sched_yield();
+ self->pt_blocking--;
+
+ return error;
+}
diff --git a/lib/libpthread/pthread_mutex2.c b/lib/libpthread/pthread_mutex2.c
index a115a8d50de..233d0bce528 100644
--- a/lib/libpthread/pthread_mutex2.c
+++ b/lib/libpthread/pthread_mutex2.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_mutex2.c,v 1.10 2007/10/04 01:46:49 ad Exp $ */
+/* $NetBSD: pthread_mutex2.c,v 1.11 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc.
@@ -37,7 +37,10 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_mutex2.c,v 1.10 2007/10/04 01:46:49 ad Exp $");
+__RCSID("$NetBSD: pthread_mutex2.c,v 1.11 2007/11/13 17:20:09 ad Exp $");
+
+#include <sys/types.h>
+#include <sys/lwpctl.h>
#include <errno.h>
#include <limits.h>
@@ -57,13 +60,12 @@ __RCSID("$NetBSD: pthread_mutex2.c,v 1.10 2007/10/04 01:46:49 ad Exp $");
*/
#define pt_nextwaiter pt_sleep.ptqe_next
#define ptm_waiters ptm_blocked.ptqh_first
-#define ptm_errorcheck ptm_blocked.ptqh_last
-
-#define MUTEX_WAITERS_BIT (0x01UL)
-#define MUTEX_RECURSIVE_BIT (0x02UL)
-#define MUTEX_THREAD (-16L)
+#define ptm_errorcheck ptm_lock
-#define MUTEX_DEFERRED(x) (*(char *)&(x)->ptm_lock)
+#define MUTEX_WAITERS_BIT ((uintptr_t)0x01)
+#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02)
+#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04)
+#define MUTEX_THREAD ((uintptr_t)-16L)
#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
@@ -98,8 +100,21 @@ __strong_alias(__libc_thr_once,pthread_once)
static inline int
mutex_cas(volatile void *ptr, void **old, void *new)
{
+ void *oldv;
- return pthread__atomic_cas_ptr(ptr, old, new);
+ oldv = *old;
+ *old = pthread__atomic_cas_ptr(ptr, oldv, new);
+ return *old == oldv;
+}
+
+static inline int
+mutex_cas_ni(volatile void *ptr, void **old, void *new)
+{
+ void *oldv;
+
+ oldv = *old;
+ *old = pthread__atomic_cas_ptr_ni(ptr, oldv, new);
+ return *old == oldv;
}
int
@@ -114,15 +129,15 @@ pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
switch (type) {
case PTHREAD_MUTEX_ERRORCHECK:
- ptm->ptm_errorcheck = (void *)1;
+ ptm->ptm_errorcheck = 1;
ptm->ptm_owner = NULL;
break;
case PTHREAD_MUTEX_RECURSIVE:
- ptm->ptm_errorcheck = NULL;
+ ptm->ptm_errorcheck = 0;
ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
break;
default:
- ptm->ptm_errorcheck = NULL;
+ ptm->ptm_errorcheck = 0;
ptm->ptm_owner = NULL;
break;
}
@@ -130,7 +145,6 @@ pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
ptm->ptm_magic = _PT_MUTEX_MAGIC;
ptm->ptm_waiters = NULL;
ptm->ptm_private = NULL;
- MUTEX_DEFERRED(ptm) = 0;
return 0;
}
@@ -149,19 +163,6 @@ pthread_mutex_destroy(pthread_mutex_t *ptm)
return 0;
}
-
-/*
- * Note regarding memory visibility: Pthreads has rules about memory
- * visibility and mutexes. Very roughly: Memory a thread can see when
- * it unlocks a mutex can be seen by another thread that locks the
- * same mutex.
- *
- * A memory barrier after a lock and before an unlock will provide
- * this behavior. This code relies on mutex_cas() to issue a barrier
- * after obtaining a lock, and on pthread__simple_unlock() to issue
- * a barrier before releasing a lock.
- */
-
int
pthread_mutex_lock(pthread_mutex_t *ptm)
{
@@ -173,7 +174,6 @@ pthread_mutex_lock(pthread_mutex_t *ptm)
if (__predict_true(mutex_cas(&ptm->ptm_owner, &owner, self)))
return 0;
-
return pthread__mutex_lock_slow(ptm);
}
@@ -185,12 +185,41 @@ pthread__mutex_pause(void)
pthread__smt_pause();
}
+/*
+ * Spin while the holder is running. 'lwpctl' gives us the true
+ * status of the thread. pt_blocking is set by libpthread in order
+ * to cut out system call and kernel spinlock overhead on remote CPUs
+ * (could represent many thousands of clock cycles). pt_blocking also
+ * makes this thread yield if the target is calling sched_yield().
+ */
+NOINLINE static void *
+pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner, uintptr_t mask)
+{
+ pthread_t thread;
+
+ for (;; owner = ptm->ptm_owner) {
+ thread = (pthread_t)MUTEX_OWNER(owner);
+ if (((uintptr_t)owner & mask) != 0)
+ break;
+ if (thread == NULL)
+ break;
+ if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE ||
+ thread->pt_blocking)
+ break;
+ pthread__mutex_pause();
+ pthread__mutex_pause();
+ pthread__mutex_pause();
+ pthread__mutex_pause();
+ }
+
+ return owner;
+}
+
NOINLINE static int
pthread__mutex_lock_slow(pthread_mutex_t *ptm)
{
void *waiters, *new, *owner;
pthread_t self;
- int count;
pthread__error(EINVAL, "Invalid mutex",
ptm->ptm_magic == _PT_MUTEX_MAGIC);
@@ -210,14 +239,14 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm)
return EDEADLK;
}
- /* Spin for a while. */
- count = pthread__nspins;
- while (MUTEX_OWNER(owner) != 0 && --count > 0) {
- pthread__mutex_pause();
- owner = ptm->ptm_owner;
- }
-
for (;; owner = ptm->ptm_owner) {
+ /*
+ * Spin while the owner is running, but block if the
+ * mutex is so heavily contested that there are existing
+ * waiters.
+ */
+ owner = pthread__mutex_spin(ptm, owner, MUTEX_WAITERS_BIT);
+
/* If it has become free, try to acquire it again. */
while (MUTEX_OWNER(owner) == 0) {
new = (void *)((uintptr_t)self | (uintptr_t)owner);
@@ -263,8 +292,21 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm)
break;
}
new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT);
- if (mutex_cas(&ptm->ptm_owner, &owner, new))
- break;
+ if (mutex_cas(&ptm->ptm_owner, &owner, new)) {
+ /*
+ * pthread_mutex_unlock() can do a
+ * non-interlocked CAS. We cannot
+ * know if our attempt to set the
+ * waiters bit has succeeded while
+ * the holding thread is running.
+ * There are many assumptions; see
+ * sys/kern/kern_mutex.c for details.
+ * In short, we must spin if we see
+ * that the holder is running again.
+ */
+ pthread__membar_full();
+ owner = pthread__mutex_spin(ptm, owner, 0);
+ }
}
/*
@@ -275,7 +317,9 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm)
* the thread onto the waiters list.
*/
while (self->pt_sleeponq) {
+ self->pt_blocking++;
(void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL);
+ self->pt_blocking--;
}
}
}
@@ -302,71 +346,37 @@ pthread_mutex_trylock(pthread_mutex_t *ptm)
return EBUSY;
}
-NOINLINE int
-pthread__mutex_catchup(pthread_mutex_t *ptm)
-{
- pthread_t self;
-
- self = pthread__self();
-
- if (self->pt_nwaiters == 1) {
- /*
- * If the calling thread is about to block, defer
- * unparking the target until _lwp_park() is called.
- */
- if (self->pt_willpark && self->pt_unpark == 0) {
- self->pt_unpark = self->pt_waiters[0];
- self->pt_unparkhint = &ptm->ptm_waiters;
- } else {
- (void)_lwp_unpark(self->pt_waiters[0],
- &ptm->ptm_waiters);
- }
- } else {
- (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
- &ptm->ptm_waiters);
- }
- self->pt_nwaiters = 0;
-
- return 0;
-}
-
int
pthread_mutex_unlock(pthread_mutex_t *ptm)
{
void *owner;
pthread_t self;
- char deferred;
self = pthread__self();
owner = self;
- deferred = MUTEX_DEFERRED(ptm);
- MUTEX_DEFERRED(ptm) = 0;
-
- if (__predict_false(!mutex_cas(&ptm->ptm_owner, &owner, NULL)))
- return pthread__mutex_unlock_slow(ptm);
/*
- * There were no waiters, but we may have deferred waking
- * other threads until mutex unlock - we must wake them now.
+ * Note this may be a non-interlocked CAS. See lock_slow()
+ * above and sys/kern/kern_mutex.c for details.
*/
- if (deferred)
- return pthread__mutex_catchup(ptm);
-
- return 0;
+ if (__predict_true(mutex_cas_ni(&ptm->ptm_owner, &owner, NULL)))
+ return 0;
+ return pthread__mutex_unlock_slow(ptm);
}
NOINLINE static int
pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
{
pthread_t self, owner, new;
- int weown, error;
+ int weown, error, deferred;
pthread__error(EINVAL, "Invalid mutex",
ptm->ptm_magic == _PT_MUTEX_MAGIC);
- self = pthread_self();
+ self = pthread__self();
owner = ptm->ptm_owner;
weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
+ deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT);
error = 0;
if (ptm->ptm_errorcheck) {
@@ -410,8 +420,27 @@ pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
* There were no waiters, but we may have deferred waking
* other threads until mutex unlock - we must wake them now.
*/
- if (self->pt_nwaiters != 0)
- return pthread__mutex_catchup(ptm);
+ if (!deferred)
+ return error;
+
+ if (self->pt_nwaiters == 1) {
+ /*
+ * If the calling thread is about to block, defer
+ * unparking the target until _lwp_park() is called.
+ */
+ if (self->pt_willpark && self->pt_unpark == 0) {
+ self->pt_unpark = self->pt_waiters[0];
+ self->pt_unparkhint = &ptm->ptm_waiters;
+ } else {
+ (void)_lwp_unpark(self->pt_waiters[0],
+ &ptm->ptm_waiters);
+ }
+ } else {
+ (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
+ &ptm->ptm_waiters);
+ }
+ self->pt_nwaiters = 0;
+
return error;
}
@@ -559,7 +588,9 @@ pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm)
if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread)
return 0;
- MUTEX_DEFERRED(ptm) = 1;
+ pthread__atomic_or_ulong((volatile unsigned long *)
+ (uintptr_t)&ptm->ptm_owner,
+ MUTEX_DEFERRED_BIT);
return 1;
}
diff --git a/lib/libpthread/pthread_rwlock2.c b/lib/libpthread/pthread_rwlock2.c
index 46e0cf8d234..cee55ea73a0 100644
--- a/lib/libpthread/pthread_rwlock2.c
+++ b/lib/libpthread/pthread_rwlock2.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_rwlock2.c,v 1.6 2007/11/13 15:57:13 ad Exp $ */
+/* $NetBSD: pthread_rwlock2.c,v 1.7 2007/11/13 17:20:09 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_rwlock2.c,v 1.6 2007/11/13 15:57:13 ad Exp $");
+__RCSID("$NetBSD: pthread_rwlock2.c,v 1.7 2007/11/13 17:20:09 ad Exp $");
#include <errno.h>
#include <stddef.h>
@@ -66,8 +66,12 @@ __strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy)
static inline int
rw_cas(pthread_rwlock_t *ptr, uintptr_t *o, uintptr_t n)
{
+ uintptr_t oldv;
- return pthread__atomic_cas_ptr(&ptr->ptr_owner, (void **)o, (void *)n);
+ oldv = *o;
+ *o = (uintptr_t)pthread__atomic_cas_ptr(&ptr->ptr_owner, (void *)oldv,
+ (void *)n);
+ return *o == oldv;
}
int