diff options
| author | ad <ad@NetBSD.org> | 2007-11-13 17:20:08 +0000 |
|---|---|---|
| committer | ad <ad@NetBSD.org> | 2007-11-13 17:20:08 +0000 |
| commit | 66ac2ffaf293a1766211e270fcce0ba856f8f80e (patch) | |
| tree | 14c1c20985051c159c5d7b74ce133f33a8efdd64 | |
| parent | 3c776c8aa6570d9365bd4dc5468590e6c51103bb (diff) | |
Mutexes:
- Play scrooge again and chop more cycles off acquire/release.
- Spin while the lock holder is running on another CPU (adaptive mutexes).
- Do non-atomic release.
Threadreg:
- Add the necessary hooks to use a thread register.
- Add the code for i386, using %gs.
- Leave i386 code disabled until xen and COMPAT_NETBSD32 have the changes.
| -rw-r--r-- | lib/libpthread/Makefile | 7 | ||||
| -rw-r--r-- | lib/libpthread/TODO | 13 | ||||
| -rw-r--r-- | lib/libpthread/arch/i386/_context_u.S | 23 | ||||
| -rw-r--r-- | lib/libpthread/arch/i386/pthread_md.c | 16 | ||||
| -rw-r--r-- | lib/libpthread/arch/i386/pthread_md.h | 43 | ||||
| -rw-r--r-- | lib/libpthread/arch/x86_64/_context_u.S | 20 | ||||
| -rw-r--r-- | lib/libpthread/arch/x86_64/pthread_md.h | 28 | ||||
| -rw-r--r-- | lib/libpthread/pthread.c | 77 | ||||
| -rw-r--r-- | lib/libpthread/pthread_int.h | 20 | ||||
| -rw-r--r-- | lib/libpthread/pthread_misc.c | 28 | ||||
| -rw-r--r-- | lib/libpthread/pthread_mutex2.c | 195 | ||||
| -rw-r--r-- | lib/libpthread/pthread_rwlock2.c | 10 |
12 files changed, 322 insertions, 158 deletions
diff --git a/lib/libpthread/Makefile b/lib/libpthread/Makefile index 412f4c1d0ac..d5c905ff83c 100644 --- a/lib/libpthread/Makefile +++ b/lib/libpthread/Makefile @@ -1,4 +1,4 @@ -# $NetBSD: Makefile,v 1.45 2007/10/13 20:36:43 jnemeth Exp $ +# $NetBSD: Makefile,v 1.46 2007/11/13 17:20:08 ad Exp $ # WARNS= 4 @@ -84,6 +84,11 @@ pthread_lock.po: pthread_lock.o ${_MKTARGET_CREATE} cp pthread_lock.o pthread_lock.po +COPTS.pthread_mutex2.c+= -fomit-frame-pointer -falign-functions=32 +pthread_mutex2.po: pthread_mutex2.o + ${_MKTARGET_CREATE} + cp pthread_mutex2.o pthread_mutex2.po + COPTS.pthread.c += -Wno-stack-protector _context_u.po: _context_u.o diff --git a/lib/libpthread/TODO b/lib/libpthread/TODO index 89915e9917b..dcff58bd0a1 100644 --- a/lib/libpthread/TODO +++ b/lib/libpthread/TODO @@ -1,4 +1,4 @@ -$NetBSD: TODO,v 1.11 2007/09/08 23:00:31 ad Exp $ +$NetBSD: TODO,v 1.12 2007/11/13 17:20:09 ad Exp $ Bugs to fix: @@ -15,16 +15,5 @@ Interfaces/features to implement: - Allow threads to change their stack size. This probably depends on the above item. -- Have a user/kernel shared page that: - - o Has a hint mechanism that gives us a clue about whether an LWP is - currently running on another CPU. This could be used for adaptive locks, - but would need to be cheap to do in-kernel. - - o Perhaps has a flag value that's reset when a detached LWP is into the - kernel and lwp_exit1(), meaning that its stack can be reclaimed. Again, - may or may not be worth it. Currently we test for this with _lwp_kill() - which is very inefficient. - - Keep a pool of dead LWPs so that we do not have take the full hit of _lwp_create() every time pthread_create() is called. diff --git a/lib/libpthread/arch/i386/_context_u.S b/lib/libpthread/arch/i386/_context_u.S index 3dac247c3bd..6720bd67402 100644 --- a/lib/libpthread/arch/i386/_context_u.S +++ b/lib/libpthread/arch/i386/_context_u.S @@ -1,4 +1,4 @@ -/* $NetBSD: _context_u.S,v 1.6 2007/11/13 15:57:14 ad Exp $ */ +/* $NetBSD: _context_u.S,v 1.7 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc. @@ -101,26 +101,17 @@ STUB(_swapcontext_u_xmm) movl 8(%esp), %ecx SETC -STUB(pthread__atomic_cas_ptr) +STUB(pthread__atomic_swap_ptr) movl 4(%esp), %ecx movl 8(%esp), %eax - movl 12(%esp), %edx - movl (%eax), %eax - lock - cmpxchgl %edx, (%ecx) - jnz 1f - mov $1, %eax - ret -1: - mov 8(%esp), %edx - mov %eax, (%edx) - mov $0, %eax + xchgl %eax, (%ecx) ret -STUB(pthread__atomic_swap_ptr) +STUB(pthread__atomic_or_ulong) movl 4(%esp), %ecx movl 8(%esp), %eax - xchgl %eax, (%ecx) + lock + orl %eax, (%ecx) ret STUB(pthread__membar_full) @@ -130,7 +121,7 @@ STUB(pthread__membar_full) STUB(pthread__membar_producer) /* A store is enough */ - movl $0, -4(%esp) + addl $0, -4(%esp) ret STUB(pthread__membar_consumer) diff --git a/lib/libpthread/arch/i386/pthread_md.c b/lib/libpthread/arch/i386/pthread_md.c index 0584a3b9b24..7dfabf2e0de 100644 --- a/lib/libpthread/arch/i386/pthread_md.c +++ b/lib/libpthread/arch/i386/pthread_md.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.c,v 1.3 2003/03/08 08:03:37 lukem Exp $ */ +/* $NetBSD: pthread_md.c,v 1.4 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2002 The NetBSD Foundation, Inc. @@ -37,10 +37,12 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_md.c,v 1.3 2003/03/08 08:03:37 lukem Exp $"); +__RCSID("$NetBSD: pthread_md.c,v 1.4 2007/11/13 17:20:09 ad Exp $"); #include <sys/param.h> #include <sys/sysctl.h> + +#include <machine/sysarch.h> #include <machine/cpu.h> #include "pthread.h" @@ -54,7 +56,8 @@ int (*_md_swapcontext_u)(ucontext_t *, const ucontext_t *); * Initialize the function pointers for get/setcontext to the * old-fp-save (s87) or new-fxsave (xmm) FP routines. */ -void pthread__i386_init(void) +void +pthread__i386_init(void) { int mib[2]; size_t len; @@ -77,3 +80,10 @@ void pthread__i386_init(void) } } + +void +pthread__threadreg_set(pthread_t self) +{ + + sysarch(I386_SET_GSBASE, &self); +} diff --git a/lib/libpthread/arch/i386/pthread_md.h b/lib/libpthread/arch/i386/pthread_md.h index e22fdc0c3de..c99ae7f412e 100644 --- a/lib/libpthread/arch/i386/pthread_md.h +++ b/lib/libpthread/arch/i386/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.10 2007/09/24 12:19:40 skrll Exp $ */ +/* $NetBSD: pthread_md.h,v 1.11 2007/11/13 17:20:10 ad Exp $ */ /*- * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc. @@ -183,5 +183,46 @@ pthread__sp(void) #define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory") #define PTHREAD__HAVE_ATOMIC +/* #define PTHREAD__HAVE_THREADREG */ + +void pthread__threadreg_set(pthread_t); + +static inline pthread_t +pthread__threadreg_get(void) +{ + pthread_t self; + + __asm volatile("movl %%gs:0, %0" + : "=r" (self) + :); + + return self; +} + +static inline void * +pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new) +{ + volatile uintptr_t *cast = ptr; + void *ret; + + __asm __volatile ("lock; cmpxchgl %2, %1" + : "=a" (ret), "=m" (*cast) + : "r" (new), "m" (*cast), "0" (old)); + + return ret; +} + +static inline void * +pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new) +{ + volatile uintptr_t *cast = ptr; + void *ret; + + __asm __volatile ("cmpxchgl %2, %1" + : "=a" (ret), "=m" (*cast) + : "r" (new), "m" (*cast), "0" (old)); + + return ret; +} #endif /* _LIB_PTHREAD_I386_MD_H */ diff --git a/lib/libpthread/arch/x86_64/_context_u.S b/lib/libpthread/arch/x86_64/_context_u.S index cdb5c4a502c..8719b4fff5e 100644 --- a/lib/libpthread/arch/x86_64/_context_u.S +++ b/lib/libpthread/arch/x86_64/_context_u.S @@ -1,4 +1,4 @@ -/* $NetBSD: _context_u.S,v 1.8 2007/11/13 15:57:14 ad Exp $ */ +/* $NetBSD: _context_u.S,v 1.9 2007/11/13 17:20:10 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -138,23 +138,17 @@ STUB(_swapcontext_u) movq %rsi, %rdi SETC -STUB(pthread__atomic_cas_ptr) - movq (%rsi), %rax - lock - cmpxchgq %rdx, (%rdi) - jnz 1f - movq $1, %rax - ret -1: - movq %rax, (%rsi) - movq $0, %rax - ret - STUB(pthread__atomic_swap_ptr) movq %rsi, %rax + lock xchgq %rax, (%rdi) ret +STUB(pthread__atomic_or_ulong) + lock + orq %rsi, (%rdi) + ret + STUB(pthread__membar_full) lock addq $0, -8(%rsp) diff --git a/lib/libpthread/arch/x86_64/pthread_md.h b/lib/libpthread/arch/x86_64/pthread_md.h index a3a79b847d4..7f04edfd85a 100644 --- a/lib/libpthread/arch/x86_64/pthread_md.h +++ b/lib/libpthread/arch/x86_64/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.6 2007/09/24 12:19:40 skrll Exp $ */ +/* $NetBSD: pthread_md.h,v 1.7 2007/11/13 17:20:10 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -106,4 +106,30 @@ pthread__sp(void) #define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory") #define PTHREAD__HAVE_ATOMIC +static inline void * +pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new) +{ + volatile uintptr_t *cast = ptr; + void *ret; + + __asm __volatile ("lock; cmpxchgq %2, %1" + : "=a" (ret), "=m" (*cast) + : "r" (new), "m" (*cast), "0" (old)); + + return ret; +} + +static inline void * +pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new) +{ + volatile uintptr_t *cast = ptr; + void *ret; + + __asm __volatile ("cmpxchgq %2, %1" + : "=a" (ret), "=m" (*cast) + : "r" (new), "m" (*cast), "0" (old)); + + return ret; +} + #endif /* _LIB_PTHREAD_X86_64_MD_H */ diff --git a/lib/libpthread/pthread.c b/lib/libpthread/pthread.c index 4d2a2fdb087..78de6571e5d 100644 --- a/lib/libpthread/pthread.c +++ b/lib/libpthread/pthread.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread.c,v 1.87 2007/11/13 15:57:10 ad Exp $ */ +/* $NetBSD: pthread.c,v 1.88 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,13 +37,14 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread.c,v 1.87 2007/11/13 15:57:10 ad Exp $"); +__RCSID("$NetBSD: pthread.c,v 1.88 2007/11/13 17:20:09 ad Exp $"); #define __EXPOSE_STACK 1 #include <sys/param.h> #include <sys/mman.h> #include <sys/sysctl.h> +#include <sys/lwpctl.h> #include <err.h> #include <errno.h> @@ -68,22 +69,24 @@ static int pthread__cmp(struct __pthread_st *, struct __pthread_st *); RB_PROTOTYPE_STATIC(__pthread__alltree, __pthread_st, pt_alltree, pthread__cmp) #endif -static void pthread__create_tramp(void *(*)(void *), void *); +static void pthread__create_tramp(pthread_t, void *(*)(void *), void *); static void pthread__initthread(pthread_t); static void pthread__scrubthread(pthread_t, char *, int); static int pthread__stackid_setup(void *, size_t, pthread_t *); static int pthread__stackalloc(pthread_t *); static void pthread__initmain(pthread_t *); +static void pthread__fork_callback(void); void pthread__init(void); int pthread__started; - pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; pthread_queue_t pthread__deadqueue; pthread_queue_t pthread__allqueue; static pthread_attr_t pthread_default_attr; +static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; +static pthread_t pthread__first; enum { DIAGASSERT_ABORT = 1<<0, @@ -191,6 +194,13 @@ pthread__init(void) PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); RB_INSERT(__pthread__alltree, &pthread__alltree, first); + /* + * Don't need lwpctl if running on a uniprocessor. + * Just use the dummy block. + */ + if (pthread__concurrency > 1) + (void)_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl); + /* Start subsystems */ PTHREAD_MD_INIT pthread__debug_init(); @@ -218,14 +228,27 @@ pthread__init(void) } } - /* Tell libc that we're here and it should role-play accordingly. */ + pthread__first = first; + pthread_atfork(NULL, NULL, pthread__fork_callback); __isthreaded = 1; } static void +pthread__fork_callback(void) +{ + + /* lwpctl state is not copied across fork. */ + if (pthread__concurrency > 1) { + (void)_lwp_ctl(LWPCTL_FEATURE_CURCPU, + &pthread__first->pt_lwpctl); + } +} + +static void pthread__child_callback(void) { + /* * Clean up data structures that a forked child process might * trip over. Note that if threads have been created (causing @@ -268,6 +291,8 @@ pthread__initthread(pthread_t t) t->pt_signalled = 0; t->pt_havespecific = 0; t->pt_early = NULL; + t->pt_lwpctl = &pthread__dummy_lwpctl; + t->pt_blocking = 0; memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); pthread_mutex_init(&t->pt_lock, NULL); @@ -339,8 +364,10 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, pthread_mutex_unlock(&pthread__deadqueue_lock); if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) { /* Still running? */ - if (_lwp_kill(newthread->pt_lid, 0) == 0 || - errno != ESRCH) { + if (newthread->pt_lwpctl->lc_curcpu != + LWPCTL_CPU_EXITED && + (_lwp_kill(newthread->pt_lid, 0) == 0 || + errno != ESRCH)) { pthread_mutex_lock( &pthread__deadqueue_lock); PTQ_INSERT_TAIL(&pthread__deadqueue, @@ -388,8 +415,8 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, * Create the new LWP. */ pthread__scrubthread(newthread, name, nattr.pta_flags); - makecontext(&newthread->pt_uc, pthread__create_tramp, 2, - startfunc, arg); + makecontext(&newthread->pt_uc, pthread__create_tramp, 3, + newthread, startfunc, arg); flag = 0; if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0) @@ -413,11 +440,15 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, static void -pthread__create_tramp(void *(*start)(void *), void *arg) +pthread__create_tramp(pthread_t self, void *(*start)(void *), void *arg) { - pthread_t self; void *retval; +#ifdef PTHREAD__HAVE_THREADREG + /* Set up identity register. */ + pthread__threadreg_set(self); +#endif + /* * Throw away some stack in a feeble attempt to reduce cache * thrash. May help for SMT processors. XXX We should not @@ -426,16 +457,22 @@ pthread__create_tramp(void *(*start)(void *), void *arg) * that pt_lid may not be set by this point, but we don't * care. */ - self = pthread__self(); (void)alloca(((unsigned)self->pt_lid & 7) << 8); if (self->pt_name != NULL) { pthread_mutex_lock(&self->pt_lock); if (self->pt_name != NULL) - (void)_lwp_setname(_lwp_self(), self->pt_name); + (void)_lwp_setname(0, self->pt_name); pthread_mutex_unlock(&self->pt_lock); } + /* + * Don't need lwpctl if running on a uniprocessor. + * Just use the dummy block. + */ + if (pthread__concurrency > 1) + (void)_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl); + retval = (*start)(arg); pthread_exit(retval); @@ -947,6 +984,14 @@ pthread__park(pthread_t self, pthread_spin_t *lock, /* Clear the willpark flag, since we're about to block. */ self->pt_willpark = 0; + /* + * For non-interlocked release of mutexes we need a store + * barrier before incrementing pt_blocking away from zero. + * This is provided by the caller (it will release an + * interlock, or do an explicit barrier). + */ + self->pt_blocking++; + /* * Kernels before 4.99.27 can't park and unpark in one step, * so take care of it now if on an old kernel. @@ -1034,6 +1079,7 @@ pthread__park(pthread_t self, pthread_spin_t *lock, pthread__spinunlock(self, lock); } self->pt_early = NULL; + self->pt_blocking--; return rv; } @@ -1260,6 +1306,11 @@ pthread__initmain(pthread_t *newt) } *newt = t; + +#ifdef PTHREAD__HAVE_THREADREG + /* Set up identity register. */ + pthread__threadreg_set(t); +#endif } static int diff --git a/lib/libpthread/pthread_int.h b/lib/libpthread/pthread_int.h index 77b7ee371d4..913d4a4a380 100644 --- a/lib/libpthread/pthread_int.h +++ b/lib/libpthread/pthread_int.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_int.h,v 1.60 2007/11/13 15:57:11 ad Exp $ */ +/* $NetBSD: pthread_int.h,v 1.61 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -130,6 +130,8 @@ struct __pthread_st { * on other CPUs will access this data frequently. */ int pt_dummy1 __aligned(128); + struct lwpctl *pt_lwpctl; /* Kernel/user comms area */ + volatile int pt_blocking; /* Blocking in userspace */ volatile int pt_rwlocked; /* Handed rwlock successfully */ volatile int pt_sleeponq; /* On a sleep queue */ volatile int pt_signalled; /* Received pthread_cond_signal() */ @@ -252,18 +254,16 @@ int pthread__find(pthread_t) PTHREAD_HIDE; _INITCONTEXT_U_MD(ucp) \ } while (/*CONSTCOND*/0) -#ifdef PTHREAD_MACHINE_HAS_ID_REGISTER -#define pthread__id(reg) (reg) -#else /* Stack location of pointer to a particular thread */ #define pthread__id(sp) \ ((pthread_t) (((vaddr_t)(sp)) & pthread__threadmask)) -#define pthread__id_reg() pthread__sp() +#ifdef PTHREAD__HAVE_THREADREG +#define pthread__self() pthread__threadreg_get() +#else +#define pthread__self() (pthread__id(pthread__sp())) #endif -#define pthread__self() (pthread__id(pthread__id_reg())) - #define pthread__abort() \ pthread__assertfunc(__FILE__, __LINE__, __func__, "unreachable") @@ -286,14 +286,14 @@ void pthread__errorfunc(const char *, int, const char *, const char *) PTHREAD_HIDE; char *pthread__getenv(const char *) PTHREAD_HIDE; -int pthread__atomic_cas_ptr(volatile void *, void **, void *) PTHREAD_HIDE; -void *pthread__atomic_swap_ptr(volatile void *, void *) PTHREAD_HIDE; +void *pthread__atomic_cas_ptr(volatile void *, const void *, const void *) PTHREAD_HIDE; +void *pthread__atomic_swap_ptr(volatile void *, const void *) PTHREAD_HIDE; +void pthread__atomic_or_ulong(volatile unsigned long *, unsigned long) PTHREAD_HIDE; void pthread__membar_full(void) PTHREAD_HIDE; void pthread__membar_producer(void) PTHREAD_HIDE; void pthread__membar_consumer(void) PTHREAD_HIDE; int pthread__mutex_deferwake(pthread_t, pthread_mutex_t *) PTHREAD_HIDE; -int pthread__mutex_catchup(pthread_mutex_t *) PTHREAD_HIDE; #ifndef pthread__smt_pause #define pthread__smt_pause() /* nothing */ diff --git a/lib/libpthread/pthread_misc.c b/lib/libpthread/pthread_misc.c index 62089aa7664..47b01eaeb07 100644 --- a/lib/libpthread/pthread_misc.c +++ b/lib/libpthread/pthread_misc.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_misc.c,v 1.2 2007/08/16 13:54:17 ad Exp $ */ +/* $NetBSD: pthread_misc.c,v 1.3 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_misc.c,v 1.2 2007/08/16 13:54:17 ad Exp $"); +__RCSID("$NetBSD: pthread_misc.c,v 1.3 2007/11/13 17:20:09 ad Exp $"); #include <sys/types.h> #include <sys/signal.h> @@ -49,13 +49,16 @@ __RCSID("$NetBSD: pthread_misc.c,v 1.2 2007/08/16 13:54:17 ad Exp $"); #include "pthread.h" #include "pthread_int.h" +int pthread__sched_yield(void); + int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); int _sys_nanosleep(const struct timespec *, struct timespec *); +int _sys_sched_yield(void); __strong_alias(_nanosleep, nanosleep) __strong_alias(__libc_thr_sigsetmask,pthread_sigmask) __strong_alias(__sigprocmask14,pthread_sigmask) -__strong_alias(__libc_thr_yield,_sys_sched_yield) +__strong_alias(__libc_thr_yield,pthread__sched_yield) /*ARGSUSED*/ int @@ -109,3 +112,22 @@ nanosleep(const struct timespec *rqtp, struct timespec *rmtp) */ return _sys_nanosleep(rqtp, rmtp); } + +int +pthread__sched_yield(void) +{ + pthread_t self; + int error; + + self = pthread__self(); + +#ifdef PTHREAD__HAVE_ATOMIC + /* Memory barrier for unlocked mutex release. */ + pthread__membar_producer(); +#endif + self->pt_blocking++; + error = _sys_sched_yield(); + self->pt_blocking--; + + return error; +} diff --git a/lib/libpthread/pthread_mutex2.c b/lib/libpthread/pthread_mutex2.c index a115a8d50de..233d0bce528 100644 --- a/lib/libpthread/pthread_mutex2.c +++ b/lib/libpthread/pthread_mutex2.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_mutex2.c,v 1.10 2007/10/04 01:46:49 ad Exp $ */ +/* $NetBSD: pthread_mutex2.c,v 1.11 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,10 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_mutex2.c,v 1.10 2007/10/04 01:46:49 ad Exp $"); +__RCSID("$NetBSD: pthread_mutex2.c,v 1.11 2007/11/13 17:20:09 ad Exp $"); + +#include <sys/types.h> +#include <sys/lwpctl.h> #include <errno.h> #include <limits.h> @@ -57,13 +60,12 @@ __RCSID("$NetBSD: pthread_mutex2.c,v 1.10 2007/10/04 01:46:49 ad Exp $"); */ #define pt_nextwaiter pt_sleep.ptqe_next #define ptm_waiters ptm_blocked.ptqh_first -#define ptm_errorcheck ptm_blocked.ptqh_last - -#define MUTEX_WAITERS_BIT (0x01UL) -#define MUTEX_RECURSIVE_BIT (0x02UL) -#define MUTEX_THREAD (-16L) +#define ptm_errorcheck ptm_lock -#define MUTEX_DEFERRED(x) (*(char *)&(x)->ptm_lock) +#define MUTEX_WAITERS_BIT ((uintptr_t)0x01) +#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02) +#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04) +#define MUTEX_THREAD ((uintptr_t)-16L) #define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT) #define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT) @@ -98,8 +100,21 @@ __strong_alias(__libc_thr_once,pthread_once) static inline int mutex_cas(volatile void *ptr, void **old, void *new) { + void *oldv; - return pthread__atomic_cas_ptr(ptr, old, new); + oldv = *old; + *old = pthread__atomic_cas_ptr(ptr, oldv, new); + return *old == oldv; +} + +static inline int +mutex_cas_ni(volatile void *ptr, void **old, void *new) +{ + void *oldv; + + oldv = *old; + *old = pthread__atomic_cas_ptr_ni(ptr, oldv, new); + return *old == oldv; } int @@ -114,15 +129,15 @@ pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr) switch (type) { case PTHREAD_MUTEX_ERRORCHECK: - ptm->ptm_errorcheck = (void *)1; + ptm->ptm_errorcheck = 1; ptm->ptm_owner = NULL; break; case PTHREAD_MUTEX_RECURSIVE: - ptm->ptm_errorcheck = NULL; + ptm->ptm_errorcheck = 0; ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT; break; default: - ptm->ptm_errorcheck = NULL; + ptm->ptm_errorcheck = 0; ptm->ptm_owner = NULL; break; } @@ -130,7 +145,6 @@ pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr) ptm->ptm_magic = _PT_MUTEX_MAGIC; ptm->ptm_waiters = NULL; ptm->ptm_private = NULL; - MUTEX_DEFERRED(ptm) = 0; return 0; } @@ -149,19 +163,6 @@ pthread_mutex_destroy(pthread_mutex_t *ptm) return 0; } - -/* - * Note regarding memory visibility: Pthreads has rules about memory - * visibility and mutexes. Very roughly: Memory a thread can see when - * it unlocks a mutex can be seen by another thread that locks the - * same mutex. - * - * A memory barrier after a lock and before an unlock will provide - * this behavior. This code relies on mutex_cas() to issue a barrier - * after obtaining a lock, and on pthread__simple_unlock() to issue - * a barrier before releasing a lock. - */ - int pthread_mutex_lock(pthread_mutex_t *ptm) { @@ -173,7 +174,6 @@ pthread_mutex_lock(pthread_mutex_t *ptm) if (__predict_true(mutex_cas(&ptm->ptm_owner, &owner, self))) return 0; - return pthread__mutex_lock_slow(ptm); } @@ -185,12 +185,41 @@ pthread__mutex_pause(void) pthread__smt_pause(); } +/* + * Spin while the holder is running. 'lwpctl' gives us the true + * status of the thread. pt_blocking is set by libpthread in order + * to cut out system call and kernel spinlock overhead on remote CPUs + * (could represent many thousands of clock cycles). pt_blocking also + * makes this thread yield if the target is calling sched_yield(). + */ +NOINLINE static void * +pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner, uintptr_t mask) +{ + pthread_t thread; + + for (;; owner = ptm->ptm_owner) { + thread = (pthread_t)MUTEX_OWNER(owner); + if (((uintptr_t)owner & mask) != 0) + break; + if (thread == NULL) + break; + if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE || + thread->pt_blocking) + break; + pthread__mutex_pause(); + pthread__mutex_pause(); + pthread__mutex_pause(); + pthread__mutex_pause(); + } + + return owner; +} + NOINLINE static int pthread__mutex_lock_slow(pthread_mutex_t *ptm) { void *waiters, *new, *owner; pthread_t self; - int count; pthread__error(EINVAL, "Invalid mutex", ptm->ptm_magic == _PT_MUTEX_MAGIC); @@ -210,14 +239,14 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm) return EDEADLK; } - /* Spin for a while. */ - count = pthread__nspins; - while (MUTEX_OWNER(owner) != 0 && --count > 0) { - pthread__mutex_pause(); - owner = ptm->ptm_owner; - } - for (;; owner = ptm->ptm_owner) { + /* + * Spin while the owner is running, but block if the + * mutex is so heavily contested that there are existing + * waiters. + */ + owner = pthread__mutex_spin(ptm, owner, MUTEX_WAITERS_BIT); + /* If it has become free, try to acquire it again. */ while (MUTEX_OWNER(owner) == 0) { new = (void *)((uintptr_t)self | (uintptr_t)owner); @@ -263,8 +292,21 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm) break; } new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT); - if (mutex_cas(&ptm->ptm_owner, &owner, new)) - break; + if (mutex_cas(&ptm->ptm_owner, &owner, new)) { + /* + * pthread_mutex_unlock() can do a + * non-interlocked CAS. We cannot + * know if our attempt to set the + * waiters bit has succeeded while + * the holding thread is running. + * There are many assumptions; see + * sys/kern/kern_mutex.c for details. + * In short, we must spin if we see + * that the holder is running again. + */ + pthread__membar_full(); + owner = pthread__mutex_spin(ptm, owner, 0); + } } /* @@ -275,7 +317,9 @@ pthread__mutex_lock_slow(pthread_mutex_t *ptm) * the thread onto the waiters list. */ while (self->pt_sleeponq) { + self->pt_blocking++; (void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL); + self->pt_blocking--; } } } @@ -302,71 +346,37 @@ pthread_mutex_trylock(pthread_mutex_t *ptm) return EBUSY; } -NOINLINE int -pthread__mutex_catchup(pthread_mutex_t *ptm) -{ - pthread_t self; - - self = pthread__self(); - - if (self->pt_nwaiters == 1) { - /* - * If the calling thread is about to block, defer - * unparking the target until _lwp_park() is called. - */ - if (self->pt_willpark && self->pt_unpark == 0) { - self->pt_unpark = self->pt_waiters[0]; - self->pt_unparkhint = &ptm->ptm_waiters; - } else { - (void)_lwp_unpark(self->pt_waiters[0], - &ptm->ptm_waiters); - } - } else { - (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, - &ptm->ptm_waiters); - } - self->pt_nwaiters = 0; - - return 0; -} - int pthread_mutex_unlock(pthread_mutex_t *ptm) { void *owner; pthread_t self; - char deferred; self = pthread__self(); owner = self; - deferred = MUTEX_DEFERRED(ptm); - MUTEX_DEFERRED(ptm) = 0; - - if (__predict_false(!mutex_cas(&ptm->ptm_owner, &owner, NULL))) - return pthread__mutex_unlock_slow(ptm); /* - * There were no waiters, but we may have deferred waking - * other threads until mutex unlock - we must wake them now. + * Note this may be a non-interlocked CAS. See lock_slow() + * above and sys/kern/kern_mutex.c for details. */ - if (deferred) - return pthread__mutex_catchup(ptm); - - return 0; + if (__predict_true(mutex_cas_ni(&ptm->ptm_owner, &owner, NULL))) + return 0; + return pthread__mutex_unlock_slow(ptm); } NOINLINE static int pthread__mutex_unlock_slow(pthread_mutex_t *ptm) { pthread_t self, owner, new; - int weown, error; + int weown, error, deferred; pthread__error(EINVAL, "Invalid mutex", ptm->ptm_magic == _PT_MUTEX_MAGIC); - self = pthread_self(); + self = pthread__self(); owner = ptm->ptm_owner; weown = (MUTEX_OWNER(owner) == (uintptr_t)self); + deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT); error = 0; if (ptm->ptm_errorcheck) { @@ -410,8 +420,27 @@ pthread__mutex_unlock_slow(pthread_mutex_t *ptm) * There were no waiters, but we may have deferred waking * other threads until mutex unlock - we must wake them now. */ - if (self->pt_nwaiters != 0) - return pthread__mutex_catchup(ptm); + if (!deferred) + return error; + + if (self->pt_nwaiters == 1) { + /* + * If the calling thread is about to block, defer + * unparking the target until _lwp_park() is called. + */ + if (self->pt_willpark && self->pt_unpark == 0) { + self->pt_unpark = self->pt_waiters[0]; + self->pt_unparkhint = &ptm->ptm_waiters; + } else { + (void)_lwp_unpark(self->pt_waiters[0], + &ptm->ptm_waiters); + } + } else { + (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, + &ptm->ptm_waiters); + } + self->pt_nwaiters = 0; + return error; } @@ -559,7 +588,9 @@ pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm) if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread) return 0; - MUTEX_DEFERRED(ptm) = 1; + pthread__atomic_or_ulong((volatile unsigned long *) + (uintptr_t)&ptm->ptm_owner, + MUTEX_DEFERRED_BIT); return 1; } diff --git a/lib/libpthread/pthread_rwlock2.c b/lib/libpthread/pthread_rwlock2.c index 46e0cf8d234..cee55ea73a0 100644 --- a/lib/libpthread/pthread_rwlock2.c +++ b/lib/libpthread/pthread_rwlock2.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_rwlock2.c,v 1.6 2007/11/13 15:57:13 ad Exp $ */ +/* $NetBSD: pthread_rwlock2.c,v 1.7 2007/11/13 17:20:09 ad Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_rwlock2.c,v 1.6 2007/11/13 15:57:13 ad Exp $"); +__RCSID("$NetBSD: pthread_rwlock2.c,v 1.7 2007/11/13 17:20:09 ad Exp $"); #include <errno.h> #include <stddef.h> @@ -66,8 +66,12 @@ __strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy) static inline int rw_cas(pthread_rwlock_t *ptr, uintptr_t *o, uintptr_t n) { + uintptr_t oldv; - return pthread__atomic_cas_ptr(&ptr->ptr_owner, (void **)o, (void *)n); + oldv = *o; + *o = (uintptr_t)pthread__atomic_cas_ptr(&ptr->ptr_owner, (void *)oldv, + (void *)n); + return *o == oldv; } int |
