diff options
| author | ad <ad@NetBSD.org> | 2008-02-10 18:50:54 +0000 |
|---|---|---|
| committer | ad <ad@NetBSD.org> | 2008-02-10 18:50:54 +0000 |
| commit | a67e1e347544393a4828fa21d5cf6f670590fd76 (patch) | |
| tree | c200cca9ccceecd65fe50f0b925bb5ad7831fa14 | |
| parent | 749400b7044c2a3ca87455ac26cc1184b678a353 (diff) | |
- Remove libpthread's atomic ops.
- Remove the old spinlock-based mutex and rwlock implementations.
- Use the atomic ops from libc.
| -rw-r--r-- | lib/libpthread/Makefile | 10 | ||||
| -rw-r--r-- | lib/libpthread/arch/hppa/pthread_md.h | 5 | ||||
| -rw-r--r-- | lib/libpthread/arch/i386/_context_u.S | 30 | ||||
| -rw-r--r-- | lib/libpthread/arch/i386/pthread_md.h | 34 | ||||
| -rw-r--r-- | lib/libpthread/arch/m68k/pthread_md.h | 5 | ||||
| -rw-r--r-- | lib/libpthread/arch/sh3/pthread_md.h | 5 | ||||
| -rw-r--r-- | lib/libpthread/arch/sparc/pthread_md.h | 5 | ||||
| -rw-r--r-- | lib/libpthread/arch/vax/pthread_md.h | 5 | ||||
| -rw-r--r-- | lib/libpthread/arch/x86_64/_context_u.S | 28 | ||||
| -rw-r--r-- | lib/libpthread/arch/x86_64/pthread_md.h | 32 | ||||
| -rw-r--r-- | lib/libpthread/pthread_int.h | 16 | ||||
| -rw-r--r-- | lib/libpthread/pthread_misc.c | 9 | ||||
| -rw-r--r-- | lib/libpthread/pthread_mutex.c | 638 | ||||
| -rw-r--r-- | lib/libpthread/pthread_mutex2.c | 623 | ||||
| -rw-r--r-- | lib/libpthread/pthread_rwlock.c | 686 | ||||
| -rw-r--r-- | lib/libpthread/pthread_rwlock2.c | 584 |
16 files changed, 843 insertions, 1872 deletions
diff --git a/lib/libpthread/Makefile b/lib/libpthread/Makefile index a9c9f6e1379..ae9521dc901 100644 --- a/lib/libpthread/Makefile +++ b/lib/libpthread/Makefile @@ -1,4 +1,4 @@ -# $NetBSD: Makefile,v 1.49 2008/01/15 03:37:14 rmind Exp $ +# $NetBSD: Makefile,v 1.50 2008/02/10 18:50:54 ad Exp $ # WARNS= 4 @@ -54,9 +54,7 @@ SRCS+= pthread_cond.c SRCS+= pthread_lock.c SRCS+= pthread_misc.c SRCS+= pthread_mutex.c -SRCS+= pthread_mutex2.c SRCS+= pthread_rwlock.c -SRCS+= pthread_rwlock2.c SRCS+= pthread_specific.c SRCS+= pthread_spin.c SRCS+= pthread_tsd.c @@ -82,10 +80,10 @@ pthread_lock.po: pthread_lock.o ${_MKTARGET_CREATE} cp pthread_lock.o pthread_lock.po -COPTS.pthread_mutex2.c+= -fomit-frame-pointer -falign-functions=32 -pthread_mutex2.po: pthread_mutex2.o +COPTS.pthread_mutex.c+= -fomit-frame-pointer -falign-functions=32 +pthread_mutex.po: pthread_mutex.o ${_MKTARGET_CREATE} - cp pthread_mutex2.o pthread_mutex2.po + cp pthread_mutex.o pthread_mutex.po COPTS.pthread.c += -Wno-stack-protector diff --git a/lib/libpthread/arch/hppa/pthread_md.h b/lib/libpthread/arch/hppa/pthread_md.h index 5e106df0621..bd7371cf486 100644 --- a/lib/libpthread/arch/hppa/pthread_md.h +++ b/lib/libpthread/arch/hppa/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.4 2007/09/24 12:19:39 skrll Exp $ */ +/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -98,4 +98,7 @@ do { \ (uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \ } while (/*CONSTCOND*/0) +/* Don't need additional memory barriers. */ +#define PTHREAD__ATOMIC_IS_MEMBAR + #endif /* !_LIB_PTHREAD_HPPA_MD_H */ diff --git a/lib/libpthread/arch/i386/_context_u.S b/lib/libpthread/arch/i386/_context_u.S index 6720bd67402..8d9cba7737a 100644 --- a/lib/libpthread/arch/i386/_context_u.S +++ b/lib/libpthread/arch/i386/_context_u.S @@ -1,4 +1,4 @@ -/* $NetBSD: _context_u.S,v 1.7 2007/11/13 17:20:09 ad Exp $ */ +/* $NetBSD: _context_u.S,v 1.8 2008/02/10 18:50:55 ad Exp $ */ /*- * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc. @@ -100,31 +100,3 @@ STUB(_swapcontext_u_xmm) movl %eax, (UC_REGS + _REG_UESP * 4)(%ecx) movl 8(%esp), %ecx SETC - -STUB(pthread__atomic_swap_ptr) - movl 4(%esp), %ecx - movl 8(%esp), %eax - xchgl %eax, (%ecx) - ret - -STUB(pthread__atomic_or_ulong) - movl 4(%esp), %ecx - movl 8(%esp), %eax - lock - orl %eax, (%ecx) - ret - -STUB(pthread__membar_full) - lock - addl $0, -4(%esp) - ret - -STUB(pthread__membar_producer) - /* A store is enough */ - addl $0, -4(%esp) - ret - -STUB(pthread__membar_consumer) - lock - addl $0, -4(%esp) - ret diff --git a/lib/libpthread/arch/i386/pthread_md.h b/lib/libpthread/arch/i386/pthread_md.h index c99ae7f412e..6128b557c18 100644 --- a/lib/libpthread/arch/i386/pthread_md.h +++ b/lib/libpthread/arch/i386/pthread_md.h @@ -1,7 +1,7 @@ -/* $NetBSD: pthread_md.h,v 1.11 2007/11/13 17:20:10 ad Exp $ */ +/* $NetBSD: pthread_md.h,v 1.12 2008/02/10 18:50:55 ad Exp $ */ /*- - * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2001, 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -182,9 +182,11 @@ pthread__sp(void) } while (/*CONSTCOND*/0) #define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory") -#define PTHREAD__HAVE_ATOMIC /* #define PTHREAD__HAVE_THREADREG */ +/* Don't need additional memory barriers. */ +#define PTHREAD__ATOMIC_IS_MEMBAR + void pthread__threadreg_set(pthread_t); static inline pthread_t @@ -199,30 +201,4 @@ pthread__threadreg_get(void) return self; } -static inline void * -pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new) -{ - volatile uintptr_t *cast = ptr; - void *ret; - - __asm __volatile ("lock; cmpxchgl %2, %1" - : "=a" (ret), "=m" (*cast) - : "r" (new), "m" (*cast), "0" (old)); - - return ret; -} - -static inline void * -pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new) -{ - volatile uintptr_t *cast = ptr; - void *ret; - - __asm __volatile ("cmpxchgl %2, %1" - : "=a" (ret), "=m" (*cast) - : "r" (new), "m" (*cast), "0" (old)); - - return ret; -} - #endif /* _LIB_PTHREAD_I386_MD_H */ diff --git a/lib/libpthread/arch/m68k/pthread_md.h b/lib/libpthread/arch/m68k/pthread_md.h index 3bcfe9b9954..4f873a1ff85 100644 --- a/lib/libpthread/arch/m68k/pthread_md.h +++ b/lib/libpthread/arch/m68k/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:16 perry Exp $ */ +/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -99,4 +99,7 @@ pthread__sp(void) (uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \ } while (/*CONSTCOND*/0) +/* m68k will not go SMP */ +#define PTHREAD__ATOMIC_IS_MEMBAR + #endif /* _LIB_PTHREAD_M68K_MD_H */ diff --git a/lib/libpthread/arch/sh3/pthread_md.h b/lib/libpthread/arch/sh3/pthread_md.h index 8f5db27e859..59933c3f66d 100644 --- a/lib/libpthread/arch/sh3/pthread_md.h +++ b/lib/libpthread/arch/sh3/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.3 2005/12/24 21:11:17 perry Exp $ */ +/* $NetBSD: pthread_md.h,v 1.4 2008/02/10 18:50:55 ad Exp $ */ /* * Copyright 2003 Wasabi Systems, Inc. @@ -90,4 +90,7 @@ pthread__sp(void) #define PTHREAD_FPREG_TO_UCONTEXT(uc, freg) #endif +/* sh3 will not go SMP */ +#define PTHREAD__ATOMIC_IS_MEMBAR + #endif /* _LIB_PTHREAD_SH3_MD_H */ diff --git a/lib/libpthread/arch/sparc/pthread_md.h b/lib/libpthread/arch/sparc/pthread_md.h index af2210747a9..4d105d7e698 100644 --- a/lib/libpthread/arch/sparc/pthread_md.h +++ b/lib/libpthread/arch/sparc/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:17 perry Exp $ */ +/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */ /*- * Copyright (c) 2002 The NetBSD Foundation, Inc. @@ -92,4 +92,7 @@ pthread__sp(void) (uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \ } while (/*CONSTCOND*/0) +/* Don't need additional memory barriers. */ +#define PTHREAD__ATOMIC_IS_MEMBAR + #endif /* _LIB_PTHREAD_SPARC_MD_H */ diff --git a/lib/libpthread/arch/vax/pthread_md.h b/lib/libpthread/arch/vax/pthread_md.h index 7ffe5220a4a..e2ed4cca1d3 100644 --- a/lib/libpthread/arch/vax/pthread_md.h +++ b/lib/libpthread/arch/vax/pthread_md.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:17 perry Exp $ */ +/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -85,4 +85,7 @@ pthread__sp(void) #define PTHREAD_FPREG_TO_UCONTEXT(uc, reg) do { \ } while (/*CONSTCOND*/0) +/* Don't need additional memory barriers. */ +#define PTHREAD__ATOMIC_IS_MEMBAR + #endif /* _LIB_PTHREAD_VAX_MD_H */ diff --git a/lib/libpthread/arch/x86_64/_context_u.S b/lib/libpthread/arch/x86_64/_context_u.S index 8719b4fff5e..b9ec75ccf4e 100644 --- a/lib/libpthread/arch/x86_64/_context_u.S +++ b/lib/libpthread/arch/x86_64/_context_u.S @@ -1,4 +1,4 @@ -/* $NetBSD: _context_u.S,v 1.9 2007/11/13 17:20:10 ad Exp $ */ +/* $NetBSD: _context_u.S,v 1.10 2008/02/10 18:50:55 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -137,29 +137,3 @@ STUB(_swapcontext_u) movq %rax, (UC_REGS + _REG_URSP * 8)(%rdi) movq %rsi, %rdi SETC - -STUB(pthread__atomic_swap_ptr) - movq %rsi, %rax - lock - xchgq %rax, (%rdi) - ret - -STUB(pthread__atomic_or_ulong) - lock - orq %rsi, (%rdi) - ret - -STUB(pthread__membar_full) - lock - addq $0, -8(%rsp) - ret - -STUB(pthread__membar_producer) - /* A store is enough */ - movq $0, -8(%rsp) - ret - -STUB(pthread__membar_consumer) - lock - addq $0, -8(%rsp) - ret diff --git a/lib/libpthread/arch/x86_64/pthread_md.h b/lib/libpthread/arch/x86_64/pthread_md.h index 7f04edfd85a..6db24f0faf0 100644 --- a/lib/libpthread/arch/x86_64/pthread_md.h +++ b/lib/libpthread/arch/x86_64/pthread_md.h @@ -1,7 +1,7 @@ -/* $NetBSD: pthread_md.h,v 1.7 2007/11/13 17:20:10 ad Exp $ */ +/* $NetBSD: pthread_md.h,v 1.8 2008/02/10 18:50:55 ad Exp $ */ /*- - * Copyright (c) 2001 The NetBSD Foundation, Inc. + * Copyright (c) 2001, 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -104,32 +104,8 @@ pthread__sp(void) } while (/*CONSTCOND*/0) #define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory") -#define PTHREAD__HAVE_ATOMIC -static inline void * -pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new) -{ - volatile uintptr_t *cast = ptr; - void *ret; - - __asm __volatile ("lock; cmpxchgq %2, %1" - : "=a" (ret), "=m" (*cast) - : "r" (new), "m" (*cast), "0" (old)); - - return ret; -} - -static inline void * -pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new) -{ - volatile uintptr_t *cast = ptr; - void *ret; - - __asm __volatile ("cmpxchgq %2, %1" - : "=a" (ret), "=m" (*cast) - : "r" (new), "m" (*cast), "0" (old)); - - return ret; -} +/* Don't need additional memory barriers. */ +#define PTHREAD__ATOMIC_IS_MEMBAR #endif /* _LIB_PTHREAD_X86_64_MD_H */ diff --git a/lib/libpthread/pthread_int.h b/lib/libpthread/pthread_int.h index 5e4d113bc80..82c5d21d006 100644 --- a/lib/libpthread/pthread_int.h +++ b/lib/libpthread/pthread_int.h @@ -1,7 +1,7 @@ -/* $NetBSD: pthread_int.h,v 1.65 2008/01/08 20:56:08 christos Exp $ */ +/* $NetBSD: pthread_int.h,v 1.66 2008/02/10 18:50:54 ad Exp $ */ /*- - * Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -51,6 +51,10 @@ #include "pthread_queue.h" #include "pthread_md.h" +/* Need to use libc-private names for atomic operations. */ +#include "../../common/lib/libc/atomic/atomic_op_namespace.h" + +#include <sys/atomic.h> #include <sys/tree.h> #include <lwp.h> @@ -289,14 +293,6 @@ void pthread__errorfunc(const char *, int, const char *, const char *) PTHREAD_HIDE; char *pthread__getenv(const char *) PTHREAD_HIDE; void pthread__cancelled(void) PTHREAD_HIDE; - -void *pthread__atomic_cas_ptr(volatile void *, const void *, const void *) PTHREAD_HIDE; -void *pthread__atomic_swap_ptr(volatile void *, const void *) PTHREAD_HIDE; -void pthread__atomic_or_ulong(volatile unsigned long *, unsigned long) PTHREAD_HIDE; -void pthread__membar_full(void) PTHREAD_HIDE; -void pthread__membar_producer(void) PTHREAD_HIDE; -void pthread__membar_consumer(void) PTHREAD_HIDE; - int pthread__mutex_deferwake(pthread_t, pthread_mutex_t *) PTHREAD_HIDE; #ifndef pthread__smt_pause diff --git a/lib/libpthread/pthread_misc.c b/lib/libpthread/pthread_misc.c index 4a5b3930f10..198225ec0f3 100644 --- a/lib/libpthread/pthread_misc.c +++ b/lib/libpthread/pthread_misc.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_misc.c,v 1.6 2008/02/09 17:07:54 yamt Exp $ */ +/* $NetBSD: pthread_misc.c,v 1.7 2008/02/10 18:50:54 ad Exp $ */ /*- * Copyright (c) 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_misc.c,v 1.6 2008/02/09 17:07:54 yamt Exp $"); +__RCSID("$NetBSD: pthread_misc.c,v 1.7 2008/02/10 18:50:54 ad Exp $"); #include <errno.h> #include <string.h> @@ -173,13 +173,12 @@ pthread__sched_yield(void) self = pthread__self(); -#ifdef PTHREAD__HAVE_ATOMIC /* Memory barrier for unlocked mutex release. */ - pthread__membar_producer(); -#endif + membar_producer(); self->pt_blocking++; error = _sys_sched_yield(); self->pt_blocking--; + membar_sync(); return error; } diff --git a/lib/libpthread/pthread_mutex.c b/lib/libpthread/pthread_mutex.c index 26e2fb92904..54cf1dc434f 100644 --- a/lib/libpthread/pthread_mutex.c +++ b/lib/libpthread/pthread_mutex.c @@ -1,7 +1,7 @@ -/* $NetBSD: pthread_mutex.c,v 1.43 2008/01/25 02:12:10 rafal Exp $ */ +/* $NetBSD: pthread_mutex.c,v 1.44 2008/02/10 18:50:54 ad Exp $ */ /*- - * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2001, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -37,24 +37,51 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_mutex.c,v 1.43 2008/01/25 02:12:10 rafal Exp $"); +__RCSID("$NetBSD: pthread_mutex.c,v 1.44 2008/02/10 18:50:54 ad Exp $"); #include <sys/types.h> - -#include <machine/lock.h> +#include <sys/lwpctl.h> #include <errno.h> #include <limits.h> #include <stdlib.h> #include <string.h> +#include <stdio.h> #include "pthread.h" #include "pthread_int.h" -#ifndef PTHREAD__HAVE_ATOMIC - -static int pthread_mutex_lock_slow(pthread_t, pthread_mutex_t *); -static void once_cleanup(void *); +/* + * Note that it's important to use the address of ptm_waiters as + * the list head in order for the hint arguments to _lwp_park / + * _lwp_unpark_all to match. + */ +#define pt_nextwaiter pt_sleep.ptqe_next +#define ptm_waiters ptm_blocked.ptqh_first +#define ptm_errorcheck ptm_lock + +#define MUTEX_WAITERS_BIT ((uintptr_t)0x01) +#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02) +#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04) +#define MUTEX_THREAD ((uintptr_t)-16L) + +#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT) +#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT) +#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD) +#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private) +#define MUTEX_SET_RECURSE(ptm, delta) \ + ((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta)) + +#if __GNUC_PREREQ__(3, 0) +#define NOINLINE __attribute ((noinline)) +#else +#define NOINLINE /* nothing */ +#endif + +static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *); +static int pthread__mutex_lock_slow(pthread_mutex_t *); +static int pthread__mutex_unlock_slow(pthread_mutex_t *); +static void pthread__mutex_pause(void); int _pthread_mutex_held_np(pthread_mutex_t *); pthread_t _pthread_mutex_owner_np(pthread_mutex_t *); @@ -74,309 +101,431 @@ __strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype) __strong_alias(__libc_thr_once,pthread_once) -struct mutex_private { - int type; - int recursecount; -}; - -static const struct mutex_private mutex_private_default = { - PTHREAD_MUTEX_DEFAULT, - 0, -}; - -struct mutexattr_private { - int type; -}; - -static const struct mutexattr_private mutexattr_private_default = { - PTHREAD_MUTEX_DEFAULT, -}; - int -pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr) +pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr) { - struct mutexattr_private *map; - struct mutex_private *mp; + intptr_t type; - pthread__error(EINVAL, "Invalid mutex attribute", - (attr == NULL) || (attr->ptma_magic == _PT_MUTEXATTR_MAGIC)); - - if (attr != NULL && (map = attr->ptma_private) != NULL && - memcmp(map, &mutexattr_private_default, sizeof(*map)) != 0) { - mp = malloc(sizeof(*mp)); - if (mp == NULL) - return ENOMEM; + if (attr == NULL) + type = PTHREAD_MUTEX_NORMAL; + else + type = (intptr_t)attr->ptma_private; - mp->type = map->type; - mp->recursecount = 0; - } else { - /* LINTED cast away const */ - mp = (struct mutex_private *) &mutex_private_default; + switch (type) { + case PTHREAD_MUTEX_ERRORCHECK: + ptm->ptm_errorcheck = 1; + ptm->ptm_owner = NULL; + break; + case PTHREAD_MUTEX_RECURSIVE: + ptm->ptm_errorcheck = 0; + ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT; + break; + default: + ptm->ptm_errorcheck = 0; + ptm->ptm_owner = NULL; + break; } - mutex->ptm_magic = _PT_MUTEX_MAGIC; - mutex->ptm_owner = NULL; - pthread_lockinit(&mutex->ptm_lock); - pthread_lockinit(&mutex->ptm_interlock); - PTQ_INIT(&mutex->ptm_blocked); - mutex->ptm_private = mp; + ptm->ptm_magic = _PT_MUTEX_MAGIC; + ptm->ptm_waiters = NULL; + ptm->ptm_private = NULL; return 0; } int -pthread_mutex_destroy(pthread_mutex_t *mutex) +pthread_mutex_destroy(pthread_mutex_t *ptm) { pthread__error(EINVAL, "Invalid mutex", - mutex->ptm_magic == _PT_MUTEX_MAGIC); + ptm->ptm_magic == _PT_MUTEX_MAGIC); pthread__error(EBUSY, "Destroying locked mutex", - __SIMPLELOCK_UNLOCKED_P(&mutex->ptm_lock)); - - mutex->ptm_magic = _PT_MUTEX_DEAD; - if (mutex->ptm_private != NULL && - mutex->ptm_private != (const void *)&mutex_private_default) - free(mutex->ptm_private); + MUTEX_OWNER(ptm->ptm_owner) == 0); + ptm->ptm_magic = _PT_MUTEX_DEAD; return 0; } - -/* - * Note regarding memory visibility: Pthreads has rules about memory - * visibility and mutexes. Very roughly: Memory a thread can see when - * it unlocks a mutex can be seen by another thread that locks the - * same mutex. - * - * A memory barrier after a lock and before an unlock will provide - * this behavior. This code relies on pthread__spintrylock() to issue - * a barrier after obtaining a lock, and on pthread__spinunlock() to - * issue a barrier before releasing a lock. - */ - int -pthread_mutex_lock(pthread_mutex_t *mutex) +pthread_mutex_lock(pthread_mutex_t *ptm) { pthread_t self; - int error; + void *val; self = pthread__self(); - - /* - * Note that if we get the lock, we don't have to deal with any - * non-default lock type handling. - */ - if (__predict_false(pthread__spintrylock(self, &mutex->ptm_lock) == 0)) { - error = pthread_mutex_lock_slow(self, mutex); - if (error) - return error; + val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self); + if (__predict_true(val == NULL)) { +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; } + return pthread__mutex_lock_slow(ptm); +} - /* - * We have the lock! - */ - mutex->ptm_owner = self; +/* We want function call overhead. */ +NOINLINE static void +pthread__mutex_pause(void) +{ - return 0; + pthread__smt_pause(); } +/* + * Spin while the holder is running. 'lwpctl' gives us the true + * status of the thread. pt_blocking is set by libpthread in order + * to cut out system call and kernel spinlock overhead on remote CPUs + * (could represent many thousands of clock cycles). pt_blocking also + * makes this thread yield if the target is calling sched_yield(). + */ +NOINLINE static void * +pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner) +{ + pthread_t thread; + unsigned int count, i; + + for (count = 2;; owner = ptm->ptm_owner) { + thread = (pthread_t)MUTEX_OWNER(owner); + if (thread == NULL) + break; + if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE || + thread->pt_blocking) + break; + if (count < 128) + count += count; + for (i = count; i != 0; i--) + pthread__mutex_pause(); + } + + return owner; +} -static int -pthread_mutex_lock_slow(pthread_t self, pthread_mutex_t *mutex) +NOINLINE static int +pthread__mutex_lock_slow(pthread_mutex_t *ptm) { - struct mutex_private *mp; - int count; + void *waiters, *new, *owner, *next; + pthread_t self; pthread__error(EINVAL, "Invalid mutex", - mutex->ptm_magic == _PT_MUTEX_MAGIC); + ptm->ptm_magic == _PT_MUTEX_MAGIC); - for (;;) { - /* Spin for a while. */ - count = pthread__nspins; - while (__SIMPLELOCK_LOCKED_P(&mutex->ptm_lock) && --count > 0) - pthread__smt_pause(); - if (count > 0) { - if (pthread__spintrylock(self, &mutex->ptm_lock) != 0) - break; - continue; + owner = ptm->ptm_owner; + self = pthread__self(); + + /* Recursive or errorcheck? */ + if (MUTEX_OWNER(owner) == (uintptr_t)self) { + if (MUTEX_RECURSIVE(owner)) { + if (MUTEX_GET_RECURSE(ptm) == INT_MAX) + return EAGAIN; + MUTEX_SET_RECURSE(ptm, +1); + return 0; } + if (ptm->ptm_errorcheck) + return EDEADLK; + } - /* Okay, didn't look free. Get the interlock... */ - pthread__spinlock(self, &mutex->ptm_interlock); + for (;; owner = ptm->ptm_owner) { + /* Spin while the owner is running. */ + owner = pthread__mutex_spin(ptm, owner); + + /* If it has become free, try to acquire it again. */ + if (MUTEX_OWNER(owner) == 0) { + for (; MUTEX_OWNER(owner) == 0; owner = next) { + new = (void *) + ((uintptr_t)self | (uintptr_t)owner); + next = atomic_cas_ptr(&ptm->ptm_owner, owner, + new); + if (next == owner) { +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; + } + } + /* + * We have lost the race to acquire the mutex. + * The new owner could be running on another + * CPU, in which case we should spin and avoid + * the overhead of blocking. + */ + if (!MUTEX_HAS_WAITERS(owner)) + continue; + } /* - * The mutex_unlock routine will get the interlock - * before looking at the list of sleepers, so if the - * lock is held we can safely put ourselves on the - * sleep queue. If it's not held, we can try taking it - * again. + * Nope, still held. Add thread to the list of waiters. + * Issue a memory barrier to ensure sleeponq/nextwaiter + * are visible before we enter the waiters list. */ - PTQ_INSERT_HEAD(&mutex->ptm_blocked, self, pt_sleep); - if (__SIMPLELOCK_UNLOCKED_P(&mutex->ptm_lock)) { - PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); - pthread__spinunlock(self, &mutex->ptm_interlock); - continue; + self->pt_sleeponq = 1; + for (waiters = ptm->ptm_waiters;; waiters = next) { + self->pt_nextwaiter = waiters; + membar_producer(); + next = atomic_cas_ptr(&ptm->ptm_waiters, waiters, self); + if (next == waiters) + break; } - mp = mutex->ptm_private; - if (mutex->ptm_owner == self && mp != NULL) { - switch (mp->type) { - case PTHREAD_MUTEX_ERRORCHECK: - PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); - pthread__spinunlock(self, &mutex->ptm_interlock); - return EDEADLK; - - case PTHREAD_MUTEX_RECURSIVE: + /* + * Set the waiters bit and block. + * + * Note that the mutex can become unlocked before we set + * the waiters bit. If that happens it's not safe to sleep + * as we may never be awoken: we must remove the current + * thread from the waiters list and try again. + * + * Because we are doing this atomically, we can't remove + * one waiter: we must remove all waiters and awken them, + * then sleep in _lwp_park() until we have been awoken. + * + * Issue a memory barrier to ensure that we are reading + * the value of ptm_owner/pt_sleeponq after we have entered + * the waiters list (the CAS itself must be atomic). + */ + membar_consumer(); + for (owner = ptm->ptm_owner;; owner = next) { + if (MUTEX_HAS_WAITERS(owner)) + break; + if (MUTEX_OWNER(owner) == 0) { + pthread__mutex_wakeup(self, ptm); + break; + } + new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT); + next = atomic_cas_ptr(&ptm->ptm_owner, owner, new); + if (next == owner) { /* - * It's safe to do this without - * holding the interlock, because - * we only modify it if we know we - * own the mutex. + * pthread_mutex_unlock() can do a + * non-interlocked CAS. We cannot + * know if our attempt to set the + * waiters bit has succeeded while + * the holding thread is running. + * There are many assumptions; see + * sys/kern/kern_mutex.c for details. + * In short, we must spin if we see + * that the holder is running again. */ - PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); - pthread__spinunlock(self, &mutex->ptm_interlock); - if (mp->recursecount == INT_MAX) - return EAGAIN; - mp->recursecount++; - return 0; + membar_sync(); + next = pthread__mutex_spin(ptm, owner); } } /* - * Locking a mutex is not a cancellation - * point, so we don't need to do the - * test-cancellation dance. We may get woken - * up spuriously by pthread_cancel or signals, - * but it's okay since we're just going to - * retry. + * We may have been awoken by the current thread above, + * or will be awoken by the current holder of the mutex. + * The key requirement is that we must not proceed until + * told that we are no longer waiting (via pt_sleeponq + * being set to zero). Otherwise it is unsafe to re-enter + * the thread onto the waiters list. */ - self->pt_sleeponq = 1; - self->pt_sleepobj = &mutex->ptm_blocked; - pthread__spinunlock(self, &mutex->ptm_interlock); - (void)pthread__park(self, &mutex->ptm_interlock, - &mutex->ptm_blocked, NULL, 0, &mutex->ptm_blocked); + while (self->pt_sleeponq) { + self->pt_blocking++; + (void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL); + self->pt_blocking--; + membar_sync(); + } } - - return 0; } - int -pthread_mutex_trylock(pthread_mutex_t *mutex) +pthread_mutex_trylock(pthread_mutex_t *ptm) { - struct mutex_private *mp; pthread_t self; - - pthread__error(EINVAL, "Invalid mutex", - mutex->ptm_magic == _PT_MUTEX_MAGIC); + void *val; self = pthread__self(); - - if (pthread__spintrylock(self, &mutex->ptm_lock) == 0) { - /* - * These tests can be performed without holding the - * interlock because these fields are only modified - * if we know we own the mutex. - */ - mp = mutex->ptm_private; - if (mp != NULL && mp->type == PTHREAD_MUTEX_RECURSIVE && - mutex->ptm_owner == self) { - if (mp->recursecount == INT_MAX) - return EAGAIN; - mp->recursecount++; - return 0; - } - - return EBUSY; + val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self); + if (__predict_true(val == NULL)) { +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; } - mutex->ptm_owner = self; + if (MUTEX_OWNER(val) == (uintptr_t)self && MUTEX_RECURSIVE(val)) { + if (MUTEX_GET_RECURSE(ptm) == INT_MAX) + return EAGAIN; + MUTEX_SET_RECURSE(ptm, +1); + return 0; + } - return 0; + return EBUSY; } - int -pthread_mutex_unlock(pthread_mutex_t *mutex) +pthread_mutex_unlock(pthread_mutex_t *ptm) { - struct mutex_private *mp; pthread_t self; - int weown; - - pthread__error(EINVAL, "Invalid mutex", - mutex->ptm_magic == _PT_MUTEX_MAGIC); + void *value; /* - * These tests can be performed without holding the - * interlock because these fields are only modified - * if we know we own the mutex. + * Note this may be a non-interlocked CAS. See lock_slow() + * above and sys/kern/kern_mutex.c for details. */ +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_exit(); +#endif self = pthread__self(); - weown = (mutex->ptm_owner == self); - mp = mutex->ptm_private; - - if (mp == NULL) { - if (__predict_false(!weown)) { - pthread__error(EPERM, "Unlocking unlocked mutex", - (mutex->ptm_owner != 0)); - pthread__error(EPERM, - "Unlocking mutex owned by another thread", weown); + value = atomic_cas_ptr_ni(&ptm->ptm_owner, self, NULL); + if (__predict_true(value == self)) + return 0; + return pthread__mutex_unlock_slow(ptm); +} + +NOINLINE static int +pthread__mutex_unlock_slow(pthread_mutex_t *ptm) +{ + pthread_t self, owner, new; + int weown, error, deferred; + + pthread__error(EINVAL, "Invalid mutex", + ptm->ptm_magic == _PT_MUTEX_MAGIC); + + self = pthread__self(); + owner = ptm->ptm_owner; + weown = (MUTEX_OWNER(owner) == (uintptr_t)self); + deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT); + error = 0; + + if (ptm->ptm_errorcheck) { + if (!weown) { + error = EPERM; + new = owner; + } else { + new = NULL; + } + } else if (MUTEX_RECURSIVE(owner)) { + if (!weown) { + error = EPERM; + new = owner; + } else if (MUTEX_GET_RECURSE(ptm) != 0) { + MUTEX_SET_RECURSE(ptm, -1); + new = owner; + } else { + new = (pthread_t)MUTEX_RECURSIVE_BIT; } - } else if (mp->type == PTHREAD_MUTEX_RECURSIVE) { - if (!weown) - return EPERM; - if (mp->recursecount != 0) { - mp->recursecount--; + } else { + pthread__error(EPERM, + "Unlocking unlocked mutex", (owner != NULL)); + pthread__error(EPERM, + "Unlocking mutex owned by another thread", weown); + new = NULL; + } + + /* + * Release the mutex. If there appear to be waiters, then + * wake them up. + */ + if (new != owner) { + owner = atomic_swap_ptr(&ptm->ptm_owner, new); + if (MUTEX_HAS_WAITERS(owner) != 0) { + pthread__mutex_wakeup(self, ptm); return 0; } - } else if (mp->type == PTHREAD_MUTEX_ERRORCHECK) { - if (!weown) - return EPERM; - if (__predict_false(!weown)) { - pthread__error(EPERM, "Unlocking unlocked mutex", - (mutex->ptm_owner != 0)); - pthread__error(EPERM, - "Unlocking mutex owned by another thread", weown); + } + + /* + * There were no waiters, but we may have deferred waking + * other threads until mutex unlock - we must wake them now. + */ + if (!deferred) + return error; + + if (self->pt_nwaiters == 1) { + /* + * If the calling thread is about to block, defer + * unparking the target until _lwp_park() is called. + */ + if (self->pt_willpark && self->pt_unpark == 0) { + self->pt_unpark = self->pt_waiters[0]; + self->pt_unparkhint = &ptm->ptm_waiters; + } else { + (void)_lwp_unpark(self->pt_waiters[0], + &ptm->ptm_waiters); } + } else { + (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, + &ptm->ptm_waiters); } + self->pt_nwaiters = 0; - mutex->ptm_owner = NULL; - pthread__spinunlock(self, &mutex->ptm_lock); + return error; +} + +static void +pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm) +{ + pthread_t thread, next; + ssize_t n, rv; /* - * Do a double-checked locking dance to see if there are any - * waiters. If we don't see any waiters, we can exit, because - * we've already released the lock. If we do see waiters, they - * were probably waiting on us... there's a slight chance that - * they are waiting on a different thread's ownership of the - * lock that happened between the unlock above and this - * examination of the queue; if so, no harm is done, as the - * waiter will loop and see that the mutex is still locked. + * Take ownership of the current set of waiters. No + * need for a memory barrier following this, all loads + * are dependent upon 'thread'. */ - pthread__spinlock(self, &mutex->ptm_interlock); - pthread__unpark_all(self, &mutex->ptm_interlock, &mutex->ptm_blocked); - return 0; -} + thread = atomic_swap_ptr(&ptm->ptm_waiters, NULL); + + for (;;) { + /* + * Pull waiters from the queue and add to our list. + * Use a memory barrier to ensure that we safely + * read the value of pt_nextwaiter before 'thread' + * sees pt_sleeponq being cleared. + */ + for (n = self->pt_nwaiters, self->pt_nwaiters = 0; + n < pthread__unpark_max && thread != NULL; + thread = next) { + next = thread->pt_nextwaiter; + if (thread != self) { + self->pt_waiters[n++] = thread->pt_lid; + membar_sync(); + } + thread->pt_sleeponq = 0; + /* No longer safe to touch 'thread' */ + } + switch (n) { + case 0: + return; + case 1: + /* + * If the calling thread is about to block, + * defer unparking the target until _lwp_park() + * is called. + */ + if (self->pt_willpark && self->pt_unpark == 0) { + self->pt_unpark = self->pt_waiters[0]; + self->pt_unparkhint = &ptm->ptm_waiters; + return; + } + rv = (ssize_t)_lwp_unpark(self->pt_waiters[0], + &ptm->ptm_waiters); + if (rv != 0 && errno != EALREADY && errno != EINTR && + errno != ESRCH) { + pthread__errorfunc(__FILE__, __LINE__, + __func__, "_lwp_unpark failed"); + } + return; + default: + rv = _lwp_unpark_all(self->pt_waiters, (size_t)n, + &ptm->ptm_waiters); + if (rv != 0 && errno != EINTR) { + pthread__errorfunc(__FILE__, __LINE__, + __func__, "_lwp_unpark_all failed"); + } + break; + } + } +} int pthread_mutexattr_init(pthread_mutexattr_t *attr) { - struct mutexattr_private *map; - - map = malloc(sizeof(*map)); - if (map == NULL) - return ENOMEM; - - *map = mutexattr_private_default; attr->ptma_magic = _PT_MUTEXATTR_MAGIC; - attr->ptma_private = map; - + attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT; return 0; } - int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) { @@ -384,10 +533,6 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr) pthread__error(EINVAL, "Invalid mutex attribute", attr->ptma_magic == _PT_MUTEXATTR_MAGIC); - attr->ptma_magic = _PT_MUTEXATTR_DEAD; - if (attr->ptma_private != NULL) - free(attr->ptma_private); - return 0; } @@ -395,15 +540,11 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr) int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep) { - struct mutexattr_private *map; pthread__error(EINVAL, "Invalid mutex attribute", attr->ptma_magic == _PT_MUTEXATTR_MAGIC); - map = attr->ptma_private; - - *typep = map->type; - + *typep = (int)(intptr_t)attr->ptma_private; return 0; } @@ -411,25 +552,19 @@ pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep) int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { - struct mutexattr_private *map; pthread__error(EINVAL, "Invalid mutex attribute", attr->ptma_magic == _PT_MUTEXATTR_MAGIC); - map = attr->ptma_private; - switch (type) { case PTHREAD_MUTEX_NORMAL: case PTHREAD_MUTEX_ERRORCHECK: case PTHREAD_MUTEX_RECURSIVE: - map->type = type; - break; - + attr->ptma_private = (void *)(intptr_t)type; + return 0; default: return EINVAL; } - - return 0; } @@ -459,24 +594,27 @@ pthread_once(pthread_once_t *once_control, void (*routine)(void)) } int -pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *mutex) +pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm) { - return mutex->ptm_owner == thread; + if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread) + return 0; + atomic_or_ulong((volatile unsigned long *) + (uintptr_t)&ptm->ptm_owner, + (unsigned long)MUTEX_DEFERRED_BIT); + return 1; } int -_pthread_mutex_held_np(pthread_mutex_t *mutex) +_pthread_mutex_held_np(pthread_mutex_t *ptm) { - return mutex->ptm_owner == pthread__self(); + return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self(); } pthread_t -_pthread_mutex_owner_np(pthread_mutex_t *mutex) +_pthread_mutex_owner_np(pthread_mutex_t *ptm) { - return (pthread_t)mutex->ptm_owner; + return (pthread_t)MUTEX_OWNER(ptm->ptm_owner); } - -#endif /* !PTHREAD__HAVE_ATOMIC */ diff --git a/lib/libpthread/pthread_mutex2.c b/lib/libpthread/pthread_mutex2.c deleted file mode 100644 index 50d6da6326d..00000000000 --- a/lib/libpthread/pthread_mutex2.c +++ /dev/null @@ -1,623 +0,0 @@ -/* $NetBSD: pthread_mutex2.c,v 1.17 2007/12/24 14:46:29 ad Exp $ */ - -/*- - * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_mutex2.c,v 1.17 2007/12/24 14:46:29 ad Exp $"); - -#include <sys/types.h> -#include <sys/lwpctl.h> - -#include <errno.h> -#include <limits.h> -#include <stdlib.h> -#include <string.h> -#include <stdio.h> - -#include "pthread.h" -#include "pthread_int.h" - -#ifdef PTHREAD__HAVE_ATOMIC - -/* - * Note that it's important to use the address of ptm_waiters as - * the list head in order for the hint arguments to _lwp_park / - * _lwp_unpark_all to match. - */ -#define pt_nextwaiter pt_sleep.ptqe_next -#define ptm_waiters ptm_blocked.ptqh_first -#define ptm_errorcheck ptm_lock - -#define MUTEX_WAITERS_BIT ((uintptr_t)0x01) -#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02) -#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04) -#define MUTEX_THREAD ((uintptr_t)-16L) - -#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT) -#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT) -#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD) -#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private) -#define MUTEX_SET_RECURSE(ptm, delta) \ - ((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta)) - -#if __GNUC_PREREQ__(3, 0) -#define NOINLINE __attribute ((noinline)) -#else -#define NOINLINE /* nothing */ -#endif - -static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *); -static int pthread__mutex_lock_slow(pthread_mutex_t *); -static int pthread__mutex_unlock_slow(pthread_mutex_t *); -static void pthread__mutex_pause(void); - -int _pthread_mutex_held_np(pthread_mutex_t *); -pthread_t _pthread_mutex_owner_np(pthread_mutex_t *); - -__weak_alias(pthread_mutex_held_np,_pthread_mutex_held_np) -__weak_alias(pthread_mutex_owner_np,_pthread_mutex_owner_np) - -__strong_alias(__libc_mutex_init,pthread_mutex_init) -__strong_alias(__libc_mutex_lock,pthread_mutex_lock) -__strong_alias(__libc_mutex_trylock,pthread_mutex_trylock) -__strong_alias(__libc_mutex_unlock,pthread_mutex_unlock) -__strong_alias(__libc_mutex_destroy,pthread_mutex_destroy) - -__strong_alias(__libc_mutexattr_init,pthread_mutexattr_init) -__strong_alias(__libc_mutexattr_destroy,pthread_mutexattr_destroy) -__strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype) - -__strong_alias(__libc_thr_once,pthread_once) - -static inline int -mutex_cas(volatile void *ptr, void **old, void *new) -{ - void *oldv; - - oldv = *old; - *old = pthread__atomic_cas_ptr(ptr, oldv, new); - return *old == oldv; -} - -static inline int -mutex_cas_ni(volatile void *ptr, void **old, void *new) -{ - void *oldv; - - oldv = *old; - *old = pthread__atomic_cas_ptr_ni(ptr, oldv, new); - return *old == oldv; -} - -int -pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr) -{ - intptr_t type; - - if (attr == NULL) - type = PTHREAD_MUTEX_NORMAL; - else - type = (intptr_t)attr->ptma_private; - - switch (type) { - case PTHREAD_MUTEX_ERRORCHECK: - ptm->ptm_errorcheck = 1; - ptm->ptm_owner = NULL; - break; - case PTHREAD_MUTEX_RECURSIVE: - ptm->ptm_errorcheck = 0; - ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT; - break; - default: - ptm->ptm_errorcheck = 0; - ptm->ptm_owner = NULL; - break; - } - - ptm->ptm_magic = _PT_MUTEX_MAGIC; - ptm->ptm_waiters = NULL; - ptm->ptm_private = NULL; - - return 0; -} - - -int -pthread_mutex_destroy(pthread_mutex_t *ptm) -{ - - pthread__error(EINVAL, "Invalid mutex", - ptm->ptm_magic == _PT_MUTEX_MAGIC); - pthread__error(EBUSY, "Destroying locked mutex", - MUTEX_OWNER(ptm->ptm_owner) == 0); - - ptm->ptm_magic = _PT_MUTEX_DEAD; - return 0; -} - -int -pthread_mutex_lock(pthread_mutex_t *ptm) -{ - void *owner; - pthread_t self; - - owner = NULL; - self = pthread__self(); - - if (__predict_true(mutex_cas(&ptm->ptm_owner, &owner, self))) - return 0; - return pthread__mutex_lock_slow(ptm); -} - -/* We want function call overhead. */ -NOINLINE static void -pthread__mutex_pause(void) -{ - - pthread__smt_pause(); -} - -/* - * Spin while the holder is running. 'lwpctl' gives us the true - * status of the thread. pt_blocking is set by libpthread in order - * to cut out system call and kernel spinlock overhead on remote CPUs - * (could represent many thousands of clock cycles). pt_blocking also - * makes this thread yield if the target is calling sched_yield(). - */ -NOINLINE static void * -pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner) -{ - pthread_t thread; - - for (;; owner = ptm->ptm_owner) { - thread = (pthread_t)MUTEX_OWNER(owner); - if (thread == NULL) - break; - if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE || - thread->pt_blocking) - break; - pthread__mutex_pause(); - pthread__mutex_pause(); - pthread__mutex_pause(); - pthread__mutex_pause(); - } - - return owner; -} - -NOINLINE static int -pthread__mutex_lock_slow(pthread_mutex_t *ptm) -{ - void *waiters, *new, *owner; - pthread_t self; - - pthread__error(EINVAL, "Invalid mutex", - ptm->ptm_magic == _PT_MUTEX_MAGIC); - - owner = ptm->ptm_owner; - self = pthread__self(); - - /* Recursive or errorcheck? */ - if (MUTEX_OWNER(owner) == (uintptr_t)self) { - if (MUTEX_RECURSIVE(owner)) { - if (MUTEX_GET_RECURSE(ptm) == INT_MAX) - return EAGAIN; - MUTEX_SET_RECURSE(ptm, +1); - return 0; - } - if (ptm->ptm_errorcheck) - return EDEADLK; - } - - for (;; owner = ptm->ptm_owner) { - /* Spin while the owner is running. */ - owner = pthread__mutex_spin(ptm, owner); - - /* If it has become free, try to acquire it again. */ - if (MUTEX_OWNER(owner) == 0) { - while (MUTEX_OWNER(owner) == 0) { - new = (void *) - ((uintptr_t)self | (uintptr_t)owner); - if (mutex_cas(&ptm->ptm_owner, &owner, new)) - return 0; - } - /* - * We have lost the race to acquire the mutex. - * The new owner could be running on another - * CPU, in which case we should spin and avoid - * the overhead of blocking. - */ - continue; - } - - /* - * Nope, still held. Add thread to the list of waiters. - * Issue a memory barrier to ensure sleeponq/nextwaiter - * are visible before we enter the waiters list. - */ - self->pt_sleeponq = 1; - for (waiters = ptm->ptm_waiters;;) { - self->pt_nextwaiter = waiters; - pthread__membar_producer(); - if (mutex_cas(&ptm->ptm_waiters, &waiters, self)) - break; - } - - /* - * Set the waiters bit and block. - * - * Note that the mutex can become unlocked before we set - * the waiters bit. If that happens it's not safe to sleep - * as we may never be awoken: we must remove the current - * thread from the waiters list and try again. - * - * Because we are doing this atomically, we can't remove - * one waiter: we must remove all waiters and awken them, - * then sleep in _lwp_park() until we have been awoken. - * - * Issue a memory barrier to ensure that we are reading - * the value of ptm_owner/pt_sleeponq after we have entered - * the waiters list (the CAS itself must be atomic). - */ - pthread__membar_consumer(); - for (owner = ptm->ptm_owner;;) { - if (MUTEX_HAS_WAITERS(owner)) - break; - if (MUTEX_OWNER(owner) == 0) { - pthread__mutex_wakeup(self, ptm); - break; - } - new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT); - if (mutex_cas(&ptm->ptm_owner, &owner, new)) { - /* - * pthread_mutex_unlock() can do a - * non-interlocked CAS. We cannot - * know if our attempt to set the - * waiters bit has succeeded while - * the holding thread is running. - * There are many assumptions; see - * sys/kern/kern_mutex.c for details. - * In short, we must spin if we see - * that the holder is running again. - */ - pthread__membar_full(); - owner = pthread__mutex_spin(ptm, owner); - } - } - - /* - * We may have been awoken by the current thread above, - * or will be awoken by the current holder of the mutex. - * The key requirement is that we must not proceed until - * told that we are no longer waiting (via pt_sleeponq - * being set to zero). Otherwise it is unsafe to re-enter - * the thread onto the waiters list. - */ - while (self->pt_sleeponq) { - self->pt_blocking++; - (void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL); - self->pt_blocking--; - } - } -} - -int -pthread_mutex_trylock(pthread_mutex_t *ptm) -{ - pthread_t self; - void *value; - - self = pthread__self(); - value = NULL; - - if (mutex_cas(&ptm->ptm_owner, &value, self)) - return 0; - - if (MUTEX_OWNER(value) == (uintptr_t)self && MUTEX_RECURSIVE(value)) { - if (MUTEX_GET_RECURSE(ptm) == INT_MAX) - return EAGAIN; - MUTEX_SET_RECURSE(ptm, +1); - return 0; - } - - return EBUSY; -} - -int -pthread_mutex_unlock(pthread_mutex_t *ptm) -{ - void *owner; - pthread_t self; - - self = pthread__self(); - owner = self; - - /* - * Note this may be a non-interlocked CAS. See lock_slow() - * above and sys/kern/kern_mutex.c for details. - */ - if (__predict_true(mutex_cas_ni(&ptm->ptm_owner, &owner, NULL))) - return 0; - return pthread__mutex_unlock_slow(ptm); -} - -NOINLINE static int -pthread__mutex_unlock_slow(pthread_mutex_t *ptm) -{ - pthread_t self, owner, new; - int weown, error, deferred; - - pthread__error(EINVAL, "Invalid mutex", - ptm->ptm_magic == _PT_MUTEX_MAGIC); - - self = pthread__self(); - owner = ptm->ptm_owner; - weown = (MUTEX_OWNER(owner) == (uintptr_t)self); - deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT); - error = 0; - - if (ptm->ptm_errorcheck) { - if (!weown) { - error = EPERM; - new = owner; - } else { - new = NULL; - } - } else if (MUTEX_RECURSIVE(owner)) { - if (!weown) { - error = EPERM; - new = owner; - } else if (MUTEX_GET_RECURSE(ptm) != 0) { - MUTEX_SET_RECURSE(ptm, -1); - new = owner; - } else { - new = (pthread_t)MUTEX_RECURSIVE_BIT; - } - } else { - pthread__error(EPERM, - "Unlocking unlocked mutex", (owner != NULL)); - pthread__error(EPERM, - "Unlocking mutex owned by another thread", weown); - new = NULL; - } - - /* - * Release the mutex. If there appear to be waiters, then - * wake them up. - */ - if (new != owner) { - owner = pthread__atomic_swap_ptr(&ptm->ptm_owner, new); - if (MUTEX_HAS_WAITERS(owner) != 0) { - pthread__mutex_wakeup(self, ptm); - return 0; - } - } - - /* - * There were no waiters, but we may have deferred waking - * other threads until mutex unlock - we must wake them now. - */ - if (!deferred) - return error; - - if (self->pt_nwaiters == 1) { - /* - * If the calling thread is about to block, defer - * unparking the target until _lwp_park() is called. - */ - if (self->pt_willpark && self->pt_unpark == 0) { - self->pt_unpark = self->pt_waiters[0]; - self->pt_unparkhint = &ptm->ptm_waiters; - } else { - (void)_lwp_unpark(self->pt_waiters[0], - &ptm->ptm_waiters); - } - } else { - (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, - &ptm->ptm_waiters); - } - self->pt_nwaiters = 0; - - return error; -} - -static void -pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm) -{ - pthread_t thread, next; - ssize_t n, rv; - - /* - * Take ownership of the current set of waiters. No - * need for a memory barrier following this, all loads - * are dependent upon 'thread'. - */ - thread = pthread__atomic_swap_ptr(&ptm->ptm_waiters, NULL); - - for (;;) { - /* - * Pull waiters from the queue and add to our list. - * Use a memory barrier to ensure that we safely - * read the value of pt_nextwaiter before 'thread' - * sees pt_sleeponq being cleared. - */ - for (n = self->pt_nwaiters, self->pt_nwaiters = 0; - n < pthread__unpark_max && thread != NULL; - thread = next) { - next = thread->pt_nextwaiter; - self->pt_waiters[n++] = thread->pt_lid; - pthread__membar_full(); - thread->pt_sleeponq = 0; - /* No longer safe to touch 'thread' */ - } - - switch (n) { - case 0: - return; - case 1: - /* - * If the calling thread is about to block, - * defer unparking the target until _lwp_park() - * is called. - */ - if (self->pt_willpark && self->pt_unpark == 0) { - self->pt_unpark = self->pt_waiters[0]; - self->pt_unparkhint = &ptm->ptm_waiters; - return; - } - rv = (ssize_t)_lwp_unpark(self->pt_waiters[0], - &ptm->ptm_waiters); - if (rv != 0 && errno != EALREADY && errno != EINTR && - errno != ESRCH) { - pthread__errorfunc(__FILE__, __LINE__, - __func__, "_lwp_unpark failed"); - } - return; - default: - rv = _lwp_unpark_all(self->pt_waiters, (size_t)n, - &ptm->ptm_waiters); - if (rv != 0 && errno != EINTR) { - pthread__errorfunc(__FILE__, __LINE__, - __func__, "_lwp_unpark_all failed"); - } - break; - } - } -} -int -pthread_mutexattr_init(pthread_mutexattr_t *attr) -{ - - attr->ptma_magic = _PT_MUTEXATTR_MAGIC; - attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT; - return 0; -} - -int -pthread_mutexattr_destroy(pthread_mutexattr_t *attr) -{ - - pthread__error(EINVAL, "Invalid mutex attribute", - attr->ptma_magic == _PT_MUTEXATTR_MAGIC); - - return 0; -} - - -int -pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep) -{ - - pthread__error(EINVAL, "Invalid mutex attribute", - attr->ptma_magic == _PT_MUTEXATTR_MAGIC); - - *typep = (int)(intptr_t)attr->ptma_private; - return 0; -} - - -int -pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) -{ - - pthread__error(EINVAL, "Invalid mutex attribute", - attr->ptma_magic == _PT_MUTEXATTR_MAGIC); - - switch (type) { - case PTHREAD_MUTEX_NORMAL: - case PTHREAD_MUTEX_ERRORCHECK: - case PTHREAD_MUTEX_RECURSIVE: - attr->ptma_private = (void *)(intptr_t)type; - return 0; - default: - return EINVAL; - } -} - - -static void -once_cleanup(void *closure) -{ - - pthread_mutex_unlock((pthread_mutex_t *)closure); -} - - -int -pthread_once(pthread_once_t *once_control, void (*routine)(void)) -{ - - if (once_control->pto_done == 0) { - pthread_mutex_lock(&once_control->pto_mutex); - pthread_cleanup_push(&once_cleanup, &once_control->pto_mutex); - if (once_control->pto_done == 0) { - routine(); - once_control->pto_done = 1; - } - pthread_cleanup_pop(1); - } - - return 0; -} - -int -pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm) -{ - - if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread) - return 0; - pthread__atomic_or_ulong((volatile unsigned long *) - (uintptr_t)&ptm->ptm_owner, - (unsigned long)MUTEX_DEFERRED_BIT); - return 1; -} - -int -_pthread_mutex_held_np(pthread_mutex_t *ptm) -{ - - return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self(); -} - -pthread_t -_pthread_mutex_owner_np(pthread_mutex_t *ptm) -{ - - return (pthread_t)MUTEX_OWNER(ptm->ptm_owner); -} - -#endif /* PTHREAD__HAVE_ATOMIC */ diff --git a/lib/libpthread/pthread_rwlock.c b/lib/libpthread/pthread_rwlock.c index 123b1537739..2d6f2a6739b 100644 --- a/lib/libpthread/pthread_rwlock.c +++ b/lib/libpthread/pthread_rwlock.c @@ -1,11 +1,11 @@ -/* $NetBSD: pthread_rwlock.c,v 1.26 2008/01/31 11:50:40 ad Exp $ */ +/* $NetBSD: pthread_rwlock.c,v 1.27 2008/02/10 18:50:55 ad Exp $ */ /*- - * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation - * by Nathan J. Williams and Andrew Doran. + * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,23 +37,32 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_rwlock.c,v 1.26 2008/01/31 11:50:40 ad Exp $"); +__RCSID("$NetBSD: pthread_rwlock.c,v 1.27 2008/02/10 18:50:55 ad Exp $"); #include <errno.h> +#include <stddef.h> #include "pthread.h" #include "pthread_int.h" -#ifndef PTHREAD__HAVE_ATOMIC +#define _RW_LOCKED 0 +#define _RW_WANT_WRITE 1 +#define _RW_WANT_READ 2 -__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np) -__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np) -__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np) +static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *); +static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *); +static void pthread__rwlock_early(void *); int _pthread_rwlock_held_np(pthread_rwlock_t *); int _pthread_rwlock_rdheld_np(pthread_rwlock_t *); int _pthread_rwlock_wrheld_np(pthread_rwlock_t *); +#ifndef lint +__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np); +__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np); +__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np); +#endif + __strong_alias(__libc_rwlock_init,pthread_rwlock_init) __strong_alias(__libc_rwlock_rdlock,pthread_rwlock_rdlock) __strong_alias(__libc_rwlock_wrlock,pthread_rwlock_wrlock) @@ -62,406 +71,531 @@ __strong_alias(__libc_rwlock_trywrlock,pthread_rwlock_trywrlock) __strong_alias(__libc_rwlock_unlock,pthread_rwlock_unlock) __strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy) +static inline uintptr_t +rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n) +{ + + return (uintptr_t)atomic_cas_ptr(&ptr->ptr_owner, (void *)o, + (void *)n); +} + int -pthread_rwlock_init(pthread_rwlock_t *rwlock, +pthread_rwlock_init(pthread_rwlock_t *ptr, const pthread_rwlockattr_t *attr) { -#ifdef ERRORCHECK - if ((rwlock == NULL) || - (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))) + + if (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)) return EINVAL; -#endif - rwlock->ptr_magic = _PT_RWLOCK_MAGIC; - pthread_lockinit(&rwlock->ptr_interlock); - PTQ_INIT(&rwlock->ptr_rblocked); - PTQ_INIT(&rwlock->ptr_wblocked); - rwlock->ptr_nreaders = 0; - rwlock->ptr_writer = NULL; + ptr->ptr_magic = _PT_RWLOCK_MAGIC; + pthread_lockinit(&ptr->ptr_interlock); + PTQ_INIT(&ptr->ptr_rblocked); + PTQ_INIT(&ptr->ptr_wblocked); + ptr->ptr_nreaders = 0; + ptr->ptr_owner = NULL; return 0; } int -pthread_rwlock_destroy(pthread_rwlock_t *rwlock) +pthread_rwlock_destroy(pthread_rwlock_t *ptr) { -#ifdef ERRORCHECK - if ((rwlock == NULL) || - (rwlock->ptr_magic != _PT_RWLOCK_MAGIC) || - (!PTQ_EMPTY(&rwlock->ptr_rblocked)) || - (!PTQ_EMPTY(&rwlock->ptr_wblocked)) || - (rwlock->ptr_nreaders != 0) || - (rwlock->ptr_writer != NULL)) + + if ((ptr->ptr_magic != _PT_RWLOCK_MAGIC) || + (!PTQ_EMPTY(&ptr->ptr_rblocked)) || + (!PTQ_EMPTY(&ptr->ptr_wblocked)) || + (ptr->ptr_nreaders != 0) || + (ptr->ptr_owner != NULL)) return EINVAL; -#endif - rwlock->ptr_magic = _PT_RWLOCK_DEAD; + ptr->ptr_magic = _PT_RWLOCK_DEAD; return 0; } - -int -pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) +static int +pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts) { + uintptr_t owner, next; pthread_t self; + int error; + + self = pthread__self(); + #ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) + if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) return EINVAL; #endif - self = pthread__self(); - - pthread__spinlock(self, &rwlock->ptr_interlock); -#ifdef ERRORCHECK - if (rwlock->ptr_writer == self) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EDEADLK; - } + + for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { + /* + * Read the lock owner field. If the need-to-wait + * indicator is clear, then try to acquire the lock. + */ + if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) == 0) { + next = rw_cas(ptr, owner, owner + RW_READ_INCR); + if (owner == next) { + /* Got it! */ +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); #endif - /* - * Don't get a readlock if there is a writer or if there are waiting - * writers; i.e. prefer writers to readers. This strategy is dictated - * by SUSv3. - */ - while ((rwlock->ptr_writer != NULL) || - (!PTQ_EMPTY(&rwlock->ptr_wblocked))) { - PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep); + return 0; + } + + /* + * Didn't get it -- spin around again (we'll + * probably sleep on the next iteration). + */ + continue; + } + + if ((owner & RW_THREAD) == (uintptr_t)self) + return EDEADLK; + + /* + * Grab the interlock. Once we have that, we + * can adjust the waiter bits and sleep queue. + */ + pthread__spinlock(self, &ptr->ptr_interlock); + + /* + * Mark the rwlock as having waiters. If the set fails, + * then we may not need to sleep and should spin again. + */ + next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS); + if (owner != next) { + pthread__spinunlock(self, &ptr->ptr_interlock); + continue; + } + + /* The waiters bit is set - it's safe to sleep. */ + PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep); + ptr->ptr_nreaders++; + self->pt_rwlocked = _RW_WANT_READ; self->pt_sleeponq = 1; - self->pt_sleepobj = &rwlock->ptr_rblocked; - pthread__spinunlock(self, &rwlock->ptr_interlock); - (void)pthread__park(self, &rwlock->ptr_interlock, - &rwlock->ptr_rblocked, NULL, 0, &rwlock->ptr_rblocked); - pthread__spinlock(self, &rwlock->ptr_interlock); - } - - rwlock->ptr_nreaders++; - pthread__spinunlock(self, &rwlock->ptr_interlock); + self->pt_sleepobj = &ptr->ptr_rblocked; + self->pt_early = pthread__rwlock_early; + pthread__spinunlock(self, &ptr->ptr_interlock); - return 0; + error = pthread__park(self, &ptr->ptr_interlock, + &ptr->ptr_rblocked, ts, 0, &ptr->ptr_rblocked); + + /* Did we get the lock? */ + if (self->pt_rwlocked == _RW_LOCKED) { +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; + } + if (error != 0) + return error; + + pthread__errorfunc(__FILE__, __LINE__, __func__, + "direct handoff failure"); + } } int -pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) +pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr) { - pthread_t self; + uintptr_t owner, next; #ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) + if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) return EINVAL; #endif - - self = pthread__self(); - pthread__spinlock(self, &rwlock->ptr_interlock); + /* * Don't get a readlock if there is a writer or if there are waiting * writers; i.e. prefer writers to readers. This strategy is dictated * by SUSv3. */ - if ((rwlock->ptr_writer != NULL) || - (!PTQ_EMPTY(&rwlock->ptr_wblocked))) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EBUSY; + for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { + if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0) + return EBUSY; + next = rw_cas(ptr, owner, owner + RW_READ_INCR); + if (owner == next) { + /* Got it! */ +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; + } } - - rwlock->ptr_nreaders++; - pthread__spinunlock(self, &rwlock->ptr_interlock); - - return 0; } - -int -pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) +static int +pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts) { + uintptr_t owner, next; pthread_t self; - extern int pthread__started; + int error; + + self = pthread__self(); #ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) + if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) return EINVAL; #endif - self = pthread__self(); - - pthread__spinlock(self, &rwlock->ptr_interlock); -#ifdef ERRORCHECK - if (rwlock->ptr_writer == self) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EDEADLK; - } + + for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { + /* + * Read the lock owner field. If the need-to-wait + * indicator is clear, then try to acquire the lock. + */ + if ((owner & RW_THREAD) == 0) { + next = rw_cas(ptr, owner, + (uintptr_t)self | RW_WRITE_LOCKED); + if (owner == next) { + /* Got it! */ +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); #endif - /* - * Prefer writers to readers here; permit writers even if there are - * waiting readers. - */ - while ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL)) { -#ifdef ERRORCHECK - if (pthread__started == 0) { - pthread__spinunlock(self, &rwlock->ptr_interlock); + return 0; + } + + /* + * Didn't get it -- spin around again (we'll + * probably sleep on the next iteration). + */ + continue; + } + + if ((owner & RW_THREAD) == (uintptr_t)self) return EDEADLK; + + /* + * Grab the interlock. Once we have that, we + * can adjust the waiter bits and sleep queue. + */ + pthread__spinlock(self, &ptr->ptr_interlock); + + /* + * Mark the rwlock as having waiters. If the set fails, + * then we may not need to sleep and should spin again. + */ + next = rw_cas(ptr, owner, + owner | RW_HAS_WAITERS | RW_WRITE_WANTED); + if (owner != next) { + pthread__spinunlock(self, &ptr->ptr_interlock); + continue; } -#endif - PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep); + + /* The waiters bit is set - it's safe to sleep. */ + PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep); + self->pt_rwlocked = _RW_WANT_WRITE; self->pt_sleeponq = 1; - self->pt_sleepobj = &rwlock->ptr_wblocked; - pthread__spinunlock(self, &rwlock->ptr_interlock); - (void)pthread__park(self, &rwlock->ptr_interlock, - &rwlock->ptr_wblocked, NULL, 0, &rwlock->ptr_wblocked); - pthread__spinlock(self, &rwlock->ptr_interlock); - } + self->pt_sleepobj = &ptr->ptr_wblocked; + self->pt_early = pthread__rwlock_early; + pthread__spinunlock(self, &ptr->ptr_interlock); - rwlock->ptr_writer = self; - pthread__spinunlock(self, &rwlock->ptr_interlock); + error = pthread__park(self, &ptr->ptr_interlock, + &ptr->ptr_wblocked, ts, 0, &ptr->ptr_wblocked); - return 0; + /* Did we get the lock? */ + if (self->pt_rwlocked == _RW_LOCKED) { +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; + } + if (error != 0) + return error; + + pthread__errorfunc(__FILE__, __LINE__, __func__, + "direct handoff failure"); + } } int -pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) +pthread_rwlock_trywrlock(pthread_rwlock_t *ptr) { + uintptr_t owner, next; pthread_t self; + #ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) + if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) return EINVAL; #endif + self = pthread__self(); - - pthread__spinlock(self, &rwlock->ptr_interlock); - /* - * Prefer writers to readers here; permit writers even if there are - * waiting readers. - */ - if ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL)) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EBUSY; + + for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { + if (owner != 0) + return EBUSY; + next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED); + if (owner == next) { + /* Got it! */ +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_enter(); +#endif + return 0; + } } +} - rwlock->ptr_writer = self; - pthread__spinunlock(self, &rwlock->ptr_interlock); +int +pthread_rwlock_rdlock(pthread_rwlock_t *ptr) +{ - return 0; + return pthread__rwlock_rdlock(ptr, NULL); } - int -pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlock, - const struct timespec *abs_timeout) +pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr, + const struct timespec *abs_timeout) { - pthread_t self; - int retval; -#ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) - return EINVAL; if (abs_timeout == NULL) return EINVAL; -#endif if ((abs_timeout->tv_nsec >= 1000000000) || (abs_timeout->tv_nsec < 0) || (abs_timeout->tv_sec < 0)) return EINVAL; - self = pthread__self(); - pthread__spinlock(self, &rwlock->ptr_interlock); -#ifdef ERRORCHECK - if (rwlock->ptr_writer == self) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EDEADLK; - } -#endif - /* - * Don't get a readlock if there is a writer or if there are waiting - * writers; i.e. prefer writers to readers. This strategy is dictated - * by SUSv3. - */ - retval = 0; - while ((retval == 0) && ((rwlock->ptr_writer != NULL) || - (!PTQ_EMPTY(&rwlock->ptr_wblocked)))) { - PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep); - self->pt_sleeponq = 1; - self->pt_sleepobj = &rwlock->ptr_rblocked; - pthread__spinunlock(self, &rwlock->ptr_interlock); - retval = pthread__park(self, &rwlock->ptr_interlock, - &rwlock->ptr_rblocked, abs_timeout, 0, - &rwlock->ptr_rblocked); - pthread__spinlock(self, &rwlock->ptr_interlock); - } + return pthread__rwlock_rdlock(ptr, abs_timeout); +} - /* One last chance to get the lock, in case it was released between - the alarm firing and when this thread got rescheduled, or in case - a signal handler kept it busy */ - if ((rwlock->ptr_writer == NULL) && - (PTQ_EMPTY(&rwlock->ptr_wblocked))) { - rwlock->ptr_nreaders++; - retval = 0; - } - pthread__spinunlock(self, &rwlock->ptr_interlock); +int +pthread_rwlock_wrlock(pthread_rwlock_t *ptr) +{ - return retval; + return pthread__rwlock_wrlock(ptr, NULL); } - int -pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlock, - const struct timespec *abs_timeout) +pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr, + const struct timespec *abs_timeout) { - pthread_t self; - int retval; - extern int pthread__started; -#ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) - return EINVAL; if (abs_timeout == NULL) return EINVAL; -#endif if ((abs_timeout->tv_nsec >= 1000000000) || (abs_timeout->tv_nsec < 0) || (abs_timeout->tv_sec < 0)) return EINVAL; - self = pthread__self(); - pthread__spinlock(self, &rwlock->ptr_interlock); -#ifdef ERRORCHECK - if (rwlock->ptr_writer == self) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EDEADLK; - } -#endif - /* - * Prefer writers to readers here; permit writers even if there are - * waiting readers. - */ - retval = 0; - while (retval == 0 && - ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL))) { -#ifdef ERRORCHECK - if (pthread__started == 0) { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EDEADLK; - } -#endif - PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep); - self->pt_sleeponq = 1; - self->pt_sleepobj = &rwlock->ptr_wblocked; - pthread__spinunlock(self, &rwlock->ptr_interlock); - retval = pthread__park(self, &rwlock->ptr_interlock, - &rwlock->ptr_wblocked, abs_timeout, 0, - &rwlock->ptr_wblocked); - pthread__spinlock(self, &rwlock->ptr_interlock); - } - - if ((rwlock->ptr_nreaders == 0) && (rwlock->ptr_writer == NULL)) { - rwlock->ptr_writer = self; - retval = 0; - } - pthread__spinunlock(self, &rwlock->ptr_interlock); - - return retval; + return pthread__rwlock_wrlock(ptr, abs_timeout); } int -pthread_rwlock_unlock(pthread_rwlock_t *rwlock) +pthread_rwlock_unlock(pthread_rwlock_t *ptr) { - pthread_t self, writer; + uintptr_t owner, decr, new, next; + pthread_t self, thread; + #ifdef ERRORCHECK - if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC)) + if ((ptr == NULL) || (ptr->ptr_magic != _PT_RWLOCK_MAGIC)) return EINVAL; #endif - writer = NULL; + +#ifndef PTHREAD__ATOMIC_IS_MEMBAR + membar_exit(); +#endif + + /* + * Since we used an add operation to set the required lock + * bits, we can use a subtract to clear them, which makes + * the read-release and write-release path similar. + */ self = pthread__self(); - - pthread__spinlock(self, &rwlock->ptr_interlock); - if (rwlock->ptr_writer != NULL) { - /* Releasing a write lock. */ -#ifdef ERRORCHECK - if (rwlock->ptr_writer != self) { - pthread__spinunlock(self, &rwlock->ptr_interlock); + owner = (uintptr_t)ptr->ptr_owner; + if ((owner & RW_WRITE_LOCKED) != 0) { + decr = (uintptr_t)self | RW_WRITE_LOCKED; + if ((owner & RW_THREAD) != (uintptr_t)self) { return EPERM; } -#endif - rwlock->ptr_writer = NULL; - writer = PTQ_FIRST(&rwlock->ptr_wblocked); - if (writer != NULL) { - PTQ_REMOVE(&rwlock->ptr_wblocked, writer, pt_sleep); + } else { + decr = RW_READ_INCR; + if (owner == 0) { + return EPERM; } - } else -#ifdef ERRORCHECK - if (rwlock->ptr_nreaders > 0) -#endif - { - /* Releasing a read lock. */ - rwlock->ptr_nreaders--; - if (rwlock->ptr_nreaders == 0) { - writer = PTQ_FIRST(&rwlock->ptr_wblocked); - if (writer != NULL) - PTQ_REMOVE(&rwlock->ptr_wblocked, writer, - pt_sleep); + } + + for (;; owner = next) { + /* + * Compute what we expect the new value of the lock to be. + * Only proceed to do direct handoff if there are waiters, + * and if the lock would become unowned. + */ + new = (owner - decr); + if ((new & (RW_THREAD | RW_HAS_WAITERS)) != RW_HAS_WAITERS) { + next = rw_cas(ptr, owner, new); + if (owner == next) { + /* Released! */ + return 0; + } + continue; } -#ifdef ERRORCHECK - } else { - pthread__spinunlock(self, &rwlock->ptr_interlock); - return EPERM; -#endif + + /* + * Grab the interlock. Once we have that, we can adjust + * the waiter bits. We must check to see if there are + * still waiters before proceeding. + */ + pthread__spinlock(self, &ptr->ptr_interlock); + owner = (uintptr_t)ptr->ptr_owner; + if ((owner & RW_HAS_WAITERS) == 0) { + pthread__spinunlock(self, &ptr->ptr_interlock); + next = owner; + continue; + } + + /* + * Give the lock away. SUSv3 dictates that we must give + * preference to writers. + */ + if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) { + new = (uintptr_t)thread | RW_WRITE_LOCKED; + + if (PTQ_NEXT(thread, pt_sleep) != NULL) + new |= RW_HAS_WAITERS | RW_WRITE_WANTED; + else if (ptr->ptr_nreaders != 0) + new |= RW_HAS_WAITERS; + + /* + * Set in the new value. The lock becomes owned + * by the writer that we are about to wake. + */ + (void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new); + + /* Wake the writer. */ + PTQ_REMOVE(&ptr->ptr_wblocked, thread, pt_sleep); + thread->pt_rwlocked = _RW_LOCKED; + pthread__unpark(self, &ptr->ptr_interlock, + &ptr->ptr_wblocked, thread); + } else { + new = 0; + PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) { + /* + * May have already been handed the lock, + * since pthread__unpark_all() can release + * our interlock before awakening all + * threads. + */ + if (thread->pt_sleepobj == NULL) + continue; + new += RW_READ_INCR; + thread->pt_rwlocked = _RW_LOCKED; + } + + /* + * Set in the new value. The lock becomes owned + * by the readers that we are about to wake. + */ + (void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new); + + /* Wake up all sleeping readers. */ + ptr->ptr_nreaders = 0; + pthread__unpark_all(self, &ptr->ptr_interlock, + &ptr->ptr_rblocked); + } + + return 0; } +} - if (writer != NULL) - pthread__unpark(self, &rwlock->ptr_interlock, - &rwlock->ptr_wblocked, writer); +/* + * Called when a timedlock awakens early to adjust the waiter bits. + * The rwlock's interlock is held on entry, and the caller has been + * removed from the waiters lists. + */ +static void +pthread__rwlock_early(void *obj) +{ + uintptr_t owner, set, new, next; + pthread_rwlock_t *ptr; + pthread_t self; + u_int off; + + self = pthread__self(); + + switch (self->pt_rwlocked) { + case _RW_WANT_READ: + off = offsetof(pthread_rwlock_t, ptr_rblocked); + break; + case _RW_WANT_WRITE: + off = offsetof(pthread_rwlock_t, ptr_wblocked); + break; + default: + pthread__errorfunc(__FILE__, __LINE__, __func__, + "bad value of pt_rwlocked"); + off = 0; + /* NOTREACHED */ + break; + } + + /* LINTED mind your own business */ + ptr = (pthread_rwlock_t *)((uint8_t *)obj - off); + owner = (uintptr_t)ptr->ptr_owner; + + if ((owner & RW_THREAD) == 0) { + pthread__errorfunc(__FILE__, __LINE__, __func__, + "lock not held"); + } + + if (!PTQ_EMPTY(&ptr->ptr_wblocked)) + set = RW_HAS_WAITERS | RW_WRITE_WANTED; + else if (ptr->ptr_nreaders != 0) + set = RW_HAS_WAITERS; else - pthread__unpark_all(self, &rwlock->ptr_interlock, - &rwlock->ptr_rblocked); + set = 0; - return 0; + for (;; owner = next) { + new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set; + next = rw_cas(ptr, owner, new); + if (owner == next) + break; + } } - int -pthread_rwlockattr_init(pthread_rwlockattr_t *attr) +_pthread_rwlock_held_np(pthread_rwlock_t *ptr) { -#ifdef ERRORCHECK - if (attr == NULL) - return EINVAL; -#endif - attr->ptra_magic = _PT_RWLOCKATTR_MAGIC; + uintptr_t owner = (uintptr_t)ptr->ptr_owner; - return 0; + return (owner & RW_THREAD) != 0; } - int -pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr) +_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr) { -#ifdef ERRORCHECK - if ((attr == NULL) || - (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)) - return EINVAL; -#endif - attr->ptra_magic = _PT_RWLOCKATTR_DEAD; + uintptr_t owner = (uintptr_t)ptr->ptr_owner; - return 0; + return (owner & RW_THREAD) != 0 && (owner & RW_WRITE_LOCKED) == 0; } int -_pthread_rwlock_held_np(pthread_rwlock_t *ptr) +_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr) { + uintptr_t owner = (uintptr_t)ptr->ptr_owner; - return ptr->ptr_writer != NULL || ptr->ptr_nreaders != 0; + return (owner & (RW_THREAD | RW_WRITE_LOCKED)) == + ((uintptr_t)pthread__self() | RW_WRITE_LOCKED); } int -_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr) +pthread_rwlockattr_init(pthread_rwlockattr_t *attr) { - return ptr->ptr_nreaders != 0; + if (attr == NULL) + return EINVAL; + attr->ptra_magic = _PT_RWLOCKATTR_MAGIC; + + return 0; } + int -_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr) +pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr) { - return ptr->ptr_writer == pthread__self(); -} + if ((attr == NULL) || + (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)) + return EINVAL; + attr->ptra_magic = _PT_RWLOCKATTR_DEAD; -#endif /* !PTHREAD__HAVE_ATOMIC */ + return 0; +} diff --git a/lib/libpthread/pthread_rwlock2.c b/lib/libpthread/pthread_rwlock2.c deleted file mode 100644 index cd5953ed77b..00000000000 --- a/lib/libpthread/pthread_rwlock2.c +++ /dev/null @@ -1,584 +0,0 @@ -/* $NetBSD: pthread_rwlock2.c,v 1.9 2008/01/31 11:50:40 ad Exp $ */ - -/*- - * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_rwlock2.c,v 1.9 2008/01/31 11:50:40 ad Exp $"); - -#include <errno.h> -#include <stddef.h> - -#include "pthread.h" -#include "pthread_int.h" - -#ifdef PTHREAD__HAVE_ATOMIC - -#define _RW_LOCKED 0 -#define _RW_WANT_WRITE 1 -#define _RW_WANT_READ 2 - -static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *); -static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *); -static void pthread__rwlock_early(void *); - -int _pthread_rwlock_held_np(pthread_rwlock_t *); -int _pthread_rwlock_rdheld_np(pthread_rwlock_t *); -int _pthread_rwlock_wrheld_np(pthread_rwlock_t *); - -#ifndef lint -__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np); -__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np); -__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np); -#endif - -__strong_alias(__libc_rwlock_init,pthread_rwlock_init) -__strong_alias(__libc_rwlock_rdlock,pthread_rwlock_rdlock) -__strong_alias(__libc_rwlock_wrlock,pthread_rwlock_wrlock) -__strong_alias(__libc_rwlock_tryrdlock,pthread_rwlock_tryrdlock) -__strong_alias(__libc_rwlock_trywrlock,pthread_rwlock_trywrlock) -__strong_alias(__libc_rwlock_unlock,pthread_rwlock_unlock) -__strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy) - -static inline uintptr_t -rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n) -{ - - return (uintptr_t)pthread__atomic_cas_ptr(&ptr->ptr_owner, (void *)o, - (void *)n); -} - -int -pthread_rwlock_init(pthread_rwlock_t *ptr, - const pthread_rwlockattr_t *attr) -{ - - if (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)) - return EINVAL; - ptr->ptr_magic = _PT_RWLOCK_MAGIC; - pthread_lockinit(&ptr->ptr_interlock); - PTQ_INIT(&ptr->ptr_rblocked); - PTQ_INIT(&ptr->ptr_wblocked); - ptr->ptr_nreaders = 0; - ptr->ptr_owner = NULL; - - return 0; -} - - -int -pthread_rwlock_destroy(pthread_rwlock_t *ptr) -{ - - if ((ptr->ptr_magic != _PT_RWLOCK_MAGIC) || - (!PTQ_EMPTY(&ptr->ptr_rblocked)) || - (!PTQ_EMPTY(&ptr->ptr_wblocked)) || - (ptr->ptr_nreaders != 0) || - (ptr->ptr_owner != NULL)) - return EINVAL; - ptr->ptr_magic = _PT_RWLOCK_DEAD; - - return 0; -} - -static int -pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts) -{ - uintptr_t owner, next; - pthread_t self; - int error; - - self = pthread__self(); - -#ifdef ERRORCHECK - if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) - return EINVAL; -#endif - - for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { - /* - * Read the lock owner field. If the need-to-wait - * indicator is clear, then try to acquire the lock. - */ - if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) == 0) { - next = rw_cas(ptr, owner, owner + RW_READ_INCR); - if (owner == next) { - /* Got it! */ - return 0; - } - - /* - * Didn't get it -- spin around again (we'll - * probably sleep on the next iteration). - */ - continue; - } - - if ((owner & RW_THREAD) == (uintptr_t)self) - return EDEADLK; - - /* - * Grab the interlock. Once we have that, we - * can adjust the waiter bits and sleep queue. - */ - pthread__spinlock(self, &ptr->ptr_interlock); - - /* - * Mark the rwlock as having waiters. If the set fails, - * then we may not need to sleep and should spin again. - */ - next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS); - if (owner != next) { - pthread__spinunlock(self, &ptr->ptr_interlock); - continue; - } - - /* The waiters bit is set - it's safe to sleep. */ - PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep); - ptr->ptr_nreaders++; - self->pt_rwlocked = _RW_WANT_READ; - self->pt_sleeponq = 1; - self->pt_sleepobj = &ptr->ptr_rblocked; - self->pt_early = pthread__rwlock_early; - pthread__spinunlock(self, &ptr->ptr_interlock); - - error = pthread__park(self, &ptr->ptr_interlock, - &ptr->ptr_rblocked, ts, 0, &ptr->ptr_rblocked); - - /* Did we get the lock? */ - if (self->pt_rwlocked == _RW_LOCKED) - return 0; - if (error != 0) - return error; - - pthread__errorfunc(__FILE__, __LINE__, __func__, - "direct handoff failure"); - } -} - - -int -pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr) -{ - uintptr_t owner, next; - -#ifdef ERRORCHECK - if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) - return EINVAL; -#endif - - /* - * Don't get a readlock if there is a writer or if there are waiting - * writers; i.e. prefer writers to readers. This strategy is dictated - * by SUSv3. - */ - for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { - if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0) - return EBUSY; - next = rw_cas(ptr, owner, owner + RW_READ_INCR); - if (owner == next) { - /* Got it! */ - return 0; - } - } -} - -static int -pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts) -{ - uintptr_t owner, next; - pthread_t self; - int error; - - self = pthread__self(); - -#ifdef ERRORCHECK - if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) - return EINVAL; -#endif - - for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { - /* - * Read the lock owner field. If the need-to-wait - * indicator is clear, then try to acquire the lock. - */ - if ((owner & RW_THREAD) == 0) { - next = rw_cas(ptr, owner, - (uintptr_t)self | RW_WRITE_LOCKED); - if (owner == next) { - /* Got it! */ - return 0; - } - - /* - * Didn't get it -- spin around again (we'll - * probably sleep on the next iteration). - */ - continue; - } - - if ((owner & RW_THREAD) == (uintptr_t)self) - return EDEADLK; - - /* - * Grab the interlock. Once we have that, we - * can adjust the waiter bits and sleep queue. - */ - pthread__spinlock(self, &ptr->ptr_interlock); - - /* - * Mark the rwlock as having waiters. If the set fails, - * then we may not need to sleep and should spin again. - */ - next = rw_cas(ptr, owner, - owner | RW_HAS_WAITERS | RW_WRITE_WANTED); - if (owner != next) { - pthread__spinunlock(self, &ptr->ptr_interlock); - continue; - } - - /* The waiters bit is set - it's safe to sleep. */ - PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep); - self->pt_rwlocked = _RW_WANT_WRITE; - self->pt_sleeponq = 1; - self->pt_sleepobj = &ptr->ptr_wblocked; - self->pt_early = pthread__rwlock_early; - pthread__spinunlock(self, &ptr->ptr_interlock); - - error = pthread__park(self, &ptr->ptr_interlock, - &ptr->ptr_wblocked, ts, 0, &ptr->ptr_wblocked); - - /* Did we get the lock? */ - if (self->pt_rwlocked == _RW_LOCKED) - return 0; - if (error != 0) - return error; - - pthread__errorfunc(__FILE__, __LINE__, __func__, - "direct handoff failure"); - } -} - - -int -pthread_rwlock_trywrlock(pthread_rwlock_t *ptr) -{ - uintptr_t owner, next; - pthread_t self; - -#ifdef ERRORCHECK - if (ptr->ptr_magic != _PT_RWLOCK_MAGIC) - return EINVAL; -#endif - - self = pthread__self(); - - for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) { - if (owner != 0) - return EBUSY; - next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED); - if (owner == next) { - /* Got it! */ - return 0; - } - } -} - -int -pthread_rwlock_rdlock(pthread_rwlock_t *ptr) -{ - - return pthread__rwlock_rdlock(ptr, NULL); -} - -int -pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr, - const struct timespec *abs_timeout) -{ - - if (abs_timeout == NULL) - return EINVAL; - if ((abs_timeout->tv_nsec >= 1000000000) || - (abs_timeout->tv_nsec < 0) || - (abs_timeout->tv_sec < 0)) - return EINVAL; - - return pthread__rwlock_rdlock(ptr, abs_timeout); -} - -int -pthread_rwlock_wrlock(pthread_rwlock_t *ptr) -{ - - return pthread__rwlock_wrlock(ptr, NULL); -} - -int -pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr, - const struct timespec *abs_timeout) -{ - - if (abs_timeout == NULL) - return EINVAL; - if ((abs_timeout->tv_nsec >= 1000000000) || - (abs_timeout->tv_nsec < 0) || - (abs_timeout->tv_sec < 0)) - return EINVAL; - - return pthread__rwlock_wrlock(ptr, abs_timeout); -} - - -int -pthread_rwlock_unlock(pthread_rwlock_t *ptr) -{ - uintptr_t owner, decr, new, next; - pthread_t self, thread; - -#ifdef ERRORCHECK - if ((ptr == NULL) || (ptr->ptr_magic != _PT_RWLOCK_MAGIC)) - return EINVAL; -#endif - - self = pthread__self(); - - /* - * Since we used an add operation to set the required lock - * bits, we can use a subtract to clear them, which makes - * the read-release and write-release path similar. - */ - owner = (uintptr_t)ptr->ptr_owner; - if ((owner & RW_WRITE_LOCKED) != 0) { - decr = (uintptr_t)self | RW_WRITE_LOCKED; - if ((owner & RW_THREAD) != (uintptr_t)self) { - return EPERM; - } - } else { - decr = RW_READ_INCR; - if (owner == 0) { - return EPERM; - } - } - - for (;; owner = next) { - /* - * Compute what we expect the new value of the lock to be. - * Only proceed to do direct handoff if there are waiters, - * and if the lock would become unowned. - */ - new = (owner - decr); - if ((new & (RW_THREAD | RW_HAS_WAITERS)) != RW_HAS_WAITERS) { - next = rw_cas(ptr, owner, new); - if (owner == next) { - /* Released! */ - return 0; - } - continue; - } - - /* - * Grab the interlock. Once we have that, we can adjust - * the waiter bits. We must check to see if there are - * still waiters before proceeding. - */ - pthread__spinlock(self, &ptr->ptr_interlock); - owner = (uintptr_t)ptr->ptr_owner; - if ((owner & RW_HAS_WAITERS) == 0) { - pthread__spinunlock(self, &ptr->ptr_interlock); - next = owner; - continue; - } - - /* - * Give the lock away. SUSv3 dictates that we must give - * preference to writers. - */ - if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) { - new = (uintptr_t)thread | RW_WRITE_LOCKED; - - if (PTQ_NEXT(thread, pt_sleep) != NULL) - new |= RW_HAS_WAITERS | RW_WRITE_WANTED; - else if (ptr->ptr_nreaders != 0) - new |= RW_HAS_WAITERS; - - /* - * Set in the new value. The lock becomes owned - * by the writer that we are about to wake. - */ - (void)pthread__atomic_swap_ptr(&ptr->ptr_owner, - (void *)new); - - /* Wake the writer. */ - PTQ_REMOVE(&ptr->ptr_wblocked, thread, pt_sleep); - thread->pt_rwlocked = _RW_LOCKED; - pthread__unpark(self, &ptr->ptr_interlock, - &ptr->ptr_wblocked, thread); - } else { - new = 0; - PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) { - /* - * May have already been handed the lock, - * since pthread__unpark_all() can release - * our interlock before awakening all - * threads. - */ - if (thread->pt_sleepobj == NULL) - continue; - new += RW_READ_INCR; - thread->pt_rwlocked = _RW_LOCKED; - } - - /* - * Set in the new value. The lock becomes owned - * by the readers that we are about to wake. - */ - (void)pthread__atomic_swap_ptr(&ptr->ptr_owner, - (void *)new); - - /* Wake up all sleeping readers. */ - ptr->ptr_nreaders = 0; - pthread__unpark_all(self, &ptr->ptr_interlock, - &ptr->ptr_rblocked); - } - - return 0; - } -} - -/* - * Called when a timedlock awakens early to adjust the waiter bits. - * The rwlock's interlock is held on entry, and the caller has been - * removed from the waiters lists. - */ -static void -pthread__rwlock_early(void *obj) -{ - uintptr_t owner, set, new, next; - pthread_rwlock_t *ptr; - pthread_t self; - u_int off; - - self = pthread__self(); - - switch (self->pt_rwlocked) { - case _RW_WANT_READ: - off = offsetof(pthread_rwlock_t, ptr_rblocked); - break; - case _RW_WANT_WRITE: - off = offsetof(pthread_rwlock_t, ptr_wblocked); - break; - default: - pthread__errorfunc(__FILE__, __LINE__, __func__, - "bad value of pt_rwlocked"); - off = 0; - /* NOTREACHED */ - break; - } - - /* LINTED mind your own business */ - ptr = (pthread_rwlock_t *)((uint8_t *)obj - off); - owner = (uintptr_t)ptr->ptr_owner; - - if ((owner & RW_THREAD) == 0) { - pthread__errorfunc(__FILE__, __LINE__, __func__, - "lock not held"); - } - - if (!PTQ_EMPTY(&ptr->ptr_wblocked)) - set = RW_HAS_WAITERS | RW_WRITE_WANTED; - else if (ptr->ptr_nreaders != 0) - set = RW_HAS_WAITERS; - else - set = 0; - - for (;; owner = next) { - new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set; - next = rw_cas(ptr, owner, new); - if (owner == next) - break; - } -} - -int -_pthread_rwlock_held_np(pthread_rwlock_t *ptr) -{ - uintptr_t owner = (uintptr_t)ptr->ptr_owner; - - return (owner & RW_THREAD) != 0; -} - -int -_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr) -{ - uintptr_t owner = (uintptr_t)ptr->ptr_owner; - - return (owner & RW_THREAD) != 0 && (owner & RW_WRITE_LOCKED) == 0; -} - -int -_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr) -{ - uintptr_t owner = (uintptr_t)ptr->ptr_owner; - - return (owner & (RW_THREAD | RW_WRITE_LOCKED)) == - ((uintptr_t)pthread__self() | RW_WRITE_LOCKED); -} - -int -pthread_rwlockattr_init(pthread_rwlockattr_t *attr) -{ - - if (attr == NULL) - return EINVAL; - attr->ptra_magic = _PT_RWLOCKATTR_MAGIC; - - return 0; -} - - -int -pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr) -{ - - if ((attr == NULL) || - (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)) - return EINVAL; - attr->ptra_magic = _PT_RWLOCKATTR_DEAD; - - return 0; -} - -#endif /* PTHREAD__HAVE_ATOMIC */ |
