summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorad <ad@NetBSD.org>2008-02-10 18:50:54 +0000
committerad <ad@NetBSD.org>2008-02-10 18:50:54 +0000
commita67e1e347544393a4828fa21d5cf6f670590fd76 (patch)
treec200cca9ccceecd65fe50f0b925bb5ad7831fa14
parent749400b7044c2a3ca87455ac26cc1184b678a353 (diff)
- Remove libpthread's atomic ops.
- Remove the old spinlock-based mutex and rwlock implementations. - Use the atomic ops from libc.
-rw-r--r--lib/libpthread/Makefile10
-rw-r--r--lib/libpthread/arch/hppa/pthread_md.h5
-rw-r--r--lib/libpthread/arch/i386/_context_u.S30
-rw-r--r--lib/libpthread/arch/i386/pthread_md.h34
-rw-r--r--lib/libpthread/arch/m68k/pthread_md.h5
-rw-r--r--lib/libpthread/arch/sh3/pthread_md.h5
-rw-r--r--lib/libpthread/arch/sparc/pthread_md.h5
-rw-r--r--lib/libpthread/arch/vax/pthread_md.h5
-rw-r--r--lib/libpthread/arch/x86_64/_context_u.S28
-rw-r--r--lib/libpthread/arch/x86_64/pthread_md.h32
-rw-r--r--lib/libpthread/pthread_int.h16
-rw-r--r--lib/libpthread/pthread_misc.c9
-rw-r--r--lib/libpthread/pthread_mutex.c638
-rw-r--r--lib/libpthread/pthread_mutex2.c623
-rw-r--r--lib/libpthread/pthread_rwlock.c686
-rw-r--r--lib/libpthread/pthread_rwlock2.c584
16 files changed, 843 insertions, 1872 deletions
diff --git a/lib/libpthread/Makefile b/lib/libpthread/Makefile
index a9c9f6e1379..ae9521dc901 100644
--- a/lib/libpthread/Makefile
+++ b/lib/libpthread/Makefile
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.49 2008/01/15 03:37:14 rmind Exp $
+# $NetBSD: Makefile,v 1.50 2008/02/10 18:50:54 ad Exp $
#
WARNS= 4
@@ -54,9 +54,7 @@ SRCS+= pthread_cond.c
SRCS+= pthread_lock.c
SRCS+= pthread_misc.c
SRCS+= pthread_mutex.c
-SRCS+= pthread_mutex2.c
SRCS+= pthread_rwlock.c
-SRCS+= pthread_rwlock2.c
SRCS+= pthread_specific.c
SRCS+= pthread_spin.c
SRCS+= pthread_tsd.c
@@ -82,10 +80,10 @@ pthread_lock.po: pthread_lock.o
${_MKTARGET_CREATE}
cp pthread_lock.o pthread_lock.po
-COPTS.pthread_mutex2.c+= -fomit-frame-pointer -falign-functions=32
-pthread_mutex2.po: pthread_mutex2.o
+COPTS.pthread_mutex.c+= -fomit-frame-pointer -falign-functions=32
+pthread_mutex.po: pthread_mutex.o
${_MKTARGET_CREATE}
- cp pthread_mutex2.o pthread_mutex2.po
+ cp pthread_mutex.o pthread_mutex.po
COPTS.pthread.c += -Wno-stack-protector
diff --git a/lib/libpthread/arch/hppa/pthread_md.h b/lib/libpthread/arch/hppa/pthread_md.h
index 5e106df0621..bd7371cf486 100644
--- a/lib/libpthread/arch/hppa/pthread_md.h
+++ b/lib/libpthread/arch/hppa/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.4 2007/09/24 12:19:39 skrll Exp $ */
+/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@@ -98,4 +98,7 @@ do { \
(uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \
} while (/*CONSTCOND*/0)
+/* Don't need additional memory barriers. */
+#define PTHREAD__ATOMIC_IS_MEMBAR
+
#endif /* !_LIB_PTHREAD_HPPA_MD_H */
diff --git a/lib/libpthread/arch/i386/_context_u.S b/lib/libpthread/arch/i386/_context_u.S
index 6720bd67402..8d9cba7737a 100644
--- a/lib/libpthread/arch/i386/_context_u.S
+++ b/lib/libpthread/arch/i386/_context_u.S
@@ -1,4 +1,4 @@
-/* $NetBSD: _context_u.S,v 1.7 2007/11/13 17:20:09 ad Exp $ */
+/* $NetBSD: _context_u.S,v 1.8 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
@@ -100,31 +100,3 @@ STUB(_swapcontext_u_xmm)
movl %eax, (UC_REGS + _REG_UESP * 4)(%ecx)
movl 8(%esp), %ecx
SETC
-
-STUB(pthread__atomic_swap_ptr)
- movl 4(%esp), %ecx
- movl 8(%esp), %eax
- xchgl %eax, (%ecx)
- ret
-
-STUB(pthread__atomic_or_ulong)
- movl 4(%esp), %ecx
- movl 8(%esp), %eax
- lock
- orl %eax, (%ecx)
- ret
-
-STUB(pthread__membar_full)
- lock
- addl $0, -4(%esp)
- ret
-
-STUB(pthread__membar_producer)
- /* A store is enough */
- addl $0, -4(%esp)
- ret
-
-STUB(pthread__membar_consumer)
- lock
- addl $0, -4(%esp)
- ret
diff --git a/lib/libpthread/arch/i386/pthread_md.h b/lib/libpthread/arch/i386/pthread_md.h
index c99ae7f412e..6128b557c18 100644
--- a/lib/libpthread/arch/i386/pthread_md.h
+++ b/lib/libpthread/arch/i386/pthread_md.h
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_md.h,v 1.11 2007/11/13 17:20:10 ad Exp $ */
+/* $NetBSD: pthread_md.h,v 1.12 2008/02/10 18:50:55 ad Exp $ */
/*-
- * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -182,9 +182,11 @@ pthread__sp(void)
} while (/*CONSTCOND*/0)
#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory")
-#define PTHREAD__HAVE_ATOMIC
/* #define PTHREAD__HAVE_THREADREG */
+/* Don't need additional memory barriers. */
+#define PTHREAD__ATOMIC_IS_MEMBAR
+
void pthread__threadreg_set(pthread_t);
static inline pthread_t
@@ -199,30 +201,4 @@ pthread__threadreg_get(void)
return self;
}
-static inline void *
-pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new)
-{
- volatile uintptr_t *cast = ptr;
- void *ret;
-
- __asm __volatile ("lock; cmpxchgl %2, %1"
- : "=a" (ret), "=m" (*cast)
- : "r" (new), "m" (*cast), "0" (old));
-
- return ret;
-}
-
-static inline void *
-pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new)
-{
- volatile uintptr_t *cast = ptr;
- void *ret;
-
- __asm __volatile ("cmpxchgl %2, %1"
- : "=a" (ret), "=m" (*cast)
- : "r" (new), "m" (*cast), "0" (old));
-
- return ret;
-}
-
#endif /* _LIB_PTHREAD_I386_MD_H */
diff --git a/lib/libpthread/arch/m68k/pthread_md.h b/lib/libpthread/arch/m68k/pthread_md.h
index 3bcfe9b9954..4f873a1ff85 100644
--- a/lib/libpthread/arch/m68k/pthread_md.h
+++ b/lib/libpthread/arch/m68k/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:16 perry Exp $ */
+/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@@ -99,4 +99,7 @@ pthread__sp(void)
(uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \
} while (/*CONSTCOND*/0)
+/* m68k will not go SMP */
+#define PTHREAD__ATOMIC_IS_MEMBAR
+
#endif /* _LIB_PTHREAD_M68K_MD_H */
diff --git a/lib/libpthread/arch/sh3/pthread_md.h b/lib/libpthread/arch/sh3/pthread_md.h
index 8f5db27e859..59933c3f66d 100644
--- a/lib/libpthread/arch/sh3/pthread_md.h
+++ b/lib/libpthread/arch/sh3/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.3 2005/12/24 21:11:17 perry Exp $ */
+/* $NetBSD: pthread_md.h,v 1.4 2008/02/10 18:50:55 ad Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@@ -90,4 +90,7 @@ pthread__sp(void)
#define PTHREAD_FPREG_TO_UCONTEXT(uc, freg)
#endif
+/* sh3 will not go SMP */
+#define PTHREAD__ATOMIC_IS_MEMBAR
+
#endif /* _LIB_PTHREAD_SH3_MD_H */
diff --git a/lib/libpthread/arch/sparc/pthread_md.h b/lib/libpthread/arch/sparc/pthread_md.h
index af2210747a9..4d105d7e698 100644
--- a/lib/libpthread/arch/sparc/pthread_md.h
+++ b/lib/libpthread/arch/sparc/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:17 perry Exp $ */
+/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -92,4 +92,7 @@ pthread__sp(void)
(uc)->uc_flags = ((uc)->uc_flags | _UC_FPU) & ~_UC_USER; \
} while (/*CONSTCOND*/0)
+/* Don't need additional memory barriers. */
+#define PTHREAD__ATOMIC_IS_MEMBAR
+
#endif /* _LIB_PTHREAD_SPARC_MD_H */
diff --git a/lib/libpthread/arch/vax/pthread_md.h b/lib/libpthread/arch/vax/pthread_md.h
index 7ffe5220a4a..e2ed4cca1d3 100644
--- a/lib/libpthread/arch/vax/pthread_md.h
+++ b/lib/libpthread/arch/vax/pthread_md.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_md.h,v 1.4 2005/12/24 21:11:17 perry Exp $ */
+/* $NetBSD: pthread_md.h,v 1.5 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@@ -85,4 +85,7 @@ pthread__sp(void)
#define PTHREAD_FPREG_TO_UCONTEXT(uc, reg) do { \
} while (/*CONSTCOND*/0)
+/* Don't need additional memory barriers. */
+#define PTHREAD__ATOMIC_IS_MEMBAR
+
#endif /* _LIB_PTHREAD_VAX_MD_H */
diff --git a/lib/libpthread/arch/x86_64/_context_u.S b/lib/libpthread/arch/x86_64/_context_u.S
index 8719b4fff5e..b9ec75ccf4e 100644
--- a/lib/libpthread/arch/x86_64/_context_u.S
+++ b/lib/libpthread/arch/x86_64/_context_u.S
@@ -1,4 +1,4 @@
-/* $NetBSD: _context_u.S,v 1.9 2007/11/13 17:20:10 ad Exp $ */
+/* $NetBSD: _context_u.S,v 1.10 2008/02/10 18:50:55 ad Exp $ */
/*-
* Copyright (c) 2001 The NetBSD Foundation, Inc.
@@ -137,29 +137,3 @@ STUB(_swapcontext_u)
movq %rax, (UC_REGS + _REG_URSP * 8)(%rdi)
movq %rsi, %rdi
SETC
-
-STUB(pthread__atomic_swap_ptr)
- movq %rsi, %rax
- lock
- xchgq %rax, (%rdi)
- ret
-
-STUB(pthread__atomic_or_ulong)
- lock
- orq %rsi, (%rdi)
- ret
-
-STUB(pthread__membar_full)
- lock
- addq $0, -8(%rsp)
- ret
-
-STUB(pthread__membar_producer)
- /* A store is enough */
- movq $0, -8(%rsp)
- ret
-
-STUB(pthread__membar_consumer)
- lock
- addq $0, -8(%rsp)
- ret
diff --git a/lib/libpthread/arch/x86_64/pthread_md.h b/lib/libpthread/arch/x86_64/pthread_md.h
index 7f04edfd85a..6db24f0faf0 100644
--- a/lib/libpthread/arch/x86_64/pthread_md.h
+++ b/lib/libpthread/arch/x86_64/pthread_md.h
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_md.h,v 1.7 2007/11/13 17:20:10 ad Exp $ */
+/* $NetBSD: pthread_md.h,v 1.8 2008/02/10 18:50:55 ad Exp $ */
/*-
- * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -104,32 +104,8 @@ pthread__sp(void)
} while (/*CONSTCOND*/0)
#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory")
-#define PTHREAD__HAVE_ATOMIC
-static inline void *
-pthread__atomic_cas_ptr(volatile void *ptr, const void *old, const void *new)
-{
- volatile uintptr_t *cast = ptr;
- void *ret;
-
- __asm __volatile ("lock; cmpxchgq %2, %1"
- : "=a" (ret), "=m" (*cast)
- : "r" (new), "m" (*cast), "0" (old));
-
- return ret;
-}
-
-static inline void *
-pthread__atomic_cas_ptr_ni(volatile void *ptr, const void *old, const void *new)
-{
- volatile uintptr_t *cast = ptr;
- void *ret;
-
- __asm __volatile ("cmpxchgq %2, %1"
- : "=a" (ret), "=m" (*cast)
- : "r" (new), "m" (*cast), "0" (old));
-
- return ret;
-}
+/* Don't need additional memory barriers. */
+#define PTHREAD__ATOMIC_IS_MEMBAR
#endif /* _LIB_PTHREAD_X86_64_MD_H */
diff --git a/lib/libpthread/pthread_int.h b/lib/libpthread/pthread_int.h
index 5e4d113bc80..82c5d21d006 100644
--- a/lib/libpthread/pthread_int.h
+++ b/lib/libpthread/pthread_int.h
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_int.h,v 1.65 2008/01/08 20:56:08 christos Exp $ */
+/* $NetBSD: pthread_int.h,v 1.66 2008/02/10 18:50:54 ad Exp $ */
/*-
- * Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -51,6 +51,10 @@
#include "pthread_queue.h"
#include "pthread_md.h"
+/* Need to use libc-private names for atomic operations. */
+#include "../../common/lib/libc/atomic/atomic_op_namespace.h"
+
+#include <sys/atomic.h>
#include <sys/tree.h>
#include <lwp.h>
@@ -289,14 +293,6 @@ void pthread__errorfunc(const char *, int, const char *, const char *)
PTHREAD_HIDE;
char *pthread__getenv(const char *) PTHREAD_HIDE;
void pthread__cancelled(void) PTHREAD_HIDE;
-
-void *pthread__atomic_cas_ptr(volatile void *, const void *, const void *) PTHREAD_HIDE;
-void *pthread__atomic_swap_ptr(volatile void *, const void *) PTHREAD_HIDE;
-void pthread__atomic_or_ulong(volatile unsigned long *, unsigned long) PTHREAD_HIDE;
-void pthread__membar_full(void) PTHREAD_HIDE;
-void pthread__membar_producer(void) PTHREAD_HIDE;
-void pthread__membar_consumer(void) PTHREAD_HIDE;
-
int pthread__mutex_deferwake(pthread_t, pthread_mutex_t *) PTHREAD_HIDE;
#ifndef pthread__smt_pause
diff --git a/lib/libpthread/pthread_misc.c b/lib/libpthread/pthread_misc.c
index 4a5b3930f10..198225ec0f3 100644
--- a/lib/libpthread/pthread_misc.c
+++ b/lib/libpthread/pthread_misc.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_misc.c,v 1.6 2008/02/09 17:07:54 yamt Exp $ */
+/* $NetBSD: pthread_misc.c,v 1.7 2008/02/10 18:50:54 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_misc.c,v 1.6 2008/02/09 17:07:54 yamt Exp $");
+__RCSID("$NetBSD: pthread_misc.c,v 1.7 2008/02/10 18:50:54 ad Exp $");
#include <errno.h>
#include <string.h>
@@ -173,13 +173,12 @@ pthread__sched_yield(void)
self = pthread__self();
-#ifdef PTHREAD__HAVE_ATOMIC
/* Memory barrier for unlocked mutex release. */
- pthread__membar_producer();
-#endif
+ membar_producer();
self->pt_blocking++;
error = _sys_sched_yield();
self->pt_blocking--;
+ membar_sync();
return error;
}
diff --git a/lib/libpthread/pthread_mutex.c b/lib/libpthread/pthread_mutex.c
index 26e2fb92904..54cf1dc434f 100644
--- a/lib/libpthread/pthread_mutex.c
+++ b/lib/libpthread/pthread_mutex.c
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_mutex.c,v 1.43 2008/01/25 02:12:10 rafal Exp $ */
+/* $NetBSD: pthread_mutex.c,v 1.44 2008/02/10 18:50:54 ad Exp $ */
/*-
- * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -37,24 +37,51 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_mutex.c,v 1.43 2008/01/25 02:12:10 rafal Exp $");
+__RCSID("$NetBSD: pthread_mutex.c,v 1.44 2008/02/10 18:50:54 ad Exp $");
#include <sys/types.h>
-
-#include <machine/lock.h>
+#include <sys/lwpctl.h>
#include <errno.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
+#include <stdio.h>
#include "pthread.h"
#include "pthread_int.h"
-#ifndef PTHREAD__HAVE_ATOMIC
-
-static int pthread_mutex_lock_slow(pthread_t, pthread_mutex_t *);
-static void once_cleanup(void *);
+/*
+ * Note that it's important to use the address of ptm_waiters as
+ * the list head in order for the hint arguments to _lwp_park /
+ * _lwp_unpark_all to match.
+ */
+#define pt_nextwaiter pt_sleep.ptqe_next
+#define ptm_waiters ptm_blocked.ptqh_first
+#define ptm_errorcheck ptm_lock
+
+#define MUTEX_WAITERS_BIT ((uintptr_t)0x01)
+#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02)
+#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04)
+#define MUTEX_THREAD ((uintptr_t)-16L)
+
+#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
+#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
+#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD)
+#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private)
+#define MUTEX_SET_RECURSE(ptm, delta) \
+ ((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta))
+
+#if __GNUC_PREREQ__(3, 0)
+#define NOINLINE __attribute ((noinline))
+#else
+#define NOINLINE /* nothing */
+#endif
+
+static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
+static int pthread__mutex_lock_slow(pthread_mutex_t *);
+static int pthread__mutex_unlock_slow(pthread_mutex_t *);
+static void pthread__mutex_pause(void);
int _pthread_mutex_held_np(pthread_mutex_t *);
pthread_t _pthread_mutex_owner_np(pthread_mutex_t *);
@@ -74,309 +101,431 @@ __strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype)
__strong_alias(__libc_thr_once,pthread_once)
-struct mutex_private {
- int type;
- int recursecount;
-};
-
-static const struct mutex_private mutex_private_default = {
- PTHREAD_MUTEX_DEFAULT,
- 0,
-};
-
-struct mutexattr_private {
- int type;
-};
-
-static const struct mutexattr_private mutexattr_private_default = {
- PTHREAD_MUTEX_DEFAULT,
-};
-
int
-pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
+pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
{
- struct mutexattr_private *map;
- struct mutex_private *mp;
+ intptr_t type;
- pthread__error(EINVAL, "Invalid mutex attribute",
- (attr == NULL) || (attr->ptma_magic == _PT_MUTEXATTR_MAGIC));
-
- if (attr != NULL && (map = attr->ptma_private) != NULL &&
- memcmp(map, &mutexattr_private_default, sizeof(*map)) != 0) {
- mp = malloc(sizeof(*mp));
- if (mp == NULL)
- return ENOMEM;
+ if (attr == NULL)
+ type = PTHREAD_MUTEX_NORMAL;
+ else
+ type = (intptr_t)attr->ptma_private;
- mp->type = map->type;
- mp->recursecount = 0;
- } else {
- /* LINTED cast away const */
- mp = (struct mutex_private *) &mutex_private_default;
+ switch (type) {
+ case PTHREAD_MUTEX_ERRORCHECK:
+ ptm->ptm_errorcheck = 1;
+ ptm->ptm_owner = NULL;
+ break;
+ case PTHREAD_MUTEX_RECURSIVE:
+ ptm->ptm_errorcheck = 0;
+ ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
+ break;
+ default:
+ ptm->ptm_errorcheck = 0;
+ ptm->ptm_owner = NULL;
+ break;
}
- mutex->ptm_magic = _PT_MUTEX_MAGIC;
- mutex->ptm_owner = NULL;
- pthread_lockinit(&mutex->ptm_lock);
- pthread_lockinit(&mutex->ptm_interlock);
- PTQ_INIT(&mutex->ptm_blocked);
- mutex->ptm_private = mp;
+ ptm->ptm_magic = _PT_MUTEX_MAGIC;
+ ptm->ptm_waiters = NULL;
+ ptm->ptm_private = NULL;
return 0;
}
int
-pthread_mutex_destroy(pthread_mutex_t *mutex)
+pthread_mutex_destroy(pthread_mutex_t *ptm)
{
pthread__error(EINVAL, "Invalid mutex",
- mutex->ptm_magic == _PT_MUTEX_MAGIC);
+ ptm->ptm_magic == _PT_MUTEX_MAGIC);
pthread__error(EBUSY, "Destroying locked mutex",
- __SIMPLELOCK_UNLOCKED_P(&mutex->ptm_lock));
-
- mutex->ptm_magic = _PT_MUTEX_DEAD;
- if (mutex->ptm_private != NULL &&
- mutex->ptm_private != (const void *)&mutex_private_default)
- free(mutex->ptm_private);
+ MUTEX_OWNER(ptm->ptm_owner) == 0);
+ ptm->ptm_magic = _PT_MUTEX_DEAD;
return 0;
}
-
-/*
- * Note regarding memory visibility: Pthreads has rules about memory
- * visibility and mutexes. Very roughly: Memory a thread can see when
- * it unlocks a mutex can be seen by another thread that locks the
- * same mutex.
- *
- * A memory barrier after a lock and before an unlock will provide
- * this behavior. This code relies on pthread__spintrylock() to issue
- * a barrier after obtaining a lock, and on pthread__spinunlock() to
- * issue a barrier before releasing a lock.
- */
-
int
-pthread_mutex_lock(pthread_mutex_t *mutex)
+pthread_mutex_lock(pthread_mutex_t *ptm)
{
pthread_t self;
- int error;
+ void *val;
self = pthread__self();
-
- /*
- * Note that if we get the lock, we don't have to deal with any
- * non-default lock type handling.
- */
- if (__predict_false(pthread__spintrylock(self, &mutex->ptm_lock) == 0)) {
- error = pthread_mutex_lock_slow(self, mutex);
- if (error)
- return error;
+ val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
+ if (__predict_true(val == NULL)) {
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
}
+ return pthread__mutex_lock_slow(ptm);
+}
- /*
- * We have the lock!
- */
- mutex->ptm_owner = self;
+/* We want function call overhead. */
+NOINLINE static void
+pthread__mutex_pause(void)
+{
- return 0;
+ pthread__smt_pause();
}
+/*
+ * Spin while the holder is running. 'lwpctl' gives us the true
+ * status of the thread. pt_blocking is set by libpthread in order
+ * to cut out system call and kernel spinlock overhead on remote CPUs
+ * (could represent many thousands of clock cycles). pt_blocking also
+ * makes this thread yield if the target is calling sched_yield().
+ */
+NOINLINE static void *
+pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner)
+{
+ pthread_t thread;
+ unsigned int count, i;
+
+ for (count = 2;; owner = ptm->ptm_owner) {
+ thread = (pthread_t)MUTEX_OWNER(owner);
+ if (thread == NULL)
+ break;
+ if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE ||
+ thread->pt_blocking)
+ break;
+ if (count < 128)
+ count += count;
+ for (i = count; i != 0; i--)
+ pthread__mutex_pause();
+ }
+
+ return owner;
+}
-static int
-pthread_mutex_lock_slow(pthread_t self, pthread_mutex_t *mutex)
+NOINLINE static int
+pthread__mutex_lock_slow(pthread_mutex_t *ptm)
{
- struct mutex_private *mp;
- int count;
+ void *waiters, *new, *owner, *next;
+ pthread_t self;
pthread__error(EINVAL, "Invalid mutex",
- mutex->ptm_magic == _PT_MUTEX_MAGIC);
+ ptm->ptm_magic == _PT_MUTEX_MAGIC);
- for (;;) {
- /* Spin for a while. */
- count = pthread__nspins;
- while (__SIMPLELOCK_LOCKED_P(&mutex->ptm_lock) && --count > 0)
- pthread__smt_pause();
- if (count > 0) {
- if (pthread__spintrylock(self, &mutex->ptm_lock) != 0)
- break;
- continue;
+ owner = ptm->ptm_owner;
+ self = pthread__self();
+
+ /* Recursive or errorcheck? */
+ if (MUTEX_OWNER(owner) == (uintptr_t)self) {
+ if (MUTEX_RECURSIVE(owner)) {
+ if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
+ return EAGAIN;
+ MUTEX_SET_RECURSE(ptm, +1);
+ return 0;
}
+ if (ptm->ptm_errorcheck)
+ return EDEADLK;
+ }
- /* Okay, didn't look free. Get the interlock... */
- pthread__spinlock(self, &mutex->ptm_interlock);
+ for (;; owner = ptm->ptm_owner) {
+ /* Spin while the owner is running. */
+ owner = pthread__mutex_spin(ptm, owner);
+
+ /* If it has become free, try to acquire it again. */
+ if (MUTEX_OWNER(owner) == 0) {
+ for (; MUTEX_OWNER(owner) == 0; owner = next) {
+ new = (void *)
+ ((uintptr_t)self | (uintptr_t)owner);
+ next = atomic_cas_ptr(&ptm->ptm_owner, owner,
+ new);
+ if (next == owner) {
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
+ }
+ }
+ /*
+ * We have lost the race to acquire the mutex.
+ * The new owner could be running on another
+ * CPU, in which case we should spin and avoid
+ * the overhead of blocking.
+ */
+ if (!MUTEX_HAS_WAITERS(owner))
+ continue;
+ }
/*
- * The mutex_unlock routine will get the interlock
- * before looking at the list of sleepers, so if the
- * lock is held we can safely put ourselves on the
- * sleep queue. If it's not held, we can try taking it
- * again.
+ * Nope, still held. Add thread to the list of waiters.
+ * Issue a memory barrier to ensure sleeponq/nextwaiter
+ * are visible before we enter the waiters list.
*/
- PTQ_INSERT_HEAD(&mutex->ptm_blocked, self, pt_sleep);
- if (__SIMPLELOCK_UNLOCKED_P(&mutex->ptm_lock)) {
- PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep);
- pthread__spinunlock(self, &mutex->ptm_interlock);
- continue;
+ self->pt_sleeponq = 1;
+ for (waiters = ptm->ptm_waiters;; waiters = next) {
+ self->pt_nextwaiter = waiters;
+ membar_producer();
+ next = atomic_cas_ptr(&ptm->ptm_waiters, waiters, self);
+ if (next == waiters)
+ break;
}
- mp = mutex->ptm_private;
- if (mutex->ptm_owner == self && mp != NULL) {
- switch (mp->type) {
- case PTHREAD_MUTEX_ERRORCHECK:
- PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep);
- pthread__spinunlock(self, &mutex->ptm_interlock);
- return EDEADLK;
-
- case PTHREAD_MUTEX_RECURSIVE:
+ /*
+ * Set the waiters bit and block.
+ *
+ * Note that the mutex can become unlocked before we set
+ * the waiters bit. If that happens it's not safe to sleep
+ * as we may never be awoken: we must remove the current
+ * thread from the waiters list and try again.
+ *
+ * Because we are doing this atomically, we can't remove
+ * one waiter: we must remove all waiters and awken them,
+ * then sleep in _lwp_park() until we have been awoken.
+ *
+ * Issue a memory barrier to ensure that we are reading
+ * the value of ptm_owner/pt_sleeponq after we have entered
+ * the waiters list (the CAS itself must be atomic).
+ */
+ membar_consumer();
+ for (owner = ptm->ptm_owner;; owner = next) {
+ if (MUTEX_HAS_WAITERS(owner))
+ break;
+ if (MUTEX_OWNER(owner) == 0) {
+ pthread__mutex_wakeup(self, ptm);
+ break;
+ }
+ new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT);
+ next = atomic_cas_ptr(&ptm->ptm_owner, owner, new);
+ if (next == owner) {
/*
- * It's safe to do this without
- * holding the interlock, because
- * we only modify it if we know we
- * own the mutex.
+ * pthread_mutex_unlock() can do a
+ * non-interlocked CAS. We cannot
+ * know if our attempt to set the
+ * waiters bit has succeeded while
+ * the holding thread is running.
+ * There are many assumptions; see
+ * sys/kern/kern_mutex.c for details.
+ * In short, we must spin if we see
+ * that the holder is running again.
*/
- PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep);
- pthread__spinunlock(self, &mutex->ptm_interlock);
- if (mp->recursecount == INT_MAX)
- return EAGAIN;
- mp->recursecount++;
- return 0;
+ membar_sync();
+ next = pthread__mutex_spin(ptm, owner);
}
}
/*
- * Locking a mutex is not a cancellation
- * point, so we don't need to do the
- * test-cancellation dance. We may get woken
- * up spuriously by pthread_cancel or signals,
- * but it's okay since we're just going to
- * retry.
+ * We may have been awoken by the current thread above,
+ * or will be awoken by the current holder of the mutex.
+ * The key requirement is that we must not proceed until
+ * told that we are no longer waiting (via pt_sleeponq
+ * being set to zero). Otherwise it is unsafe to re-enter
+ * the thread onto the waiters list.
*/
- self->pt_sleeponq = 1;
- self->pt_sleepobj = &mutex->ptm_blocked;
- pthread__spinunlock(self, &mutex->ptm_interlock);
- (void)pthread__park(self, &mutex->ptm_interlock,
- &mutex->ptm_blocked, NULL, 0, &mutex->ptm_blocked);
+ while (self->pt_sleeponq) {
+ self->pt_blocking++;
+ (void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL);
+ self->pt_blocking--;
+ membar_sync();
+ }
}
-
- return 0;
}
-
int
-pthread_mutex_trylock(pthread_mutex_t *mutex)
+pthread_mutex_trylock(pthread_mutex_t *ptm)
{
- struct mutex_private *mp;
pthread_t self;
-
- pthread__error(EINVAL, "Invalid mutex",
- mutex->ptm_magic == _PT_MUTEX_MAGIC);
+ void *val;
self = pthread__self();
-
- if (pthread__spintrylock(self, &mutex->ptm_lock) == 0) {
- /*
- * These tests can be performed without holding the
- * interlock because these fields are only modified
- * if we know we own the mutex.
- */
- mp = mutex->ptm_private;
- if (mp != NULL && mp->type == PTHREAD_MUTEX_RECURSIVE &&
- mutex->ptm_owner == self) {
- if (mp->recursecount == INT_MAX)
- return EAGAIN;
- mp->recursecount++;
- return 0;
- }
-
- return EBUSY;
+ val = atomic_cas_ptr(&ptm->ptm_owner, NULL, self);
+ if (__predict_true(val == NULL)) {
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
}
- mutex->ptm_owner = self;
+ if (MUTEX_OWNER(val) == (uintptr_t)self && MUTEX_RECURSIVE(val)) {
+ if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
+ return EAGAIN;
+ MUTEX_SET_RECURSE(ptm, +1);
+ return 0;
+ }
- return 0;
+ return EBUSY;
}
-
int
-pthread_mutex_unlock(pthread_mutex_t *mutex)
+pthread_mutex_unlock(pthread_mutex_t *ptm)
{
- struct mutex_private *mp;
pthread_t self;
- int weown;
-
- pthread__error(EINVAL, "Invalid mutex",
- mutex->ptm_magic == _PT_MUTEX_MAGIC);
+ void *value;
/*
- * These tests can be performed without holding the
- * interlock because these fields are only modified
- * if we know we own the mutex.
+ * Note this may be a non-interlocked CAS. See lock_slow()
+ * above and sys/kern/kern_mutex.c for details.
*/
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_exit();
+#endif
self = pthread__self();
- weown = (mutex->ptm_owner == self);
- mp = mutex->ptm_private;
-
- if (mp == NULL) {
- if (__predict_false(!weown)) {
- pthread__error(EPERM, "Unlocking unlocked mutex",
- (mutex->ptm_owner != 0));
- pthread__error(EPERM,
- "Unlocking mutex owned by another thread", weown);
+ value = atomic_cas_ptr_ni(&ptm->ptm_owner, self, NULL);
+ if (__predict_true(value == self))
+ return 0;
+ return pthread__mutex_unlock_slow(ptm);
+}
+
+NOINLINE static int
+pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
+{
+ pthread_t self, owner, new;
+ int weown, error, deferred;
+
+ pthread__error(EINVAL, "Invalid mutex",
+ ptm->ptm_magic == _PT_MUTEX_MAGIC);
+
+ self = pthread__self();
+ owner = ptm->ptm_owner;
+ weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
+ deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT);
+ error = 0;
+
+ if (ptm->ptm_errorcheck) {
+ if (!weown) {
+ error = EPERM;
+ new = owner;
+ } else {
+ new = NULL;
+ }
+ } else if (MUTEX_RECURSIVE(owner)) {
+ if (!weown) {
+ error = EPERM;
+ new = owner;
+ } else if (MUTEX_GET_RECURSE(ptm) != 0) {
+ MUTEX_SET_RECURSE(ptm, -1);
+ new = owner;
+ } else {
+ new = (pthread_t)MUTEX_RECURSIVE_BIT;
}
- } else if (mp->type == PTHREAD_MUTEX_RECURSIVE) {
- if (!weown)
- return EPERM;
- if (mp->recursecount != 0) {
- mp->recursecount--;
+ } else {
+ pthread__error(EPERM,
+ "Unlocking unlocked mutex", (owner != NULL));
+ pthread__error(EPERM,
+ "Unlocking mutex owned by another thread", weown);
+ new = NULL;
+ }
+
+ /*
+ * Release the mutex. If there appear to be waiters, then
+ * wake them up.
+ */
+ if (new != owner) {
+ owner = atomic_swap_ptr(&ptm->ptm_owner, new);
+ if (MUTEX_HAS_WAITERS(owner) != 0) {
+ pthread__mutex_wakeup(self, ptm);
return 0;
}
- } else if (mp->type == PTHREAD_MUTEX_ERRORCHECK) {
- if (!weown)
- return EPERM;
- if (__predict_false(!weown)) {
- pthread__error(EPERM, "Unlocking unlocked mutex",
- (mutex->ptm_owner != 0));
- pthread__error(EPERM,
- "Unlocking mutex owned by another thread", weown);
+ }
+
+ /*
+ * There were no waiters, but we may have deferred waking
+ * other threads until mutex unlock - we must wake them now.
+ */
+ if (!deferred)
+ return error;
+
+ if (self->pt_nwaiters == 1) {
+ /*
+ * If the calling thread is about to block, defer
+ * unparking the target until _lwp_park() is called.
+ */
+ if (self->pt_willpark && self->pt_unpark == 0) {
+ self->pt_unpark = self->pt_waiters[0];
+ self->pt_unparkhint = &ptm->ptm_waiters;
+ } else {
+ (void)_lwp_unpark(self->pt_waiters[0],
+ &ptm->ptm_waiters);
}
+ } else {
+ (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
+ &ptm->ptm_waiters);
}
+ self->pt_nwaiters = 0;
- mutex->ptm_owner = NULL;
- pthread__spinunlock(self, &mutex->ptm_lock);
+ return error;
+}
+
+static void
+pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
+{
+ pthread_t thread, next;
+ ssize_t n, rv;
/*
- * Do a double-checked locking dance to see if there are any
- * waiters. If we don't see any waiters, we can exit, because
- * we've already released the lock. If we do see waiters, they
- * were probably waiting on us... there's a slight chance that
- * they are waiting on a different thread's ownership of the
- * lock that happened between the unlock above and this
- * examination of the queue; if so, no harm is done, as the
- * waiter will loop and see that the mutex is still locked.
+ * Take ownership of the current set of waiters. No
+ * need for a memory barrier following this, all loads
+ * are dependent upon 'thread'.
*/
- pthread__spinlock(self, &mutex->ptm_interlock);
- pthread__unpark_all(self, &mutex->ptm_interlock, &mutex->ptm_blocked);
- return 0;
-}
+ thread = atomic_swap_ptr(&ptm->ptm_waiters, NULL);
+
+ for (;;) {
+ /*
+ * Pull waiters from the queue and add to our list.
+ * Use a memory barrier to ensure that we safely
+ * read the value of pt_nextwaiter before 'thread'
+ * sees pt_sleeponq being cleared.
+ */
+ for (n = self->pt_nwaiters, self->pt_nwaiters = 0;
+ n < pthread__unpark_max && thread != NULL;
+ thread = next) {
+ next = thread->pt_nextwaiter;
+ if (thread != self) {
+ self->pt_waiters[n++] = thread->pt_lid;
+ membar_sync();
+ }
+ thread->pt_sleeponq = 0;
+ /* No longer safe to touch 'thread' */
+ }
+ switch (n) {
+ case 0:
+ return;
+ case 1:
+ /*
+ * If the calling thread is about to block,
+ * defer unparking the target until _lwp_park()
+ * is called.
+ */
+ if (self->pt_willpark && self->pt_unpark == 0) {
+ self->pt_unpark = self->pt_waiters[0];
+ self->pt_unparkhint = &ptm->ptm_waiters;
+ return;
+ }
+ rv = (ssize_t)_lwp_unpark(self->pt_waiters[0],
+ &ptm->ptm_waiters);
+ if (rv != 0 && errno != EALREADY && errno != EINTR &&
+ errno != ESRCH) {
+ pthread__errorfunc(__FILE__, __LINE__,
+ __func__, "_lwp_unpark failed");
+ }
+ return;
+ default:
+ rv = _lwp_unpark_all(self->pt_waiters, (size_t)n,
+ &ptm->ptm_waiters);
+ if (rv != 0 && errno != EINTR) {
+ pthread__errorfunc(__FILE__, __LINE__,
+ __func__, "_lwp_unpark_all failed");
+ }
+ break;
+ }
+ }
+}
int
pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
- struct mutexattr_private *map;
-
- map = malloc(sizeof(*map));
- if (map == NULL)
- return ENOMEM;
-
- *map = mutexattr_private_default;
attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
- attr->ptma_private = map;
-
+ attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
return 0;
}
-
int
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
{
@@ -384,10 +533,6 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
- attr->ptma_magic = _PT_MUTEXATTR_DEAD;
- if (attr->ptma_private != NULL)
- free(attr->ptma_private);
-
return 0;
}
@@ -395,15 +540,11 @@ pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
int
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
{
- struct mutexattr_private *map;
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
- map = attr->ptma_private;
-
- *typep = map->type;
-
+ *typep = (int)(intptr_t)attr->ptma_private;
return 0;
}
@@ -411,25 +552,19 @@ pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
int
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
{
- struct mutexattr_private *map;
pthread__error(EINVAL, "Invalid mutex attribute",
attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
- map = attr->ptma_private;
-
switch (type) {
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_RECURSIVE:
- map->type = type;
- break;
-
+ attr->ptma_private = (void *)(intptr_t)type;
+ return 0;
default:
return EINVAL;
}
-
- return 0;
}
@@ -459,24 +594,27 @@ pthread_once(pthread_once_t *once_control, void (*routine)(void))
}
int
-pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *mutex)
+pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm)
{
- return mutex->ptm_owner == thread;
+ if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread)
+ return 0;
+ atomic_or_ulong((volatile unsigned long *)
+ (uintptr_t)&ptm->ptm_owner,
+ (unsigned long)MUTEX_DEFERRED_BIT);
+ return 1;
}
int
-_pthread_mutex_held_np(pthread_mutex_t *mutex)
+_pthread_mutex_held_np(pthread_mutex_t *ptm)
{
- return mutex->ptm_owner == pthread__self();
+ return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self();
}
pthread_t
-_pthread_mutex_owner_np(pthread_mutex_t *mutex)
+_pthread_mutex_owner_np(pthread_mutex_t *ptm)
{
- return (pthread_t)mutex->ptm_owner;
+ return (pthread_t)MUTEX_OWNER(ptm->ptm_owner);
}
-
-#endif /* !PTHREAD__HAVE_ATOMIC */
diff --git a/lib/libpthread/pthread_mutex2.c b/lib/libpthread/pthread_mutex2.c
deleted file mode 100644
index 50d6da6326d..00000000000
--- a/lib/libpthread/pthread_mutex2.c
+++ /dev/null
@@ -1,623 +0,0 @@
-/* $NetBSD: pthread_mutex2.c,v 1.17 2007/12/24 14:46:29 ad Exp $ */
-
-/*-
- * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * This product includes software developed by the NetBSD
- * Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_mutex2.c,v 1.17 2007/12/24 14:46:29 ad Exp $");
-
-#include <sys/types.h>
-#include <sys/lwpctl.h>
-
-#include <errno.h>
-#include <limits.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-#include "pthread.h"
-#include "pthread_int.h"
-
-#ifdef PTHREAD__HAVE_ATOMIC
-
-/*
- * Note that it's important to use the address of ptm_waiters as
- * the list head in order for the hint arguments to _lwp_park /
- * _lwp_unpark_all to match.
- */
-#define pt_nextwaiter pt_sleep.ptqe_next
-#define ptm_waiters ptm_blocked.ptqh_first
-#define ptm_errorcheck ptm_lock
-
-#define MUTEX_WAITERS_BIT ((uintptr_t)0x01)
-#define MUTEX_RECURSIVE_BIT ((uintptr_t)0x02)
-#define MUTEX_DEFERRED_BIT ((uintptr_t)0x04)
-#define MUTEX_THREAD ((uintptr_t)-16L)
-
-#define MUTEX_HAS_WAITERS(x) ((uintptr_t)(x) & MUTEX_WAITERS_BIT)
-#define MUTEX_RECURSIVE(x) ((uintptr_t)(x) & MUTEX_RECURSIVE_BIT)
-#define MUTEX_OWNER(x) ((uintptr_t)(x) & MUTEX_THREAD)
-#define MUTEX_GET_RECURSE(ptm) ((intptr_t)(ptm)->ptm_private)
-#define MUTEX_SET_RECURSE(ptm, delta) \
- ((ptm)->ptm_private = (void *)((intptr_t)(ptm)->ptm_private + delta))
-
-#if __GNUC_PREREQ__(3, 0)
-#define NOINLINE __attribute ((noinline))
-#else
-#define NOINLINE /* nothing */
-#endif
-
-static void pthread__mutex_wakeup(pthread_t, pthread_mutex_t *);
-static int pthread__mutex_lock_slow(pthread_mutex_t *);
-static int pthread__mutex_unlock_slow(pthread_mutex_t *);
-static void pthread__mutex_pause(void);
-
-int _pthread_mutex_held_np(pthread_mutex_t *);
-pthread_t _pthread_mutex_owner_np(pthread_mutex_t *);
-
-__weak_alias(pthread_mutex_held_np,_pthread_mutex_held_np)
-__weak_alias(pthread_mutex_owner_np,_pthread_mutex_owner_np)
-
-__strong_alias(__libc_mutex_init,pthread_mutex_init)
-__strong_alias(__libc_mutex_lock,pthread_mutex_lock)
-__strong_alias(__libc_mutex_trylock,pthread_mutex_trylock)
-__strong_alias(__libc_mutex_unlock,pthread_mutex_unlock)
-__strong_alias(__libc_mutex_destroy,pthread_mutex_destroy)
-
-__strong_alias(__libc_mutexattr_init,pthread_mutexattr_init)
-__strong_alias(__libc_mutexattr_destroy,pthread_mutexattr_destroy)
-__strong_alias(__libc_mutexattr_settype,pthread_mutexattr_settype)
-
-__strong_alias(__libc_thr_once,pthread_once)
-
-static inline int
-mutex_cas(volatile void *ptr, void **old, void *new)
-{
- void *oldv;
-
- oldv = *old;
- *old = pthread__atomic_cas_ptr(ptr, oldv, new);
- return *old == oldv;
-}
-
-static inline int
-mutex_cas_ni(volatile void *ptr, void **old, void *new)
-{
- void *oldv;
-
- oldv = *old;
- *old = pthread__atomic_cas_ptr_ni(ptr, oldv, new);
- return *old == oldv;
-}
-
-int
-pthread_mutex_init(pthread_mutex_t *ptm, const pthread_mutexattr_t *attr)
-{
- intptr_t type;
-
- if (attr == NULL)
- type = PTHREAD_MUTEX_NORMAL;
- else
- type = (intptr_t)attr->ptma_private;
-
- switch (type) {
- case PTHREAD_MUTEX_ERRORCHECK:
- ptm->ptm_errorcheck = 1;
- ptm->ptm_owner = NULL;
- break;
- case PTHREAD_MUTEX_RECURSIVE:
- ptm->ptm_errorcheck = 0;
- ptm->ptm_owner = (void *)MUTEX_RECURSIVE_BIT;
- break;
- default:
- ptm->ptm_errorcheck = 0;
- ptm->ptm_owner = NULL;
- break;
- }
-
- ptm->ptm_magic = _PT_MUTEX_MAGIC;
- ptm->ptm_waiters = NULL;
- ptm->ptm_private = NULL;
-
- return 0;
-}
-
-
-int
-pthread_mutex_destroy(pthread_mutex_t *ptm)
-{
-
- pthread__error(EINVAL, "Invalid mutex",
- ptm->ptm_magic == _PT_MUTEX_MAGIC);
- pthread__error(EBUSY, "Destroying locked mutex",
- MUTEX_OWNER(ptm->ptm_owner) == 0);
-
- ptm->ptm_magic = _PT_MUTEX_DEAD;
- return 0;
-}
-
-int
-pthread_mutex_lock(pthread_mutex_t *ptm)
-{
- void *owner;
- pthread_t self;
-
- owner = NULL;
- self = pthread__self();
-
- if (__predict_true(mutex_cas(&ptm->ptm_owner, &owner, self)))
- return 0;
- return pthread__mutex_lock_slow(ptm);
-}
-
-/* We want function call overhead. */
-NOINLINE static void
-pthread__mutex_pause(void)
-{
-
- pthread__smt_pause();
-}
-
-/*
- * Spin while the holder is running. 'lwpctl' gives us the true
- * status of the thread. pt_blocking is set by libpthread in order
- * to cut out system call and kernel spinlock overhead on remote CPUs
- * (could represent many thousands of clock cycles). pt_blocking also
- * makes this thread yield if the target is calling sched_yield().
- */
-NOINLINE static void *
-pthread__mutex_spin(pthread_mutex_t *ptm, pthread_t owner)
-{
- pthread_t thread;
-
- for (;; owner = ptm->ptm_owner) {
- thread = (pthread_t)MUTEX_OWNER(owner);
- if (thread == NULL)
- break;
- if (thread->pt_lwpctl->lc_curcpu == LWPCTL_CPU_NONE ||
- thread->pt_blocking)
- break;
- pthread__mutex_pause();
- pthread__mutex_pause();
- pthread__mutex_pause();
- pthread__mutex_pause();
- }
-
- return owner;
-}
-
-NOINLINE static int
-pthread__mutex_lock_slow(pthread_mutex_t *ptm)
-{
- void *waiters, *new, *owner;
- pthread_t self;
-
- pthread__error(EINVAL, "Invalid mutex",
- ptm->ptm_magic == _PT_MUTEX_MAGIC);
-
- owner = ptm->ptm_owner;
- self = pthread__self();
-
- /* Recursive or errorcheck? */
- if (MUTEX_OWNER(owner) == (uintptr_t)self) {
- if (MUTEX_RECURSIVE(owner)) {
- if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
- return EAGAIN;
- MUTEX_SET_RECURSE(ptm, +1);
- return 0;
- }
- if (ptm->ptm_errorcheck)
- return EDEADLK;
- }
-
- for (;; owner = ptm->ptm_owner) {
- /* Spin while the owner is running. */
- owner = pthread__mutex_spin(ptm, owner);
-
- /* If it has become free, try to acquire it again. */
- if (MUTEX_OWNER(owner) == 0) {
- while (MUTEX_OWNER(owner) == 0) {
- new = (void *)
- ((uintptr_t)self | (uintptr_t)owner);
- if (mutex_cas(&ptm->ptm_owner, &owner, new))
- return 0;
- }
- /*
- * We have lost the race to acquire the mutex.
- * The new owner could be running on another
- * CPU, in which case we should spin and avoid
- * the overhead of blocking.
- */
- continue;
- }
-
- /*
- * Nope, still held. Add thread to the list of waiters.
- * Issue a memory barrier to ensure sleeponq/nextwaiter
- * are visible before we enter the waiters list.
- */
- self->pt_sleeponq = 1;
- for (waiters = ptm->ptm_waiters;;) {
- self->pt_nextwaiter = waiters;
- pthread__membar_producer();
- if (mutex_cas(&ptm->ptm_waiters, &waiters, self))
- break;
- }
-
- /*
- * Set the waiters bit and block.
- *
- * Note that the mutex can become unlocked before we set
- * the waiters bit. If that happens it's not safe to sleep
- * as we may never be awoken: we must remove the current
- * thread from the waiters list and try again.
- *
- * Because we are doing this atomically, we can't remove
- * one waiter: we must remove all waiters and awken them,
- * then sleep in _lwp_park() until we have been awoken.
- *
- * Issue a memory barrier to ensure that we are reading
- * the value of ptm_owner/pt_sleeponq after we have entered
- * the waiters list (the CAS itself must be atomic).
- */
- pthread__membar_consumer();
- for (owner = ptm->ptm_owner;;) {
- if (MUTEX_HAS_WAITERS(owner))
- break;
- if (MUTEX_OWNER(owner) == 0) {
- pthread__mutex_wakeup(self, ptm);
- break;
- }
- new = (void *)((uintptr_t)owner | MUTEX_WAITERS_BIT);
- if (mutex_cas(&ptm->ptm_owner, &owner, new)) {
- /*
- * pthread_mutex_unlock() can do a
- * non-interlocked CAS. We cannot
- * know if our attempt to set the
- * waiters bit has succeeded while
- * the holding thread is running.
- * There are many assumptions; see
- * sys/kern/kern_mutex.c for details.
- * In short, we must spin if we see
- * that the holder is running again.
- */
- pthread__membar_full();
- owner = pthread__mutex_spin(ptm, owner);
- }
- }
-
- /*
- * We may have been awoken by the current thread above,
- * or will be awoken by the current holder of the mutex.
- * The key requirement is that we must not proceed until
- * told that we are no longer waiting (via pt_sleeponq
- * being set to zero). Otherwise it is unsafe to re-enter
- * the thread onto the waiters list.
- */
- while (self->pt_sleeponq) {
- self->pt_blocking++;
- (void)_lwp_park(NULL, 0, &ptm->ptm_waiters, NULL);
- self->pt_blocking--;
- }
- }
-}
-
-int
-pthread_mutex_trylock(pthread_mutex_t *ptm)
-{
- pthread_t self;
- void *value;
-
- self = pthread__self();
- value = NULL;
-
- if (mutex_cas(&ptm->ptm_owner, &value, self))
- return 0;
-
- if (MUTEX_OWNER(value) == (uintptr_t)self && MUTEX_RECURSIVE(value)) {
- if (MUTEX_GET_RECURSE(ptm) == INT_MAX)
- return EAGAIN;
- MUTEX_SET_RECURSE(ptm, +1);
- return 0;
- }
-
- return EBUSY;
-}
-
-int
-pthread_mutex_unlock(pthread_mutex_t *ptm)
-{
- void *owner;
- pthread_t self;
-
- self = pthread__self();
- owner = self;
-
- /*
- * Note this may be a non-interlocked CAS. See lock_slow()
- * above and sys/kern/kern_mutex.c for details.
- */
- if (__predict_true(mutex_cas_ni(&ptm->ptm_owner, &owner, NULL)))
- return 0;
- return pthread__mutex_unlock_slow(ptm);
-}
-
-NOINLINE static int
-pthread__mutex_unlock_slow(pthread_mutex_t *ptm)
-{
- pthread_t self, owner, new;
- int weown, error, deferred;
-
- pthread__error(EINVAL, "Invalid mutex",
- ptm->ptm_magic == _PT_MUTEX_MAGIC);
-
- self = pthread__self();
- owner = ptm->ptm_owner;
- weown = (MUTEX_OWNER(owner) == (uintptr_t)self);
- deferred = (int)((uintptr_t)owner & MUTEX_DEFERRED_BIT);
- error = 0;
-
- if (ptm->ptm_errorcheck) {
- if (!weown) {
- error = EPERM;
- new = owner;
- } else {
- new = NULL;
- }
- } else if (MUTEX_RECURSIVE(owner)) {
- if (!weown) {
- error = EPERM;
- new = owner;
- } else if (MUTEX_GET_RECURSE(ptm) != 0) {
- MUTEX_SET_RECURSE(ptm, -1);
- new = owner;
- } else {
- new = (pthread_t)MUTEX_RECURSIVE_BIT;
- }
- } else {
- pthread__error(EPERM,
- "Unlocking unlocked mutex", (owner != NULL));
- pthread__error(EPERM,
- "Unlocking mutex owned by another thread", weown);
- new = NULL;
- }
-
- /*
- * Release the mutex. If there appear to be waiters, then
- * wake them up.
- */
- if (new != owner) {
- owner = pthread__atomic_swap_ptr(&ptm->ptm_owner, new);
- if (MUTEX_HAS_WAITERS(owner) != 0) {
- pthread__mutex_wakeup(self, ptm);
- return 0;
- }
- }
-
- /*
- * There were no waiters, but we may have deferred waking
- * other threads until mutex unlock - we must wake them now.
- */
- if (!deferred)
- return error;
-
- if (self->pt_nwaiters == 1) {
- /*
- * If the calling thread is about to block, defer
- * unparking the target until _lwp_park() is called.
- */
- if (self->pt_willpark && self->pt_unpark == 0) {
- self->pt_unpark = self->pt_waiters[0];
- self->pt_unparkhint = &ptm->ptm_waiters;
- } else {
- (void)_lwp_unpark(self->pt_waiters[0],
- &ptm->ptm_waiters);
- }
- } else {
- (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters,
- &ptm->ptm_waiters);
- }
- self->pt_nwaiters = 0;
-
- return error;
-}
-
-static void
-pthread__mutex_wakeup(pthread_t self, pthread_mutex_t *ptm)
-{
- pthread_t thread, next;
- ssize_t n, rv;
-
- /*
- * Take ownership of the current set of waiters. No
- * need for a memory barrier following this, all loads
- * are dependent upon 'thread'.
- */
- thread = pthread__atomic_swap_ptr(&ptm->ptm_waiters, NULL);
-
- for (;;) {
- /*
- * Pull waiters from the queue and add to our list.
- * Use a memory barrier to ensure that we safely
- * read the value of pt_nextwaiter before 'thread'
- * sees pt_sleeponq being cleared.
- */
- for (n = self->pt_nwaiters, self->pt_nwaiters = 0;
- n < pthread__unpark_max && thread != NULL;
- thread = next) {
- next = thread->pt_nextwaiter;
- self->pt_waiters[n++] = thread->pt_lid;
- pthread__membar_full();
- thread->pt_sleeponq = 0;
- /* No longer safe to touch 'thread' */
- }
-
- switch (n) {
- case 0:
- return;
- case 1:
- /*
- * If the calling thread is about to block,
- * defer unparking the target until _lwp_park()
- * is called.
- */
- if (self->pt_willpark && self->pt_unpark == 0) {
- self->pt_unpark = self->pt_waiters[0];
- self->pt_unparkhint = &ptm->ptm_waiters;
- return;
- }
- rv = (ssize_t)_lwp_unpark(self->pt_waiters[0],
- &ptm->ptm_waiters);
- if (rv != 0 && errno != EALREADY && errno != EINTR &&
- errno != ESRCH) {
- pthread__errorfunc(__FILE__, __LINE__,
- __func__, "_lwp_unpark failed");
- }
- return;
- default:
- rv = _lwp_unpark_all(self->pt_waiters, (size_t)n,
- &ptm->ptm_waiters);
- if (rv != 0 && errno != EINTR) {
- pthread__errorfunc(__FILE__, __LINE__,
- __func__, "_lwp_unpark_all failed");
- }
- break;
- }
- }
-}
-int
-pthread_mutexattr_init(pthread_mutexattr_t *attr)
-{
-
- attr->ptma_magic = _PT_MUTEXATTR_MAGIC;
- attr->ptma_private = (void *)PTHREAD_MUTEX_DEFAULT;
- return 0;
-}
-
-int
-pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
-{
-
- pthread__error(EINVAL, "Invalid mutex attribute",
- attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
-
- return 0;
-}
-
-
-int
-pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
-{
-
- pthread__error(EINVAL, "Invalid mutex attribute",
- attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
-
- *typep = (int)(intptr_t)attr->ptma_private;
- return 0;
-}
-
-
-int
-pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
-{
-
- pthread__error(EINVAL, "Invalid mutex attribute",
- attr->ptma_magic == _PT_MUTEXATTR_MAGIC);
-
- switch (type) {
- case PTHREAD_MUTEX_NORMAL:
- case PTHREAD_MUTEX_ERRORCHECK:
- case PTHREAD_MUTEX_RECURSIVE:
- attr->ptma_private = (void *)(intptr_t)type;
- return 0;
- default:
- return EINVAL;
- }
-}
-
-
-static void
-once_cleanup(void *closure)
-{
-
- pthread_mutex_unlock((pthread_mutex_t *)closure);
-}
-
-
-int
-pthread_once(pthread_once_t *once_control, void (*routine)(void))
-{
-
- if (once_control->pto_done == 0) {
- pthread_mutex_lock(&once_control->pto_mutex);
- pthread_cleanup_push(&once_cleanup, &once_control->pto_mutex);
- if (once_control->pto_done == 0) {
- routine();
- once_control->pto_done = 1;
- }
- pthread_cleanup_pop(1);
- }
-
- return 0;
-}
-
-int
-pthread__mutex_deferwake(pthread_t thread, pthread_mutex_t *ptm)
-{
-
- if (MUTEX_OWNER(ptm->ptm_owner) != (uintptr_t)thread)
- return 0;
- pthread__atomic_or_ulong((volatile unsigned long *)
- (uintptr_t)&ptm->ptm_owner,
- (unsigned long)MUTEX_DEFERRED_BIT);
- return 1;
-}
-
-int
-_pthread_mutex_held_np(pthread_mutex_t *ptm)
-{
-
- return MUTEX_OWNER(ptm->ptm_owner) == (uintptr_t)pthread__self();
-}
-
-pthread_t
-_pthread_mutex_owner_np(pthread_mutex_t *ptm)
-{
-
- return (pthread_t)MUTEX_OWNER(ptm->ptm_owner);
-}
-
-#endif /* PTHREAD__HAVE_ATOMIC */
diff --git a/lib/libpthread/pthread_rwlock.c b/lib/libpthread/pthread_rwlock.c
index 123b1537739..2d6f2a6739b 100644
--- a/lib/libpthread/pthread_rwlock.c
+++ b/lib/libpthread/pthread_rwlock.c
@@ -1,11 +1,11 @@
-/* $NetBSD: pthread_rwlock.c,v 1.26 2008/01/31 11:50:40 ad Exp $ */
+/* $NetBSD: pthread_rwlock.c,v 1.27 2008/02/10 18:50:55 ad Exp $ */
/*-
- * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
- * by Nathan J. Williams and Andrew Doran.
+ * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -37,23 +37,32 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_rwlock.c,v 1.26 2008/01/31 11:50:40 ad Exp $");
+__RCSID("$NetBSD: pthread_rwlock.c,v 1.27 2008/02/10 18:50:55 ad Exp $");
#include <errno.h>
+#include <stddef.h>
#include "pthread.h"
#include "pthread_int.h"
-#ifndef PTHREAD__HAVE_ATOMIC
+#define _RW_LOCKED 0
+#define _RW_WANT_WRITE 1
+#define _RW_WANT_READ 2
-__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np)
-__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np)
-__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np)
+static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *);
+static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *);
+static void pthread__rwlock_early(void *);
int _pthread_rwlock_held_np(pthread_rwlock_t *);
int _pthread_rwlock_rdheld_np(pthread_rwlock_t *);
int _pthread_rwlock_wrheld_np(pthread_rwlock_t *);
+#ifndef lint
+__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np);
+__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np);
+__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np);
+#endif
+
__strong_alias(__libc_rwlock_init,pthread_rwlock_init)
__strong_alias(__libc_rwlock_rdlock,pthread_rwlock_rdlock)
__strong_alias(__libc_rwlock_wrlock,pthread_rwlock_wrlock)
@@ -62,406 +71,531 @@ __strong_alias(__libc_rwlock_trywrlock,pthread_rwlock_trywrlock)
__strong_alias(__libc_rwlock_unlock,pthread_rwlock_unlock)
__strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy)
+static inline uintptr_t
+rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n)
+{
+
+ return (uintptr_t)atomic_cas_ptr(&ptr->ptr_owner, (void *)o,
+ (void *)n);
+}
+
int
-pthread_rwlock_init(pthread_rwlock_t *rwlock,
+pthread_rwlock_init(pthread_rwlock_t *ptr,
const pthread_rwlockattr_t *attr)
{
-#ifdef ERRORCHECK
- if ((rwlock == NULL) ||
- (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC)))
+
+ if (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
return EINVAL;
-#endif
- rwlock->ptr_magic = _PT_RWLOCK_MAGIC;
- pthread_lockinit(&rwlock->ptr_interlock);
- PTQ_INIT(&rwlock->ptr_rblocked);
- PTQ_INIT(&rwlock->ptr_wblocked);
- rwlock->ptr_nreaders = 0;
- rwlock->ptr_writer = NULL;
+ ptr->ptr_magic = _PT_RWLOCK_MAGIC;
+ pthread_lockinit(&ptr->ptr_interlock);
+ PTQ_INIT(&ptr->ptr_rblocked);
+ PTQ_INIT(&ptr->ptr_wblocked);
+ ptr->ptr_nreaders = 0;
+ ptr->ptr_owner = NULL;
return 0;
}
int
-pthread_rwlock_destroy(pthread_rwlock_t *rwlock)
+pthread_rwlock_destroy(pthread_rwlock_t *ptr)
{
-#ifdef ERRORCHECK
- if ((rwlock == NULL) ||
- (rwlock->ptr_magic != _PT_RWLOCK_MAGIC) ||
- (!PTQ_EMPTY(&rwlock->ptr_rblocked)) ||
- (!PTQ_EMPTY(&rwlock->ptr_wblocked)) ||
- (rwlock->ptr_nreaders != 0) ||
- (rwlock->ptr_writer != NULL))
+
+ if ((ptr->ptr_magic != _PT_RWLOCK_MAGIC) ||
+ (!PTQ_EMPTY(&ptr->ptr_rblocked)) ||
+ (!PTQ_EMPTY(&ptr->ptr_wblocked)) ||
+ (ptr->ptr_nreaders != 0) ||
+ (ptr->ptr_owner != NULL))
return EINVAL;
-#endif
- rwlock->ptr_magic = _PT_RWLOCK_DEAD;
+ ptr->ptr_magic = _PT_RWLOCK_DEAD;
return 0;
}
-
-int
-pthread_rwlock_rdlock(pthread_rwlock_t *rwlock)
+static int
+pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
+ uintptr_t owner, next;
pthread_t self;
+ int error;
+
+ self = pthread__self();
+
#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
+ if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
- self = pthread__self();
-
- pthread__spinlock(self, &rwlock->ptr_interlock);
-#ifdef ERRORCHECK
- if (rwlock->ptr_writer == self) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EDEADLK;
- }
+
+ for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
+ /*
+ * Read the lock owner field. If the need-to-wait
+ * indicator is clear, then try to acquire the lock.
+ */
+ if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) == 0) {
+ next = rw_cas(ptr, owner, owner + RW_READ_INCR);
+ if (owner == next) {
+ /* Got it! */
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
#endif
- /*
- * Don't get a readlock if there is a writer or if there are waiting
- * writers; i.e. prefer writers to readers. This strategy is dictated
- * by SUSv3.
- */
- while ((rwlock->ptr_writer != NULL) ||
- (!PTQ_EMPTY(&rwlock->ptr_wblocked))) {
- PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep);
+ return 0;
+ }
+
+ /*
+ * Didn't get it -- spin around again (we'll
+ * probably sleep on the next iteration).
+ */
+ continue;
+ }
+
+ if ((owner & RW_THREAD) == (uintptr_t)self)
+ return EDEADLK;
+
+ /*
+ * Grab the interlock. Once we have that, we
+ * can adjust the waiter bits and sleep queue.
+ */
+ pthread__spinlock(self, &ptr->ptr_interlock);
+
+ /*
+ * Mark the rwlock as having waiters. If the set fails,
+ * then we may not need to sleep and should spin again.
+ */
+ next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS);
+ if (owner != next) {
+ pthread__spinunlock(self, &ptr->ptr_interlock);
+ continue;
+ }
+
+ /* The waiters bit is set - it's safe to sleep. */
+ PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep);
+ ptr->ptr_nreaders++;
+ self->pt_rwlocked = _RW_WANT_READ;
self->pt_sleeponq = 1;
- self->pt_sleepobj = &rwlock->ptr_rblocked;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- (void)pthread__park(self, &rwlock->ptr_interlock,
- &rwlock->ptr_rblocked, NULL, 0, &rwlock->ptr_rblocked);
- pthread__spinlock(self, &rwlock->ptr_interlock);
- }
-
- rwlock->ptr_nreaders++;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
+ self->pt_sleepobj = &ptr->ptr_rblocked;
+ self->pt_early = pthread__rwlock_early;
+ pthread__spinunlock(self, &ptr->ptr_interlock);
- return 0;
+ error = pthread__park(self, &ptr->ptr_interlock,
+ &ptr->ptr_rblocked, ts, 0, &ptr->ptr_rblocked);
+
+ /* Did we get the lock? */
+ if (self->pt_rwlocked == _RW_LOCKED) {
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
+ }
+ if (error != 0)
+ return error;
+
+ pthread__errorfunc(__FILE__, __LINE__, __func__,
+ "direct handoff failure");
+ }
}
int
-pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
+pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr)
{
- pthread_t self;
+ uintptr_t owner, next;
#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
+ if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
-
- self = pthread__self();
- pthread__spinlock(self, &rwlock->ptr_interlock);
+
/*
* Don't get a readlock if there is a writer or if there are waiting
* writers; i.e. prefer writers to readers. This strategy is dictated
* by SUSv3.
*/
- if ((rwlock->ptr_writer != NULL) ||
- (!PTQ_EMPTY(&rwlock->ptr_wblocked))) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EBUSY;
+ for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
+ if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0)
+ return EBUSY;
+ next = rw_cas(ptr, owner, owner + RW_READ_INCR);
+ if (owner == next) {
+ /* Got it! */
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
+ }
}
-
- rwlock->ptr_nreaders++;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
-
- return 0;
}
-
-int
-pthread_rwlock_wrlock(pthread_rwlock_t *rwlock)
+static int
+pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts)
{
+ uintptr_t owner, next;
pthread_t self;
- extern int pthread__started;
+ int error;
+
+ self = pthread__self();
#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
+ if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
- self = pthread__self();
-
- pthread__spinlock(self, &rwlock->ptr_interlock);
-#ifdef ERRORCHECK
- if (rwlock->ptr_writer == self) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EDEADLK;
- }
+
+ for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
+ /*
+ * Read the lock owner field. If the need-to-wait
+ * indicator is clear, then try to acquire the lock.
+ */
+ if ((owner & RW_THREAD) == 0) {
+ next = rw_cas(ptr, owner,
+ (uintptr_t)self | RW_WRITE_LOCKED);
+ if (owner == next) {
+ /* Got it! */
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
#endif
- /*
- * Prefer writers to readers here; permit writers even if there are
- * waiting readers.
- */
- while ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL)) {
-#ifdef ERRORCHECK
- if (pthread__started == 0) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
+ return 0;
+ }
+
+ /*
+ * Didn't get it -- spin around again (we'll
+ * probably sleep on the next iteration).
+ */
+ continue;
+ }
+
+ if ((owner & RW_THREAD) == (uintptr_t)self)
return EDEADLK;
+
+ /*
+ * Grab the interlock. Once we have that, we
+ * can adjust the waiter bits and sleep queue.
+ */
+ pthread__spinlock(self, &ptr->ptr_interlock);
+
+ /*
+ * Mark the rwlock as having waiters. If the set fails,
+ * then we may not need to sleep and should spin again.
+ */
+ next = rw_cas(ptr, owner,
+ owner | RW_HAS_WAITERS | RW_WRITE_WANTED);
+ if (owner != next) {
+ pthread__spinunlock(self, &ptr->ptr_interlock);
+ continue;
}
-#endif
- PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep);
+
+ /* The waiters bit is set - it's safe to sleep. */
+ PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);
+ self->pt_rwlocked = _RW_WANT_WRITE;
self->pt_sleeponq = 1;
- self->pt_sleepobj = &rwlock->ptr_wblocked;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- (void)pthread__park(self, &rwlock->ptr_interlock,
- &rwlock->ptr_wblocked, NULL, 0, &rwlock->ptr_wblocked);
- pthread__spinlock(self, &rwlock->ptr_interlock);
- }
+ self->pt_sleepobj = &ptr->ptr_wblocked;
+ self->pt_early = pthread__rwlock_early;
+ pthread__spinunlock(self, &ptr->ptr_interlock);
- rwlock->ptr_writer = self;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
+ error = pthread__park(self, &ptr->ptr_interlock,
+ &ptr->ptr_wblocked, ts, 0, &ptr->ptr_wblocked);
- return 0;
+ /* Did we get the lock? */
+ if (self->pt_rwlocked == _RW_LOCKED) {
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
+ }
+ if (error != 0)
+ return error;
+
+ pthread__errorfunc(__FILE__, __LINE__, __func__,
+ "direct handoff failure");
+ }
}
int
-pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
+pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)
{
+ uintptr_t owner, next;
pthread_t self;
+
#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
+ if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
return EINVAL;
#endif
+
self = pthread__self();
-
- pthread__spinlock(self, &rwlock->ptr_interlock);
- /*
- * Prefer writers to readers here; permit writers even if there are
- * waiting readers.
- */
- if ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL)) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EBUSY;
+
+ for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
+ if (owner != 0)
+ return EBUSY;
+ next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED);
+ if (owner == next) {
+ /* Got it! */
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_enter();
+#endif
+ return 0;
+ }
}
+}
- rwlock->ptr_writer = self;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
+int
+pthread_rwlock_rdlock(pthread_rwlock_t *ptr)
+{
- return 0;
+ return pthread__rwlock_rdlock(ptr, NULL);
}
-
int
-pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlock,
- const struct timespec *abs_timeout)
+pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr,
+ const struct timespec *abs_timeout)
{
- pthread_t self;
- int retval;
-#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
- return EINVAL;
if (abs_timeout == NULL)
return EINVAL;
-#endif
if ((abs_timeout->tv_nsec >= 1000000000) ||
(abs_timeout->tv_nsec < 0) ||
(abs_timeout->tv_sec < 0))
return EINVAL;
- self = pthread__self();
- pthread__spinlock(self, &rwlock->ptr_interlock);
-#ifdef ERRORCHECK
- if (rwlock->ptr_writer == self) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EDEADLK;
- }
-#endif
- /*
- * Don't get a readlock if there is a writer or if there are waiting
- * writers; i.e. prefer writers to readers. This strategy is dictated
- * by SUSv3.
- */
- retval = 0;
- while ((retval == 0) && ((rwlock->ptr_writer != NULL) ||
- (!PTQ_EMPTY(&rwlock->ptr_wblocked)))) {
- PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep);
- self->pt_sleeponq = 1;
- self->pt_sleepobj = &rwlock->ptr_rblocked;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- retval = pthread__park(self, &rwlock->ptr_interlock,
- &rwlock->ptr_rblocked, abs_timeout, 0,
- &rwlock->ptr_rblocked);
- pthread__spinlock(self, &rwlock->ptr_interlock);
- }
+ return pthread__rwlock_rdlock(ptr, abs_timeout);
+}
- /* One last chance to get the lock, in case it was released between
- the alarm firing and when this thread got rescheduled, or in case
- a signal handler kept it busy */
- if ((rwlock->ptr_writer == NULL) &&
- (PTQ_EMPTY(&rwlock->ptr_wblocked))) {
- rwlock->ptr_nreaders++;
- retval = 0;
- }
- pthread__spinunlock(self, &rwlock->ptr_interlock);
+int
+pthread_rwlock_wrlock(pthread_rwlock_t *ptr)
+{
- return retval;
+ return pthread__rwlock_wrlock(ptr, NULL);
}
-
int
-pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlock,
- const struct timespec *abs_timeout)
+pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr,
+ const struct timespec *abs_timeout)
{
- pthread_t self;
- int retval;
- extern int pthread__started;
-#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
- return EINVAL;
if (abs_timeout == NULL)
return EINVAL;
-#endif
if ((abs_timeout->tv_nsec >= 1000000000) ||
(abs_timeout->tv_nsec < 0) ||
(abs_timeout->tv_sec < 0))
return EINVAL;
- self = pthread__self();
- pthread__spinlock(self, &rwlock->ptr_interlock);
-#ifdef ERRORCHECK
- if (rwlock->ptr_writer == self) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EDEADLK;
- }
-#endif
- /*
- * Prefer writers to readers here; permit writers even if there are
- * waiting readers.
- */
- retval = 0;
- while (retval == 0 &&
- ((rwlock->ptr_nreaders > 0) || (rwlock->ptr_writer != NULL))) {
-#ifdef ERRORCHECK
- if (pthread__started == 0) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EDEADLK;
- }
-#endif
- PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep);
- self->pt_sleeponq = 1;
- self->pt_sleepobj = &rwlock->ptr_wblocked;
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- retval = pthread__park(self, &rwlock->ptr_interlock,
- &rwlock->ptr_wblocked, abs_timeout, 0,
- &rwlock->ptr_wblocked);
- pthread__spinlock(self, &rwlock->ptr_interlock);
- }
-
- if ((rwlock->ptr_nreaders == 0) && (rwlock->ptr_writer == NULL)) {
- rwlock->ptr_writer = self;
- retval = 0;
- }
- pthread__spinunlock(self, &rwlock->ptr_interlock);
-
- return retval;
+ return pthread__rwlock_wrlock(ptr, abs_timeout);
}
int
-pthread_rwlock_unlock(pthread_rwlock_t *rwlock)
+pthread_rwlock_unlock(pthread_rwlock_t *ptr)
{
- pthread_t self, writer;
+ uintptr_t owner, decr, new, next;
+ pthread_t self, thread;
+
#ifdef ERRORCHECK
- if ((rwlock == NULL) || (rwlock->ptr_magic != _PT_RWLOCK_MAGIC))
+ if ((ptr == NULL) || (ptr->ptr_magic != _PT_RWLOCK_MAGIC))
return EINVAL;
#endif
- writer = NULL;
+
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_exit();
+#endif
+
+ /*
+ * Since we used an add operation to set the required lock
+ * bits, we can use a subtract to clear them, which makes
+ * the read-release and write-release path similar.
+ */
self = pthread__self();
-
- pthread__spinlock(self, &rwlock->ptr_interlock);
- if (rwlock->ptr_writer != NULL) {
- /* Releasing a write lock. */
-#ifdef ERRORCHECK
- if (rwlock->ptr_writer != self) {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
+ owner = (uintptr_t)ptr->ptr_owner;
+ if ((owner & RW_WRITE_LOCKED) != 0) {
+ decr = (uintptr_t)self | RW_WRITE_LOCKED;
+ if ((owner & RW_THREAD) != (uintptr_t)self) {
return EPERM;
}
-#endif
- rwlock->ptr_writer = NULL;
- writer = PTQ_FIRST(&rwlock->ptr_wblocked);
- if (writer != NULL) {
- PTQ_REMOVE(&rwlock->ptr_wblocked, writer, pt_sleep);
+ } else {
+ decr = RW_READ_INCR;
+ if (owner == 0) {
+ return EPERM;
}
- } else
-#ifdef ERRORCHECK
- if (rwlock->ptr_nreaders > 0)
-#endif
- {
- /* Releasing a read lock. */
- rwlock->ptr_nreaders--;
- if (rwlock->ptr_nreaders == 0) {
- writer = PTQ_FIRST(&rwlock->ptr_wblocked);
- if (writer != NULL)
- PTQ_REMOVE(&rwlock->ptr_wblocked, writer,
- pt_sleep);
+ }
+
+ for (;; owner = next) {
+ /*
+ * Compute what we expect the new value of the lock to be.
+ * Only proceed to do direct handoff if there are waiters,
+ * and if the lock would become unowned.
+ */
+ new = (owner - decr);
+ if ((new & (RW_THREAD | RW_HAS_WAITERS)) != RW_HAS_WAITERS) {
+ next = rw_cas(ptr, owner, new);
+ if (owner == next) {
+ /* Released! */
+ return 0;
+ }
+ continue;
}
-#ifdef ERRORCHECK
- } else {
- pthread__spinunlock(self, &rwlock->ptr_interlock);
- return EPERM;
-#endif
+
+ /*
+ * Grab the interlock. Once we have that, we can adjust
+ * the waiter bits. We must check to see if there are
+ * still waiters before proceeding.
+ */
+ pthread__spinlock(self, &ptr->ptr_interlock);
+ owner = (uintptr_t)ptr->ptr_owner;
+ if ((owner & RW_HAS_WAITERS) == 0) {
+ pthread__spinunlock(self, &ptr->ptr_interlock);
+ next = owner;
+ continue;
+ }
+
+ /*
+ * Give the lock away. SUSv3 dictates that we must give
+ * preference to writers.
+ */
+ if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) {
+ new = (uintptr_t)thread | RW_WRITE_LOCKED;
+
+ if (PTQ_NEXT(thread, pt_sleep) != NULL)
+ new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
+ else if (ptr->ptr_nreaders != 0)
+ new |= RW_HAS_WAITERS;
+
+ /*
+ * Set in the new value. The lock becomes owned
+ * by the writer that we are about to wake.
+ */
+ (void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new);
+
+ /* Wake the writer. */
+ PTQ_REMOVE(&ptr->ptr_wblocked, thread, pt_sleep);
+ thread->pt_rwlocked = _RW_LOCKED;
+ pthread__unpark(self, &ptr->ptr_interlock,
+ &ptr->ptr_wblocked, thread);
+ } else {
+ new = 0;
+ PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) {
+ /*
+ * May have already been handed the lock,
+ * since pthread__unpark_all() can release
+ * our interlock before awakening all
+ * threads.
+ */
+ if (thread->pt_sleepobj == NULL)
+ continue;
+ new += RW_READ_INCR;
+ thread->pt_rwlocked = _RW_LOCKED;
+ }
+
+ /*
+ * Set in the new value. The lock becomes owned
+ * by the readers that we are about to wake.
+ */
+ (void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new);
+
+ /* Wake up all sleeping readers. */
+ ptr->ptr_nreaders = 0;
+ pthread__unpark_all(self, &ptr->ptr_interlock,
+ &ptr->ptr_rblocked);
+ }
+
+ return 0;
}
+}
- if (writer != NULL)
- pthread__unpark(self, &rwlock->ptr_interlock,
- &rwlock->ptr_wblocked, writer);
+/*
+ * Called when a timedlock awakens early to adjust the waiter bits.
+ * The rwlock's interlock is held on entry, and the caller has been
+ * removed from the waiters lists.
+ */
+static void
+pthread__rwlock_early(void *obj)
+{
+ uintptr_t owner, set, new, next;
+ pthread_rwlock_t *ptr;
+ pthread_t self;
+ u_int off;
+
+ self = pthread__self();
+
+ switch (self->pt_rwlocked) {
+ case _RW_WANT_READ:
+ off = offsetof(pthread_rwlock_t, ptr_rblocked);
+ break;
+ case _RW_WANT_WRITE:
+ off = offsetof(pthread_rwlock_t, ptr_wblocked);
+ break;
+ default:
+ pthread__errorfunc(__FILE__, __LINE__, __func__,
+ "bad value of pt_rwlocked");
+ off = 0;
+ /* NOTREACHED */
+ break;
+ }
+
+ /* LINTED mind your own business */
+ ptr = (pthread_rwlock_t *)((uint8_t *)obj - off);
+ owner = (uintptr_t)ptr->ptr_owner;
+
+ if ((owner & RW_THREAD) == 0) {
+ pthread__errorfunc(__FILE__, __LINE__, __func__,
+ "lock not held");
+ }
+
+ if (!PTQ_EMPTY(&ptr->ptr_wblocked))
+ set = RW_HAS_WAITERS | RW_WRITE_WANTED;
+ else if (ptr->ptr_nreaders != 0)
+ set = RW_HAS_WAITERS;
else
- pthread__unpark_all(self, &rwlock->ptr_interlock,
- &rwlock->ptr_rblocked);
+ set = 0;
- return 0;
+ for (;; owner = next) {
+ new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set;
+ next = rw_cas(ptr, owner, new);
+ if (owner == next)
+ break;
+ }
}
-
int
-pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
+_pthread_rwlock_held_np(pthread_rwlock_t *ptr)
{
-#ifdef ERRORCHECK
- if (attr == NULL)
- return EINVAL;
-#endif
- attr->ptra_magic = _PT_RWLOCKATTR_MAGIC;
+ uintptr_t owner = (uintptr_t)ptr->ptr_owner;
- return 0;
+ return (owner & RW_THREAD) != 0;
}
-
int
-pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
+_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)
{
-#ifdef ERRORCHECK
- if ((attr == NULL) ||
- (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
- return EINVAL;
-#endif
- attr->ptra_magic = _PT_RWLOCKATTR_DEAD;
+ uintptr_t owner = (uintptr_t)ptr->ptr_owner;
- return 0;
+ return (owner & RW_THREAD) != 0 && (owner & RW_WRITE_LOCKED) == 0;
}
int
-_pthread_rwlock_held_np(pthread_rwlock_t *ptr)
+_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)
{
+ uintptr_t owner = (uintptr_t)ptr->ptr_owner;
- return ptr->ptr_writer != NULL || ptr->ptr_nreaders != 0;
+ return (owner & (RW_THREAD | RW_WRITE_LOCKED)) ==
+ ((uintptr_t)pthread__self() | RW_WRITE_LOCKED);
}
int
-_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)
+pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
{
- return ptr->ptr_nreaders != 0;
+ if (attr == NULL)
+ return EINVAL;
+ attr->ptra_magic = _PT_RWLOCKATTR_MAGIC;
+
+ return 0;
}
+
int
-_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)
+pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
{
- return ptr->ptr_writer == pthread__self();
-}
+ if ((attr == NULL) ||
+ (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
+ return EINVAL;
+ attr->ptra_magic = _PT_RWLOCKATTR_DEAD;
-#endif /* !PTHREAD__HAVE_ATOMIC */
+ return 0;
+}
diff --git a/lib/libpthread/pthread_rwlock2.c b/lib/libpthread/pthread_rwlock2.c
deleted file mode 100644
index cd5953ed77b..00000000000
--- a/lib/libpthread/pthread_rwlock2.c
+++ /dev/null
@@ -1,584 +0,0 @@
-/* $NetBSD: pthread_rwlock2.c,v 1.9 2008/01/31 11:50:40 ad Exp $ */
-
-/*-
- * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by Nathan J. Williams, by Jason R. Thorpe, and by Andrew Doran.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * This product includes software developed by the NetBSD
- * Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_rwlock2.c,v 1.9 2008/01/31 11:50:40 ad Exp $");
-
-#include <errno.h>
-#include <stddef.h>
-
-#include "pthread.h"
-#include "pthread_int.h"
-
-#ifdef PTHREAD__HAVE_ATOMIC
-
-#define _RW_LOCKED 0
-#define _RW_WANT_WRITE 1
-#define _RW_WANT_READ 2
-
-static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *);
-static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *);
-static void pthread__rwlock_early(void *);
-
-int _pthread_rwlock_held_np(pthread_rwlock_t *);
-int _pthread_rwlock_rdheld_np(pthread_rwlock_t *);
-int _pthread_rwlock_wrheld_np(pthread_rwlock_t *);
-
-#ifndef lint
-__weak_alias(pthread_rwlock_held_np,_pthread_rwlock_held_np);
-__weak_alias(pthread_rwlock_rdheld_np,_pthread_rwlock_rdheld_np);
-__weak_alias(pthread_rwlock_wrheld_np,_pthread_rwlock_wrheld_np);
-#endif
-
-__strong_alias(__libc_rwlock_init,pthread_rwlock_init)
-__strong_alias(__libc_rwlock_rdlock,pthread_rwlock_rdlock)
-__strong_alias(__libc_rwlock_wrlock,pthread_rwlock_wrlock)
-__strong_alias(__libc_rwlock_tryrdlock,pthread_rwlock_tryrdlock)
-__strong_alias(__libc_rwlock_trywrlock,pthread_rwlock_trywrlock)
-__strong_alias(__libc_rwlock_unlock,pthread_rwlock_unlock)
-__strong_alias(__libc_rwlock_destroy,pthread_rwlock_destroy)
-
-static inline uintptr_t
-rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n)
-{
-
- return (uintptr_t)pthread__atomic_cas_ptr(&ptr->ptr_owner, (void *)o,
- (void *)n);
-}
-
-int
-pthread_rwlock_init(pthread_rwlock_t *ptr,
- const pthread_rwlockattr_t *attr)
-{
-
- if (attr && (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
- return EINVAL;
- ptr->ptr_magic = _PT_RWLOCK_MAGIC;
- pthread_lockinit(&ptr->ptr_interlock);
- PTQ_INIT(&ptr->ptr_rblocked);
- PTQ_INIT(&ptr->ptr_wblocked);
- ptr->ptr_nreaders = 0;
- ptr->ptr_owner = NULL;
-
- return 0;
-}
-
-
-int
-pthread_rwlock_destroy(pthread_rwlock_t *ptr)
-{
-
- if ((ptr->ptr_magic != _PT_RWLOCK_MAGIC) ||
- (!PTQ_EMPTY(&ptr->ptr_rblocked)) ||
- (!PTQ_EMPTY(&ptr->ptr_wblocked)) ||
- (ptr->ptr_nreaders != 0) ||
- (ptr->ptr_owner != NULL))
- return EINVAL;
- ptr->ptr_magic = _PT_RWLOCK_DEAD;
-
- return 0;
-}
-
-static int
-pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts)
-{
- uintptr_t owner, next;
- pthread_t self;
- int error;
-
- self = pthread__self();
-
-#ifdef ERRORCHECK
- if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
- return EINVAL;
-#endif
-
- for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
- /*
- * Read the lock owner field. If the need-to-wait
- * indicator is clear, then try to acquire the lock.
- */
- if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) == 0) {
- next = rw_cas(ptr, owner, owner + RW_READ_INCR);
- if (owner == next) {
- /* Got it! */
- return 0;
- }
-
- /*
- * Didn't get it -- spin around again (we'll
- * probably sleep on the next iteration).
- */
- continue;
- }
-
- if ((owner & RW_THREAD) == (uintptr_t)self)
- return EDEADLK;
-
- /*
- * Grab the interlock. Once we have that, we
- * can adjust the waiter bits and sleep queue.
- */
- pthread__spinlock(self, &ptr->ptr_interlock);
-
- /*
- * Mark the rwlock as having waiters. If the set fails,
- * then we may not need to sleep and should spin again.
- */
- next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS);
- if (owner != next) {
- pthread__spinunlock(self, &ptr->ptr_interlock);
- continue;
- }
-
- /* The waiters bit is set - it's safe to sleep. */
- PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep);
- ptr->ptr_nreaders++;
- self->pt_rwlocked = _RW_WANT_READ;
- self->pt_sleeponq = 1;
- self->pt_sleepobj = &ptr->ptr_rblocked;
- self->pt_early = pthread__rwlock_early;
- pthread__spinunlock(self, &ptr->ptr_interlock);
-
- error = pthread__park(self, &ptr->ptr_interlock,
- &ptr->ptr_rblocked, ts, 0, &ptr->ptr_rblocked);
-
- /* Did we get the lock? */
- if (self->pt_rwlocked == _RW_LOCKED)
- return 0;
- if (error != 0)
- return error;
-
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "direct handoff failure");
- }
-}
-
-
-int
-pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr)
-{
- uintptr_t owner, next;
-
-#ifdef ERRORCHECK
- if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
- return EINVAL;
-#endif
-
- /*
- * Don't get a readlock if there is a writer or if there are waiting
- * writers; i.e. prefer writers to readers. This strategy is dictated
- * by SUSv3.
- */
- for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
- if ((owner & (RW_WRITE_LOCKED | RW_WRITE_WANTED)) != 0)
- return EBUSY;
- next = rw_cas(ptr, owner, owner + RW_READ_INCR);
- if (owner == next) {
- /* Got it! */
- return 0;
- }
- }
-}
-
-static int
-pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts)
-{
- uintptr_t owner, next;
- pthread_t self;
- int error;
-
- self = pthread__self();
-
-#ifdef ERRORCHECK
- if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
- return EINVAL;
-#endif
-
- for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
- /*
- * Read the lock owner field. If the need-to-wait
- * indicator is clear, then try to acquire the lock.
- */
- if ((owner & RW_THREAD) == 0) {
- next = rw_cas(ptr, owner,
- (uintptr_t)self | RW_WRITE_LOCKED);
- if (owner == next) {
- /* Got it! */
- return 0;
- }
-
- /*
- * Didn't get it -- spin around again (we'll
- * probably sleep on the next iteration).
- */
- continue;
- }
-
- if ((owner & RW_THREAD) == (uintptr_t)self)
- return EDEADLK;
-
- /*
- * Grab the interlock. Once we have that, we
- * can adjust the waiter bits and sleep queue.
- */
- pthread__spinlock(self, &ptr->ptr_interlock);
-
- /*
- * Mark the rwlock as having waiters. If the set fails,
- * then we may not need to sleep and should spin again.
- */
- next = rw_cas(ptr, owner,
- owner | RW_HAS_WAITERS | RW_WRITE_WANTED);
- if (owner != next) {
- pthread__spinunlock(self, &ptr->ptr_interlock);
- continue;
- }
-
- /* The waiters bit is set - it's safe to sleep. */
- PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);
- self->pt_rwlocked = _RW_WANT_WRITE;
- self->pt_sleeponq = 1;
- self->pt_sleepobj = &ptr->ptr_wblocked;
- self->pt_early = pthread__rwlock_early;
- pthread__spinunlock(self, &ptr->ptr_interlock);
-
- error = pthread__park(self, &ptr->ptr_interlock,
- &ptr->ptr_wblocked, ts, 0, &ptr->ptr_wblocked);
-
- /* Did we get the lock? */
- if (self->pt_rwlocked == _RW_LOCKED)
- return 0;
- if (error != 0)
- return error;
-
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "direct handoff failure");
- }
-}
-
-
-int
-pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)
-{
- uintptr_t owner, next;
- pthread_t self;
-
-#ifdef ERRORCHECK
- if (ptr->ptr_magic != _PT_RWLOCK_MAGIC)
- return EINVAL;
-#endif
-
- self = pthread__self();
-
- for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {
- if (owner != 0)
- return EBUSY;
- next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED);
- if (owner == next) {
- /* Got it! */
- return 0;
- }
- }
-}
-
-int
-pthread_rwlock_rdlock(pthread_rwlock_t *ptr)
-{
-
- return pthread__rwlock_rdlock(ptr, NULL);
-}
-
-int
-pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr,
- const struct timespec *abs_timeout)
-{
-
- if (abs_timeout == NULL)
- return EINVAL;
- if ((abs_timeout->tv_nsec >= 1000000000) ||
- (abs_timeout->tv_nsec < 0) ||
- (abs_timeout->tv_sec < 0))
- return EINVAL;
-
- return pthread__rwlock_rdlock(ptr, abs_timeout);
-}
-
-int
-pthread_rwlock_wrlock(pthread_rwlock_t *ptr)
-{
-
- return pthread__rwlock_wrlock(ptr, NULL);
-}
-
-int
-pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr,
- const struct timespec *abs_timeout)
-{
-
- if (abs_timeout == NULL)
- return EINVAL;
- if ((abs_timeout->tv_nsec >= 1000000000) ||
- (abs_timeout->tv_nsec < 0) ||
- (abs_timeout->tv_sec < 0))
- return EINVAL;
-
- return pthread__rwlock_wrlock(ptr, abs_timeout);
-}
-
-
-int
-pthread_rwlock_unlock(pthread_rwlock_t *ptr)
-{
- uintptr_t owner, decr, new, next;
- pthread_t self, thread;
-
-#ifdef ERRORCHECK
- if ((ptr == NULL) || (ptr->ptr_magic != _PT_RWLOCK_MAGIC))
- return EINVAL;
-#endif
-
- self = pthread__self();
-
- /*
- * Since we used an add operation to set the required lock
- * bits, we can use a subtract to clear them, which makes
- * the read-release and write-release path similar.
- */
- owner = (uintptr_t)ptr->ptr_owner;
- if ((owner & RW_WRITE_LOCKED) != 0) {
- decr = (uintptr_t)self | RW_WRITE_LOCKED;
- if ((owner & RW_THREAD) != (uintptr_t)self) {
- return EPERM;
- }
- } else {
- decr = RW_READ_INCR;
- if (owner == 0) {
- return EPERM;
- }
- }
-
- for (;; owner = next) {
- /*
- * Compute what we expect the new value of the lock to be.
- * Only proceed to do direct handoff if there are waiters,
- * and if the lock would become unowned.
- */
- new = (owner - decr);
- if ((new & (RW_THREAD | RW_HAS_WAITERS)) != RW_HAS_WAITERS) {
- next = rw_cas(ptr, owner, new);
- if (owner == next) {
- /* Released! */
- return 0;
- }
- continue;
- }
-
- /*
- * Grab the interlock. Once we have that, we can adjust
- * the waiter bits. We must check to see if there are
- * still waiters before proceeding.
- */
- pthread__spinlock(self, &ptr->ptr_interlock);
- owner = (uintptr_t)ptr->ptr_owner;
- if ((owner & RW_HAS_WAITERS) == 0) {
- pthread__spinunlock(self, &ptr->ptr_interlock);
- next = owner;
- continue;
- }
-
- /*
- * Give the lock away. SUSv3 dictates that we must give
- * preference to writers.
- */
- if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) {
- new = (uintptr_t)thread | RW_WRITE_LOCKED;
-
- if (PTQ_NEXT(thread, pt_sleep) != NULL)
- new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
- else if (ptr->ptr_nreaders != 0)
- new |= RW_HAS_WAITERS;
-
- /*
- * Set in the new value. The lock becomes owned
- * by the writer that we are about to wake.
- */
- (void)pthread__atomic_swap_ptr(&ptr->ptr_owner,
- (void *)new);
-
- /* Wake the writer. */
- PTQ_REMOVE(&ptr->ptr_wblocked, thread, pt_sleep);
- thread->pt_rwlocked = _RW_LOCKED;
- pthread__unpark(self, &ptr->ptr_interlock,
- &ptr->ptr_wblocked, thread);
- } else {
- new = 0;
- PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) {
- /*
- * May have already been handed the lock,
- * since pthread__unpark_all() can release
- * our interlock before awakening all
- * threads.
- */
- if (thread->pt_sleepobj == NULL)
- continue;
- new += RW_READ_INCR;
- thread->pt_rwlocked = _RW_LOCKED;
- }
-
- /*
- * Set in the new value. The lock becomes owned
- * by the readers that we are about to wake.
- */
- (void)pthread__atomic_swap_ptr(&ptr->ptr_owner,
- (void *)new);
-
- /* Wake up all sleeping readers. */
- ptr->ptr_nreaders = 0;
- pthread__unpark_all(self, &ptr->ptr_interlock,
- &ptr->ptr_rblocked);
- }
-
- return 0;
- }
-}
-
-/*
- * Called when a timedlock awakens early to adjust the waiter bits.
- * The rwlock's interlock is held on entry, and the caller has been
- * removed from the waiters lists.
- */
-static void
-pthread__rwlock_early(void *obj)
-{
- uintptr_t owner, set, new, next;
- pthread_rwlock_t *ptr;
- pthread_t self;
- u_int off;
-
- self = pthread__self();
-
- switch (self->pt_rwlocked) {
- case _RW_WANT_READ:
- off = offsetof(pthread_rwlock_t, ptr_rblocked);
- break;
- case _RW_WANT_WRITE:
- off = offsetof(pthread_rwlock_t, ptr_wblocked);
- break;
- default:
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "bad value of pt_rwlocked");
- off = 0;
- /* NOTREACHED */
- break;
- }
-
- /* LINTED mind your own business */
- ptr = (pthread_rwlock_t *)((uint8_t *)obj - off);
- owner = (uintptr_t)ptr->ptr_owner;
-
- if ((owner & RW_THREAD) == 0) {
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "lock not held");
- }
-
- if (!PTQ_EMPTY(&ptr->ptr_wblocked))
- set = RW_HAS_WAITERS | RW_WRITE_WANTED;
- else if (ptr->ptr_nreaders != 0)
- set = RW_HAS_WAITERS;
- else
- set = 0;
-
- for (;; owner = next) {
- new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set;
- next = rw_cas(ptr, owner, new);
- if (owner == next)
- break;
- }
-}
-
-int
-_pthread_rwlock_held_np(pthread_rwlock_t *ptr)
-{
- uintptr_t owner = (uintptr_t)ptr->ptr_owner;
-
- return (owner & RW_THREAD) != 0;
-}
-
-int
-_pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)
-{
- uintptr_t owner = (uintptr_t)ptr->ptr_owner;
-
- return (owner & RW_THREAD) != 0 && (owner & RW_WRITE_LOCKED) == 0;
-}
-
-int
-_pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)
-{
- uintptr_t owner = (uintptr_t)ptr->ptr_owner;
-
- return (owner & (RW_THREAD | RW_WRITE_LOCKED)) ==
- ((uintptr_t)pthread__self() | RW_WRITE_LOCKED);
-}
-
-int
-pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
-{
-
- if (attr == NULL)
- return EINVAL;
- attr->ptra_magic = _PT_RWLOCKATTR_MAGIC;
-
- return 0;
-}
-
-
-int
-pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
-{
-
- if ((attr == NULL) ||
- (attr->ptra_magic != _PT_RWLOCKATTR_MAGIC))
- return EINVAL;
- attr->ptra_magic = _PT_RWLOCKATTR_DEAD;
-
- return 0;
-}
-
-#endif /* PTHREAD__HAVE_ATOMIC */