diff options
| author | ad <ad@NetBSD.org> | 2007-08-04 13:37:48 +0000 |
|---|---|---|
| committer | ad <ad@NetBSD.org> | 2007-08-04 13:37:48 +0000 |
| commit | 50fa8db4e44b07accbfd5f2821987476be3bf05c (patch) | |
| tree | 04cfc58059fb658923f9a7b24583dd0b3e4437d1 /lib | |
| parent | b108f67c054e0f87f9b653c4d194652a4bfc40cc (diff) | |
Some significant performance improvements, and a fix for a race with pthread
detach/join.
- Make mutex acquire spin for a short time, as done with spinlocks.
- Make the number of spins controllable with the env var PTHREAD_NSPINS.
- Reduce the amount of time that libpthread internal spinlocks are held.
- Rely more on the barrier effects of park/unpark to avoid taking spinlocks.
- Simplify the locking around pthreads and the global queues.
- Align per-thread sync data on a 128 byte boundary.
- Offset thread stacks by a small amount to try and reduce cache thrash.
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/libpthread/pthread.c | 284 | ||||
| -rw-r--r-- | lib/libpthread/pthread_barrier.c | 6 | ||||
| -rw-r--r-- | lib/libpthread/pthread_cond.c | 101 | ||||
| -rw-r--r-- | lib/libpthread/pthread_int.h | 81 | ||||
| -rw-r--r-- | lib/libpthread/pthread_lock.c | 31 | ||||
| -rw-r--r-- | lib/libpthread/pthread_mutex.c | 128 | ||||
| -rw-r--r-- | lib/libpthread/pthread_rwlock.c | 18 | ||||
| -rw-r--r-- | lib/libpthread/pthread_types.h | 15 | ||||
| -rw-r--r-- | lib/libpthread/sem.c | 10 |
9 files changed, 340 insertions, 334 deletions
diff --git a/lib/libpthread/pthread.c b/lib/libpthread/pthread.c index 5ab952d5333..c210077b6c7 100644 --- a/lib/libpthread/pthread.c +++ b/lib/libpthread/pthread.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread.c,v 1.68 2007/03/24 18:51:59 ad Exp $ */ +/* $NetBSD: pthread.c,v 1.69 2007/08/04 13:37:48 ad Exp $ */ /*- * Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread.c,v 1.68 2007/03/24 18:51:59 ad Exp $"); +__RCSID("$NetBSD: pthread.c,v 1.69 2007/08/04 13:37:48 ad Exp $"); #include <err.h> #include <errno.h> @@ -68,21 +68,15 @@ __RCSID("$NetBSD: pthread.c,v 1.68 2007/03/24 18:51:59 ad Exp $"); /* How many times to try acquiring spin locks on MP systems. */ #define PTHREAD__NSPINS 1024 -static void pthread__create_tramp(void *(*start)(void *), void *arg); -static void pthread__dead(pthread_t, pthread_t); +static void pthread__create_tramp(void *(*)(void *), void *); +static void pthread__initthread(pthread_t); int pthread__started; -pthread_spin_t pthread__allqueue_lock = __SIMPLELOCK_UNLOCKED; -struct pthread_queue_t pthread__allqueue; +pthread_spin_t pthread__queue_lock = __SIMPLELOCK_UNLOCKED; +pthread_queue_t pthread__allqueue; +pthread_queue_t pthread__deadqueue; -pthread_spin_t pthread__deadqueue_lock = __SIMPLELOCK_UNLOCKED; -struct pthread_queue_t pthread__deadqueue; -struct pthread_queue_t *pthread__reidlequeue; - -static int nthreads; -static int nextthread; -static pthread_spin_t nextthread_lock = __SIMPLELOCK_UNLOCKED; static pthread_attr_t pthread_default_attr; enum { @@ -153,6 +147,8 @@ pthread_init(void) pthread__nspins = PTHREAD__NSPINS; else pthread__nspins = 1; + if ((p = getenv("PTHREAD_NSPINS")) != NULL) + pthread__nspins = atoi(p); i = (int)_lwp_unpark_all(NULL, 0, NULL); if (i == -1) err(1, "_lwp_unpark_all"); @@ -163,10 +159,9 @@ pthread_init(void) pthread_attr_init(&pthread_default_attr); PTQ_INIT(&pthread__allqueue); PTQ_INIT(&pthread__deadqueue); - nthreads = 1; /* Create the thread structure corresponding to main() */ pthread__initmain(&first); - pthread__initthread(first, first); + pthread__initthread(first); first->pt_state = PT_STATE_RUNNING; first->pt_lid = _lwp_self(); @@ -240,19 +235,12 @@ pthread__start(void) /* General-purpose thread data structure sanitization. */ -void -pthread__initthread(pthread_t self, pthread_t t) +/* ARGSUSED */ +static void +pthread__initthread(pthread_t t) { - int id; - - pthread_spinlock(self, &nextthread_lock); - id = nextthread; - nextthread++; - pthread_spinunlock(self, &nextthread_lock); - t->pt_num = id; t->pt_magic = PT_MAGIC; - pthread_lockinit(&t->pt_flaglock); t->pt_spinlocks = 0; t->pt_exitval = NULL; t->pt_flags = 0; @@ -260,10 +248,7 @@ pthread__initthread(pthread_t self, pthread_t t) t->pt_errno = 0; t->pt_state = PT_STATE_RUNNING; - pthread_lockinit(&t->pt_statelock); - - PTQ_INIT(&t->pt_joiners); - pthread_lockinit(&t->pt_join_lock); + pthread_lockinit(&t->pt_lock); PTQ_INIT(&t->pt_cleanup_stack); memset(&t->pt_specific, 0, sizeof(int) * PTHREAD_KEYS_MAX); t->pt_name = NULL; @@ -307,16 +292,24 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, self = pthread__self(); - pthread_spinlock(self, &pthread__deadqueue_lock); + pthread_spinlock(self, &pthread__queue_lock); newthread = PTQ_FIRST(&pthread__deadqueue); if (newthread != NULL) { - if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0 && - (_lwp_kill(newthread->pt_lid, 0) == 0 || errno != ESRCH)) - newthread = NULL; - else + if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) { + PTQ_REMOVE(&pthread__deadqueue, newthread, pt_allq); + pthread_spinunlock(self, &pthread__queue_lock); + if (_lwp_kill(newthread->pt_lid, 0) == 0 || + errno != ESRCH) { + pthread_spinlock(self, &pthread__queue_lock); + PTQ_INSERT_TAIL(&pthread__deadqueue, + newthread, pt_allq); + pthread_spinunlock(self, &pthread__queue_lock); + newthread = NULL; + } + } else PTQ_REMOVE(&pthread__deadqueue, newthread, pt_allq); } - pthread_spinunlock(self, &pthread__deadqueue_lock); + pthread_spinunlock(self, &pthread__queue_lock); if (newthread == NULL) { /* Set up a stack and allocate space for a pthread_st. */ ret = pthread__stackalloc(&newthread); @@ -328,7 +321,7 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, } /* 2. Set up state. */ - pthread__initthread(self, newthread); + pthread__initthread(newthread); newthread->pt_flags = nattr.pta_flags; /* 3. Set up misc. attributes. */ @@ -350,10 +343,9 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, startfunc, arg); /* 5. Add to list of all threads. */ - pthread_spinlock(self, &pthread__allqueue_lock); + pthread_spinlock(self, &pthread__queue_lock); PTQ_INSERT_HEAD(&pthread__allqueue, newthread, pt_allq); - nthreads++; - pthread_spinunlock(self, &pthread__allqueue_lock); + pthread_spinunlock(self, &pthread__queue_lock); /* 5a. Create the new LWP. */ newthread->pt_sleeponq = 0; @@ -367,16 +359,16 @@ pthread_create(pthread_t *thread, const pthread_attr_t *attr, SDPRINTF(("(pthread_create %p) _lwp_create: %s\n", strerror(errno))); free(name); - pthread_spinlock(self, &pthread__allqueue_lock); + pthread_spinlock(self, &pthread__queue_lock); PTQ_REMOVE(&pthread__allqueue, newthread, pt_allq); - nthreads--; - pthread_spinunlock(self, &pthread__allqueue_lock); - pthread_spinlock(self, &pthread__deadqueue_lock); PTQ_INSERT_HEAD(&pthread__deadqueue, newthread, pt_allq); - pthread_spinunlock(self, &pthread__deadqueue_lock); + pthread_spinunlock(self, &pthread__queue_lock); return ret; } + /* XXX must die */ + newthread->pt_num = newthread->pt_lid; + SDPRINTF(("(pthread_create %p) new thread %p (name %p, lid %d).\n", self, newthread, newthread->pt_name, (int)newthread->pt_lid)); @@ -392,6 +384,14 @@ pthread__create_tramp(void *(*start)(void *), void *arg) { void *retval; + /* + * Throw away some stack in a feeble attempt to reduce cache + * thrash. May help for SMT processors. XXX We should not + * be allocating stacks on fixed 2MB boundaries. Needs a + * thread register or decent thread local storage. + */ + (void)alloca(((unsigned)pthread__self()->pt_lid & 7) << 8); + retval = (*start)(arg); pthread_exit(retval); @@ -439,17 +439,16 @@ pthread_exit(void *retval) pthread_t self; struct pt_clean_t *cleanup; char *name; - int nt; self = pthread__self(); SDPRINTF(("(pthread_exit %p) status %p, flags %x, cancel %d\n", self, retval, self->pt_flags, self->pt_cancel)); /* Disable cancellability. */ - pthread_spinlock(self, &self->pt_flaglock); + pthread_spinlock(self, &self->pt_lock); self->pt_flags |= PT_FLAG_CS_DISABLED; self->pt_cancel = 0; - pthread_spinunlock(self, &self->pt_flaglock); + pthread_spinunlock(self, &self->pt_lock); /* Call any cancellation cleanup handlers */ while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { @@ -463,49 +462,23 @@ pthread_exit(void *retval) self->pt_exitval = retval; - /* - * it's safe to check PT_FLAG_DETACHED without pt_flaglock - * because it's only set by pthread_detach with pt_join_lock held. - */ - pthread_spinlock(self, &self->pt_join_lock); + pthread_spinlock(self, &self->pt_lock); if (self->pt_flags & PT_FLAG_DETACHED) { self->pt_state = PT_STATE_DEAD; - pthread_spinunlock(self, &self->pt_join_lock); name = self->pt_name; self->pt_name = NULL; - - if (name != NULL) - free(name); - - pthread_spinlock(self, &pthread__allqueue_lock); + pthread_spinlock(self, &pthread__queue_lock); PTQ_REMOVE(&pthread__allqueue, self, pt_allq); - nthreads--; - nt = nthreads; - pthread_spinunlock(self, &pthread__allqueue_lock); - - if (nt == 0) { - /* Whoah, we're the last one. Time to go. */ - exit(0); - } - - /* Yeah, yeah, doing work while we're dead is tacky. */ - pthread_spinlock(self, &pthread__deadqueue_lock); PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_allq); - pthread_spinunlock(self, &pthread__deadqueue_lock); + pthread_spinunlock(self, &pthread__queue_lock); + pthread_spinunlock(self, &self->pt_lock); + if (name != NULL) + free(name); _lwp_exit(); } else { self->pt_state = PT_STATE_ZOMBIE; - + pthread_spinunlock(self, &self->pt_lock); /* Note: name will be freed by the joiner. */ - pthread_spinlock(self, &pthread__allqueue_lock); - nthreads--; - nt = nthreads; - pthread_spinunlock(self, &pthread__allqueue_lock); - if (nt == 0) { - /* Whoah, we're the last one. Time to go. */ - exit(0); - } - pthread_spinunlock(self, &self->pt_join_lock); _lwp_exit(); } @@ -537,10 +510,10 @@ pthread_join(pthread_t thread, void **valptr) retval = 0; name = NULL; again: - pthread_spinlock(self, &thread->pt_join_lock); + pthread_spinlock(self, &thread->pt_lock); switch (thread->pt_state) { case PT_STATE_RUNNING: - pthread_spinunlock(self, &thread->pt_join_lock); + pthread_spinunlock(self, &thread->pt_lock); /* * IEEE Std 1003.1, 2004 Edition: @@ -559,22 +532,21 @@ pthread_join(pthread_t thread, void **valptr) thread->pt_name = NULL; } thread->pt_state = PT_STATE_DEAD; - pthread_spinunlock(self, &thread->pt_join_lock); + pthread_spinlock(self, &pthread__queue_lock); + PTQ_REMOVE(&pthread__allqueue, thread, pt_allq); + PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_allq); + pthread_spinunlock(self, &pthread__queue_lock); + pthread_spinunlock(self, &thread->pt_lock); + SDPRINTF(("(pthread_join %p) Joined %p.\n", self, thread)); + if (name != NULL) + free(name); (void)_lwp_detach(thread->pt_lid); - break; + return retval; default: - pthread_spinunlock(self, &thread->pt_join_lock); + pthread_spinunlock(self, &thread->pt_lock); return EINVAL; } - SDPRINTF(("(pthread_join %p) Joined %p.\n", self, thread)); - - pthread__dead(self, thread); - - if (name != NULL) - free(name); - - return retval; } @@ -600,33 +572,14 @@ pthread_detach(pthread_t thread) if (thread->pt_magic != PT_MAGIC) return EINVAL; - pthread_spinlock(self, &self->pt_join_lock); + pthread_spinlock(self, &self->pt_lock); thread->pt_flags |= PT_FLAG_DETACHED; - pthread_spinunlock(self, &self->pt_join_lock); + pthread_spinunlock(self, &self->pt_lock); return _lwp_detach(thread->pt_lid); } -static void -pthread__dead(pthread_t self, pthread_t thread) -{ - - SDPRINTF(("(pthread__dead %p) Reclaimed %p.\n", self, thread)); - pthread__assert(thread->pt_state == PT_STATE_DEAD); - pthread__assert(thread->pt_name == NULL); - - /* Cleanup time. Move the dead thread from allqueue to the deadqueue */ - pthread_spinlock(self, &pthread__allqueue_lock); - PTQ_REMOVE(&pthread__allqueue, thread, pt_allq); - pthread_spinunlock(self, &pthread__allqueue_lock); - - pthread_spinlock(self, &pthread__deadqueue_lock); - PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_allq); - pthread_spinunlock(self, &pthread__deadqueue_lock); -} - - int pthread_getname_np(pthread_t thread, char *name, size_t len) { @@ -640,12 +593,12 @@ pthread_getname_np(pthread_t thread, char *name, size_t len) if (thread->pt_magic != PT_MAGIC) return EINVAL; - pthread_spinlock(self, &thread->pt_join_lock); + pthread_spinlock(self, &thread->pt_lock); if (thread->pt_name == NULL) name[0] = '\0'; else strlcpy(name, thread->pt_name, len); - pthread_spinunlock(self, &thread->pt_join_lock); + pthread_spinunlock(self, &thread->pt_lock); return 0; } @@ -673,11 +626,10 @@ pthread_setname_np(pthread_t thread, const char *name, void *arg) if (cp == NULL) return ENOMEM; - pthread_spinlock(self, &thread->pt_join_lock); + pthread_spinlock(self, &thread->pt_lock); oldname = thread->pt_name; thread->pt_name = cp; - - pthread_spinunlock(self, &thread->pt_join_lock); + pthread_spinunlock(self, &thread->pt_lock); if (oldname != NULL) free(oldname); @@ -705,18 +657,16 @@ pthread_cancel(pthread_t thread) pthread_t self; self = pthread__self(); -#ifdef ERRORCHECK if (pthread__find(self, thread) != 0) return ESRCH; -#endif - pthread_spinlock(self, &thread->pt_flaglock); + pthread_spinlock(self, &thread->pt_lock); thread->pt_flags |= PT_FLAG_CS_PENDING; if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { thread->pt_cancel = 1; - pthread_spinunlock(self, &thread->pt_flaglock); + pthread_spinunlock(self, &thread->pt_lock); _lwp_wakeup(thread->pt_lid); } else - pthread_spinunlock(self, &thread->pt_flaglock); + pthread_spinunlock(self, &thread->pt_lock); return 0; } @@ -731,7 +681,8 @@ pthread_setcancelstate(int state, int *oldstate) self = pthread__self(); retval = 0; - pthread_spinlock(self, &self->pt_flaglock); + pthread_spinlock(self, &self->pt_lock); + if (oldstate != NULL) { if (self->pt_flags & PT_FLAG_CS_DISABLED) *oldstate = PTHREAD_CANCEL_DISABLE; @@ -756,14 +707,15 @@ pthread_setcancelstate(int state, int *oldstate) self->pt_cancel = 1; /* This is not a deferred cancellation point. */ if (self->pt_flags & PT_FLAG_CS_ASYNC) { - pthread_spinunlock(self, &self->pt_flaglock); + pthread_spinunlock(self, &self->pt_lock); pthread_exit(PTHREAD_CANCELED); } } } else retval = EINVAL; - pthread_spinunlock(self, &self->pt_flaglock); + pthread_spinunlock(self, &self->pt_lock); + return retval; } @@ -777,7 +729,7 @@ pthread_setcanceltype(int type, int *oldtype) self = pthread__self(); retval = 0; - pthread_spinlock(self, &self->pt_flaglock); + pthread_spinlock(self, &self->pt_lock); if (oldtype != NULL) { if (self->pt_flags & PT_FLAG_CS_ASYNC) @@ -789,7 +741,7 @@ pthread_setcanceltype(int type, int *oldtype) if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { self->pt_flags |= PT_FLAG_CS_ASYNC; if (self->pt_cancel) { - pthread_spinunlock(self, &self->pt_flaglock); + pthread_spinunlock(self, &self->pt_lock); pthread_exit(PTHREAD_CANCELED); } } else if (type == PTHREAD_CANCEL_DEFERRED) @@ -797,7 +749,8 @@ pthread_setcanceltype(int type, int *oldtype) else retval = EINVAL; - pthread_spinunlock(self, &self->pt_flaglock); + pthread_spinunlock(self, &self->pt_lock); + return retval; } @@ -825,11 +778,11 @@ pthread__find(pthread_t self, pthread_t id) { pthread_t target; - pthread_spinlock(self, &pthread__allqueue_lock); + pthread_spinlock(self, &pthread__queue_lock); PTQ_FOREACH(target, &pthread__allqueue, pt_allq) if (target == id) break; - pthread_spinunlock(self, &pthread__allqueue_lock); + pthread_spinunlock(self, &pthread__queue_lock); if (target == NULL) return ESRCH; @@ -963,9 +916,8 @@ pthread__errorfunc(const char *file, int line, const char *function, int pthread__park(pthread_t self, pthread_spin_t *lock, - struct pthread_queue_t *queue, - const struct timespec *abstime, int cancelpt, - const void *hint) + pthread_queue_t *queue, const struct timespec *abstime, + int cancelpt, const void *hint) { int rv; @@ -975,10 +927,27 @@ pthread__park(pthread_t self, pthread_spin_t *lock, * Wait until we are awoken by a pending unpark operation, * a signal, an unpark posted after we have gone asleep, * or an expired timeout. + * + * It is fine to test the value of both pt_sleepobj and + * pt_sleeponq without holding any locks, because: + * + * o Only the blocking thread (this thread) ever sets them + * to a non-NULL value. + * + * o Other threads may set them NULL, but if they do so they + * must also make this thread return from _lwp_park. + * + * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system + * calls and all make use of spinlocks in the kernel. So + * these system calls act as full memory barriers, and will + * ensure that the calling CPU's store buffers are drained. + * In combination with the spinlock release before unpark, + * this means that modification of pt_sleepobj/onq by another + * thread will become globally visible before that thread + * schedules an unpark operation on this thread. */ rv = 0; - do { - pthread_spinunlock(self, lock); + while (self->pt_sleepobj != NULL && rv == 0) { if (_lwp_park(abstime, NULL, hint) != 0) { switch (rv = errno) { case EINTR: @@ -1001,22 +970,26 @@ pthread__park(pthread_t self, pthread_spin_t *lock, * _lwp_park/_lwp_wakeup also provide a * barrier. */ - pthread_spinlock(self, &self->pt_flaglock); + pthread_spinlock(self, &self->pt_lock); if (self->pt_cancel) rv = EINTR; - pthread_spinunlock(self, &self->pt_flaglock); + pthread_spinunlock(self, &self->pt_lock); } - pthread_spinlock(self, lock); - } while (self->pt_sleepobj != NULL && rv == 0); + } /* * If we have been awoken early but are still on the queue, - * then remove ourself. + * then remove ourself. Again, it's safe to do the test + * without holding any locks. */ if (self->pt_sleeponq) { - PTQ_REMOVE(queue, self, pt_sleep); - self->pt_sleepobj = NULL; - self->pt_sleeponq = 0; + pthread_spinlock(self, lock); + if (self->pt_sleeponq) { + PTQ_REMOVE(queue, self, pt_sleep); + self->pt_sleepobj = NULL; + self->pt_sleeponq = 0; + } + pthread_spinunlock(self, lock); } SDPRINTF(("(pthread__park %p) queue %p exit\n", self, queue)); @@ -1026,7 +999,7 @@ pthread__park(pthread_t self, pthread_spin_t *lock, void pthread__unpark(pthread_t self, pthread_spin_t *lock, - struct pthread_queue_t *queue, pthread_t target) + pthread_queue_t *queue, pthread_t target) { int rv; @@ -1044,6 +1017,13 @@ pthread__unpark(pthread_t self, pthread_spin_t *lock, */ target->pt_sleepobj = NULL; target->pt_sleeponq = 0; + + /* + * Releasing the spinlock serves as a store barrier, + * which ensures that all our modifications are visible + * to the thread in pthread__park() before the unpark + * operation is set in motion. + */ pthread_spinunlock(self, lock); rv = _lwp_unpark(target->pt_lid, queue); @@ -1056,7 +1036,7 @@ pthread__unpark(pthread_t self, pthread_spin_t *lock, void pthread__unpark_all(pthread_t self, pthread_spin_t *lock, - struct pthread_queue_t *queue) + pthread_queue_t *queue) { lwpid_t waiters[PTHREAD__UNPARK_MAX]; ssize_t n, rv; @@ -1096,8 +1076,6 @@ pthread__unpark_all(pthread_t self, pthread_spin_t *lock, * * In both cases we shouldn't remove the * thread from the queue. - * - * XXXLWP basic fairness issues here. */ next = PTQ_NEXT(thread, pt_sleep); if (thread->pt_sleepobj != NULL) @@ -1109,6 +1087,12 @@ pthread__unpark_all(pthread_t self, pthread_spin_t *lock, "unpark %p\n", self, queue, thread)); } + /* + * Releasing the spinlock serves as a store barrier, + * which ensures that all our modifications are visible + * to the thread in pthread__park() before the unpark + * operation is set in motion. + */ pthread_spinunlock(self, lock); switch (n) { case 0: @@ -1122,7 +1106,7 @@ pthread__unpark_all(pthread_t self, pthread_spin_t *lock, } return; default: - rv = _lwp_unpark_all(waiters, n, queue); + rv = _lwp_unpark_all(waiters, (size_t)n, queue); if (rv != 0 && errno != EINTR) { OOPS("_lwp_unpark_all failed"); SDPRINTF(("(pthread__unpark_all %p) " diff --git a/lib/libpthread/pthread_barrier.c b/lib/libpthread/pthread_barrier.c index 2562339bc20..6897f63c3c5 100644 --- a/lib/libpthread/pthread_barrier.c +++ b/lib/libpthread/pthread_barrier.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_barrier.c,v 1.12 2007/03/24 18:52:00 ad Exp $ */ +/* $NetBSD: pthread_barrier.c,v 1.13 2007/08/04 13:37:49 ad Exp $ */ /*- * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_barrier.c,v 1.12 2007/03/24 18:52:00 ad Exp $"); +__RCSID("$NetBSD: pthread_barrier.c,v 1.13 2007/08/04 13:37:49 ad Exp $"); #include <errno.h> @@ -180,11 +180,13 @@ pthread_barrier_wait(pthread_barrier_t *barrier) PTQ_INSERT_TAIL(&barrier->ptb_waiters, self, pt_sleep); self->pt_sleeponq = 1; self->pt_sleepobj = &barrier->ptb_waiters; + pthread_spinunlock(self, &barrier->ptb_lock); (void)pthread__park(self, &barrier->ptb_lock, &barrier->ptb_waiters, NULL, 0, &barrier->ptb_waiters); SDPRINTF(("(barrier wait %p) Woke up on %p\n", self, barrier)); + pthread_spinlock(self, &barrier->ptb_lock); } pthread_spinunlock(self, &barrier->ptb_lock); diff --git a/lib/libpthread/pthread_cond.c b/lib/libpthread/pthread_cond.c index d6914dadb75..9d32c98160c 100644 --- a/lib/libpthread/pthread_cond.c +++ b/lib/libpthread/pthread_cond.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_cond.c,v 1.31 2007/04/12 21:36:06 ad Exp $ */ +/* $NetBSD: pthread_cond.c,v 1.32 2007/08/04 13:37:49 ad Exp $ */ /*- * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_cond.c,v 1.31 2007/04/12 21:36:06 ad Exp $"); +__RCSID("$NetBSD: pthread_cond.c,v 1.32 2007/08/04 13:37:49 ad Exp $"); #include <errno.h> #include <sys/time.h> @@ -46,12 +46,6 @@ __RCSID("$NetBSD: pthread_cond.c,v 1.31 2007/04/12 21:36:06 ad Exp $"); #include "pthread.h" #include "pthread_int.h" -#ifdef PTHREAD_COND_DEBUG -#define SDPRINTF(x) DPRINTF(x) -#else -#define SDPRINTF(x) -#endif - int _sys_nanosleep(const struct timespec *, struct timespec *); extern int pthread__started; @@ -116,10 +110,14 @@ pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) if (__predict_false(pthread__started == 0)) return pthread_cond_wait_nothread(self, mutex, NULL); - SDPRINTF(("(cond wait %p) Waiting on %p, mutex %p\n", - self, cond, mutex)); if (__predict_false(self->pt_cancel)) pthread_exit(PTHREAD_CANCELED); + + /* + * Note this thread as waiting on the CV. To ensure good + * performance it's critical that the spinlock is held for + * as short a time as possible - that means no system calls. + */ pthread_spinlock(self, &cond->ptc_lock); if (cond->ptc_mutex == NULL) cond->ptc_mutex = mutex; @@ -134,23 +132,40 @@ pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) self->pt_signalled = 0; self->pt_sleeponq = 1; self->pt_sleepobj = &cond->ptc_waiters; + pthread_spinunlock(self, &cond->ptc_lock); + + /* Once recorded as a waiter release the mutex and sleep. */ pthread_mutex_unlock(mutex); (void)pthread__park(self, &cond->ptc_lock, &cond->ptc_waiters, NULL, 1, &mutex->ptm_blocked); - if (PTQ_EMPTY(&cond->ptc_waiters)) - cond->ptc_mutex = NULL; - pthread_spinunlock(self, &cond->ptc_lock); - pthread_mutex_lock(mutex); + /* + * If we awoke abnormally the waiters list will have been + * made empty by the current thread (in pthread__park()), + * so we can check the value safely without locking. + * + * Otherwise, it will have been updated by whichever thread + * last issued a wakeup. + */ + if (PTQ_EMPTY(&cond->ptc_waiters) && cond->ptc_mutex != NULL) { + pthread_spinlock(self, &cond->ptc_lock); + if (PTQ_EMPTY(&cond->ptc_waiters)) + cond->ptc_mutex = NULL; + pthread_spinunlock(self, &cond->ptc_lock); + } + + /* + * Re-acquire the mutex and return to the caller. If we + * have cancelled then exit. POSIX dictates that the mutex + * must be held when we action the cancellation. + */ + pthread_mutex_lock(mutex); if (__predict_false(self->pt_cancel)) { if (self->pt_signalled) pthread_cond_signal(cond); pthread_exit(PTHREAD_CANCELED); } - SDPRINTF(("(cond wait %p) Woke up on %p, mutex %p\n", - self, cond, mutex)); - return 0; } @@ -178,11 +193,14 @@ pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, if (__predict_false(pthread__started == 0)) return pthread_cond_wait_nothread(self, mutex, abstime); - SDPRINTF(("(cond timed wait %p) Waiting on %p until %d.%06ld\n", - self, cond, abstime->tv_sec, abstime->tv_nsec/1000)); - if (__predict_false(self->pt_cancel)) pthread_exit(PTHREAD_CANCELED); + + /* + * Note this thread as waiting on the CV. To ensure good + * performance it's critical that the spinlock is held for + * as short a time as possible - that means no system calls. + */ pthread_spinlock(self, &cond->ptc_lock); if (cond->ptc_mutex == NULL) cond->ptc_mutex = mutex; @@ -197,17 +215,33 @@ pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, self->pt_signalled = 0; self->pt_sleeponq = 1; self->pt_sleepobj = &cond->ptc_waiters; + pthread_spinunlock(self, &cond->ptc_lock); + + /* Once recorded as a waiter release the mutex and sleep. */ pthread_mutex_unlock(mutex); retval = pthread__park(self, &cond->ptc_lock, &cond->ptc_waiters, abstime, 1, &mutex->ptm_blocked); - if (PTQ_EMPTY(&cond->ptc_waiters)) - cond->ptc_mutex = NULL; - pthread_spinunlock(self, &cond->ptc_lock); - SDPRINTF(("(cond timed wait %p) Woke up on %p, mutex %p\n", - self, cond)); - SDPRINTF(("(cond timed wait %p) %s\n", - self, (retval == ETIMEDOUT) ? "(timed out)" : "")); + /* + * If we awoke abnormally the waiters list will have been + * made empty by the current thread (in pthread__park()), + * so we can check the value safely without locking. + * + * Otherwise, it will have been updated by whichever thread + * last issued a wakeup. + */ + if (PTQ_EMPTY(&cond->ptc_waiters) && cond->ptc_mutex != NULL) { + pthread_spinlock(self, &cond->ptc_lock); + if (PTQ_EMPTY(&cond->ptc_waiters)) + cond->ptc_mutex = NULL; + pthread_spinunlock(self, &cond->ptc_lock); + } + + /* + * Re-acquire the mutex and return to the caller. If we + * have cancelled then exit. POSIX dictates that the mutex + * must be held when we action the cancellation. + */ pthread_mutex_lock(mutex); if (__predict_false(self->pt_cancel | retval)) { if (self->pt_signalled) @@ -229,9 +263,6 @@ pthread_cond_signal(pthread_cond_t *cond) cond->ptc_magic == _PT_COND_MAGIC); PTHREADD_ADD(PTHREADD_COND_SIGNAL); - SDPRINTF(("(cond signal %p) Signaling %p\n", - pthread__self(), cond)); - if (PTQ_EMPTY(&cond->ptc_waiters)) return 0; @@ -258,10 +289,10 @@ pthread_cond_signal(pthread_cond_t *cond) * * After resuming execution, the thread must check to see if it * has been restarted as a result of pthread_cond_signal(). If it - * has, but cannot take the wakeup (because of eg a pending Unix - * signal or timeout) then try to ensure that another thread sees - * it. This is necessary because there may be multiple waiters, - * and at least one should take the wakeup if possible. + * has, but cannot take the wakeup (because of eg a timeout) then + * try to ensure that another thread sees it. This is necessary + * because there may be multiple waiters, and at least one should + * take the wakeup if possible. */ PTQ_REMOVE(&cond->ptc_waiters, signaled, pt_sleep); mutex = cond->ptc_mutex; @@ -302,8 +333,6 @@ pthread_cond_broadcast(pthread_cond_t *cond) cond->ptc_magic == _PT_COND_MAGIC); PTHREADD_ADD(PTHREADD_COND_BROADCAST); - SDPRINTF(("(cond signal %p) Broadcasting %p\n", - pthread__self(), cond)); if (PTQ_EMPTY(&cond->ptc_waiters)) return 0; diff --git a/lib/libpthread/pthread_int.h b/lib/libpthread/pthread_int.h index b610ee67c67..2ccfc130135 100644 --- a/lib/libpthread/pthread_int.h +++ b/lib/libpthread/pthread_int.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_int.h,v 1.42 2007/04/12 21:36:06 ad Exp $ */ +/* $NetBSD: pthread_int.h,v 1.43 2007/08/04 13:37:49 ad Exp $ */ /*- * Copyright (c) 2001, 2002, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -71,35 +71,15 @@ struct pthread_attr_private { }; struct __pthread_st { - unsigned int pt_magic; - /* Identifier, for debugging and for preventing recycling. */ - int pt_num; - - lwpid_t pt_lid; /* LWP ID */ - int pt_state; /* running, blocked, etc. */ - pthread_spin_t pt_statelock; /* lock on pt_state */ - int pt_flags; /* see PT_FLAG_* below */ - pthread_spin_t pt_flaglock; /* lock on pt_flag */ - int pt_cancel; /* Deferred cancellation */ - int pt_spinlocks; /* Number of spinlocks held. */ - void *pt_mutexhint; /* Last mutex held. */ - int pt_sleeponq; /* on a sleep queue */ - int pt_errno; /* Thread-specific errno. */ - int pt_signalled; /* Received pthread_cond_signal() */ - - /* Entry on the list of all threads */ - PTQ_ENTRY(__pthread_st) pt_allq; - /* Entry on the sleep queue (xxx should be same as run queue?) */ - PTQ_ENTRY(__pthread_st) pt_sleep; - /* - * Object we're sleeping on. For 1:1 threads (!SA), this is - * protected by the interlock on the object that the thread is - * sleeping on. - */ - void *pt_sleepobj; - /* Queue we're sleeping on */ - struct pthread_queue_t *pt_sleepq; - + unsigned int pt_magic; /* Magic number */ + int pt_num; /* ID XXX should die */ + int pt_state; /* running, blocked, etc. */ + pthread_spin_t pt_lock; /* lock on state */ + int pt_flags; /* see PT_FLAG_* below */ + int pt_cancel; /* Deferred cancellation */ + int pt_spinlocks; /* Number of spinlocks held. */ + void *pt_mutexhint; /* Last mutex held. */ + int pt_errno; /* Thread-specific errno. */ stack_t pt_stack; /* Our stack */ ucontext_t *pt_uc; /* Saved context when we're stopped */ void *pt_exitval; /* Read by pthread_join() */ @@ -108,15 +88,24 @@ struct __pthread_st { /* Stack of cancellation cleanup handlers and their arguments */ PTQ_HEAD(, pt_clean_t) pt_cleanup_stack; - /* Other threads trying to pthread_join() us. */ - struct pthread_queue_t pt_joiners; - /* Lock for above, and for changing pt_state to ZOMBIE or DEAD, - * and for setting the DETACHED flag. Also protects pt_name. - */ - pthread_spin_t pt_join_lock; - /* Thread-specific data */ - void* pt_specific[PTHREAD_KEYS_MAX]; + void *pt_specific[PTHREAD_KEYS_MAX]; + + /* LWP ID and entry on the list of all threads. */ + lwpid_t pt_lid; + PTQ_ENTRY(__pthread_st) pt_allq; + + /* + * General synchronization data. We try to align, as threads + * on other CPUs will access this data frequently. + */ + int pt_dummy1 __aligned(128); + int pt_sleeponq; /* on a sleep queue */ + int pt_signalled; /* Received pthread_cond_signal() */ + void *pt_sleepobj; /* object slept on */ + pthread_queue_t *pt_sleepq; /* sleep queue */ + PTQ_ENTRY(__pthread_st) pt_sleep; + int pt_dummy2 __aligned(128); }; /* Thread states */ @@ -169,17 +158,15 @@ void pthread_init(void) __attribute__ ((__constructor__)); /* Utility functions */ -/* Set up/clean up a thread's basic state. */ -void pthread__initthread(pthread_t self, pthread_t t); /* Get offset from stack start to struct sa_stackinfo */ ssize_t pthread__stackinfo_offset(void); void pthread__unpark_all(pthread_t self, pthread_spin_t *lock, - struct pthread_queue_t *threadq); + pthread_queue_t *threadq); void pthread__unpark(pthread_t self, pthread_spin_t *lock, - struct pthread_queue_t *queue, pthread_t target); + pthread_queue_t *queue, pthread_t target); int pthread__park(pthread_t self, pthread_spin_t *lock, - struct pthread_queue_t *threadq, + pthread_queue_t *threadq, const struct timespec *abs_timeout, int cancelpt, const void *hint); @@ -258,4 +245,12 @@ void pthread__assertfunc(const char *file, int line, const char *function, void pthread__errorfunc(const char *file, int line, const char *function, const char *msg); +#if defined(i386) || defined(__x86_64__) +#define pthread__smt_pause() __asm __volatile("rep; nop" ::: "memory") +#else +#define pthread__smt_pause() /* nothing */ +#endif + +extern int pthread__nspins; + #endif /* _LIB_PTHREAD_INT_H */ diff --git a/lib/libpthread/pthread_lock.c b/lib/libpthread/pthread_lock.c index 2232e0eb441..c5d4b91a8aa 100644 --- a/lib/libpthread/pthread_lock.c +++ b/lib/libpthread/pthread_lock.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_lock.c,v 1.20 2007/03/24 18:52:00 ad Exp $ */ +/* $NetBSD: pthread_lock.c,v 1.21 2007/08/04 13:37:49 ad Exp $ */ /*- * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_lock.c,v 1.20 2007/03/24 18:52:00 ad Exp $"); +__RCSID("$NetBSD: pthread_lock.c,v 1.21 2007/08/04 13:37:49 ad Exp $"); #include <sys/types.h> #include <sys/lock.h> @@ -56,14 +56,6 @@ __RCSID("$NetBSD: pthread_lock.c,v 1.20 2007/03/24 18:52:00 ad Exp $"); #define SDPRINTF(x) #endif -/* This does not belong here. */ -#if defined(i386) || defined(__x86_64__) -#define smt_pause() __asm __volatile("rep; nop" ::: "memory") -#else -#define smt_pause() /* nothing */ -#endif - -extern int pthread__nspins; static int pthread__atomic; RAS_DECL(pthread__lock); @@ -139,9 +131,8 @@ pthread_lockinit(pthread_spin_t *lock) void pthread_spinlock(pthread_t thread, pthread_spin_t *lock) { - int count, ret; + int count; - count = pthread__nspins; SDPRINTF(("(pthread_spinlock %p) spinlock %p (count %d)\n", thread, lock, thread->pt_spinlocks)); #ifdef PTHREAD_SPIN_DEBUG @@ -155,14 +146,15 @@ pthread_spinlock(pthread_t thread, pthread_spin_t *lock) } do { - while ((ret = pthread__simple_lock_try(lock)) == 0 && - --count) { - smt_pause(); + count = pthread__nspins; + while (*lock == __SIMPLELOCK_LOCKED && --count > 0) + pthread__smt_pause(); + if (count > 0) { + if (pthread__simple_lock_try(lock)) + break; + continue; } - if (ret == 1) - break; - SDPRINTF(("(pthread_spinlock %p) retrying spinlock %p " "(count %d)\n", thread, lock, thread->pt_spinlocks)); @@ -170,7 +162,6 @@ pthread_spinlock(pthread_t thread, pthread_spin_t *lock) /* XXXLWP far from ideal */ sched_yield(); - count = pthread__nspins; thread->pt_spinlocks++; } while (/*CONSTCOND*/ 1); @@ -260,7 +251,7 @@ pthread_spin_lock(pthread_spinlock_t *lock) #endif while (pthread__simple_lock_try(&lock->pts_spin) == 0) { - smt_pause(); + pthread__smt_pause(); } return 0; diff --git a/lib/libpthread/pthread_mutex.c b/lib/libpthread/pthread_mutex.c index ee1c6304614..bb9024faf00 100644 --- a/lib/libpthread/pthread_mutex.c +++ b/lib/libpthread/pthread_mutex.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_mutex.c,v 1.28 2007/03/24 18:52:00 ad Exp $ */ +/* $NetBSD: pthread_mutex.c,v 1.29 2007/08/04 13:37:49 ad Exp $ */ /*- * Copyright (c) 2001, 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_mutex.c,v 1.28 2007/03/24 18:52:00 ad Exp $"); +__RCSID("$NetBSD: pthread_mutex.c,v 1.29 2007/08/04 13:37:49 ad Exp $"); #include <errno.h> #include <limits.h> @@ -189,15 +189,26 @@ static int pthread_mutex_lock_slow(pthread_t self, pthread_mutex_t *mutex) { extern int pthread__started; + struct mutex_private *mp; + sigset_t ss; + int count; pthread__error(EINVAL, "Invalid mutex", mutex->ptm_magic == _PT_MUTEX_MAGIC); PTHREADD_ADD(PTHREADD_MUTEX_LOCK_SLOW); - while (/*CONSTCOND*/1) { - if (pthread__simple_lock_try(&mutex->ptm_lock)) - break; /* got it! */ - + + for (;;) { + /* Spin for a while. */ + count = pthread__nspins; + while (mutex->ptm_lock == __SIMPLELOCK_LOCKED && --count > 0) + pthread__smt_pause(); + if (count > 0) { + if (pthread__simple_lock_try(&mutex->ptm_lock) != 0) + break; + continue; + } + /* Okay, didn't look free. Get the interlock... */ pthread_spinlock(self, &mutex->ptm_interlock); @@ -209,71 +220,60 @@ pthread_mutex_lock_slow(pthread_t self, pthread_mutex_t *mutex) * again. */ PTQ_INSERT_HEAD(&mutex->ptm_blocked, self, pt_sleep); - if (mutex->ptm_lock == __SIMPLELOCK_LOCKED) { - struct mutex_private *mp; - - GET_MUTEX_PRIVATE(mutex, mp); - - if (mutex->ptm_owner == self) { - switch (mp->type) { - case PTHREAD_MUTEX_ERRORCHECK: - PTQ_REMOVE(&mutex->ptm_blocked, self, - pt_sleep); - pthread_spinunlock(self, - &mutex->ptm_interlock); - return EDEADLK; - - case PTHREAD_MUTEX_RECURSIVE: - /* - * It's safe to do this without - * holding the interlock, because - * we only modify it if we know we - * own the mutex. - */ - PTQ_REMOVE(&mutex->ptm_blocked, self, - pt_sleep); - pthread_spinunlock(self, - &mutex->ptm_interlock); - if (mp->recursecount == INT_MAX) - return EAGAIN; - mp->recursecount++; - return 0; - } - } + if (mutex->ptm_lock != __SIMPLELOCK_LOCKED) { + PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); + pthread_spinunlock(self, &mutex->ptm_interlock); + continue; + } + + GET_MUTEX_PRIVATE(mutex, mp); - if (pthread__started == 0) { - sigset_t ss; + if (mutex->ptm_owner == self) { + switch (mp->type) { + case PTHREAD_MUTEX_ERRORCHECK: + PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); + pthread_spinunlock(self, &mutex->ptm_interlock); + return EDEADLK; + case PTHREAD_MUTEX_RECURSIVE: /* - * The spec says we must deadlock, so... + * It's safe to do this without + * holding the interlock, because + * we only modify it if we know we + * own the mutex. */ - pthread__assert(mp->type == - PTHREAD_MUTEX_NORMAL); - (void) sigprocmask(SIG_SETMASK, NULL, &ss); - for (;;) { - sigsuspend(&ss); - } - /*NOTREACHED*/ + PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); + pthread_spinunlock(self, &mutex->ptm_interlock); + if (mp->recursecount == INT_MAX) + return EAGAIN; + mp->recursecount++; + return 0; } + } - /* - * Locking a mutex is not a cancellation - * point, so we don't need to do the - * test-cancellation dance. We may get woken - * up spuriously by pthread_cancel or signals, - * but it's okay since we're just going to - * retry. - */ - self->pt_sleeponq = 1; - self->pt_sleepobj = &mutex->ptm_blocked; - (void)pthread__park(self, &mutex->ptm_interlock, - &mutex->ptm_blocked, NULL, 0, &mutex->ptm_blocked); - pthread_spinunlock(self, &mutex->ptm_interlock); - } else { - PTQ_REMOVE(&mutex->ptm_blocked, self, pt_sleep); - pthread_spinunlock(self, &mutex->ptm_interlock); + if (pthread__started == 0) { + /* The spec says we must deadlock, so... */ + pthread__assert(mp->type == PTHREAD_MUTEX_NORMAL); + (void) sigprocmask(SIG_SETMASK, NULL, &ss); + for (;;) { + sigsuspend(&ss); + } + /*NOTREACHED*/ } - /* Go around for another try. */ + + /* + * Locking a mutex is not a cancellation + * point, so we don't need to do the + * test-cancellation dance. We may get woken + * up spuriously by pthread_cancel or signals, + * but it's okay since we're just going to + * retry. + */ + self->pt_sleeponq = 1; + self->pt_sleepobj = &mutex->ptm_blocked; + pthread_spinunlock(self, &mutex->ptm_interlock); + (void)pthread__park(self, &mutex->ptm_interlock, + &mutex->ptm_blocked, NULL, 0, &mutex->ptm_blocked); } return 0; diff --git a/lib/libpthread/pthread_rwlock.c b/lib/libpthread/pthread_rwlock.c index 405fceaaaa6..f7f5c6c879c 100644 --- a/lib/libpthread/pthread_rwlock.c +++ b/lib/libpthread/pthread_rwlock.c @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_rwlock.c,v 1.18 2007/03/24 18:52:00 ad Exp $ */ +/* $NetBSD: pthread_rwlock.c,v 1.19 2007/08/04 13:37:50 ad Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: pthread_rwlock.c,v 1.18 2007/03/24 18:52:00 ad Exp $"); +__RCSID("$NetBSD: pthread_rwlock.c,v 1.19 2007/08/04 13:37:50 ad Exp $"); #include <errno.h> @@ -117,8 +117,10 @@ pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep); self->pt_sleeponq = 1; self->pt_sleepobj = &rwlock->ptr_rblocked; + pthread_spinunlock(self, &rwlock->ptr_interlock); (void)pthread__park(self, &rwlock->ptr_interlock, &rwlock->ptr_rblocked, NULL, 0, &rwlock->ptr_rblocked); + pthread_spinlock(self, &rwlock->ptr_interlock); } rwlock->ptr_nreaders++; @@ -190,8 +192,10 @@ pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep); self->pt_sleeponq = 1; self->pt_sleepobj = &rwlock->ptr_wblocked; + pthread_spinunlock(self, &rwlock->ptr_interlock); (void)pthread__park(self, &rwlock->ptr_interlock, &rwlock->ptr_wblocked, NULL, 0, &rwlock->ptr_wblocked); + pthread_spinlock(self, &rwlock->ptr_interlock); } rwlock->ptr_writer = self; @@ -228,12 +232,6 @@ pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) } -struct pthread_rwlock__waitarg { - pthread_t ptw_thread; - pthread_rwlock_t *ptw_rwlock; - struct pthread_queue_t *ptw_queue; -}; - int pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlock, const struct timespec *abs_timeout) @@ -271,9 +269,11 @@ pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlock, PTQ_INSERT_TAIL(&rwlock->ptr_rblocked, self, pt_sleep); self->pt_sleeponq = 1; self->pt_sleepobj = &rwlock->ptr_rblocked; + pthread_spinunlock(self, &rwlock->ptr_interlock); retval = pthread__park(self, &rwlock->ptr_interlock, &rwlock->ptr_rblocked, abs_timeout, 0, &rwlock->ptr_rblocked); + pthread_spinlock(self, &rwlock->ptr_interlock); } /* One last chance to get the lock, in case it was released between @@ -333,9 +333,11 @@ pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlock, PTQ_INSERT_TAIL(&rwlock->ptr_wblocked, self, pt_sleep); self->pt_sleeponq = 1; self->pt_sleepobj = &rwlock->ptr_wblocked; + pthread_spinunlock(self, &rwlock->ptr_interlock); retval = pthread__park(self, &rwlock->ptr_interlock, &rwlock->ptr_wblocked, abs_timeout, 0, &rwlock->ptr_wblocked); + pthread_spinlock(self, &rwlock->ptr_interlock); } if ((rwlock->ptr_nreaders == 0) && (rwlock->ptr_writer == NULL)) { diff --git a/lib/libpthread/pthread_types.h b/lib/libpthread/pthread_types.h index c8fe3d2103e..fd9da70ea87 100644 --- a/lib/libpthread/pthread_types.h +++ b/lib/libpthread/pthread_types.h @@ -1,4 +1,4 @@ -/* $NetBSD: pthread_types.h,v 1.6 2007/05/02 21:54:16 ad Exp $ */ +/* $NetBSD: pthread_types.h,v 1.7 2007/08/04 13:37:50 ad Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -54,7 +54,8 @@ struct name { \ struct type **ptqh_last;/* addr of last next element */ \ } -_PTQ_HEAD(pthread_queue_t, __pthread_st); +_PTQ_HEAD(pthread_queue_struct_t, __pthread_st); +typedef struct pthread_queue_struct_t pthread_queue_t; struct __pthread_st; struct __pthread_attr_st; @@ -98,7 +99,7 @@ struct __pthread_mutex_st { pthread_spin_t ptm_lock; pthread_spin_t ptm_interlock; pthread_t ptm_owner; - struct pthread_queue_t ptm_blocked; + pthread_queue_t ptm_blocked; void *ptm_private; }; @@ -128,7 +129,7 @@ struct __pthread_cond_st { /* Protects the queue of waiters */ pthread_spin_t ptc_lock; - struct pthread_queue_t ptc_waiters; + pthread_queue_t ptc_waiters; pthread_mutex_t *ptc_mutex; /* Current mutex */ void *ptc_private; @@ -181,8 +182,8 @@ struct __pthread_rwlock_st { /* Protects data below */ pthread_spin_t ptr_interlock; - struct pthread_queue_t ptr_rblocked; - struct pthread_queue_t ptr_wblocked; + pthread_queue_t ptr_rblocked; + pthread_queue_t ptr_wblocked; unsigned int ptr_nreaders; pthread_t ptr_writer; void *ptr_private; @@ -214,7 +215,7 @@ struct __pthread_barrier_st { /* Protects data below */ pthread_spin_t ptb_lock; - struct pthread_queue_t ptb_waiters; + pthread_queue_t ptb_waiters; unsigned int ptb_initcount; unsigned int ptb_curcount; unsigned int ptb_generation; diff --git a/lib/libpthread/sem.c b/lib/libpthread/sem.c index 44fe2940102..b659cf33719 100644 --- a/lib/libpthread/sem.c +++ b/lib/libpthread/sem.c @@ -1,4 +1,4 @@ -/* $NetBSD: sem.c,v 1.14 2007/03/24 18:52:00 ad Exp $ */ +/* $NetBSD: sem.c,v 1.15 2007/08/04 13:37:50 ad Exp $ */ /*- * Copyright (c) 2003, 2006, 2007 The NetBSD Foundation, Inc. @@ -66,7 +66,7 @@ */ #include <sys/cdefs.h> -__RCSID("$NetBSD: sem.c,v 1.14 2007/03/24 18:52:00 ad Exp $"); +__RCSID("$NetBSD: sem.c,v 1.15 2007/08/04 13:37:50 ad Exp $"); #include <sys/types.h> #include <sys/ksem.h> @@ -92,7 +92,7 @@ struct _sem_st { /* Protects data below. */ pthread_spin_t usem_interlock; - struct pthread_queue_t usem_waiters; + pthread_queue_t usem_waiters; unsigned int usem_count; }; @@ -287,7 +287,7 @@ sem_wait(sem_t *sem) { pthread_t self; extern int pthread__started; - struct pthread_queue_t *queue; + pthread_queue_t *queue; #ifdef ERRORCHECK if (sem == NULL || *sem == NULL || (*sem)->usem_magic != USEM_MAGIC) { @@ -333,8 +333,10 @@ sem_wait(sem_t *sem) PTQ_INSERT_TAIL(queue, self, pt_sleep); self->pt_sleeponq = 1; self->pt_sleepobj = queue, + pthread_spinunlock(self, &(*sem)->usem_interlock); (void)pthread__park(self, &(*sem)->usem_interlock, queue, NULL, 1, queue); + pthread_spinlock(self, &(*sem)->usem_interlock); } (*sem)->usem_count--; pthread_spinunlock(self, &(*sem)->usem_interlock); |
