diff options
| author | cl <cl@NetBSD.org> | 2004-03-14 01:08:47 +0000 |
|---|---|---|
| committer | cl <cl@NetBSD.org> | 2004-03-14 01:08:47 +0000 |
| commit | ea5ec0212d715373eb56dbdeb218b9504e1139f2 (patch) | |
| tree | cb57ac9e27b581a85616dd80ef89dfd4aa130a8e | |
| parent | f1bacc8b382d29f3679efbbb152b03c00e19767b (diff) | |
add kernel part of concurrency support for SA on MP systems
- move per VP data into struct sadata_vp referenced from l->l_savp
* VP id
* lock on VP data
* LWP on VP
* recently blocked LWP on VP
* queue of LWPs woken which ran on this VP before sleep
* faultaddr
* LWP cache for upcalls
* upcall queue
- add current concurrency and requested concurrency variables
- make process exit run LWP on all VPs
- make signal delivery consider all VPs
- make timer events consider all VPs
- add sa_newsavp to allocate new sadata_vp structure
- add sa_increaseconcurrency to prepare new VP
- make sys_sa_setconcurrency request new VP or wakeup idle VP
- make sa_yield lower current concurrency
- set sa_cpu = VP id in upcalls
- maintain cached LWPs per VP
35 files changed, 539 insertions, 331 deletions
diff --git a/sys/arch/alpha/alpha/trap.c b/sys/arch/alpha/alpha/trap.c index 1b463bcbfa4..06de0521f6a 100644 --- a/sys/arch/alpha/alpha/trap.c +++ b/sys/arch/alpha/alpha/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.90 2004/02/19 17:06:06 drochner Exp $ */ +/* $NetBSD: trap.c,v 1.91 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 2000, 2001 The NetBSD Foundation, Inc. @@ -100,7 +100,7 @@ #include <sys/cdefs.h> /* RCS ID & Copyright macro defns */ -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.90 2004/02/19 17:06:06 drochner Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.91 2004/03/14 01:08:47 cl Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -438,8 +438,7 @@ trap(const u_long a0, const u_long a1, const u_long a2, const u_long entry, if (user) { KERNEL_PROC_LOCK(l); if (l->l_flag & L_SA) { - KDASSERT(l->l_proc != NULL && l->l_proc->p_sa != NULL); - l->l_proc->p_sa->sa_vp_faultaddr = (vaddr_t)a0; + l->l_savp->savp_faultaddr = (vaddr_t)a0; l->l_flag |= L_SA_PAGEFAULT; } } else { diff --git a/sys/arch/amd64/amd64/trap.c b/sys/arch/amd64/amd64/trap.c index c823e72701e..87a6c3d20a1 100644 --- a/sys/arch/amd64/amd64/trap.c +++ b/sys/arch/amd64/amd64/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.16 2004/02/19 17:18:38 drochner Exp $ */ +/* $NetBSD: trap.c,v 1.17 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 1998, 2000 The NetBSD Foundation, Inc. @@ -75,7 +75,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.16 2004/02/19 17:18:38 drochner Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.17 2004/03/14 01:08:47 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -437,8 +437,7 @@ copyfault: cr2 = rcr2(); KERNEL_PROC_LOCK(l); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)cr2; + l->l_savp->savp_faultaddr = (vaddr_t)cr2; l->l_flag |= L_SA_PAGEFAULT; } faultcommon: diff --git a/sys/arch/amiga/amiga/trap.c b/sys/arch/amiga/amiga/trap.c index 6da161d7629..b4a3cf0d6d3 100644 --- a/sys/arch/amiga/amiga/trap.c +++ b/sys/arch/amiga/amiga/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.99 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.100 2004/03/14 01:08:47 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990 The Regents of the University of California. @@ -83,7 +83,7 @@ #include "opt_fpu_emulate.h" #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.99 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.100 2004/03/14 01:08:47 cl Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -405,8 +405,7 @@ trapmmufault(type, code, v, fp, l, sticks) else { map = &vm->vm_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/arm/arm32/fault.c b/sys/arch/arm/arm32/fault.c index 2cf5d967586..474b4e21cbb 100644 --- a/sys/arch/arm/arm32/fault.c +++ b/sys/arch/arm/arm32/fault.c @@ -1,4 +1,4 @@ -/* $NetBSD: fault.c,v 1.48 2004/02/13 11:36:11 wiz Exp $ */ +/* $NetBSD: fault.c,v 1.49 2004/03/14 01:08:47 cl Exp $ */ /* * Copyright 2003 Wasabi Systems, Inc. @@ -81,7 +81,7 @@ #include "opt_kgdb.h" #include <sys/types.h> -__KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.48 2004/02/13 11:36:11 wiz Exp $"); +__KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.49 2004/03/14 01:08:47 cl Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -349,8 +349,7 @@ data_abort_handler(trapframe_t *tf) } else { map = &l->l_proc->p_vmspace->vm_map; if (l->l_flag & L_SA) { - KDASSERT(l->l_proc->p_sa != NULL); - l->l_proc->p_sa->sa_vp_faultaddr = (vaddr_t)far; + l->l_savp->savp_faultaddr = (vaddr_t)far; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/atari/atari/trap.c b/sys/arch/atari/atari/trap.c index cf54393c410..51337210a97 100644 --- a/sys/arch/atari/atari/trap.c +++ b/sys/arch/atari/atari/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.75 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.76 2004/03/14 01:08:47 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990 The Regents of the University of California. @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.75 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.76 2004/03/14 01:08:47 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -655,8 +655,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/cesfic/cesfic/trap.c b/sys/arch/cesfic/cesfic/trap.c index 2b317705838..29f3933d8de 100644 --- a/sys/arch/cesfic/cesfic/trap.c +++ b/sys/arch/cesfic/cesfic/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.18 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.19 2004/03/14 01:08:47 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.18 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.19 2004/03/14 01:08:47 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -653,8 +653,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/hp300/hp300/trap.c b/sys/arch/hp300/hp300/trap.c index 323cea99186..cc536485c16 100644 --- a/sys/arch/hp300/hp300/trap.c +++ b/sys/arch/hp300/hp300/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.109 2003/11/17 14:37:59 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.110 2004/03/14 01:08:47 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.109 2003/11/17 14:37:59 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.110 2004/03/14 01:08:47 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -639,8 +639,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/hppa/hppa/trap.c b/sys/arch/hppa/hppa/trap.c index 855d3375688..ba4fcd48b9e 100644 --- a/sys/arch/hppa/hppa/trap.c +++ b/sys/arch/hppa/hppa/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.14 2003/11/28 19:02:25 chs Exp $ */ +/* $NetBSD: trap.c,v 1.15 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 2001, 2002 The NetBSD Foundation, Inc. @@ -69,7 +69,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.14 2003/11/28 19:02:25 chs Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.15 2004/03/14 01:08:47 cl Exp $"); /* #define INTRDEBUG */ /* #define TRAPDEBUG */ @@ -723,8 +723,7 @@ trap(int type, struct trapframe *frame) else { map = &vm->vm_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = va; + l->l_savp->savp_faultaddr = va; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/i386/i386/trap.c b/sys/arch/i386/i386/trap.c index 4d300bfaeef..d46a1a761a0 100644 --- a/sys/arch/i386/i386/trap.c +++ b/sys/arch/i386/i386/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.199 2004/03/11 11:42:04 yamt Exp $ */ +/* $NetBSD: trap.c,v 1.200 2004/03/14 01:08:48 cl Exp $ */ /*- * Copyright (c) 1998, 2000 The NetBSD Foundation, Inc. @@ -75,7 +75,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.199 2004/03/11 11:42:04 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.200 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -555,8 +555,7 @@ copyfault: cr2 = rcr2(); KERNEL_PROC_LOCK(l); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)cr2; + l->l_savp->savp_faultaddr = (vaddr_t)cr2; l->l_flag |= L_SA_PAGEFAULT; } faultcommon: diff --git a/sys/arch/luna68k/luna68k/trap.c b/sys/arch/luna68k/luna68k/trap.c index dda3b93592c..87aec2068aa 100644 --- a/sys/arch/luna68k/luna68k/trap.c +++ b/sys/arch/luna68k/luna68k/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.29 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.30 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -78,7 +78,7 @@ #include <sys/cdefs.h> /* RCS ID & Copyright macro defns */ -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.29 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.30 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -575,8 +575,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/mac68k/mac68k/trap.c b/sys/arch/mac68k/mac68k/trap.c index df8827a9663..784d3609a81 100644 --- a/sys/arch/mac68k/mac68k/trap.c +++ b/sys/arch/mac68k/mac68k/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.110 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.111 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.110 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.111 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -627,8 +627,7 @@ copyfault: else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/mips/mips/trap.c b/sys/arch/mips/mips/trap.c index 297289630e7..772fc7b3556 100644 --- a/sys/arch/mips/mips/trap.c +++ b/sys/arch/mips/mips/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.191 2004/02/13 11:36:15 wiz Exp $ */ +/* $NetBSD: trap.c,v 1.192 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1992, 1993 @@ -78,7 +78,7 @@ #include <sys/cdefs.h> /* RCS ID & Copyright macro defns */ -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.191 2004/02/13 11:36:15 wiz Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.192 2004/03/14 01:08:48 cl Exp $"); #include "opt_cputype.h" /* which mips CPU levels do we support? */ #include "opt_ktrace.h" @@ -374,8 +374,7 @@ trap(unsigned status, unsigned cause, unsigned vaddr, unsigned opc, va = trunc_page(vaddr); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)vaddr; + l->l_savp->savp_faultaddr = (vaddr_t)vaddr; l->l_flag |= L_SA_PAGEFAULT; } diff --git a/sys/arch/mvme68k/mvme68k/trap.c b/sys/arch/mvme68k/mvme68k/trap.c index 564c5037ed6..41c2e7cbcc0 100644 --- a/sys/arch/mvme68k/mvme68k/trap.c +++ b/sys/arch/mvme68k/mvme68k/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.73 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.74 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.73 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.74 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -648,8 +648,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/news68k/news68k/trap.c b/sys/arch/news68k/news68k/trap.c index 5aa73e02a10..155fe7efee4 100644 --- a/sys/arch/news68k/news68k/trap.c +++ b/sys/arch/news68k/news68k/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.35 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.36 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.35 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.36 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -600,8 +600,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/next68k/next68k/trap.c b/sys/arch/next68k/next68k/trap.c index 40132d75935..b6e31b60c24 100644 --- a/sys/arch/next68k/next68k/trap.c +++ b/sys/arch/next68k/next68k/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.49 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.50 2004/03/14 01:08:48 cl Exp $ */ /* * This file was taken from mvme68k/mvme68k/trap.c @@ -84,7 +84,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.49 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.50 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -654,8 +654,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/pc532/pc532/trap.c b/sys/arch/pc532/pc532/trap.c index 0104efd989d..d96c335d000 100644 --- a/sys/arch/pc532/pc532/trap.c +++ b/sys/arch/pc532/pc532/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.63 2004/01/23 04:12:39 simonb Exp $ */ +/* $NetBSD: trap.c,v 1.64 2004/03/14 01:08:48 cl Exp $ */ /*- * Copyright (c) 1990 The Regents of the University of California. @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.63 2004/01/23 04:12:39 simonb Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.64 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -449,8 +449,7 @@ trap(frame) else { map = &vm->vm_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = + l->l_savp->savp_faultaddr = (vaddr_t)frame.tf_tear; l->l_flag |= L_SA_PAGEFAULT; } diff --git a/sys/arch/powerpc/ibm4xx/trap.c b/sys/arch/powerpc/ibm4xx/trap.c index 3ed9bec4556..1ef40550b5c 100644 --- a/sys/arch/powerpc/ibm4xx/trap.c +++ b/sys/arch/powerpc/ibm4xx/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.21 2004/02/24 18:31:46 drochner Exp $ */ +/* $NetBSD: trap.c,v 1.22 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright 2001 Wasabi Systems, Inc. @@ -67,7 +67,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.21 2004/02/24 18:31:46 drochner Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.22 2004/03/14 01:08:48 cl Exp $"); #include "opt_altivec.h" #include "opt_ddb.h" @@ -193,8 +193,7 @@ trap(struct trapframe *frame) } else { map = &p->p_vmspace->vm_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = va; + l->l_savp->savp_faultaddr = va; l->l_flag |= L_SA_PAGEFAULT; } } @@ -242,8 +241,7 @@ trap(struct trapframe *frame) frame->dar, frame->tf_xtra[TF_ESR])); KASSERT(l == curlwp && (l->l_stat == LSONPROC)); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)frame->dar; + l->l_savp->savp_faultaddr = (vaddr_t)frame->dar; l->l_flag |= L_SA_PAGEFAULT; } rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(frame->dar), @@ -274,8 +272,7 @@ trap(struct trapframe *frame) case EXC_ISI|EXC_USER: KERNEL_PROC_LOCK(l); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)frame->srr0; + l->l_savp->savp_faultaddr = (vaddr_t)frame->srr0; l->l_flag |= L_SA_PAGEFAULT; } ftype = VM_PROT_EXECUTE; diff --git a/sys/arch/powerpc/powerpc/trap.c b/sys/arch/powerpc/powerpc/trap.c index 7766f80ac21..caacc124531 100644 --- a/sys/arch/powerpc/powerpc/trap.c +++ b/sys/arch/powerpc/powerpc/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.97 2004/02/24 18:25:27 drochner Exp $ */ +/* $NetBSD: trap.c,v 1.98 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (C) 1995, 1996 Wolfgang Solfrank. @@ -32,7 +32,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.97 2004/02/24 18:25:27 drochner Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.98 2004/03/14 01:08:48 cl Exp $"); #include "opt_altivec.h" #include "opt_ddb.h" @@ -151,8 +151,7 @@ trap(struct trapframe *frame) return; } if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = va; + l->l_savp->savp_faultaddr = va; l->l_flag |= L_SA_PAGEFAULT; } } else { @@ -238,8 +237,7 @@ trap(struct trapframe *frame) } if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)frame->dar; + l->l_savp->savp_faultaddr = (vaddr_t)frame->dar; l->l_flag |= L_SA_PAGEFAULT; } rv = uvm_fault(map, trunc_page(frame->dar), 0, ftype); @@ -313,8 +311,7 @@ trap(struct trapframe *frame) } if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)frame->srr0; + l->l_savp->savp_faultaddr = (vaddr_t)frame->srr0; l->l_flag |= L_SA_PAGEFAULT; } ftype = VM_PROT_EXECUTE; diff --git a/sys/arch/sh3/sh3/exception.c b/sys/arch/sh3/sh3/exception.c index 950e9a8e20a..9d5fa1443e5 100644 --- a/sys/arch/sh3/sh3/exception.c +++ b/sys/arch/sh3/sh3/exception.c @@ -1,4 +1,4 @@ -/* $NetBSD: exception.c,v 1.17 2003/11/24 03:11:16 uwe Exp $ */ +/* $NetBSD: exception.c,v 1.18 2004/03/14 01:08:48 cl Exp $ */ /*- * Copyright (c) 1990 The Regents of the University of California. @@ -79,7 +79,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: exception.c,v 1.17 2003/11/24 03:11:16 uwe Exp $"); +__KERNEL_RCSID(0, "$NetBSD: exception.c,v 1.18 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -508,8 +508,7 @@ do { \ } if ((map != kernel_map) && (l->l_flag & L_SA)) { - KDASSERT(l->l_proc != NULL && l->l_proc->p_sa != NULL); - l->l_proc->p_sa->sa_vp_faultaddr = (vaddr_t)va; + l->l_savp->savp_faultaddr = (vaddr_t)va; l->l_flag |= L_SA_PAGEFAULT; } diff --git a/sys/arch/sh5/sh5/trap.c b/sys/arch/sh5/sh5/trap.c index e66d16e53df..2554134a063 100644 --- a/sys/arch/sh5/sh5/trap.c +++ b/sys/arch/sh5/sh5/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.31 2003/10/31 16:44:35 cl Exp $ */ +/* $NetBSD: trap.c,v 1.32 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright 2002 Wasabi Systems, Inc. @@ -111,7 +111,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.31 2003/10/31 16:44:35 cl Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.32 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" @@ -307,8 +307,7 @@ trap(struct lwp *l, struct trapframe *tf) map = &vm->vm_map; va = trunc_page(vaddr); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)vaddr; + l->l_savp->savp_faultaddr = (vaddr_t)vaddr; l->l_flag |= L_SA_PAGEFAULT; } rv = uvm_fault(map, va, 0, ftype); diff --git a/sys/arch/sparc/sparc/trap.c b/sys/arch/sparc/sparc/trap.c index 12d9f07cad1..81ba03f930b 100644 --- a/sys/arch/sparc/sparc/trap.c +++ b/sys/arch/sparc/sparc/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.152 2004/02/13 11:36:18 wiz Exp $ */ +/* $NetBSD: trap.c,v 1.153 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1996 @@ -49,7 +49,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.152 2004/02/13 11:36:18 wiz Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.153 2004/03/14 01:08:48 cl Exp $"); #include "opt_ddb.h" #include "opt_ktrace.h" @@ -976,8 +976,7 @@ mem_access_fault(type, ser, v, pc, psr, tf) } else { l->l_md.md_tf = tf; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } @@ -1295,8 +1294,7 @@ mem_access_fault4m(type, sfsr, sfva, tf) } else { l->l_md.md_tf = tf; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)sfva; + l->l_savp->savp_faultaddr = (vaddr_t)sfva; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/sparc64/sparc64/trap.c b/sys/arch/sparc64/sparc64/trap.c index 8138fa9eaed..80d3c524297 100644 --- a/sys/arch/sparc64/sparc64/trap.c +++ b/sys/arch/sparc64/sparc64/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.115 2004/03/12 13:27:03 drochner Exp $ */ +/* $NetBSD: trap.c,v 1.116 2004/03/14 01:08:48 cl Exp $ */ /* * Copyright (c) 1996-2002 Eduardo Horvath. All rights reserved. @@ -50,7 +50,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.115 2004/03/12 13:27:03 drochner Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.116 2004/03/14 01:08:48 cl Exp $"); #define NEW_FPSTATE @@ -1184,8 +1184,7 @@ data_access_fault(tf, type, pc, addr, sfva, sfsr) } else { l->l_md.md_tf = tf; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = addr; + l->l_savp->savp_faultaddr = addr; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/sun2/sun2/trap.c b/sys/arch/sun2/sun2/trap.c index e368faa5888..02a4f9097f9 100644 --- a/sys/arch/sun2/sun2/trap.c +++ b/sys/arch/sun2/sun2/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.16 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.17 2004/03/14 01:08:49 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -78,7 +78,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.16 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.17 2004/03/14 01:08:49 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -528,8 +528,7 @@ trap(type, code, v, tf) if ((l->l_addr->u_pcb.pcb_onfault == NULL) || KDFAULT(code)) map = kernel_map; } else if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } diff --git a/sys/arch/sun3/sun3/trap.c b/sys/arch/sun3/sun3/trap.c index 86d03db69dc..abd4ac7ac75 100644 --- a/sys/arch/sun3/sun3/trap.c +++ b/sys/arch/sun3/sun3/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.115 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.116 2004/03/14 01:08:49 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -78,7 +78,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.115 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.116 2004/03/14 01:08:49 cl Exp $"); #include "opt_ddb.h" #include "opt_execfmt.h" @@ -539,8 +539,7 @@ trap(type, code, v, tf) if ((l->l_addr->u_pcb.pcb_onfault == NULL) || KDFAULT(code)) map = kernel_map; } else if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } diff --git a/sys/arch/vax/vax/trap.c b/sys/arch/vax/vax/trap.c index 6338121fda6..8aac7306b4c 100644 --- a/sys/arch/vax/vax/trap.c +++ b/sys/arch/vax/vax/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.92 2004/02/13 18:27:14 drochner Exp $ */ +/* $NetBSD: trap.c,v 1.93 2004/03/14 01:08:49 cl Exp $ */ /* * Copyright (c) 1994 Ludd, University of Lule}, Sweden. @@ -33,7 +33,7 @@ /* All bugs are subject to removal without further notice */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.92 2004/02/13 18:27:14 drochner Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.93 2004/03/14 01:08:49 cl Exp $"); #include "opt_ddb.h" #include "opt_ktrace.h" @@ -271,8 +271,7 @@ if(faultdebug)printf("trap accflt type %lx, code %lx, pc %lx, psl %lx\n", if (umode) { KERNEL_PROC_LOCK(l); if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)frame->code; + l->l_savp->savp_faultaddr = (vaddr_t)frame->code; l->l_flag |= L_SA_PAGEFAULT; } } else diff --git a/sys/arch/x68k/x68k/trap.c b/sys/arch/x68k/x68k/trap.c index 4054617d385..6f34c67a257 100644 --- a/sys/arch/x68k/x68k/trap.c +++ b/sys/arch/x68k/x68k/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.71 2003/11/08 12:17:25 tsutsui Exp $ */ +/* $NetBSD: trap.c,v 1.72 2004/03/14 01:08:49 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1990, 1993 @@ -77,7 +77,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.71 2003/11/08 12:17:25 tsutsui Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.72 2004/03/14 01:08:49 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -691,8 +691,7 @@ trap(type, code, v, frame) else { map = vm ? &vm->vm_map : kernel_map; if (l->l_flag & L_SA) { - KDASSERT(p != NULL && p->p_sa != NULL); - p->p_sa->sa_vp_faultaddr = (vaddr_t)v; + l->l_savp->savp_faultaddr = (vaddr_t)v; l->l_flag |= L_SA_PAGEFAULT; } } diff --git a/sys/arch/xen/i386/trap.c b/sys/arch/xen/i386/trap.c index 4a69d5d60d0..b1bf421cbd2 100644 --- a/sys/arch/xen/i386/trap.c +++ b/sys/arch/xen/i386/trap.c @@ -1,4 +1,4 @@ -/* $NetBSD: trap.c,v 1.1 2004/03/11 21:44:08 cl Exp $ */ +/* $NetBSD: trap.c,v 1.2 2004/03/14 01:08:50 cl Exp $ */ /* NetBSD: trap.c,v 1.199 2004/03/11 11:42:04 yamt Exp */ /*- @@ -76,7 +76,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.1 2004/03/11 21:44:08 cl Exp $"); +__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.2 2004/03/14 01:08:50 cl Exp $"); #include "opt_ddb.h" #include "opt_kgdb.h" @@ -557,7 +557,7 @@ copyfault: cr2 = ((uint32_t *)(void *)&frame)[1]; KERNEL_PROC_LOCK(l); if (l->l_flag & L_SA) { - p->p_sa->sa_vp_faultaddr = (vaddr_t)cr2; + l->l_savp->savp_faultaddr = (vaddr_t)cr2; l->l_flag |= L_SA_PAGEFAULT; } faultcommon: diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index a6eb5541d7f..30f65d08819 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_exit.c,v 1.138 2004/03/05 07:27:22 dbj Exp $ */ +/* $NetBSD: kern_exit.c,v 1.139 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 1998, 1999 The NetBSD Foundation, Inc. @@ -74,7 +74,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_exit.c,v 1.138 2004/03/05 07:27:22 dbj Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_exit.c,v 1.139 2004/03/14 01:08:47 cl Exp $"); #include "opt_ktrace.h" #include "opt_perfctrs.h" @@ -532,6 +532,7 @@ exit_lwps(struct lwp *l) { struct proc *p; struct lwp *l2; + struct sadata_vp *vp; int s, error; lwpid_t waited; @@ -545,23 +546,28 @@ exit_lwps(struct lwp *l) p->p_userret_arg = NULL; if (p->p_sa) { - /* - * Make SA-cached LWPs normal process runnable LWPs so - * that they'll also self-destruct. - */ - DPRINTF(("exit_lwps: Making cached LWPs of %d runnable: ", - p->p_pid)); - SCHED_LOCK(s); - while ((l2 = sa_getcachelwp(p)) != 0) { - l2->l_priority = l2->l_usrpri; - setrunnable(l2); - DPRINTF(("%d ", l2->l_lid)); - } - DPRINTF(("\n")); - SCHED_UNLOCK(s); + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + /* + * Make SA-cached LWPs normal process runnable + * LWPs so that they'll also self-destruct. + */ + DPRINTF(("exit_lwps: Making cached LWPs of %d on VP %d runnable: ", + p->p_pid, vp->savp_id)); + SCHED_LOCK(s); + while ((l2 = sa_getcachelwp(vp)) != 0) { + l2->l_priority = l2->l_usrpri; + setrunnable(l2); + DPRINTF(("%d ", l2->l_lid)); + } + SCHED_UNLOCK(s); + DPRINTF(("\n")); - /* Clear wokenq, the LWPs on the queue will run below */ - p->p_sa->sa_wokenq_head = NULL; + /* + * Clear wokenq, the LWPs on the queue will + * run below. + */ + vp->savp_wokenq_head = NULL; + } } /* @@ -572,9 +578,6 @@ exit_lwps(struct lwp *l) LIST_FOREACH(l2, &p->p_lwps, l_sibling) { l2->l_flag &= ~(L_DETACHED|L_SA); - if (p->p_sa && l2->l_wchan == p->p_sa) - l2->l_flag |= L_SINTR; - if ((l2->l_stat == LSSLEEP && (l2->l_flag & L_SINTR)) || l2->l_stat == LSSUSPENDED || l2->l_stat == LSSTOP) { SCHED_LOCK(s); diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 0614bf3828e..18eac8c8adf 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_proc.c,v 1.74 2004/02/27 02:43:25 junyoung Exp $ */ +/* $NetBSD: kern_proc.c,v 1.75 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 1999 The NetBSD Foundation, Inc. @@ -69,7 +69,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.74 2004/02/27 02:43:25 junyoung Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.75 2004/03/14 01:08:47 cl Exp $"); #include "opt_kstack.h" @@ -178,6 +178,7 @@ struct pool ras_pool; struct pool sadata_pool; struct pool saupcall_pool; struct pool sastack_pool; +struct pool savp_pool; struct pool ptimer_pool; MALLOC_DEFINE(M_EMULDATA, "emuldata", "Per-process emulation data"); @@ -260,6 +261,8 @@ procinit(void) "saupcpl", &pool_allocator_nointr); pool_init(&sastack_pool, sizeof(struct sastack), 0, 0, 0, "sastackpl", &pool_allocator_nointr); + pool_init(&savp_pool, sizeof(struct sadata_vp), 0, 0, 0, "savppl", + &pool_allocator_nointr); pool_init(&ptimer_pool, sizeof(struct ptimer), 0, 0, 0, "ptimerpl", &pool_allocator_nointr); } diff --git a/sys/kern/kern_sa.c b/sys/kern/kern_sa.c index f93466650b4..301a647a253 100644 --- a/sys/kern/kern_sa.c +++ b/sys/kern/kern_sa.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_sa.c,v 1.48 2004/03/14 00:45:21 cl Exp $ */ +/* $NetBSD: kern_sa.c,v 1.49 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.48 2004/03/14 00:45:21 cl Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.49 2004/03/14 01:08:47 cl Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -53,11 +53,15 @@ __KERNEL_RCSID(0, "$NetBSD: kern_sa.c,v 1.48 2004/03/14 00:45:21 cl Exp $"); #include <uvm/uvm_extern.h> +static struct sadata_vp *sa_newsavp(struct sadata *); static __inline int sa_stackused(struct sastack *, struct sadata *); static __inline void sa_setstackfree(struct sastack *, struct sadata *); static struct sastack *sa_getstack(struct sadata *); static __inline struct sastack *sa_getstack0(struct sadata *); static __inline int sast_compare(struct sastack *, struct sastack *); +#ifdef MULTIPROCESSOR +static int sa_increaseconcurrency(struct lwp *, int); +#endif static void sa_setwoken(struct lwp *); static void sa_switchcall(void *); static int sa_newcachelwp(struct lwp *); @@ -141,6 +145,45 @@ sadata_upcall_free(struct sadata_upcall *sau) pool_put(&saupcall_pool, sau); } +static struct sadata_vp * +sa_newsavp(struct sadata *sa) +{ + struct sadata_vp *vp, *qvp; + + /* Allocate virtual processor data structure */ + vp = pool_get(&savp_pool, PR_WAITOK); + /* Initialize. */ + memset(vp, 0, sizeof(*vp)); + simple_lock_init(&vp->savp_lock); + vp->savp_lwp = NULL; + vp->savp_wokenq_head = NULL; + vp->savp_faultaddr = 0; + vp->savp_ofaultaddr = 0; + LIST_INIT(&vp->savp_lwpcache); + vp->savp_ncached = 0; + SIMPLEQ_INIT(&vp->savp_upcalls); + + simple_lock(&sa->sa_lock); + /* find first free savp_id and add vp to sorted slist */ + if (SLIST_EMPTY(&sa->sa_vps) || + SLIST_FIRST(&sa->sa_vps)->savp_id != 0) { + vp->savp_id = 0; + SLIST_INSERT_HEAD(&sa->sa_vps, vp, savp_next); + } else { + SLIST_FOREACH(qvp, &sa->sa_vps, savp_next) { + if (SLIST_NEXT(qvp, savp_next) == NULL || + SLIST_NEXT(qvp, savp_next)->savp_id != + qvp->savp_id + 1) + break; + } + vp->savp_id = qvp->savp_id + 1; + SLIST_INSERT_AFTER(qvp, vp, savp_next); + } + simple_unlock(&sa->sa_lock); + + return (vp); +} + int sys_sa_register(struct lwp *l, void *v, register_t *retval) { @@ -162,8 +205,7 @@ sys_sa_register(struct lwp *l, void *v, register_t *retval) memset(sa, 0, sizeof(*sa)); simple_lock_init(&sa->sa_lock); sa->sa_flag = SCARG(uap, flags) & SA_FLAG_ALL; - sa->sa_vp = NULL; - sa->sa_wokenq_head = NULL; + sa->sa_maxconcurrency = 1; sa->sa_concurrency = 1; SPLAY_INIT(&sa->sa_stackstree); sa->sa_stacknext = NULL; @@ -172,11 +214,11 @@ sys_sa_register(struct lwp *l, void *v, register_t *retval) else sa->sa_stackinfo_offset = 0; sa->sa_nstacks = 0; - sa->sa_vp_faultaddr = 0; - sa->sa_vp_ofaultaddr = 0; - LIST_INIT(&sa->sa_lwpcache); - SIMPLEQ_INIT(&sa->sa_upcalls); + SLIST_INIT(&sa->sa_vps); p->p_sa = sa; + } + if (l->l_savp == NULL) { + l->l_savp = sa_newsavp(p->p_sa); sa_newcachelwp(l); } @@ -197,6 +239,7 @@ sa_release(struct proc *p) { struct sadata *sa; struct sastack *sast, *next; + struct sadata_vp *vp; sa = p->p_sa; KDASSERT(sa != NULL); @@ -209,6 +252,10 @@ sa_release(struct proc *p) } p->p_flag &= ~P_SA; + while ((vp = SLIST_FIRST(&p->p_sa->sa_vps)) != NULL) { + SLIST_REMOVE_HEAD(&p->p_sa->sa_vps, savp_next); + pool_put(&savp_pool, vp); + } pool_put(&sadata_pool, sa); p->p_sa = NULL; } @@ -375,13 +422,14 @@ sys_sa_enable(struct lwp *l, void *v, register_t *retval) { struct proc *p = l->l_proc; struct sadata *sa = p->p_sa; + struct sadata_vp *vp = l->l_savp; int error; DPRINTF(("sys_sa_enable(%d.%d)\n", l->l_proc->p_pid, l->l_lid)); /* We have to be using scheduler activations */ - if (sa == NULL) + if (sa == NULL || vp == NULL) return (EINVAL); if (p->p_flag & P_SA) /* Already running! */ @@ -392,7 +440,7 @@ sys_sa_enable(struct lwp *l, void *v, register_t *retval) return (error); /* Assign this LWP to the virtual processor */ - sa->sa_vp = l; + vp->savp_lwp = l; p->p_flag |= P_SA; l->l_flag |= L_SA; /* We are now an activation LWP */ @@ -402,6 +450,80 @@ sys_sa_enable(struct lwp *l, void *v, register_t *retval) } +#ifdef MULTIPROCESSOR +static int +sa_increaseconcurrency(struct lwp *l, int concurrency) +{ + struct proc *p; + struct lwp *l2; + struct sadata *sa; + vaddr_t uaddr; + boolean_t inmem; + int addedconcurrency, error, s; + + p = l->l_proc; + sa = p->p_sa; + + addedconcurrency = 0; + simple_lock(&sa->sa_lock); + while (sa->sa_maxconcurrency < concurrency) { + sa->sa_maxconcurrency++; + sa->sa_concurrency++; + simple_unlock(&sa->sa_lock); + + inmem = uvm_uarea_alloc(&uaddr); + if (__predict_false(uaddr == 0)) { + /* reset concurrency */ + simple_lock(&sa->sa_lock); + sa->sa_maxconcurrency--; + sa->sa_concurrency--; + simple_unlock(&sa->sa_lock); + return (addedconcurrency); + } else { + newlwp(l, p, uaddr, inmem, 0, NULL, 0, + child_return, 0, &l2); + l2->l_flag |= L_SA; + l2->l_savp = sa_newsavp(sa); + if (l2->l_savp) { + l2->l_savp->savp_lwp = l2; + cpu_setfunc(l2, sa_switchcall, NULL); + error = sa_upcall(l2, SA_UPCALL_NEWPROC, + NULL, NULL, 0, NULL); + if (error) { + /* free new savp */ + SLIST_REMOVE(&sa->sa_vps, l2->l_savp, + sadata_vp, savp_next); + pool_put(&savp_pool, l2->l_savp); + } + } else + error = 1; + if (error) { + /* put l2 into l's LWP cache */ + l2->l_savp = l->l_savp; + PHOLD(l2); + SCHED_LOCK(s); + sa_putcachelwp(p, l2); + SCHED_UNLOCK(s); + /* reset concurrency */ + simple_lock(&sa->sa_lock); + sa->sa_maxconcurrency--; + sa->sa_concurrency--; + simple_unlock(&sa->sa_lock); + return (addedconcurrency); + } + SCHED_LOCK(s); + setrunnable(l2); + SCHED_UNLOCK(s); + addedconcurrency++; + } + simple_lock(&sa->sa_lock); + } + simple_unlock(&sa->sa_lock); + + return (addedconcurrency); +} +#endif + int sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval) { @@ -409,9 +531,15 @@ sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval) syscallarg(int) concurrency; } */ *uap = v; struct sadata *sa = l->l_proc->p_sa; +#ifdef MULTIPROCESSOR + struct sadata_vp *vp = l->l_savp; + int ncpus, s; + struct cpu_info *ci; + CPU_INFO_ITERATOR cii; +#endif - DPRINTF(("sys_sa_concurrency(%d.%d)\n", l->l_proc->p_pid, - l->l_lid)); + DPRINTFN(11,("sys_sa_concurrency(%d.%d)\n", l->l_proc->p_pid, + l->l_lid)); /* We have to be using scheduler activations */ if (sa == NULL) @@ -420,14 +548,51 @@ sys_sa_setconcurrency(struct lwp *l, void *v, register_t *retval) if (SCARG(uap, concurrency) < 1) return (EINVAL); - *retval = sa->sa_concurrency; + *retval = 0; /* * Concurrency greater than the number of physical CPUs does * not make sense. * XXX Should we ever support hot-plug CPUs, this will need * adjustment. */ - sa->sa_concurrency = min(SCARG(uap, concurrency), 1 /* XXX ncpus */); +#ifdef MULTIPROCESSOR + if (SCARG(uap, concurrency) > sa->sa_maxconcurrency) { + ncpus = 0; + for (CPU_INFO_FOREACH(cii, ci)) + ncpus++; + *retval += sa_increaseconcurrency(l, + min(SCARG(uap, concurrency), ncpus)); + } +#endif + + DPRINTFN(11,("sys_sa_concurrency(%d.%d) want %d, have %d, max %d\n", + l->l_proc->p_pid, l->l_lid, SCARG(uap, concurrency), + sa->sa_concurrency, sa->sa_maxconcurrency)); +#ifdef MULTIPROCESSOR + if (SCARG(uap, concurrency) > sa->sa_concurrency) { + SCHED_LOCK(s); + SLIST_FOREACH(vp, &sa->sa_vps, savp_next) { + if (vp->savp_lwp->l_flag & L_SA_IDLE) { + vp->savp_lwp->l_flag &= + ~(L_SA_IDLE|L_SA_YIELD|L_SINTR); + SCHED_UNLOCK(s); + DPRINTFN(11,("sys_sa_concurrency(%d.%d) NEWPROC vp %d\n", + l->l_proc->p_pid, l->l_lid, vp->savp_id)); + cpu_setfunc(vp->savp_lwp, sa_switchcall, NULL); + /* error = */ sa_upcall(vp->savp_lwp, SA_UPCALL_NEWPROC, + NULL, NULL, 0, NULL); + SCHED_LOCK(s); + sa->sa_concurrency++; + setrunnable(vp->savp_lwp); + KDASSERT((vp->savp_lwp->l_flag & L_SINTR) == 0); + (*retval)++; + } + if (sa->sa_concurrency == SCARG(uap, concurrency)) + break; + } + SCHED_UNLOCK(s); + } +#endif return (0); } @@ -453,13 +618,14 @@ sa_yield(struct lwp *l) { struct proc *p = l->l_proc; struct sadata *sa = p->p_sa; - int ret, s; + struct sadata_vp *vp = l->l_savp; + int ret; #if defined(MULTIPROCESSOR) KDASSERT(l->l_flag & L_BIGLOCK); #endif - if (sa->sa_vp != l) { + if (vp->savp_lwp != l) { /* * We lost the VP on our way here, this happens for * instance when we sleep in systrace. This will end @@ -482,27 +648,44 @@ sa_yield(struct lwp *l) * A signal will probably wake us up. Worst case, the upcall * happens and just causes the process to yield again. */ - s = splsched(); /* Protect from timer expirations */ - KDASSERT(sa->sa_vp == l); + /* s = splsched(); */ /* Protect from timer expirations */ + KDASSERT(vp->savp_lwp == l); /* * If we were told to make an upcall or exit before * the splsched(), make sure we process it instead of * going to sleep. It might make more sense for this to * be handled inside of tsleep.... */ - ret = 0; - while (ret == 0 && p->p_userret == NULL && - (l->l_flag & L_SA_UPCALL) == 0) { - l->l_flag |= L_SA_YIELD; + ret = 0; + l->l_flag |= L_SA_YIELD; + if (l->l_flag & L_SA_UPCALL) { + /* KERNEL_PROC_UNLOCK(l); in upcallret() */ + upcallret(l); + KERNEL_PROC_LOCK(l); + } + while (l->l_flag & L_SA_YIELD) { + DPRINTFN(1,("sa_yield(%d.%d) really going dormant\n", + p->p_pid, l->l_lid)); + + simple_lock(&sa->sa_lock); + sa->sa_concurrency--; + simple_unlock(&sa->sa_lock); + ret = tsleep((caddr_t) l, PUSER | PCATCH, "sawait", 0); - l->l_flag &= ~L_SA_YIELD; - if (p->p_flag & P_WEXIT) - lwp_exit(l); - KDASSERT(sa->sa_vp == l); + + simple_lock(&sa->sa_lock); + sa->sa_concurrency++; + simple_unlock(&sa->sa_lock); + + KDASSERT(vp->savp_lwp == l || p->p_flag & P_WEXIT); + + /* KERNEL_PROC_UNLOCK(l); in upcallret() */ + upcallret(l); + KERNEL_PROC_LOCK(l); } - splx(s); - DPRINTFN(1,("sa_yield(%d.%d) returned\n", - p->p_pid, l->l_lid)); + /* splx(s); */ + DPRINTFN(1,("sa_yield(%d.%d) returned, ret %d, userret %p\n", + p->p_pid, l->l_lid, ret, p->p_userret)); } @@ -545,24 +728,23 @@ sa_upcall(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted, { struct sadata_upcall *sau; struct sadata *sa = l->l_proc->p_sa; + struct sadata_vp *vp = l->l_savp; struct sastack *sast; int error, f; /* XXX prevent recursive upcalls if we sleep for memory */ SA_LWP_STATE_LOCK(l, f); sast = sa_getstack(sa); - sau = sadata_upcall_alloc(1); SA_LWP_STATE_UNLOCK(l, f); if (sast == NULL) { - /* assign to assure that it gets freed */ - sau->sau_type = type & SA_UPCALL_TYPE_MASK; - sau->sau_arg = arg; - sadata_upcall_free(sau); return (ENOMEM); } DPRINTFN(9,("sa_upcall(%d.%d) using stack %p\n", l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp)); + SA_LWP_STATE_LOCK(l, f); + sau = sadata_upcall_alloc(1); + SA_LWP_STATE_UNLOCK(l, f); error = sa_upcall0(l, type, event, interrupted, argsize, arg, sau); if (error) { sadata_upcall_free(sau); @@ -571,7 +753,7 @@ sa_upcall(struct lwp *l, int type, struct lwp *event, struct lwp *interrupted, } sau->sau_stack = sast->sast_stack; - SIMPLEQ_INSERT_TAIL(&sa->sa_upcalls, sau, sau_next); + SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); l->l_flag |= L_SA_UPCALL; return (0); @@ -618,7 +800,7 @@ sa_upcall_getstate(union sau_state *ss, struct lwp *l) #endif ); ss->ss_captured.ss_sa.sa_id = l->l_lid; - ss->ss_captured.ss_sa.sa_cpu = 0; /* XXX extract from l_cpu */ + ss->ss_captured.ss_sa.sa_cpu = l->l_savp->savp_id; } else ss->ss_captured.ss_sa.sa_context = NULL; } @@ -636,14 +818,16 @@ sa_pagefault(struct lwp *l, ucontext_t *l_ctx) { struct proc *p; struct sadata *sa; + struct sadata_vp *vp; struct sastack sast; p = l->l_proc; sa = p->p_sa; + vp = l->l_savp; - KDASSERT(sa->sa_vp == l); + KDASSERT(vp->savp_lwp == l); - if (sa->sa_vp_faultaddr == sa->sa_vp_ofaultaddr) { + if (vp->savp_faultaddr == vp->savp_ofaultaddr) { DPRINTFN(10,("sa_pagefault(%d.%d) double page fault\n", p->p_pid, l->l_lid)); return 1; @@ -658,7 +842,7 @@ sa_pagefault(struct lwp *l, ucontext_t *l_ctx) return 1; } - sa->sa_vp_ofaultaddr = sa->sa_vp_faultaddr; + vp->savp_ofaultaddr = vp->savp_faultaddr; return 0; } @@ -674,13 +858,13 @@ void sa_switch(struct lwp *l, int type) { struct proc *p = l->l_proc; - struct sadata *sa = p->p_sa; + struct sadata_vp *vp = l->l_savp; struct sadata_upcall *sau; struct lwp *l2; int error, s; DPRINTFN(4,("sa_switch(%d.%d type %d VP %d)\n", p->p_pid, l->l_lid, - type, sa->sa_vp ? sa->sa_vp->l_lid : 0)); + type, vp->savp_lwp ? vp->savp_lwp->l_lid : 0)); SCHED_ASSERT_LOCKED(); if (p->p_flag & P_WEXIT) { @@ -692,7 +876,7 @@ sa_switch(struct lwp *l, int type) /* * Case 0: we're blocking in sa_yield */ - if (sa->sa_wokenq_head == NULL) { + if (vp->savp_wokenq_head == NULL && p->p_userret == NULL) { l->l_flag |= L_SA_IDLE; mi_switch(l, NULL); } else { @@ -704,7 +888,7 @@ sa_switch(struct lwp *l, int type) SCHED_UNLOCK(s); } return; - } else if (sa->sa_vp == l) { + } else if (vp->savp_lwp == l) { /* * Case 1: we're blocking for the first time; generate * a SA_BLOCKED upcall and allocate resources for the @@ -717,7 +901,7 @@ sa_switch(struct lwp *l, int type) * would be Bad. Therefore, we must use a cached new * LWP. The first thing that this new LWP must do is * allocate another LWP for the cache. */ - l2 = sa_getcachelwp(p); + l2 = sa_getcachelwp(vp); if (l2 == NULL) { /* XXXSMP */ /* No upcall for you! */ @@ -781,8 +965,8 @@ sa_switch(struct lwp *l, int type) mi_switch(l, NULL); DPRINTFN(10,("sa_switch(%d.%d) page fault resolved\n", p->p_pid, l->l_lid)); - if (sa->sa_vp_faultaddr == sa->sa_vp_ofaultaddr) - sa->sa_vp_ofaultaddr = -1; + if (vp->savp_faultaddr == vp->savp_ofaultaddr) + vp->savp_ofaultaddr = -1; return; } @@ -791,13 +975,13 @@ sa_switch(struct lwp *l, int type) l->l_flag |= L_SA_BLOCKING; l2->l_priority = l2->l_usrpri; - sa->sa_vp_blocker = l; - sa->sa_vp = l2; + vp->savp_blocker = l; + vp->savp_lwp = l2; setrunnable(l2); PRELE(l2); /* Remove the artificial hold-count */ KDASSERT(l2 != l); - } else if (sa->sa_vp != NULL) { + } else if (vp->savp_lwp != NULL) { /* * Case 2: We've been woken up while another LWP was * on the VP, but we're going back to sleep without @@ -808,10 +992,10 @@ sa_switch(struct lwp *l, int type) * go. If the LWP on the VP was idling, don't make it * run again, though. */ - if (sa->sa_vp->l_flag & L_SA_YIELD) + if (vp->savp_lwp->l_flag & L_SA_YIELD) l2 = NULL; else { - l2 = sa->sa_vp; /* XXXUPSXXX Unfair advantage for l2 ? */ + l2 = vp->savp_lwp; /* XXXUPSXXX Unfair advantage for l2 ? */ if((l2->l_stat != LSRUN) || ((l2->l_flag & L_INMEM) == 0)) l2 = NULL; } @@ -836,51 +1020,54 @@ sa_switchcall(void *arg) { struct lwp *l, *l2; struct proc *p; - struct sadata *sa; + struct sadata_vp *vp; struct sadata_upcall *sau; struct sastack *sast; int s; l2 = curlwp; p = l2->l_proc; - sa = p->p_sa; + vp = l2->l_savp; if (p->p_flag & P_WEXIT) lwp_exit(l2); - l = sa->sa_vp_blocker; - KDASSERT(sa->sa_vp == l2); + KDASSERT(vp->savp_lwp == l2); sau = arg; DPRINTFN(6,("sa_switchcall(%d.%d)\n", p->p_pid, l2->l_lid)); l2->l_flag &= ~L_SA; - if (LIST_EMPTY(&sa->sa_lwpcache)) { + if (LIST_EMPTY(&vp->savp_lwpcache)) { /* Allocate the next cache LWP */ DPRINTFN(6,("sa_switchcall(%d.%d) allocating LWP\n", p->p_pid, l2->l_lid)); sa_newcachelwp(l2); } - sast = sa_getstack(sa); - if (sast) { - sau->sau_stack = sast->sast_stack; - SIMPLEQ_INSERT_TAIL(&sa->sa_upcalls, sau, sau_next); - l2->l_flag |= L_SA|L_SA_UPCALL; - } else { + if (sau) { + l = vp->savp_blocker; + sast = sa_getstack(p->p_sa); + if (sast) { + sau->sau_stack = sast->sast_stack; + SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); + l2->l_flag |= L_SA_UPCALL; + } else { #ifdef DIAGNOSTIC - printf("sa_switchcall(%d.%d flag %x): Not enough stacks.\n", - p->p_pid, l->l_lid, l->l_flag); + printf("sa_switchcall(%d.%d flag %x): Not enough stacks.\n", + p->p_pid, l->l_lid, l->l_flag); #endif - sadata_upcall_free(sau); - PHOLD(l2); - SCHED_LOCK(s); - sa_putcachelwp(p, l2); /* sets L_SA */ - sa->sa_vp = l; - mi_switch(l2, NULL); - /* mostly NOTREACHED */ - SCHED_ASSERT_UNLOCKED(); - splx(s); + sadata_upcall_free(sau); + PHOLD(l2); + SCHED_LOCK(s); + sa_putcachelwp(p, l2); /* sets L_SA */ + vp->savp_lwp = l; + mi_switch(l2, NULL); + /* mostly NOTREACHED */ + SCHED_ASSERT_UNLOCKED(); + splx(s); + } } + l2->l_flag |= L_SA; upcallret(l2); } @@ -909,6 +1096,7 @@ sa_newcachelwp(struct lwp *l) */ PHOLD(l2); SCHED_LOCK(s); + l2->l_savp = l->l_savp; sa_putcachelwp(p, l2); SCHED_UNLOCK(s); } @@ -923,11 +1111,11 @@ sa_newcachelwp(struct lwp *l) void sa_putcachelwp(struct proc *p, struct lwp *l) { - struct sadata *sa; + struct sadata_vp *vp; SCHED_ASSERT_LOCKED(); - sa = p->p_sa; + vp = l->l_savp; LIST_REMOVE(l, l_sibling); p->p_nlwps--; @@ -936,8 +1124,8 @@ sa_putcachelwp(struct proc *p, struct lwp *l) /* XXX lock sadata */ DPRINTFN(5,("sa_putcachelwp(%d.%d) Adding LWP %d to cache\n", p->p_pid, curlwp->l_lid, l->l_lid)); - LIST_INSERT_HEAD(&sa->sa_lwpcache, l, l_sibling); - sa->sa_ncached++; + LIST_INSERT_HEAD(&vp->savp_lwpcache, l, l_sibling); + vp->savp_ncached++; /* XXX unlock */ } @@ -945,20 +1133,20 @@ sa_putcachelwp(struct proc *p, struct lwp *l) * Fetch a LWP from the cache. */ struct lwp * -sa_getcachelwp(struct proc *p) +sa_getcachelwp(struct sadata_vp *vp) { - struct sadata *sa; struct lwp *l; + struct proc *p; SCHED_ASSERT_LOCKED(); l = NULL; - sa = p->p_sa; /* XXX lock sadata */ - if (sa->sa_ncached > 0) { - sa->sa_ncached--; - l = LIST_FIRST(&sa->sa_lwpcache); + if (vp->savp_ncached > 0) { + vp->savp_ncached--; + l = LIST_FIRST(&vp->savp_lwpcache); LIST_REMOVE(l, l_sibling); + p = l->l_proc; LIST_INSERT_HEAD(&p->p_lwps, l, l_sibling); p->p_nlwps++; DPRINTFN(5,("sa_getcachelwp(%d.%d) Got LWP %d from cache.\n", @@ -975,12 +1163,14 @@ sa_unblock_userret(struct lwp *l) struct proc *p; struct lwp *l2; struct sadata *sa; + struct sadata_vp *vp; struct sadata_upcall *sau; struct sastack *sast; int f, s; p = l->l_proc; sa = p->p_sa; + vp = l->l_savp; if (p->p_flag & P_WEXIT) return; @@ -997,7 +1187,7 @@ sa_unblock_userret(struct lwp *l) /* maybe NOTREACHED */ SCHED_LOCK(s); - if (l != sa->sa_vp) { + if (l != vp->savp_lwp) { /* Invoke an "unblocked" upcall */ DPRINTFN(8,("sa_unblock_userret(%d.%d) unblocking\n", p->p_pid, l->l_lid)); @@ -1046,7 +1236,7 @@ sa_unblock_userret(struct lwp *l) sau->sau_stack = sast->sast_stack; SCHED_LOCK(s); - SIMPLEQ_INSERT_TAIL(&sa->sa_upcalls, sau, sau_next); + SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); l->l_flag |= L_SA_UPCALL; l->l_flag &= ~L_SA_BLOCKING; sa_putcachelwp(p, l2); @@ -1063,12 +1253,14 @@ sa_upcall_userret(struct lwp *l) struct lwp *l2; struct proc *p; struct sadata *sa; + struct sadata_vp *vp; struct sadata_upcall *sau; struct sastack *sast; int f, s; p = l->l_proc; sa = p->p_sa; + vp = l->l_savp; SCHED_ASSERT_UNLOCKED(); @@ -1081,14 +1273,14 @@ sa_upcall_userret(struct lwp *l) KDASSERT((l->l_flag & L_SA_BLOCKING) == 0); sast = NULL; - if (SIMPLEQ_EMPTY(&sa->sa_upcalls) && sa->sa_wokenq_head != NULL) + if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL) sast = sa_getstack(sa); SCHED_LOCK(s); - if (SIMPLEQ_EMPTY(&sa->sa_upcalls) && sa->sa_wokenq_head != NULL && + if (SIMPLEQ_EMPTY(&vp->savp_upcalls) && vp->savp_wokenq_head != NULL && sast != NULL) { /* Invoke an "unblocked" upcall */ - l2 = sa->sa_wokenq_head; - sa->sa_wokenq_head = l2->l_forw; + l2 = vp->savp_wokenq_head; + vp->savp_wokenq_head = l2->l_forw; DPRINTFN(9,("sa_upcall_userret(%d.%d) using stack %p\n", l->l_proc->p_pid, l->l_lid, sast->sast_stack.ss_sp)); @@ -1123,7 +1315,7 @@ sa_upcall_userret(struct lwp *l) } sau->sau_stack = sast->sast_stack; - SIMPLEQ_INSERT_TAIL(&sa->sa_upcalls, sau, sau_next); + SIMPLEQ_INSERT_TAIL(&vp->savp_upcalls, sau, sau_next); l2->l_flag &= ~L_SA_BLOCKING; SCHED_LOCK(s); @@ -1135,12 +1327,12 @@ sa_upcall_userret(struct lwp *l) sa_setstackfree(sast, sa); } - KDASSERT(sa->sa_vp == l); + KDASSERT(vp->savp_lwp == l); - while (!SIMPLEQ_EMPTY(&sa->sa_upcalls)) + while (!SIMPLEQ_EMPTY(&vp->savp_upcalls)) sa_makeupcalls(l); - if (sa->sa_wokenq_head == NULL) + if (vp->savp_wokenq_head == NULL) l->l_flag &= ~L_SA_UPCALL; SA_LWP_STATE_UNLOCK(l, f); @@ -1154,6 +1346,7 @@ sa_makeupcalls(struct lwp *l) struct lwp *l2, *eventq; struct proc *p; struct sadata *sa; + struct sadata_vp *vp; struct sa_t **sapp, *sap; struct sa_t self_sa; struct sa_t *sas[3], *sasp; @@ -1165,9 +1358,10 @@ sa_makeupcalls(struct lwp *l) p = l->l_proc; sa = p->p_sa; + vp = l->l_savp; - sau = SIMPLEQ_FIRST(&sa->sa_upcalls); - SIMPLEQ_REMOVE_HEAD(&sa->sa_upcalls, sau_next); + sau = SIMPLEQ_FIRST(&vp->savp_upcalls); + SIMPLEQ_REMOVE_HEAD(&vp->savp_upcalls, sau_next); if (sau->sau_flags & SAU_FLAG_DEFERRED_EVENT) sa_upcall_getstate(&sau->sau_event, @@ -1181,7 +1375,7 @@ sa_makeupcalls(struct lwp *l) & ~ALIGNBYTES); self_sa.sa_id = l->l_lid; - self_sa.sa_cpu = 0; /* XXX l->l_cpu; */ + self_sa.sa_cpu = vp->savp_id; sas[0] = &self_sa; nevents = 0; nint = 0; @@ -1220,8 +1414,8 @@ sa_makeupcalls(struct lwp *l) eventq = NULL; if (sau->sau_type == SA_UPCALL_UNBLOCKED) { SCHED_LOCK(s); - eventq = sa->sa_wokenq_head; - sa->sa_wokenq_head = NULL; + eventq = vp->savp_wokenq_head; + vp->savp_wokenq_head = NULL; SCHED_UNLOCK(s); l2 = eventq; while (l2 != NULL) { @@ -1239,7 +1433,8 @@ sa_makeupcalls(struct lwp *l) sadata_upcall_free(sau); #ifdef DIAGNOSTIC printf("sa_makeupcalls: couldn't copyout activation" - " ucontext for %d.%d\n", l->l_proc->p_pid, l->l_lid); + " ucontext for %d.%d to %p\n", l->l_proc->p_pid, l->l_lid, + up); #endif sigexit(l, SIGILL); /* NOTREACHED */ @@ -1329,6 +1524,8 @@ sa_makeupcalls(struct lwp *l) l->l_lid, type)); cpu_upcall(l, type, nevents, nint, sapp, ap, stack, sa->sa_upcall); + + l->l_flag &= ~L_SA_YIELD; } static void @@ -1337,6 +1534,7 @@ sa_setwoken(struct lwp *l) struct lwp *l2, *vp_lwp; struct proc *p; struct sadata *sa; + struct sadata_vp *vp; int s; SCHED_LOCK(s); @@ -1348,7 +1546,8 @@ sa_setwoken(struct lwp *l) p = l->l_proc; sa = p->p_sa; - vp_lwp = sa->sa_vp; + vp = l->l_savp; + vp_lwp = vp->savp_lwp; l2 = NULL; KDASSERT(vp_lwp != NULL); @@ -1359,7 +1558,7 @@ sa_setwoken(struct lwp *l) #if notyet if (vp_lwp->l_flag & L_SA_IDLE) { KDASSERT((vp_lwp->l_flag & L_SA_UPCALL) == 0); - KDASSERT(sa->sa_wokenq_head == NULL); + KDASSERT(vp->savp_wokenq_head == NULL); DPRINTFN(3,("sa_setwoken(%d.%d) repossess: idle vp_lwp %d state %d\n", l->l_proc->p_pid, l->l_lid, vp_lwp->l_lid, vp_lwp->l_stat)); @@ -1374,11 +1573,11 @@ sa_setwoken(struct lwp *l) vp_lwp->l_stat)); PHOLD(l); - if (sa->sa_wokenq_head == NULL) - sa->sa_wokenq_head = l; + if (vp->savp_wokenq_head == NULL) + vp->savp_wokenq_head = l; else - *sa->sa_wokenq_tailp = l; - *(sa->sa_wokenq_tailp = &l->l_forw) = NULL; + *vp->savp_wokenq_tailp = l; + *(vp->savp_wokenq_tailp = &l->l_forw) = NULL; switch (vp_lwp->l_stat) { case LSONPROC: @@ -1445,7 +1644,7 @@ sa_vp_repossess(struct lwp *l) { struct lwp *l2; struct proc *p = l->l_proc; - struct sadata *sa = p->p_sa; + struct sadata_vp *vp = l->l_savp; SCHED_ASSERT_LOCKED(); @@ -1453,8 +1652,8 @@ sa_vp_repossess(struct lwp *l) * Put ourselves on the virtual processor and note that the * previous occupant of that position was interrupted. */ - l2 = sa->sa_vp; - sa->sa_vp = l; + l2 = vp->savp_lwp; + vp->savp_lwp = l; if (l2->l_flag & L_SA_YIELD) l2->l_flag &= ~(L_SA_YIELD|L_SA_IDLE); @@ -1516,6 +1715,7 @@ debug_print_sa(struct proc *p) { struct lwp *l; struct sadata *sa; + struct sadata_vp *vp; printf("Process %d (%s), state %d, address %p, flags %x\n", p->p_pid, p->p_comm, p->p_stat, p, p->p_flag); @@ -1525,14 +1725,16 @@ debug_print_sa(struct proc *p) debug_print_lwp(l); sa = p->p_sa; if (sa) { - if (sa->sa_vp) - printf("SA VP: %d %s\n", sa->sa_vp->l_lid, - sa->sa_vp->l_flag & L_SA_YIELD ? - (sa->sa_vp->l_flag & L_SA_IDLE ? - "idle" : "yielding") : ""); - printf("SAs: %d cached LWPs\n", sa->sa_ncached); - LIST_FOREACH(l, &sa->sa_lwpcache, l_sibling) - debug_print_lwp(l); + SLIST_FOREACH(vp, &sa->sa_vps, savp_next) { + if (vp->savp_lwp) + printf("SA VP: %d %s\n", vp->savp_lwp->l_lid, + vp->savp_lwp->l_flag & L_SA_YIELD ? + (vp->savp_lwp->l_flag & L_SA_IDLE ? + "idle" : "yielding") : ""); + printf("SAs: %d cached LWPs\n", vp->savp_ncached); + LIST_FOREACH(l, &vp->savp_lwpcache, l_sibling) + debug_print_lwp(l); + } } return 0; diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c index 70de3e8ed82..7017019355d 100644 --- a/sys/kern/kern_sig.c +++ b/sys/kern/kern_sig.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_sig.c,v 1.186 2004/03/11 22:34:26 christos Exp $ */ +/* $NetBSD: kern_sig.c,v 1.187 2004/03/14 01:08:47 cl Exp $ */ /* * Copyright (c) 1982, 1986, 1989, 1991, 1993 @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_sig.c,v 1.186 2004/03/11 22:34:26 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_sig.c,v 1.187 2004/03/14 01:08:47 cl Exp $"); #include "opt_ktrace.h" #include "opt_compat_sunos.h" @@ -993,6 +993,7 @@ static void kpsignal2(struct proc *p, const ksiginfo_t *ksi, int dolock) { struct lwp *l, *suspended = NULL; + struct sadata_vp *vp; int s = 0, prop, allsusp; sig_t action; int signum = ksi->ksi_signo; @@ -1095,17 +1096,24 @@ kpsignal2(struct proc *p, const ksiginfo_t *ksi, int dolock) SCHED_LOCK(s); if (p->p_flag & P_SA) { - l = p->p_sa->sa_vp; allsusp = 0; + l = NULL; if (p->p_stat == SACTIVE) { - KDASSERT(l != NULL); - if (l->l_flag & L_SA_IDLE) { - /* wakeup idle LWP */ - } else if (l->l_flag & L_SA_YIELD) { - /* idle LWP is already waking up */ - goto out; - /*NOTREACHED*/ - } else { + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + l = vp->savp_lwp; + KDASSERT(l != NULL); + if (l->l_flag & L_SA_IDLE) { + /* wakeup idle LWP */ + goto found; + /*NOTREACHED*/ + } else if (l->l_flag & L_SA_YIELD) { + /* idle LWP is already waking up */ + goto out; + /*NOTREACHED*/ + } + } + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + l = vp->savp_lwp; if (l->l_stat == LSRUN || l->l_stat == LSONPROC) { signotify(p); @@ -1115,21 +1123,18 @@ kpsignal2(struct proc *p, const ksiginfo_t *ksi, int dolock) if (l->l_stat == LSSLEEP && l->l_flag & L_SINTR) { /* ok to signal vp lwp */ - } else if (signum == SIGKILL) { - /* - * get a suspended lwp from - * the cache to send KILL - * signal - * XXXcl add signal checks at resume points - */ - suspended = sa_getcachelwp(p); - allsusp = 1; } else l = NULL; } + if (l == NULL) + allsusp = 1; } else if (p->p_stat == SSTOP) { - if (l->l_stat != LSSLEEP || (l->l_flag & L_SINTR) == 0) + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + l = vp->savp_lwp; + if (l->l_stat == LSSLEEP && (l->l_flag & L_SINTR) != 0) + break; l = NULL; + } } } else if (p->p_nrlwps > 0 && (p->p_stat != SSTOP)) { /* @@ -1161,6 +1166,7 @@ kpsignal2(struct proc *p, const ksiginfo_t *ksi, int dolock) } } + found: switch (p->p_stat) { case SACTIVE: @@ -1210,8 +1216,19 @@ kpsignal2(struct proc *p, const ksiginfo_t *ksi, int dolock) * make this happen by unsuspending one of * them. */ - if (allsusp && (signum == SIGKILL)) + if (allsusp && (signum == SIGKILL)) { + if (p->p_flag & P_SA) { + /* + * get a suspended lwp from + * the cache to send KILL + * signal + * XXXcl add signal checks at resume points + */ + suspended = sa_getcachelwp + (SLIST_FIRST(&p->p_sa->sa_vps)); + } lwp_continue(suspended); + } goto done; } /* @@ -1403,13 +1420,9 @@ issignal(struct lwp *l) int dolock = (l->l_flag & L_SINTR) == 0, locked = !dolock; sigset_t ss; - if (l->l_flag & L_SA) { - struct sadata *sa = p->p_sa; - - /* Bail out if we do not own the virtual processor */ - if (sa->sa_vp != l) - return 0; - } + /* Bail out if we do not own the virtual processor */ + if (l->l_flag & L_SA && l->l_savp->savp_lwp != l) + return 0; if (p->p_stat == SSTOP) { /* @@ -1598,6 +1611,7 @@ proc_stop(struct proc *p, int wakeup) { struct lwp *l; struct proc *parent; + struct sadata_vp *vp; SCHED_ASSERT_LOCKED(); @@ -1615,19 +1629,21 @@ proc_stop(struct proc *p, int wakeup) * because the VP-LWP in stopped state cannot be * repossessed. */ - l = p->p_sa->sa_vp; - if (l->l_stat == LSONPROC && l->l_cpu == curcpu()) { - l->l_stat = LSSTOP; - p->p_nrlwps--; - } else if (l->l_stat == LSRUN) { - /* Remove LWP from the run queue */ - remrunqueue(l); - l->l_stat = LSSTOP; - p->p_nrlwps--; - } else if (l->l_stat == LSSLEEP && - l->l_flag & L_SA_IDLE) { - l->l_flag &= ~L_SA_IDLE; - l->l_stat = LSSTOP; + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + l = vp->savp_lwp; + if (l->l_stat == LSONPROC && l->l_cpu == curcpu()) { + l->l_stat = LSSTOP; + p->p_nrlwps--; + } else if (l->l_stat == LSRUN) { + /* Remove LWP from the run queue */ + remrunqueue(l); + l->l_stat = LSSTOP; + p->p_nrlwps--; + } else if (l->l_stat == LSSLEEP && + l->l_flag & L_SA_IDLE) { + l->l_flag &= ~L_SA_IDLE; + l->l_stat = LSSTOP; + } } goto out; } @@ -1701,6 +1717,7 @@ struct lwp * proc_unstop(struct proc *p) { struct lwp *l, *lr = NULL; + struct sadata_vp *vp; int cantake = 0; SCHED_ASSERT_LOCKED(); @@ -1737,12 +1754,15 @@ proc_unstop(struct proc *p) } if (p->p_flag & P_SA) { /* Only consider returning the LWP on the VP. */ - lr = p->p_sa->sa_vp; - if (lr->l_stat == LSSLEEP) { - if (lr->l_flag & L_SA_YIELD) - setrunnable(lr); - else if (lr->l_flag & L_SINTR) - return lr; + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + lr = vp->savp_lwp; + if (lr->l_stat == LSSLEEP) { + if (lr->l_flag & L_SA_YIELD) { + setrunnable(lr); + break; + } else if (lr->l_flag & L_SINTR) + return lr; + } } return NULL; } diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 8e6eaf7bb6d..27cb5e23023 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_synch.c,v 1.141 2004/02/13 11:36:23 wiz Exp $ */ +/* $NetBSD: kern_synch.c,v 1.142 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. @@ -74,7 +74,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.141 2004/02/13 11:36:23 wiz Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.142 2004/03/14 01:08:47 cl Exp $"); #include "opt_ddb.h" #include "opt_ktrace.h" @@ -611,11 +611,10 @@ unsleep(struct lwp *l) __inline void sa_awaken(struct lwp *l) { - struct sadata *sa = l->l_proc->p_sa; - + SCHED_ASSERT_LOCKED(); - if (l == sa->sa_vp && l->l_flag & L_SA_YIELD) + if (l == l->l_savp->savp_lwp && l->l_flag & L_SA_YIELD) l->l_flag &= ~L_SA_IDLE; } diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c index d6ba519e74e..cb606765790 100644 --- a/sys/kern/kern_time.c +++ b/sys/kern/kern_time.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_time.c,v 1.81 2004/01/02 18:52:17 cl Exp $ */ +/* $NetBSD: kern_time.c,v 1.82 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 2000 The NetBSD Foundation, Inc. @@ -68,7 +68,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_time.c,v 1.81 2004/01/02 18:52:17 cl Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_time.c,v 1.82 2004/03/14 01:08:47 cl Exp $"); #include "fs_nfs.h" #include "opt_nfs.h" @@ -862,7 +862,7 @@ timerupcall(struct lwp *l, void *arg) KDASSERT(l->l_proc->p_sa); /* Bail out if we do not own the virtual processor */ - if (l->l_proc->p_sa->sa_vp != l) + if (l->l_savp->savp_lwp != l) return ; KERNEL_PROC_LOCK(l); @@ -1194,7 +1194,9 @@ void itimerfire(struct ptimer *pt) { struct proc *p = pt->pt_proc; + struct sadata_vp *vp; int s; + unsigned int i; if (pt->pt_ev.sigev_notify == SIGEV_SIGNAL) { /* @@ -1216,8 +1218,6 @@ itimerfire(struct ptimer *pt) } } else if (pt->pt_ev.sigev_notify == SIGEV_SA && (p->p_flag & P_SA)) { /* Cause the process to generate an upcall when it returns. */ - struct sadata *sa = p->p_sa; - unsigned int i; if (p->p_userret == NULL) { /* @@ -1234,9 +1234,12 @@ itimerfire(struct ptimer *pt) p->p_userret_arg = p->p_timers; SCHED_LOCK(s); - if (sa->sa_vp->l_flag & L_SA_IDLE) { - sa->sa_vp->l_flag &= ~L_SA_IDLE; - sched_wakeup(sa->sa_vp); + SLIST_FOREACH(vp, &p->p_sa->sa_vps, savp_next) { + if (vp->savp_lwp->l_flag & L_SA_IDLE) { + vp->savp_lwp->l_flag &= ~L_SA_IDLE; + sched_wakeup(vp->savp_lwp); + break; + } } SCHED_UNLOCK(s); } else if (p->p_userret == timerupcall) { diff --git a/sys/sys/lwp.h b/sys/sys/lwp.h index d5cd59b33b8..1538ddc6cde 100644 --- a/sys/sys/lwp.h +++ b/sys/sys/lwp.h @@ -1,4 +1,4 @@ -/* $NetBSD: lwp.h,v 1.20 2004/03/14 00:45:21 cl Exp $ */ +/* $NetBSD: lwp.h,v 1.21 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -73,7 +73,7 @@ struct lwp { int l_holdcnt; /* If non-zero, don't swap. */ void *l_ctxlink; /* uc_link {get,set}context */ int l_dupfd; /* Sideways return value from cloning devices XXX */ - void *l_fill1; /* XXXcl temporarily unused */ + struct sadata_vp *l_savp; /* SA "virtual processor" */ #define l_endzero l_priority diff --git a/sys/sys/savar.h b/sys/sys/savar.h index 2e788bdcb3c..8e4be147ad9 100644 --- a/sys/sys/savar.h +++ b/sys/sys/savar.h @@ -1,4 +1,4 @@ -/* $NetBSD: savar.h,v 1.14 2004/01/02 18:52:17 cl Exp $ */ +/* $NetBSD: savar.h,v 1.15 2004/03/14 01:08:47 cl Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. @@ -84,24 +84,32 @@ struct sastack { unsigned int sast_gen; }; +struct sadata_vp { + int savp_id; /* "virtual processor" identifier */ + SLIST_ENTRY(sadata_vp) savp_next; /* link to next sadata_vp */ + struct simplelock savp_lock; /* lock on these fields */ + struct lwp *savp_lwp; /* lwp on "virtual processor" */ + struct lwp *savp_blocker; /* recently blocked lwp */ + struct lwp *savp_wokenq_head; /* list of woken lwps */ + struct lwp **savp_wokenq_tailp; /* list of woken lwps */ + vaddr_t savp_faultaddr; /* page fault address */ + vaddr_t savp_ofaultaddr; /* old page fault address */ + LIST_HEAD(, lwp) savp_lwpcache; /* list of available lwps */ + int savp_ncached; /* list length */ + SIMPLEQ_HEAD(, sadata_upcall) savp_upcalls; /* pending upcalls */ +}; + struct sadata { struct simplelock sa_lock; /* lock on these fields */ int sa_flag; /* SA_* flags */ sa_upcall_t sa_upcall; /* upcall entry point */ - struct lwp *sa_vp; /* "virtual processor" allocation */ - struct lwp *sa_vp_blocker; /* recently blocked lwp */ - struct lwp *sa_wokenq_head; /* list of woken lwps */ - struct lwp **sa_wokenq_tailp; /* list of woken lwps */ - vaddr_t sa_vp_faultaddr; /* page fault address */ - vaddr_t sa_vp_ofaultaddr; /* old page fault address */ - int sa_concurrency; /* desired concurrency */ - LIST_HEAD(, lwp) sa_lwpcache; /* list of available lwps */ - int sa_ncached; /* list length */ + int sa_concurrency; /* current concurrency */ + int sa_maxconcurrency; /* requested concurrency */ SPLAY_HEAD(sasttree, sastack) sa_stackstree; /* tree of upcall stacks */ struct sastack *sa_stacknext; /* next free stack */ ssize_t sa_stackinfo_offset; /* offset from ss_sp to stackinfo data */ int sa_nstacks; /* number of upcall stacks */ - SIMPLEQ_HEAD(, sadata_upcall) sa_upcalls; /* pending upcalls */ + SLIST_HEAD(, sadata_vp) sa_vps; /* list of "virtual processors" */ }; #define SA_FLAG_ALL SA_FLAG_PREEMPT @@ -109,6 +117,7 @@ struct sadata { extern struct pool sadata_pool; /* memory pool for sadata structures */ extern struct pool saupcall_pool; /* memory pool for pending upcalls */ extern struct pool sastack_pool; /* memory pool for sastack structs */ +extern struct pool savp_pool; /* memory pool for sadata_vp structures */ #ifdef _KERNEL #include <sys/mallocvar.h> @@ -128,7 +137,7 @@ void sa_yield(struct lwp *); int sa_upcall(struct lwp *, int, struct lwp *, struct lwp *, size_t, void *); void sa_putcachelwp(struct proc *, struct lwp *); -struct lwp *sa_getcachelwp(struct proc *); +struct lwp *sa_getcachelwp(struct sadata_vp *); void sa_unblock_userret(struct lwp *); |
