diff options
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/arch/arm/arm/arm_machdep.c | 30 | ||||
| -rw-r--r-- | sys/arch/arm/arm32/cpuswitch.S | 6 | ||||
| -rw-r--r-- | sys/arch/arm/arm32/db_machdep.c | 6 | ||||
| -rw-r--r-- | sys/arch/arm/arm32/genassym.cf | 4 | ||||
| -rw-r--r-- | sys/arch/arm/include/arm32/frame.h | 47 | ||||
| -rw-r--r-- | sys/arch/arm/include/cpu.h | 50 | ||||
| -rw-r--r-- | sys/arch/arm/include/locore.h | 7 | ||||
| -rw-r--r-- | sys/arch/arm/include/proc.h | 3 | ||||
| -rw-r--r-- | sys/arch/evbarm/conf/std.generic | 4 |
9 files changed, 87 insertions, 70 deletions
diff --git a/sys/arch/arm/arm/arm_machdep.c b/sys/arch/arm/arm/arm_machdep.c index 25ec91b8285..2badde549e2 100644 --- a/sys/arch/arm/arm/arm_machdep.c +++ b/sys/arch/arm/arm/arm_machdep.c @@ -1,4 +1,4 @@ -/* $NetBSD: arm_machdep.c,v 1.63 2020/02/15 08:16:10 skrll Exp $ */ +/* $NetBSD: arm_machdep.c,v 1.64 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright (c) 2001 Wasabi Systems, Inc. @@ -80,7 +80,7 @@ #include <sys/param.h> -__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.63 2020/02/15 08:16:10 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.64 2020/08/14 16:18:36 skrll Exp $"); #include <sys/atomic.h> #include <sys/cpu.h> @@ -241,17 +241,39 @@ cpu_need_resched(struct cpu_info *ci, struct lwp *l, int flags) if (flags & RESCHED_REMOTE) { intr_ipi_send(ci->ci_kcpuset, IPI_KPREEMPT); } else { - atomic_or_uint(&ci->ci_astpending, __BIT(1)); + l->l_md.md_astpending |= __BIT(1); } #endif /* __HAVE_PREEMPTION */ return; } + + KASSERT((flags & RESCHED_UPREEMPT) != 0); if (flags & RESCHED_REMOTE) { #ifdef MULTIPROCESSOR intr_ipi_send(ci->ci_kcpuset, IPI_AST); #endif /* MULTIPROCESSOR */ } else { - setsoftast(ci); + l->l_md.md_astpending |= __BIT(0); + } +} + + +/* + * Notify the current lwp (l) that it has a signal pending, + * process as soon as possible. + */ +void +cpu_signotify(struct lwp *l) +{ + + KASSERT(kpreempt_disabled()); + + if (l->l_cpu != curcpu()) { +#ifdef MULTIPROCESSOR + intr_ipi_send(l->l_cpu->ci_kcpuset, IPI_AST); +#endif + } else { + l->l_md.md_astpending |= __BIT(0); } } diff --git a/sys/arch/arm/arm32/cpuswitch.S b/sys/arch/arm/arm32/cpuswitch.S index 0d0725dc49e..60a511a28a3 100644 --- a/sys/arch/arm/arm32/cpuswitch.S +++ b/sys/arch/arm/arm32/cpuswitch.S @@ -1,4 +1,4 @@ -/* $NetBSD: cpuswitch.S,v 1.101 2020/07/10 12:25:09 skrll Exp $ */ +/* $NetBSD: cpuswitch.S,v 1.102 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright 2003 Wasabi Systems, Inc. @@ -87,7 +87,7 @@ #include <arm/asm.h> #include <arm/locore.h> - RCSID("$NetBSD: cpuswitch.S,v 1.101 2020/07/10 12:25:09 skrll Exp $") + RCSID("$NetBSD: cpuswitch.S,v 1.102 2020/08/14 16:18:36 skrll Exp $") /* LINTSTUB: include <sys/param.h> */ @@ -191,6 +191,7 @@ ENTRY(cpu_switchto) /* We have a new curlwp now so make a note of it */ str r6, [r5, #(CI_CURLWP)] + dmb /* see comments in kern_mutex.c */ /* Get the new pcb */ ldr r7, [r6, #(L_PCB)] @@ -388,6 +389,7 @@ ENTRY_NP(softint_switch) mcr p15, 0, r5, c13, c0, 4 /* save new lwp */ #endif str r5, [r7, #(CI_CURLWP)] /* save new lwp */ + dmb /* see comments in kern_mutex.c */ #ifdef KASAN mov r0, r5 diff --git a/sys/arch/arm/arm32/db_machdep.c b/sys/arch/arm/arm32/db_machdep.c index 8678c61e333..f63c7346c4b 100644 --- a/sys/arch/arm/arm32/db_machdep.c +++ b/sys/arch/arm/arm32/db_machdep.c @@ -1,4 +1,4 @@ -/* $NetBSD: db_machdep.c,v 1.34 2020/07/03 10:19:18 jmcneill Exp $ */ +/* $NetBSD: db_machdep.c,v 1.35 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright (c) 1996 Mark Brinicombe @@ -34,7 +34,7 @@ #endif #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: db_machdep.c,v 1.34 2020/07/03 10:19:18 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: db_machdep.c,v 1.35 2020/08/14 16:18:36 skrll Exp $"); #include <sys/param.h> @@ -521,8 +521,6 @@ show_cpuinfo(struct cpu_info *kci) &ci->ci_cpl, cpuid, ci->ci_cpl); db_printf("%p cpu[%lu].ci_softints = 0x%08x\n", &ci->ci_softints, cpuid, ci->ci_softints); - db_printf("%p cpu[%lu].ci_astpending = 0x%08x\n", - &ci->ci_astpending, cpuid, ci->ci_astpending); db_printf("%p cpu[%lu].ci_intr_depth = %u\n", &ci->ci_intr_depth, cpuid, ci->ci_intr_depth); diff --git a/sys/arch/arm/arm32/genassym.cf b/sys/arch/arm/arm32/genassym.cf index 42878b6e76e..4f88d8c79a4 100644 --- a/sys/arch/arm/arm32/genassym.cf +++ b/sys/arch/arm/arm32/genassym.cf @@ -1,4 +1,4 @@ -# $NetBSD: genassym.cf,v 1.93 2020/07/08 10:18:00 skrll Exp $ +# $NetBSD: genassym.cf,v 1.94 2020/08/14 16:18:36 skrll Exp $ # Copyright (c) 1982, 1990 The Regents of the University of California. # All rights reserved. @@ -166,6 +166,7 @@ define L_CPU offsetof(struct lwp, l_cpu) define L_PROC offsetof(struct lwp, l_proc) define L_PRIVATE offsetof(struct lwp, l_private) define L_FLAG offsetof(struct lwp, l_flag) +define L_MD_ASTPENDING offsetof(struct lwp, l_md.md_astpending) define L_MD_FLAGS offsetof(struct lwp, l_md.md_flags) define L_MD_TF offsetof(struct lwp, l_md.md_tf) define MDLWP_NOALIGNFLT MDLWP_NOALIGNFLT @@ -225,7 +226,6 @@ define CF_CONTROL offsetof(struct cpu_functions, cf_control) define CPU_INFO_SIZE sizeof(struct cpu_info) define CI_ARM_CPUID offsetof(struct cpu_info, ci_arm_cpuid) -define CI_ASTPENDING offsetof(struct cpu_info, ci_astpending) define CI_CPL offsetof(struct cpu_info, ci_cpl) define CI_CURLWP offsetof(struct cpu_info, ci_curlwp) define CI_INTR_DEPTH offsetof(struct cpu_info, ci_intr_depth) diff --git a/sys/arch/arm/include/arm32/frame.h b/sys/arch/arm/include/arm32/frame.h index dd21082ba18..b3d59fde306 100644 --- a/sys/arch/arm/include/arm32/frame.h +++ b/sys/arch/arm/include/arm32/frame.h @@ -1,4 +1,4 @@ -/* $NetBSD: frame.h,v 1.47 2018/10/28 14:46:59 skrll Exp $ */ +/* $NetBSD: frame.h,v 1.48 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright (c) 1994-1997 Mark Brinicombe. @@ -151,26 +151,16 @@ void validate_trapframe(trapframe_t *, int); msr cpsr_c, ra /* Restore interrupts */ #endif -#ifdef __HAVE_PREEMPTION -#define DO_CLEAR_ASTPENDING \ - mvn r1, #1 /* complement of 1 */ ;\ - add r0, r4, #CI_ASTPENDING /* address of astpending */ ;\ - bl _C_LABEL(atomic_and_uint) /* clear AST */ -#else -#define DO_CLEAR_ASTPENDING \ - mov r0, #0 ;\ - str r0, [r4, #CI_ASTPENDING] /* clear AST */ -#endif - #define DO_PENDING_AST(lbl) ;\ -1: ldr r1, [r4, #CI_ASTPENDING] /* Pending AST? */ ;\ - tst r1, #0x00000001 ;\ +1: ldr r1, [r5, #L_MD_ASTPENDING] /* Pending AST? */ ;\ + tst r1, #1 ;\ beq lbl /* Nope. Just bail */ ;\ - DO_CLEAR_ASTPENDING ;\ - CPSIE_I(r5, r5) /* Restore interrupts */ ;\ + bic r0, r1, #1 /* clear AST */ ;\ + str r0, [r5, #L_MD_ASTPENDING] ;\ + CPSIE_I(r6, r6) /* Restore interrupts */ ;\ mov r0, sp ;\ bl _C_LABEL(ast) /* ast(frame) */ ;\ - CPSID_I(r0, r5) /* Disable interrupts */ ;\ + CPSID_I(r0, r6) /* Disable interrupts */ ;\ b 1b /* test again */ /* @@ -179,8 +169,8 @@ void validate_trapframe(trapframe_t *, int); * alignment faults when executing old a.out ARM binaries. * * Note that when ENABLE_ALIGNMENTS_FAULTS finishes r4 will contain - * pointer to the cpu's cpu_info. DO_AST_AND_RESTORE_ALIGNMENT_FAULTS - * relies on r4 being preserved. + * curcpu() and r5 containing curlwp. DO_AST_AND_RESTORE_ALIGNMENT_FAULTS + * relies on r4 and r5 being preserved. */ #ifdef EXEC_AOUT #define AST_ALIGNMENT_FAULT_LOCALS \ @@ -198,10 +188,9 @@ void validate_trapframe(trapframe_t *, int); #define ENABLE_ALIGNMENT_FAULTS \ and r7, r0, #(PSR_MODE) /* Test for USR32 mode */ ;\ cmp r7, #(PSR_USR32_MODE) ;\ - GET_CURCPU(r4) /* r4 = cpuinfo */ ;\ + GET_CURX(r4, r5) /* r4 = curcpu, r5 = curlwp */ ;\ bne 1f /* Not USR mode skip AFLT */ ;\ - ldr r1, [r4, #CI_CURLWP] /* get curlwp from cpu_info */ ;\ - ldr r1, [r1, #L_MD_FLAGS] /* Fetch l_md.md_flags */ ;\ + ldr r1, [r5, #L_MD_FLAGS] /* Fetch l_md.md_flags */ ;\ tst r1, #MDLWP_NOALIGNFLT ;\ beq 1f /* AFLTs already enabled */ ;\ ldr r2, .Laflt_cpufuncs ;\ @@ -213,13 +202,13 @@ void validate_trapframe(trapframe_t *, int); /* * This macro must be invoked just before PULLFRAMEFROMSVCANDEXIT or * PULLFRAME at the end of interrupt/exception handlers. We know that - * r4 points to cpu_info since that is what ENABLE_ALIGNMENT_FAULTS did - * for use. + * r4 points to curcpu() and r5 points to curlwp since that is what + * ENABLE_ALIGNMENT_FAULTS did for us. */ #define DO_AST_AND_RESTORE_ALIGNMENT_FAULTS \ DO_PENDING_SOFTINTS ;\ - GET_CPSR(r5) /* save CPSR */ ;\ - CPSID_I(r1, r5) /* Disable interrupts */ ;\ + GET_CPSR(r6) /* save CPSR */ ;\ + CPSID_I(r1, r6) /* Disable interrupts */ ;\ cmp r7, #(PSR_USR32_MODE) /* Returning to USR mode? */ ;\ bne 3f /* Nope, get out now */ ;\ DO_PENDING_AST(2f) /* Pending AST? */ ;\ @@ -240,13 +229,13 @@ void validate_trapframe(trapframe_t *, int); #define ENABLE_ALIGNMENT_FAULTS \ and r7, r0, #(PSR_MODE) /* Test for USR32 mode */ ;\ - GET_CURCPU(r4) /* r4 = cpuinfo */ + GET_CURX(r4, r5) /* r4 = curcpu, r5 = curlwp */ #define DO_AST_AND_RESTORE_ALIGNMENT_FAULTS \ DO_PENDING_SOFTINTS ;\ - GET_CPSR(r5) /* save CPSR */ ;\ - CPSID_I(r1, r5) /* Disable interrupts */ ;\ + GET_CPSR(r6) /* save CPSR */ ;\ + CPSID_I(r1, r6) /* Disable interrupts */ ;\ cmp r7, #(PSR_USR32_MODE) ;\ bne 2f /* Nope, get out now */ ;\ DO_PENDING_AST(2f) /* Pending AST? */ ;\ diff --git a/sys/arch/arm/include/cpu.h b/sys/arch/arm/include/cpu.h index b01826cf96d..795af9918d3 100644 --- a/sys/arch/arm/include/cpu.h +++ b/sys/arch/arm/include/cpu.h @@ -1,4 +1,4 @@ -/* $NetBSD: cpu.h,v 1.111 2020/06/29 23:54:06 riastradh Exp $ */ +/* $NetBSD: cpu.h,v 1.112 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright (c) 1994-1996 Mark Brinicombe. @@ -154,6 +154,7 @@ static inline void cpu_dosoftints(void); #include <sys/cpu_data.h> #include <sys/device_if.h> #include <sys/evcnt.h> +#include <sys/param.h> struct cpu_info { struct cpu_data ci_data; /* MI per-cpu data */ @@ -163,22 +164,32 @@ struct cpu_info { uint32_t ci_arm_cputype; /* CPU type */ uint32_t ci_arm_cpurev; /* CPU revision */ uint32_t ci_ctrl; /* The CPU control register */ - int ci_cpl; /* current processor level (spl) */ - volatile int ci_astpending; /* */ - int ci_want_resched;/* resched() was called */ - int ci_intr_depth; /* */ - int ci_kfpu_spl; + /* + * the following are in their own cache line, as they are stored to + * regularly by remote CPUs; when they were mixed with other fields + * we observed frequent cache misses. + */ + int ci_want_resched __aligned(COHERENCY_UNIT); + /* resched() was called */ + lwp_t * ci_curlwp __aligned(COHERENCY_UNIT); + /* current lwp */ + lwp_t * ci_onproc; /* current user LWP / kthread */ + + /* + * largely CPU-private. + */ + lwp_t * ci_softlwps[SOFTINT_COUNT] __aligned(COHERENCY_UNIT); struct cpu_softc * ci_softc; /* platform softc */ - lwp_t * ci_softlwps[SOFTINT_COUNT]; - volatile uint32_t - ci_softints; + int ci_cpl; /* current processor level (spl) */ + int ci_kfpu_spl; + + volatile u_int ci_intr_depth; /* */ + volatile u_int ci_softints; - lwp_t * ci_curlwp; /* current lwp */ - lwp_t * ci_onproc; /* current user LWP / kthread */ lwp_t * ci_lastlwp; /* last lwp */ struct evcnt ci_arm700bugcount; @@ -315,19 +326,8 @@ cpu_dosoftints(void) /* * Scheduling glue */ - -#ifdef __HAVE_PREEMPTION -#define setsoftast(ci) atomic_or_uint(&(ci)->ci_astpending, __BIT(0)) -#else -#define setsoftast(ci) ((ci)->ci_astpending = __BIT(0)) -#endif - -/* - * Notify the current process (p) that it has a signal pending, - * process as soon as possible. - */ - -#define cpu_signotify(l) setsoftast((l)->l_cpu) +void cpu_signotify(struct lwp *); +#define setsoftast(ci) (cpu_signotify((ci)->ci_onproc)) /* * Give a profiling tick to the current process when the user profiling @@ -335,7 +335,7 @@ cpu_dosoftints(void) * through trap(), marking the proc as needing a profiling tick. */ #define cpu_need_proftick(l) ((l)->l_pflag |= LP_OWEUPC, \ - setsoftast((l)->l_cpu)) + setsoftast(lwp_getcpu(l))) /* * We've already preallocated the stack for the idlelwps for additional CPUs. diff --git a/sys/arch/arm/include/locore.h b/sys/arch/arm/include/locore.h index 453a17ba6f3..ced9b5ccf58 100644 --- a/sys/arch/arm/include/locore.h +++ b/sys/arch/arm/include/locore.h @@ -1,4 +1,4 @@ -/* $NetBSD: locore.h,v 1.32 2020/02/15 08:16:11 skrll Exp $ */ +/* $NetBSD: locore.h,v 1.33 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright (c) 1994-1996 Mark Brinicombe. @@ -87,18 +87,23 @@ #if defined (TPIDRPRW_IS_CURCPU) #define GET_CURCPU(rX) mrc p15, 0, rX, c13, c0, 4 #define GET_CURLWP(rX) GET_CURCPU(rX); ldr rX, [rX, #CI_CURLWP] +#define GET_CURX(rCPU, rLWP) GET_CURCPU(rCPU); ldr rLWP, [rCPU, #CI_CURLWP] #elif defined (TPIDRPRW_IS_CURLWP) #define GET_CURLWP(rX) mrc p15, 0, rX, c13, c0, 4 #if defined (MULTIPROCESSOR) #define GET_CURCPU(rX) GET_CURLWP(rX); ldr rX, [rX, #L_CPU] +#define GET_CURX(rCPU, rLWP) GET_CURLWP(rLWP); ldr rCPU, [rLWP, #L_CPU] #elif defined(_ARM_ARCH_7) #define GET_CURCPU(rX) movw rX, #:lower16:cpu_info_store; movt rX, #:upper16:cpu_info_store +#define GET_CURX(rCPU, rLWP) GET_CURLWP(rLWP); GET_CURCPU(rCPU) #else #define GET_CURCPU(rX) ldr rX, =_C_LABEL(cpu_info_store) +#define GET_CURX(rCPU, rLWP) GET_CURLWP(rLWP); ldr rCPU, [rLWP, #L_CPU] #endif #elif !defined(MULTIPROCESSOR) #define GET_CURCPU(rX) ldr rX, =_C_LABEL(cpu_info_store) #define GET_CURLWP(rX) GET_CURCPU(rX); ldr rX, [rX, #CI_CURLWP] +#define GET_CURX(rCPU, rLWP) GET_CURCPU(rCPU); ldr rLWP, [rCPU, #CI_CURLWP] #endif #define GET_CURPCB(rX) GET_CURLWP(rX); ldr rX, [rX, #L_PCB] diff --git a/sys/arch/arm/include/proc.h b/sys/arch/arm/include/proc.h index a2c69794da9..cf6a3b59768 100644 --- a/sys/arch/arm/include/proc.h +++ b/sys/arch/arm/include/proc.h @@ -1,4 +1,4 @@ -/* $NetBSD: proc.h,v 1.18 2017/10/31 12:37:23 martin Exp $ */ +/* $NetBSD: proc.h,v 1.19 2020/08/14 16:18:36 skrll Exp $ */ /* * Copyright (c) 1994 Mark Brinicombe. @@ -45,6 +45,7 @@ struct lwp; struct mdlwp { struct trapframe *md_tf; int md_flags; + volatile uint32_t md_astpending; }; /* Flags setttings for md_flags */ diff --git a/sys/arch/evbarm/conf/std.generic b/sys/arch/evbarm/conf/std.generic index f3d1c697ffb..b133a1a0474 100644 --- a/sys/arch/evbarm/conf/std.generic +++ b/sys/arch/evbarm/conf/std.generic @@ -1,4 +1,4 @@ -# $NetBSD: std.generic,v 1.7 2020/01/25 18:10:11 skrll Exp $ +# $NetBSD: std.generic,v 1.8 2020/08/14 16:18:37 skrll Exp $ # # generic NetBSD/evbarm with FDT support @@ -22,7 +22,7 @@ options FPU_VFP options MODULAR options MODULAR_DEFAULT_AUTOLOAD options PCI_NETBSD_CONFIGURE -options TPIDRPRW_IS_CURCPU +options TPIDRPRW_IS_CURLWP options __BUS_SPACE_HAS_STREAM_METHODS options __HAVE_CPU_COUNTER options __HAVE_CPU_UAREA_ALLOC_IDLELWP |
