summaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys')
-rw-r--r--sys/arch/arm/arm/arm_machdep.c30
-rw-r--r--sys/arch/arm/arm32/cpuswitch.S6
-rw-r--r--sys/arch/arm/arm32/db_machdep.c6
-rw-r--r--sys/arch/arm/arm32/genassym.cf4
-rw-r--r--sys/arch/arm/include/arm32/frame.h47
-rw-r--r--sys/arch/arm/include/cpu.h50
-rw-r--r--sys/arch/arm/include/locore.h7
-rw-r--r--sys/arch/arm/include/proc.h3
-rw-r--r--sys/arch/evbarm/conf/std.generic4
9 files changed, 87 insertions, 70 deletions
diff --git a/sys/arch/arm/arm/arm_machdep.c b/sys/arch/arm/arm/arm_machdep.c
index 25ec91b8285..2badde549e2 100644
--- a/sys/arch/arm/arm/arm_machdep.c
+++ b/sys/arch/arm/arm/arm_machdep.c
@@ -1,4 +1,4 @@
-/* $NetBSD: arm_machdep.c,v 1.63 2020/02/15 08:16:10 skrll Exp $ */
+/* $NetBSD: arm_machdep.c,v 1.64 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright (c) 2001 Wasabi Systems, Inc.
@@ -80,7 +80,7 @@
#include <sys/param.h>
-__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.63 2020/02/15 08:16:10 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: arm_machdep.c,v 1.64 2020/08/14 16:18:36 skrll Exp $");
#include <sys/atomic.h>
#include <sys/cpu.h>
@@ -241,17 +241,39 @@ cpu_need_resched(struct cpu_info *ci, struct lwp *l, int flags)
if (flags & RESCHED_REMOTE) {
intr_ipi_send(ci->ci_kcpuset, IPI_KPREEMPT);
} else {
- atomic_or_uint(&ci->ci_astpending, __BIT(1));
+ l->l_md.md_astpending |= __BIT(1);
}
#endif /* __HAVE_PREEMPTION */
return;
}
+
+ KASSERT((flags & RESCHED_UPREEMPT) != 0);
if (flags & RESCHED_REMOTE) {
#ifdef MULTIPROCESSOR
intr_ipi_send(ci->ci_kcpuset, IPI_AST);
#endif /* MULTIPROCESSOR */
} else {
- setsoftast(ci);
+ l->l_md.md_astpending |= __BIT(0);
+ }
+}
+
+
+/*
+ * Notify the current lwp (l) that it has a signal pending,
+ * process as soon as possible.
+ */
+void
+cpu_signotify(struct lwp *l)
+{
+
+ KASSERT(kpreempt_disabled());
+
+ if (l->l_cpu != curcpu()) {
+#ifdef MULTIPROCESSOR
+ intr_ipi_send(l->l_cpu->ci_kcpuset, IPI_AST);
+#endif
+ } else {
+ l->l_md.md_astpending |= __BIT(0);
}
}
diff --git a/sys/arch/arm/arm32/cpuswitch.S b/sys/arch/arm/arm32/cpuswitch.S
index 0d0725dc49e..60a511a28a3 100644
--- a/sys/arch/arm/arm32/cpuswitch.S
+++ b/sys/arch/arm/arm32/cpuswitch.S
@@ -1,4 +1,4 @@
-/* $NetBSD: cpuswitch.S,v 1.101 2020/07/10 12:25:09 skrll Exp $ */
+/* $NetBSD: cpuswitch.S,v 1.102 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@@ -87,7 +87,7 @@
#include <arm/asm.h>
#include <arm/locore.h>
- RCSID("$NetBSD: cpuswitch.S,v 1.101 2020/07/10 12:25:09 skrll Exp $")
+ RCSID("$NetBSD: cpuswitch.S,v 1.102 2020/08/14 16:18:36 skrll Exp $")
/* LINTSTUB: include <sys/param.h> */
@@ -191,6 +191,7 @@ ENTRY(cpu_switchto)
/* We have a new curlwp now so make a note of it */
str r6, [r5, #(CI_CURLWP)]
+ dmb /* see comments in kern_mutex.c */
/* Get the new pcb */
ldr r7, [r6, #(L_PCB)]
@@ -388,6 +389,7 @@ ENTRY_NP(softint_switch)
mcr p15, 0, r5, c13, c0, 4 /* save new lwp */
#endif
str r5, [r7, #(CI_CURLWP)] /* save new lwp */
+ dmb /* see comments in kern_mutex.c */
#ifdef KASAN
mov r0, r5
diff --git a/sys/arch/arm/arm32/db_machdep.c b/sys/arch/arm/arm32/db_machdep.c
index 8678c61e333..f63c7346c4b 100644
--- a/sys/arch/arm/arm32/db_machdep.c
+++ b/sys/arch/arm/arm32/db_machdep.c
@@ -1,4 +1,4 @@
-/* $NetBSD: db_machdep.c,v 1.34 2020/07/03 10:19:18 jmcneill Exp $ */
+/* $NetBSD: db_machdep.c,v 1.35 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright (c) 1996 Mark Brinicombe
@@ -34,7 +34,7 @@
#endif
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_machdep.c,v 1.34 2020/07/03 10:19:18 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_machdep.c,v 1.35 2020/08/14 16:18:36 skrll Exp $");
#include <sys/param.h>
@@ -521,8 +521,6 @@ show_cpuinfo(struct cpu_info *kci)
&ci->ci_cpl, cpuid, ci->ci_cpl);
db_printf("%p cpu[%lu].ci_softints = 0x%08x\n",
&ci->ci_softints, cpuid, ci->ci_softints);
- db_printf("%p cpu[%lu].ci_astpending = 0x%08x\n",
- &ci->ci_astpending, cpuid, ci->ci_astpending);
db_printf("%p cpu[%lu].ci_intr_depth = %u\n",
&ci->ci_intr_depth, cpuid, ci->ci_intr_depth);
diff --git a/sys/arch/arm/arm32/genassym.cf b/sys/arch/arm/arm32/genassym.cf
index 42878b6e76e..4f88d8c79a4 100644
--- a/sys/arch/arm/arm32/genassym.cf
+++ b/sys/arch/arm/arm32/genassym.cf
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.93 2020/07/08 10:18:00 skrll Exp $
+# $NetBSD: genassym.cf,v 1.94 2020/08/14 16:18:36 skrll Exp $
# Copyright (c) 1982, 1990 The Regents of the University of California.
# All rights reserved.
@@ -166,6 +166,7 @@ define L_CPU offsetof(struct lwp, l_cpu)
define L_PROC offsetof(struct lwp, l_proc)
define L_PRIVATE offsetof(struct lwp, l_private)
define L_FLAG offsetof(struct lwp, l_flag)
+define L_MD_ASTPENDING offsetof(struct lwp, l_md.md_astpending)
define L_MD_FLAGS offsetof(struct lwp, l_md.md_flags)
define L_MD_TF offsetof(struct lwp, l_md.md_tf)
define MDLWP_NOALIGNFLT MDLWP_NOALIGNFLT
@@ -225,7 +226,6 @@ define CF_CONTROL offsetof(struct cpu_functions, cf_control)
define CPU_INFO_SIZE sizeof(struct cpu_info)
define CI_ARM_CPUID offsetof(struct cpu_info, ci_arm_cpuid)
-define CI_ASTPENDING offsetof(struct cpu_info, ci_astpending)
define CI_CPL offsetof(struct cpu_info, ci_cpl)
define CI_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CI_INTR_DEPTH offsetof(struct cpu_info, ci_intr_depth)
diff --git a/sys/arch/arm/include/arm32/frame.h b/sys/arch/arm/include/arm32/frame.h
index dd21082ba18..b3d59fde306 100644
--- a/sys/arch/arm/include/arm32/frame.h
+++ b/sys/arch/arm/include/arm32/frame.h
@@ -1,4 +1,4 @@
-/* $NetBSD: frame.h,v 1.47 2018/10/28 14:46:59 skrll Exp $ */
+/* $NetBSD: frame.h,v 1.48 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright (c) 1994-1997 Mark Brinicombe.
@@ -151,26 +151,16 @@ void validate_trapframe(trapframe_t *, int);
msr cpsr_c, ra /* Restore interrupts */
#endif
-#ifdef __HAVE_PREEMPTION
-#define DO_CLEAR_ASTPENDING \
- mvn r1, #1 /* complement of 1 */ ;\
- add r0, r4, #CI_ASTPENDING /* address of astpending */ ;\
- bl _C_LABEL(atomic_and_uint) /* clear AST */
-#else
-#define DO_CLEAR_ASTPENDING \
- mov r0, #0 ;\
- str r0, [r4, #CI_ASTPENDING] /* clear AST */
-#endif
-
#define DO_PENDING_AST(lbl) ;\
-1: ldr r1, [r4, #CI_ASTPENDING] /* Pending AST? */ ;\
- tst r1, #0x00000001 ;\
+1: ldr r1, [r5, #L_MD_ASTPENDING] /* Pending AST? */ ;\
+ tst r1, #1 ;\
beq lbl /* Nope. Just bail */ ;\
- DO_CLEAR_ASTPENDING ;\
- CPSIE_I(r5, r5) /* Restore interrupts */ ;\
+ bic r0, r1, #1 /* clear AST */ ;\
+ str r0, [r5, #L_MD_ASTPENDING] ;\
+ CPSIE_I(r6, r6) /* Restore interrupts */ ;\
mov r0, sp ;\
bl _C_LABEL(ast) /* ast(frame) */ ;\
- CPSID_I(r0, r5) /* Disable interrupts */ ;\
+ CPSID_I(r0, r6) /* Disable interrupts */ ;\
b 1b /* test again */
/*
@@ -179,8 +169,8 @@ void validate_trapframe(trapframe_t *, int);
* alignment faults when executing old a.out ARM binaries.
*
* Note that when ENABLE_ALIGNMENTS_FAULTS finishes r4 will contain
- * pointer to the cpu's cpu_info. DO_AST_AND_RESTORE_ALIGNMENT_FAULTS
- * relies on r4 being preserved.
+ * curcpu() and r5 containing curlwp. DO_AST_AND_RESTORE_ALIGNMENT_FAULTS
+ * relies on r4 and r5 being preserved.
*/
#ifdef EXEC_AOUT
#define AST_ALIGNMENT_FAULT_LOCALS \
@@ -198,10 +188,9 @@ void validate_trapframe(trapframe_t *, int);
#define ENABLE_ALIGNMENT_FAULTS \
and r7, r0, #(PSR_MODE) /* Test for USR32 mode */ ;\
cmp r7, #(PSR_USR32_MODE) ;\
- GET_CURCPU(r4) /* r4 = cpuinfo */ ;\
+ GET_CURX(r4, r5) /* r4 = curcpu, r5 = curlwp */ ;\
bne 1f /* Not USR mode skip AFLT */ ;\
- ldr r1, [r4, #CI_CURLWP] /* get curlwp from cpu_info */ ;\
- ldr r1, [r1, #L_MD_FLAGS] /* Fetch l_md.md_flags */ ;\
+ ldr r1, [r5, #L_MD_FLAGS] /* Fetch l_md.md_flags */ ;\
tst r1, #MDLWP_NOALIGNFLT ;\
beq 1f /* AFLTs already enabled */ ;\
ldr r2, .Laflt_cpufuncs ;\
@@ -213,13 +202,13 @@ void validate_trapframe(trapframe_t *, int);
/*
* This macro must be invoked just before PULLFRAMEFROMSVCANDEXIT or
* PULLFRAME at the end of interrupt/exception handlers. We know that
- * r4 points to cpu_info since that is what ENABLE_ALIGNMENT_FAULTS did
- * for use.
+ * r4 points to curcpu() and r5 points to curlwp since that is what
+ * ENABLE_ALIGNMENT_FAULTS did for us.
*/
#define DO_AST_AND_RESTORE_ALIGNMENT_FAULTS \
DO_PENDING_SOFTINTS ;\
- GET_CPSR(r5) /* save CPSR */ ;\
- CPSID_I(r1, r5) /* Disable interrupts */ ;\
+ GET_CPSR(r6) /* save CPSR */ ;\
+ CPSID_I(r1, r6) /* Disable interrupts */ ;\
cmp r7, #(PSR_USR32_MODE) /* Returning to USR mode? */ ;\
bne 3f /* Nope, get out now */ ;\
DO_PENDING_AST(2f) /* Pending AST? */ ;\
@@ -240,13 +229,13 @@ void validate_trapframe(trapframe_t *, int);
#define ENABLE_ALIGNMENT_FAULTS \
and r7, r0, #(PSR_MODE) /* Test for USR32 mode */ ;\
- GET_CURCPU(r4) /* r4 = cpuinfo */
+ GET_CURX(r4, r5) /* r4 = curcpu, r5 = curlwp */
#define DO_AST_AND_RESTORE_ALIGNMENT_FAULTS \
DO_PENDING_SOFTINTS ;\
- GET_CPSR(r5) /* save CPSR */ ;\
- CPSID_I(r1, r5) /* Disable interrupts */ ;\
+ GET_CPSR(r6) /* save CPSR */ ;\
+ CPSID_I(r1, r6) /* Disable interrupts */ ;\
cmp r7, #(PSR_USR32_MODE) ;\
bne 2f /* Nope, get out now */ ;\
DO_PENDING_AST(2f) /* Pending AST? */ ;\
diff --git a/sys/arch/arm/include/cpu.h b/sys/arch/arm/include/cpu.h
index b01826cf96d..795af9918d3 100644
--- a/sys/arch/arm/include/cpu.h
+++ b/sys/arch/arm/include/cpu.h
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.111 2020/06/29 23:54:06 riastradh Exp $ */
+/* $NetBSD: cpu.h,v 1.112 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -154,6 +154,7 @@ static inline void cpu_dosoftints(void);
#include <sys/cpu_data.h>
#include <sys/device_if.h>
#include <sys/evcnt.h>
+#include <sys/param.h>
struct cpu_info {
struct cpu_data ci_data; /* MI per-cpu data */
@@ -163,22 +164,32 @@ struct cpu_info {
uint32_t ci_arm_cputype; /* CPU type */
uint32_t ci_arm_cpurev; /* CPU revision */
uint32_t ci_ctrl; /* The CPU control register */
- int ci_cpl; /* current processor level (spl) */
- volatile int ci_astpending; /* */
- int ci_want_resched;/* resched() was called */
- int ci_intr_depth; /* */
- int ci_kfpu_spl;
+ /*
+ * the following are in their own cache line, as they are stored to
+ * regularly by remote CPUs; when they were mixed with other fields
+ * we observed frequent cache misses.
+ */
+ int ci_want_resched __aligned(COHERENCY_UNIT);
+ /* resched() was called */
+ lwp_t * ci_curlwp __aligned(COHERENCY_UNIT);
+ /* current lwp */
+ lwp_t * ci_onproc; /* current user LWP / kthread */
+
+ /*
+ * largely CPU-private.
+ */
+ lwp_t * ci_softlwps[SOFTINT_COUNT] __aligned(COHERENCY_UNIT);
struct cpu_softc *
ci_softc; /* platform softc */
- lwp_t * ci_softlwps[SOFTINT_COUNT];
- volatile uint32_t
- ci_softints;
+ int ci_cpl; /* current processor level (spl) */
+ int ci_kfpu_spl;
+
+ volatile u_int ci_intr_depth; /* */
+ volatile u_int ci_softints;
- lwp_t * ci_curlwp; /* current lwp */
- lwp_t * ci_onproc; /* current user LWP / kthread */
lwp_t * ci_lastlwp; /* last lwp */
struct evcnt ci_arm700bugcount;
@@ -315,19 +326,8 @@ cpu_dosoftints(void)
/*
* Scheduling glue
*/
-
-#ifdef __HAVE_PREEMPTION
-#define setsoftast(ci) atomic_or_uint(&(ci)->ci_astpending, __BIT(0))
-#else
-#define setsoftast(ci) ((ci)->ci_astpending = __BIT(0))
-#endif
-
-/*
- * Notify the current process (p) that it has a signal pending,
- * process as soon as possible.
- */
-
-#define cpu_signotify(l) setsoftast((l)->l_cpu)
+void cpu_signotify(struct lwp *);
+#define setsoftast(ci) (cpu_signotify((ci)->ci_onproc))
/*
* Give a profiling tick to the current process when the user profiling
@@ -335,7 +335,7 @@ cpu_dosoftints(void)
* through trap(), marking the proc as needing a profiling tick.
*/
#define cpu_need_proftick(l) ((l)->l_pflag |= LP_OWEUPC, \
- setsoftast((l)->l_cpu))
+ setsoftast(lwp_getcpu(l)))
/*
* We've already preallocated the stack for the idlelwps for additional CPUs.
diff --git a/sys/arch/arm/include/locore.h b/sys/arch/arm/include/locore.h
index 453a17ba6f3..ced9b5ccf58 100644
--- a/sys/arch/arm/include/locore.h
+++ b/sys/arch/arm/include/locore.h
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.h,v 1.32 2020/02/15 08:16:11 skrll Exp $ */
+/* $NetBSD: locore.h,v 1.33 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -87,18 +87,23 @@
#if defined (TPIDRPRW_IS_CURCPU)
#define GET_CURCPU(rX) mrc p15, 0, rX, c13, c0, 4
#define GET_CURLWP(rX) GET_CURCPU(rX); ldr rX, [rX, #CI_CURLWP]
+#define GET_CURX(rCPU, rLWP) GET_CURCPU(rCPU); ldr rLWP, [rCPU, #CI_CURLWP]
#elif defined (TPIDRPRW_IS_CURLWP)
#define GET_CURLWP(rX) mrc p15, 0, rX, c13, c0, 4
#if defined (MULTIPROCESSOR)
#define GET_CURCPU(rX) GET_CURLWP(rX); ldr rX, [rX, #L_CPU]
+#define GET_CURX(rCPU, rLWP) GET_CURLWP(rLWP); ldr rCPU, [rLWP, #L_CPU]
#elif defined(_ARM_ARCH_7)
#define GET_CURCPU(rX) movw rX, #:lower16:cpu_info_store; movt rX, #:upper16:cpu_info_store
+#define GET_CURX(rCPU, rLWP) GET_CURLWP(rLWP); GET_CURCPU(rCPU)
#else
#define GET_CURCPU(rX) ldr rX, =_C_LABEL(cpu_info_store)
+#define GET_CURX(rCPU, rLWP) GET_CURLWP(rLWP); ldr rCPU, [rLWP, #L_CPU]
#endif
#elif !defined(MULTIPROCESSOR)
#define GET_CURCPU(rX) ldr rX, =_C_LABEL(cpu_info_store)
#define GET_CURLWP(rX) GET_CURCPU(rX); ldr rX, [rX, #CI_CURLWP]
+#define GET_CURX(rCPU, rLWP) GET_CURCPU(rCPU); ldr rLWP, [rCPU, #CI_CURLWP]
#endif
#define GET_CURPCB(rX) GET_CURLWP(rX); ldr rX, [rX, #L_PCB]
diff --git a/sys/arch/arm/include/proc.h b/sys/arch/arm/include/proc.h
index a2c69794da9..cf6a3b59768 100644
--- a/sys/arch/arm/include/proc.h
+++ b/sys/arch/arm/include/proc.h
@@ -1,4 +1,4 @@
-/* $NetBSD: proc.h,v 1.18 2017/10/31 12:37:23 martin Exp $ */
+/* $NetBSD: proc.h,v 1.19 2020/08/14 16:18:36 skrll Exp $ */
/*
* Copyright (c) 1994 Mark Brinicombe.
@@ -45,6 +45,7 @@ struct lwp;
struct mdlwp {
struct trapframe *md_tf;
int md_flags;
+ volatile uint32_t md_astpending;
};
/* Flags setttings for md_flags */
diff --git a/sys/arch/evbarm/conf/std.generic b/sys/arch/evbarm/conf/std.generic
index f3d1c697ffb..b133a1a0474 100644
--- a/sys/arch/evbarm/conf/std.generic
+++ b/sys/arch/evbarm/conf/std.generic
@@ -1,4 +1,4 @@
-# $NetBSD: std.generic,v 1.7 2020/01/25 18:10:11 skrll Exp $
+# $NetBSD: std.generic,v 1.8 2020/08/14 16:18:37 skrll Exp $
#
# generic NetBSD/evbarm with FDT support
@@ -22,7 +22,7 @@ options FPU_VFP
options MODULAR
options MODULAR_DEFAULT_AUTOLOAD
options PCI_NETBSD_CONFIGURE
-options TPIDRPRW_IS_CURCPU
+options TPIDRPRW_IS_CURLWP
options __BUS_SPACE_HAS_STREAM_METHODS
options __HAVE_CPU_COUNTER
options __HAVE_CPU_UAREA_ALLOC_IDLELWP