summaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys')
-rw-r--r--sys/arch/i386/i386/i386_trap.S2
-rw-r--r--sys/arch/i386/i386/vector.S92
-rw-r--r--sys/arch/x86/x86/syscall.c13
-rw-r--r--sys/lockdoc/init_data.c10
-rw-r--r--sys/lockdoc/log.c86
-rw-r--r--sys/sys/lockdoc.h21
6 files changed, 170 insertions, 54 deletions
diff --git a/sys/arch/i386/i386/i386_trap.S b/sys/arch/i386/i386/i386_trap.S
index 239bbbf4e77..344619bf1a4 100644
--- a/sys/arch/i386/i386/i386_trap.S
+++ b/sys/arch/i386/i386/i386_trap.S
@@ -400,6 +400,8 @@ END(trap_return_fault_return)
/* LINTSTUB: Ignore */
ENTRY(alltraps)
INTRENTRY
+ movl $__trace_irqs_off_asm,%eax
+ call *%eax
STI(%eax)
calltrap:
diff --git a/sys/arch/i386/i386/vector.S b/sys/arch/i386/i386/vector.S
index 5ba3e9c37cc..00c4b2eb566 100644
--- a/sys/arch/i386/i386/vector.S
+++ b/sys/arch/i386/i386/vector.S
@@ -310,10 +310,14 @@ IDTVEC(resume_lapic_ltimer)
pushl %ebx
IDEPTH_INCR
movb $IPL_CLOCK,CPUVAR(ILEVEL)
+ movl $__trace_irqs_off_asm,%eax
+ call *%eax
sti
pushl $0
call _C_LABEL(lapic_clockintr)
addl $4,%esp
+ movl $__trace_irqs_on_asm,%eax
+ call *%eax
cli
jmp _C_LABEL(Xdoreti)
2:
@@ -370,6 +374,7 @@ IDTVEC_END(resume_hyperv_hypercall)
* for specific PICs.
*/
+#ifdef LOCKDOC
#define INTRSTUB1(name, num, sub, off, early_ack, late_ack, mask, unmask, level_mask) \
IDTVEC(recurse_ ## name ## num) ;\
INTR_RECURSE_HWFRAME ;\
@@ -406,6 +411,8 @@ IDTVEC(intr_ ## name ## num) ;\
movb %bl,CPUVAR(ILEVEL) ;\
/* switch stack if necessary, and push a ptr to our intrframe */ \
IDEPTH_INCR ;\
+ movl $__trace_irqs_off_asm,%eax ;\
+ call *%eax ;\
sti ;\
movl IS_HANDLERS(%ebp),%ebx ;\
cmpl $0,IS_MASK_COUNT(%ebp) /* source currently masked? */ ;\
@@ -430,6 +437,8 @@ IDTVEC(intr_ ## name ## num) ;\
jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\
7: \
cli ;\
+ movl $__trace_irqs_on_asm,%eax ;\
+ call *%eax ;\
btsl $(num - sub),CPUVAR(IPENDING) + off ;\
8: level_mask(num) ;\
late_ack(num) ;\
@@ -451,6 +460,89 @@ IDTVEC(intr_ ## name ## num) ;\
addl $4,%esp ;\
INTRFASTEXIT ;\
IDTVEC_END(intr_ ## name ## num)
+#else
+#define INTRSTUB1(name, num, sub, off, early_ack, late_ack, mask, unmask, level_mask) \
+IDTVEC(recurse_ ## name ## num) ;\
+ INTR_RECURSE_HWFRAME ;\
+ subl $4,%esp ;\
+ pushl $T_ASTFLT /* trap # for doing ASTs */ ;\
+ INTRENTRY ;\
+IDTVEC_END(recurse_ ## name ## num) ;\
+IDTVEC(resume_ ## name ## num) \
+ movl $IREENT_MAGIC,TF_ERR(%esp) ;\
+ movl %ebx,%esi ;\
+ movl CPUVAR(ISOURCES) + (num) * 4,%ebp ;\
+ movl IS_MAXLEVEL(%ebp),%ebx ;\
+ jmp 1f ;\
+IDTVEC_END(resume_ ## name ## num) ;\
+IDTVEC(intr_ ## name ## num) ;\
+ pushl $0 /* dummy error code */ ;\
+ pushl $T_ASTFLT /* trap # for doing ASTs */ ;\
+ INTRENTRY ;\
+ movl CPUVAR(ISOURCES) + (num) * 4,%ebp ;\
+ mask(num) /* mask it in hardware */ ;\
+ early_ack(num) /* and allow other intrs */ ;\
+ testl %ebp,%ebp ;\
+ jz 9f /* stray */ ;\
+ movl IS_MAXLEVEL(%ebp),%ebx ;\
+ movzbl CPUVAR(ILEVEL),%esi ;\
+ cmpl %ebx,%esi ;\
+ jae 10f /* currently masked; hold it */ ;\
+ addl $1,CPUVAR(NINTR) /* statistical info */ ;\
+ adcl $0,CPUVAR(NINTR)+4 ;\
+ addl $1,IS_EVCNTLO(%ebp) /* inc event counter */ ;\
+ adcl $0,IS_EVCNTHI(%ebp) ;\
+1: \
+ pushl %esi /* if_ppi */ ;\
+ movb %bl,CPUVAR(ILEVEL) ;\
+ /* switch stack if necessary, and push a ptr to our intrframe */ \
+ IDEPTH_INCR ;\
+ sti ;\
+ movl IS_HANDLERS(%ebp),%ebx ;\
+ cmpl $0,IS_MASK_COUNT(%ebp) /* source currently masked? */ ;\
+ jne 12f /* yes, hold it */ ;\
+6: \
+ movl IH_LEVEL(%ebx),%eax ;\
+ cmpl %esi,%eax ;\
+ jle 7f ;\
+ pushl IH_ARG(%ebx) ;\
+ movl IH_FUN(%ebx),%edi ;\
+ movb %al,CPUVAR(ILEVEL) ;\
+ movl IH_NEXT(%ebx),%ebx /* next handler in chain */ ;\
+ call *%edi /* call it */ ;\
+ addl $4,%esp /* toss the arg */ ;\
+ testl %ebx,%ebx ;\
+ jnz 6b ;\
+ cmpl $0,IS_MASK_COUNT(%ebp) /* source now masked? */ ;\
+ jne 12f /* yes, deal */ ;\
+ cli ;\
+ unmask(num) /* unmask it in hardware */ ;\
+ late_ack(num) ;\
+ jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\
+7: \
+ cli ;\
+ btsl $(num - sub),CPUVAR(IPENDING) + off ;\
+8: level_mask(num) ;\
+ late_ack(num) ;\
+ jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\
+12: \
+ cli ;\
+ btsl $(num - sub),CPUVAR(IMASKED) + off ;\
+ btrl $(num - sub),CPUVAR(IPENDING) + off ;\
+ jmp 8b ;\
+10: \
+ btsl $(num - sub),CPUVAR(IPENDING) + off ;\
+ level_mask(num) ;\
+ late_ack(num) ;\
+ INTRFASTEXIT ;\
+9: \
+ pushl %esp /* for unmask */ ;\
+ unmask(num) ;\
+ late_ack(num) ;\
+ addl $4,%esp ;\
+ INTRFASTEXIT ;\
+IDTVEC_END(intr_ ## name ## num)
+#endif
#define INTRSTUB(name, num, early_ack, late_ack, mask, unmask, level_mask) \
INTRSTUB1(name, num, 0, 0, early_ack, late_ack, mask, unmask, level_mask)
diff --git a/sys/arch/x86/x86/syscall.c b/sys/arch/x86/x86/syscall.c
index 13ffda378c7..ccda9d609b9 100644
--- a/sys/arch/x86/x86/syscall.c
+++ b/sys/arch/x86/x86/syscall.c
@@ -49,6 +49,10 @@ __KERNEL_RCSID(0, "$NetBSD: syscall.c,v 1.21 2022/03/17 22:22:49 riastradh Exp $
#include "opt_dtrace.h"
+#ifdef LOCKDOC
+#include <sys/lockdoc.h>
+#endif
+
#ifndef __x86_64__
int x86_copyargs(void *, void *, size_t);
#endif
@@ -111,6 +115,10 @@ syscall(struct trapframe *frame)
code = X86_TF_RAX(frame) & (SYS_NSYSENT - 1);
callp = p->p_emul->e_sysent + code;
+#ifdef LOCKDOC
+ trace_irqs_off_check(frame->tf_eflags);
+#endif
+
SYSCALL_COUNT(syscall_counts, code);
SYSCALL_TIME_SYS_ENTRY(l, syscall_times, code);
@@ -163,6 +171,11 @@ syscall(struct trapframe *frame)
}
SYSCALL_TIME_SYS_EXIT(l);
+
+#ifdef LOCKDOC
+ trace_irqs_on_check(frame->tf_eflags);
+#endif
+
userret(l);
}
diff --git a/sys/lockdoc/init_data.c b/sys/lockdoc/init_data.c
index bce4ea7a20f..12d7b752547 100644
--- a/sys/lockdoc/init_data.c
+++ b/sys/lockdoc/init_data.c
@@ -1,4 +1,6 @@
#include <arch/x86/include/cpufunc.h>
+#include <arch/i386/include/pio.h>
+
#include <sys/lwp.h>
#include <sys/lockdoc.h>
@@ -15,7 +17,7 @@ void lockdoc_send_current_task_addr(void) {
la_buffer.action = LOCKDOC_CURRENT_TASK;
la_buffer.ptr = (uint32_t)((uint32_t)curcpu()->ci_self + offsetof(struct cpu_info, ci_curlwp));
- outb_(PING_CHAR,IO_PORT_LOG);
+ lockdoc_outb(PING_CHAR,IO_PORT_LOG);
x86_write_flags(flags);
}
@@ -30,7 +32,7 @@ void lockdoc_send_lwp_flag_offset(void) {
la_buffer.action = LOCKDOC_LWP_FLAG_OFFSET;
la_buffer.ptr = offsetof(struct lwp, l_pflag);
- outb_(PING_CHAR,IO_PORT_LOG);
+ lockdoc_outb(PING_CHAR,IO_PORT_LOG);
x86_write_flags(flags);
}
@@ -45,7 +47,7 @@ void lockdoc_send_pid_offset(void) {
la_buffer.action = LOCKDOC_PID_OFFSET;
la_buffer.ptr = offsetof(struct lwp, l_lid);
- outb_(PING_CHAR,IO_PORT_LOG);
+ lockdoc_outb(PING_CHAR,IO_PORT_LOG);
x86_write_flags(flags);
}
@@ -60,7 +62,7 @@ void lockdoc_send_kernel_version(void) {
snprintf((char*)&la_buffer.type, LOG_CHAR_BUFFER_LEN, "%s", "notimplemented"); // TODO Implement
la_buffer.action = LOCKDOC_KERNEL_VERSION;
- outb_(PING_CHAR,IO_PORT_LOG);
+ lockdoc_outb(PING_CHAR,IO_PORT_LOG);
x86_write_flags(flags);
}
diff --git a/sys/lockdoc/log.c b/sys/lockdoc/log.c
index b8207e54d46..7454713a353 100644
--- a/sys/lockdoc/log.c
+++ b/sys/lockdoc/log.c
@@ -12,28 +12,28 @@ struct log_action la_buffer;
// Mutexes are always considered write locks
inline void __mutex_enter(kmutex_t * lock, const char* file, int line, const char* func) {
_mutex_enter(lock);
- lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0);
}
inline void __mutex_exit(kmutex_t * lock, const char* file, int line, const char* func) {
- lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0);
_mutex_exit(lock);
}
inline void __mutex_spin_enter(kmutex_t * lock, const char* file, int line, const char* func) {
_mutex_spin_enter(lock);
- lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0);
}
inline void __mutex_spin_exit(kmutex_t * lock, const char* file, int line, const char* func) {
- lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0);
_mutex_spin_exit(lock);
}
inline int __mutex_tryenter(kmutex_t * lock, const char* file, int line, const char* func) {
int enter = _mutex_tryenter(lock);
if(enter != 0) {
- lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0);
}
return enter;
}
@@ -44,46 +44,42 @@ inline int __mutex_tryenter(kmutex_t * lock, const char* file, int line, const c
inline int __rw_tryupgrade(krwlock_t * lock, const char* file, int line, const char* func) {
int upgrade = _rw_tryupgrade(lock);
if(upgrade != 0) {
- lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
- lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, 0);
+ lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0);
}
return upgrade;
}
inline void __rw_downgrade(krwlock_t * lock, const char* file, int line, const char* func) {
_rw_downgrade(lock);
- lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
- lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0);
+ lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, 0);
}
inline void __rw_enter(krwlock_t * lock, const krw_t type, const char* file, int line, const char* func) {
_rw_enter(lock, type);
if(type == RW_READER)
- lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, 0);
else
- lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0);
}
inline int __rw_tryenter(krwlock_t * lock, const krw_t type, const char* file, int line, const char* func) {
int enter = _rw_tryenter(lock, type);
if(enter != 0) {
if(type == RW_READER)
- lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, 0);
else
- lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0);
}
return enter;
}
inline void __rw_exit(krwlock_t * lock, const char* file, int line, const char* func) {
- /*
- * See sys/sys/rwlock.h
- * We cannot include this directly due to circular dependencies.
- */
if(rw_lock_op(lock) == RW_WRITER)
- lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0);
else
- lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE);
+ lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, 0);
_rw_exit(lock);
}
#undef LOCK_TYPE
@@ -93,7 +89,7 @@ inline void __x86_disable_intr(const char* file, int line, const char* func) {
eflags = x86_read_flags();
if (eflags & (1 << 9)){ // Check if interrupts were enabled in the first place
- lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE);
+ lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0);
}
_x86_disable_intr();
@@ -104,43 +100,53 @@ inline void __x86_enable_intr(const char* file, int line, const char* func) {
eflags = x86_read_flags();
if (!(eflags & (1 << 9))){ // Check if interrupts were disabled in the first place
- lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE);
+ lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0);
}
_x86_enable_intr();
}
-int32_t lockdoc_get_ctx(void) {
- if (curlwp == NULL) {
- return 0;
- } else if (curlwp->l_pflag & LP_INTR) {
- return -1;
- } else {
- return curlwp->l_lid;
+void __trace_irqs_on(const char *file, int line) {
+ lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0);
+}
+
+void __trace_irqs_on_check(int tf_eflags, const char *file, int line) {
+ u_long cur_eflags;
+ cur_eflags = x86_read_flags();
+ if ((tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) {
+ lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0);
}
}
-void __trace_irqs_on(struct trapframe *frame, const char *file, int line) {
- lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE);
+/* Since we cannot use the C DEFINES in interrupt handling asm routines */
+void __trace_irqs_on_asm(void) {
+ __trace_irqs_on("asm", 0);
}
-void __trace_irqs_on_check(struct trapframe *frame, const char *file, int line) {
+void __trace_irqs_off(const char *file, int line) {
+ lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0);
+}
+
+void __trace_irqs_off_check(int tf_eflags, const char *file, int line) {
u_long cur_eflags;
cur_eflags = x86_read_flags();
- if ((frame->tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) {
- lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE);
+ if ((tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) {
+ lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0);
}
}
-void __trace_irqs_off(struct trapframe *frame, const char *file, int line) {
- lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", frame->tf_trapno);
+/* Since we cannot use the C DEFINES in interrupt handling asm routines */
+void __trace_irqs_off_asm(void) {
+ __trace_irqs_off("asm", 0);
}
-void __trace_irqs_off_check(struct trapframe *frame, const char *file, int line) {
- u_long cur_eflags;
- cur_eflags = x86_read_flags();
- if ((frame->tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) {
- lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", frame->tf_trapno);
+int32_t lockdoc_get_ctx(void) {
+ if (curlwp == NULL) {
+ return 0;
+ } else if (curlwp->l_pflag & LP_INTR) {
+ return -1;
+ } else {
+ return curlwp->l_lid;
}
}
diff --git a/sys/sys/lockdoc.h b/sys/sys/lockdoc.h
index 6d8ed6ef4ff..abd17d6b21a 100644
--- a/sys/sys/lockdoc.h
+++ b/sys/sys/lockdoc.h
@@ -48,10 +48,12 @@ void __rw_exit(krwlock_t *, const char*, int, const char*);
void __x86_disable_intr(const char*, int, const char*);
void __x86_enable_intr(const char*, int, const char*);
-void __trace_irqs_on(struct trapframe *, const char *, int);
-void __trace_irqs_on_check(struct trapframe *, const char *, int);
-void __trace_irqs_off(struct trapframe *, const char *, int);
-void __trace_irqs_off_check(struct trapframe *, const char *, int);
+void __trace_irqs_on(const char *, int);
+void __trace_irqs_on_check(int, const char *, int);
+void __trace_irqs_on_asm(void);
+void __trace_irqs_off(const char *, int);
+void __trace_irqs_off_check(int, const char *, int);
+void __trace_irqs_off_asm(void);
/* Helper function to get a lock type (backported from NetBSD 10.0-STABLE) */
krw_t rw_lock_op(krwlock_t *rw);
@@ -76,14 +78,13 @@ krw_t rw_lock_op(krwlock_t *rw);
#define x86_disable_intr() __x86_disable_intr(__FILE__, __LINE__, __func__)
#define x86_enable_intr() __x86_enable_intr(__FILE__, __LINE__, __func__)
-#define trace_irqs_on(frame) __trace_irqs_on(frame, __FILE__, __LINE__)
+#define trace_irqs_on() __trace_irqs_on(__FILE__, __LINE__)
#define trace_irqs_on_check(frame) __trace_irqs_on_check(frame, __FILE__, __LINE__)
-#define trace_irqs_off(frame) __trace_irqs_off(frame, __FILE__, __LINE__)
+#define trace_irqs_off() __trace_irqs_off(__FILE__, __LINE__)
#define trace_irqs_off_check(frame) __trace_irqs_off_check(frame, __FILE__, __LINE__)
-/* Basic port I/O */
-static inline void outb_(u_int8_t v, u_int16_t port)
+static inline void lockdoc_outb(u_int8_t v, u_int16_t port)
{
__asm __volatile("outb %0,%1" : : "a" (v), "dN" (port));
}
@@ -110,7 +111,7 @@ static inline void lockdoc_log_memory(int alloc, const char *datatype, const voi
strncpy(la_buffer.type,datatype,LOG_CHAR_BUFFER_LEN);
la_buffer.type[LOG_CHAR_BUFFER_LEN - 1] = '\0';
- outb_(PING_CHAR,IO_PORT_LOG);
+ lockdoc_outb(PING_CHAR,IO_PORT_LOG);
x86_write_flags(eflags);
}
@@ -161,7 +162,7 @@ static inline void lockdoc_log_lock(int lock_op, const volatile void* ptr, const
// irq_sync
la_buffer.irq_sync = irq_sync;
- outb_(PING_CHAR,IO_PORT_LOG);
+ lockdoc_outb(PING_CHAR,IO_PORT_LOG);
x86_write_flags(eflags);
}