diff options
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/arch/i386/i386/i386_trap.S | 2 | ||||
| -rw-r--r-- | sys/arch/i386/i386/vector.S | 92 | ||||
| -rw-r--r-- | sys/arch/x86/x86/syscall.c | 13 | ||||
| -rw-r--r-- | sys/lockdoc/init_data.c | 10 | ||||
| -rw-r--r-- | sys/lockdoc/log.c | 86 | ||||
| -rw-r--r-- | sys/sys/lockdoc.h | 21 |
6 files changed, 170 insertions, 54 deletions
diff --git a/sys/arch/i386/i386/i386_trap.S b/sys/arch/i386/i386/i386_trap.S index 239bbbf4e77..344619bf1a4 100644 --- a/sys/arch/i386/i386/i386_trap.S +++ b/sys/arch/i386/i386/i386_trap.S @@ -400,6 +400,8 @@ END(trap_return_fault_return) /* LINTSTUB: Ignore */ ENTRY(alltraps) INTRENTRY + movl $__trace_irqs_off_asm,%eax + call *%eax STI(%eax) calltrap: diff --git a/sys/arch/i386/i386/vector.S b/sys/arch/i386/i386/vector.S index 5ba3e9c37cc..00c4b2eb566 100644 --- a/sys/arch/i386/i386/vector.S +++ b/sys/arch/i386/i386/vector.S @@ -310,10 +310,14 @@ IDTVEC(resume_lapic_ltimer) pushl %ebx IDEPTH_INCR movb $IPL_CLOCK,CPUVAR(ILEVEL) + movl $__trace_irqs_off_asm,%eax + call *%eax sti pushl $0 call _C_LABEL(lapic_clockintr) addl $4,%esp + movl $__trace_irqs_on_asm,%eax + call *%eax cli jmp _C_LABEL(Xdoreti) 2: @@ -370,6 +374,7 @@ IDTVEC_END(resume_hyperv_hypercall) * for specific PICs. */ +#ifdef LOCKDOC #define INTRSTUB1(name, num, sub, off, early_ack, late_ack, mask, unmask, level_mask) \ IDTVEC(recurse_ ## name ## num) ;\ INTR_RECURSE_HWFRAME ;\ @@ -406,6 +411,8 @@ IDTVEC(intr_ ## name ## num) ;\ movb %bl,CPUVAR(ILEVEL) ;\ /* switch stack if necessary, and push a ptr to our intrframe */ \ IDEPTH_INCR ;\ + movl $__trace_irqs_off_asm,%eax ;\ + call *%eax ;\ sti ;\ movl IS_HANDLERS(%ebp),%ebx ;\ cmpl $0,IS_MASK_COUNT(%ebp) /* source currently masked? */ ;\ @@ -430,6 +437,8 @@ IDTVEC(intr_ ## name ## num) ;\ jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\ 7: \ cli ;\ + movl $__trace_irqs_on_asm,%eax ;\ + call *%eax ;\ btsl $(num - sub),CPUVAR(IPENDING) + off ;\ 8: level_mask(num) ;\ late_ack(num) ;\ @@ -451,6 +460,89 @@ IDTVEC(intr_ ## name ## num) ;\ addl $4,%esp ;\ INTRFASTEXIT ;\ IDTVEC_END(intr_ ## name ## num) +#else +#define INTRSTUB1(name, num, sub, off, early_ack, late_ack, mask, unmask, level_mask) \ +IDTVEC(recurse_ ## name ## num) ;\ + INTR_RECURSE_HWFRAME ;\ + subl $4,%esp ;\ + pushl $T_ASTFLT /* trap # for doing ASTs */ ;\ + INTRENTRY ;\ +IDTVEC_END(recurse_ ## name ## num) ;\ +IDTVEC(resume_ ## name ## num) \ + movl $IREENT_MAGIC,TF_ERR(%esp) ;\ + movl %ebx,%esi ;\ + movl CPUVAR(ISOURCES) + (num) * 4,%ebp ;\ + movl IS_MAXLEVEL(%ebp),%ebx ;\ + jmp 1f ;\ +IDTVEC_END(resume_ ## name ## num) ;\ +IDTVEC(intr_ ## name ## num) ;\ + pushl $0 /* dummy error code */ ;\ + pushl $T_ASTFLT /* trap # for doing ASTs */ ;\ + INTRENTRY ;\ + movl CPUVAR(ISOURCES) + (num) * 4,%ebp ;\ + mask(num) /* mask it in hardware */ ;\ + early_ack(num) /* and allow other intrs */ ;\ + testl %ebp,%ebp ;\ + jz 9f /* stray */ ;\ + movl IS_MAXLEVEL(%ebp),%ebx ;\ + movzbl CPUVAR(ILEVEL),%esi ;\ + cmpl %ebx,%esi ;\ + jae 10f /* currently masked; hold it */ ;\ + addl $1,CPUVAR(NINTR) /* statistical info */ ;\ + adcl $0,CPUVAR(NINTR)+4 ;\ + addl $1,IS_EVCNTLO(%ebp) /* inc event counter */ ;\ + adcl $0,IS_EVCNTHI(%ebp) ;\ +1: \ + pushl %esi /* if_ppi */ ;\ + movb %bl,CPUVAR(ILEVEL) ;\ + /* switch stack if necessary, and push a ptr to our intrframe */ \ + IDEPTH_INCR ;\ + sti ;\ + movl IS_HANDLERS(%ebp),%ebx ;\ + cmpl $0,IS_MASK_COUNT(%ebp) /* source currently masked? */ ;\ + jne 12f /* yes, hold it */ ;\ +6: \ + movl IH_LEVEL(%ebx),%eax ;\ + cmpl %esi,%eax ;\ + jle 7f ;\ + pushl IH_ARG(%ebx) ;\ + movl IH_FUN(%ebx),%edi ;\ + movb %al,CPUVAR(ILEVEL) ;\ + movl IH_NEXT(%ebx),%ebx /* next handler in chain */ ;\ + call *%edi /* call it */ ;\ + addl $4,%esp /* toss the arg */ ;\ + testl %ebx,%ebx ;\ + jnz 6b ;\ + cmpl $0,IS_MASK_COUNT(%ebp) /* source now masked? */ ;\ + jne 12f /* yes, deal */ ;\ + cli ;\ + unmask(num) /* unmask it in hardware */ ;\ + late_ack(num) ;\ + jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\ +7: \ + cli ;\ + btsl $(num - sub),CPUVAR(IPENDING) + off ;\ +8: level_mask(num) ;\ + late_ack(num) ;\ + jmp _C_LABEL(Xdoreti) /* lower spl and do ASTs */ ;\ +12: \ + cli ;\ + btsl $(num - sub),CPUVAR(IMASKED) + off ;\ + btrl $(num - sub),CPUVAR(IPENDING) + off ;\ + jmp 8b ;\ +10: \ + btsl $(num - sub),CPUVAR(IPENDING) + off ;\ + level_mask(num) ;\ + late_ack(num) ;\ + INTRFASTEXIT ;\ +9: \ + pushl %esp /* for unmask */ ;\ + unmask(num) ;\ + late_ack(num) ;\ + addl $4,%esp ;\ + INTRFASTEXIT ;\ +IDTVEC_END(intr_ ## name ## num) +#endif #define INTRSTUB(name, num, early_ack, late_ack, mask, unmask, level_mask) \ INTRSTUB1(name, num, 0, 0, early_ack, late_ack, mask, unmask, level_mask) diff --git a/sys/arch/x86/x86/syscall.c b/sys/arch/x86/x86/syscall.c index 13ffda378c7..ccda9d609b9 100644 --- a/sys/arch/x86/x86/syscall.c +++ b/sys/arch/x86/x86/syscall.c @@ -49,6 +49,10 @@ __KERNEL_RCSID(0, "$NetBSD: syscall.c,v 1.21 2022/03/17 22:22:49 riastradh Exp $ #include "opt_dtrace.h" +#ifdef LOCKDOC +#include <sys/lockdoc.h> +#endif + #ifndef __x86_64__ int x86_copyargs(void *, void *, size_t); #endif @@ -111,6 +115,10 @@ syscall(struct trapframe *frame) code = X86_TF_RAX(frame) & (SYS_NSYSENT - 1); callp = p->p_emul->e_sysent + code; +#ifdef LOCKDOC + trace_irqs_off_check(frame->tf_eflags); +#endif + SYSCALL_COUNT(syscall_counts, code); SYSCALL_TIME_SYS_ENTRY(l, syscall_times, code); @@ -163,6 +171,11 @@ syscall(struct trapframe *frame) } SYSCALL_TIME_SYS_EXIT(l); + +#ifdef LOCKDOC + trace_irqs_on_check(frame->tf_eflags); +#endif + userret(l); } diff --git a/sys/lockdoc/init_data.c b/sys/lockdoc/init_data.c index bce4ea7a20f..12d7b752547 100644 --- a/sys/lockdoc/init_data.c +++ b/sys/lockdoc/init_data.c @@ -1,4 +1,6 @@ #include <arch/x86/include/cpufunc.h> +#include <arch/i386/include/pio.h> + #include <sys/lwp.h> #include <sys/lockdoc.h> @@ -15,7 +17,7 @@ void lockdoc_send_current_task_addr(void) { la_buffer.action = LOCKDOC_CURRENT_TASK; la_buffer.ptr = (uint32_t)((uint32_t)curcpu()->ci_self + offsetof(struct cpu_info, ci_curlwp)); - outb_(PING_CHAR,IO_PORT_LOG); + lockdoc_outb(PING_CHAR,IO_PORT_LOG); x86_write_flags(flags); } @@ -30,7 +32,7 @@ void lockdoc_send_lwp_flag_offset(void) { la_buffer.action = LOCKDOC_LWP_FLAG_OFFSET; la_buffer.ptr = offsetof(struct lwp, l_pflag); - outb_(PING_CHAR,IO_PORT_LOG); + lockdoc_outb(PING_CHAR,IO_PORT_LOG); x86_write_flags(flags); } @@ -45,7 +47,7 @@ void lockdoc_send_pid_offset(void) { la_buffer.action = LOCKDOC_PID_OFFSET; la_buffer.ptr = offsetof(struct lwp, l_lid); - outb_(PING_CHAR,IO_PORT_LOG); + lockdoc_outb(PING_CHAR,IO_PORT_LOG); x86_write_flags(flags); } @@ -60,7 +62,7 @@ void lockdoc_send_kernel_version(void) { snprintf((char*)&la_buffer.type, LOG_CHAR_BUFFER_LEN, "%s", "notimplemented"); // TODO Implement la_buffer.action = LOCKDOC_KERNEL_VERSION; - outb_(PING_CHAR,IO_PORT_LOG); + lockdoc_outb(PING_CHAR,IO_PORT_LOG); x86_write_flags(flags); } diff --git a/sys/lockdoc/log.c b/sys/lockdoc/log.c index b8207e54d46..7454713a353 100644 --- a/sys/lockdoc/log.c +++ b/sys/lockdoc/log.c @@ -12,28 +12,28 @@ struct log_action la_buffer; // Mutexes are always considered write locks inline void __mutex_enter(kmutex_t * lock, const char* file, int line, const char* func) { _mutex_enter(lock); - lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0); } inline void __mutex_exit(kmutex_t * lock, const char* file, int line, const char* func) { - lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0); _mutex_exit(lock); } inline void __mutex_spin_enter(kmutex_t * lock, const char* file, int line, const char* func) { _mutex_spin_enter(lock); - lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0); } inline void __mutex_spin_exit(kmutex_t * lock, const char* file, int line, const char* func) { - lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0); _mutex_spin_exit(lock); } inline int __mutex_tryenter(kmutex_t * lock, const char* file, int line, const char* func) { int enter = _mutex_tryenter(lock); if(enter != 0) { - lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0); } return enter; } @@ -44,46 +44,42 @@ inline int __mutex_tryenter(kmutex_t * lock, const char* file, int line, const c inline int __rw_tryupgrade(krwlock_t * lock, const char* file, int line, const char* func) { int upgrade = _rw_tryupgrade(lock); if(upgrade != 0) { - lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE); - lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, 0); + lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0); } return upgrade; } inline void __rw_downgrade(krwlock_t * lock, const char* file, int line, const char* func) { _rw_downgrade(lock); - lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); - lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0); + lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, 0); } inline void __rw_enter(krwlock_t * lock, const krw_t type, const char* file, int line, const char* func) { _rw_enter(lock, type); if(type == RW_READER) - lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, 0); else - lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0); } inline int __rw_tryenter(krwlock_t * lock, const krw_t type, const char* file, int line, const char* func) { int enter = _rw_tryenter(lock, type); if(enter != 0) { if(type == RW_READER) - lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_READ, lock, file, line, func, LOCK_TYPE, 0); else - lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(P_WRITE, lock, file, line, func, LOCK_TYPE, 0); } return enter; } inline void __rw_exit(krwlock_t * lock, const char* file, int line, const char* func) { - /* - * See sys/sys/rwlock.h - * We cannot include this directly due to circular dependencies. - */ if(rw_lock_op(lock) == RW_WRITER) - lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(V_WRITE, lock, file, line, func, LOCK_TYPE, 0); else - lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, LOCK_NONE); + lockdoc_log_lock(V_READ, lock, file, line, func, LOCK_TYPE, 0); _rw_exit(lock); } #undef LOCK_TYPE @@ -93,7 +89,7 @@ inline void __x86_disable_intr(const char* file, int line, const char* func) { eflags = x86_read_flags(); if (eflags & (1 << 9)){ // Check if interrupts were enabled in the first place - lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE); + lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0); } _x86_disable_intr(); @@ -104,43 +100,53 @@ inline void __x86_enable_intr(const char* file, int line, const char* func) { eflags = x86_read_flags(); if (!(eflags & (1 << 9))){ // Check if interrupts were disabled in the first place - lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE); + lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0); } _x86_enable_intr(); } -int32_t lockdoc_get_ctx(void) { - if (curlwp == NULL) { - return 0; - } else if (curlwp->l_pflag & LP_INTR) { - return -1; - } else { - return curlwp->l_lid; +void __trace_irqs_on(const char *file, int line) { + lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0); +} + +void __trace_irqs_on_check(int tf_eflags, const char *file, int line) { + u_long cur_eflags; + cur_eflags = x86_read_flags(); + if ((tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) { + lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0); } } -void __trace_irqs_on(struct trapframe *frame, const char *file, int line) { - lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE); +/* Since we cannot use the C DEFINES in interrupt handling asm routines */ +void __trace_irqs_on_asm(void) { + __trace_irqs_on("asm", 0); } -void __trace_irqs_on_check(struct trapframe *frame, const char *file, int line) { +void __trace_irqs_off(const char *file, int line) { + lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0); +} + +void __trace_irqs_off_check(int tf_eflags, const char *file, int line) { u_long cur_eflags; cur_eflags = x86_read_flags(); - if ((frame->tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) { - lockdoc_log_lock(V_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", LOCK_NONE); + if ((tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) { + lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", 0); } } -void __trace_irqs_off(struct trapframe *frame, const char *file, int line) { - lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", frame->tf_trapno); +/* Since we cannot use the C DEFINES in interrupt handling asm routines */ +void __trace_irqs_off_asm(void) { + __trace_irqs_off("asm", 0); } -void __trace_irqs_off_check(struct trapframe *frame, const char *file, int line) { - u_long cur_eflags; - cur_eflags = x86_read_flags(); - if ((frame->tf_eflags & (1 << 9)) && !(cur_eflags & (1 << 9))) { - lockdoc_log_lock(P_WRITE, (void *)PSEUDOLOCK_ADDR_HARDIRQ, file, line, "dummy", "dummy", frame->tf_trapno); +int32_t lockdoc_get_ctx(void) { + if (curlwp == NULL) { + return 0; + } else if (curlwp->l_pflag & LP_INTR) { + return -1; + } else { + return curlwp->l_lid; } } diff --git a/sys/sys/lockdoc.h b/sys/sys/lockdoc.h index 6d8ed6ef4ff..abd17d6b21a 100644 --- a/sys/sys/lockdoc.h +++ b/sys/sys/lockdoc.h @@ -48,10 +48,12 @@ void __rw_exit(krwlock_t *, const char*, int, const char*); void __x86_disable_intr(const char*, int, const char*); void __x86_enable_intr(const char*, int, const char*); -void __trace_irqs_on(struct trapframe *, const char *, int); -void __trace_irqs_on_check(struct trapframe *, const char *, int); -void __trace_irqs_off(struct trapframe *, const char *, int); -void __trace_irqs_off_check(struct trapframe *, const char *, int); +void __trace_irqs_on(const char *, int); +void __trace_irqs_on_check(int, const char *, int); +void __trace_irqs_on_asm(void); +void __trace_irqs_off(const char *, int); +void __trace_irqs_off_check(int, const char *, int); +void __trace_irqs_off_asm(void); /* Helper function to get a lock type (backported from NetBSD 10.0-STABLE) */ krw_t rw_lock_op(krwlock_t *rw); @@ -76,14 +78,13 @@ krw_t rw_lock_op(krwlock_t *rw); #define x86_disable_intr() __x86_disable_intr(__FILE__, __LINE__, __func__) #define x86_enable_intr() __x86_enable_intr(__FILE__, __LINE__, __func__) -#define trace_irqs_on(frame) __trace_irqs_on(frame, __FILE__, __LINE__) +#define trace_irqs_on() __trace_irqs_on(__FILE__, __LINE__) #define trace_irqs_on_check(frame) __trace_irqs_on_check(frame, __FILE__, __LINE__) -#define trace_irqs_off(frame) __trace_irqs_off(frame, __FILE__, __LINE__) +#define trace_irqs_off() __trace_irqs_off(__FILE__, __LINE__) #define trace_irqs_off_check(frame) __trace_irqs_off_check(frame, __FILE__, __LINE__) -/* Basic port I/O */ -static inline void outb_(u_int8_t v, u_int16_t port) +static inline void lockdoc_outb(u_int8_t v, u_int16_t port) { __asm __volatile("outb %0,%1" : : "a" (v), "dN" (port)); } @@ -110,7 +111,7 @@ static inline void lockdoc_log_memory(int alloc, const char *datatype, const voi strncpy(la_buffer.type,datatype,LOG_CHAR_BUFFER_LEN); la_buffer.type[LOG_CHAR_BUFFER_LEN - 1] = '\0'; - outb_(PING_CHAR,IO_PORT_LOG); + lockdoc_outb(PING_CHAR,IO_PORT_LOG); x86_write_flags(eflags); } @@ -161,7 +162,7 @@ static inline void lockdoc_log_lock(int lock_op, const volatile void* ptr, const // irq_sync la_buffer.irq_sync = irq_sync; - outb_(PING_CHAR,IO_PORT_LOG); + lockdoc_outb(PING_CHAR,IO_PORT_LOG); x86_write_flags(eflags); } |
