summaryrefslogtreecommitdiff
path: root/sys/arch/amd64/include/frameasm.h
blob: 4544a4e8aa8a22d16f42ba9a07a614b10f991a6e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
/*	$NetBSD: frameasm.h,v 1.55 2022/07/30 14:11:00 riastradh Exp $	*/

#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H

#ifdef _KERNEL_OPT
#include "opt_xen.h"
#include "opt_svs.h"
#include "opt_kcov.h"
#include "opt_kmsan.h"
#endif

/*
 * Macros to define pushing/popping frames for interrupts, traps
 * and system calls. Currently all the same; will diverge later.
 */

#ifdef XENPV
#define HYPERVISOR_iret hypercall_page + (__HYPERVISOR_iret * 32)
/* Xen do not need swapgs, done by hypervisor */
#define swapgs
#define iretq	pushq $0 ; jmp HYPERVISOR_iret
#define	XEN_ONLY2(x,y)	x,y
#define	NOT_XEN(x)

#define CLI(temp_reg) \
 	movq CPUVAR(VCPU),%r ## temp_reg ;			\
	movb $1,EVTCHN_UPCALL_MASK(%r ## temp_reg);

#define STI(temp_reg) \
 	movq CPUVAR(VCPU),%r ## temp_reg ;			\
	movb $0,EVTCHN_UPCALL_MASK(%r ## temp_reg);

#define PUSHF(temp_reg) \
 	movq CPUVAR(VCPU),%r ## temp_reg ;			\
	movzbl EVTCHN_UPCALL_MASK(%r ## temp_reg), %e ## temp_reg; \
	pushq %r ## temp_reg

#define POPF \
	popq %rdi; \
	call _C_LABEL(xen_write_psl)


#else /* XENPV */
#define	XEN_ONLY2(x,y)
#define	NOT_XEN(x)	x
#define CLI(temp_reg) cli
#define STI(temp_reg) sti
#define PUSHF(temp_reg) pushf
#define POPL popl
#endif	/* XENPV */

#define HP_NAME_CLAC		1
#define HP_NAME_STAC		2
#define HP_NAME_NOLOCK		3
#define HP_NAME_RETFENCE	4
#define HP_NAME_SVS_ENTER	5
#define HP_NAME_SVS_LEAVE	6
#define HP_NAME_SVS_ENTER_ALT	7
#define HP_NAME_SVS_LEAVE_ALT	8
#define HP_NAME_IBRS_ENTER	9
#define HP_NAME_IBRS_LEAVE	10
#define HP_NAME_SVS_ENTER_NMI	11
#define HP_NAME_SVS_LEAVE_NMI	12
#define HP_NAME_MDS_LEAVE	13

#define HOTPATCH(name, size) \
123:						; \
	.pushsection	.rodata.hotpatch, "a"	; \
	.byte		name			; \
	.byte		size			; \
	.quad		123b			; \
	.popsection

#define SMAP_ENABLE \
	HOTPATCH(HP_NAME_CLAC, 3)		; \
	.byte 0x0F, 0x1F, 0x00			; \

#define SMAP_DISABLE \
	HOTPATCH(HP_NAME_STAC, 3)		; \
	.byte 0x0F, 0x1F, 0x00			; \

/*
 * IBRS
 */

#define IBRS_ENTER_BYTES	12
#define IBRS_ENTER \
	HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES)		; \
	NOIBRS_ENTER
#define NOIBRS_ENTER \
	.byte 0xEB, (IBRS_ENTER_BYTES-2)	/* jmp */	; \
	.fill	(IBRS_ENTER_BYTES-2),1,0xCC

#define IBRS_LEAVE_BYTES	12
#define IBRS_LEAVE \
	HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES)		; \
	NOIBRS_LEAVE
#define NOIBRS_LEAVE \
	.byte 0xEB, (IBRS_LEAVE_BYTES-2)	/* jmp */	; \
	.fill	(IBRS_LEAVE_BYTES-2),1,0xCC

/*
 * MDS
 */

#define MDS_LEAVE_BYTES	10
#define MDS_LEAVE \
	HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES)		; \
	NOMDS_LEAVE
#define NOMDS_LEAVE \
	.byte 0xEB, (MDS_LEAVE_BYTES-2)	/* jmp */		; \
	.fill	(MDS_LEAVE_BYTES-2),1,0xCC

#define	SWAPGS	NOT_XEN(swapgs)

/*
 * These are used on interrupt or trap entry or exit.
 */
#define INTR_SAVE_GPRS \
	movq	%rdi,TF_RDI(%rsp)	; \
	movq	%rsi,TF_RSI(%rsp)	; \
	movq	%rdx,TF_RDX(%rsp)	; \
	movq	%rcx,TF_RCX(%rsp)	; \
	movq	%r8,TF_R8(%rsp)		; \
	movq	%r9,TF_R9(%rsp)		; \
	movq	%r10,TF_R10(%rsp)	; \
	movq	%r11,TF_R11(%rsp)	; \
	movq	%r12,TF_R12(%rsp)	; \
	movq	%r13,TF_R13(%rsp)	; \
	movq	%r14,TF_R14(%rsp)	; \
	movq	%r15,TF_R15(%rsp)	; \
	movq	%rbp,TF_RBP(%rsp)	; \
	movq	%rbx,TF_RBX(%rsp)	; \
	movq	%rax,TF_RAX(%rsp)

#define	INTR_RESTORE_GPRS \
	movq	TF_RDI(%rsp),%rdi	; \
	movq	TF_RSI(%rsp),%rsi	; \
	movq	TF_RDX(%rsp),%rdx	; \
	movq	TF_RCX(%rsp),%rcx	; \
	movq	TF_R8(%rsp),%r8		; \
	movq	TF_R9(%rsp),%r9		; \
	movq	TF_R10(%rsp),%r10	; \
	movq	TF_R11(%rsp),%r11	; \
	movq	TF_R12(%rsp),%r12	; \
	movq	TF_R13(%rsp),%r13	; \
	movq	TF_R14(%rsp),%r14	; \
	movq	TF_R15(%rsp),%r15	; \
	movq	TF_RBP(%rsp),%rbp	; \
	movq	TF_RBX(%rsp),%rbx	; \
	movq	TF_RAX(%rsp),%rax

#define TEXT_USER_BEGIN	.pushsection	.text.user, "ax"
#define TEXT_USER_END	.popsection

#ifdef SVS

/* XXX: put this somewhere else */
#define SVS_UTLS		0xffffff0000000000 /* PMAP_PCPU_BASE */
#define UTLS_KPDIRPA		0
#define UTLS_SCRATCH		8
#define UTLS_RSP0		16

#define SVS_ENTER_BYTES	22
#define NOSVS_ENTER \
	.byte 0xEB, (SVS_ENTER_BYTES-2)	/* jmp */	; \
	.fill	(SVS_ENTER_BYTES-2),1,0xCC
#define SVS_ENTER \
	HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES)	; \
	NOSVS_ENTER

#define SVS_LEAVE_BYTES	21
#define NOSVS_LEAVE \
	.byte 0xEB, (SVS_LEAVE_BYTES-2)	/* jmp */	; \
	.fill	(SVS_LEAVE_BYTES-2),1,0xCC
#define SVS_LEAVE \
	HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES)	; \
	NOSVS_LEAVE

#define SVS_ENTER_ALT_BYTES	23
#define NOSVS_ENTER_ALTSTACK \
	.byte 0xEB, (SVS_ENTER_ALT_BYTES-2)	/* jmp */	; \
	.fill	(SVS_ENTER_ALT_BYTES-2),1,0xCC
#define SVS_ENTER_ALTSTACK \
	HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES)	; \
	NOSVS_ENTER_ALTSTACK

#define SVS_LEAVE_ALT_BYTES	22
#define NOSVS_LEAVE_ALTSTACK \
	.byte 0xEB, (SVS_LEAVE_ALT_BYTES-2)	/* jmp */	; \
	.fill	(SVS_LEAVE_ALT_BYTES-2),1,0xCC
#define SVS_LEAVE_ALTSTACK \
	HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES)	; \
	NOSVS_LEAVE_ALTSTACK

#define SVS_ENTER_NMI_BYTES	22
#define NOSVS_ENTER_NMI \
	.byte 0xEB, (SVS_ENTER_NMI_BYTES-2)	/* jmp */	; \
	.fill	(SVS_ENTER_NMI_BYTES-2),1,0xCC
#define SVS_ENTER_NMI \
	HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES)	; \
	NOSVS_ENTER_NMI

#define SVS_LEAVE_NMI_BYTES	11
#define NOSVS_LEAVE_NMI \
	.byte 0xEB, (SVS_LEAVE_NMI_BYTES-2)	/* jmp */	; \
	.fill	(SVS_LEAVE_NMI_BYTES-2),1,0xCC
#define SVS_LEAVE_NMI \
	HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES)	; \
	NOSVS_LEAVE_NMI

#else
#define SVS_ENTER	/* nothing */
#define SVS_ENTER_NMI	/* nothing */
#define SVS_LEAVE	/* nothing */
#define SVS_LEAVE_NMI	/* nothing */
#define SVS_ENTER_ALTSTACK	/* nothing */
#define SVS_LEAVE_ALTSTACK	/* nothing */
#endif

#ifdef KMSAN
/* XXX this belongs somewhere else. */
#define KMSAN_ENTER	\
	movq	%rsp,%rdi		; \
	movq	$TF_REGSIZE+16+40,%rsi	; \
	xorq	%rdx,%rdx		; \
	callq	kmsan_mark		; \
	callq	kmsan_intr_enter
#define KMSAN_LEAVE	\
	pushq	%rbp			; \
	movq	%rsp,%rbp		; \
	callq	kmsan_intr_leave	; \
	popq	%rbp
#define KMSAN_INIT_ARG(sz)	\
	pushq	%rax			; \
	pushq	%rcx			; \
	pushq	%rdx			; \
	pushq	%rsi			; \
	pushq	%rdi			; \
	pushq	%r8			; \
	pushq	%r9			; \
	pushq	%r10			; \
	pushq	%r11			; \
	movq	$sz,%rdi		; \
	callq	_C_LABEL(kmsan_init_arg); \
	popq	%r11			; \
	popq	%r10			; \
	popq	%r9			; \
	popq	%r8			; \
	popq	%rdi			; \
	popq	%rsi			; \
	popq	%rdx			; \
	popq	%rcx			; \
	popq	%rax
#define KMSAN_INIT_RET(sz)	\
	pushq	%rax			; \
	pushq	%rcx			; \
	pushq	%rdx			; \
	pushq	%rsi			; \
	pushq	%rdi			; \
	pushq	%r8			; \
	pushq	%r9			; \
	pushq	%r10			; \
	pushq	%r11			; \
	movq	$sz,%rdi		; \
	callq	_C_LABEL(kmsan_init_ret); \
	popq	%r11			; \
	popq	%r10			; \
	popq	%r9			; \
	popq	%r8			; \
	popq	%rdi			; \
	popq	%rsi			; \
	popq	%rdx			; \
	popq	%rcx			; \
	popq	%rax
#else
#define KMSAN_ENTER		/* nothing */
#define KMSAN_LEAVE		/* nothing */
#define KMSAN_INIT_ARG(sz)	/* nothing */
#define KMSAN_INIT_RET(sz)	/* nothing */
#endif

#ifdef KCOV
#define KCOV_DISABLE			\
	incl	CPUVAR(IDEPTH)
#define KCOV_ENABLE			\
	decl	CPUVAR(IDEPTH)
#else
#define KCOV_DISABLE		/* nothing */
#define KCOV_ENABLE		/* nothing */
#endif

#define	INTRENTRY \
	subq	$TF_REGSIZE,%rsp	; \
	INTR_SAVE_GPRS			; \
	cld				; \
	SMAP_ENABLE			; \
	testb	$SEL_UPL,TF_CS(%rsp)	; \
	je	98f			; \
	SWAPGS				; \
	IBRS_ENTER			; \
	SVS_ENTER			; \
	movw	%gs,TF_GS(%rsp)		; \
	movw	%fs,TF_FS(%rsp)		; \
	movw	%es,TF_ES(%rsp)		; \
	movw	%ds,TF_DS(%rsp)		; \
98:	KMSAN_ENTER

#define INTRFASTEXIT \
	jmp	intrfastexit

#define INTR_RECURSE_HWFRAME \
	movq	%rsp,%r10		; \
	movl	%ss,%r11d		; \
	pushq	%r11			; \
	pushq	%r10			; \
	pushfq				; \
	pushq	$GSEL(GCODE_SEL,SEL_KPL); \
/* XEN: We must fixup CS, as even kernel mode runs at CPL 3 */ \
 	XEN_ONLY2(andb	$0xfc,(%rsp);)	  \
	pushq	%r13			;

#define INTR_RECURSE_ENTRY \
	subq	$TF_REGSIZE,%rsp	; \
	INTR_SAVE_GPRS			; \
	cld				; \
	KMSAN_ENTER

#define	CHECK_DEFERRED_SWITCH \
	cmpl	$0, CPUVAR(WANT_PMAPLOAD)

#define CHECK_ASTPENDING(reg)	cmpl	$0, L_MD_ASTPENDING(reg)
#define CLEAR_ASTPENDING(reg)	movl	$0, L_MD_ASTPENDING(reg)

/*
 * If the FPU state is not in the CPU, restore it. Executed with interrupts
 * disabled.
 *
 *     %r14 is curlwp, must not be modified
 *     %rbx must not be modified
 */
#define HANDLE_DEFERRED_FPU	\
	testl	$MDL_FPU_IN_CPU,L_MD_FLAGS(%r14)	; \
	jnz	1f					; \
	call	_C_LABEL(fpu_handle_deferred)		; \
	orl	$MDL_FPU_IN_CPU,L_MD_FLAGS(%r14)	; \
1:

#endif /* _AMD64_MACHINE_FRAMEASM_H */