summaryrefslogtreecommitdiff
path: root/sys/arch/aarch64/include/pmap_machdep.h
blob: db42e8b511b2d072f259265fdbee7ca68b6ae0e2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
/*	$NetBSD: pmap_machdep.h,v 1.3 2023/04/20 08:28:03 skrll Exp $	*/

/*-
 * Copyright (c) 2022 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Nick Hudson
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef	_AARCH64_PMAP_MACHDEP_H_
#define	_AARCH64_PMAP_MACHDEP_H_

#include <arm/cpufunc.h>

#define	PMAP_HWPAGEWALKER		1

#define	PMAP_PDETABSIZE	(PAGE_SIZE / sizeof(pd_entry_t))
#define	PMAP_SEGTABSIZE	NSEGPG

#define	PMAP_INVALID_PDETAB_ADDRESS	((pmap_pdetab_t *)(VM_MIN_KERNEL_ADDRESS - PAGE_SIZE))
#define	PMAP_INVALID_SEGTAB_ADDRESS	((pmap_segtab_t *)(VM_MIN_KERNEL_ADDRESS - PAGE_SIZE))

#define	NPTEPG		(PAGE_SIZE / sizeof(pt_entry_t))
#define	NPDEPG		(PAGE_SIZE / sizeof(pd_entry_t))

#define	PTPSHIFT	3
#define	PTPLENGTH	(PGSHIFT - PTPSHIFT)
#define	SEGSHIFT	(PGSHIFT + PTPLENGTH)	/* LOG2(NBSEG) */

#define	NBSEG		(1 << SEGSHIFT)		/* bytes/segment */
#define	SEGOFSET	(NBSEG - 1)		/* byte offset into segment */

#define	SEGLENGTH	(PGSHIFT - 3)

#define	XSEGSHIFT	(SEGSHIFT + SEGLENGTH + SEGLENGTH)
						/* LOG2(NBXSEG) */

#define	NBXSEG		(1UL << XSEGSHIFT)	/* bytes/xsegment */
#define	XSEGOFSET	(NBXSEG - 1)		/* byte offset into xsegment */
#define	XSEGLENGTH	(PGSHIFT - 3)
#define	NXSEGPG		(1 << XSEGLENGTH)
#define	NSEGPG		(1 << SEGLENGTH)


#ifndef __BSD_PTENTRY_T__
#define	__BSD_PTENTRY_T__
#define	PRIxPTE         PRIx64
#endif /* __BSD_PTENTRY_T__ */

#define	KERNEL_PID	0

#define	__HAVE_PMAP_PV_TRACK
#define	__HAVE_PMAP_MD

/* XXX temporary */
#define	__HAVE_UNLOCKED_PMAP

#define	PMAP_PAGE_INIT(pp)				\
do {							\
	(pp)->pp_md.mdpg_first.pv_next = NULL;		\
	(pp)->pp_md.mdpg_first.pv_pmap = NULL;		\
	(pp)->pp_md.mdpg_first.pv_va = 0;		\
	(pp)->pp_md.mdpg_attrs = 0;			\
	VM_PAGEMD_PVLIST_LOCK_INIT(&(pp)->pp_md);	\
} while (/* CONSTCOND */ 0)

struct pmap_md {
	paddr_t			pmd_l0_pa;
};

#define	pm_l0_pa	pm_md.pmd_l0_pa

void pmap_md_pdetab_init(struct pmap *);
void pmap_md_pdetab_fini(struct pmap *);

vaddr_t pmap_md_map_poolpage(paddr_t, size_t);
paddr_t pmap_md_unmap_poolpage(vaddr_t, size_t);
struct vm_page *pmap_md_alloc_poolpage(int);

bool	pmap_md_kernel_vaddr_p(vaddr_t);
paddr_t	pmap_md_kernel_vaddr_to_paddr(vaddr_t);
bool	pmap_md_direct_mapped_vaddr_p(vaddr_t);
paddr_t	pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t);
vaddr_t	pmap_md_direct_map_paddr(paddr_t);
bool	pmap_md_io_vaddr_p(vaddr_t);

void	pmap_md_activate_efirt(void);
void	pmap_md_deactivate_efirt(void);

void	pmap_icache_sync_range(pmap_t, vaddr_t, vaddr_t);

vsize_t	pmap_kenter_range(vaddr_t, paddr_t, vsize_t, vm_prot_t, u_int flags);

#include <uvm/pmap/vmpagemd.h>
#include <uvm/pmap/pmap.h>
#include <uvm/pmap/pmap_pvt.h>
#include <uvm/pmap/pmap_tlb.h>
#include <uvm/pmap/pmap_synci.h>
#include <uvm/pmap/tlb.h>

#include <uvm/uvm_page.h>

#define	POOL_VTOPHYS(va)	vtophys((vaddr_t)(va))

struct pmap_page {
	struct vm_page_md	pp_md;
};

#define	PMAP_PAGE_TO_MD(ppage)	(&((ppage)->pp_md))

#define	PVLIST_EMPTY_P(pg)	VM_PAGEMD_PVLIST_EMPTY_P(VM_PAGE_TO_MD(pg))

#define	LX_BLKPAG_OS_MODIFIED	LX_BLKPAG_OS_0

#define	PMAP_PTE_OS0	"modified"
#define	PMAP_PTE_OS1	"(unk)"

static inline paddr_t
pmap_l0pa(struct pmap *pm)
{
        return pm->pm_l0_pa;
}

#if defined(__PMAP_PRIVATE)

#include <uvm/uvm_physseg.h>
struct vm_page_md;

void	pmap_md_icache_sync_all(void);
void	pmap_md_icache_sync_range_index(vaddr_t, vsize_t);
void	pmap_md_page_syncicache(struct vm_page_md *, const kcpuset_t *);
bool	pmap_md_vca_add(struct vm_page_md *, vaddr_t, pt_entry_t *);
void	pmap_md_vca_clean(struct vm_page_md *, int);
void	pmap_md_vca_remove(struct vm_page_md *, vaddr_t, bool, bool);
bool	pmap_md_ok_to_steal_p(const uvm_physseg_t, size_t);

void	pmap_md_xtab_activate(pmap_t, struct lwp *);
void	pmap_md_xtab_deactivate(pmap_t);

vaddr_t pmap_md_direct_map_paddr(paddr_t);


#ifdef MULTIPROCESSOR
#define	PMAP_NO_PV_UNCACHED
#endif

static inline void
pmap_md_init(void)
{
	// nothing
}


static inline bool
pmap_md_tlb_check_entry(void *ctx, vaddr_t va, tlb_asid_t asid, pt_entry_t pte)
{
	// TLB not walked and so not called.
	return false;
}


static inline bool
pmap_md_virtual_cache_aliasing_p(void)
{
	return false;
}


static inline vsize_t
pmap_md_cache_prefer_mask(void)
{
	return 0;
}


static inline pt_entry_t *
pmap_md_nptep(pt_entry_t *ptep)
{

	return ptep + 1;
}

#endif	/* __PMAP_PRIVATE */

#ifdef __PMAP_PRIVATE
static __inline paddr_t
pte_to_paddr(pt_entry_t pte)
{

	return l3pte_pa(pte);
}


static inline bool
pte_valid_p(pt_entry_t pte)
{

	return l3pte_valid(pte);
}


static inline void
pmap_md_clean_page(struct vm_page_md *md, bool is_src)
{
}


static inline bool
pte_modified_p(pt_entry_t pte)
{

	return (pte & LX_BLKPAG_OS_MODIFIED) != 0;
}


static inline bool
pte_wired_p(pt_entry_t pte)
{

        return (pte & LX_BLKPAG_OS_WIRED) != 0;
}


static inline pt_entry_t
pte_wire_entry(pt_entry_t pte)
{

        return pte | LX_BLKPAG_OS_WIRED;
}


static inline pt_entry_t
pte_unwire_entry(pt_entry_t pte)
{

        return pte & ~LX_BLKPAG_OS_WIRED;
}


static inline uint64_t
pte_value(pt_entry_t pte)
{

	return pte;
}

static inline bool
pte_cached_p(pt_entry_t pte)
{

	return ((pte & LX_BLKPAG_ATTR_MASK) == LX_BLKPAG_ATTR_NORMAL_WB);
}

static inline bool
pte_deferred_exec_p(pt_entry_t pte)
{

	return false;
}

static inline pt_entry_t
pte_nv_entry(bool kernel_p)
{

	/* Not valid entry */
	return kernel_p ? 0 : 0;
}

static inline pt_entry_t
pte_prot_downgrade(pt_entry_t pte, vm_prot_t prot)
{

	return (pte & ~LX_BLKPAG_AP)
	    | (((prot) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ ? LX_BLKPAG_AP_RO : LX_BLKPAG_AP_RW);
}

static inline pt_entry_t
pte_prot_nowrite(pt_entry_t pte)
{

	return pte & ~LX_BLKPAG_AF;
}

static inline pt_entry_t
pte_cached_change(pt_entry_t pte, bool cached)
{
	pte &= ~LX_BLKPAG_ATTR_MASK;
	pte |= (cached ? LX_BLKPAG_ATTR_NORMAL_WB : LX_BLKPAG_ATTR_NORMAL_NC);

	return pte;
}

static inline void
pte_set(pt_entry_t *ptep, pt_entry_t pte)
{

	*ptep = pte;
	dsb(ishst);
	/*
	 * if this mapping is going to be used by userland then the eret *can* act
	 * as the isb, but might not (apple m1).
	 *
	 * if this mapping is kernel then the isb is always needed (for some micro-
	 * architectures)
	 */

	isb();
}

static inline pd_entry_t
pte_invalid_pde(void)
{

	return 0;
}


static inline pd_entry_t
pte_pde_pdetab(paddr_t pa, bool kernel_p)
{

	return LX_VALID | LX_TYPE_TBL | (kernel_p ? 0 : LX_BLKPAG_NG) | pa;
}


static inline pd_entry_t
pte_pde_ptpage(paddr_t pa, bool kernel_p)
{

	return LX_VALID | LX_TYPE_TBL | (kernel_p ? 0 : LX_BLKPAG_NG) | pa;
}


static inline bool
pte_pde_valid_p(pd_entry_t pde)
{

	return lxpde_valid(pde);
}


static inline paddr_t
pte_pde_to_paddr(pd_entry_t pde)
{

	return lxpde_pa(pde);
}


static inline pd_entry_t
pte_pde_cas(pd_entry_t *pdep, pd_entry_t opde, pt_entry_t npde)
{
#ifdef MULTIPROCESSOR
	opde = atomic_cas_64(pdep, opde, npde);
	dsb(ishst);
#else
	*pdep = npde;
#endif
	return opde;
}


static inline void
pte_pde_set(pd_entry_t *pdep, pd_entry_t npde)
{

	*pdep = npde;
}



static inline pt_entry_t
pte_memattr(u_int flags)
{

	switch (flags & (PMAP_DEV_MASK | PMAP_CACHE_MASK)) {
	case PMAP_DEV_NP ... PMAP_DEV_NP | PMAP_CACHE_MASK:
		/* Device-nGnRnE */
		return LX_BLKPAG_ATTR_DEVICE_MEM_NP;
	case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK:
		/* Device-nGnRE */
		return LX_BLKPAG_ATTR_DEVICE_MEM;
	case PMAP_NOCACHE:
	case PMAP_NOCACHE_OVR:
	case PMAP_WRITE_COMBINE:
		/* only no-cache */
		return LX_BLKPAG_ATTR_NORMAL_NC;
	case PMAP_WRITE_BACK:
	case 0:
	default:
		return LX_BLKPAG_ATTR_NORMAL_WB;
	}
}


static inline pt_entry_t
pte_make_kenter_pa(paddr_t pa, struct vm_page_md *mdpg, vm_prot_t prot,
    u_int flags)
{
	KASSERTMSG((pa & ~L3_PAG_OA) == 0, "pa %" PRIxPADDR, pa);

	pt_entry_t pte = pa
	    | LX_VALID
#ifdef MULTIPROCESSOR
	    | LX_BLKPAG_SH_IS
#endif
	    | L3_TYPE_PAG
	    | LX_BLKPAG_AF
	    | LX_BLKPAG_UXN | LX_BLKPAG_PXN
	    | (((prot) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ ? LX_BLKPAG_AP_RO : LX_BLKPAG_AP_RW)
	    | LX_BLKPAG_OS_WIRED;

	if (prot & VM_PROT_EXECUTE)
		pte &= ~LX_BLKPAG_PXN;

	pte &= ~LX_BLKPAG_ATTR_MASK;
	pte |= pte_memattr(flags);

	return pte;
}

static inline pt_entry_t
pte_make_enter_efirt(paddr_t pa, vm_prot_t prot, u_int flags)
{
	KASSERTMSG((pa & ~L3_PAG_OA) == 0, "pa %" PRIxPADDR, pa);

	pt_entry_t npte = pa
	    | LX_VALID
#ifdef MULTIPROCESSOR
	    | LX_BLKPAG_SH_IS
#endif
	    | L3_TYPE_PAG
	    | LX_BLKPAG_AF
	    | LX_BLKPAG_NG /* | LX_BLKPAG_APUSER */
	    | LX_BLKPAG_UXN | LX_BLKPAG_PXN
	    | (((prot) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ ? LX_BLKPAG_AP_RO : LX_BLKPAG_AP_RW);

	if (prot & VM_PROT_EXECUTE)
		npte &= ~LX_BLKPAG_PXN;

	npte &= ~LX_BLKPAG_ATTR_MASK;
	npte |= pte_memattr(flags);

	return npte;
}

static inline pt_entry_t
pte_make_enter(paddr_t pa, const struct vm_page_md *mdpg, vm_prot_t prot,
    u_int flags, bool is_kernel_pmap_p)
{
	KASSERTMSG((pa & ~L3_PAG_OA) == 0, "pa %" PRIxPADDR, pa);

	pt_entry_t npte = pa
	    | LX_VALID
#ifdef MULTIPROCESSOR
	    | LX_BLKPAG_SH_IS
#endif
	    | L3_TYPE_PAG
	    | LX_BLKPAG_UXN | LX_BLKPAG_PXN
	    | (((prot) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ ? LX_BLKPAG_AP_RO : LX_BLKPAG_AP_RW);

	if ((prot & VM_PROT_WRITE) != 0 &&
	    ((flags & VM_PROT_WRITE) != 0 || VM_PAGEMD_MODIFIED_P(mdpg))) {
		/*
		 * This is a writable mapping, and the page's mod state
		 * indicates it has already been modified.  No need for
		 * modified emulation.
		 */
		npte |= LX_BLKPAG_AF;
	} else if ((flags & VM_PROT_ALL) || VM_PAGEMD_REFERENCED_P(mdpg)) {
		/*
		 * - The access type indicates that we don't need to do
		 *   referenced emulation.
		 * OR
		 * - The physical page has already been referenced so no need
		 *   to re-do referenced emulation here.
		 */
		npte |= LX_BLKPAG_AF;
	}

	if (prot & VM_PROT_EXECUTE)
		npte &= (is_kernel_pmap_p ? ~LX_BLKPAG_PXN : ~LX_BLKPAG_UXN);

	npte &= ~LX_BLKPAG_ATTR_MASK;
	npte |= pte_memattr(flags);

	/*
	 * Make sure userland mappings get the right permissions
	 */
	if (!is_kernel_pmap_p) {
		npte |= LX_BLKPAG_NG | LX_BLKPAG_APUSER;
	}

	return npte;
}
#endif /* __PMAP_PRIVATE */

#endif	/* _AARCH64_PMAP_MACHDEP_H_ */