1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
|
/* $NetBSD: cpufunc.h,v 1.26 2023/04/20 08:28:03 skrll Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _AARCH64_CPUFUNC_H_
#define _AARCH64_CPUFUNC_H_
#ifdef _KERNEL
#include <arm/armreg.h>
#include <sys/device_if.h>
#include <sys/cpu.h>
#include <uvm/uvm_extern.h>
#include <uvm/pmap/pmap_devmap.h>
extern u_int aarch64_cache_vindexsize; /* cachesize/way (VIVT/VIPT) */
extern u_int aarch64_cache_prefer_mask;
extern u_int cputype; /* compat arm */
extern int aarch64_bti_enabled;
extern int aarch64_hafdbs_enabled;
extern int aarch64_pan_enabled;
extern int aarch64_pac_enabled;
void aarch64_hafdbs_init(int);
void aarch64_pan_init(int);
int aarch64_pac_init(int);
int set_cpufuncs(void);
int aarch64_setcpufuncs(struct cpu_info *);
void aarch64_getcacheinfo(struct cpu_info *);
void aarch64_parsecacheinfo(struct cpu_info *);
void aarch64_printcacheinfo(device_t, struct cpu_info *);
void aarch64_dcache_wbinv_all(void);
void aarch64_dcache_inv_all(void);
void aarch64_dcache_wb_all(void);
void aarch64_icache_inv_all(void);
/* cache op in cpufunc_asm_armv8.S */
void aarch64_nullop(void);
uint32_t aarch64_cpuid(void);
void aarch64_icache_sync_range(vaddr_t, vsize_t);
void aarch64_icache_inv_range(vaddr_t, vsize_t);
void aarch64_icache_barrier_range(vaddr_t, vsize_t);
void aarch64_idcache_wbinv_range(vaddr_t, vsize_t);
void aarch64_dcache_wbinv_range(vaddr_t, vsize_t);
void aarch64_dcache_inv_range(vaddr_t, vsize_t);
void aarch64_dcache_wb_range(vaddr_t, vsize_t);
void aarch64_icache_inv_all(void);
void aarch64_drain_writebuf(void);
/* tlb op in cpufunc_asm_armv8.S */
#define cpu_set_ttbr0(t) curcpu()->ci_cpufuncs.cf_set_ttbr0((t))
void aarch64_set_ttbr0(uint64_t);
void aarch64_set_ttbr0_thunderx(uint64_t);
void aarch64_tlbi_all(void); /* all ASID, all VA */
void aarch64_tlbi_by_asid(int); /* an ASID, all VA */
void aarch64_tlbi_by_va(vaddr_t); /* all ASID, a VA */
void aarch64_tlbi_by_va_ll(vaddr_t); /* all ASID, a VA, lastlevel */
void aarch64_tlbi_by_asid_va(int, vaddr_t); /* an ASID, a VA */
void aarch64_tlbi_by_asid_va_ll(int, vaddr_t); /* an ASID, a VA, lastlevel */
/* misc */
#define cpu_idnum() aarch64_cpuid()
/* cache op */
#define cpu_dcache_wbinv_all() aarch64_dcache_wbinv_all()
#define cpu_dcache_inv_all() aarch64_dcache_inv_all()
#define cpu_dcache_wb_all() aarch64_dcache_wb_all()
#define cpu_idcache_wbinv_all() \
(aarch64_dcache_wbinv_all(), aarch64_icache_inv_all())
#define cpu_icache_sync_all() \
(aarch64_dcache_wb_all(), aarch64_icache_inv_all())
#define cpu_icache_inv_all() aarch64_icache_inv_all()
#define cpu_dcache_wbinv_range(v,s) aarch64_dcache_wbinv_range((v),(s))
#define cpu_dcache_inv_range(v,s) aarch64_dcache_inv_range((v),(s))
#define cpu_dcache_wb_range(v,s) aarch64_dcache_wb_range((v),(s))
#define cpu_idcache_wbinv_range(v,s) aarch64_idcache_wbinv_range((v),(s))
#define cpu_icache_sync_range(v,s) \
curcpu()->ci_cpufuncs.cf_icache_sync_range((v),(s))
#define cpu_sdcache_wbinv_range(v,p,s) ((void)0)
#define cpu_sdcache_inv_range(v,p,s) ((void)0)
#define cpu_sdcache_wb_range(v,p,s) ((void)0)
/* others */
#define cpu_drain_writebuf() aarch64_drain_writebuf()
extern u_int arm_dcache_align;
extern u_int arm_dcache_align_mask;
static inline bool
cpu_gtmr_exists_p(void)
{
return true;
}
static inline u_int
cpu_clusterid(void)
{
return __SHIFTOUT(reg_mpidr_el1_read(), MPIDR_AFF1);
}
static inline bool
cpu_earlydevice_va_p(void)
{
/* This function may be called before enabling MMU, or mapping KVA */
if ((reg_sctlr_el1_read() & SCTLR_M) == 0)
return false;
/* device mapping will be available after pmap_devmap_bootstrap() */
if (!pmap_devmap_bootstrapped_p())
return false;
return true;
}
#endif /* _KERNEL */
/* definitions of TAG and PAC in pointers */
#define AARCH64_ADDRTOP_TAG_BIT 55
#define AARCH64_ADDRTOP_TAG __BIT(55) /* ECR_EL1.TBI[01]=1 */
#define AARCH64_ADDRTOP_MSB __BIT(63) /* ECR_EL1.TBI[01]=0 */
#define AARCH64_ADDRESS_TAG_MASK __BITS(63,56) /* if TCR.TBI[01]=1 */
#define AARCH64_ADDRESS_PAC_MASK __BITS(54,48) /* depend on VIRT_BIT */
#define AARCH64_ADDRESS_TAGPAC_MASK \
(AARCH64_ADDRESS_TAG_MASK|AARCH64_ADDRESS_PAC_MASK)
#ifdef _KERNEL
/*
* Which is the address space of this VA?
* return the space considering TBI. (PAC is not yet)
*
* return value: AARCH64_ADDRSPACE_{LOWER,UPPER}{_OUTOFRANGE}?
*/
#define AARCH64_ADDRSPACE_LOWER 0 /* -> TTBR0 */
#define AARCH64_ADDRSPACE_UPPER 1 /* -> TTBR1 */
#define AARCH64_ADDRSPACE_LOWER_OUTOFRANGE -1 /* certainly fault */
#define AARCH64_ADDRSPACE_UPPER_OUTOFRANGE -2 /* certainly fault */
static inline int
aarch64_addressspace(vaddr_t va)
{
uint64_t addrtop, tbi;
addrtop = va & AARCH64_ADDRTOP_TAG;
tbi = addrtop ? TCR_TBI1 : TCR_TBI0;
if (reg_tcr_el1_read() & tbi) {
if (addrtop == 0) {
/* lower address, and TBI0 enabled */
if ((va & AARCH64_ADDRESS_PAC_MASK) != 0)
return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE;
return AARCH64_ADDRSPACE_LOWER;
}
/* upper address, and TBI1 enabled */
if ((va & AARCH64_ADDRESS_PAC_MASK) != AARCH64_ADDRESS_PAC_MASK)
return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE;
return AARCH64_ADDRSPACE_UPPER;
}
addrtop = va & AARCH64_ADDRTOP_MSB;
if (addrtop == 0) {
/* lower address, and TBI0 disabled */
if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != 0)
return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE;
return AARCH64_ADDRSPACE_LOWER;
}
/* upper address, and TBI1 disabled */
if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != AARCH64_ADDRESS_TAGPAC_MASK)
return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE;
return AARCH64_ADDRSPACE_UPPER;
}
static inline vaddr_t
aarch64_untag_address(vaddr_t va)
{
uint64_t addrtop, tbi;
addrtop = va & AARCH64_ADDRTOP_TAG;
tbi = addrtop ? TCR_TBI1 : TCR_TBI0;
if (reg_tcr_el1_read() & tbi) {
if (addrtop == 0) {
/* lower address, and TBI0 enabled */
return va & ~AARCH64_ADDRESS_TAG_MASK;
}
/* upper address, and TBI1 enabled */
return va | AARCH64_ADDRESS_TAG_MASK;
}
/* TBI[01] is disabled, nothing to do */
return va;
}
#endif /* _KERNEL */
static __inline uint64_t
aarch64_strip_pac(uint64_t __val)
{
if (__val & AARCH64_ADDRTOP_TAG)
return __val | AARCH64_ADDRESS_TAGPAC_MASK;
return __val & ~AARCH64_ADDRESS_TAGPAC_MASK;
}
#endif /* _AARCH64_CPUFUNC_H_ */
|