1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
|
/* $NetBSD: nvmevar.h,v 1.28 2022/08/14 12:08:57 jmcneill Exp $ */
/* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */
/*
* Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <sys/bus.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/mutex.h>
#include <sys/pool.h>
#include <sys/queue.h>
#include <sys/buf.h>
struct nvme_dmamem {
bus_dmamap_t ndm_map;
bus_dma_segment_t ndm_seg;
size_t ndm_size;
void *ndm_kva;
};
#define NVME_DMA_MAP(_ndm) ((_ndm)->ndm_map)
#define NVME_DMA_LEN(_ndm) ((_ndm)->ndm_map->dm_segs[0].ds_len)
#define NVME_DMA_DVA(_ndm) ((uint64_t)(_ndm)->ndm_map->dm_segs[0].ds_addr)
#define NVME_DMA_KVA(_ndm) ((void *)(_ndm)->ndm_kva)
struct nvme_softc;
struct nvme_queue;
typedef void (*nvme_nnc_done)(void *, struct buf *, uint16_t, uint32_t);
struct nvme_ccb {
SIMPLEQ_ENTRY(nvme_ccb) ccb_entry;
/* DMA handles */
bus_dmamap_t ccb_dmamap;
bus_addr_t ccb_prpl_off;
uint64_t ccb_prpl_dva;
uint64_t *ccb_prpl;
/* command context */
uint16_t ccb_id;
void *ccb_cookie;
#define NVME_CCB_FREE 0xbeefdeed
void (*ccb_done)(struct nvme_queue *,
struct nvme_ccb *, struct nvme_cqe *);
/* namespace context */
void *nnc_cookie;
nvme_nnc_done nnc_done;
uint16_t nnc_nsid;
uint16_t nnc_flags;
#define NVME_NS_CTX_F_READ __BIT(0)
#define NVME_NS_CTX_F_POLL __BIT(1)
#define NVME_NS_CTX_F_FUA __BIT(2)
struct buf *nnc_buf;
daddr_t nnc_blkno;
size_t nnc_datasize;
int nnc_secsize;
};
struct nvme_queue {
struct nvme_softc *q_sc;
kmutex_t q_sq_mtx;
kmutex_t q_cq_mtx;
struct nvme_dmamem *q_sq_dmamem;
struct nvme_dmamem *q_cq_dmamem;
struct nvme_dmamem *q_nvmmu_dmamem; /* for apple m1 nvme */
bus_size_t q_sqtdbl; /* submission queue tail doorbell */
bus_size_t q_cqhdbl; /* completion queue head doorbell */
uint16_t q_id;
uint32_t q_entries;
uint32_t q_sq_tail;
uint32_t q_cq_head;
uint16_t q_cq_phase;
kmutex_t q_ccb_mtx;
kcondvar_t q_ccb_wait; /* wait for ccb avail/finish */
bool q_ccb_waiting; /* whether there are waiters */
uint16_t q_nccbs; /* total number of ccbs */
struct nvme_ccb *q_ccbs;
SIMPLEQ_HEAD(, nvme_ccb) q_ccb_list;
struct nvme_dmamem *q_ccb_prpls;
};
struct nvme_namespace {
struct nvm_identify_namespace *ident;
device_t dev;
uint32_t flags;
#define NVME_NS_F_OPEN __BIT(0)
};
struct nvme_ops {
void (*op_enable)(struct nvme_softc *);
int (*op_q_alloc)(struct nvme_softc *,
struct nvme_queue *);
void (*op_q_free)(struct nvme_softc *,
struct nvme_queue *);
uint32_t (*op_sq_enter)(struct nvme_softc *,
struct nvme_queue *, struct nvme_ccb *);
void (*op_sq_leave)(struct nvme_softc *,
struct nvme_queue *, struct nvme_ccb *);
uint32_t (*op_sq_enter_locked)(struct nvme_softc *,
struct nvme_queue *, struct nvme_ccb *);
void (*op_sq_leave_locked)(struct nvme_softc *,
struct nvme_queue *, struct nvme_ccb *);
void (*op_cq_done)(struct nvme_softc *,
struct nvme_queue *, struct nvme_ccb *);
};
struct nvme_softc {
device_t sc_dev;
const struct nvme_ops *sc_ops;
bus_space_tag_t sc_iot;
bus_space_handle_t sc_ioh;
bus_size_t sc_ios;
bus_dma_tag_t sc_dmat;
int (*sc_intr_establish)(struct nvme_softc *,
uint16_t qid, struct nvme_queue *);
int (*sc_intr_disestablish)(struct nvme_softc *,
uint16_t qid);
void **sc_ih; /* interrupt handlers */
void **sc_softih; /* softintr handlers */
u_int sc_rdy_to; /* RDY timeout */
size_t sc_mps; /* memory page size */
size_t sc_mdts; /* max data trasfer size */
u_int sc_max_sgl; /* max S/G segments */
u_int sc_dstrd;
struct nvm_identify_controller
sc_identify;
u_int sc_nn; /* namespace count */
struct nvme_namespace *sc_namespaces;
bool sc_use_mq;
u_int sc_nq; /* # of io queue (sc_q) */
struct nvme_queue *sc_admin_q;
struct nvme_queue **sc_q;
uint32_t sc_flags;
#define NVME_F_ATTACHED __BIT(0)
#define NVME_F_OPEN __BIT(1)
uint32_t sc_quirks;
#define NVME_QUIRK_DELAY_B4_CHK_RDY __BIT(0)
#define NVME_QUIRK_NOMSI __BIT(1)
char sc_modelname[81];
};
#define lemtoh16(p) le16toh(*((uint16_t *)(p)))
#define lemtoh32(p) le32toh(*((uint32_t *)(p)))
#define lemtoh64(p) le64toh(*((uint64_t *)(p)))
#define htolem16(p, x) (*((uint16_t *)(p)) = htole16(x))
#define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x))
#define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x))
struct nvme_attach_args {
uint16_t naa_nsid;
uint32_t naa_qentries; /* total number of queue slots */
uint32_t naa_maxphys; /* maximum device transfer size */
const char *naa_typename; /* identifier */
};
int nvme_attach(struct nvme_softc *);
int nvme_detach(struct nvme_softc *, int flags);
int nvme_rescan(device_t, const char *, const int *);
void nvme_childdet(device_t, device_t);
int nvme_suspend(struct nvme_softc *);
int nvme_resume(struct nvme_softc *);
int nvme_intr(void *);
void nvme_softintr_intx(void *);
int nvme_intr_msi(void *);
void nvme_softintr_msi(void *);
static __inline struct nvme_queue *
nvme_get_q(struct nvme_softc *sc)
{
return sc->sc_q[cpu_index(curcpu()) % sc->sc_nq];
}
/*
* namespace
*/
static __inline struct nvme_namespace *
nvme_ns_get(struct nvme_softc *sc, uint16_t nsid)
{
if (nsid == 0 || nsid - 1 >= sc->sc_nn)
return NULL;
return &sc->sc_namespaces[nsid - 1];
}
#define nvme_read4(_s, _r) \
bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r))
#define nvme_write4(_s, _r, _v) \
bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
uint64_t
nvme_read8(struct nvme_softc *, bus_size_t);
void nvme_write8(struct nvme_softc *, bus_size_t, uint64_t);
#define nvme_barrier(_s, _r, _l, _f) \
bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f))
struct nvme_dmamem *
nvme_dmamem_alloc(struct nvme_softc *, size_t);
void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *);
void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int);
int nvme_ns_identify(struct nvme_softc *, uint16_t);
void nvme_ns_free(struct nvme_softc *, uint16_t);
int nvme_ns_dobio(struct nvme_softc *, uint16_t, void *,
struct buf *, void *, size_t, int, daddr_t, int, nvme_nnc_done);
int nvme_ns_sync(struct nvme_softc *, uint16_t, int);
int nvme_admin_getcache(struct nvme_softc *, int *);
int nvme_admin_setcache(struct nvme_softc *, int);
|