diff options
| author | skrll <skrll@NetBSD.org> | 2022-05-07 08:20:03 +0000 |
|---|---|---|
| committer | skrll <skrll@NetBSD.org> | 2022-05-07 08:20:03 +0000 |
| commit | e19f86ea808ac23c166bcc90d51d55c724d6db0b (patch) | |
| tree | 8a1ab0dbcd0c9c7134f9115a8d6681b00769a746 /sys/dev | |
| parent | f128a5caf3631edeed0700cb4dc1238dd9c58b62 (diff) | |
Add support for Apple silicon NVME. Ported from OpenBSD.
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/ic/nvme.c | 118 | ||||
| -rw-r--r-- | sys/dev/ic/nvmevar.h | 43 |
2 files changed, 133 insertions, 28 deletions
diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c index 11b5be8c065..66696835472 100644 --- a/sys/dev/ic/nvme.c +++ b/sys/dev/ic/nvme.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvme.c,v 1.59 2021/11/16 06:58:33 skrll Exp $ */ +/* $NetBSD: nvme.c,v 1.60 2022/05/07 08:20:04 skrll Exp $ */ /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ /* @@ -18,7 +18,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.59 2021/11/16 06:58:33 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.60 2022/05/07 08:20:04 skrll Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -55,6 +55,27 @@ static int nvme_enable(struct nvme_softc *, u_int); static int nvme_disable(struct nvme_softc *); static int nvme_shutdown(struct nvme_softc *); +uint32_t nvme_op_sq_enter(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +void nvme_op_sq_leave(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +uint32_t nvme_op_sq_enter_locked(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +void nvme_op_sq_leave_locked(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + +void nvme_op_cq_done(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + +static const struct nvme_ops nvme_ops = { + .op_sq_enter = nvme_op_sq_enter, + .op_sq_leave = nvme_op_sq_leave, + .op_sq_enter_locked = nvme_op_sq_enter_locked, + .op_sq_leave_locked = nvme_op_sq_leave_locked, + + .op_cq_done = nvme_op_cq_done, +}; + #ifdef NVME_DEBUG static void nvme_dumpregs(struct nvme_softc *); #endif @@ -92,12 +113,6 @@ static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, bool (*)(void *), void *); -static struct nvme_dmamem * - nvme_dmamem_alloc(struct nvme_softc *, size_t); -static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); -static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, - int); - static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, void *); static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, @@ -126,15 +141,11 @@ static int nvme_set_number_of_queues(struct nvme_softc *, u_int, u_int *, #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ #define NVME_TIMO_SY 60 /* sync cache timeout */ -#define nvme_read4(_s, _r) \ - bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) -#define nvme_write4(_s, _r, _v) \ - bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) /* * Some controllers, at least Apple NVMe, always require split * transfers, so don't use bus_space_{read,write}_8() on LP64. */ -static inline uint64_t +uint64_t nvme_read8(struct nvme_softc *sc, bus_size_t r) { uint64_t v; @@ -151,7 +162,7 @@ nvme_read8(struct nvme_softc *sc, bus_size_t r) return v; } -static inline void +void nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) { uint32_t *a = (uint32_t *)&v; @@ -164,8 +175,6 @@ nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) nvme_write4(sc, r + 4, a[0]); #endif } -#define nvme_barrier(_s, _r, _l, _f) \ - bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) #ifdef NVME_DEBUG static __used void @@ -261,6 +270,9 @@ nvme_enable(struct nvme_softc *sc, u_int mps) return error; } + if (sc->sc_ops->op_enable != NULL) + sc->sc_ops->op_enable(sc); + nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); delay(5000); @@ -346,6 +358,9 @@ nvme_attach(struct nvme_softc *sc) uint16_t ioq_entries = nvme_ioq_size; int i; + if (sc->sc_ops == NULL) + sc->sc_ops = &nvme_ops; + reg = nvme_read4(sc, NVME_VS); if (reg == 0xffffffff) { aprint_error_dev(sc->sc_dev, "invalid mapping\n"); @@ -1312,6 +1327,44 @@ kmem_free: return error; } +uint32_t +nvme_op_sq_enter(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + mutex_enter(&q->q_sq_mtx); + + return nvme_op_sq_enter_locked(sc, q, ccb); +} + +uint32_t +nvme_op_sq_enter_locked(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + return q->q_sq_tail; +} + +void +nvme_op_sq_leave_locked(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + uint32_t tail; + + tail = ++q->q_sq_tail; + if (tail >= q->q_entries) + tail = 0; + q->q_sq_tail = tail; + nvme_write4(sc, q->q_sqtdbl, tail); +} + +void +nvme_op_sq_leave(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + nvme_op_sq_leave_locked(sc, q, ccb); + + mutex_exit(&q->q_sq_mtx); +} + static void nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) @@ -1319,10 +1372,7 @@ nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); uint32_t tail; - mutex_enter(&q->q_sq_mtx); - tail = q->q_sq_tail; - if (++q->q_sq_tail >= q->q_entries) - q->q_sq_tail = 0; + tail = sc->sc_ops->op_sq_enter(sc, q, ccb); sqe += tail; @@ -1334,8 +1384,7 @@ nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); - nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); - mutex_exit(&q->q_sq_mtx); + sc->sc_ops->op_sq_leave(sc, q, ccb); } struct nvme_poll_state { @@ -1432,6 +1481,13 @@ nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, { } +void +nvme_op_cq_done(struct nvme_softc *sc, + struct nvme_queue *q, struct nvme_ccb *ccb) +{ + /* nop */ +} + static int nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) { @@ -1471,6 +1527,8 @@ nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) rv++; + sc->sc_ops->op_cq_done(sc, q, ccb); + /* * Unlock the mutex before calling the ccb_done callback * and re-lock afterwards. The callback triggers lddone() @@ -1869,6 +1927,11 @@ nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) q->q_cq_head = 0; q->q_cq_phase = NVME_CQE_PHASE; + if (sc->sc_ops->op_q_alloc != NULL) { + if (sc->sc_ops->op_q_alloc(sc, q) != 0) + goto free_cq; + } + nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); @@ -1901,9 +1964,6 @@ nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q) memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); - q->q_sqtdbl = NVME_SQTDBL(q->q_id, sc->sc_dstrd); - q->q_cqhdbl = NVME_CQHDBL(q->q_id, sc->sc_dstrd); - q->q_sq_tail = 0; q->q_cq_head = 0; q->q_cq_phase = NVME_CQE_PHASE; @@ -1920,6 +1980,10 @@ nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) mutex_destroy(&q->q_cq_mtx); nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); + + if (sc->sc_ops->op_q_alloc != NULL) + sc->sc_ops->op_q_free(sc, q); + nvme_dmamem_free(sc, q->q_cq_dmamem); nvme_dmamem_free(sc, q->q_sq_dmamem); kmem_free(q, sizeof(*q)); @@ -1989,7 +2053,7 @@ nvme_softintr_msi(void *xq) nvme_q_complete(sc, q); } -static struct nvme_dmamem * +struct nvme_dmamem * nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) { struct nvme_dmamem *ndm; @@ -2033,7 +2097,7 @@ ndmfree: return NULL; } -static void +void nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) { bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), diff --git a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h index dca4d5c42f6..ce67e6dc038 100644 --- a/sys/dev/ic/nvmevar.h +++ b/sys/dev/ic/nvmevar.h @@ -1,4 +1,4 @@ -/* $NetBSD: nvmevar.h,v 1.23 2021/11/16 06:58:33 skrll Exp $ */ +/* $NetBSD: nvmevar.h,v 1.24 2022/05/07 08:20:04 skrll Exp $ */ /* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ /* @@ -79,6 +79,8 @@ struct nvme_queue { kmutex_t q_cq_mtx; struct nvme_dmamem *q_sq_dmamem; struct nvme_dmamem *q_cq_dmamem; + struct nvme_dmamem *q_nvmmu_dmamem; /* for apple m1 nvme */ + bus_size_t q_sqtdbl; /* submission queue tail doorbell */ bus_size_t q_cqhdbl; /* completion queue head doorbell */ uint16_t q_id; @@ -103,9 +105,32 @@ struct nvme_namespace { #define NVME_NS_F_OPEN __BIT(0) }; +struct nvme_ops { + void (*op_enable)(struct nvme_softc *); + + int (*op_q_alloc)(struct nvme_softc *, + struct nvme_queue *); + void (*op_q_free)(struct nvme_softc *, + struct nvme_queue *); + + uint32_t (*op_sq_enter)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + void (*op_sq_leave)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + uint32_t (*op_sq_enter_locked)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + void (*op_sq_leave_locked)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); + + void (*op_cq_done)(struct nvme_softc *, + struct nvme_queue *, struct nvme_ccb *); +}; + struct nvme_softc { device_t sc_dev; + const struct nvme_ops *sc_ops; + bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; bus_size_t sc_ios; @@ -199,6 +224,22 @@ nvme_ns_get(struct nvme_softc *sc, uint16_t nsid) return &sc->sc_namespaces[nsid - 1]; } +#define nvme_read4(_s, _r) \ + bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) +#define nvme_write4(_s, _r, _v) \ + bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) +uint64_t + nvme_read8(struct nvme_softc *, bus_size_t); +void nvme_write8(struct nvme_softc *, bus_size_t, uint64_t); + +#define nvme_barrier(_s, _r, _l, _f) \ + bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) + +struct nvme_dmamem * + nvme_dmamem_alloc(struct nvme_softc *, size_t); +void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); +void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int); + int nvme_ns_identify(struct nvme_softc *, uint16_t); void nvme_ns_free(struct nvme_softc *, uint16_t); int nvme_ns_dobio(struct nvme_softc *, uint16_t, void *, |
