diff options
| author | jmcneill <jmcneill@NetBSD.org> | 2022-08-14 12:08:57 +0000 |
|---|---|---|
| committer | jmcneill <jmcneill@NetBSD.org> | 2022-08-14 12:08:57 +0000 |
| commit | 4112efb4c8998b430ec658d0ba9f0aab307668b6 (patch) | |
| tree | 188c660dc6c158fa34d8a7c86c4fea7426fc47ab /sys/dev | |
| parent | 8aa7595054747ff91337f2a379df0a41b2777a91 (diff) | |
nvme: Make sure that q_ccb_list is always accessed with the q lock held.
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/ic/nvme.c | 49 | ||||
| -rw-r--r-- | sys/dev/ic/nvmevar.h | 17 |
2 files changed, 46 insertions, 20 deletions
diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c index 55c4e1b30d5..77232c3f18d 100644 --- a/sys/dev/ic/nvme.c +++ b/sys/dev/ic/nvme.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvme.c,v 1.61 2022/07/31 12:02:28 mlelstv Exp $ */ +/* $NetBSD: nvme.c,v 1.62 2022/08/14 12:08:57 jmcneill Exp $ */ /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ /* @@ -18,7 +18,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.61 2022/07/31 12:02:28 mlelstv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.62 2022/08/14 12:08:57 jmcneill Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -88,6 +88,9 @@ static void nvme_ccbs_free(struct nvme_queue *); static struct nvme_ccb * nvme_ccb_get(struct nvme_queue *, bool); +static struct nvme_ccb * + nvme_ccb_get_bio(struct nvme_softc *, struct buf *, + struct nvme_queue **); static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); static int nvme_poll(struct nvme_softc *, struct nvme_queue *, @@ -768,12 +771,12 @@ nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, struct buf *bp, void *data, size_t datasize, int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) { - struct nvme_queue *q = nvme_get_q(sc, bp, false); + struct nvme_queue *q; struct nvme_ccb *ccb; bus_dmamap_t dmap; int i, error; - ccb = nvme_ccb_get(q, false); + ccb = nvme_ccb_get_bio(sc, bp, &q); if (ccb == NULL) return EAGAIN; @@ -910,7 +913,7 @@ nvme_ns_sync_finished(void *cookie) int nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) { - struct nvme_queue *q = nvme_get_q(sc, NULL, true); + struct nvme_queue *q = nvme_get_q(sc); struct nvme_ccb *ccb; int result = 0; @@ -1277,7 +1280,7 @@ nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) return EINVAL; - q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc, NULL, true); + q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); ccb = nvme_ccb_get(q, true); KASSERT(ccb != NULL); @@ -1852,6 +1855,40 @@ again: return ccb; } +static struct nvme_ccb * +nvme_ccb_get_bio(struct nvme_softc *sc, struct buf *bp, + struct nvme_queue **selq) +{ + u_int cpuindex = cpu_index(bp->b_ci); + + /* + * Find a queue with available ccbs, preferring the originating + * CPU's queue. + */ + + for (u_int qoff = 0; qoff < sc->sc_nq; qoff++) { + struct nvme_queue *q = sc->sc_q[(cpuindex + qoff) % sc->sc_nq]; + struct nvme_ccb *ccb; + + mutex_enter(&q->q_ccb_mtx); + ccb = SIMPLEQ_FIRST(&q->q_ccb_list); + if (ccb != NULL) { + SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); +#ifdef DEBUG + ccb->ccb_cookie = NULL; +#endif + } + mutex_exit(&q->q_ccb_mtx); + + if (ccb != NULL) { + *selq = q; + return ccb; + } + } + + return NULL; +} + static void nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) { diff --git a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h index e696f48ee2c..0920ab64476 100644 --- a/sys/dev/ic/nvmevar.h +++ b/sys/dev/ic/nvmevar.h @@ -1,4 +1,4 @@ -/* $NetBSD: nvmevar.h,v 1.27 2022/08/01 08:09:30 mlelstv Exp $ */ +/* $NetBSD: nvmevar.h,v 1.28 2022/08/14 12:08:57 jmcneill Exp $ */ /* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ /* @@ -197,20 +197,9 @@ int nvme_intr_msi(void *); void nvme_softintr_msi(void *); static __inline struct nvme_queue * -nvme_get_q(struct nvme_softc *sc, struct buf *bp, bool waitok) +nvme_get_q(struct nvme_softc *sc) { - struct cpu_info *ci = (bp && bp->b_ci) ? bp->b_ci : curcpu(); - - /* - * Find a queue with available ccbs, preferring the originating CPU's queue. - */ - - for (u_int qoff = 0; qoff < sc->sc_nq; qoff++) { - struct nvme_queue *q = sc->sc_q[(cpu_index(ci) + qoff) % sc->sc_nq]; - if (!SIMPLEQ_EMPTY(&q->q_ccb_list) || waitok) - return q; - } - return NULL; + return sc->sc_q[cpu_index(curcpu()) % sc->sc_nq]; } /* |
