diff options
| author | jdolecek <jdolecek@NetBSD.org> | 2016-09-19 22:11:41 +0000 |
|---|---|---|
| committer | jdolecek <jdolecek@NetBSD.org> | 2016-09-19 22:11:41 +0000 |
| commit | c02068e3773fc4e9735cf9e5adaf8830f41a50e0 (patch) | |
| tree | eab2cd32eac93d9b7a09631c93629f6c01289311 /sys/dev | |
| parent | a8d40635b0ce45865edb3c2083df758598c59cda (diff) | |
slightly optimize memory access - change struct nvme_queue so that the
struct dmamem members are allocated as part of it, instead of separate
kmem_alloc()s
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/ic/nvme.c | 112 | ||||
| -rw-r--r-- | sys/dev/ic/nvmevar.h | 16 |
2 files changed, 62 insertions, 66 deletions
diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c index 5d7044a2027..466a280a3e1 100644 --- a/sys/dev/ic/nvme.c +++ b/sys/dev/ic/nvme.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvme.c,v 1.11 2016/09/19 20:33:51 jdolecek Exp $ */ +/* $NetBSD: nvme.c,v 1.12 2016/09/19 22:11:41 jdolecek Exp $ */ /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ /* @@ -18,7 +18,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.11 2016/09/19 20:33:51 jdolecek Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.12 2016/09/19 22:11:41 jdolecek Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -85,11 +85,9 @@ static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); -static struct nvme_dmamem * - nvme_dmamem_alloc(struct nvme_softc *, size_t); +static int nvme_dmamem_alloc(struct nvme_softc *, size_t, + struct nvme_dmamem *); static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); -static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, - int); static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, void *); @@ -153,8 +151,12 @@ nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) #endif } #endif /* __LP64__ */ + #define nvme_barrier(_s, _r, _l, _f) \ bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) +#define nvme_dmamem_sync(sc, mem, ops) \ + bus_dmamap_sync((sc)->sc_dmat, NVME_DMA_MAP(mem), \ + 0, NVME_DMA_LEN(mem), (ops)); static void nvme_version(struct nvme_softc *sc, uint32_t ver) @@ -546,19 +548,19 @@ nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) { struct nvme_sqe sqe; struct nvm_identify_namespace *identify; - struct nvme_dmamem *mem; + struct nvme_dmamem mem; struct nvme_ccb *ccb; struct nvme_namespace *ns; - int rv; + int error; KASSERT(nsid > 0); ccb = nvme_ccb_get(sc->sc_admin_q); KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ - mem = nvme_dmamem_alloc(sc, sizeof(*identify)); - if (mem == NULL) - return ENOMEM; + error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); + if (error) + return error; memset(&sqe, 0, sizeof(sqe)); sqe.opcode = NVM_ADMIN_IDENTIFY; @@ -570,13 +572,14 @@ nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) ccb->ccb_cookie = &sqe; nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); - rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); + error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, + NVME_TIMO_IDENT); nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); nvme_ccb_put(sc->sc_admin_q, ccb); - if (rv != 0) { - rv = EIO; + if (error != 0) { + error = EIO; goto done; } @@ -590,9 +593,9 @@ nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) ns->ident = identify; done: - nvme_dmamem_free(sc, mem); + nvme_dmamem_free(sc, &mem); - return rv; + return error; } int @@ -1073,29 +1076,29 @@ nvme_identify(struct nvme_softc *sc, u_int mps) { char sn[41], mn[81], fr[17]; struct nvm_identify_controller *identify; - struct nvme_dmamem *mem; + struct nvme_dmamem mem; struct nvme_ccb *ccb; u_int mdts; - int rv = 1; + int error; ccb = nvme_ccb_get(sc->sc_admin_q); KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ - mem = nvme_dmamem_alloc(sc, sizeof(*identify)); - if (mem == NULL) - return 1; + error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); + if (error) + return error; ccb->ccb_done = nvme_empty_done; - ccb->ccb_cookie = mem; + ccb->ccb_cookie = &mem; nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); - rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, + error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, NVME_TIMO_IDENT); nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); nvme_ccb_put(sc->sc_admin_q, ccb); - if (rv != 0) + if (error != 0) goto done; identify = NVME_DMA_KVA(mem); @@ -1120,9 +1123,9 @@ nvme_identify(struct nvme_softc *sc, u_int mps) memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); done: - nvme_dmamem_free(sc, mem); + nvme_dmamem_free(sc, &mem); - return rv; + return error; } static int @@ -1225,7 +1228,7 @@ nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) struct nvme_dmamem *mem = ccb->ccb_cookie; sqe->opcode = NVM_ADMIN_IDENTIFY; - htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); + htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(*mem)); htolem32(&sqe->cdw10, 1); } @@ -1237,6 +1240,7 @@ nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) bus_addr_t off; uint64_t *prpl; u_int i; + int error; mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); SIMPLEQ_INIT(&q->q_ccb_list); @@ -1246,8 +1250,12 @@ nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) return 1; q->q_nccbs = nccbs; - q->q_ccb_prpls = nvme_dmamem_alloc(sc, - sizeof(*prpl) * sc->sc_max_sgl * nccbs); + error = nvme_dmamem_alloc(sc, sizeof(*prpl) * sc->sc_max_sgl * nccbs, + &q->q_ccb_prpls); + if (error) { + kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); + return error; + } prpl = NVME_DMA_KVA(q->q_ccb_prpls); off = 0; @@ -1315,7 +1323,7 @@ nvme_ccbs_free(struct nvme_queue *q) } mutex_exit(&q->q_ccb_mtx); - nvme_dmamem_free(sc, q->q_ccb_prpls); + nvme_dmamem_free(sc, &q->q_ccb_prpls); kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); q->q_ccbs = NULL; mutex_destroy(&q->q_ccb_mtx); @@ -1325,20 +1333,21 @@ static struct nvme_queue * nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) { struct nvme_queue *q; + int error; q = kmem_alloc(sizeof(*q), KM_SLEEP); if (q == NULL) return NULL; q->q_sc = sc; - q->q_sq_dmamem = nvme_dmamem_alloc(sc, - sizeof(struct nvme_sqe) * entries); - if (q->q_sq_dmamem == NULL) + error = nvme_dmamem_alloc(sc, sizeof(struct nvme_sqe) * entries, + &q->q_sq_dmamem); + if (error) goto free; - q->q_cq_dmamem = nvme_dmamem_alloc(sc, - sizeof(struct nvme_cqe) * entries); - if (q->q_cq_dmamem == NULL) + error = nvme_dmamem_alloc(sc, sizeof(struct nvme_cqe) * entries, + &q->q_cq_dmamem); + if (error) goto free_sq; memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); @@ -1365,9 +1374,9 @@ nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) return q; free_cq: - nvme_dmamem_free(sc, q->q_cq_dmamem); + nvme_dmamem_free(sc, &q->q_cq_dmamem); free_sq: - nvme_dmamem_free(sc, q->q_sq_dmamem); + nvme_dmamem_free(sc, &q->q_sq_dmamem); free: kmem_free(q, sizeof(*q)); @@ -1382,8 +1391,8 @@ nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) mutex_destroy(&q->q_cq_mtx); nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); - nvme_dmamem_free(sc, q->q_cq_dmamem); - nvme_dmamem_free(sc, q->q_sq_dmamem); + nvme_dmamem_free(sc, &q->q_cq_dmamem); + nvme_dmamem_free(sc, &q->q_sq_dmamem); kmem_free(q, sizeof(*q)); } @@ -1429,16 +1438,12 @@ nvme_softintr_msi(void *xq) nvme_q_complete(sc, q); } -static struct nvme_dmamem * -nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) +static int +nvme_dmamem_alloc(struct nvme_softc *sc, size_t size, struct nvme_dmamem *ndm) { - struct nvme_dmamem *ndm; int nsegs; - ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); - if (ndm == NULL) - return NULL; - + memset(ndm, 0, sizeof(*ndm)); ndm->ndm_size = size; if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, @@ -1458,7 +1463,7 @@ nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) NULL, BUS_DMA_WAITOK) != 0) goto unmap; - return ndm; + return 0; unmap: bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); @@ -1467,15 +1472,7 @@ free: destroy: bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); ndmfree: - kmem_free(ndm, sizeof(*ndm)); - return NULL; -} - -static void -nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) -{ - bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), - 0, NVME_DMA_LEN(mem), ops); + return ENOMEM; } void @@ -1485,7 +1482,6 @@ nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); - kmem_free(ndm, sizeof(*ndm)); } /* diff --git a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h index 6d1aac2a7ed..f14bebd1bf7 100644 --- a/sys/dev/ic/nvmevar.h +++ b/sys/dev/ic/nvmevar.h @@ -1,4 +1,4 @@ -/* $NetBSD: nvmevar.h,v 1.4 2016/09/19 20:33:51 jdolecek Exp $ */ +/* $NetBSD: nvmevar.h,v 1.5 2016/09/19 22:11:41 jdolecek Exp $ */ /* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ /* @@ -30,10 +30,10 @@ struct nvme_dmamem { size_t ndm_size; void *ndm_kva; }; -#define NVME_DMA_MAP(_ndm) ((_ndm)->ndm_map) -#define NVME_DMA_LEN(_ndm) ((_ndm)->ndm_map->dm_segs[0].ds_len) -#define NVME_DMA_DVA(_ndm) ((uint64_t)(_ndm)->ndm_map->dm_segs[0].ds_addr) -#define NVME_DMA_KVA(_ndm) ((void *)(_ndm)->ndm_kva) +#define NVME_DMA_MAP(_ndm) ((_ndm).ndm_map) +#define NVME_DMA_LEN(_ndm) ((_ndm).ndm_map->dm_segs[0].ds_len) +#define NVME_DMA_DVA(_ndm) ((uint64_t)(_ndm).ndm_map->dm_segs[0].ds_addr) +#define NVME_DMA_KVA(_ndm) ((void *)(_ndm).ndm_kva) struct nvme_softc; struct nvme_queue; @@ -74,8 +74,8 @@ struct nvme_queue { struct nvme_softc *q_sc; kmutex_t q_sq_mtx; kmutex_t q_cq_mtx; - struct nvme_dmamem *q_sq_dmamem; - struct nvme_dmamem *q_cq_dmamem; + struct nvme_dmamem q_sq_dmamem; + struct nvme_dmamem q_cq_dmamem; bus_size_t q_sqtdbl; /* submission queue tail doorbell */ bus_size_t q_cqhdbl; /* completion queue head doorbell */ uint16_t q_id; @@ -88,7 +88,7 @@ struct nvme_queue { u_int q_nccbs; struct nvme_ccb *q_ccbs; SIMPLEQ_HEAD(, nvme_ccb) q_ccb_list; - struct nvme_dmamem *q_ccb_prpls; + struct nvme_dmamem q_ccb_prpls; }; struct nvme_namespace { |
