diff options
| author | nonaka <nonaka@NetBSD.org> | 2016-06-04 16:11:50 +0000 |
|---|---|---|
| committer | nonaka <nonaka@NetBSD.org> | 2016-06-04 16:11:50 +0000 |
| commit | 5bfc0c6cda163f7112609ae50fefd2c5f3950828 (patch) | |
| tree | 2301521be278b1ff934b0bf5c812f3bf2b4d8bba /sys/dev | |
| parent | fb7041c48b747bb4619f20873f464609327dbbd2 (diff) | |
Add NVMe command passthrough support.
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/ic/Makefile | 5 | ||||
| -rw-r--r-- | sys/dev/ic/nvme.c | 331 | ||||
| -rw-r--r-- | sys/dev/ic/nvmeio.h | 97 | ||||
| -rw-r--r-- | sys/dev/ic/nvmereg.h | 7 | ||||
| -rw-r--r-- | sys/dev/ic/nvmevar.h | 5 |
5 files changed, 439 insertions, 6 deletions
diff --git a/sys/dev/ic/Makefile b/sys/dev/ic/Makefile index e853501e60f..25bdf25295c 100644 --- a/sys/dev/ic/Makefile +++ b/sys/dev/ic/Makefile @@ -1,9 +1,10 @@ -# $NetBSD: Makefile,v 1.24 2005/12/11 12:21:25 christos Exp $ +# $NetBSD: Makefile,v 1.25 2016/06/04 16:11:51 nonaka Exp $ INCSDIR= /usr/include/dev/ic # Only install includes which are used by userland INCS= athioctl.h bt8xx.h hd44780var.h icpreg.h icp_ioctl.h isp_ioctl.h \ - mlxreg.h mlxio.h rrunnerreg.h rrunnervar.h wdcreg.h wi_ieee.h + mlxreg.h mlxio.h nvmeio.h nvmereg.h rrunnerreg.h rrunnervar.h \ + wdcreg.h wi_ieee.h .include <bsd.kinc.mk> diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c index 3f8e0cb19ee..0e3e8a2d6c1 100644 --- a/sys/dev/ic/nvme.c +++ b/sys/dev/ic/nvme.c @@ -1,4 +1,4 @@ -/* $NetBSD: nvme.c,v 1.2 2016/05/02 19:18:29 christos Exp $ */ +/* $NetBSD: nvme.c,v 1.3 2016/06/04 16:11:51 nonaka Exp $ */ /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ /* @@ -18,7 +18,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.2 2016/05/02 19:18:29 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.3 2016/06/04 16:11:51 nonaka Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -26,14 +26,19 @@ __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.2 2016/05/02 19:18:29 christos Exp $"); #include <sys/atomic.h> #include <sys/bus.h> #include <sys/buf.h> +#include <sys/conf.h> #include <sys/device.h> #include <sys/kmem.h> #include <sys/once.h> +#include <sys/proc.h> #include <sys/queue.h> #include <sys/mutex.h> +#include <uvm/uvm_extern.h> + #include <dev/ic/nvmereg.h> #include <dev/ic/nvmevar.h> +#include <dev/ic/nvmeio.h> int nvme_adminq_size = 128; int nvme_ioq_size = 128; @@ -95,6 +100,13 @@ static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); +static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, + void *); +static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, + struct nvme_cqe *); +static int nvme_command_passthrough(struct nvme_softc *, + struct nvme_pt_command *, uint16_t, struct lwp *, bool); + #define nvme_read4(_s, _r) \ bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) #define nvme_write4(_s, _r, _v) \ @@ -729,6 +741,136 @@ nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) } static void +nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) +{ + struct nvme_softc *sc = q->q_sc; + struct nvme_sqe *sqe = slot; + struct nvme_pt_command *pt = ccb->ccb_cookie; + bus_dmamap_t dmap = ccb->ccb_dmamap; + int i; + + sqe->opcode = pt->cmd.opcode; + htolem32(&sqe->nsid, pt->cmd.nsid); + + if (pt->buf != NULL && pt->len > 0) { + htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); + switch (dmap->dm_nsegs) { + case 1: + break; + case 2: + htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); + break; + default: + for (i = 1; i < dmap->dm_nsegs; i++) { + htolem64(&ccb->ccb_prpl[i - 1], + dmap->dm_segs[i].ds_addr); + } + bus_dmamap_sync(sc->sc_dmat, + NVME_DMA_MAP(q->q_ccb_prpls), + ccb->ccb_prpl_off, + sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, + BUS_DMASYNC_PREWRITE); + htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); + break; + } + } + + htolem32(&sqe->cdw10, pt->cmd.cdw10); + htolem32(&sqe->cdw11, pt->cmd.cdw11); + htolem32(&sqe->cdw12, pt->cmd.cdw12); + htolem32(&sqe->cdw13, pt->cmd.cdw13); + htolem32(&sqe->cdw14, pt->cmd.cdw14); + htolem32(&sqe->cdw15, pt->cmd.cdw15); +} + +static void +nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) +{ + struct nvme_softc *sc = q->q_sc; + struct nvme_pt_command *pt = ccb->ccb_cookie; + bus_dmamap_t dmap = ccb->ccb_dmamap; + + if (pt->buf != NULL && pt->len > 0) { + if (dmap->dm_nsegs > 2) { + bus_dmamap_sync(sc->sc_dmat, + NVME_DMA_MAP(q->q_ccb_prpls), + ccb->ccb_prpl_off, + sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, + BUS_DMASYNC_POSTWRITE); + } + + bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, + pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(sc->sc_dmat, dmap); + } + + pt->cpl.cdw0 = cqe->cdw0; + pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; +} + +static int +nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, + uint16_t nsid, struct lwp *l, bool is_adminq) +{ + struct nvme_queue *q; + struct nvme_ccb *ccb; + void *buf = NULL; + int error; + + if ((pt->buf == NULL && pt->len > 0) || + (pt->buf != NULL && pt->len == 0)) + return EINVAL; + + q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); + ccb = nvme_ccb_get(q); + if (ccb == NULL) + return EBUSY; + + if (pt->buf != NULL && pt->len > 0) { + buf = kmem_alloc(pt->len, KM_SLEEP); + if (buf == NULL) { + error = ENOMEM; + goto ccb_put; + } + if (!pt->is_read) { + error = copyin(pt->buf, buf, pt->len); + if (error) + goto kmem_free; + } + error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, + pt->len, NULL, + BUS_DMA_WAITOK | + (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); + if (error) + goto kmem_free; + bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, + 0, ccb->ccb_dmamap->dm_mapsize, + pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); + } + + ccb->ccb_done = nvme_pt_done; + ccb->ccb_cookie = pt; + + pt->cmd.nsid = nsid; + if (nvme_poll(sc, q, ccb, nvme_pt_fill)) { + error = EIO; + goto out; + } + + error = 0; +out: + if (buf != NULL) { + if (error == 0 && pt->is_read) + error = copyout(buf, pt->buf, pt->len); +kmem_free: + kmem_free(buf, pt->len); + } +ccb_put: + nvme_ccb_put(q, ccb); + return error; +} + +static void nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) { @@ -1289,3 +1431,188 @@ nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); kmem_free(ndm, sizeof(*ndm)); } + +/* + * ioctl + */ + +/* nvme */ +dev_type_open(nvmeopen); +dev_type_close(nvmeclose); +dev_type_ioctl(nvmeioctl); + +const struct cdevsw nvme_cdevsw = { + .d_open = nvmeopen, + .d_close = nvmeclose, + .d_read = noread, + .d_write = nowrite, + .d_ioctl = nvmeioctl, + .d_stop = nostop, + .d_tty = notty, + .d_poll = nopoll, + .d_mmap = nommap, + .d_kqfilter = nokqfilter, + .d_discard = nodiscard, + .d_flag = D_OTHER, +}; + +extern struct cfdriver nvme_cd; + +/* + * Accept an open operation on the control device. + */ +int +nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) +{ + struct nvme_softc *sc; + int unit = minor(dev); + + if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) + return ENXIO; + if ((sc->sc_flags & NVME_F_ATTACHED) == 0) + return ENXIO; + if (ISSET(sc->sc_flags, NVME_F_OPEN)) + return EBUSY; + + SET(sc->sc_flags, NVME_F_OPEN); + return 0; +} + +/* + * Accept the last close on the control device. + */ +int +nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) +{ + struct nvme_softc *sc; + int unit = minor(dev); + + sc = device_lookup_private(&nvme_cd, unit); + if (sc == NULL) + return ENXIO; + + CLR(sc->sc_flags, NVME_F_OPEN); + return 0; +} + +/* + * Handle control operations. + */ +int +nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) +{ + struct nvme_softc *sc; + struct nvme_pt_command *pt; + int unit = minor(dev); + + sc = device_lookup_private(&nvme_cd, unit); + if (sc == NULL) + return ENXIO; + + switch (cmd) { + case NVME_PASSTHROUGH_CMD: + pt = (struct nvme_pt_command *)data; + return nvme_command_passthrough(sc, pt, pt->cmd.nsid, l, true); + } + + return ENOTTY; +} + +/* nvmens */ +dev_type_open(nvmensopen); +dev_type_close(nvmensclose); +dev_type_ioctl(nvmensioctl); + +const struct cdevsw nvmens_cdevsw = { + .d_open = nvmensopen, + .d_close = nvmensclose, + .d_read = noread, + .d_write = nowrite, + .d_ioctl = nvmensioctl, + .d_stop = nostop, + .d_tty = notty, + .d_poll = nopoll, + .d_mmap = nommap, + .d_kqfilter = nokqfilter, + .d_discard = nodiscard, + .d_flag = D_OTHER, +}; + +extern struct cfdriver nvmens_cd; + +/* + * Accept an open operation on the control device. + */ +int +nvmensopen(dev_t dev, int flag, int mode, struct lwp *l) +{ + struct nvme_softc *sc; + int unit = minor(dev) / 0x10000; + int nsid = minor(dev) & 0xffff; + int nsidx; + + if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) + return ENXIO; + if ((sc->sc_flags & NVME_F_ATTACHED) == 0) + return ENXIO; + if (nsid == 0) + return ENXIO; + + nsidx = nsid - 1; + if (nsidx > sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) + return ENXIO; + if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) + return EBUSY; + + SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); + return 0; +} + +/* + * Accept the last close on the control device. + */ +int +nvmensclose(dev_t dev, int flag, int mode, struct lwp *l) +{ + struct nvme_softc *sc; + int unit = minor(dev) / 0x10000; + int nsid = minor(dev) & 0xffff; + int nsidx; + + sc = device_lookup_private(&nvme_cd, unit); + if (sc == NULL) + return ENXIO; + if (nsid == 0) + return ENXIO; + + nsidx = nsid - 1; + if (nsidx > sc->sc_nn) + return ENXIO; + + CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); + return 0; +} + +/* + * Handle control operations. + */ +int +nvmensioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) +{ + struct nvme_softc *sc; + int unit = minor(dev) / 0x10000; + int nsid = minor(dev) & 0xffff; + + sc = device_lookup_private(&nvme_cd, unit); + if (sc == NULL) + return ENXIO; + if (nsid == 0) + return ENXIO; + + switch (cmd) { + case NVME_PASSTHROUGH_CMD: + return nvme_command_passthrough(sc, data, nsid, l, false); + } + + return ENOTTY; +} diff --git a/sys/dev/ic/nvmeio.h b/sys/dev/ic/nvmeio.h new file mode 100644 index 00000000000..dbfe8e9970f --- /dev/null +++ b/sys/dev/ic/nvmeio.h @@ -0,0 +1,97 @@ +/* $NetBSD: nvmeio.h,v 1.1 2016/06/04 16:11:51 nonaka Exp $ */ + +/*- + * Copyright (C) 2012-2013 Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/dev/nvme/nvme.h 296617 2016-03-10 17:13:10Z mav $ + */ + +#ifndef __NVMEIO_H__ +#define __NVMEIO_H__ + +#include <sys/ioccom.h> +#include <dev/ic/nvmereg.h> + +#define NVME_PASSTHROUGH_CMD _IOWR('n', 0, struct nvme_pt_command) + +#define nvme_completion_is_error(cpl) \ + ((NVME_CQE_SC((cpl)->flags) != NVME_CQE_SC_SUCCESS) \ + || (NVME_CQE_SCT((cpl)->flags) != NVME_CQE_SCT_GENERIC)) + +struct nvme_pt_command { + + /* + * cmd is used to specify a passthrough command to a controller or + * namespace. + * + * The following fields from cmd may be specified by the caller: + * * opcode + * * nsid (namespace id) - for admin commands only + * * cdw10-cdw15 + * + * Remaining fields must be set to 0 by the caller. + */ + struct nvme_sqe cmd; + + /* + * cpl returns completion status for the passthrough command + * specified by cmd. + * + * The following fields will be filled out by the driver, for + * consumption by the caller: + * * cdw0 + * * flags (except for phase) + * + * Remaining fields will be set to 0 by the driver. + */ + struct nvme_cqe cpl; + + /* buf is the data buffer associated with this passthrough command. */ + void *buf; + + /* + * len is the length of the data buffer associated with this + * passthrough command. + */ + uint32_t len; + + /* + * is_read = 1 if the passthrough command will read data into the + * supplied buffer from the controller. + * + * is_read = 0 if the passthrough command will write data from the + * supplied buffer to the controller. + */ + uint32_t is_read; + + /* + * timeout (unit: ms) + * + * 0: use default timeout value + */ + uint32_t timeout; +}; + +#endif /* __NVMEIO_H__ */ diff --git a/sys/dev/ic/nvmereg.h b/sys/dev/ic/nvmereg.h index 8ac10b4b747..c4663f9b3d2 100644 --- a/sys/dev/ic/nvmereg.h +++ b/sys/dev/ic/nvmereg.h @@ -1,4 +1,4 @@ -/* $NetBSD: nvmereg.h,v 1.1 2016/05/01 10:21:02 nonaka Exp $ */ +/* $NetBSD: nvmereg.h,v 1.2 2016/06/04 16:11:51 nonaka Exp $ */ /* $OpenBSD: nvmereg.h,v 1.10 2016/04/14 11:18:32 dlg Exp $ */ /* @@ -17,6 +17,9 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#ifndef __NVMEREG_H__ +#define __NVMEREG_H__ + #define NVME_CAP 0x0000 /* Controller Capabilities */ #define NVME_CAP_MPSMAX(_r) (12 + (((_r) >> 52) & 0xf)) /* shift */ #define NVME_CAP_MPSMIN(_r) (12 + (((_r) >> 48) & 0xf)) /* shift */ @@ -385,3 +388,5 @@ struct nvm_identify_namespace { uint8_t vs[3712]; } __packed __aligned(8); + +#endif /* __NVMEREG_H__ */ diff --git a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h index bf9187a6dd9..f00f6cbe21d 100644 --- a/sys/dev/ic/nvmevar.h +++ b/sys/dev/ic/nvmevar.h @@ -1,4 +1,4 @@ -/* $NetBSD: nvmevar.h,v 1.1 2016/05/01 10:21:02 nonaka Exp $ */ +/* $NetBSD: nvmevar.h,v 1.2 2016/06/04 16:11:51 nonaka Exp $ */ /* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ /* @@ -79,6 +79,8 @@ struct nvme_queue { struct nvme_namespace { struct nvm_identify_namespace *ident; device_t dev; + uint32_t flags; +#define NVME_NS_F_OPEN __BIT(0) }; struct nvme_softc { @@ -113,6 +115,7 @@ struct nvme_softc { uint32_t sc_flags; #define NVME_F_ATTACHED __BIT(0) +#define NVME_F_OPEN __BIT(1) }; #define lemtoh16(p) le16toh(*((uint16_t *)(p))) |
