summaryrefslogtreecommitdiff
path: root/sys/dev
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev')
-rw-r--r--sys/dev/ic/ld_nvme.c124
-rw-r--r--sys/dev/ic/nvme.c201
-rw-r--r--sys/dev/ic/nvmevar.h9
3 files changed, 170 insertions, 164 deletions
diff --git a/sys/dev/ic/ld_nvme.c b/sys/dev/ic/ld_nvme.c
index 5d1a1984247..257bb72c60b 100644
--- a/sys/dev/ic/ld_nvme.c
+++ b/sys/dev/ic/ld_nvme.c
@@ -1,4 +1,4 @@
-/* $NetBSD: ld_nvme.c,v 1.18 2018/01/23 22:42:29 pgoyette Exp $ */
+/* $NetBSD: ld_nvme.c,v 1.19 2018/03/16 23:31:19 jdolecek Exp $ */
/*-
* Copyright (C) 2016 NONAKA Kimihiro <nonaka@netbsd.org>
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ld_nvme.c,v 1.18 2018/01/23 22:42:29 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ld_nvme.c,v 1.19 2018/03/16 23:31:19 jdolecek Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -49,14 +49,6 @@ struct ld_nvme_softc {
struct nvme_softc *sc_nvme;
uint16_t sc_nsid;
-
- /* getcache handling */
- kmutex_t sc_getcache_lock;
- kcondvar_t sc_getcache_cv;
- kcondvar_t sc_getcache_ready_cv;
- bool sc_getcache_waiting;
- bool sc_getcache_ready;
- int sc_getcache_result;
};
static int ld_nvme_match(device_t, cfdata_t, void *);
@@ -73,8 +65,6 @@ static int ld_nvme_getcache(struct ld_softc *, int *);
static int ld_nvme_ioctl(struct ld_softc *, u_long, void *, int32_t, bool);
static void ld_nvme_biodone(void *, struct buf *, uint16_t, uint32_t);
-static void ld_nvme_syncdone(void *, struct buf *, uint16_t, uint32_t);
-static void ld_nvme_getcache_done(void *, struct buf *, uint16_t, uint32_t);
static int
ld_nvme_match(device_t parent, cfdata_t match, void *aux)
@@ -103,10 +93,6 @@ ld_nvme_attach(device_t parent, device_t self, void *aux)
sc->sc_nvme = nsc;
sc->sc_nsid = naa->naa_nsid;
- mutex_init(&sc->sc_getcache_lock, MUTEX_DEFAULT, IPL_SOFTBIO);
- cv_init(&sc->sc_getcache_cv, "nvmegcq");
- cv_init(&sc->sc_getcache_ready_cv, "nvmegcr");
-
aprint_naive("\n");
aprint_normal("\n");
@@ -203,116 +189,16 @@ ld_nvme_flush(struct ld_softc *ld, bool poll)
{
struct ld_nvme_softc *sc = device_private(ld->sc_dv);
- if (!nvme_has_volatile_write_cache(sc->sc_nvme)) {
- /* cache not present, no value in trying to flush it */
- return 0;
- }
-
- return nvme_ns_sync(sc->sc_nvme, sc->sc_nsid, sc,
- poll ? NVME_NS_CTX_F_POLL : 0,
- ld_nvme_syncdone);
-}
-
-static void
-ld_nvme_syncdone(void *xc, struct buf *bp, uint16_t cmd_status, uint32_t cdw0)
-{
- /* nothing to do */
+ return nvme_ns_sync(sc->sc_nvme, sc->sc_nsid,
+ poll ? NVME_NS_CTX_F_POLL : 0);
}
static int
ld_nvme_getcache(struct ld_softc *ld, int *addr)
{
- int error;
struct ld_nvme_softc *sc = device_private(ld->sc_dv);
- /*
- * DPO not supported, Dataset Management (DSM) field doesn't specify
- * the same semantics.
- */
- *addr = DKCACHE_FUA;
-
- if (!nvme_has_volatile_write_cache(sc->sc_nvme)) {
- /* cache simply not present */
- return 0;
- }
-
- /*
- * This is admin queue request. The queue is relatively limited in size,
- * and this is not performance critical call, so have at most one pending
- * cache request at a time to avoid spurious EWOULDBLOCK failures.
- */
- mutex_enter(&sc->sc_getcache_lock);
- while (sc->sc_getcache_waiting) {
- error = cv_wait_sig(&sc->sc_getcache_cv, &sc->sc_getcache_lock);
- if (error)
- goto out;
- }
- sc->sc_getcache_waiting = true;
- sc->sc_getcache_ready = false;
- mutex_exit(&sc->sc_getcache_lock);
-
- error = nvme_admin_getcache(sc->sc_nvme, sc, ld_nvme_getcache_done);
- if (error) {
- mutex_enter(&sc->sc_getcache_lock);
- goto out;
- }
-
- mutex_enter(&sc->sc_getcache_lock);
- while (!sc->sc_getcache_ready) {
- error = cv_wait_sig(&sc->sc_getcache_ready_cv,
- &sc->sc_getcache_lock);
- if (error)
- goto out;
- }
-
- KDASSERT(sc->sc_getcache_ready);
-
- if (sc->sc_getcache_result >= 0)
- *addr |= sc->sc_getcache_result;
- else
- error = EINVAL;
-
- out:
- sc->sc_getcache_waiting = false;
-
- /* wake one of eventual waiters */
- cv_signal(&sc->sc_getcache_cv);
-
- mutex_exit(&sc->sc_getcache_lock);
-
- return error;
-}
-
-static void
-ld_nvme_getcache_done(void *xc, struct buf *bp, uint16_t cmd_status, uint32_t cdw0)
-{
- struct ld_nvme_softc *sc = xc;
- uint16_t status = NVME_CQE_SC(cmd_status);
- int result;
-
- if (status == NVME_CQE_SC_SUCCESS) {
- result = 0;
-
- if (cdw0 & NVME_CQE_CDW0_VWC_WCE)
- result |= DKCACHE_WRITE;
-
- /*
- * If volatile write cache is present, the flag shall also be
- * settable.
- */
- result |= DKCACHE_WCHANGE;
- } else {
- result = -1;
- }
-
- mutex_enter(&sc->sc_getcache_lock);
- sc->sc_getcache_result = result;
- sc->sc_getcache_ready = true;
-
- /* wake up the waiter */
- cv_signal(&sc->sc_getcache_ready_cv);
-
- mutex_exit(&sc->sc_getcache_lock);
+ return nvme_admin_getcache(sc->sc_nvme, addr);
}
static int
diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c
index e097f85185c..176c91189c9 100644
--- a/sys/dev/ic/nvme.c
+++ b/sys/dev/ic/nvme.c
@@ -1,4 +1,4 @@
-/* $NetBSD: nvme.c,v 1.33 2018/03/16 18:49:18 jdolecek Exp $ */
+/* $NetBSD: nvme.c,v 1.34 2018/03/16 23:31:19 jdolecek Exp $ */
/* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
/*
@@ -18,7 +18,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.33 2018/03/16 18:49:18 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.34 2018/03/16 23:31:19 jdolecek Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -63,7 +63,7 @@ static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t);
static void nvme_ccbs_free(struct nvme_queue *);
static struct nvme_ccb *
- nvme_ccb_get(struct nvme_queue *);
+ nvme_ccb_get(struct nvme_queue *, bool);
static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *);
static int nvme_poll(struct nvme_softc *, struct nvme_queue *,
@@ -85,6 +85,8 @@ static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *,
struct nvme_ccb *, void *));
static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q);
static void nvme_q_free(struct nvme_softc *, struct nvme_queue *);
+static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *,
+ bool (*)(void *), void *);
static struct nvme_dmamem *
nvme_dmamem_alloc(struct nvme_softc *, size_t);
@@ -566,7 +568,7 @@ nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid)
KASSERT(nsid > 0);
- ccb = nvme_ccb_get(sc->sc_admin_q);
+ ccb = nvme_ccb_get(sc->sc_admin_q, false);
KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
mem = nvme_dmamem_alloc(sc, sizeof(*identify));
@@ -622,7 +624,7 @@ nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie,
bus_dmamap_t dmap;
int i, error;
- ccb = nvme_ccb_get(q);
+ ccb = nvme_ccb_get(q, false);
if (ccb == NULL)
return EAGAIN;
@@ -741,31 +743,44 @@ nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb,
* If there is no volatile write cache, it makes no sense to issue
* flush commands or query for the status.
*/
-bool
+static bool
nvme_has_volatile_write_cache(struct nvme_softc *sc)
{
/* sc_identify is filled during attachment */
return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0);
}
+static bool
+nvme_ns_sync_finished(void *cookie)
+{
+ int *result = cookie;
+
+ return (*result != 0);
+}
+
int
-nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie,
- int flags, nvme_nnc_done nnc_done)
+nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags)
{
struct nvme_queue *q = nvme_get_q(sc);
struct nvme_ccb *ccb;
+ int result = 0;
+
+ if (!nvme_has_volatile_write_cache(sc)) {
+ /* cache not present, no value in trying to flush it */
+ return 0;
+ }
- ccb = nvme_ccb_get(q);
+ ccb = nvme_ccb_get(q, true);
if (ccb == NULL)
return EAGAIN;
ccb->ccb_done = nvme_ns_sync_done;
- ccb->ccb_cookie = cookie;
+ ccb->ccb_cookie = &result;
/* namespace context */
ccb->nnc_nsid = nsid;
ccb->nnc_flags = flags;
- ccb->nnc_done = nnc_done;
+ ccb->nnc_done = NULL;
if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0)
@@ -774,7 +789,12 @@ nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie,
}
nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill);
- return 0;
+
+ /* wait for completion */
+ nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result);
+ KASSERT(result != 0);
+
+ return (result > 0) ? 0 : EIO;
}
static void
@@ -790,36 +810,64 @@ static void
nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb,
struct nvme_cqe *cqe)
{
- void *cookie = ccb->ccb_cookie;
- nvme_nnc_done nnc_done = ccb->nnc_done;
+ int *result = ccb->ccb_cookie;
+ uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags));
+
+ if (status == NVME_CQE_SC_SUCCESS)
+ *result = 1;
+ else
+ *result = -1;
nvme_ccb_put(q, ccb);
+}
+
+static bool
+nvme_getcache_finished(void *xc)
+{
+ int *addr = xc;
- nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
+ return (*addr != 0);
}
/*
* Get status of volatile write cache. Always asynchronous.
*/
int
-nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done)
+nvme_admin_getcache(struct nvme_softc *sc, int *addr)
{
struct nvme_ccb *ccb;
struct nvme_queue *q = sc->sc_admin_q;
+ int result = 0, error;
- ccb = nvme_ccb_get(q);
- if (ccb == NULL)
- return EAGAIN;
+ if (!nvme_has_volatile_write_cache(sc)) {
+ /* cache simply not present */
+ *addr = 0;
+ return 0;
+ }
+
+ ccb = nvme_ccb_get(q, true);
+ KASSERT(ccb != NULL);
ccb->ccb_done = nvme_getcache_done;
- ccb->ccb_cookie = cookie;
+ ccb->ccb_cookie = &result;
/* namespace context */
ccb->nnc_flags = 0;
- ccb->nnc_done = nnc_done;
+ ccb->nnc_done = NULL;
nvme_q_submit(sc, q, ccb, nvme_getcache_fill);
- return 0;
+
+ /* wait for completion */
+ nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result);
+ KASSERT(result != 0);
+
+ if (result > 0) {
+ *addr = result;
+ error = 0;
+ } else
+ error = EINVAL;
+
+ return error;
}
static void
@@ -835,12 +883,35 @@ static void
nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb,
struct nvme_cqe *cqe)
{
- void *cookie = ccb->ccb_cookie;
- nvme_nnc_done nnc_done = ccb->nnc_done;
+ int *addr = ccb->ccb_cookie;
+ uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags));
+ uint32_t cdw0 = lemtoh32(&cqe->cdw0);
+ int result;
- nvme_ccb_put(q, ccb);
+ if (status == NVME_CQE_SC_SUCCESS) {
+ result = 0;
+
+ /*
+ * DPO not supported, Dataset Management (DSM) field doesn't
+ * specify the same semantics. FUA is always supported.
+ */
+ result = DKCACHE_FUA;
+
+ if (cdw0 & NVME_CQE_CDW0_VWC_WCE)
+ result |= DKCACHE_WRITE;
+
+ /*
+ * If volatile write cache is present, the flag shall also be
+ * settable.
+ */
+ result |= DKCACHE_WCHANGE;
+ } else {
+ result = -1;
+ }
+
+ *addr = result;
- nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
+ nvme_ccb_put(q, ccb);
}
void
@@ -941,9 +1012,8 @@ nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt,
return EINVAL;
q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc);
- ccb = nvme_ccb_get(q);
- if (ccb == NULL)
- return EBUSY;
+ ccb = nvme_ccb_get(q, true);
+ KASSERT(ccb != NULL);
if (pt->buf != NULL) {
KASSERT(pt->len > 0);
@@ -1014,6 +1084,8 @@ nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
struct nvme_poll_state {
struct nvme_sqe s;
struct nvme_cqe c;
+ void *cookie;
+ void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *);
};
static int
@@ -1021,8 +1093,6 @@ nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec)
{
struct nvme_poll_state state;
- void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *);
- void *cookie;
uint16_t flags;
int step = 10;
int maxloop = timo_sec * 1000000 / step;
@@ -1031,8 +1101,8 @@ nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
memset(&state, 0, sizeof(state));
(*fill)(q, ccb, &state.s);
- done = ccb->ccb_done;
- cookie = ccb->ccb_cookie;
+ state.done = ccb->ccb_done;
+ state.cookie = ccb->ccb_cookie;
ccb->ccb_done = nvme_poll_done;
ccb->ccb_cookie = &state;
@@ -1048,13 +1118,22 @@ nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
}
}
- ccb->ccb_cookie = cookie;
- done(q, ccb, &state.c);
-
if (error == 0) {
flags = lemtoh16(&state.c.flags);
return flags & ~NVME_CQE_PHASE;
} else {
+ /*
+ * If it succeds later, it would hit ccb which will have been
+ * already reused for something else. Not good. Cross
+ * fingers and hope for best. XXX do controller reset?
+ */
+ aprint_error_dev(sc->sc_dev, "polled command timed out\n");
+
+ /* Invoke the callback to clean state anyway */
+ struct nvme_cqe cqe;
+ memset(&cqe, 0, sizeof(cqe));
+ ccb->ccb_done(q, ccb, &cqe);
+
return 1;
}
}
@@ -1076,6 +1155,9 @@ nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb,
SET(cqe->flags, htole16(NVME_CQE_PHASE));
state->c = *cqe;
+
+ ccb->ccb_cookie = state->cookie;
+ state->done(q, ccb, &state->c);
}
static void
@@ -1152,6 +1234,26 @@ nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q)
return rv;
}
+static void
+nvme_q_wait_complete(struct nvme_softc *sc,
+ struct nvme_queue *q, bool (*finished)(void *), void *cookie)
+{
+ mutex_enter(&q->q_ccb_mtx);
+ if (finished(cookie))
+ goto out;
+
+ for(;;) {
+ q->q_ccb_waiting = true;
+ cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx);
+
+ if (finished(cookie))
+ break;
+ }
+
+out:
+ mutex_exit(&q->q_ccb_mtx);
+}
+
static int
nvme_identify(struct nvme_softc *sc, u_int mps)
{
@@ -1162,7 +1264,7 @@ nvme_identify(struct nvme_softc *sc, u_int mps)
u_int mdts;
int rv = 1;
- ccb = nvme_ccb_get(sc->sc_admin_q);
+ ccb = nvme_ccb_get(sc->sc_admin_q, false);
KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
mem = nvme_dmamem_alloc(sc, sizeof(*identify));
@@ -1219,7 +1321,7 @@ nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q)
if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0)
return 1;
- ccb = nvme_ccb_get(sc->sc_admin_q);
+ ccb = nvme_ccb_get(sc->sc_admin_q, false);
KASSERT(ccb != NULL);
ccb->ccb_done = nvme_empty_done;
@@ -1265,7 +1367,7 @@ nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q)
struct nvme_ccb *ccb;
int rv;
- ccb = nvme_ccb_get(sc->sc_admin_q);
+ ccb = nvme_ccb_get(sc->sc_admin_q, false);
KASSERT(ccb != NULL);
ccb->ccb_done = nvme_empty_done;
@@ -1320,7 +1422,7 @@ nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap)
uint16_t ncqa, nsqa;
int rv;
- ccb = nvme_ccb_get(sc->sc_admin_q);
+ ccb = nvme_ccb_get(sc->sc_admin_q, false);
KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
memset(&pt, 0, sizeof(pt));
@@ -1356,6 +1458,8 @@ nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs)
u_int i;
mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO);
+ cv_init(&q->q_ccb_wait, "nvmeqw");
+ q->q_ccb_waiting = false;
SIMPLEQ_INIT(&q->q_ccb_list);
q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP);
@@ -1395,17 +1499,24 @@ free_maps:
}
static struct nvme_ccb *
-nvme_ccb_get(struct nvme_queue *q)
+nvme_ccb_get(struct nvme_queue *q, bool wait)
{
struct nvme_ccb *ccb = NULL;
mutex_enter(&q->q_ccb_mtx);
+again:
ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
if (ccb != NULL) {
SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
#ifdef DEBUG
ccb->ccb_cookie = NULL;
#endif
+ } else {
+ if (__predict_false(wait)) {
+ q->q_ccb_waiting = true;
+ cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx);
+ goto again;
+ }
}
mutex_exit(&q->q_ccb_mtx);
@@ -1421,6 +1532,13 @@ nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb)
ccb->ccb_cookie = (void *)NVME_CCB_FREE;
#endif
SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry);
+
+ /* It's unlikely there are any waiters, it's not used for regular I/O */
+ if (__predict_false(q->q_ccb_waiting)) {
+ q->q_ccb_waiting = false;
+ cv_broadcast(&q->q_ccb_wait);
+ }
+
mutex_exit(&q->q_ccb_mtx);
}
@@ -1440,6 +1558,7 @@ nvme_ccbs_free(struct nvme_queue *q)
nvme_dmamem_free(sc, q->q_ccb_prpls);
kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs);
q->q_ccbs = NULL;
+ cv_destroy(&q->q_ccb_wait);
mutex_destroy(&q->q_ccb_mtx);
}
diff --git a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h
index 19da75a6095..8a0ec0914f6 100644
--- a/sys/dev/ic/nvmevar.h
+++ b/sys/dev/ic/nvmevar.h
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmevar.h,v 1.14 2018/03/16 18:49:18 jdolecek Exp $ */
+/* $NetBSD: nvmevar.h,v 1.15 2018/03/16 23:31:19 jdolecek Exp $ */
/* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */
/*
@@ -87,6 +87,8 @@ struct nvme_queue {
uint16_t q_cq_phase;
kmutex_t q_ccb_mtx;
+ kcondvar_t q_ccb_wait; /* wait for ccb avail/finish */
+ bool q_ccb_waiting; /* whether there are waiters */
uint16_t q_nccbs; /* total number of ccbs */
struct nvme_ccb *q_ccbs;
SIMPLEQ_HEAD(, nvme_ccb) q_ccb_list;
@@ -179,6 +181,5 @@ int nvme_ns_identify(struct nvme_softc *, uint16_t);
void nvme_ns_free(struct nvme_softc *, uint16_t);
int nvme_ns_dobio(struct nvme_softc *, uint16_t, void *,
struct buf *, void *, size_t, int, daddr_t, int, nvme_nnc_done);
-int nvme_ns_sync(struct nvme_softc *, uint16_t, void *, int, nvme_nnc_done);
-bool nvme_has_volatile_write_cache(struct nvme_softc *);
-int nvme_admin_getcache(struct nvme_softc *, void *, nvme_nnc_done);
+int nvme_ns_sync(struct nvme_softc *, uint16_t, int);
+int nvme_admin_getcache(struct nvme_softc *, int *);