diff options
| author | nonaka <nonaka@NetBSD.org> | 2022-05-20 13:55:16 +0000 |
|---|---|---|
| committer | nonaka <nonaka@NetBSD.org> | 2022-05-20 13:55:16 +0000 |
| commit | c2ac0d610a6a8512d4d5ac55b43783dc75d86aaa (patch) | |
| tree | e77911c31d081d878761ad55021732d774ae61c5 /sys/dev/hyperv | |
| parent | 75b8c1c8296bd48dfee1231a2108a2e8a7bd2eca (diff) | |
Improve Hyper-V support.
vmbus(4):
- Added support for multichannel.
hvn(4):
- Added support for multichannel.
- Added support for change MTU.
- Added support for TX aggregation.
- Improve VLAN support.
- Improve checksum offload support.
Diffstat (limited to 'sys/dev/hyperv')
| -rw-r--r-- | sys/dev/hyperv/files.hyperv | 7 | ||||
| -rw-r--r-- | sys/dev/hyperv/hyperv_common.c | 25 | ||||
| -rw-r--r-- | sys/dev/hyperv/hypervreg.h | 3 | ||||
| -rw-r--r-- | sys/dev/hyperv/hypervvar.h | 7 | ||||
| -rw-r--r-- | sys/dev/hyperv/if_hvn.c | 4511 | ||||
| -rw-r--r-- | sys/dev/hyperv/if_hvnreg.h | 19 | ||||
| -rw-r--r-- | sys/dev/hyperv/vmbus.c | 228 | ||||
| -rw-r--r-- | sys/dev/hyperv/vmbusvar.h | 16 |
8 files changed, 4178 insertions, 638 deletions
diff --git a/sys/dev/hyperv/files.hyperv b/sys/dev/hyperv/files.hyperv index 385b52dcb3f..f482b35b651 100644 --- a/sys/dev/hyperv/files.hyperv +++ b/sys/dev/hyperv/files.hyperv @@ -1,4 +1,4 @@ -# $NetBSD: files.hyperv,v 1.2 2019/05/24 14:28:48 nonaka Exp $ +# $NetBSD: files.hyperv,v 1.3 2022/05/20 13:55:17 nonaka Exp $ define hypervvmbus {} device vmbus: hypervvmbus @@ -14,6 +14,11 @@ file dev/hyperv/hvkbd.c hvkbd needs-flag device hvn: ether, ifnet, arp attach hvn at hypervvmbus file dev/hyperv/if_hvn.c hvn +defparam opt_if_hvn.h HVN_UDP_CKSUM_FIXUP_MTU_DEFAULT + HVN_CHANNEL_MAX_COUNT_DEFAULT + HVN_CHANNEL_COUNT_DEFAULT + HVN_TX_RING_COUNT_DEFAULT + HVN_LINK_STATE_CHANGE_DELAY device hvs: scsi attach hvs at hypervvmbus diff --git a/sys/dev/hyperv/hyperv_common.c b/sys/dev/hyperv/hyperv_common.c index 4347dc47f86..6305eeeed40 100644 --- a/sys/dev/hyperv/hyperv_common.c +++ b/sys/dev/hyperv/hyperv_common.c @@ -1,4 +1,4 @@ -/* $NetBSD: hyperv_common.c,v 1.5 2019/12/10 12:20:20 nonaka Exp $ */ +/* $NetBSD: hyperv_common.c,v 1.6 2022/05/20 13:55:17 nonaka Exp $ */ /*- * Copyright (c) 2009-2012,2016-2017 Microsoft Corp. @@ -29,7 +29,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: hyperv_common.c,v 1.5 2019/12/10 12:20:20 nonaka Exp $"); +__KERNEL_RCSID(0, "$NetBSD: hyperv_common.c,v 1.6 2022/05/20 13:55:17 nonaka Exp $"); #include "hyperv.h" @@ -42,6 +42,7 @@ __KERNEL_RCSID(0, "$NetBSD: hyperv_common.c,v 1.5 2019/12/10 12:20:20 nonaka Exp #include <dev/hyperv/hypervreg.h> #include <dev/hyperv/hypervvar.h> +u_int hyperv_ver_major; hyperv_tc64_t hyperv_tc64; int hyperv_nullop(void); @@ -111,46 +112,40 @@ hyperv_guid2str(const struct hyperv_guid *guid, char *buf, size_t sz) */ void * hyperv_dma_alloc(bus_dma_tag_t dmat, struct hyperv_dma *dma, bus_size_t size, - bus_size_t alignment, bus_size_t boundary, int nsegs, int flags) + bus_size_t alignment, bus_size_t boundary, int nsegs) { - const int waitok = (flags & HYPERV_DMA_NOSLEEP) != HYPERV_DMA_NOSLEEP; - const int kmemflags = waitok ? KM_SLEEP: KM_NOSLEEP; - const int dmaflags = waitok ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; int rseg, error; KASSERT(dma != NULL); KASSERT(dma->segs == NULL); KASSERT(nsegs > 0); - dma->segs = kmem_intr_zalloc(sizeof(*dma->segs) * nsegs, kmemflags); - if (dma->segs == NULL) - return NULL; - + dma->segs = kmem_zalloc(sizeof(*dma->segs) * nsegs, KM_SLEEP); dma->nsegs = nsegs; error = bus_dmamem_alloc(dmat, size, alignment, boundary, dma->segs, - nsegs, &rseg, dmaflags); + nsegs, &rseg, BUS_DMA_WAITOK); if (error) { printf("%s: bus_dmamem_alloc failed: error=%d\n", __func__, error); goto fail1; } error = bus_dmamem_map(dmat, dma->segs, rseg, size, &dma->addr, - dmaflags); + BUS_DMA_WAITOK); if (error) { printf("%s: bus_dmamem_map failed: error=%d\n", __func__, error); goto fail2; } - error = bus_dmamap_create(dmat, size, rseg, size, boundary, dmaflags, - &dma->map); + error = bus_dmamap_create(dmat, size, rseg, size, boundary, + BUS_DMA_WAITOK, &dma->map); if (error) { printf("%s: bus_dmamap_create failed: error=%d\n", __func__, error); goto fail3; } error = bus_dmamap_load(dmat, dma->map, dma->addr, size, NULL, - BUS_DMA_READ | BUS_DMA_WRITE | dmaflags); + BUS_DMA_READ | BUS_DMA_WRITE | BUS_DMA_WAITOK); if (error) { printf("%s: bus_dmamap_load failed: error=%d\n", __func__, error); diff --git a/sys/dev/hyperv/hypervreg.h b/sys/dev/hyperv/hypervreg.h index 68d10160239..8b5acca58b1 100644 --- a/sys/dev/hyperv/hypervreg.h +++ b/sys/dev/hyperv/hypervreg.h @@ -1,4 +1,4 @@ -/* $NetBSD: hypervreg.h,v 1.1 2019/02/15 08:54:01 nonaka Exp $ */ +/* $NetBSD: hypervreg.h,v 1.2 2022/05/20 13:55:17 nonaka Exp $ */ /* $OpenBSD: hypervreg.h,v 1.10 2017/01/05 13:17:22 mikeb Exp $ */ /*- @@ -258,6 +258,7 @@ struct vmbus_bufring { uint8_t br_rsvd[4084]; uint8_t br_data[0]; } __packed; +__CTASSERT(sizeof(struct vmbus_bufring) == PAGE_SIZE); /* * Channel diff --git a/sys/dev/hyperv/hypervvar.h b/sys/dev/hyperv/hypervvar.h index 3c7eb420f8c..3ebd98d7a45 100644 --- a/sys/dev/hyperv/hypervvar.h +++ b/sys/dev/hyperv/hypervvar.h @@ -1,4 +1,4 @@ -/* $NetBSD: hypervvar.h,v 1.5 2021/12/23 04:06:51 yamaguchi Exp $ */ +/* $NetBSD: hypervvar.h,v 1.6 2022/05/20 13:55:17 nonaka Exp $ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD @@ -59,6 +59,7 @@ __CTASSERT(sizeof(struct hyperv_reftsc) == PAGE_SIZE); #endif #if defined(_KERNEL) +extern u_int hyperv_ver_major; int hyperv_hypercall_enabled(void); int hyperv_synic_supported(void); @@ -108,10 +109,8 @@ hyperv_dma_get_paddr(struct hyperv_dma *dma) return dma->map->dm_segs[0].ds_addr; } -#define HYPERV_DMA_SLEEPOK 0 -#define HYPERV_DMA_NOSLEEP __BIT(0) void *hyperv_dma_alloc(bus_dma_tag_t, struct hyperv_dma *, bus_size_t, - bus_size_t, bus_size_t, int, int); + bus_size_t, bus_size_t, int); void hyperv_dma_free(bus_dma_tag_t, struct hyperv_dma *); #endif /* _KERNEL */ diff --git a/sys/dev/hyperv/if_hvn.c b/sys/dev/hyperv/if_hvn.c index 41f8495c4e7..eee00a66b4b 100644 --- a/sys/dev/hyperv/if_hvn.c +++ b/sys/dev/hyperv/if_hvn.c @@ -1,4 +1,4 @@ -/* $NetBSD: if_hvn.c,v 1.21 2021/06/16 00:21:18 riastradh Exp $ */ +/* $NetBSD: if_hvn.c,v 1.22 2022/05/20 13:55:17 nonaka Exp $ */ /* $OpenBSD: if_hvn.c,v 1.39 2018/03/11 14:31:34 mikeb Exp $ */ /*- @@ -35,9 +35,10 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: if_hvn.c,v 1.21 2021/06/16 00:21:18 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: if_hvn.c,v 1.22 2022/05/20 13:55:17 nonaka Exp $"); #ifdef _KERNEL_OPT +#include "opt_if_hvn.h" #include "opt_inet.h" #include "opt_inet6.h" #include "opt_net_mpsafe.h" @@ -47,14 +48,28 @@ __KERNEL_RCSID(0, "$NetBSD: if_hvn.c,v 1.21 2021/06/16 00:21:18 riastradh Exp $" #include <sys/systm.h> #include <sys/kernel.h> #include <sys/device.h> -#include <sys/atomic.h> +#include <sys/bitops.h> #include <sys/bus.h> +#include <sys/condvar.h> +#include <sys/cpu.h> +#include <sys/evcnt.h> #include <sys/intr.h> #include <sys/kmem.h> +#include <sys/kthread.h> +#include <sys/mutex.h> +#include <sys/pcq.h> +#include <sys/sysctl.h> +#include <sys/workqueue.h> #include <net/if.h> #include <net/if_ether.h> #include <net/if_media.h> +#include <net/if_vlanvar.h> +#include <net/rss_config.h> +#include <netinet/in.h> +#include <netinet/ip.h> +#include <netinet/ip6.h> +#include <netinet/udp.h> #include <net/bpf.h> @@ -71,9 +86,24 @@ __KERNEL_RCSID(0, "$NetBSD: if_hvn.c,v 1.21 2021/06/16 00:21:18 riastradh Exp $" #define EVL_CFI_BITS 12 #endif +#define HVN_CHIM_SIZE (15 * 1024 * 1024) + #define HVN_NVS_MSGSIZE 32 #define HVN_NVS_BUFSIZE PAGE_SIZE +#define HVN_RING_BUFSIZE (128 * PAGE_SIZE) +#define HVN_RING_IDX2CPU(sc, idx) ((idx) % ncpu) + +#ifndef HVN_CHANNEL_MAX_COUNT_DEFAULT +#define HVN_CHANNEL_MAX_COUNT_DEFAULT 8 +#endif + +#ifndef HVN_LINK_STATE_CHANGE_DELAY +#define HVN_LINK_STATE_CHANGE_DELAY 5000 +#endif + +#define HVN_WORKQUEUE_PRI PRI_SOFTNET + /* * RNDIS control interface */ @@ -93,17 +123,36 @@ struct rndis_cmd { uint8_t rc_cmpbuf[HVN_RNDIS_BUFSIZE]; int rc_done; TAILQ_ENTRY(rndis_cmd) rc_entry; + kmutex_t rc_lock; + kcondvar_t rc_cv; }; TAILQ_HEAD(rndis_queue, rndis_cmd); -#define HVN_MAXMTU (9 * 1024) +#define HVN_MTU_MIN 68 +#define HVN_MTU_MAX (65535 - ETHER_ADDR_LEN) #define HVN_RNDIS_XFER_SIZE 2048 +#define HVN_NDIS_TXCSUM_CAP_IP4 \ + (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) +#define HVN_NDIS_TXCSUM_CAP_TCP4 \ + (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) +#define HVN_NDIS_TXCSUM_CAP_TCP6 \ + (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ + NDIS_TXCSUM_CAP_IP6EXT) +#define HVN_NDIS_TXCSUM_CAP_UDP6 \ + (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) +#define HVN_NDIS_LSOV2_CAP_IP6 \ + (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) + +#define HVN_RNDIS_CMD_NORESP __BIT(0) + +#define HVN_NVS_CMD_NORESP __BIT(0) + /* * Tx ring */ -#define HVN_TX_DESC 256 +#define HVN_TX_DESC 512 #define HVN_TX_FRAGS 15 /* 31 is the max */ #define HVN_TX_FRAG_SIZE PAGE_SIZE #define HVN_TX_PKT_SIZE 16384 @@ -113,71 +162,291 @@ TAILQ_HEAD(rndis_queue, rndis_cmd); sizeof(struct rndis_pktinfo) + NDIS_VLAN_INFO_SIZE + \ sizeof(struct rndis_pktinfo) + NDIS_TXCSUM_INFO_SIZE) +#define HVN_PKTSIZE_MIN(align) \ + roundup2(ETHER_MIN_LEN + ETHER_VLAN_ENCAP_LEN - ETHER_CRC_LEN + \ + HVN_RNDIS_PKT_LEN, (align)) +#define HVN_PKTSIZE(m, align) \ + roundup2((m)->m_pkthdr.len + HVN_RNDIS_PKT_LEN, (align)) + struct hvn_tx_desc { uint32_t txd_id; - int txd_ready; struct vmbus_gpa txd_sgl[HVN_TX_FRAGS + 1]; int txd_nsge; struct mbuf *txd_buf; bus_dmamap_t txd_dmap; struct vmbus_gpa txd_gpa; struct rndis_packet_msg *txd_req; + TAILQ_ENTRY(hvn_tx_desc) txd_entry; + u_int txd_refs; + uint32_t txd_flags; +#define HVN_TXD_FLAG_ONAGG __BIT(0) +#define HVN_TXD_FLAG_DMAMAP __BIT(1) + uint32_t txd_chim_index; + int txd_chim_size; + STAILQ_ENTRY(hvn_tx_desc) txd_agg_entry; + STAILQ_HEAD(, hvn_tx_desc) txd_agg_list; }; +struct hvn_softc; +struct hvn_rx_ring; + +struct hvn_tx_ring { + struct hvn_softc *txr_softc; + struct vmbus_channel *txr_chan; + struct hvn_rx_ring *txr_rxr; + void *txr_si; + char txr_name[16]; + + int txr_id; + int txr_oactive; + int txr_suspended; + int txr_csum_assist; + uint64_t txr_caps_assist; + uint32_t txr_flags; +#define HVN_TXR_FLAG_UDP_HASH __BIT(0) + + struct evcnt txr_evpkts; + struct evcnt txr_evsends; + struct evcnt txr_evnodesc; + struct evcnt txr_evdmafailed; + struct evcnt txr_evdefrag; + struct evcnt txr_evpcqdrop; + struct evcnt txr_evtransmitdefer; + struct evcnt txr_evflushfailed; + struct evcnt txr_evchimneytried; + struct evcnt txr_evchimney; + struct evcnt txr_evvlanfixup; + struct evcnt txr_evvlanhwtagging; + struct evcnt txr_evvlantap; + + kmutex_t txr_lock; + pcq_t *txr_interq; + + uint32_t txr_avail; + TAILQ_HEAD(, hvn_tx_desc) txr_list; + struct hvn_tx_desc txr_desc[HVN_TX_DESC]; + uint8_t *txr_msgs; + struct hyperv_dma txr_dma; + + int txr_chim_size; + + /* Applied packet transmission aggregation limits. */ + int txr_agg_szmax; + short txr_agg_pktmax; + short txr_agg_align; + + /* Packet transmission aggregation states. */ + struct hvn_tx_desc *txr_agg_txd; + int txr_agg_szleft; + short txr_agg_pktleft; + struct rndis_packet_msg *txr_agg_prevpkt; + + /* Temporary stats for each sends. */ + int txr_stat_pkts; + int txr_stat_size; + int txr_stat_mcasts; + + int (*txr_sendpkt)(struct hvn_tx_ring *, + struct hvn_tx_desc *); +} __aligned(CACHE_LINE_SIZE); + +struct hvn_rx_ring { + struct hvn_softc *rxr_softc; + struct vmbus_channel *rxr_chan; + struct hvn_tx_ring *rxr_txr; + void *rxr_si; + bool rxr_workqueue; + char rxr_name[16]; + + struct work rxr_wk; + volatile bool rxr_onlist; + volatile bool rxr_onproc; + kmutex_t rxr_onwork_lock; + kcondvar_t rxr_onwork_cv; + + uint32_t rxr_flags; +#define HVN_RXR_FLAG_UDP_HASH __BIT(0) + + kmutex_t rxr_lock; + + struct evcnt rxr_evpkts; + struct evcnt rxr_evcsum_ip; + struct evcnt rxr_evcsum_tcp; + struct evcnt rxr_evcsum_udp; + struct evcnt rxr_evvlanhwtagging; + struct evcnt rxr_evintr; + struct evcnt rxr_evdefer; + struct evcnt rxr_evdeferreq; + struct evcnt rxr_evredeferreq; + + /* NVS */ + uint8_t *rxr_nvsbuf; +} __aligned(CACHE_LINE_SIZE); + struct hvn_softc { device_t sc_dev; struct vmbus_softc *sc_vmbus; - struct vmbus_channel *sc_chan; + struct vmbus_channel *sc_prichan; bus_dma_tag_t sc_dmat; struct ethercom sc_ec; struct ifmedia sc_media; - kmutex_t sc_media_lock; /* XXX */ struct if_percpuq *sc_ipq; + struct workqueue *sc_wq; + bool sc_txrx_workqueue; + kmutex_t sc_core_lock; + + kmutex_t sc_link_lock; + kcondvar_t sc_link_cv; + callout_t sc_link_tmout; + lwp_t *sc_link_lwp; + uint32_t sc_link_ev; +#define HVN_LINK_EV_STATE_CHANGE __BIT(0) +#define HVN_LINK_EV_NETWORK_CHANGE_TMOUT __BIT(1) +#define HVN_LINK_EV_NETWORK_CHANGE __BIT(2) +#define HVN_LINK_EV_RESUME_NETWORK __BIT(3) +#define HVN_LINK_EV_EXIT_THREAD __BIT(4) int sc_link_state; - int sc_promisc; + bool sc_link_onproc; + bool sc_link_pending; + bool sc_link_suspend; + + int sc_tx_process_limit; + int sc_rx_process_limit; + int sc_tx_intr_process_limit; + int sc_rx_intr_process_limit; + + struct sysctllog *sc_sysctllog; + + uint32_t sc_caps; +#define HVN_CAPS_VLAN __BIT(0) +#define HVN_CAPS_MTU __BIT(1) +#define HVN_CAPS_IPCS __BIT(2) +#define HVN_CAPS_TCP4CS __BIT(3) +#define HVN_CAPS_TCP6CS __BIT(4) +#define HVN_CAPS_UDP4CS __BIT(5) +#define HVN_CAPS_UDP6CS __BIT(6) +#define HVN_CAPS_TSO4 __BIT(7) +#define HVN_CAPS_TSO6 __BIT(8) +#define HVN_CAPS_HASHVAL __BIT(9) +#define HVN_CAPS_UDPHASH __BIT(10) uint32_t sc_flags; -#define HVN_SCF_ATTACHED __BIT(0) +#define HVN_SCF_ATTACHED __BIT(0) +#define HVN_SCF_RXBUF_CONNECTED __BIT(1) +#define HVN_SCF_CHIM_CONNECTED __BIT(2) +#define HVN_SCF_REVOKED __BIT(3) +#define HVN_SCF_HAS_RSSKEY __BIT(4) +#define HVN_SCF_HAS_RSSIND __BIT(5) /* NVS protocol */ int sc_proto; uint32_t sc_nvstid; uint8_t sc_nvsrsp[HVN_NVS_MSGSIZE]; - uint8_t *sc_nvsbuf; int sc_nvsdone; + kmutex_t sc_nvsrsp_lock; + kcondvar_t sc_nvsrsp_cv; /* RNDIS protocol */ int sc_ndisver; uint32_t sc_rndisrid; + int sc_tso_szmax; + int sc_tso_sgmin; + uint32_t sc_rndis_agg_size; + uint32_t sc_rndis_agg_pkts; + uint32_t sc_rndis_agg_align; struct rndis_queue sc_cntl_sq; /* submission queue */ kmutex_t sc_cntl_sqlck; struct rndis_queue sc_cntl_cq; /* completion queue */ kmutex_t sc_cntl_cqlck; struct rndis_queue sc_cntl_fq; /* free queue */ kmutex_t sc_cntl_fqlck; + kcondvar_t sc_cntl_fqcv; struct rndis_cmd sc_cntl_msgs[HVN_RNDIS_CTLREQS]; struct hvn_nvs_rndis sc_data_msg; + int sc_rss_ind_size; + uint32_t sc_rss_hash; /* setting, NDIS_HASH_ */ + uint32_t sc_rss_hcap; /* caps, NDIS_HASH_ */ + struct ndis_rssprm_toeplitz sc_rss; + /* Rx ring */ uint8_t *sc_rx_ring; int sc_rx_size; uint32_t sc_rx_hndl; struct hyperv_dma sc_rx_dma; + struct hvn_rx_ring *sc_rxr; + int sc_nrxr; + int sc_nrxr_inuse; /* Tx ring */ - uint32_t sc_tx_next; - uint32_t sc_tx_avail; - struct hvn_tx_desc sc_tx_desc[HVN_TX_DESC]; - bus_dmamap_t sc_tx_rmap; - uint8_t *sc_tx_msgs; - bus_dma_segment_t sc_tx_mseg; + struct hvn_tx_ring *sc_txr; + int sc_ntxr; + int sc_ntxr_inuse; + + /* chimney sending buffers */ + uint8_t *sc_chim; + uint32_t sc_chim_hndl; + struct hyperv_dma sc_chim_dma; + kmutex_t sc_chim_bmap_lock; + u_long *sc_chim_bmap; + int sc_chim_bmap_cnt; + int sc_chim_cnt; + int sc_chim_szmax; + + /* Packet transmission aggregation user settings. */ + int sc_agg_size; + int sc_agg_pkts; }; #define SC2IFP(_sc_) (&(_sc_)->sc_ec.ec_if) #define IFP2SC(_ifp_) ((_ifp_)->if_softc) +#ifndef HVN_TX_PROCESS_LIMIT_DEFAULT +#define HVN_TX_PROCESS_LIMIT_DEFAULT 128 +#endif +#ifndef HVN_RX_PROCESS_LIMIT_DEFAULT +#define HVN_RX_PROCESS_LIMIT_DEFAULT 128 +#endif +#ifndef HVN_TX_INTR_PROCESS_LIMIT_DEFAULT +#define HVN_TX_INTR_PROCESS_LIMIT_DEFAULT 256 +#endif +#ifndef HVN_RX_INTR_PROCESS_LIMIT_DEFAULT +#define HVN_RX_INTR_PROCESS_LIMIT_DEFAULT 256 +#endif + +/* + * See hvn_set_hlen(). + * + * This value is for Azure. For Hyper-V, set this above + * 65536 to disable UDP datagram checksum fixup. + */ +#ifndef HVN_UDP_CKSUM_FIXUP_MTU_DEFAULT +#define HVN_UDP_CKSUM_FIXUP_MTU_DEFAULT 1420 +#endif +static int hvn_udpcs_fixup_mtu = HVN_UDP_CKSUM_FIXUP_MTU_DEFAULT; + +/* Limit chimney send size */ +static int hvn_tx_chimney_size = 0; + +/* # of channels to use; each channel has one RX ring and one TX ring */ +#ifndef HVN_CHANNEL_COUNT_DEFAULT +#define HVN_CHANNEL_COUNT_DEFAULT 0 +#endif +static int hvn_channel_cnt = HVN_CHANNEL_COUNT_DEFAULT; + +/* # of transmit rings to use */ +#ifndef HVN_TX_RING_COUNT_DEFAULT +#define HVN_TX_RING_COUNT_DEFAULT 0 +#endif +static int hvn_tx_ring_cnt = HVN_TX_RING_COUNT_DEFAULT; + +/* Packet transmission aggregation size limit */ +static int hvn_tx_agg_size = -1; + +/* Packet transmission aggregation count limit */ +static int hvn_tx_agg_pkts = -1; static int hvn_match(device_t, cfdata_t, void *); static void hvn_attach(device_t, device_t, void *); @@ -189,38 +458,103 @@ CFATTACH_DECL_NEW(hvn, sizeof(struct hvn_softc), static int hvn_ioctl(struct ifnet *, u_long, void *); static int hvn_media_change(struct ifnet *); static void hvn_media_status(struct ifnet *, struct ifmediareq *); -static int hvn_iff(struct hvn_softc *); +static void hvn_link_task(void *); +static void hvn_link_event(struct hvn_softc *, uint32_t); +static void hvn_link_netchg_tmout_cb(void *); static int hvn_init(struct ifnet *); +static int hvn_init_locked(struct ifnet *); static void hvn_stop(struct ifnet *, int); +static void hvn_stop_locked(struct ifnet *); static void hvn_start(struct ifnet *); -static int hvn_encap(struct hvn_softc *, struct mbuf *, - struct hvn_tx_desc **); -static void hvn_decap(struct hvn_softc *, struct hvn_tx_desc *); -static void hvn_txeof(struct hvn_softc *, uint64_t); -static int hvn_rx_ring_create(struct hvn_softc *); +static int hvn_transmit(struct ifnet *, struct mbuf *); +static void hvn_deferred_transmit(void *); +static int hvn_flush_txagg(struct hvn_tx_ring *); +static int hvn_encap(struct hvn_tx_ring *, struct hvn_tx_desc *, + struct mbuf *, int); +static int hvn_txpkt(struct hvn_tx_ring *, struct hvn_tx_desc *); +static void hvn_txeof(struct hvn_tx_ring *, uint64_t); +static int hvn_rx_ring_create(struct hvn_softc *, int); static int hvn_rx_ring_destroy(struct hvn_softc *); -static int hvn_tx_ring_create(struct hvn_softc *); +static void hvn_fixup_rx_data(struct hvn_softc *); +static int hvn_tx_ring_create(struct hvn_softc *, int); static void hvn_tx_ring_destroy(struct hvn_softc *); -static int hvn_set_capabilities(struct hvn_softc *); +static void hvn_set_chim_size(struct hvn_softc *, int); +static uint32_t hvn_chim_alloc(struct hvn_softc *); +static void hvn_chim_free(struct hvn_softc *, uint32_t); +static void hvn_fixup_tx_data(struct hvn_softc *); +static struct mbuf * + hvn_set_hlen(struct mbuf *, int *); +static int hvn_txd_peek(struct hvn_tx_ring *); +static struct hvn_tx_desc * + hvn_txd_get(struct hvn_tx_ring *); +static void hvn_txd_put(struct hvn_tx_ring *, struct hvn_tx_desc *); +static void hvn_txd_gc(struct hvn_tx_ring *, struct hvn_tx_desc *); +static void hvn_txd_hold(struct hvn_tx_desc *); +static void hvn_txd_agg(struct hvn_tx_desc *, struct hvn_tx_desc *); +static int hvn_tx_ring_pending(struct hvn_tx_ring *); +static void hvn_tx_ring_qflush(struct hvn_softc *, struct hvn_tx_ring *); +static int hvn_get_rsscaps(struct hvn_softc *, int *); +static int hvn_set_rss(struct hvn_softc *, uint16_t); +static void hvn_fixup_rss_ind(struct hvn_softc *); +static int hvn_get_hwcaps(struct hvn_softc *, struct ndis_offload *); +static int hvn_set_capabilities(struct hvn_softc *, int); static int hvn_get_lladdr(struct hvn_softc *, uint8_t *); -static void hvn_get_link_status(struct hvn_softc *); +static void hvn_update_link_status(struct hvn_softc *); +static int hvn_get_mtu(struct hvn_softc *, uint32_t *); +static int hvn_channel_attach(struct hvn_softc *, struct vmbus_channel *); +static void hvn_channel_detach(struct hvn_softc *, struct vmbus_channel *); +static void hvn_channel_detach_all(struct hvn_softc *); +static int hvn_subchannel_attach(struct hvn_softc *); +static int hvn_synth_alloc_subchannels(struct hvn_softc *, int *); +static int hvn_synth_attachable(const struct hvn_softc *); +static int hvn_synth_attach(struct hvn_softc *, int); +static void hvn_synth_detach(struct hvn_softc *); +static void hvn_set_ring_inuse(struct hvn_softc *, int); +static void hvn_disable_rx(struct hvn_softc *); +static void hvn_drain_rxtx(struct hvn_softc *, int ); +static void hvn_suspend_data(struct hvn_softc *); +static void hvn_suspend_mgmt(struct hvn_softc *); +static void hvn_suspend(struct hvn_softc *) __unused; +static void hvn_resume_tx(struct hvn_softc *, int); +static void hvn_resume_data(struct hvn_softc *); +static void hvn_resume_mgmt(struct hvn_softc *); +static void hvn_resume(struct hvn_softc *) __unused; +static void hvn_init_sysctls(struct hvn_softc *); /* NSVP */ -static int hvn_nvs_attach(struct hvn_softc *); +static int hvn_nvs_init(struct hvn_softc *); +static void hvn_nvs_destroy(struct hvn_softc *); +static int hvn_nvs_attach(struct hvn_softc *, int); +static int hvn_nvs_connect_rxbuf(struct hvn_softc *); +static int hvn_nvs_disconnect_rxbuf(struct hvn_softc *); +static int hvn_nvs_connect_chim(struct hvn_softc *); +static int hvn_nvs_disconnect_chim(struct hvn_softc *); +static void hvn_handle_ring_work(struct work *, void *); +static void hvn_nvs_softintr(void *); static void hvn_nvs_intr(void *); -static int hvn_nvs_cmd(struct hvn_softc *, void *, size_t, uint64_t, int); -static int hvn_nvs_ack(struct hvn_softc *, uint64_t); +static void hvn_nvs_intr1(struct hvn_rx_ring *, int, int); +static int hvn_nvs_cmd(struct hvn_softc *, void *, size_t, uint64_t, + u_int); +static int hvn_nvs_ack(struct hvn_rx_ring *, uint64_t); static void hvn_nvs_detach(struct hvn_softc *); +static int hvn_nvs_alloc_subchannels(struct hvn_softc *, int *); /* RNDIS */ -static int hvn_rndis_attach(struct hvn_softc *); -static int hvn_rndis_cmd(struct hvn_softc *, struct rndis_cmd *, int); -static void hvn_rndis_input(struct hvn_softc *, uint64_t, void *); -static void hvn_rxeof(struct hvn_softc *, uint8_t *, uint32_t); +static int hvn_rndis_init(struct hvn_softc *); +static void hvn_rndis_destroy(struct hvn_softc *); +static int hvn_rndis_attach(struct hvn_softc *, int); +static int hvn_rndis_cmd(struct hvn_softc *, struct rndis_cmd *, u_int); +static int hvn_rndis_input(struct hvn_rx_ring *, uint64_t, void *); +static int hvn_rxeof(struct hvn_rx_ring *, uint8_t *, uint32_t); static void hvn_rndis_complete(struct hvn_softc *, uint8_t *, uint32_t); -static int hvn_rndis_output(struct hvn_softc *, struct hvn_tx_desc *); +static int hvn_rndis_output_sgl(struct hvn_tx_ring *, + struct hvn_tx_desc *); +static int hvn_rndis_output_chim(struct hvn_tx_ring *, + struct hvn_tx_desc *); static void hvn_rndis_status(struct hvn_softc *, uint8_t *, uint32_t); static int hvn_rndis_query(struct hvn_softc *, uint32_t, void *, size_t *); +static int hvn_rndis_query2(struct hvn_softc *, uint32_t, const void *, + size_t, void *, size_t *, size_t); static int hvn_rndis_set(struct hvn_softc *, uint32_t, void *, size_t); static int hvn_rndis_open(struct hvn_softc *); static int hvn_rndis_close(struct hvn_softc *); @@ -242,93 +576,155 @@ hvn_attach(device_t parent, device_t self, void *aux) struct hvn_softc *sc = device_private(self); struct vmbus_attach_args *aa = aux; struct ifnet *ifp = SC2IFP(sc); + char xnamebuf[32]; uint8_t enaddr[ETHER_ADDR_LEN]; + uint32_t mtu; + int tx_ring_cnt, ring_cnt; + int error; sc->sc_dev = self; sc->sc_vmbus = (struct vmbus_softc *)device_private(parent); - sc->sc_chan = aa->aa_chan; + sc->sc_prichan = aa->aa_chan; sc->sc_dmat = sc->sc_vmbus->sc_dmat; aprint_naive("\n"); aprint_normal(": Hyper-V NetVSC\n"); - if (hvn_nvs_attach(sc)) { - aprint_error_dev(self, "failed to init NVSP\n"); + sc->sc_txrx_workqueue = true; + sc->sc_tx_process_limit = HVN_TX_PROCESS_LIMIT_DEFAULT; + sc->sc_rx_process_limit = HVN_RX_PROCESS_LIMIT_DEFAULT; + sc->sc_tx_intr_process_limit = HVN_TX_INTR_PROCESS_LIMIT_DEFAULT; + sc->sc_rx_intr_process_limit = HVN_RX_INTR_PROCESS_LIMIT_DEFAULT; + sc->sc_agg_size = hvn_tx_agg_size; + sc->sc_agg_pkts = hvn_tx_agg_pkts; + + mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_SOFTNET); + mutex_init(&sc->sc_link_lock, MUTEX_DEFAULT, IPL_NET); + cv_init(&sc->sc_link_cv, "hvnknkcv"); + callout_init(&sc->sc_link_tmout, CALLOUT_MPSAFE); + callout_setfunc(&sc->sc_link_tmout, hvn_link_netchg_tmout_cb, sc); + if (kthread_create(PRI_NONE, KTHREAD_MUSTJOIN | KTHREAD_MPSAFE, NULL, + hvn_link_task, sc, &sc->sc_link_lwp, "%slink", + device_xname(self))) { + aprint_error_dev(self, "failed to create link thread\n"); return; } - if (hvn_rx_ring_create(sc)) { - aprint_error_dev(self, "failed to create Rx ring\n"); - goto fail1; + snprintf(xnamebuf, sizeof(xnamebuf), "%srxtx", device_xname(self)); + if (workqueue_create(&sc->sc_wq, xnamebuf, hvn_handle_ring_work, + sc, HVN_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE)) { + aprint_error_dev(self, "failed to create workqueue\n"); + sc->sc_wq = NULL; + goto destroy_link_thread; } - if (hvn_tx_ring_create(sc)) { + ring_cnt = hvn_channel_cnt; + if (ring_cnt <= 0) { + ring_cnt = ncpu; + if (ring_cnt > HVN_CHANNEL_MAX_COUNT_DEFAULT) + ring_cnt = HVN_CHANNEL_MAX_COUNT_DEFAULT; + } else if (ring_cnt > ncpu) + ring_cnt = ncpu; + + tx_ring_cnt = hvn_tx_ring_cnt; + if (tx_ring_cnt <= 0 || tx_ring_cnt > ring_cnt) + tx_ring_cnt = ring_cnt; + + if (hvn_tx_ring_create(sc, tx_ring_cnt)) { aprint_error_dev(self, "failed to create Tx ring\n"); - goto fail2; + goto destroy_wq; + } + + if (hvn_rx_ring_create(sc, ring_cnt)) { + aprint_error_dev(self, "failed to create Rx ring\n"); + goto destroy_tx_ring; } strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ); ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_extflags = IFEF_MPSAFE; ifp->if_ioctl = hvn_ioctl; ifp->if_start = hvn_start; + ifp->if_transmit = hvn_transmit; ifp->if_init = hvn_init; ifp->if_stop = hvn_stop; - ifp->if_capabilities = IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx; - ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx; - ifp->if_capabilities |= IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx; - if (sc->sc_ndisver > NDIS_VERSION_6_30) { - ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx; - ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx; - ifp->if_capabilities |= IFCAP_CSUM_UDPv6_Tx; - ifp->if_capabilities |= IFCAP_CSUM_UDPv6_Rx; - } - if (sc->sc_proto >= HVN_NVS_PROTO_VERSION_2) { - sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; - sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; - sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING; - } + ifp->if_baudrate = IF_Gbps(10); - IFQ_SET_MAXLEN(&ifp->if_snd, HVN_TX_DESC - 1); + IFQ_SET_MAXLEN(&ifp->if_snd, uimax(HVN_TX_DESC - 1, IFQ_MAXLEN)); IFQ_SET_READY(&ifp->if_snd); /* Initialize ifmedia structures. */ sc->sc_ec.ec_ifmedia = &sc->sc_media; - /* XXX media locking needs revisiting */ - mutex_init(&sc->sc_media_lock, MUTEX_DEFAULT, IPL_SOFTNET); ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, - hvn_media_change, hvn_media_status, &sc->sc_media_lock); - ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_MANUAL, 0, NULL); - ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_MANUAL); + hvn_media_change, hvn_media_status, &sc->sc_core_lock); + ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); + ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_10G_T, 0, NULL); + ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); if_initialize(ifp); sc->sc_ipq = if_percpuq_create(ifp); if_deferred_start_init(ifp, NULL); - if (hvn_rndis_attach(sc)) { - aprint_error_dev(self, "failed to init RNDIS\n"); - goto fail3; + hvn_nvs_init(sc); + hvn_rndis_init(sc); + if (hvn_synth_attach(sc, ETHERMTU)) { + aprint_error_dev(self, "failed to attach synth\n"); + goto destroy_if_percpuq; } aprint_normal_dev(self, "NVS %d.%d NDIS %d.%d\n", sc->sc_proto >> 16, sc->sc_proto & 0xffff, sc->sc_ndisver >> 16 , sc->sc_ndisver & 0xffff); - if (hvn_set_capabilities(sc)) { - aprint_error_dev(self, "failed to setup offloading\n"); - goto fail4; - } - if (hvn_get_lladdr(sc, enaddr)) { aprint_error_dev(self, "failed to obtain an ethernet address\n"); - goto fail4; + goto detach_synth; } aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(enaddr)); + /* + * Fixup TX/RX stuffs after synthetic parts are attached. + */ + hvn_fixup_tx_data(sc); + hvn_fixup_rx_data(sc); + + ifp->if_capabilities |= sc->sc_txr[0].txr_caps_assist & + (IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | + IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | + IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | + IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | + IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx); + /* XXX TSOv4, TSOv6 */ + if (sc->sc_caps & HVN_CAPS_VLAN) { + /* XXX not sure about VLAN_MTU. */ + sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; + sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; + } + sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; + ether_ifattach(ifp, enaddr); + + error = hvn_get_mtu(sc, &mtu); + if (error) + mtu = ETHERMTU; + if (mtu < ETHERMTU) { + DPRINTF("%s: fixup mtu %u -> %u\n", device_xname(sc->sc_dev), + ETHERMTU, mtu); + ifp->if_mtu = mtu; + } + if_register(ifp); + /* + * Kick off link status check. + */ + hvn_link_event(sc, HVN_LINK_EV_STATE_CHANGE); + + hvn_init_sysctls(sc); + if (pmf_device_register(self, NULL, NULL)) pmf_class_network_register(self, ifp); else @@ -337,13 +733,25 @@ hvn_attach(device_t parent, device_t self, void *aux) SET(sc->sc_flags, HVN_SCF_ATTACHED); return; -fail4: hvn_rndis_detach(sc); +detach_synth: + hvn_synth_detach(sc); + hvn_rndis_destroy(sc); + hvn_nvs_destroy(sc); +destroy_if_percpuq: if_percpuq_destroy(sc->sc_ipq); -fail3: ifmedia_fini(&sc->sc_media); - mutex_destroy(&sc->sc_media_lock); + hvn_rx_ring_destroy(sc); +destroy_tx_ring: hvn_tx_ring_destroy(sc); -fail2: hvn_rx_ring_destroy(sc); -fail1: hvn_nvs_detach(sc); +destroy_wq: + workqueue_destroy(sc->sc_wq); + sc->sc_wq = NULL; +destroy_link_thread: + hvn_link_event(sc, HVN_LINK_EV_EXIT_THREAD); + kthread_join(sc->sc_link_lwp); + callout_destroy(&sc->sc_link_tmout); + cv_destroy(&sc->sc_link_cv); + mutex_destroy(&sc->sc_link_lock); + mutex_destroy(&sc->sc_core_lock); } static int @@ -355,21 +763,46 @@ hvn_detach(device_t self, int flags) if (!ISSET(sc->sc_flags, HVN_SCF_ATTACHED)) return 0; - if (ifp->if_flags & IFF_RUNNING) - hvn_stop(ifp, 1); + if (vmbus_channel_is_revoked(sc->sc_prichan)) + SET(sc->sc_flags, HVN_SCF_REVOKED); pmf_device_deregister(self); + mutex_enter(&sc->sc_core_lock); + + if (ifp->if_flags & IFF_RUNNING) + hvn_stop_locked(ifp); + /* + * NOTE: + * hvn_stop() only suspends data, so managment + * stuffs have to be suspended manually here. + */ + hvn_suspend_mgmt(sc); + ether_ifdetach(ifp); if_detach(ifp); - ifmedia_fini(&sc->sc_media); - mutex_destroy(&sc->sc_media_lock); if_percpuq_destroy(sc->sc_ipq); - hvn_rndis_detach(sc); + hvn_link_event(sc, HVN_LINK_EV_EXIT_THREAD); + kthread_join(sc->sc_link_lwp); + callout_halt(&sc->sc_link_tmout, NULL); + + hvn_synth_detach(sc); + hvn_rndis_destroy(sc); + hvn_nvs_destroy(sc); + + mutex_exit(&sc->sc_core_lock); + hvn_rx_ring_destroy(sc); hvn_tx_ring_destroy(sc); - hvn_nvs_detach(sc); + + workqueue_destroy(sc->sc_wq); + callout_destroy(&sc->sc_link_tmout); + cv_destroy(&sc->sc_link_cv); + mutex_destroy(&sc->sc_link_lock); + mutex_destroy(&sc->sc_core_lock); + + sysctl_teardown(&sc->sc_sysctllog); return 0; } @@ -378,18 +811,99 @@ static int hvn_ioctl(struct ifnet *ifp, u_long command, void * data) { struct hvn_softc *sc = IFP2SC(ifp); + struct ifreq *ifr = (struct ifreq *)data; + uint32_t mtu; int s, error = 0; - s = splnet(); + switch (command) { + case SIOCSIFMTU: + if (ifr->ifr_mtu < HVN_MTU_MIN || ifr->ifr_mtu > HVN_MTU_MAX) { + error = EINVAL; + break; + } - error = ether_ioctl(ifp, command, data); - if (error == ENETRESET) { - if (ifp->if_flags & IFF_RUNNING) - hvn_iff(sc); - error = 0; - } + mutex_enter(&sc->sc_core_lock); - splx(s); + if (!(sc->sc_caps & HVN_CAPS_MTU)) { + /* Can't change MTU */ + mutex_exit(&sc->sc_core_lock); + error = EOPNOTSUPP; + break; + } + + if (ifp->if_mtu == ifr->ifr_mtu) { + mutex_exit(&sc->sc_core_lock); + break; + } + + /* + * Suspend this interface before the synthetic parts + * are ripped. + */ + hvn_suspend(sc); + + /* + * Detach the synthetics parts, i.e. NVS and RNDIS. + */ + hvn_synth_detach(sc); + + /* + * Reattach the synthetic parts, i.e. NVS and RNDIS, + * with the new MTU setting. + */ + error = hvn_synth_attach(sc, ifr->ifr_mtu); + if (error) { + mutex_exit(&sc->sc_core_lock); + break; + } + + error = hvn_get_mtu(sc, &mtu); + if (error) + mtu = ifr->ifr_mtu; + DPRINTF("%s: RNDIS mtu=%d\n", device_xname(sc->sc_dev), mtu); + + /* + * Commit the requested MTU, after the synthetic parts + * have been successfully attached. + */ + if (mtu >= ifr->ifr_mtu) { + mtu = ifr->ifr_mtu; + } else { + DPRINTF("%s: fixup mtu %d -> %u\n", + device_xname(sc->sc_dev), ifr->ifr_mtu, mtu); + } + ifp->if_mtu = mtu; + + /* + * Synthetic parts' reattach may change the chimney + * sending size; update it. + */ + if (sc->sc_txr[0].txr_chim_size > sc->sc_chim_szmax) + hvn_set_chim_size(sc, sc->sc_chim_szmax); + + /* + * All done! Resume the interface now. + */ + hvn_resume(sc); + + mutex_exit(&sc->sc_core_lock); + break; + default: + s = splnet(); + if (command == SIOCGIFMEDIA || command == SIOCSIFMEDIA) + error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, command); + else + error = ether_ioctl(ifp, command, data); + splx(s); + if (error == ENETRESET) { + mutex_enter(&sc->sc_core_lock); + if (ifp->if_flags & IFF_RUNNING) + hvn_init_locked(ifp); + mutex_exit(&sc->sc_core_lock); + error = 0; + } + break; + } return error; } @@ -397,7 +911,19 @@ hvn_ioctl(struct ifnet *ifp, u_long command, void * data) static int hvn_media_change(struct ifnet *ifp) { + struct hvn_softc *sc = IFP2SC(ifp); + struct ifmedia *ifm = &sc->sc_media; + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return EINVAL; + + switch (IFM_SUBTYPE(ifm->ifm_media)) { + case IFM_AUTO: + break; + default: + device_printf(sc->sc_dev, "Only auto media type\n"); + return EINVAL; + } return 0; } @@ -405,27 +931,109 @@ static void hvn_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct hvn_softc *sc = IFP2SC(ifp); - int link_state; - - link_state = sc->sc_link_state; - hvn_get_link_status(sc); - if (link_state != sc->sc_link_state) - if_link_state_change(ifp, sc->sc_link_state); ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER | IFM_MANUAL; - if (sc->sc_link_state == LINK_STATE_UP) - ifmr->ifm_status |= IFM_ACTIVE; + ifmr->ifm_active = IFM_ETHER; + + if (sc->sc_link_state != LINK_STATE_UP) { + ifmr->ifm_active |= IFM_NONE; + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + ifmr->ifm_active |= IFM_10G_T | IFM_FDX; } -static int -hvn_iff(struct hvn_softc *sc) +static void +hvn_link_task(void *arg) { + struct hvn_softc *sc = arg; + struct ifnet *ifp = SC2IFP(sc); + uint32_t event; + int old_link_state; - /* XXX */ - sc->sc_promisc = 0; + mutex_enter(&sc->sc_link_lock); + sc->sc_link_onproc = false; + for (;;) { + if (sc->sc_link_ev == 0) { + cv_wait(&sc->sc_link_cv, &sc->sc_link_lock); + continue; + } - return 0; + sc->sc_link_onproc = true; + event = sc->sc_link_ev; + sc->sc_link_ev = 0; + mutex_exit(&sc->sc_link_lock); + + if (event & HVN_LINK_EV_EXIT_THREAD) + break; + + if (sc->sc_link_suspend) + goto next; + + if (event & HVN_LINK_EV_RESUME_NETWORK) { + if (sc->sc_link_pending) + event |= HVN_LINK_EV_NETWORK_CHANGE; + else + event |= HVN_LINK_EV_STATE_CHANGE; + } + + if (event & HVN_LINK_EV_NETWORK_CHANGE) { + /* Prevent any link status checks from running. */ + sc->sc_link_pending = true; + + /* + * Fake up a [link down --> link up] state change; + * 5 seconds delay is used, which closely simulates + * miibus reaction upon link down event. + */ + old_link_state = sc->sc_link_state; + sc->sc_link_state = LINK_STATE_DOWN; + if (old_link_state != sc->sc_link_state) { + if_link_state_change(ifp, LINK_STATE_DOWN); + } +#if defined(HVN_LINK_STATE_CHANGE_DELAY) && HVN_LINK_STATE_CHANGE_DELAY > 0 + callout_schedule(&sc->sc_link_tmout, + mstohz(HVN_LINK_STATE_CHANGE_DELAY)); +#else + hvn_link_event(sc, HVN_LINK_EV_NETWORK_CHANGE_TMOUT); +#endif + } else if (event & HVN_LINK_EV_NETWORK_CHANGE_TMOUT) { + /* Re-allow link status checks. */ + sc->sc_link_pending = false; + hvn_update_link_status(sc); + } else if (event & HVN_LINK_EV_STATE_CHANGE) { + if (!sc->sc_link_pending) + hvn_update_link_status(sc); + } + next: + mutex_enter(&sc->sc_link_lock); + sc->sc_link_onproc = false; + } + + mutex_enter(&sc->sc_link_lock); + sc->sc_link_onproc = false; + mutex_exit(&sc->sc_link_lock); + + kthread_exit(0); +} + +static void +hvn_link_event(struct hvn_softc *sc, uint32_t ev) +{ + + mutex_enter(&sc->sc_link_lock); + SET(sc->sc_link_ev, ev); + cv_signal(&sc->sc_link_cv); + mutex_exit(&sc->sc_link_lock); +} + +static void +hvn_link_netchg_tmout_cb(void *arg) +{ + struct hvn_softc *sc = arg; + + hvn_link_event(sc, HVN_LINK_EV_NETWORK_CHANGE_TMOUT); } static int @@ -434,18 +1042,37 @@ hvn_init(struct ifnet *ifp) struct hvn_softc *sc = IFP2SC(ifp); int error; - hvn_stop(ifp, 0); + mutex_enter(&sc->sc_core_lock); + error = hvn_init_locked(ifp); + mutex_exit(&sc->sc_core_lock); + + return error; +} + +static int +hvn_init_locked(struct ifnet *ifp) +{ + struct hvn_softc *sc = IFP2SC(ifp); + int error; - error = hvn_iff(sc); + KASSERT(mutex_owned(&sc->sc_core_lock)); + + hvn_stop_locked(ifp); + + error = hvn_rndis_open(sc); if (error) return error; - error = hvn_rndis_open(sc); - if (error == 0) { - ifp->if_flags |= IFF_RUNNING; - ifp->if_flags &= ~IFF_OACTIVE; - } - return error; + /* Clear OACTIVE bit. */ + ifp->if_flags &= ~IFF_OACTIVE; + + /* Clear TX 'suspended' bit. */ + hvn_resume_tx(sc, sc->sc_ntxr_inuse); + + /* Everything is ready; unleash! */ + ifp->if_flags |= IFF_RUNNING; + + return 0; } static void @@ -453,50 +1080,162 @@ hvn_stop(struct ifnet *ifp, int disable) { struct hvn_softc *sc = IFP2SC(ifp); - hvn_rndis_close(sc); - - ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); + mutex_enter(&sc->sc_core_lock); + hvn_stop_locked(ifp); + mutex_exit(&sc->sc_core_lock); } static void -hvn_start(struct ifnet *ifp) +hvn_stop_locked(struct ifnet *ifp) { struct hvn_softc *sc = IFP2SC(ifp); + int i; + + KASSERT(mutex_owned(&sc->sc_core_lock)); + + /* Clear RUNNING bit ASAP. */ + ifp->if_flags &= ~IFF_RUNNING; + + /* Suspend data transfers. */ + hvn_suspend_data(sc); + + /* Clear OACTIVE bit. */ + ifp->if_flags &= ~IFF_OACTIVE; + for (i = 0; i < sc->sc_ntxr_inuse; i++) + sc->sc_txr[i].txr_oactive = 0; +} + +static void +hvn_transmit_common(struct ifnet *ifp, struct hvn_tx_ring *txr, + bool is_transmit) +{ struct hvn_tx_desc *txd; struct mbuf *m; + int l2hlen = ETHER_HDR_LEN; - if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) + KASSERT(mutex_owned(&txr->txr_lock)); + + if (!(ifp->if_flags & IFF_RUNNING)) + return; + if (!is_transmit && (ifp->if_flags & IFF_OACTIVE)) + return; + if (txr->txr_oactive) + return; + if (txr->txr_suspended) return; for (;;) { - if (!sc->sc_tx_avail) { + if (!hvn_txd_peek(txr)) { /* transient */ - ifp->if_flags |= IFF_OACTIVE; + if (!is_transmit) + ifp->if_flags |= IFF_OACTIVE; + txr->txr_oactive = 1; + txr->txr_evnodesc.ev_count++; break; } - IFQ_DEQUEUE(&ifp->if_snd, m); + if (is_transmit) + m = pcq_get(txr->txr_interq); + else + IFQ_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; - if (hvn_encap(sc, m, &txd)) { +#if defined(INET) || defined(INET6) + if (m->m_pkthdr.csum_flags & + (M_CSUM_TCPv4|M_CSUM_UDPv4|M_CSUM_TCPv6|M_CSUM_UDPv6)) { + m = hvn_set_hlen(m, &l2hlen); + if (__predict_false(m == NULL)) { + if_statinc(ifp, if_oerrors); + continue; + } + } +#endif + + txd = hvn_txd_get(txr); + if (hvn_encap(txr, txd, m, l2hlen)) { /* the chain is too large */ if_statinc(ifp, if_oerrors); + hvn_txd_put(txr, txd); m_freem(m); continue; } - bpf_mtap(ifp, m, BPF_D_OUT); - - if (hvn_rndis_output(sc, txd)) { - hvn_decap(sc, txd); - if_statinc(ifp, if_oerrors); - m_freem(m); - continue; + if (txr->txr_agg_pktleft == 0) { + if (txr->txr_agg_txd != NULL) { + hvn_flush_txagg(txr); + } else { + if (hvn_txpkt(txr, txd)) { + /* txd is freed, but m is not. */ + m_freem(m); + if_statinc(ifp, if_oerrors); + } + } } + } + + /* Flush pending aggerated transmission. */ + if (txr->txr_agg_txd != NULL) + hvn_flush_txagg(txr); +} + +static void +hvn_start(struct ifnet *ifp) +{ + struct hvn_softc *sc = IFP2SC(ifp); + struct hvn_tx_ring *txr = &sc->sc_txr[0]; + + mutex_enter(&txr->txr_lock); + hvn_transmit_common(ifp, txr, false); + mutex_exit(&txr->txr_lock); +} + +static int +hvn_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused) +{ + struct hvn_softc *sc = IFP2SC(ifp); + u_int cpu; + + cpu = cpu_index(curcpu()); + + return cpu % sc->sc_ntxr_inuse; +} - sc->sc_tx_next++; +static int +hvn_transmit(struct ifnet *ifp, struct mbuf *m) +{ + struct hvn_softc *sc = IFP2SC(ifp); + struct hvn_tx_ring *txr; + int qid; + + qid = hvn_select_txqueue(ifp, m); + txr = &sc->sc_txr[qid]; + + if (__predict_false(!pcq_put(txr->txr_interq, m))) { + mutex_enter(&txr->txr_lock); + txr->txr_evpcqdrop.ev_count++; + mutex_exit(&txr->txr_lock); + m_freem(m); + return ENOBUFS; } + + kpreempt_disable(); + softint_schedule(txr->txr_si); + kpreempt_enable(); + return 0; +} + +static void +hvn_deferred_transmit(void *arg) +{ + struct hvn_tx_ring *txr = arg; + struct hvn_softc *sc = txr->txr_softc; + struct ifnet *ifp = SC2IFP(sc); + + mutex_enter(&txr->txr_lock); + txr->txr_evtransmitdefer.ev_count++; + hvn_transmit_common(ifp, txr, true); + mutex_exit(&txr->txr_lock); } static inline char * @@ -522,22 +1261,282 @@ hvn_rndis_pktinfo_append(struct rndis_packet_msg *pkt, size_t pktsize, return (char *)pi->rm_data; } +static struct mbuf * +hvn_pullup_hdr(struct mbuf *m, int len) +{ + struct mbuf *mn; + + if (__predict_false(m->m_len < len)) { + mn = m_pullup(m, len); + if (mn == NULL) + return NULL; + m = mn; + } + return m; +} + +/* + * NOTE: If this function failed, the m would be freed. + */ +static struct mbuf * +hvn_set_hlen(struct mbuf *m, int *l2hlenp) +{ + const struct ether_header *eh; + int l2hlen, off; + + m = hvn_pullup_hdr(m, sizeof(*eh)); + if (m == NULL) + return NULL; + + eh = mtod(m, const struct ether_header *); + if (eh->ether_type == ntohs(ETHERTYPE_VLAN)) + l2hlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + else + l2hlen = ETHER_HDR_LEN; + +#if defined(INET) + if (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4)) { + const struct ip *ip; + + off = l2hlen + sizeof(*ip); + m = hvn_pullup_hdr(m, off); + if (m == NULL) + return NULL; + + ip = (struct ip *)((mtod(m, uint8_t *)) + off); + + /* + * UDP checksum offload does not work in Azure, if the + * following conditions meet: + * - sizeof(IP hdr + UDP hdr + payload) > 1420. + * - IP_DF is not set in the IP hdr. + * + * Fallback to software checksum for these UDP datagrams. + */ + if ((m->m_pkthdr.csum_flags & M_CSUM_UDPv4) && + m->m_pkthdr.len > hvn_udpcs_fixup_mtu + l2hlen && + !(ntohs(ip->ip_off) & IP_DF)) { + uint16_t *csump; + + off = l2hlen + + M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); + m = hvn_pullup_hdr(m, off + sizeof(struct udphdr)); + if (m == NULL) + return NULL; + + csump = (uint16_t *)(mtod(m, uint8_t *) + off + + M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data)); + *csump = cpu_in_cksum(m, m->m_pkthdr.len - off, off, 0); + m->m_pkthdr.csum_flags &= ~M_CSUM_UDPv4; + } + } +#endif /* INET */ +#if defined(INET) && defined(INET6) + else +#endif /* INET && INET6 */ +#if defined(INET6) + { + const struct ip6_hdr *ip6; + + off = l2hlen + sizeof(*ip6); + m = hvn_pullup_hdr(m, off); + if (m == NULL) + return NULL; + + ip6 = (struct ip6_hdr *)((mtod(m, uint8_t *)) + l2hlen); + if (ip6->ip6_nxt != IPPROTO_TCP && + ip6->ip6_nxt != IPPROTO_UDP) { + m_freem(m); + return NULL; + } + } +#endif /* INET6 */ + + *l2hlenp = l2hlen; + + return m; +} + static int -hvn_encap(struct hvn_softc *sc, struct mbuf *m, struct hvn_tx_desc **txd0) +hvn_flush_txagg(struct hvn_tx_ring *txr) { + struct hvn_softc *sc = txr->txr_softc; + struct ifnet *ifp = SC2IFP(sc); struct hvn_tx_desc *txd; + struct mbuf *m; + int error, pkts; + + txd = txr->txr_agg_txd; + KASSERTMSG(txd != NULL, "no aggregate txdesc"); + + /* + * Since hvn_txpkt() will reset this temporary stat, save + * it now, so that oerrors can be updated properly, if + * hvn_txpkt() ever fails. + */ + pkts = txr->txr_stat_pkts; + + /* + * Since txd's mbuf will _not_ be freed upon hvn_txpkt() + * failure, save it for later freeing, if hvn_txpkt() ever + * fails. + */ + m = txd->txd_buf; + error = hvn_txpkt(txr, txd); + if (__predict_false(error)) { + /* txd is freed, but m is not. */ + m_freem(m); + txr->txr_evflushfailed.ev_count++; + if_statadd(ifp, if_oerrors, pkts); + } + + /* Reset all aggregation states. */ + txr->txr_agg_txd = NULL; + txr->txr_agg_szleft = 0; + txr->txr_agg_pktleft = 0; + txr->txr_agg_prevpkt = NULL; + + return error; +} + +static void * +hvn_try_txagg(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd, int pktsz) +{ + struct hvn_softc *sc = txr->txr_softc; + struct hvn_tx_desc *agg_txd; + struct rndis_packet_msg *pkt; + void *chim; + int olen; + + if (txr->txr_agg_txd != NULL) { + if (txr->txr_agg_pktleft > 0 && txr->txr_agg_szleft > pktsz) { + agg_txd = txr->txr_agg_txd; + pkt = txr->txr_agg_prevpkt; + + /* + * Update the previous RNDIS packet's total length, + * it can be increased due to the mandatory alignment + * padding for this RNDIS packet. And update the + * aggregating txdesc's chimney sending buffer size + * accordingly. + * + * XXX + * Zero-out the padding, as required by the RNDIS spec. + */ + olen = pkt->rm_len; + pkt->rm_len = roundup2(olen, txr->txr_agg_align); + agg_txd->txd_chim_size += pkt->rm_len - olen; + + /* Link this txdesc to the parent. */ + hvn_txd_agg(agg_txd, txd); + + chim = (uint8_t *)pkt + pkt->rm_len; + /* Save the current packet for later fixup. */ + txr->txr_agg_prevpkt = chim; + + txr->txr_agg_pktleft--; + txr->txr_agg_szleft -= pktsz; + if (txr->txr_agg_szleft <= + HVN_PKTSIZE_MIN(txr->txr_agg_align)) { + /* + * Probably can't aggregate more packets, + * flush this aggregating txdesc proactively. + */ + txr->txr_agg_pktleft = 0; + } + + /* Done! */ + return chim; + } + hvn_flush_txagg(txr); + } + + txr->txr_evchimneytried.ev_count++; + txd->txd_chim_index = hvn_chim_alloc(sc); + if (txd->txd_chim_index == HVN_NVS_CHIM_IDX_INVALID) + return NULL; + txr->txr_evchimney.ev_count++; + + chim = sc->sc_chim + (txd->txd_chim_index * sc->sc_chim_szmax); + + if (txr->txr_agg_pktmax > 1 && + txr->txr_agg_szmax > pktsz + HVN_PKTSIZE_MIN(txr->txr_agg_align)) { + txr->txr_agg_txd = txd; + txr->txr_agg_pktleft = txr->txr_agg_pktmax - 1; + txr->txr_agg_szleft = txr->txr_agg_szmax - pktsz; + txr->txr_agg_prevpkt = chim; + } + + return chim; +} + +static int +hvn_encap(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd, struct mbuf *m, + int l2hlen) +{ + /* Used to pad ethernet frames with < ETHER_MIN_LEN bytes */ + static const char zero_pad[ETHER_MIN_LEN]; + struct hvn_softc *sc = txr->txr_softc; struct rndis_packet_msg *pkt; bus_dma_segment_t *seg; - size_t pktlen; + void *chim = NULL; + size_t pktlen, pktsize; + int l3hlen; int i, rv; - do { - txd = &sc->sc_tx_desc[sc->sc_tx_next % HVN_TX_DESC]; - sc->sc_tx_next++; - } while (!txd->txd_ready); - txd->txd_ready = 0; + if (ISSET(sc->sc_caps, HVN_CAPS_VLAN) && !vlan_has_tag(m)) { + struct ether_vlan_header *evl; + + m = hvn_pullup_hdr(m, sizeof(*evl)); + if (m == NULL) { + DPRINTF("%s: failed to pullup mbuf\n", + device_xname(sc->sc_dev)); + return -1; + } + + evl = mtod(m, struct ether_vlan_header *); + if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + struct ether_header *eh; + uint16_t proto = evl->evl_proto; + + vlan_set_tag(m, ntohs(evl->evl_tag)); + + /* + * Trim VLAN tag from header. + */ + memmove((uint8_t *)evl + ETHER_VLAN_ENCAP_LEN, + evl, ETHER_HDR_LEN); + m_adj(m, ETHER_VLAN_ENCAP_LEN); + + eh = mtod(m, struct ether_header *); + eh->ether_type = proto; + + /* + * Re-padding. See sys/net/if_vlan.c:vlan_start(). + */ + if (m->m_pkthdr.len < (ETHER_MIN_LEN - ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN)) { + m_copyback(m, m->m_pkthdr.len, + (ETHER_MIN_LEN - ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN) - + m->m_pkthdr.len, zero_pad); + } + + txr->txr_evvlanfixup.ev_count++; + } + } pkt = txd->txd_req; + pktsize = HVN_PKTSIZE(m, txr->txr_agg_align); + if (pktsize < txr->txr_chim_size) { + chim = hvn_try_txagg(txr, txd, pktsize); + if (chim != NULL) + pkt = chim; + } else { + if (txr->txr_agg_txd != NULL) + hvn_flush_txagg(txr); + } + memset(pkt, 0, HVN_RNDIS_PKT_LEN); pkt->rm_type = REMOTE_NDIS_PACKET_MSG; pkt->rm_len = sizeof(*pkt) + m->m_pkthdr.len; @@ -546,22 +1545,18 @@ hvn_encap(struct hvn_softc *sc, struct mbuf *m, struct hvn_tx_desc **txd0) pkt->rm_pktinfooffset = sizeof(*pkt); /* adjusted below */ pkt->rm_pktinfolen = 0; - rv = bus_dmamap_load_mbuf(sc->sc_dmat, txd->txd_dmap, m, BUS_DMA_READ | - BUS_DMA_NOWAIT); - switch (rv) { - case 0: - break; - case EFBIG: - if (m_defrag(m, M_NOWAIT) != NULL && - bus_dmamap_load_mbuf(sc->sc_dmat, txd->txd_dmap, m, - BUS_DMA_READ | BUS_DMA_NOWAIT) == 0) - break; - /* FALLTHROUGH */ - default: - DPRINTF("%s: failed to load mbuf\n", device_xname(sc->sc_dev)); - return -1; + if (txr->txr_flags & HVN_TXR_FLAG_UDP_HASH) { + char *cp; + + /* + * Set the hash value for this packet, so that the host could + * dispatch the TX done event for this packet back to this TX + * ring's channel. + */ + cp = hvn_rndis_pktinfo_append(pkt, HVN_RNDIS_PKT_LEN, + HVN_NDIS_HASH_VALUE_SIZE, HVN_NDIS_PKTINFO_TYPE_HASHVAL); + memcpy(cp, &txr->txr_id, HVN_NDIS_HASH_VALUE_SIZE); } - txd->txd_buf = m; if (vlan_has_tag(m)) { uint32_t vlan; @@ -574,19 +1569,34 @@ hvn_encap(struct hvn_softc *sc, struct mbuf *m, struct hvn_tx_desc **txd0) cp = hvn_rndis_pktinfo_append(pkt, HVN_RNDIS_PKT_LEN, NDIS_VLAN_INFO_SIZE, NDIS_PKTINFO_TYPE_VLAN); memcpy(cp, &vlan, NDIS_VLAN_INFO_SIZE); + txr->txr_evvlanhwtagging.ev_count++; } - if (m->m_pkthdr.csum_flags & (M_CSUM_IPv4 | M_CSUM_UDPv4 | - M_CSUM_TCPv4)) { - uint32_t csum = NDIS_TXCSUM_INFO_IPV4; + if (m->m_pkthdr.csum_flags & txr->txr_csum_assist) { + uint32_t csum; char *cp; - if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) - csum |= NDIS_TXCSUM_INFO_IPCS; - if (m->m_pkthdr.csum_flags & M_CSUM_TCPv4) - csum |= NDIS_TXCSUM_INFO_TCPCS; - if (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) - csum |= NDIS_TXCSUM_INFO_UDPCS; + if (m->m_pkthdr.csum_flags & (M_CSUM_TCPv6 | M_CSUM_UDPv6)) { + csum = NDIS_TXCSUM_INFO_IPV6; + l3hlen = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); + if (m->m_pkthdr.csum_flags & M_CSUM_TCPv6) + csum |= NDIS_TXCSUM_INFO_MKTCPCS(l2hlen + + l3hlen); + if (m->m_pkthdr.csum_flags & M_CSUM_UDPv6) + csum |= NDIS_TXCSUM_INFO_MKUDPCS(l2hlen + + l3hlen); + } else { + csum = NDIS_TXCSUM_INFO_IPV4; + l3hlen = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); + if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) + csum |= NDIS_TXCSUM_INFO_IPCS; + if (m->m_pkthdr.csum_flags & M_CSUM_TCPv4) + csum |= NDIS_TXCSUM_INFO_MKTCPCS(l2hlen + + l3hlen); + if (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) + csum |= NDIS_TXCSUM_INFO_MKUDPCS(l2hlen + + l3hlen); + } cp = hvn_rndis_pktinfo_append(pkt, HVN_RNDIS_PKT_LEN, NDIS_TXCSUM_INFO_SIZE, NDIS_PKTINFO_TYPE_CSUM); memcpy(cp, &csum, NDIS_TXCSUM_INFO_SIZE); @@ -595,6 +1605,53 @@ hvn_encap(struct hvn_softc *sc, struct mbuf *m, struct hvn_tx_desc **txd0) pktlen = pkt->rm_pktinfooffset + pkt->rm_pktinfolen; pkt->rm_pktinfooffset -= RNDIS_HEADER_OFFSET; + /* + * Fast path: Chimney sending. + */ + if (chim != NULL) { + struct hvn_tx_desc *tgt_txd; + + tgt_txd = (txr->txr_agg_txd != NULL) ? txr->txr_agg_txd : txd; + + KASSERTMSG(pkt == chim, + "RNDIS pkt not in chimney sending buffer"); + KASSERTMSG(tgt_txd->txd_chim_index != HVN_NVS_CHIM_IDX_INVALID, + "chimney sending buffer is not used"); + + tgt_txd->txd_chim_size += pkt->rm_len; + m_copydata(m, 0, m->m_pkthdr.len, (uint8_t *)chim + pktlen); + + txr->txr_sendpkt = hvn_rndis_output_chim; + goto done; + } + + KASSERTMSG(txr->txr_agg_txd == NULL, "aggregating sglist txdesc"); + KASSERTMSG(txd->txd_chim_index == HVN_NVS_CHIM_IDX_INVALID, + "chimney buffer is used"); + KASSERTMSG(pkt == txd->txd_req, "RNDIS pkt not in txdesc"); + + rv = bus_dmamap_load_mbuf(sc->sc_dmat, txd->txd_dmap, m, BUS_DMA_READ | + BUS_DMA_NOWAIT); + switch (rv) { + case 0: + break; + case EFBIG: + if (m_defrag(m, M_NOWAIT) != NULL) { + txr->txr_evdefrag.ev_count++; + if (bus_dmamap_load_mbuf(sc->sc_dmat, txd->txd_dmap, m, + BUS_DMA_READ | BUS_DMA_NOWAIT) == 0) + break; + } + /* FALLTHROUGH */ + default: + DPRINTF("%s: failed to load mbuf\n", device_xname(sc->sc_dev)); + txr->txr_evdmafailed.ev_count++; + return -1; + } + bus_dmamap_sync(sc->sc_dmat, txd->txd_dmap, + 0, txd->txd_dmap->dm_mapsize, BUS_DMASYNC_PREWRITE); + SET(txd->txd_flags, HVN_TXD_FLAG_DMAMAP); + /* Attach an RNDIS message to the first slot */ txd->txd_sgl[0].gpa_page = txd->txd_gpa.gpa_page; txd->txd_sgl[0].gpa_ofs = txd->txd_gpa.gpa_ofs; @@ -608,35 +1665,115 @@ hvn_encap(struct hvn_softc *sc, struct mbuf *m, struct hvn_tx_desc **txd0) txd->txd_sgl[1 + i].gpa_len = seg->ds_len; } - *txd0 = txd; + txd->txd_chim_index = HVN_NVS_CHIM_IDX_INVALID; + txd->txd_chim_size = 0; + txr->txr_sendpkt = hvn_rndis_output_sgl; +done: + txd->txd_buf = m; - atomic_dec_uint(&sc->sc_tx_avail); + /* Update temporary stats for later use. */ + txr->txr_stat_pkts++; + txr->txr_stat_size += m->m_pkthdr.len; + if (m->m_flags & M_MCAST) + txr->txr_stat_mcasts++; return 0; } static void -hvn_decap(struct hvn_softc *sc, struct hvn_tx_desc *txd) +hvn_bpf_mtap(struct hvn_tx_ring *txr, struct mbuf *m, u_int direction) { + struct hvn_softc *sc = txr->txr_softc; struct ifnet *ifp = SC2IFP(sc); + struct ether_header *eh; + struct ether_vlan_header evl; - bus_dmamap_sync(sc->sc_dmat, txd->txd_dmap, - 0, txd->txd_dmap->dm_mapsize, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(sc->sc_dmat, txd->txd_dmap); - txd->txd_buf = NULL; - txd->txd_nsge = 0; - txd->txd_ready = 1; - atomic_inc_uint(&sc->sc_tx_avail); - ifp->if_flags &= ~IFF_OACTIVE; + if (!vlan_has_tag(m)) { + bpf_mtap(ifp, m, direction); + return; + } + + if (ifp->if_bpf == NULL) + return; + + txr->txr_evvlantap.ev_count++; + + /* + * Restore a VLAN tag for bpf. + * + * Do not modify contents of the original mbuf, + * because Tx processing on the mbuf is still in progress. + */ + + eh = mtod(m, struct ether_header *); + memcpy(evl.evl_dhost, eh->ether_dhost, ETHER_ADDR_LEN * 2); + evl.evl_encap_proto = htons(ETHERTYPE_VLAN); + evl.evl_tag = htons(vlan_get_tag(m)); + evl.evl_proto = eh->ether_type; + + /* Do not tap ether header of the original mbuf. */ + m_adj(m, sizeof(*eh)); + + bpf_mtap2(ifp->if_bpf, &evl, sizeof(evl), m, direction); + + /* Cannot restore ether header of the original mbuf, + * but do not worry about it because just free it. */ } -static void -hvn_txeof(struct hvn_softc *sc, uint64_t tid) +static int +hvn_txpkt(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd) { + struct hvn_softc *sc = txr->txr_softc; struct ifnet *ifp = SC2IFP(sc); + const struct hvn_tx_desc *tmp_txd; + int error; + + /* + * Make sure that this txd and any aggregated txds are not + * freed before bpf_mtap. + */ + hvn_txd_hold(txd); + + error = (*txr->txr_sendpkt)(txr, txd); + if (error == 0) { + hvn_bpf_mtap(txr, txd->txd_buf, BPF_D_OUT); + STAILQ_FOREACH(tmp_txd, &txd->txd_agg_list, txd_agg_entry) + hvn_bpf_mtap(txr, tmp_txd->txd_buf, BPF_D_OUT); + + if_statadd(ifp, if_opackets, txr->txr_stat_pkts); + if_statadd(ifp, if_obytes, txr->txr_stat_size); + if (txr->txr_stat_mcasts != 0) + if_statadd(ifp, if_omcasts, txr->txr_stat_mcasts); + txr->txr_evpkts.ev_count += txr->txr_stat_pkts; + txr->txr_evsends.ev_count++; + } + + hvn_txd_put(txr, txd); + + if (__predict_false(error)) { + /* + * Caller will perform further processing on the + * associated mbuf, so don't free it in hvn_txd_put(); + * only unload it from the DMA map in hvn_txd_put(), + * if it was loaded. + */ + txd->txd_buf = NULL; + hvn_txd_put(txr, txd); + } + + /* Reset temporary stats, after this sending is done. */ + txr->txr_stat_pkts = 0; + txr->txr_stat_size = 0; + txr->txr_stat_mcasts = 0; + + return error; +} + +static void +hvn_txeof(struct hvn_tx_ring *txr, uint64_t tid) +{ + struct hvn_softc *sc = txr->txr_softc; struct hvn_tx_desc *txd; - struct mbuf *m; uint32_t id = tid >> 32; if ((tid & 0xffffffffU) != 0) @@ -648,174 +1785,257 @@ hvn_txeof(struct hvn_softc *sc, uint64_t tid) return; } - txd = &sc->sc_tx_desc[id]; + txd = &txr->txr_desc[id]; - if ((m = txd->txd_buf) == NULL) { + if (txd->txd_buf == NULL) device_printf(sc->sc_dev, "no mbuf @%u\n", id); - return; - } - txd->txd_buf = NULL; - - bus_dmamap_sync(sc->sc_dmat, txd->txd_dmap, - 0, txd->txd_dmap->dm_mapsize, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(sc->sc_dmat, txd->txd_dmap); - m_freem(m); - if_statinc(ifp, if_opackets); - txd->txd_ready = 1; - - atomic_inc_uint(&sc->sc_tx_avail); - ifp->if_flags &= ~IFF_OACTIVE; + hvn_txd_put(txr, txd); } static int -hvn_rx_ring_create(struct hvn_softc *sc) +hvn_rx_ring_create(struct hvn_softc *sc, int ring_cnt) { - struct hvn_nvs_rxbuf_conn cmd; - struct hvn_nvs_rxbuf_conn_resp *rsp; - uint64_t tid; + struct hvn_rx_ring *rxr; + int i; if (sc->sc_proto <= HVN_NVS_PROTO_VERSION_2) sc->sc_rx_size = 15 * 1024 * 1024; /* 15MB */ else sc->sc_rx_size = 16 * 1024 * 1024; /* 16MB */ sc->sc_rx_ring = hyperv_dma_alloc(sc->sc_dmat, &sc->sc_rx_dma, - sc->sc_rx_size, PAGE_SIZE, PAGE_SIZE, sc->sc_rx_size / PAGE_SIZE, - HYPERV_DMA_SLEEPOK); + sc->sc_rx_size, PAGE_SIZE, PAGE_SIZE, sc->sc_rx_size / PAGE_SIZE); if (sc->sc_rx_ring == NULL) { DPRINTF("%s: failed to allocate Rx ring buffer\n", device_xname(sc->sc_dev)); return -1; } - if (vmbus_handle_alloc(sc->sc_chan, &sc->sc_rx_dma, sc->sc_rx_size, - &sc->sc_rx_hndl)) { - DPRINTF("%s: failed to obtain a PA handle\n", - device_xname(sc->sc_dev)); - goto errout; - } - memset(&cmd, 0, sizeof(cmd)); - cmd.nvs_type = HVN_NVS_TYPE_RXBUF_CONN; - cmd.nvs_gpadl = sc->sc_rx_hndl; - cmd.nvs_sig = HVN_NVS_RXBUF_SIG; + sc->sc_rxr = kmem_zalloc(sizeof(*rxr) * ring_cnt, KM_SLEEP); + sc->sc_nrxr_inuse = sc->sc_nrxr = ring_cnt; - tid = atomic_inc_uint_nv(&sc->sc_nvstid); - if (hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 100)) - goto errout; + for (i = 0; i < sc->sc_nrxr; i++) { + rxr = &sc->sc_rxr[i]; + rxr->rxr_softc = sc; + if (i < sc->sc_ntxr) { + rxr->rxr_txr = &sc->sc_txr[i]; + rxr->rxr_txr->txr_rxr = rxr; + } - rsp = (struct hvn_nvs_rxbuf_conn_resp *)&sc->sc_nvsrsp; - if (rsp->nvs_status != HVN_NVS_STATUS_OK) { - DPRINTF("%s: failed to set up the Rx ring\n", - device_xname(sc->sc_dev)); - goto errout; - } - if (rsp->nvs_nsect > 1) { - DPRINTF("%s: invalid number of Rx ring sections: %u\n", - device_xname(sc->sc_dev), rsp->nvs_nsect); - hvn_rx_ring_destroy(sc); - return -1; + mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); + mutex_init(&rxr->rxr_onwork_lock, MUTEX_DEFAULT, IPL_NET); + cv_init(&rxr->rxr_onwork_cv, "waitonwk"); + + snprintf(rxr->rxr_name, sizeof(rxr->rxr_name), + "%s-rx%d", device_xname(sc->sc_dev), i); + evcnt_attach_dynamic(&rxr->rxr_evpkts, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "packets received"); + evcnt_attach_dynamic(&rxr->rxr_evcsum_ip, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "IP checksum"); + evcnt_attach_dynamic(&rxr->rxr_evcsum_tcp, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "TCP checksum"); + evcnt_attach_dynamic(&rxr->rxr_evcsum_udp, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "UDP checksum"); + evcnt_attach_dynamic(&rxr->rxr_evvlanhwtagging, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "VLAN H/W tagging"); + evcnt_attach_dynamic(&rxr->rxr_evintr, EVCNT_TYPE_INTR, + NULL, rxr->rxr_name, "interrupt on ring"); + evcnt_attach_dynamic(&rxr->rxr_evdefer, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "handled queue in workqueue"); + evcnt_attach_dynamic(&rxr->rxr_evdeferreq, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "requested defer on ring"); + evcnt_attach_dynamic(&rxr->rxr_evredeferreq, EVCNT_TYPE_MISC, + NULL, rxr->rxr_name, "requested defer in workqueue"); + + rxr->rxr_nvsbuf = kmem_zalloc(HVN_NVS_BUFSIZE, KM_SLEEP); + if (rxr->rxr_nvsbuf == NULL) { + DPRINTF("%s: failed to allocate channel data buffer\n", + device_xname(sc->sc_dev)); + goto errout; + } + + rxr->rxr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, + hvn_nvs_softintr, rxr); + if (rxr->rxr_si == NULL) { + DPRINTF("%s: failed to establish rx softint\n", + device_xname(sc->sc_dev)); + goto errout; + } } + return 0; errout: - if (sc->sc_rx_hndl) { - vmbus_handle_free(sc->sc_chan, sc->sc_rx_hndl); - sc->sc_rx_hndl = 0; - } - if (sc->sc_rx_ring) { - hyperv_dma_free(sc->sc_dmat, &sc->sc_rx_dma); - sc->sc_rx_ring = NULL; - } + hvn_rx_ring_destroy(sc); return -1; } static int hvn_rx_ring_destroy(struct hvn_softc *sc) { - struct hvn_nvs_rxbuf_disconn cmd; - uint64_t tid; + struct hvn_rx_ring *rxr; + int i; - if (sc->sc_rx_ring == NULL) - return 0; + if (sc->sc_rxr != NULL) { + for (i = 0; i < sc->sc_nrxr; i++) { + rxr = &sc->sc_rxr[i]; - memset(&cmd, 0, sizeof(cmd)); - cmd.nvs_type = HVN_NVS_TYPE_RXBUF_DISCONN; - cmd.nvs_sig = HVN_NVS_RXBUF_SIG; + if (rxr->rxr_si != NULL) { + softint_disestablish(rxr->rxr_si); + rxr->rxr_si = NULL; + } - tid = atomic_inc_uint_nv(&sc->sc_nvstid); - if (hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0)) - return -1; + if (rxr->rxr_nvsbuf != NULL) { + kmem_free(rxr->rxr_nvsbuf, HVN_NVS_BUFSIZE); + rxr->rxr_nvsbuf = NULL; + } - delay(100); + evcnt_detach(&rxr->rxr_evpkts); + evcnt_detach(&rxr->rxr_evcsum_ip); + evcnt_detach(&rxr->rxr_evcsum_tcp); + evcnt_detach(&rxr->rxr_evcsum_udp); + evcnt_detach(&rxr->rxr_evvlanhwtagging); + evcnt_detach(&rxr->rxr_evintr); + evcnt_detach(&rxr->rxr_evdefer); + evcnt_detach(&rxr->rxr_evdeferreq); + evcnt_detach(&rxr->rxr_evredeferreq); + + cv_destroy(&rxr->rxr_onwork_cv); + mutex_destroy(&rxr->rxr_onwork_lock); + mutex_destroy(&rxr->rxr_lock); + } + kmem_free(sc->sc_rxr, sizeof(*rxr) * sc->sc_nrxr); + sc->sc_rxr = NULL; + sc->sc_nrxr = 0; + } + if (sc->sc_rx_ring != NULL) { + hyperv_dma_free(sc->sc_dmat, &sc->sc_rx_dma); + sc->sc_rx_ring = NULL; + } - vmbus_handle_free(sc->sc_chan, sc->sc_rx_hndl); - sc->sc_rx_hndl = 0; + return 0; +} - hyperv_dma_free(sc->sc_dmat, &sc->sc_rx_dma); - sc->sc_rx_ring = NULL; +static void +hvn_fixup_rx_data(struct hvn_softc *sc) +{ + struct hvn_rx_ring *rxr; + int i; - return 0; + if (sc->sc_caps & HVN_CAPS_UDPHASH) { + for (i = 0; i < sc->sc_nrxr; i++) { + rxr = &sc->sc_rxr[i]; + rxr->rxr_flags |= HVN_RXR_FLAG_UDP_HASH; + } + } } static int -hvn_tx_ring_create(struct hvn_softc *sc) +hvn_tx_ring_create(struct hvn_softc *sc, int ring_cnt) { - const int dmaflags = cold ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK; + struct hvn_tx_ring *txr; struct hvn_tx_desc *txd; bus_dma_segment_t *seg; size_t msgsize; - int i, rsegs; + int i, j; paddr_t pa; - msgsize = roundup(HVN_RNDIS_PKT_LEN, 128); - - /* Allocate memory to store RNDIS messages */ - if (bus_dmamem_alloc(sc->sc_dmat, msgsize * HVN_TX_DESC, PAGE_SIZE, 0, - &sc->sc_tx_mseg, 1, &rsegs, dmaflags)) { - DPRINTF("%s: failed to allocate memory for RDNIS messages\n", - device_xname(sc->sc_dev)); - goto errout; - } - if (bus_dmamem_map(sc->sc_dmat, &sc->sc_tx_mseg, 1, msgsize * - HVN_TX_DESC, (void **)&sc->sc_tx_msgs, dmaflags)) { - DPRINTF("%s: failed to establish mapping for RDNIS messages\n", - device_xname(sc->sc_dev)); - goto errout; - } - memset(sc->sc_tx_msgs, 0, msgsize * HVN_TX_DESC); - if (bus_dmamap_create(sc->sc_dmat, msgsize * HVN_TX_DESC, 1, - msgsize * HVN_TX_DESC, 0, dmaflags, &sc->sc_tx_rmap)) { - DPRINTF("%s: failed to create map for RDNIS messages\n", - device_xname(sc->sc_dev)); - goto errout; - } - if (bus_dmamap_load(sc->sc_dmat, sc->sc_tx_rmap, sc->sc_tx_msgs, - msgsize * HVN_TX_DESC, NULL, dmaflags)) { - DPRINTF("%s: failed to create map for RDNIS messages\n", + /* + * Create TXBUF for chimney sending. + * + * NOTE: It is shared by all channels. + */ + sc->sc_chim = hyperv_dma_alloc(sc->sc_dmat, &sc->sc_chim_dma, + HVN_CHIM_SIZE, PAGE_SIZE, 0, 1); + if (sc->sc_chim == NULL) { + DPRINTF("%s: failed to allocate chimney sending memory", device_xname(sc->sc_dev)); goto errout; } - for (i = 0; i < HVN_TX_DESC; i++) { - txd = &sc->sc_tx_desc[i]; - if (bus_dmamap_create(sc->sc_dmat, HVN_TX_PKT_SIZE, - HVN_TX_FRAGS, HVN_TX_FRAG_SIZE, PAGE_SIZE, dmaflags, - &txd->txd_dmap)) { - DPRINTF("%s: failed to create map for TX descriptors\n", - device_xname(sc->sc_dev)); + sc->sc_txr = kmem_zalloc(sizeof(*txr) * ring_cnt, KM_SLEEP); + sc->sc_ntxr_inuse = sc->sc_ntxr = ring_cnt; + + msgsize = roundup(HVN_RNDIS_PKT_LEN, 128); + + for (j = 0; j < ring_cnt; j++) { + txr = &sc->sc_txr[j]; + txr->txr_softc = sc; + txr->txr_id = j; + + mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); + txr->txr_interq = pcq_create(HVN_TX_DESC, KM_SLEEP); + + snprintf(txr->txr_name, sizeof(txr->txr_name), + "%s-tx%d", device_xname(sc->sc_dev), j); + evcnt_attach_dynamic(&txr->txr_evpkts, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "packets transmit"); + evcnt_attach_dynamic(&txr->txr_evsends, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "sends"); + evcnt_attach_dynamic(&txr->txr_evnodesc, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "descriptor shortage"); + evcnt_attach_dynamic(&txr->txr_evdmafailed, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "DMA failure"); + evcnt_attach_dynamic(&txr->txr_evdefrag, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "mbuf defraged"); + evcnt_attach_dynamic(&txr->txr_evpcqdrop, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "dropped in pcq"); + evcnt_attach_dynamic(&txr->txr_evtransmitdefer, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "deferred transmit"); + evcnt_attach_dynamic(&txr->txr_evflushfailed, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "aggregation flush failure"); + evcnt_attach_dynamic(&txr->txr_evchimneytried, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "chimney send tried"); + evcnt_attach_dynamic(&txr->txr_evchimney, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "chimney send"); + evcnt_attach_dynamic(&txr->txr_evvlanfixup, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "VLAN fixup"); + evcnt_attach_dynamic(&txr->txr_evvlanhwtagging, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "VLAN H/W tagging"); + evcnt_attach_dynamic(&txr->txr_evvlantap, EVCNT_TYPE_MISC, + NULL, txr->txr_name, "VLAN bpf_mtap fixup"); + + txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, + hvn_deferred_transmit, txr); + if (txr->txr_si == NULL) { + aprint_error_dev(sc->sc_dev, + "failed to establish softint for tx ring\n"); + goto errout; + } + + /* Allocate memory to store RNDIS messages */ + txr->txr_msgs = hyperv_dma_alloc(sc->sc_dmat, &txr->txr_dma, + msgsize * HVN_TX_DESC, PAGE_SIZE, 0, 1); + if (txr->txr_msgs == NULL) { + DPRINTF("%s: failed to allocate memory for RDNIS " + "messages\n", device_xname(sc->sc_dev)); goto errout; } - seg = &sc->sc_tx_rmap->dm_segs[0]; - pa = seg->ds_addr + (msgsize * i); - txd->txd_gpa.gpa_page = atop(pa); - txd->txd_gpa.gpa_ofs = pa & PAGE_MASK; - txd->txd_gpa.gpa_len = msgsize; - txd->txd_req = (void *)(sc->sc_tx_msgs + (msgsize * i)); - txd->txd_id = i + HVN_NVS_CHIM_SIG; - txd->txd_ready = 1; + + TAILQ_INIT(&txr->txr_list); + for (i = 0; i < HVN_TX_DESC; i++) { + txd = &txr->txr_desc[i]; + txd->txd_chim_index = HVN_NVS_CHIM_IDX_INVALID; + txd->txd_chim_size = 0; + STAILQ_INIT(&txd->txd_agg_list); + if (bus_dmamap_create(sc->sc_dmat, HVN_TX_PKT_SIZE, + HVN_TX_FRAGS, HVN_TX_FRAG_SIZE, PAGE_SIZE, + BUS_DMA_WAITOK, &txd->txd_dmap)) { + DPRINTF("%s: failed to create map for TX " + "descriptors\n", device_xname(sc->sc_dev)); + goto errout; + } + seg = &txr->txr_dma.map->dm_segs[0]; + pa = seg->ds_addr + (msgsize * i); + txd->txd_gpa.gpa_page = atop(pa); + txd->txd_gpa.gpa_ofs = pa & PAGE_MASK; + txd->txd_gpa.gpa_len = msgsize; + txd->txd_req = (void *)(txr->txr_msgs + (msgsize * i)); + txd->txd_id = i + HVN_NVS_CHIM_SIG; + TAILQ_INSERT_TAIL(&txr->txr_list, txd, txd_entry); + } + txr->txr_avail = HVN_TX_DESC; } - sc->sc_tx_avail = HVN_TX_DESC; return 0; @@ -827,40 +2047,319 @@ hvn_tx_ring_create(struct hvn_softc *sc) static void hvn_tx_ring_destroy(struct hvn_softc *sc) { + struct hvn_tx_ring *txr; struct hvn_tx_desc *txd; + int i, j; + + if (sc->sc_txr != NULL) { + for (j = 0; j < sc->sc_ntxr; j++) { + txr = &sc->sc_txr[j]; + + mutex_enter(&txr->txr_lock); + for (i = 0; i < HVN_TX_DESC; i++) { + txd = &txr->txr_desc[i]; + hvn_txd_gc(txr, txd); + } + mutex_exit(&txr->txr_lock); + for (i = 0; i < HVN_TX_DESC; i++) { + txd = &txr->txr_desc[i]; + if (txd->txd_dmap != NULL) { + bus_dmamap_destroy(sc->sc_dmat, + txd->txd_dmap); + txd->txd_dmap = NULL; + } + } + if (txr->txr_msgs != NULL) { + hyperv_dma_free(sc->sc_dmat, &txr->txr_dma); + txr->txr_msgs = NULL; + } + if (txr->txr_si != NULL) { + softint_disestablish(txr->txr_si); + txr->txr_si = NULL; + } + if (txr->txr_interq != NULL) { + hvn_tx_ring_qflush(sc, txr); + pcq_destroy(txr->txr_interq); + txr->txr_interq = NULL; + } + + evcnt_detach(&txr->txr_evpkts); + evcnt_detach(&txr->txr_evsends); + evcnt_detach(&txr->txr_evnodesc); + evcnt_detach(&txr->txr_evdmafailed); + evcnt_detach(&txr->txr_evdefrag); + evcnt_detach(&txr->txr_evpcqdrop); + evcnt_detach(&txr->txr_evtransmitdefer); + evcnt_detach(&txr->txr_evflushfailed); + evcnt_detach(&txr->txr_evchimneytried); + evcnt_detach(&txr->txr_evchimney); + evcnt_detach(&txr->txr_evvlanfixup); + evcnt_detach(&txr->txr_evvlanhwtagging); + evcnt_detach(&txr->txr_evvlantap); + + mutex_destroy(&txr->txr_lock); + } + + kmem_free(sc->sc_txr, sizeof(*txr) * sc->sc_ntxr); + sc->sc_txr = NULL; + } + + if (sc->sc_chim != NULL) { + hyperv_dma_free(sc->sc_dmat, &sc->sc_chim_dma); + sc->sc_chim = NULL; + } +} + +static void +hvn_set_chim_size(struct hvn_softc *sc, int chim_size) +{ + struct hvn_tx_ring *txr; int i; - for (i = 0; i < HVN_TX_DESC; i++) { - txd = &sc->sc_tx_desc[i]; - if (txd->txd_dmap == NULL) + for (i = 0; i < sc->sc_ntxr_inuse; i++) { + txr = &sc->sc_txr[i]; + txr->txr_chim_size = chim_size; + } +} + +#if LONG_BIT == 64 +#define ffsl(v) ffs64(v) +#elif LONG_BIT == 32 +#define ffsl(v) ffs32(v) +#else +#error unsupport LONG_BIT +#endif /* LONG_BIT */ + +static uint32_t +hvn_chim_alloc(struct hvn_softc *sc) +{ + uint32_t chim_idx = HVN_NVS_CHIM_IDX_INVALID; + int i, idx; + + mutex_spin_enter(&sc->sc_chim_bmap_lock); + for (i = 0; i < sc->sc_chim_bmap_cnt; i++) { + idx = ffsl(~sc->sc_chim_bmap[i]); + if (idx == 0) continue; + + --idx; /* ffsl is 1-based */ + SET(sc->sc_chim_bmap[i], __BIT(idx)); + + chim_idx = i * LONG_BIT + idx; + break; + } + mutex_spin_exit(&sc->sc_chim_bmap_lock); + + return chim_idx; +} + +static void +hvn_chim_free(struct hvn_softc *sc, uint32_t chim_idx) +{ + u_long mask; + uint32_t idx; + + idx = chim_idx / LONG_BIT; + mask = __BIT(chim_idx % LONG_BIT); + + mutex_spin_enter(&sc->sc_chim_bmap_lock); + CLR(sc->sc_chim_bmap[idx], mask); + mutex_spin_exit(&sc->sc_chim_bmap_lock); +} + +static void +hvn_fixup_tx_data(struct hvn_softc *sc) +{ + struct hvn_tx_ring *txr; + uint64_t caps_assist; + int csum_assist; + int i; + + hvn_set_chim_size(sc, sc->sc_chim_szmax); + if (hvn_tx_chimney_size > 0 && hvn_tx_chimney_size < sc->sc_chim_szmax) + hvn_set_chim_size(sc, hvn_tx_chimney_size); + + caps_assist = 0; + csum_assist = 0; + if (sc->sc_caps & HVN_CAPS_IPCS) { + caps_assist |= IFCAP_CSUM_IPv4_Tx; + caps_assist |= IFCAP_CSUM_IPv4_Rx; + csum_assist |= M_CSUM_IPv4; + } + if (sc->sc_caps & HVN_CAPS_TCP4CS) { + caps_assist |= IFCAP_CSUM_TCPv4_Tx; + caps_assist |= IFCAP_CSUM_TCPv4_Rx; + csum_assist |= M_CSUM_TCPv4; + } + if (sc->sc_caps & HVN_CAPS_TCP6CS) { + caps_assist |= IFCAP_CSUM_TCPv6_Tx; + csum_assist |= M_CSUM_TCPv6; + } + if (sc->sc_caps & HVN_CAPS_UDP4CS) { + caps_assist |= IFCAP_CSUM_UDPv4_Tx; + caps_assist |= IFCAP_CSUM_UDPv4_Rx; + csum_assist |= M_CSUM_UDPv4; + } + if (sc->sc_caps & HVN_CAPS_UDP6CS) { + caps_assist |= IFCAP_CSUM_UDPv6_Tx; + csum_assist |= M_CSUM_UDPv6; + } + for (i = 0; i < sc->sc_ntxr; i++) { + txr = &sc->sc_txr[i]; + txr->txr_caps_assist = caps_assist; + txr->txr_csum_assist = csum_assist; + } + + if (sc->sc_caps & HVN_CAPS_UDPHASH) { + for (i = 0; i < sc->sc_ntxr; i++) { + txr = &sc->sc_txr[i]; + txr->txr_flags |= HVN_TXR_FLAG_UDP_HASH; + } + } +} + +static int +hvn_txd_peek(struct hvn_tx_ring *txr) +{ + + KASSERT(mutex_owned(&txr->txr_lock)); + + return txr->txr_avail; +} + +static struct hvn_tx_desc * +hvn_txd_get(struct hvn_tx_ring *txr) +{ + struct hvn_tx_desc *txd; + + KASSERT(mutex_owned(&txr->txr_lock)); + + txd = TAILQ_FIRST(&txr->txr_list); + KASSERT(txd != NULL); + TAILQ_REMOVE(&txr->txr_list, txd, txd_entry); + txr->txr_avail--; + + txd->txd_refs = 1; + + return txd; +} + +static void +hvn_txd_put(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd) +{ + struct hvn_softc *sc = txr->txr_softc; + struct hvn_tx_desc *tmp_txd; + + KASSERT(mutex_owned(&txr->txr_lock)); + KASSERTMSG(!ISSET(txd->txd_flags, HVN_TXD_FLAG_ONAGG), + "put an onagg txd %#x", txd->txd_flags); + + KASSERTMSG(txd->txd_refs > 0, "invalid txd refs %d", txd->txd_refs); + if (atomic_dec_uint_nv(&txd->txd_refs) != 0) + return; + + if (!STAILQ_EMPTY(&txd->txd_agg_list)) { + while ((tmp_txd = STAILQ_FIRST(&txd->txd_agg_list)) != NULL) { + KASSERTMSG(STAILQ_EMPTY(&tmp_txd->txd_agg_list), + "resursive aggregation on aggregated txdesc"); + KASSERTMSG( + ISSET(tmp_txd->txd_flags, HVN_TXD_FLAG_ONAGG), + "not aggregated txdesc"); + KASSERTMSG( + tmp_txd->txd_chim_index == HVN_NVS_CHIM_IDX_INVALID, + "aggregated txdesc consumes chimney sending " + "buffer: idx %u", tmp_txd->txd_chim_index); + KASSERTMSG(tmp_txd->txd_chim_size == 0, + "aggregated txdesc has non-zero chimney sending " + "size: sz %u", tmp_txd->txd_chim_size); + + STAILQ_REMOVE_HEAD(&txd->txd_agg_list, txd_agg_entry); + CLR(tmp_txd->txd_flags, HVN_TXD_FLAG_ONAGG); + hvn_txd_put(txr, tmp_txd); + } + } + + if (txd->txd_chim_index != HVN_NVS_CHIM_IDX_INVALID) { + KASSERTMSG(!ISSET(txd->txd_flags, HVN_TXD_FLAG_DMAMAP), + "chim txd uses dmamap"); + hvn_chim_free(sc, txd->txd_chim_index); + txd->txd_chim_index = HVN_NVS_CHIM_IDX_INVALID; + txd->txd_chim_size = 0; + } else if (ISSET(txd->txd_flags, HVN_TXD_FLAG_DMAMAP)) { bus_dmamap_sync(sc->sc_dmat, txd->txd_dmap, 0, txd->txd_dmap->dm_mapsize, - BUS_DMASYNC_POSTWRITE); + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->sc_dmat, txd->txd_dmap); - bus_dmamap_destroy(sc->sc_dmat, txd->txd_dmap); - txd->txd_dmap = NULL; - if (txd->txd_buf == NULL) - continue; + CLR(txd->txd_flags, HVN_TXD_FLAG_DMAMAP); + } + + if (txd->txd_buf != NULL) { m_freem(txd->txd_buf); txd->txd_buf = NULL; } - if (sc->sc_tx_rmap != NULL) { - bus_dmamap_sync(sc->sc_dmat, sc->sc_tx_rmap, - 0, sc->sc_tx_rmap->dm_mapsize, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(sc->sc_dmat, sc->sc_tx_rmap); - bus_dmamap_destroy(sc->sc_dmat, sc->sc_tx_rmap); - sc->sc_tx_rmap = NULL; - } - if (sc->sc_tx_msgs != NULL) { - size_t msgsize = roundup(HVN_RNDIS_PKT_LEN, 128); - bus_dmamem_unmap(sc->sc_dmat, sc->sc_tx_msgs, - msgsize * HVN_TX_DESC); - bus_dmamem_free(sc->sc_dmat, &sc->sc_tx_mseg, 1); - sc->sc_tx_msgs = NULL; - } + TAILQ_INSERT_TAIL(&txr->txr_list, txd, txd_entry); + txr->txr_avail++; + txr->txr_oactive = 0; +} + +static void +hvn_txd_gc(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd) +{ + + KASSERTMSG(txd->txd_refs == 0 || txd->txd_refs == 1, + "invalid txd refs %d", txd->txd_refs); + + /* Aggregated txds will be freed by their aggregating txd. */ + if (txd->txd_refs > 0 && !ISSET(txd->txd_flags, HVN_TXD_FLAG_ONAGG)) + hvn_txd_put(txr, txd); +} + +static void +hvn_txd_hold(struct hvn_tx_desc *txd) +{ + + /* 0->1 transition will never work */ + KASSERTMSG(txd->txd_refs > 0, "invalid txd refs %d", txd->txd_refs); + + atomic_inc_uint(&txd->txd_refs); +} + +static void +hvn_txd_agg(struct hvn_tx_desc *agg_txd, struct hvn_tx_desc *txd) +{ + + KASSERTMSG(!ISSET(agg_txd->txd_flags, HVN_TXD_FLAG_ONAGG), + "recursive aggregation on aggregating txdesc"); + KASSERTMSG(!ISSET(txd->txd_flags, HVN_TXD_FLAG_ONAGG), + "already aggregated"); + KASSERTMSG(STAILQ_EMPTY(&txd->txd_agg_list), + "recursive aggregation on to-be-aggregated txdesc"); + + SET(txd->txd_flags, HVN_TXD_FLAG_ONAGG); + STAILQ_INSERT_TAIL(&agg_txd->txd_agg_list, txd, txd_agg_entry); +} + +static int +hvn_tx_ring_pending(struct hvn_tx_ring *txr) +{ + int pending = 0; + + mutex_enter(&txr->txr_lock); + if (hvn_txd_peek(txr) != HVN_TX_DESC) + pending = 1; + mutex_exit(&txr->txr_lock); + + return pending; +} + +static void +hvn_tx_ring_qflush(struct hvn_softc *sc, struct hvn_tx_ring *txr) +{ + struct mbuf *m; + + while ((m = pcq_get(txr->txr_interq)) != NULL) + m_freem(m); } static int @@ -876,19 +2375,817 @@ hvn_get_lladdr(struct hvn_softc *sc, uint8_t *enaddr) } static void -hvn_get_link_status(struct hvn_softc *sc) +hvn_update_link_status(struct hvn_softc *sc) { - uint32_t state; + struct ifnet *ifp = SC2IFP(sc); + uint32_t state, old_link_state; size_t len = sizeof(state); + int rv; + + rv = hvn_rndis_query(sc, OID_GEN_MEDIA_CONNECT_STATUS, &state, &len); + if (rv != 0 || len != sizeof(state)) + return; + + old_link_state = sc->sc_link_state; + sc->sc_link_state = (state == NDIS_MEDIA_STATE_CONNECTED) ? + LINK_STATE_UP : LINK_STATE_DOWN; + if (old_link_state != sc->sc_link_state) { + if_link_state_change(ifp, sc->sc_link_state); + } +} + +static int +hvn_get_mtu(struct hvn_softc *sc, uint32_t *mtu) +{ + size_t mtusz = sizeof(*mtu); + int rv; + + rv = hvn_rndis_query(sc, OID_GEN_MAXIMUM_FRAME_SIZE, mtu, &mtusz); + if (rv == 0 && mtusz != sizeof(*mtu)) + rv = -1; + return rv; +} + +static int +hvn_channel_attach(struct hvn_softc *sc, struct vmbus_channel *chan) +{ + struct hvn_rx_ring *rxr; + struct hvn_tx_ring *txr; + int idx; + + idx = chan->ch_subidx; + if (idx < 0 || idx >= sc->sc_nrxr_inuse) { + DPRINTF("%s: invalid sub-channel %u\n", + device_xname(sc->sc_dev), idx); + return -1; + } + + rxr = &sc->sc_rxr[idx]; + rxr->rxr_chan = chan; + + if (idx < sc->sc_ntxr_inuse) { + txr = &sc->sc_txr[idx]; + txr->txr_chan = chan; + } + + /* Bind this channel to a proper CPU. */ + vmbus_channel_cpu_set(chan, HVN_RING_IDX2CPU(sc, idx)); + + chan->ch_flags &= ~CHF_BATCHED; + + /* Associate our interrupt handler with the channel */ + if (vmbus_channel_open(chan, + HVN_RING_BUFSIZE - sizeof(struct vmbus_bufring), NULL, 0, + hvn_nvs_intr, rxr)) { + DPRINTF("%s: failed to open channel\n", + device_xname(sc->sc_dev)); + return -1; + } + + return 0; +} + +static void +hvn_channel_detach(struct hvn_softc *sc, struct vmbus_channel *chan) +{ + + vmbus_channel_close_direct(chan); +} + +static void +hvn_channel_detach_all(struct hvn_softc *sc) +{ + struct vmbus_channel **subchans; + int i, subchan_cnt = sc->sc_nrxr_inuse - 1; + + if (subchan_cnt > 0) { + /* Detach the sub-channels. */ + subchans = vmbus_subchannel_get(sc->sc_prichan, subchan_cnt); + for (i = 0; i < subchan_cnt; i++) + hvn_channel_detach(sc, subchans[i]); + vmbus_subchannel_rel(subchans, subchan_cnt); + } + + /* + * Detach the primary channel, _after_ all sub-channels + * are detached. + */ + hvn_channel_detach(sc, sc->sc_prichan); + + /* Wait for sub-channels to be destroyed, if any. */ + vmbus_subchannel_drain(sc->sc_prichan); +} + +static int +hvn_subchannel_attach(struct hvn_softc *sc) +{ + struct vmbus_channel **subchans; + int subchan_cnt = sc->sc_nrxr_inuse - 1; + int i, error = 0; + + KASSERTMSG(subchan_cnt > 0, "no sub-channels"); + + /* Attach the sub-channels. */ + subchans = vmbus_subchannel_get(sc->sc_prichan, subchan_cnt); + for (i = 0; i < subchan_cnt; ++i) { + int error1; + + error1 = hvn_channel_attach(sc, subchans[i]); + if (error1) { + error = error1; + /* Move on; all channels will be detached later. */ + } + } + vmbus_subchannel_rel(subchans, subchan_cnt); + + if (error) { + aprint_error_dev(sc->sc_dev, + "sub-channels attach failed: %d\n", error); + return error; + } + + aprint_debug_dev(sc->sc_dev, "%d sub-channels attached\n", + subchan_cnt); + return 0; +} + +static int +hvn_synth_alloc_subchannels(struct hvn_softc *sc, int *nsubch) +{ + struct vmbus_channel **subchans; + int error, nchan, rxr_cnt; + + nchan = *nsubch + 1; + if (nchan < 2) { + /* Multiple RX/TX rings are not requested. */ + *nsubch = 0; + return 0; + } + + /* + * Query RSS capabilities, e.g. # of RX rings, and # of indirect + * table entries. + */ + if (hvn_get_rsscaps(sc, &rxr_cnt)) { + /* No RSS. */ + *nsubch = 0; + return 0; + } + + aprint_debug_dev(sc->sc_dev, "RX rings offered %u, requested %d\n", + rxr_cnt, nchan); + + if (nchan > rxr_cnt) + nchan = rxr_cnt; + if (nchan == 1) { + aprint_debug_dev(sc->sc_dev, + "only 1 channel is supported, no vRSS\n"); + *nsubch = 0; + return 0; + } + + *nsubch = nchan - 1; + error = hvn_nvs_alloc_subchannels(sc, nsubch); + if (error || *nsubch == 0) { + /* Failed to allocate sub-channels. */ + *nsubch = 0; + return 0; + } + + /* + * Wait for all sub-channels to become ready before moving on. + */ + subchans = vmbus_subchannel_get(sc->sc_prichan, *nsubch); + vmbus_subchannel_rel(subchans, *nsubch); + return 0; +} + +static int +hvn_synth_attachable(const struct hvn_softc *sc) +{ +#if 0 + const struct hvn_rx_ring *rxr; + int i; + + for (i = 0; i < sc->sc_nrxr; i++) { + rxr = &sc->sc_rxr[i]; + if (rxr->rxr_flags) + return 0; + } +#endif + return 1; +} + +/* + * Make sure that the RX filter is zero after the successful + * RNDIS initialization. + * + * NOTE: + * Under certain conditions on certain versions of Hyper-V, + * the RNDIS rxfilter is _not_ zero on the hypervisor side + * after the successful RNDIS initialization, which breaks + * the assumption of any following code (well, it breaks the + * RNDIS API contract actually). Clear the RNDIS rxfilter + * explicitly, drain packets sneaking through, and drain the + * interrupt taskqueues scheduled due to the stealth packets. + */ +static void +hvn_init_fixat(struct hvn_softc *sc, int nchan) +{ + + hvn_disable_rx(sc); + hvn_drain_rxtx(sc, nchan); +} + +static void +hvn_set_txagg(struct hvn_softc *sc) +{ + struct hvn_tx_ring *txr; + uint32_t size, pkts; + int i; - if (hvn_rndis_query(sc, OID_GEN_MEDIA_CONNECT_STATUS, - &state, &len) == 0) - sc->sc_link_state = (state == NDIS_MEDIA_STATE_CONNECTED) ? - LINK_STATE_UP : LINK_STATE_DOWN; + /* + * Setup aggregation size. + */ + if (sc->sc_agg_size < 0) + size = UINT32_MAX; + else + size = sc->sc_agg_size; + + if (size > sc->sc_rndis_agg_size) + size = sc->sc_rndis_agg_size; + + /* NOTE: We only aggregate packets using chimney sending buffers. */ + if (size > (uint32_t)sc->sc_chim_szmax) + size = sc->sc_chim_szmax; + + if (size <= 2 * HVN_PKTSIZE_MIN(sc->sc_rndis_agg_align)) { + /* Disable */ + size = 0; + pkts = 0; + goto done; + } + + /* NOTE: Type of the per TX ring setting is 'int'. */ + if (size > INT_MAX) + size = INT_MAX; + + /* + * Setup aggregation packet count. + */ + if (sc->sc_agg_pkts < 0) + pkts = UINT32_MAX; + else + pkts = sc->sc_agg_pkts; + + if (pkts > sc->sc_rndis_agg_pkts) + pkts = sc->sc_rndis_agg_pkts; + + if (pkts <= 1) { + /* Disable */ + size = 0; + pkts = 0; + goto done; + } + + /* NOTE: Type of the per TX ring setting is 'short'. */ + if (pkts > SHRT_MAX) + pkts = SHRT_MAX; + +done: + /* NOTE: Type of the per TX ring setting is 'short'. */ + if (sc->sc_rndis_agg_align > SHRT_MAX) { + /* Disable */ + size = 0; + pkts = 0; + } + + aprint_verbose_dev(sc->sc_dev, + "TX aggregate size %u, pkts %u, align %u\n", + size, pkts, sc->sc_rndis_agg_align); + + for (i = 0; i < sc->sc_ntxr_inuse; ++i) { + txr = &sc->sc_txr[i]; + + mutex_enter(&txr->txr_lock); + txr->txr_agg_szmax = size; + txr->txr_agg_pktmax = pkts; + txr->txr_agg_align = sc->sc_rndis_agg_align; + mutex_exit(&txr->txr_lock); + } } static int -hvn_nvs_attach(struct hvn_softc *sc) +hvn_synth_attach(struct hvn_softc *sc, int mtu) +{ + uint8_t rss_key[RSS_KEYSIZE]; + uint32_t old_caps; + int nchan = 1, nsubch; + int i, error; + + if (!hvn_synth_attachable(sc)) + return ENXIO; + + /* Save capabilities for later verification. */ + old_caps = sc->sc_caps; + sc->sc_caps = 0; + + /* Clear RSS stuffs. */ + sc->sc_rss_ind_size = 0; + sc->sc_rss_hash = 0; + sc->sc_rss_hcap = 0; + + /* + * Attach the primary channel _before_ attaching NVS and RNDIS. + */ + error = hvn_channel_attach(sc, sc->sc_prichan); + if (error) { + aprint_error_dev(sc->sc_dev, + "failed to attach primary channel\n"); + goto failed; + } + + /* + * Attach NVS. + */ + error = hvn_nvs_attach(sc, mtu); + if (error) { + aprint_error_dev(sc->sc_dev, "failed to init NVSP\n"); + goto detach_channel; + } + + /* + * Attach RNDIS _after_ NVS is attached. + */ + error = hvn_rndis_attach(sc, mtu); + if (error) { + aprint_error_dev(sc->sc_dev, "failed to init RNDIS\n"); + goto detach_nvs; + } + + error = hvn_set_capabilities(sc, mtu); + if (error) { + aprint_error_dev(sc->sc_dev, "failed to setup offloading\n"); + goto detach_rndis; + } + + if ((sc->sc_flags & HVN_SCF_ATTACHED) && old_caps != sc->sc_caps) { + device_printf(sc->sc_dev, "caps mismatch " + "old 0x%08x, new 0x%08x\n", old_caps, sc->sc_caps); + error = ENXIO; + goto detach_rndis; + } + + /* + * Allocate sub-channels for multi-TX/RX rings. + * + * NOTE: + * The # of RX rings that can be used is equivalent to the # of + * channels to be requested. + */ + nsubch = sc->sc_nrxr - 1; + error = hvn_synth_alloc_subchannels(sc, &nsubch); + if (error) { + aprint_error_dev(sc->sc_dev, + "failed to allocate sub channels\n"); + goto detach_synth; + } + + /* + * Set the # of TX/RX rings that could be used according to + * the # of channels that NVS offered. + */ + nchan = nsubch + 1; + hvn_set_ring_inuse(sc, nchan); + + if (nchan > 1) { + /* + * Attach the sub-channels. + * + * NOTE: hvn_set_ring_inuse() _must_ have been called. + */ + error = hvn_subchannel_attach(sc); + if (error) { + aprint_error_dev(sc->sc_dev, + "failed to attach sub channels\n"); + goto detach_synth; + } + + /* + * Configure RSS key and indirect table _after_ all sub-channels + * are attached. + */ + if (!(sc->sc_flags & HVN_SCF_HAS_RSSKEY)) { + /* Set the default RSS key. */ + CTASSERT(sizeof(sc->sc_rss.rss_key) == sizeof(rss_key)); + rss_getkey(rss_key); + memcpy(&sc->sc_rss.rss_key, rss_key, + sizeof(sc->sc_rss.rss_key)); + sc->sc_flags |= HVN_SCF_HAS_RSSKEY; + } + + if (!(sc->sc_flags & HVN_SCF_HAS_RSSIND)) { + /* Setup RSS indirect table in round-robin fashion. */ + for (i = 0; i < NDIS_HASH_INDCNT; i++) { + sc->sc_rss.rss_ind[i] = i % nchan; + } + sc->sc_flags |= HVN_SCF_HAS_RSSIND; + } else { + /* + * # of usable channels may be changed, so we have to + * make sure that all entries in RSS indirect table + * are valid. + * + * NOTE: hvn_set_ring_inuse() _must_ have been called. + */ + hvn_fixup_rss_ind(sc); + } + + sc->sc_rss_hash = sc->sc_rss_hcap; + error = hvn_set_rss(sc, NDIS_RSS_FLAG_NONE); + if (error) { + aprint_error_dev(sc->sc_dev, "failed to setup RSS\n"); + goto detach_synth; + } + } + + /* + * Fixup transmission aggregation setup. + */ + hvn_set_txagg(sc); + hvn_init_fixat(sc, nchan); + return 0; + +detach_synth: + hvn_init_fixat(sc, nchan); + hvn_synth_detach(sc); + return error; + +detach_rndis: + hvn_init_fixat(sc, nchan); + hvn_rndis_detach(sc); +detach_nvs: + hvn_nvs_detach(sc); +detach_channel: + hvn_channel_detach(sc, sc->sc_prichan); +failed: + /* Restore old capabilities. */ + sc->sc_caps = old_caps; + return error; +} + +static void +hvn_synth_detach(struct hvn_softc *sc) +{ + + /* Detach the RNDIS first. */ + hvn_rndis_detach(sc); + + /* Detach NVS. */ + hvn_nvs_detach(sc); + + /* Detach all of the channels. */ + hvn_channel_detach_all(sc); + + if (sc->sc_prichan->ch_sc->sc_proto >= VMBUS_VERSION_WIN10 && + sc->sc_rx_hndl) { + /* + * Host is post-Win2016, disconnect RXBUF from primary channel + * here. + */ + vmbus_handle_free(sc->sc_prichan, sc->sc_rx_hndl); + sc->sc_rx_hndl = 0; + } + + if (sc->sc_prichan->ch_sc->sc_proto >= VMBUS_VERSION_WIN10 && + sc->sc_chim_hndl) { + /* + * Host is post-Win2016, disconnect chimney sending buffer + * from primary channel here. + */ + vmbus_handle_free(sc->sc_prichan, sc->sc_chim_hndl); + sc->sc_chim_hndl = 0; + } +} + +static void +hvn_set_ring_inuse(struct hvn_softc *sc, int ring_cnt) +{ + + if (sc->sc_ntxr > ring_cnt) + sc->sc_ntxr_inuse = ring_cnt; + else + sc->sc_ntxr_inuse = sc->sc_ntxr; + sc->sc_nrxr_inuse = ring_cnt; +} + +static void +hvn_channel_drain(struct hvn_softc *sc, struct vmbus_channel *chan) +{ + struct hvn_rx_ring *rxr; + int i, s; + + for (rxr = NULL, i = 0; i < sc->sc_nrxr_inuse; i++) { + rxr = &sc->sc_rxr[i]; + if (rxr->rxr_chan == chan) + break; + } + KASSERT(i < sc->sc_nrxr_inuse); + + /* + * NOTE: + * The TX bufring will not be drained by the hypervisor, + * if the primary channel is revoked. + */ + while (!vmbus_channel_rx_empty(chan) || + (!vmbus_channel_is_revoked(sc->sc_prichan) && + !vmbus_channel_tx_empty(chan))) { + DELAY(20); + s = splnet(); + hvn_nvs_intr1(rxr, sc->sc_tx_process_limit, + sc->sc_rx_process_limit); + splx(s); + } + + mutex_enter(&rxr->rxr_onwork_lock); + while (rxr->rxr_onlist || rxr->rxr_onproc) + cv_wait(&rxr->rxr_onwork_cv, &rxr->rxr_onwork_lock); + mutex_exit(&rxr->rxr_onwork_lock); +} + +static void +hvn_disable_rx(struct hvn_softc *sc) +{ + + /* + * Disable RX by clearing RX filter forcefully. + */ + (void)hvn_rndis_close(sc); /* ignore error */ + + /* + * Give RNDIS enough time to flush all pending data packets. + */ + DELAY(200); +} + +static void +hvn_drain_rxtx(struct hvn_softc *sc, int nchan) +{ + struct vmbus_channel **subchans = NULL; + int i, nsubch; + + /* + * Drain RX/TX bufrings and interrupts. + */ + nsubch = nchan - 1; + if (nsubch > 0) + subchans = vmbus_subchannel_get(sc->sc_prichan, nsubch); + + if (subchans != NULL) { + for (i = 0; i < nsubch; ++i) + hvn_channel_drain(sc, subchans[i]); + } + hvn_channel_drain(sc, sc->sc_prichan); + + if (subchans != NULL) + vmbus_subchannel_rel(subchans, nsubch); +} + +static void +hvn_suspend_data(struct hvn_softc *sc) +{ + struct hvn_tx_ring *txr; + int i, s; + + /* + * Suspend TX. + */ + for (i = 0; i < sc->sc_ntxr_inuse; i++) { + txr = &sc->sc_txr[i]; + + mutex_enter(&txr->txr_lock); + txr->txr_suspended = 1; + mutex_exit(&txr->txr_lock); + /* No one is able send more packets now. */ + + /* + * Wait for all pending sends to finish. + * + * NOTE: + * We will _not_ receive all pending send-done, if the + * primary channel is revoked. + */ + while (hvn_tx_ring_pending(txr) && + !vmbus_channel_is_revoked(sc->sc_prichan)) { + DELAY(20); + s = splnet(); + hvn_nvs_intr1(txr->txr_rxr, sc->sc_tx_process_limit, + sc->sc_rx_process_limit); + splx(s); + } + } + + /* + * Disable RX. + */ + hvn_disable_rx(sc); + + /* + * Drain RX/TX. + */ + hvn_drain_rxtx(sc, sc->sc_nrxr_inuse); +} + +static void +hvn_suspend_mgmt(struct hvn_softc *sc) +{ + + sc->sc_link_suspend = true; + callout_halt(&sc->sc_link_tmout, NULL); + + /* Drain link state task */ + mutex_enter(&sc->sc_link_lock); + for (;;) { + if (!sc->sc_link_onproc) + break; + mutex_exit(&sc->sc_link_lock); + DELAY(20); + mutex_enter(&sc->sc_link_lock); + } + mutex_exit(&sc->sc_link_lock); +} + +static void +hvn_suspend(struct hvn_softc *sc) +{ + struct ifnet *ifp = SC2IFP(sc); + + if (ifp->if_flags & IFF_RUNNING) + hvn_suspend_data(sc); + hvn_suspend_mgmt(sc); +} + +static void +hvn_resume_tx(struct hvn_softc *sc, int ring_cnt) +{ + struct hvn_tx_ring *txr; + int i; + + for (i = 0; i < ring_cnt; i++) { + txr = &sc->sc_txr[i]; + mutex_enter(&txr->txr_lock); + txr->txr_suspended = 0; + mutex_exit(&txr->txr_lock); + } +} + +static void +hvn_resume_data(struct hvn_softc *sc) +{ + struct ifnet *ifp = SC2IFP(sc); + struct hvn_tx_ring *txr; + int i; + + /* + * Re-enable RX. + */ + hvn_rndis_open(sc); + + /* + * Make sure to clear suspend status on "all" TX rings, + * since sc_ntxr_inuse can be changed after hvn_suspend_data(). + */ + hvn_resume_tx(sc, sc->sc_ntxr); + + /* + * Flush unused mbuf, since sc_ntxr_inuse may be reduced. + */ + for (i = sc->sc_ntxr_inuse; i < sc->sc_ntxr; i++) + hvn_tx_ring_qflush(sc, &sc->sc_txr[i]); + + /* + * Kick start TX. + */ + for (i = 0; i < sc->sc_ntxr_inuse; i++) { + txr = &sc->sc_txr[i]; + mutex_enter(&txr->txr_lock); + txr->txr_oactive = 0; + + /* ALTQ */ + if (txr->txr_id == 0) + if_schedule_deferred_start(ifp); + softint_schedule(txr->txr_si); + mutex_exit(&txr->txr_lock); + } +} + +static void +hvn_resume_mgmt(struct hvn_softc *sc) +{ + + sc->sc_link_suspend = false; + hvn_link_event(sc, HVN_LINK_EV_RESUME_NETWORK); +} + +static void +hvn_resume(struct hvn_softc *sc) +{ + struct ifnet *ifp = SC2IFP(sc); + + if (ifp->if_flags & IFF_RUNNING) + hvn_resume_data(sc); + hvn_resume_mgmt(sc); +} + +static int +hvn_nvs_init(struct hvn_softc *sc) +{ + + mutex_init(&sc->sc_nvsrsp_lock, MUTEX_DEFAULT, IPL_NET); + cv_init(&sc->sc_nvsrsp_cv, "nvsrspcv"); + + return 0; +} + +static void +hvn_nvs_destroy(struct hvn_softc *sc) +{ + + mutex_destroy(&sc->sc_nvsrsp_lock); + cv_destroy(&sc->sc_nvsrsp_cv); +} + +static int +hvn_nvs_doinit(struct hvn_softc *sc, uint32_t proto) +{ + struct hvn_nvs_init cmd; + struct hvn_nvs_init_resp *rsp; + uint64_t tid; + int error; + + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_INIT; + cmd.nvs_ver_min = cmd.nvs_ver_max = proto; + + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + error = hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0); + if (error == 0) { + rsp = (struct hvn_nvs_init_resp *)&sc->sc_nvsrsp; + if (rsp->nvs_status != HVN_NVS_STATUS_OK) + error = EINVAL; + } + mutex_exit(&sc->sc_nvsrsp_lock); + + return error; +} + +static int +hvn_nvs_conf_ndis(struct hvn_softc *sc, int mtu) +{ + struct hvn_nvs_ndis_conf cmd; + uint64_t tid; + int error; + + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_NDIS_CONF; + cmd.nvs_mtu = mtu + ETHER_HDR_LEN; + cmd.nvs_caps = HVN_NVS_NDIS_CONF_VLAN; + + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + /* NOTE: No response. */ + error = hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0); + mutex_exit(&sc->sc_nvsrsp_lock); + + if (error == 0) + sc->sc_caps |= HVN_CAPS_MTU | HVN_CAPS_VLAN; + return error; +} + +static int +hvn_nvs_init_ndis(struct hvn_softc *sc) +{ + struct hvn_nvs_ndis_init cmd; + uint64_t tid; + int error; + + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_NDIS_INIT; + cmd.nvs_ndis_major = (sc->sc_ndisver & 0xffff0000) >> 16; + cmd.nvs_ndis_minor = sc->sc_ndisver & 0x0000ffff; + + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + /* NOTE: No response. */ + error = hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0); + mutex_exit(&sc->sc_nvsrsp_lock); + + return error; +} + +static int +hvn_nvs_attach(struct hvn_softc *sc, int mtu) { static const uint32_t protos[] = { HVN_NVS_PROTO_VERSION_5, @@ -896,96 +3193,364 @@ hvn_nvs_attach(struct hvn_softc *sc) HVN_NVS_PROTO_VERSION_2, HVN_NVS_PROTO_VERSION_1 }; - struct hvn_nvs_init cmd; - struct hvn_nvs_init_resp *rsp; - struct hvn_nvs_ndis_init ncmd; - struct hvn_nvs_ndis_conf ccmd; - uint32_t ndisver, ringsize; - uint64_t tid; int i; - sc->sc_nvsbuf = kmem_zalloc(HVN_NVS_BUFSIZE, KM_SLEEP); + if (hyperv_ver_major >= 10) + sc->sc_caps |= HVN_CAPS_UDPHASH; + + /* + * Initialize NVS. + */ + if (sc->sc_flags & HVN_SCF_ATTACHED) { + /* + * NVS version and NDIS version MUST NOT be changed. + */ + DPRINTF("%s: reinit NVS version %#x, NDIS version %u.%u\n", + device_xname(sc->sc_dev), sc->sc_proto, + (sc->sc_ndisver >> 16), sc->sc_ndisver & 0xffff); + + if (hvn_nvs_doinit(sc, sc->sc_proto)) { + DPRINTF("%s: failed to reinit NVSP version %#x\n", + device_xname(sc->sc_dev), sc->sc_proto); + return -1; + } + } else { + /* + * Find the supported NVS version and set NDIS version + * accordingly. + */ + for (i = 0; i < __arraycount(protos); i++) { + if (hvn_nvs_doinit(sc, protos[i]) == 0) + break; + } + if (i == __arraycount(protos)) { + DPRINTF("%s: failed to negotiate NVSP version\n", + device_xname(sc->sc_dev)); + return -1; + } + + sc->sc_proto = protos[i]; + if (sc->sc_proto <= HVN_NVS_PROTO_VERSION_4) + sc->sc_ndisver = NDIS_VERSION_6_1; + else + sc->sc_ndisver = NDIS_VERSION_6_30; - /* We need to be able to fit all RNDIS control and data messages */ - ringsize = HVN_RNDIS_CTLREQS * - (sizeof(struct hvn_nvs_rndis) + sizeof(struct vmbus_gpa)) + - HVN_TX_DESC * (sizeof(struct hvn_nvs_rndis) + - (HVN_TX_FRAGS + 1) * sizeof(struct vmbus_gpa)); + DPRINTF("%s: NVS version %#x, NDIS version %u.%u\n", + device_xname(sc->sc_dev), sc->sc_proto, + (sc->sc_ndisver >> 16), sc->sc_ndisver & 0xffff); + } - sc->sc_chan->ch_flags &= ~CHF_BATCHED; + if (sc->sc_proto >= HVN_NVS_PROTO_VERSION_5) + sc->sc_caps |= HVN_CAPS_HASHVAL; - /* Associate our interrupt handler with the channel */ - if (vmbus_channel_open(sc->sc_chan, ringsize, NULL, 0, - hvn_nvs_intr, sc)) { - DPRINTF("%s: failed to open channel\n", + if (sc->sc_proto >= HVN_NVS_PROTO_VERSION_2) { + /* + * Configure NDIS before initializing it. + */ + if (hvn_nvs_conf_ndis(sc, mtu)) + return -1; + } + + /* + * Initialize NDIS. + */ + if (hvn_nvs_init_ndis(sc)) + return -1; + + /* + * Connect RXBUF. + */ + if (hvn_nvs_connect_rxbuf(sc)) + return -1; + + /* + * Connect chimney sending buffer. + */ + if (hvn_nvs_connect_chim(sc)) + return -1; + + return 0; +} + +static int +hvn_nvs_connect_rxbuf(struct hvn_softc *sc) +{ + struct hvn_nvs_rxbuf_conn cmd; + struct hvn_nvs_rxbuf_conn_resp *rsp; + uint64_t tid; + + if (vmbus_handle_alloc(sc->sc_prichan, &sc->sc_rx_dma, sc->sc_rx_size, + &sc->sc_rx_hndl)) { + DPRINTF("%s: failed to obtain a PA handle\n", device_xname(sc->sc_dev)); - kmem_free(sc->sc_nvsbuf, HVN_NVS_BUFSIZE); return -1; } memset(&cmd, 0, sizeof(cmd)); - cmd.nvs_type = HVN_NVS_TYPE_INIT; - for (i = 0; i < __arraycount(protos); i++) { - cmd.nvs_ver_min = cmd.nvs_ver_max = protos[i]; - tid = atomic_inc_uint_nv(&sc->sc_nvstid); - if (hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 100)) - return -1; + cmd.nvs_type = HVN_NVS_TYPE_RXBUF_CONN; + cmd.nvs_gpadl = sc->sc_rx_hndl; + cmd.nvs_sig = HVN_NVS_RXBUF_SIG; - rsp = (struct hvn_nvs_init_resp *)&sc->sc_nvsrsp; - if (rsp->nvs_status == HVN_NVS_STATUS_OK) { - sc->sc_proto = protos[i]; - break; + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + if (hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0)) + goto errout; + + rsp = (struct hvn_nvs_rxbuf_conn_resp *)&sc->sc_nvsrsp; + if (rsp->nvs_status != HVN_NVS_STATUS_OK) { + DPRINTF("%s: failed to set up the Rx ring\n", + device_xname(sc->sc_dev)); + goto errout; + } + + SET(sc->sc_flags, HVN_SCF_RXBUF_CONNECTED); + + if (rsp->nvs_nsect > 1) { + DPRINTF("%s: invalid number of Rx ring sections: %u\n", + device_xname(sc->sc_dev), rsp->nvs_nsect); + goto errout; + } + mutex_exit(&sc->sc_nvsrsp_lock); + + return 0; + + errout: + mutex_exit(&sc->sc_nvsrsp_lock); + hvn_nvs_disconnect_rxbuf(sc); + return -1; +} + +static int +hvn_nvs_disconnect_rxbuf(struct hvn_softc *sc) +{ + struct hvn_nvs_rxbuf_disconn cmd; + uint64_t tid; + int s, error; + + if (ISSET(sc->sc_flags, HVN_SCF_RXBUF_CONNECTED)) { + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_RXBUF_DISCONN; + cmd.nvs_sig = HVN_NVS_RXBUF_SIG; + + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + error = hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, + HVN_NVS_CMD_NORESP); + if (error) { + device_printf(sc->sc_dev, + "failed to send rxbuf disconn: %d", error); } + CLR(sc->sc_flags, HVN_SCF_RXBUF_CONNECTED); + mutex_exit(&sc->sc_nvsrsp_lock); + + /* + * Wait for the hypervisor to receive this NVS request. + * + * NOTE: + * The TX bufring will not be drained by the hypervisor, + * if the primary channel is revoked. + */ + while (!vmbus_channel_tx_empty(sc->sc_prichan) && + !vmbus_channel_is_revoked(sc->sc_prichan)) { + DELAY(20); + s = splnet(); + hvn_nvs_intr1(&sc->sc_rxr[0], sc->sc_tx_process_limit, + sc->sc_rx_process_limit); + splx(s); + } + /* + * Linger long enough for NVS to disconnect RXBUF. + */ + DELAY(200); + } + + if (sc->sc_prichan->ch_sc->sc_proto < VMBUS_VERSION_WIN10 && + sc->sc_rx_hndl) { + /* + * Disconnect RXBUF from primary channel. + */ + vmbus_handle_free(sc->sc_prichan, sc->sc_rx_hndl); + sc->sc_rx_hndl = 0; } - if (i == __arraycount(protos)) { - DPRINTF("%s: failed to negotiate NVSP version\n", + + return 0; +} + +static int +hvn_nvs_connect_chim(struct hvn_softc *sc) +{ + struct hvn_nvs_chim_conn cmd; + const struct hvn_nvs_chim_conn_resp *rsp; + uint64_t tid; + + mutex_init(&sc->sc_chim_bmap_lock, MUTEX_DEFAULT, IPL_NET); + + /* + * Connect chimney sending buffer GPADL to the primary channel. + * + * NOTE: + * Only primary channel has chimney sending buffer connected to it. + * Sub-channels just share this chimney sending buffer. + */ + if (vmbus_handle_alloc(sc->sc_prichan, &sc->sc_chim_dma, HVN_CHIM_SIZE, + &sc->sc_chim_hndl)) { + DPRINTF("%s: failed to obtain a PA handle for chimney\n", device_xname(sc->sc_dev)); return -1; } - if (sc->sc_proto >= HVN_NVS_PROTO_VERSION_2) { - memset(&ccmd, 0, sizeof(ccmd)); - ccmd.nvs_type = HVN_NVS_TYPE_NDIS_CONF; - ccmd.nvs_mtu = HVN_MAXMTU; - ccmd.nvs_caps = HVN_NVS_NDIS_CONF_VLAN; + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_CHIM_CONN; + cmd.nvs_gpadl = sc->sc_chim_hndl; + cmd.nvs_sig = HVN_NVS_CHIM_SIG; + + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + if (hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0)) + goto errout; + + rsp = (struct hvn_nvs_chim_conn_resp *)&sc->sc_nvsrsp; + if (rsp->nvs_status != HVN_NVS_STATUS_OK) { + DPRINTF("%s: failed to set up chimney sending buffer\n", + device_xname(sc->sc_dev)); + goto errout; + } + + if (rsp->nvs_sectsz == 0 || + (rsp->nvs_sectsz % sizeof(uint32_t)) != 0) { + /* + * Can't use chimney sending buffer; done! + */ + if (rsp->nvs_sectsz == 0) { + device_printf(sc->sc_dev, + "zero chimney sending buffer section size\n"); + } else { + device_printf(sc->sc_dev, + "misaligned chimney sending buffers," + " section size: %d", rsp->nvs_sectsz); + } + sc->sc_chim_szmax = 0; + sc->sc_chim_cnt = 0; + } else { + sc->sc_chim_szmax = rsp->nvs_sectsz; + sc->sc_chim_cnt = HVN_CHIM_SIZE / sc->sc_chim_szmax; + } + + if (sc->sc_chim_szmax > 0) { + if ((HVN_CHIM_SIZE % sc->sc_chim_szmax) != 0) { + device_printf(sc->sc_dev, + "chimney sending sections are not properly " + "aligned\n"); + } + if ((sc->sc_chim_cnt % LONG_BIT) != 0) { + device_printf(sc->sc_dev, + "discard %d chimney sending sections\n", + sc->sc_chim_cnt % LONG_BIT); + } + + sc->sc_chim_bmap_cnt = sc->sc_chim_cnt / LONG_BIT; + sc->sc_chim_bmap = kmem_zalloc(sc->sc_chim_bmap_cnt * + sizeof(u_long), KM_SLEEP); + } + + /* Done! */ + SET(sc->sc_flags, HVN_SCF_CHIM_CONNECTED); + + aprint_verbose_dev(sc->sc_dev, "chimney sending buffer %d/%d\n", + sc->sc_chim_szmax, sc->sc_chim_cnt); + + mutex_exit(&sc->sc_nvsrsp_lock); + + return 0; + +errout: + mutex_exit(&sc->sc_nvsrsp_lock); + hvn_nvs_disconnect_chim(sc); + return -1; +} + +static int +hvn_nvs_disconnect_chim(struct hvn_softc *sc) +{ + struct hvn_nvs_chim_disconn cmd; + uint64_t tid; + int s, error; + + if (ISSET(sc->sc_flags, HVN_SCF_CHIM_CONNECTED)) { + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_CHIM_DISCONN; + cmd.nvs_sig = HVN_NVS_CHIM_SIG; tid = atomic_inc_uint_nv(&sc->sc_nvstid); - if (hvn_nvs_cmd(sc, &ccmd, sizeof(ccmd), tid, 100)) - return -1; + mutex_enter(&sc->sc_nvsrsp_lock); + error = hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, + HVN_NVS_CMD_NORESP); + if (error) { + device_printf(sc->sc_dev, + "failed to send chim disconn: %d", error); + } + CLR(sc->sc_flags, HVN_SCF_CHIM_CONNECTED); + mutex_exit(&sc->sc_nvsrsp_lock); + + /* + * Wait for the hypervisor to receive this NVS request. + * + * NOTE: + * The TX bufring will not be drained by the hypervisor, + * if the primary channel is revoked. + */ + while (!vmbus_channel_tx_empty(sc->sc_prichan) && + !vmbus_channel_is_revoked(sc->sc_prichan)) { + DELAY(20); + s = splnet(); + hvn_nvs_intr1(&sc->sc_rxr[0], sc->sc_tx_process_limit, + sc->sc_rx_process_limit); + splx(s); + } + /* + * Linger long enough for NVS to disconnect chimney + * sending buffer. + */ + DELAY(200); } - memset(&ncmd, 0, sizeof(ncmd)); - ncmd.nvs_type = HVN_NVS_TYPE_NDIS_INIT; - if (sc->sc_proto <= HVN_NVS_PROTO_VERSION_4) - ndisver = NDIS_VERSION_6_1; - else - ndisver = NDIS_VERSION_6_30; - ncmd.nvs_ndis_major = (ndisver & 0xffff0000) >> 16; - ncmd.nvs_ndis_minor = ndisver & 0x0000ffff; + if (sc->sc_prichan->ch_sc->sc_proto < VMBUS_VERSION_WIN10 && + sc->sc_chim_hndl) { + /* + * Disconnect chimney sending buffer from primary channel. + */ + vmbus_handle_free(sc->sc_prichan, sc->sc_chim_hndl); + sc->sc_chim_hndl = 0; + } - tid = atomic_inc_uint_nv(&sc->sc_nvstid); - if (hvn_nvs_cmd(sc, &ncmd, sizeof(ncmd), tid, 100)) - return -1; + if (sc->sc_chim_bmap != NULL) { + kmem_free(sc->sc_chim_bmap, sc->sc_chim_cnt / LONG_BIT); + sc->sc_chim_bmap = NULL; + sc->sc_chim_bmap_cnt = 0; + } - sc->sc_ndisver = ndisver; + mutex_destroy(&sc->sc_chim_bmap_lock); return 0; } -static void -hvn_nvs_intr(void *arg) +#define HVN_HANDLE_RING_DOTX __BIT(0) + +static int +hvn_handle_ring(struct hvn_rx_ring *rxr, int txlimit, int rxlimit) { - struct hvn_softc *sc = arg; - struct ifnet *ifp = SC2IFP(sc); + struct hvn_softc *sc = rxr->rxr_softc; struct vmbus_chanpkt_hdr *cph; const struct hvn_nvs_hdr *nvs; uint64_t rid; uint32_t rlen; + int n, tx = 0, rx = 0; + int result = 0; int rv; - bool dotx = false; + mutex_enter(&rxr->rxr_lock); for (;;) { - rv = vmbus_channel_recv(sc->sc_chan, sc->sc_nvsbuf, + rv = vmbus_channel_recv(rxr->rxr_chan, rxr->rxr_nvsbuf, HVN_NVS_BUFSIZE, &rlen, &rid, 1); if (rv != 0 || rlen == 0) { if (rv != EAGAIN) @@ -993,7 +3558,7 @@ hvn_nvs_intr(void *arg) "failed to receive an NVSP packet\n"); break; } - cph = (struct vmbus_chanpkt_hdr *)sc->sc_nvsbuf; + cph = (struct vmbus_chanpkt_hdr *)rxr->rxr_nvsbuf; nvs = (const struct hvn_nvs_hdr *)VMBUS_CHANPKT_CONST_DATA(cph); if (cph->cph_type == VMBUS_CHANPKT_TYPE_COMP) { @@ -1002,14 +3567,23 @@ hvn_nvs_intr(void *arg) case HVN_NVS_TYPE_RXBUF_CONNRESP: case HVN_NVS_TYPE_CHIM_CONNRESP: case HVN_NVS_TYPE_SUBCH_RESP: + mutex_enter(&sc->sc_nvsrsp_lock); /* copy the response back */ memcpy(&sc->sc_nvsrsp, nvs, HVN_NVS_MSGSIZE); sc->sc_nvsdone = 1; - wakeup(&sc->sc_nvsrsp); + cv_signal(&sc->sc_nvsrsp_cv); + mutex_exit(&sc->sc_nvsrsp_lock); break; case HVN_NVS_TYPE_RNDIS_ACK: - dotx = true; - hvn_txeof(sc, cph->cph_tid); + if (rxr->rxr_txr == NULL) + break; + + result |= HVN_HANDLE_RING_DOTX; + mutex_enter(&rxr->rxr_txr->txr_lock); + hvn_txeof(rxr->rxr_txr, cph->cph_tid); + mutex_exit(&rxr->rxr_txr->txr_lock); + if (txlimit > 0 && ++tx >= txlimit) + goto out; break; default: device_printf(sc->sc_dev, @@ -1020,7 +3594,14 @@ hvn_nvs_intr(void *arg) } else if (cph->cph_type == VMBUS_CHANPKT_TYPE_RXBUF) { switch (nvs->nvs_type) { case HVN_NVS_TYPE_RNDIS: - hvn_rndis_input(sc, cph->cph_tid, cph); + n = hvn_rndis_input(rxr, cph->cph_tid, cph); + if (rxlimit > 0) { + if (n < 0) + goto out; + rx += n; + if (rx >= rxlimit) + goto out; + } break; default: device_printf(sc->sc_dev, @@ -1042,31 +3623,149 @@ hvn_nvs_intr(void *arg) device_printf(sc->sc_dev, "unknown NVSP packet type %u\n", cph->cph_type); } +out: + mutex_exit(&rxr->rxr_lock); + + return result; +} + +static void +hvn_nvs_intr1(struct hvn_rx_ring *rxr, int txlimit, int rxlimit) +{ + struct hvn_softc *sc = rxr->rxr_softc; + struct ifnet *ifp = SC2IFP(sc); + struct hvn_tx_ring *txr = rxr->rxr_txr; + int result; + + rxr->rxr_workqueue = sc->sc_txrx_workqueue; + + result = hvn_handle_ring(rxr, txlimit, rxlimit); + + if ((result & HVN_HANDLE_RING_DOTX) && txr != NULL) { + mutex_enter(&txr->txr_lock); + /* ALTQ */ + if (txr->txr_id == 0) { + ifp->if_flags &= ~IFF_OACTIVE; + if_schedule_deferred_start(ifp); + } + softint_schedule(txr->txr_si); + mutex_exit(&txr->txr_lock); + } +} + +static void +hvn_schedule_handle_ring(struct hvn_softc *sc, struct hvn_rx_ring *rxr, + bool intr) +{ + + KASSERT(mutex_owned(&rxr->rxr_onwork_lock)); + + if (rxr->rxr_workqueue) { + if (!rxr->rxr_onlist) { + rxr->rxr_onlist = true; + if (intr) + rxr->rxr_evdeferreq.ev_count++; + else + rxr->rxr_evredeferreq.ev_count++; + workqueue_enqueue(sc->sc_wq, &rxr->rxr_wk, NULL); + } + } else { + rxr->rxr_onlist = true; + if (intr) + rxr->rxr_evdeferreq.ev_count++; + else + rxr->rxr_evredeferreq.ev_count++; + softint_schedule(rxr->rxr_si); + } +} + +static void +hvn_handle_ring_common(struct hvn_rx_ring *rxr) +{ + struct hvn_softc *sc = rxr->rxr_softc; + int txlimit = sc->sc_tx_process_limit; + int rxlimit = sc->sc_rx_process_limit; + + rxr->rxr_evdefer.ev_count++; + + mutex_enter(&rxr->rxr_onwork_lock); + rxr->rxr_onproc = true; + rxr->rxr_onlist = false; + mutex_exit(&rxr->rxr_onwork_lock); + + hvn_nvs_intr1(rxr, txlimit, rxlimit); + + mutex_enter(&rxr->rxr_onwork_lock); + if (vmbus_channel_unpause(rxr->rxr_chan)) { + vmbus_channel_pause(rxr->rxr_chan); + hvn_schedule_handle_ring(sc, rxr, false); + } + rxr->rxr_onproc = false; + cv_broadcast(&rxr->rxr_onwork_cv); + mutex_exit(&rxr->rxr_onwork_lock); +} + +static void +hvn_handle_ring_work(struct work *wk, void *arg) +{ + struct hvn_rx_ring *rxr = container_of(wk, struct hvn_rx_ring, rxr_wk); - if (dotx) - if_schedule_deferred_start(ifp); + hvn_handle_ring_common(rxr); +} + +static void +hvn_nvs_softintr(void *arg) +{ + struct hvn_rx_ring *rxr = arg; + + hvn_handle_ring_common(rxr); +} + +static void +hvn_nvs_intr(void *arg) +{ + struct hvn_rx_ring *rxr = arg; + struct hvn_softc *sc = rxr->rxr_softc; + int txlimit = cold ? 0 : sc->sc_tx_intr_process_limit; + int rxlimit = cold ? 0 : sc->sc_rx_intr_process_limit; + + rxr->rxr_evintr.ev_count++; + + KASSERT(!rxr->rxr_onproc); + KASSERT(!rxr->rxr_onlist); + + vmbus_channel_pause(rxr->rxr_chan); + + hvn_nvs_intr1(rxr, txlimit, rxlimit); + + if (vmbus_channel_unpause(rxr->rxr_chan) && !cold) { + vmbus_channel_pause(rxr->rxr_chan); + mutex_enter(&rxr->rxr_onwork_lock); + hvn_schedule_handle_ring(sc, rxr, true); + mutex_exit(&rxr->rxr_onwork_lock); + } } static int hvn_nvs_cmd(struct hvn_softc *sc, void *cmd, size_t cmdsize, uint64_t tid, - int timo) + u_int flags) { + struct hvn_rx_ring *rxr = &sc->sc_rxr[0]; /* primary channel */ struct hvn_nvs_hdr *hdr = cmd; int tries = 10; int rv, s; + KASSERT(mutex_owned(&sc->sc_nvsrsp_lock)); + sc->sc_nvsdone = 0; do { - rv = vmbus_channel_send(sc->sc_chan, cmd, cmdsize, + rv = vmbus_channel_send(rxr->rxr_chan, cmd, cmdsize, tid, VMBUS_CHANPKT_TYPE_INBAND, - timo ? VMBUS_CHANPKT_FLAG_RC : 0); + ISSET(flags, HVN_NVS_CMD_NORESP) ? 0 : + VMBUS_CHANPKT_FLAG_RC); if (rv == EAGAIN) { - if (cold) - delay(1000); - else - tsleep(cmd, PRIBIO, "nvsout", - uimax(1, mstohz(1))); + DELAY(1000); } else if (rv) { DPRINTF("%s: NVSP operation %u send error %d\n", device_xname(sc->sc_dev), hdr->nvs_type, rv); @@ -1080,31 +3779,25 @@ hvn_nvs_cmd(struct hvn_softc *sc, void *cmd, size_t cmdsize, uint64_t tid, return rv; } - if (timo == 0) + if (ISSET(flags, HVN_NVS_CMD_NORESP)) return 0; - do { - if (cold) { - delay(1000); - s = splnet(); - hvn_nvs_intr(sc); - splx(s); - } else - tsleep(sc->sc_nvsrsp, PRIBIO | PCATCH, "nvscmd", - uimax(1, mstohz(1))); - } while (--timo > 0 && sc->sc_nvsdone != 1); - - if (timo == 0 && sc->sc_nvsdone != 1) { - device_printf(sc->sc_dev, "NVSP operation %u timed out\n", - hdr->nvs_type); - return ETIMEDOUT; + while (!sc->sc_nvsdone && !ISSET(sc->sc_flags, HVN_SCF_REVOKED)) { + mutex_exit(&sc->sc_nvsrsp_lock); + DELAY(1000); + s = splnet(); + hvn_nvs_intr1(rxr, 0, 0); + splx(s); + mutex_enter(&sc->sc_nvsrsp_lock); } + return 0; } static int -hvn_nvs_ack(struct hvn_softc *sc, uint64_t tid) +hvn_nvs_ack(struct hvn_rx_ring *rxr, uint64_t tid) { + struct hvn_softc *sc __unused = rxr->rxr_softc; struct hvn_nvs_rndis_ack cmd; int tries = 5; int rv; @@ -1112,10 +3805,10 @@ hvn_nvs_ack(struct hvn_softc *sc, uint64_t tid) cmd.nvs_type = HVN_NVS_TYPE_RNDIS_ACK; cmd.nvs_status = HVN_NVS_STATUS_OK; do { - rv = vmbus_channel_send(sc->sc_chan, &cmd, sizeof(cmd), + rv = vmbus_channel_send(rxr->rxr_chan, &cmd, sizeof(cmd), tid, VMBUS_CHANPKT_TYPE_COMP, 0); if (rv == EAGAIN) - delay(10); + DELAY(10); else if (rv) { DPRINTF("%s: NVSP acknowledgement error %d\n", device_xname(sc->sc_dev), rv); @@ -1129,10 +3822,53 @@ static void hvn_nvs_detach(struct hvn_softc *sc) { - if (vmbus_channel_close(sc->sc_chan) == 0) { - kmem_free(sc->sc_nvsbuf, HVN_NVS_BUFSIZE); - sc->sc_nvsbuf = NULL; + hvn_nvs_disconnect_rxbuf(sc); + hvn_nvs_disconnect_chim(sc); +} + +static int +hvn_nvs_alloc_subchannels(struct hvn_softc *sc, int *nsubchp) +{ + struct hvn_nvs_subch_req cmd; + struct hvn_nvs_subch_resp *rsp; + uint64_t tid; + int nsubch, nsubch_req; + + nsubch_req = *nsubchp; + KASSERTMSG(nsubch_req > 0, "invalid # of sub-channels %d", nsubch_req); + + memset(&cmd, 0, sizeof(cmd)); + cmd.nvs_type = HVN_NVS_TYPE_SUBCH_REQ; + cmd.nvs_op = HVN_NVS_SUBCH_OP_ALLOC; + cmd.nvs_nsubch = nsubch_req; + + tid = atomic_inc_uint_nv(&sc->sc_nvstid); + mutex_enter(&sc->sc_nvsrsp_lock); + if (hvn_nvs_cmd(sc, &cmd, sizeof(cmd), tid, 0)) { + mutex_exit(&sc->sc_nvsrsp_lock); + return EIO; + } + + rsp = (struct hvn_nvs_subch_resp *)&sc->sc_nvsrsp; + if (rsp->nvs_status != HVN_NVS_STATUS_OK) { + mutex_exit(&sc->sc_nvsrsp_lock); + DPRINTF("%s: failed to alloc sub-channels\n", + device_xname(sc->sc_dev)); + return EIO; + } + + nsubch = rsp->nvs_nsubch; + if (nsubch > nsubch_req) { + aprint_debug_dev(sc->sc_dev, + "%u subchans are allocated, requested %d\n", + nsubch, nsubch_req); + nsubch = nsubch_req; } + mutex_exit(&sc->sc_nvsrsp_lock); + + *nsubchp = nsubch; + + return 0; } static inline struct rndis_cmd * @@ -1142,9 +3878,7 @@ hvn_alloc_cmd(struct hvn_softc *sc) mutex_enter(&sc->sc_cntl_fqlck); while ((rc = TAILQ_FIRST(&sc->sc_cntl_fq)) == NULL) - /* XXX use condvar(9) instead of mtsleep */ - mtsleep(&sc->sc_cntl_fq, PRIBIO, "nvsalloc", 1, - &sc->sc_cntl_fqlck); + cv_wait(&sc->sc_cntl_fqcv, &sc->sc_cntl_fqlck); TAILQ_REMOVE(&sc->sc_cntl_fq, rc, rc_entry); mutex_exit(&sc->sc_cntl_fqlck); return rc; @@ -1215,18 +3949,15 @@ hvn_free_cmd(struct hvn_softc *sc, struct rndis_cmd *rc) memset(&rc->rc_msg, 0, sizeof(rc->rc_msg)); mutex_enter(&sc->sc_cntl_fqlck); TAILQ_INSERT_TAIL(&sc->sc_cntl_fq, rc, rc_entry); + cv_signal(&sc->sc_cntl_fqcv); mutex_exit(&sc->sc_cntl_fqlck); - wakeup(&sc->sc_cntl_fq); } static int -hvn_rndis_attach(struct hvn_softc *sc) +hvn_rndis_init(struct hvn_softc *sc) { - const int dmaflags = cold ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK; - struct rndis_init_req *req; - struct rndis_init_comp *cmp; struct rndis_cmd *rc; - int i, rv; + int i; /* RNDIS control message queues */ TAILQ_INIT(&sc->sc_cntl_sq); @@ -1235,24 +3966,25 @@ hvn_rndis_attach(struct hvn_softc *sc) mutex_init(&sc->sc_cntl_sqlck, MUTEX_DEFAULT, IPL_NET); mutex_init(&sc->sc_cntl_cqlck, MUTEX_DEFAULT, IPL_NET); mutex_init(&sc->sc_cntl_fqlck, MUTEX_DEFAULT, IPL_NET); + cv_init(&sc->sc_cntl_fqcv, "nvsalloc"); for (i = 0; i < HVN_RNDIS_CTLREQS; i++) { rc = &sc->sc_cntl_msgs[i]; if (bus_dmamap_create(sc->sc_dmat, PAGE_SIZE, 1, PAGE_SIZE, 0, - dmaflags, &rc->rc_dmap)) { + BUS_DMA_WAITOK, &rc->rc_dmap)) { DPRINTF("%s: failed to create RNDIS command map\n", device_xname(sc->sc_dev)); goto errout; } if (bus_dmamem_alloc(sc->sc_dmat, PAGE_SIZE, PAGE_SIZE, - 0, &rc->rc_segs, 1, &rc->rc_nsegs, dmaflags)) { + 0, &rc->rc_segs, 1, &rc->rc_nsegs, BUS_DMA_WAITOK)) { DPRINTF("%s: failed to allocate RNDIS command\n", device_xname(sc->sc_dev)); bus_dmamap_destroy(sc->sc_dmat, rc->rc_dmap); goto errout; } if (bus_dmamem_map(sc->sc_dmat, &rc->rc_segs, rc->rc_nsegs, - PAGE_SIZE, (void **)&rc->rc_req, dmaflags)) { + PAGE_SIZE, (void **)&rc->rc_req, BUS_DMA_WAITOK)) { DPRINTF("%s: failed to allocate RNDIS command\n", device_xname(sc->sc_dev)); bus_dmamem_free(sc->sc_dmat, &rc->rc_segs, @@ -1262,18 +3994,67 @@ hvn_rndis_attach(struct hvn_softc *sc) } memset(rc->rc_req, 0, PAGE_SIZE); if (bus_dmamap_load(sc->sc_dmat, rc->rc_dmap, rc->rc_req, - PAGE_SIZE, NULL, dmaflags)) { + PAGE_SIZE, NULL, BUS_DMA_WAITOK)) { DPRINTF("%s: failed to load RNDIS command map\n", device_xname(sc->sc_dev)); bus_dmamem_free(sc->sc_dmat, &rc->rc_segs, rc->rc_nsegs); bus_dmamap_destroy(sc->sc_dmat, rc->rc_dmap); + rc->rc_req = NULL; goto errout; } rc->rc_gpa = atop(rc->rc_dmap->dm_segs[0].ds_addr); + mutex_init(&rc->rc_lock, MUTEX_DEFAULT, IPL_NET); + cv_init(&rc->rc_cv, "rndiscmd"); TAILQ_INSERT_TAIL(&sc->sc_cntl_fq, rc, rc_entry); } + /* Initialize RNDIS Data command */ + memset(&sc->sc_data_msg, 0, sizeof(sc->sc_data_msg)); + sc->sc_data_msg.nvs_type = HVN_NVS_TYPE_RNDIS; + sc->sc_data_msg.nvs_rndis_mtype = HVN_NVS_RNDIS_MTYPE_DATA; + sc->sc_data_msg.nvs_chim_idx = HVN_NVS_CHIM_IDX_INVALID; + + return 0; + +errout: + hvn_rndis_destroy(sc); + return -1; +} + +static void +hvn_rndis_destroy(struct hvn_softc *sc) +{ + struct rndis_cmd *rc; + int i; + + for (i = 0; i < HVN_RNDIS_CTLREQS; i++) { + rc = &sc->sc_cntl_msgs[i]; + if (rc->rc_req == NULL) + continue; + + TAILQ_REMOVE(&sc->sc_cntl_fq, rc, rc_entry); + bus_dmamem_free(sc->sc_dmat, &rc->rc_segs, rc->rc_nsegs); + rc->rc_req = NULL; + bus_dmamap_destroy(sc->sc_dmat, rc->rc_dmap); + mutex_destroy(&rc->rc_lock); + cv_destroy(&rc->rc_cv); + } + + mutex_destroy(&sc->sc_cntl_sqlck); + mutex_destroy(&sc->sc_cntl_cqlck); + mutex_destroy(&sc->sc_cntl_fqlck); + cv_destroy(&sc->sc_cntl_fqcv); +} + +static int +hvn_rndis_attach(struct hvn_softc *sc, int mtu) +{ + struct rndis_init_req *req; + struct rndis_init_comp *cmp; + struct rndis_cmd *rc; + int rv; + rc = hvn_alloc_cmd(sc); bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, @@ -1294,49 +4075,326 @@ hvn_rndis_attach(struct hvn_softc *sc) bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, BUS_DMASYNC_PREWRITE); - if ((rv = hvn_rndis_cmd(sc, rc, 500)) != 0) { + if ((rv = hvn_rndis_cmd(sc, rc, 0)) != 0) { DPRINTF("%s: INITIALIZE_MSG failed, error %d\n", device_xname(sc->sc_dev), rv); hvn_free_cmd(sc, rc); - goto errout; + return -1; } cmp = (struct rndis_init_comp *)&rc->rc_cmp; if (cmp->rm_status != RNDIS_STATUS_SUCCESS) { DPRINTF("%s: failed to init RNDIS, error %#x\n", device_xname(sc->sc_dev), cmp->rm_status); hvn_free_cmd(sc, rc); - goto errout; + return -1; } + sc->sc_rndis_agg_size = cmp->rm_pktmaxsz; + sc->sc_rndis_agg_pkts = cmp->rm_pktmaxcnt; + sc->sc_rndis_agg_align = __BIT(cmp->rm_align); + + if (sc->sc_rndis_agg_align < sizeof(uint32_t)) { + /* + * The RNDIS packet messsage encap assumes that the RNDIS + * packet message is at least 4 bytes aligned. Fix up the + * alignment here, if the remote side sets the alignment + * too low. + */ + aprint_verbose_dev(sc->sc_dev, + "fixup RNDIS aggpkt align: %u -> %zu\n", + sc->sc_rndis_agg_align, sizeof(uint32_t)); + sc->sc_rndis_agg_align = sizeof(uint32_t); + } + + aprint_verbose_dev(sc->sc_dev, + "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u\n", + cmp->rm_ver_major, cmp->rm_ver_minor, sc->sc_rndis_agg_size, + sc->sc_rndis_agg_pkts, sc->sc_rndis_agg_align); + hvn_free_cmd(sc, rc); - /* Initialize RNDIS Data command */ - memset(&sc->sc_data_msg, 0, sizeof(sc->sc_data_msg)); - sc->sc_data_msg.nvs_type = HVN_NVS_TYPE_RNDIS; - sc->sc_data_msg.nvs_rndis_mtype = HVN_NVS_RNDIS_MTYPE_DATA; - sc->sc_data_msg.nvs_chim_idx = HVN_NVS_CHIM_IDX_INVALID; + return 0; +} + +static int +hvn_get_rsscaps(struct hvn_softc *sc, int *nrxr) +{ + struct ndis_rss_caps in, caps; + size_t caps_len; + int error, rxr_cnt, indsz, hash_fnidx; + uint32_t hash_func = 0, hash_types = 0; + + *nrxr = 0; + + if (sc->sc_ndisver < NDIS_VERSION_6_20) + return EOPNOTSUPP; + + memset(&in, 0, sizeof(in)); + in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; + in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; + in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; + + caps_len = NDIS_RSS_CAPS_SIZE; + error = hvn_rndis_query2(sc, OID_GEN_RECEIVE_SCALE_CAPABILITIES, + &in, NDIS_RSS_CAPS_SIZE, &caps, &caps_len, NDIS_RSS_CAPS_SIZE_6_0); + if (error) + return error; + + /* + * Preliminary verification. + */ + if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { + DPRINTF("%s: invalid NDIS objtype 0x%02x\n", + device_xname(sc->sc_dev), caps.ndis_hdr.ndis_type); + return EINVAL; + } + if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { + DPRINTF("%s: invalid NDIS objrev 0x%02x\n", + device_xname(sc->sc_dev), caps.ndis_hdr.ndis_rev); + return EINVAL; + } + if (caps.ndis_hdr.ndis_size > caps_len) { + DPRINTF("%s: invalid NDIS objsize %u, data size %zu\n", + device_xname(sc->sc_dev), caps.ndis_hdr.ndis_size, + caps_len); + return EINVAL; + } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { + DPRINTF("%s: invalid NDIS objsize %u\n", + device_xname(sc->sc_dev), caps.ndis_hdr.ndis_size); + return EINVAL; + } + + /* + * Save information for later RSS configuration. + */ + if (caps.ndis_nrxr == 0) { + DPRINTF("%s: 0 RX rings!?\n", device_xname(sc->sc_dev)); + return EINVAL; + } + rxr_cnt = caps.ndis_nrxr; + aprint_debug_dev(sc->sc_dev, "%u Rx rings\n", rxr_cnt); + + if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && + caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { + if (caps.ndis_nind > NDIS_HASH_INDCNT) { + DPRINTF("%s: too many RSS indirect table entries %u\n", + device_xname(sc->sc_dev), caps.ndis_nind); + return EOPNOTSUPP; + } + if (!powerof2(caps.ndis_nind)) { + DPRINTF("%s: RSS indirect table size is not power-of-2:" + " %u\n", device_xname(sc->sc_dev), caps.ndis_nind); + return EOPNOTSUPP; + } + + indsz = caps.ndis_nind; + } else { + indsz = NDIS_HASH_INDCNT; + } + if (rxr_cnt > indsz) { + aprint_debug_dev(sc->sc_dev, + "# of RX rings (%u) > RSS indirect table size %u\n", + rxr_cnt, indsz); + rxr_cnt = indsz; + } + + /* + * NOTE: + * Toeplitz is at the lowest bit, and it is prefered; so ffs(), + * instead of fls(), is used here. + */ + hash_fnidx = ffs(caps.ndis_caps & NDIS_RSS_CAP_HASHFUNC_MASK); + if (hash_fnidx == 0) { + DPRINTF("%s: no hash functions, caps 0x%08x\n", + device_xname(sc->sc_dev), caps.ndis_caps); + return EOPNOTSUPP; + } + hash_func = 1 << (hash_fnidx - 1); /* ffs is 1-based */ + + if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) + hash_types |= NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4; + if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) + hash_types |= NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6; + if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) + hash_types |= NDIS_HASH_IPV6_EX | NDIS_HASH_TCP_IPV6_EX; + if (hash_types == 0) { + DPRINTF("%s: no hash types, caps 0x%08x\n", + device_xname(sc->sc_dev), caps.ndis_caps); + return EOPNOTSUPP; + } + aprint_debug_dev(sc->sc_dev, "RSS caps %#x\n", caps.ndis_caps); + + sc->sc_rss_ind_size = indsz; + sc->sc_rss_hcap = hash_func | hash_types; + if (sc->sc_caps & HVN_CAPS_UDPHASH) { + /* UDP 4-tuple hash is unconditionally enabled. */ + sc->sc_rss_hcap |= NDIS_HASH_UDP_IPV4_X; + } + *nrxr = rxr_cnt; return 0; +} -errout: - for (i = 0; i < HVN_RNDIS_CTLREQS; i++) { - rc = &sc->sc_cntl_msgs[i]; - if (rc->rc_req == NULL) - continue; - TAILQ_REMOVE(&sc->sc_cntl_fq, rc, rc_entry); - bus_dmamem_free(sc->sc_dmat, &rc->rc_segs, rc->rc_nsegs); - rc->rc_req = NULL; - bus_dmamap_destroy(sc->sc_dmat, rc->rc_dmap); +static int +hvn_set_rss(struct hvn_softc *sc, uint16_t flags) +{ + struct ndis_rssprm_toeplitz *rss = &sc->sc_rss; + struct ndis_rss_params *params = &rss->rss_params; + int len; + + /* + * Only NDIS 6.20+ is supported: + * We only support 4bytes element in indirect table, which has been + * adopted since NDIS 6.20. + */ + if (sc->sc_ndisver < NDIS_VERSION_6_20) + return 0; + + /* XXX only one can be specified through, popcnt? */ + KASSERTMSG((sc->sc_rss_hash & NDIS_HASH_FUNCTION_MASK), + "no hash func %08x", sc->sc_rss_hash); + KASSERTMSG((sc->sc_rss_hash & NDIS_HASH_STD), + "no standard hash types %08x", sc->sc_rss_hash); + KASSERTMSG(sc->sc_rss_ind_size > 0, "no indirect table size"); + + aprint_debug_dev(sc->sc_dev, "RSS indirect table size %d, hash %#x\n", + sc->sc_rss_ind_size, sc->sc_rss_hash); + + len = NDIS_RSSPRM_TOEPLITZ_SIZE(sc->sc_rss_ind_size); + + memset(params, 0, sizeof(*params)); + params->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; + params->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; + params->ndis_hdr.ndis_size = len; + params->ndis_flags = flags; + params->ndis_hash = + sc->sc_rss_hash & (NDIS_HASH_FUNCTION_MASK | NDIS_HASH_STD); + params->ndis_indsize = sizeof(rss->rss_ind[0]) * sc->sc_rss_ind_size; + params->ndis_indoffset = + offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); + params->ndis_keysize = sizeof(rss->rss_key); + params->ndis_keyoffset = + offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); + + return hvn_rndis_set(sc, OID_GEN_RECEIVE_SCALE_PARAMETERS, rss, len); +} + +static void +hvn_fixup_rss_ind(struct hvn_softc *sc) +{ + struct ndis_rssprm_toeplitz *rss = &sc->sc_rss; + int i, nchan; + + nchan = sc->sc_nrxr_inuse; + KASSERTMSG(nchan > 1, "invalid # of channels %d", nchan); + + /* + * Check indirect table to make sure that all channels in it + * can be used. + */ + for (i = 0; i < NDIS_HASH_INDCNT; i++) { + if (rss->rss_ind[i] >= nchan) { + DPRINTF("%s: RSS indirect table %d fixup: %u -> %d\n", + device_xname(sc->sc_dev), i, rss->rss_ind[i], + nchan - 1); + rss->rss_ind[i] = nchan - 1; + } } - return -1; } static int -hvn_set_capabilities(struct hvn_softc *sc) +hvn_get_hwcaps(struct hvn_softc *sc, struct ndis_offload *caps) +{ + struct ndis_offload in; + size_t caps_len, len; + int error; + + memset(&in, 0, sizeof(in)); + in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; + if (sc->sc_ndisver >= NDIS_VERSION_6_30) { + in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; + len = in.ndis_hdr.ndis_size = NDIS_OFFLOAD_SIZE; + } else if (sc->sc_ndisver >= NDIS_VERSION_6_1) { + in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; + len = in.ndis_hdr.ndis_size = NDIS_OFFLOAD_SIZE_6_1; + } else { + in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; + len = in.ndis_hdr.ndis_size = NDIS_OFFLOAD_SIZE_6_0; + } + + caps_len = NDIS_OFFLOAD_SIZE; + error = hvn_rndis_query2(sc, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, + &in, len, caps, &caps_len, NDIS_OFFLOAD_SIZE_6_0); + if (error) + return error; + + /* + * Preliminary verification. + */ + if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { + DPRINTF("%s: invalid NDIS objtype 0x%02x\n", + device_xname(sc->sc_dev), caps->ndis_hdr.ndis_type); + return EINVAL; + } + if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { + DPRINTF("%s: invalid NDIS objrev 0x%02x\n", + device_xname(sc->sc_dev), caps->ndis_hdr.ndis_rev); + return EINVAL; + } + if (caps->ndis_hdr.ndis_size > caps_len) { + DPRINTF("%s: invalid NDIS objsize %u, data size %zu\n", + device_xname(sc->sc_dev), caps->ndis_hdr.ndis_size, + caps_len); + return EINVAL; + } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { + DPRINTF("%s: invalid NDIS objsize %u\n", + device_xname(sc->sc_dev), caps->ndis_hdr.ndis_size); + return EINVAL; + } + + /* + * NOTE: + * caps->ndis_hdr.ndis_size MUST be checked before accessing + * NDIS 6.1+ specific fields. + */ + aprint_debug_dev(sc->sc_dev, "hwcaps rev %u\n", + caps->ndis_hdr.ndis_rev); + + aprint_debug_dev(sc->sc_dev, "hwcaps csum: " + "ip4 tx 0x%x/0x%x rx 0x%x/0x%x, " + "ip6 tx 0x%x/0x%x rx 0x%x/0x%x\n", + caps->ndis_csum.ndis_ip4_txcsum, caps->ndis_csum.ndis_ip4_txenc, + caps->ndis_csum.ndis_ip4_rxcsum, caps->ndis_csum.ndis_ip4_rxenc, + caps->ndis_csum.ndis_ip6_txcsum, caps->ndis_csum.ndis_ip6_txenc, + caps->ndis_csum.ndis_ip6_rxcsum, caps->ndis_csum.ndis_ip6_rxenc); + aprint_debug_dev(sc->sc_dev, "hwcaps lsov2: " + "ip4 maxsz %u minsg %u encap 0x%x, " + "ip6 maxsz %u minsg %u encap 0x%x opts 0x%x\n", + caps->ndis_lsov2.ndis_ip4_maxsz, caps->ndis_lsov2.ndis_ip4_minsg, + caps->ndis_lsov2.ndis_ip4_encap, caps->ndis_lsov2.ndis_ip6_maxsz, + caps->ndis_lsov2.ndis_ip6_minsg, caps->ndis_lsov2.ndis_ip6_encap, + caps->ndis_lsov2.ndis_ip6_opts); + + return 0; +} + +static int +hvn_set_capabilities(struct hvn_softc *sc, int mtu) { + struct ndis_offload hwcaps; struct ndis_offload_params params; - size_t len = sizeof(params); + size_t len; + uint32_t caps = 0; + int error, tso_maxsz, tso_minsg; + + error = hvn_get_hwcaps(sc, &hwcaps); + if (error) { + DPRINTF("%s: failed to query hwcaps\n", + device_xname(sc->sc_dev)); + return error; + } + /* NOTE: 0 means "no change" */ memset(¶ms, 0, sizeof(params)); params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; @@ -1348,28 +4406,149 @@ hvn_set_capabilities(struct hvn_softc *sc) len = params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; } - params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TXRX; - params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TXRX; - params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TXRX; - if (sc->sc_ndisver >= NDIS_VERSION_6_30) { - params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TXRX; - params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TXRX; + /* + * TSO4/TSO6 setup. + */ + tso_maxsz = IP_MAXPACKET; + tso_minsg = 2; + if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) { + caps |= HVN_CAPS_TSO4; + params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; + + if (hwcaps.ndis_lsov2.ndis_ip4_maxsz < tso_maxsz) + tso_maxsz = hwcaps.ndis_lsov2.ndis_ip4_maxsz; + if (hwcaps.ndis_lsov2.ndis_ip4_minsg > tso_minsg) + tso_minsg = hwcaps.ndis_lsov2.ndis_ip4_minsg; + } + if ((hwcaps.ndis_lsov2.ndis_ip6_encap & NDIS_OFFLOAD_ENCAP_8023) && + (hwcaps.ndis_lsov2.ndis_ip6_opts & HVN_NDIS_LSOV2_CAP_IP6) == + HVN_NDIS_LSOV2_CAP_IP6) { + caps |= HVN_CAPS_TSO6; + params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; + + if (hwcaps.ndis_lsov2.ndis_ip6_maxsz < tso_maxsz) + tso_maxsz = hwcaps.ndis_lsov2.ndis_ip6_maxsz; + if (hwcaps.ndis_lsov2.ndis_ip6_minsg > tso_minsg) + tso_minsg = hwcaps.ndis_lsov2.ndis_ip6_minsg; + } + sc->sc_tso_szmax = 0; + sc->sc_tso_sgmin = 0; + if (caps & (HVN_CAPS_TSO4 | HVN_CAPS_TSO6)) { + KASSERTMSG(tso_maxsz <= IP_MAXPACKET, + "invalid NDIS TSO maxsz %d", tso_maxsz); + KASSERTMSG(tso_minsg >= 2, + "invalid NDIS TSO minsg %d", tso_minsg); + if (tso_maxsz < tso_minsg * mtu) { + DPRINTF("%s: invalid NDIS TSO config: " + "maxsz %d, minsg %d, mtu %d; " + "disable TSO4 and TSO6\n", device_xname(sc->sc_dev), + tso_maxsz, tso_minsg, mtu); + caps &= ~(HVN_CAPS_TSO4 | HVN_CAPS_TSO6); + params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_OFF; + params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_OFF; + } else { + sc->sc_tso_szmax = tso_maxsz; + sc->sc_tso_sgmin = tso_minsg; + aprint_debug_dev(sc->sc_dev, + "NDIS TSO szmax %d sgmin %d\n", + sc->sc_tso_szmax, sc->sc_tso_sgmin); + } + } + + /* IPv4 checksum */ + if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HVN_NDIS_TXCSUM_CAP_IP4) == + HVN_NDIS_TXCSUM_CAP_IP4) { + caps |= HVN_CAPS_IPCS; + params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; + } + if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) { + if (params.ndis_ip4csum == NDIS_OFFLOAD_PARAM_TX) + params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TXRX; + else + params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_RX; + } + + /* TCP4 checksum */ + if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HVN_NDIS_TXCSUM_CAP_TCP4) == + HVN_NDIS_TXCSUM_CAP_TCP4) { + caps |= HVN_CAPS_TCP4CS; + params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; + } + if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) { + if (params.ndis_tcp4csum == NDIS_OFFLOAD_PARAM_TX) + params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TXRX; + else + params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_RX; + } + + /* UDP4 checksum */ + if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) { + caps |= HVN_CAPS_UDP4CS; + params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; + } + if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) { + if (params.ndis_udp4csum == NDIS_OFFLOAD_PARAM_TX) + params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TXRX; + else + params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_RX; + } + + /* TCP6 checksum */ + if ((hwcaps.ndis_csum.ndis_ip6_txcsum & HVN_NDIS_TXCSUM_CAP_TCP6) == + HVN_NDIS_TXCSUM_CAP_TCP6) { + caps |= HVN_CAPS_TCP6CS; + params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; + } + if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) { + if (params.ndis_tcp6csum == NDIS_OFFLOAD_PARAM_TX) + params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TXRX; + else + params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_RX; } - return hvn_rndis_set(sc, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, len); + /* UDP6 checksum */ + if ((hwcaps.ndis_csum.ndis_ip6_txcsum & HVN_NDIS_TXCSUM_CAP_UDP6) == + HVN_NDIS_TXCSUM_CAP_UDP6) { + caps |= HVN_CAPS_UDP6CS; + params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; + } + if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) { + if (params.ndis_udp6csum == NDIS_OFFLOAD_PARAM_TX) + params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TXRX; + else + params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_RX; + } + + aprint_debug_dev(sc->sc_dev, "offload csum: " + "ip4 %u, tcp4 %u, udp4 %u, tcp6 %u, udp6 %u\n", + params.ndis_ip4csum, params.ndis_tcp4csum, params.ndis_udp4csum, + params.ndis_tcp6csum, params.ndis_udp6csum); + aprint_debug_dev(sc->sc_dev, "offload lsov2: ip4 %u, ip6 %u\n", + params.ndis_lsov2_ip4, params.ndis_lsov2_ip6); + + error = hvn_rndis_set(sc, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, len); + if (error) { + DPRINTF("%s: offload config failed: %d\n", + device_xname(sc->sc_dev), error); + return error; + } + + aprint_debug_dev(sc->sc_dev, "offload config done\n"); + sc->sc_caps |= caps; + + return 0; } static int -hvn_rndis_cmd(struct hvn_softc *sc, struct rndis_cmd *rc, int timo) +hvn_rndis_cmd(struct hvn_softc *sc, struct rndis_cmd *rc, u_int flags) { + struct hvn_rx_ring *rxr = &sc->sc_rxr[0]; /* primary channel */ struct hvn_nvs_rndis *msg = &rc->rc_msg; struct rndis_msghdr *hdr = rc->rc_req; struct vmbus_gpa sgl[1]; int tries = 10; int rv, s; - KASSERT(timo > 0); - msg->nvs_type = HVN_NVS_TYPE_RNDIS; msg->nvs_rndis_mtype = HVN_NVS_RNDIS_MTYPE_CTRL; msg->nvs_chim_idx = HVN_NVS_CHIM_IDX_INVALID; @@ -1380,18 +4559,17 @@ hvn_rndis_cmd(struct hvn_softc *sc, struct rndis_cmd *rc, int timo) rc->rc_done = 0; + mutex_enter(&rc->rc_lock); + hvn_submit_cmd(sc, rc); do { - rv = vmbus_channel_send_sgl(sc->sc_chan, sgl, 1, &rc->rc_msg, + rv = vmbus_channel_send_sgl(rxr->rxr_chan, sgl, 1, &rc->rc_msg, sizeof(*msg), rc->rc_id); if (rv == EAGAIN) { - if (cold) - delay(1000); - else - tsleep(rc, PRIBIO, "rndisout", - uimax(1, mstohz(1))); + DELAY(1000); } else if (rv) { + mutex_exit(&rc->rc_lock); DPRINTF("%s: RNDIS operation %u send error %d\n", device_xname(sc->sc_dev), hdr->rm_type, rv); hvn_rollback_cmd(sc, rc); @@ -1400,40 +4578,42 @@ hvn_rndis_cmd(struct hvn_softc *sc, struct rndis_cmd *rc, int timo) } while (rv != 0 && --tries > 0); if (tries == 0 && rv != 0) { + mutex_exit(&rc->rc_lock); device_printf(sc->sc_dev, "RNDIS operation %u send error %d\n", hdr->rm_type, rv); + hvn_rollback_cmd(sc, rc); return rv; } - if (vmbus_channel_is_revoked(sc->sc_chan)) { + if (vmbus_channel_is_revoked(rxr->rxr_chan) || + ISSET(flags, HVN_RNDIS_CMD_NORESP)) { /* No response */ + mutex_exit(&rc->rc_lock); + if (hvn_rollback_cmd(sc, rc)) + hvn_release_cmd(sc, rc); return 0; } bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, BUS_DMASYNC_POSTWRITE); - do { - if (cold) { - delay(1000); - s = splnet(); - hvn_nvs_intr(sc); - splx(s); - } else - tsleep(rc, PRIBIO | PCATCH, "rndiscmd", - uimax(1, mstohz(1))); - } while (--timo > 0 && rc->rc_done != 1); + while (!rc->rc_done && !ISSET(sc->sc_flags, HVN_SCF_REVOKED)) { + mutex_exit(&rc->rc_lock); + DELAY(1000); + s = splnet(); + hvn_nvs_intr1(rxr, 0, 0); + splx(s); + mutex_enter(&rc->rc_lock); + } + mutex_exit(&rc->rc_lock); bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, BUS_DMASYNC_POSTREAD); - if (rc->rc_done != 1) { - rv = timo == 0 ? ETIMEDOUT : EINTR; + if (!rc->rc_done) { + rv = EINTR; if (hvn_rollback_cmd(sc, rc)) { hvn_release_cmd(sc, rc); rv = 0; - } else if (rv == ETIMEDOUT) { - device_printf(sc->sc_dev, - "RNDIS operation %u timed out\n", hdr->rm_type); } return rv; } @@ -1442,16 +4622,18 @@ hvn_rndis_cmd(struct hvn_softc *sc, struct rndis_cmd *rc, int timo) return 0; } -static void -hvn_rndis_input(struct hvn_softc *sc, uint64_t tid, void *arg) +static int +hvn_rndis_input(struct hvn_rx_ring *rxr, uint64_t tid, void *arg) { + struct hvn_softc *sc = rxr->rxr_softc; struct vmbus_chanpkt_prplist *cp = arg; uint32_t off, len, type; - int i; + int i, rv, rx = 0; + bool qfull = false; if (sc->sc_rx_ring == NULL) { DPRINTF("%s: invalid rx ring\n", device_xname(sc->sc_dev)); - return; + return 0; } for (i = 0; i < cp->cp_range_cnt; i++) { @@ -1465,7 +4647,11 @@ hvn_rndis_input(struct hvn_softc *sc, uint64_t tid, void *arg) switch (type) { /* data message */ case REMOTE_NDIS_PACKET_MSG: - hvn_rxeof(sc, sc->sc_rx_ring + off, len); + rv = hvn_rxeof(rxr, sc->sc_rx_ring + off, len); + if (rv == 1) + rx++; + else if (rv == -1) /* The receive queue is full. */ + qfull = true; break; /* completion messages */ case REMOTE_NDIS_INITIALIZE_CMPLT: @@ -1486,7 +4672,11 @@ hvn_rndis_input(struct hvn_softc *sc, uint64_t tid, void *arg) } } - hvn_nvs_ack(sc, tid); + hvn_nvs_ack(rxr, tid); + + if (qfull) + return -1; + return rx; } static inline struct mbuf * @@ -1494,7 +4684,7 @@ hvn_devget(struct hvn_softc *sc, void *buf, uint32_t len) { struct ifnet *ifp = SC2IFP(sc); struct mbuf *m; - size_t size = len + ETHER_ALIGN; + size_t size = len + ETHER_ALIGN + ETHER_VLAN_ENCAP_LEN; MGETHDR(m, M_NOWAIT, MT_DATA); if (m == NULL) @@ -1512,28 +4702,38 @@ hvn_devget(struct hvn_softc *sc, void *buf, uint32_t len) } m->m_len = m->m_pkthdr.len = size; - m_adj(m, ETHER_ALIGN); + m_adj(m, ETHER_ALIGN + ETHER_VLAN_ENCAP_LEN); m_copyback(m, 0, len, buf); m_set_rcvif(m, ifp); return m; } -static void -hvn_rxeof(struct hvn_softc *sc, uint8_t *buf, uint32_t len) +#define HVN_RXINFO_CSUM __BIT(NDIS_PKTINFO_TYPE_CSUM) +#define HVN_RXINFO_VLAN __BIT(NDIS_PKTINFO_TYPE_VLAN) +#define HVN_RXINFO_HASHVAL __BIT(HVN_NDIS_PKTINFO_TYPE_HASHVAL) +#define HVN_RXINFO_HASHINFO __BIT(HVN_NDIS_PKTINFO_TYPE_HASHINF) +#define HVN_RXINFO_ALL (HVN_RXINFO_CSUM | \ + HVN_RXINFO_VLAN | \ + HVN_RXINFO_HASHVAL | \ + HVN_RXINFO_HASHINFO) + +static int +hvn_rxeof(struct hvn_rx_ring *rxr, uint8_t *buf, uint32_t len) { + struct hvn_softc *sc = rxr->rxr_softc; struct ifnet *ifp = SC2IFP(sc); struct rndis_packet_msg *pkt; struct rndis_pktinfo *pi; - uint32_t csum, vlan; struct mbuf *m; + uint32_t mask, csum, vlan, hashval, hashinfo; if (!(ifp->if_flags & IFF_RUNNING)) - return; + return 0; if (len < sizeof(*pkt)) { device_printf(sc->sc_dev, "data packet too short: %u\n", len); - return; + return 0; } pkt = (struct rndis_packet_msg *)buf; @@ -1541,13 +4741,13 @@ hvn_rxeof(struct hvn_softc *sc, uint8_t *buf, uint32_t len) device_printf(sc->sc_dev, "data packet out of bounds: %u@%u\n", pkt->rm_dataoffset, pkt->rm_datalen); - return; + return 0; } if ((m = hvn_devget(sc, buf + RNDIS_HEADER_OFFSET + pkt->rm_dataoffset, pkt->rm_datalen)) == NULL) { if_statinc(ifp, if_ierrors); - return; + return 0; } if (pkt->rm_pktinfooffset + pkt->rm_pktinfolen > len) { @@ -1557,6 +4757,8 @@ hvn_rxeof(struct hvn_softc *sc, uint8_t *buf, uint32_t len) goto done; } + mask = csum = hashval = hashinfo = 0; + vlan = 0xffffffff; pi = (struct rndis_pktinfo *)(buf + RNDIS_HEADER_OFFSET + pkt->rm_pktinfooffset); while (pkt->rm_pktinfolen > 0) { @@ -1570,34 +4772,93 @@ hvn_rxeof(struct hvn_softc *sc, uint8_t *buf, uint32_t len) switch (pi->rm_type) { case NDIS_PKTINFO_TYPE_CSUM: memcpy(&csum, pi->rm_data, sizeof(csum)); - if (csum & NDIS_RXCSUM_INFO_IPCS_OK) - m->m_pkthdr.csum_flags |= M_CSUM_IPv4; - if (csum & NDIS_RXCSUM_INFO_TCPCS_OK) - m->m_pkthdr.csum_flags |= M_CSUM_TCPv4; - if (csum & NDIS_RXCSUM_INFO_UDPCS_OK) - m->m_pkthdr.csum_flags |= M_CSUM_UDPv4; + SET(mask, HVN_RXINFO_CSUM); break; case NDIS_PKTINFO_TYPE_VLAN: memcpy(&vlan, pi->rm_data, sizeof(vlan)); - if (vlan != 0xffffffff) { - uint16_t t = NDIS_VLAN_INFO_ID(vlan); - t |= NDIS_VLAN_INFO_PRI(vlan) << EVL_PRIO_BITS; - t |= NDIS_VLAN_INFO_CFI(vlan) << EVL_CFI_BITS; - vlan_set_tag(m, t); - } + SET(mask, HVN_RXINFO_VLAN); + break; + case HVN_NDIS_PKTINFO_TYPE_HASHVAL: + memcpy(&hashval, pi->rm_data, sizeof(hashval)); + SET(mask, HVN_RXINFO_HASHVAL); + break; + case HVN_NDIS_PKTINFO_TYPE_HASHINF: + memcpy(&hashinfo, pi->rm_data, sizeof(hashinfo)); + SET(mask, HVN_RXINFO_HASHINFO); break; default: DPRINTF("%s: unhandled pktinfo type %u\n", device_xname(sc->sc_dev), pi->rm_type); - break; + goto next; } + if (mask == HVN_RXINFO_ALL) { + /* All found; done */ + break; + } + next: pkt->rm_pktinfolen -= pi->rm_size; pi = (struct rndis_pktinfo *)((char *)pi + pi->rm_size); } + /* + * Final fixup. + * - If there is no hash value, invalidate the hash info. + */ + if (!ISSET(mask, HVN_RXINFO_HASHVAL)) + hashinfo = 0; + + if (csum != 0) { + if (ISSET(csum, NDIS_RXCSUM_INFO_IPCS_OK) && + ISSET(ifp->if_csum_flags_rx, M_CSUM_IPv4)) { + SET(m->m_pkthdr.csum_flags, M_CSUM_IPv4); + rxr->rxr_evcsum_ip.ev_count++; + } + if (ISSET(csum, NDIS_RXCSUM_INFO_TCPCS_OK) && + ISSET(ifp->if_csum_flags_rx, M_CSUM_TCPv4)) { + SET(m->m_pkthdr.csum_flags, M_CSUM_TCPv4); + rxr->rxr_evcsum_tcp.ev_count++; + } + if (ISSET(csum, NDIS_RXCSUM_INFO_UDPCS_OK) && + ISSET(ifp->if_csum_flags_rx, M_CSUM_UDPv4)) { + SET(m->m_pkthdr.csum_flags, M_CSUM_UDPv4); + rxr->rxr_evcsum_udp.ev_count++; + } + } + + if (vlan != 0xffffffff) { + uint16_t t = NDIS_VLAN_INFO_ID(vlan); + t |= NDIS_VLAN_INFO_PRI(vlan) << EVL_PRIO_BITS; + t |= NDIS_VLAN_INFO_CFI(vlan) << EVL_CFI_BITS; + + if (ISSET(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { + vlan_set_tag(m, t); + rxr->rxr_evvlanhwtagging.ev_count++; + } else { + struct ether_header eh; + struct ether_vlan_header *evl; + + KDASSERT(m->m_pkthdr.len >= sizeof(eh)); + m_copydata(m, 0, sizeof(eh), &eh); + M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT); + KDASSERT(m != NULL); + + evl = mtod(m, struct ether_vlan_header *); + memcpy(evl->evl_dhost, eh.ether_dhost, + ETHER_ADDR_LEN * 2); + evl->evl_encap_proto = htons(ETHERTYPE_VLAN); + evl->evl_tag = htons(t); + evl->evl_proto = eh.ether_type; + } + } + + /* XXX RSS hash is not supported. */ + done: + rxr->rxr_evpkts.ev_count++; if_percpuq_enqueue(sc->sc_ipq, m); + /* XXX Unable to detect that the receive queue is full. */ + return 1; } static void @@ -1608,6 +4869,7 @@ hvn_rndis_complete(struct hvn_softc *sc, uint8_t *buf, uint32_t len) memcpy(&id, buf + RNDIS_HEADER_OFFSET, sizeof(id)); if ((rc = hvn_complete_cmd(sc, id)) != NULL) { + mutex_enter(&rc->rc_lock); if (len < rc->rc_cmplen) device_printf(sc->sc_dev, "RNDIS response %u too short: %u\n", id, len); @@ -1621,7 +4883,8 @@ hvn_rndis_complete(struct hvn_softc *sc, uint8_t *buf, uint32_t len) memcpy(&rc->rc_cmpbuf, buf + rc->rc_cmplen, len - rc->rc_cmplen); rc->rc_done = 1; - wakeup(rc); + cv_signal(&rc->rc_cv); + mutex_exit(&rc->rc_lock); } else { DPRINTF("%s: failed to complete RNDIS request id %u\n", device_xname(sc->sc_dev), id); @@ -1629,12 +4892,13 @@ hvn_rndis_complete(struct hvn_softc *sc, uint8_t *buf, uint32_t len) } static int -hvn_rndis_output(struct hvn_softc *sc, struct hvn_tx_desc *txd) +hvn_rndis_output_sgl(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd) { + struct hvn_softc *sc = txr->txr_softc; uint64_t rid = (uint64_t)txd->txd_id << 32; int rv; - rv = vmbus_channel_send_sgl(sc->sc_chan, txd->txd_sgl, txd->txd_nsge, + rv = vmbus_channel_send_sgl(txr->txr_chan, txd->txd_sgl, txd->txd_nsge, &sc->sc_data_msg, sizeof(sc->sc_data_msg), rid); if (rv) { DPRINTF("%s: RNDIS data send error %d\n", @@ -1644,40 +4908,70 @@ hvn_rndis_output(struct hvn_softc *sc, struct hvn_tx_desc *txd) return 0; } +static int +hvn_rndis_output_chim(struct hvn_tx_ring *txr, struct hvn_tx_desc *txd) +{ + struct hvn_nvs_rndis rndis; + uint64_t rid = (uint64_t)txd->txd_id << 32; + int rv; + + memset(&rndis, 0, sizeof(rndis)); + rndis.nvs_type = HVN_NVS_TYPE_RNDIS; + rndis.nvs_rndis_mtype = HVN_NVS_RNDIS_MTYPE_DATA; + rndis.nvs_chim_idx = txd->txd_chim_index; + rndis.nvs_chim_sz = txd->txd_chim_size; + + rv = vmbus_channel_send(txr->txr_chan, &rndis, sizeof(rndis), + rid, VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC); + if (rv) { + DPRINTF("%s: RNDIS chimney data send error %d: idx %u, sz %u\n", + device_xname(sc->sc_dev), rv, rndis.nvs_chim_idx, + rndis.nvs_chim_sz); + return rv; + } + return 0; +} + static void hvn_rndis_status(struct hvn_softc *sc, uint8_t *buf, uint32_t len) { - struct ifnet *ifp = SC2IFP(sc); uint32_t status; - int link_state = sc->sc_link_state; memcpy(&status, buf + RNDIS_HEADER_OFFSET, sizeof(status)); switch (status) { case RNDIS_STATUS_MEDIA_CONNECT: - sc->sc_link_state = LINK_STATE_UP; - break; case RNDIS_STATUS_MEDIA_DISCONNECT: - sc->sc_link_state = LINK_STATE_DOWN; + hvn_link_event(sc, HVN_LINK_EV_STATE_CHANGE); + break; + case RNDIS_STATUS_NETWORK_CHANGE: + hvn_link_event(sc, HVN_LINK_EV_NETWORK_CHANGE); break; /* Ignore these */ case RNDIS_STATUS_OFFLOAD_CURRENT_CONFIG: + case RNDIS_STATUS_LINK_SPEED_CHANGE: return; default: DPRINTF("%s: unhandled status %#x\n", device_xname(sc->sc_dev), status); return; } - if (link_state != sc->sc_link_state) - if_link_state_change(ifp, sc->sc_link_state); } static int hvn_rndis_query(struct hvn_softc *sc, uint32_t oid, void *res, size_t *length) { + + return hvn_rndis_query2(sc, oid, NULL, 0, res, length, 0); +} + +static int +hvn_rndis_query2(struct hvn_softc *sc, uint32_t oid, const void *idata, + size_t idlen, void *odata, size_t *odlen, size_t min_odlen) +{ struct rndis_cmd *rc; struct rndis_query_req *req; struct rndis_query_comp *cmp; - size_t olength = *length; + size_t olength = *odlen; int rv; rc = hvn_alloc_cmd(sc); @@ -1689,17 +4983,22 @@ hvn_rndis_query(struct hvn_softc *sc, uint32_t oid, void *res, size_t *length) req = rc->rc_req; req->rm_type = REMOTE_NDIS_QUERY_MSG; - req->rm_len = sizeof(*req); + req->rm_len = sizeof(*req) + idlen; req->rm_rid = rc->rc_id; req->rm_oid = oid; req->rm_infobufoffset = sizeof(*req) - RNDIS_HEADER_OFFSET; + if (idlen > 0) { + KASSERT(sizeof(*req) + idlen <= PAGE_SIZE); + req->rm_infobuflen = idlen; + memcpy(req + 1, idata, idlen); + } rc->rc_cmplen = sizeof(*cmp); bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, BUS_DMASYNC_PREWRITE); - if ((rv = hvn_rndis_cmd(sc, rc, 500)) != 0) { + if ((rv = hvn_rndis_cmd(sc, rc, 0)) != 0) { DPRINTF("%s: QUERY_MSG failed, error %d\n", device_xname(sc->sc_dev), rv); hvn_free_cmd(sc, rc); @@ -1709,15 +5008,16 @@ hvn_rndis_query(struct hvn_softc *sc, uint32_t oid, void *res, size_t *length) cmp = (struct rndis_query_comp *)&rc->rc_cmp; switch (cmp->rm_status) { case RNDIS_STATUS_SUCCESS: - if (cmp->rm_infobuflen > olength) { + if (cmp->rm_infobuflen > olength || + (min_odlen > 0 && cmp->rm_infobuflen < min_odlen)) { rv = EINVAL; break; } - memcpy(res, rc->rc_cmpbuf, cmp->rm_infobuflen); - *length = cmp->rm_infobuflen; + memcpy(odata, rc->rc_cmpbuf, cmp->rm_infobuflen); + *odlen = cmp->rm_infobuflen; break; default: - *length = 0; + *odlen = 0; rv = EIO; break; } @@ -1759,7 +5059,7 @@ hvn_rndis_set(struct hvn_softc *sc, uint32_t oid, void *data, size_t length) bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, BUS_DMASYNC_PREWRITE); - if ((rv = hvn_rndis_cmd(sc, rc, 500)) != 0) { + if ((rv = hvn_rndis_cmd(sc, rc, 0)) != 0) { DPRINTF("%s: SET_MSG failed, error %d\n", device_xname(sc->sc_dev), rv); hvn_free_cmd(sc, rc); @@ -1777,15 +5077,31 @@ hvn_rndis_set(struct hvn_softc *sc, uint32_t oid, void *data, size_t length) static int hvn_rndis_open(struct hvn_softc *sc) { + struct ifnet *ifp = SC2IFP(sc); uint32_t filter; int rv; - if (sc->sc_promisc) + if (ifp->if_flags & IFF_PROMISC) { filter = RNDIS_PACKET_TYPE_PROMISCUOUS; - else - filter = RNDIS_PACKET_TYPE_BROADCAST | - RNDIS_PACKET_TYPE_ALL_MULTICAST | - RNDIS_PACKET_TYPE_DIRECTED; + } else { + filter = RNDIS_PACKET_TYPE_DIRECTED; + if (ifp->if_flags & IFF_BROADCAST) + filter |= RNDIS_PACKET_TYPE_BROADCAST; + if (ifp->if_flags & IFF_ALLMULTI) + filter |= RNDIS_PACKET_TYPE_ALL_MULTICAST; + else { + struct ethercom *ec = &sc->sc_ec; + struct ether_multi *enm; + struct ether_multistep step; + + ETHER_LOCK(ec); + ETHER_FIRST_MULTI(step, ec, enm); + /* TODO: support multicast list */ + if (enm != NULL) + filter |= RNDIS_PACKET_TYPE_ALL_MULTICAST; + ETHER_UNLOCK(ec); + } + } rv = hvn_rndis_set(sc, OID_GEN_CURRENT_PACKET_FILTER, &filter, sizeof(filter)); @@ -1833,13 +5149,146 @@ hvn_rndis_detach(struct hvn_softc *sc) bus_dmamap_sync(sc->sc_dmat, rc->rc_dmap, 0, PAGE_SIZE, BUS_DMASYNC_PREWRITE); - if ((rv = hvn_rndis_cmd(sc, rc, 500)) != 0) { + /* No RNDIS completion; rely on NVS message send completion */ + if ((rv = hvn_rndis_cmd(sc, rc, HVN_RNDIS_CMD_NORESP)) != 0) { DPRINTF("%s: HALT_MSG failed, error %d\n", device_xname(sc->sc_dev), rv); } hvn_free_cmd(sc, rc); +} - mutex_destroy(&sc->sc_cntl_sqlck); - mutex_destroy(&sc->sc_cntl_cqlck); - mutex_destroy(&sc->sc_cntl_fqlck); +static void +hvn_init_sysctls(struct hvn_softc *sc) +{ + struct sysctllog **log; + const struct sysctlnode *rnode, *cnode, *rxnode, *txnode; + const char *dvname; + int error; + + log = &sc->sc_sysctllog; + dvname = device_xname(sc->sc_dev); + + error = sysctl_createv(log, 0, NULL, &rnode, + 0, CTLTYPE_NODE, dvname, + SYSCTL_DESCR("hvn information and settings"), + NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); + if (error) + goto err; + + error = sysctl_createv(log, 0, &rnode, &cnode, + CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", + SYSCTL_DESCR("Use workqueue for packet processing"), + NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rnode, &rxnode, + 0, CTLTYPE_NODE, "rx", + SYSCTL_DESCR("hvn information and settings for Rx"), + NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rxnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", + SYSCTL_DESCR("max number of Rx packets" + " to process for interrupt processing"), + NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rxnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", + SYSCTL_DESCR("max number of Rx packets" + " to process for deferred processing"), + NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rnode, &txnode, + 0, CTLTYPE_NODE, "tx", + SYSCTL_DESCR("hvn information and settings for Tx"), + NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &txnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", + SYSCTL_DESCR("max number of Tx packets" + " to process for interrupt processing"), + NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &txnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", + SYSCTL_DESCR("max number of Tx packets" + " to process for deferred processing"), + NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + return; + +out: + sysctl_teardown(log); + sc->sc_sysctllog = NULL; +err: + aprint_error_dev(sc->sc_dev, "sysctl_createv failed (err = %d)\n", + error); +} + +SYSCTL_SETUP(sysctl_hw_hvn_setup, "sysctl hw.hvn setup") +{ + const struct sysctlnode *rnode; + const struct sysctlnode *cnode; + int error; + + error = sysctl_createv(clog, 0, NULL, &rnode, + CTLFLAG_PERMANENT, CTLTYPE_NODE, "hvn", + SYSCTL_DESCR("hvn global controls"), + NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); + if (error) + goto fail; + + error = sysctl_createv(clog, 0, &rnode, &cnode, + CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, + "udp_csum_fixup_mtu", + SYSCTL_DESCR("UDP checksum offloding fixup MTU"), + NULL, 0, &hvn_udpcs_fixup_mtu, sizeof(hvn_udpcs_fixup_mtu), + CTL_CREATE, CTL_EOL); + if (error) + goto fail; + + error = sysctl_createv(clog, 0, &rnode, &cnode, + CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, + "chimney_size", + SYSCTL_DESCR("Chimney send packet size limit"), + NULL, 0, &hvn_tx_chimney_size, sizeof(hvn_tx_chimney_size), + CTL_CREATE, CTL_EOL); + if (error) + goto fail; + + error = sysctl_createv(clog, 0, &rnode, &cnode, + CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, + "channel_count", + SYSCTL_DESCR("# of channels to use"), + NULL, 0, &hvn_channel_cnt, sizeof(hvn_channel_cnt), + CTL_CREATE, CTL_EOL); + if (error) + goto fail; + + error = sysctl_createv(clog, 0, &rnode, &cnode, + CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, + "tx_ring_count", + SYSCTL_DESCR("# of transmit rings to use"), + NULL, 0, &hvn_tx_ring_cnt, sizeof(hvn_tx_ring_cnt), + CTL_CREATE, CTL_EOL); + if (error) + goto fail; + + return; + +fail: + aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, error); } diff --git a/sys/dev/hyperv/if_hvnreg.h b/sys/dev/hyperv/if_hvnreg.h index 12f8e507a75..364b2cd2f6e 100644 --- a/sys/dev/hyperv/if_hvnreg.h +++ b/sys/dev/hyperv/if_hvnreg.h @@ -1,4 +1,4 @@ -/* $NetBSD: if_hvnreg.h,v 1.2 2021/10/21 13:21:54 andvar Exp $ */ +/* $NetBSD: if_hvnreg.h,v 1.3 2022/05/20 13:55:17 nonaka Exp $ */ /* $OpenBSD: if_hvnreg.h,v 1.3 2016/09/14 17:48:28 mikeb Exp $ */ /*- @@ -201,4 +201,21 @@ struct hvn_nvs_rndis_ack { uint8_t nvs_rsvd[24]; } __packed; +/* + * RNDIS extension + */ + +/* Per-packet hash info */ +#define HVN_NDIS_HASH_INFO_SIZE sizeof(uint32_t) +#define HVN_NDIS_PKTINFO_TYPE_HASHINF NDIS_PKTINFO_TYPE_ORIG_NBLIST +/* NDIS_HASH_ */ + +/* Per-packet hash value */ +#define HVN_NDIS_HASH_VALUE_SIZE sizeof(uint32_t) +#define HVN_NDIS_PKTINFO_TYPE_HASHVAL NDIS_PKTINFO_TYPE_PKT_CANCELID + +/* Per-packet-info size */ +#define HVN_RNDIS_PKTINFO_SIZE(dlen) \ + offsetof(struct rndis_pktinfo, rm_data[dlen]) + #endif /* _IF_HVNREG_H_ */ diff --git a/sys/dev/hyperv/vmbus.c b/sys/dev/hyperv/vmbus.c index 4eb9213234d..aeb3f3311ed 100644 --- a/sys/dev/hyperv/vmbus.c +++ b/sys/dev/hyperv/vmbus.c @@ -1,4 +1,4 @@ -/* $NetBSD: vmbus.c,v 1.17 2022/04/09 23:38:32 riastradh Exp $ */ +/* $NetBSD: vmbus.c,v 1.18 2022/05/20 13:55:17 nonaka Exp $ */ /* $OpenBSD: hyperv.c,v 1.43 2017/06/27 13:56:15 mikeb Exp $ */ /*- @@ -35,7 +35,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: vmbus.c,v 1.17 2022/04/09 23:38:32 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: vmbus.c,v 1.18 2022/05/20 13:55:17 nonaka Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -100,9 +100,6 @@ static struct vmbus_channel * static int vmbus_channel_ring_create(struct vmbus_channel *, uint32_t); static void vmbus_channel_ring_destroy(struct vmbus_channel *); static void vmbus_channel_detach(struct vmbus_channel *); -static void vmbus_channel_pause(struct vmbus_channel *); -static uint32_t vmbus_channel_unpause(struct vmbus_channel *); -static uint32_t vmbus_channel_ready(struct vmbus_channel *); static void vmbus_chevq_enqueue(struct vmbus_softc *, int, void *); static void vmbus_process_chevq(void *); static void vmbus_chevq_thread(void *); @@ -274,6 +271,9 @@ vmbus_attach(struct vmbus_softc *sc) "hvmsg", NULL, IPL_NET, NULL, NULL, NULL); hyperv_set_message_proc(vmbus_message_proc, sc); + sc->sc_chanmap = kmem_zalloc(sizeof(struct vmbus_channel *) * + VMBUS_CHAN_MAX, KM_SLEEP); + if (vmbus_alloc_dma(sc)) goto cleanup; @@ -306,6 +306,8 @@ vmbus_attach(struct vmbus_softc *sc) cleanup: vmbus_deinit_interrupts(sc); vmbus_free_dma(sc); + kmem_free(__UNVOLATILE(sc->sc_chanmap), + sizeof(struct vmbus_channel *) * VMBUS_CHAN_MAX); return -1; } @@ -326,6 +328,8 @@ vmbus_detach(struct vmbus_softc *sc, int flags) vmbus_deinit_interrupts(sc); vmbus_free_dma(sc); + kmem_free(__UNVOLATILE(sc->sc_chanmap), + sizeof(struct vmbus_channel *) * VMBUS_CHAN_MAX); return 0; } @@ -345,18 +349,18 @@ vmbus_alloc_dma(struct vmbus_softc *sc) pd = &sc->sc_percpu[cpu_index(ci)]; pd->simp = hyperv_dma_alloc(sc->sc_dmat, &pd->simp_dma, - PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); + PAGE_SIZE, PAGE_SIZE, 0, 1); if (pd->simp == NULL) return ENOMEM; pd->siep = hyperv_dma_alloc(sc->sc_dmat, &pd->siep_dma, - PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); + PAGE_SIZE, PAGE_SIZE, 0, 1); if (pd->siep == NULL) return ENOMEM; } sc->sc_events = hyperv_dma_alloc(sc->sc_dmat, &sc->sc_events_dma, - PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); + PAGE_SIZE, PAGE_SIZE, 0, 1); if (sc->sc_events == NULL) return ENOMEM; sc->sc_wevents = (u_long *)sc->sc_events; @@ -364,8 +368,7 @@ vmbus_alloc_dma(struct vmbus_softc *sc) for (i = 0; i < __arraycount(sc->sc_monitor); i++) { sc->sc_monitor[i] = hyperv_dma_alloc(sc->sc_dmat, - &sc->sc_monitor_dma[i], PAGE_SIZE, PAGE_SIZE, 0, 1, - HYPERV_DMA_SLEEPOK); + &sc->sc_monitor_dma[i], PAGE_SIZE, PAGE_SIZE, 0, 1); if (sc->sc_monitor[i] == NULL) return ENOMEM; } @@ -523,7 +526,7 @@ vmbus_connect(struct vmbus_softc *sc) for (i = 0; i < __arraycount(versions); i++) { cmd.chm_ver = versions[i]; rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), - cold ? HCF_NOSLEEP : HCF_SLEEPOK); + HCF_NOSLEEP); if (rv) { DPRINTF("%s: CONNECT failed\n", device_xname(sc->sc_dev)); @@ -549,7 +552,6 @@ static int vmbus_cmd(struct vmbus_softc *sc, void *cmd, size_t cmdlen, void *rsp, size_t rsplen, int flags) { - const int prflags = cold ? PR_NOWAIT : PR_WAITOK; struct vmbus_msg *msg; paddr_t pa; int rv; @@ -560,7 +562,7 @@ vmbus_cmd(struct vmbus_softc *sc, void *cmd, size_t cmdlen, void *rsp, return EMSGSIZE; } - msg = pool_cache_get_paddr(sc->sc_msgpool, prflags, &pa); + msg = pool_cache_get_paddr(sc->sc_msgpool, PR_WAITOK, &pa); if (msg == NULL) { device_printf(sc->sc_dev, "couldn't get msgpool\n"); return ENOMEM; @@ -588,11 +590,9 @@ vmbus_cmd(struct vmbus_softc *sc, void *cmd, size_t cmdlen, void *rsp, static int vmbus_start(struct vmbus_softc *sc, struct vmbus_msg *msg, paddr_t msg_pa) { - static const int delays[] = { - 100, 100, 100, 500, 500, 5000, 5000, 5000 - }; const char *wchan = "hvstart"; uint16_t status; + int wait_ms = 1; /* milliseconds */ int i, s; msg->msg_req.hc_connid = VMBUS_CONNID_MESSAGE; @@ -604,33 +604,45 @@ vmbus_start(struct vmbus_softc *sc, struct vmbus_msg *msg, paddr_t msg_pa) mutex_exit(&sc->sc_req_lock); } - for (i = 0; i < __arraycount(delays); i++) { + /* + * In order to cope with transient failures, e.g. insufficient + * resources on host side, we retry the post message Hypercall + * several times. 20 retries seem sufficient. + */ +#define HC_RETRY_MAX 20 +#define HC_WAIT_MAX (2 * 1000) /* 2s */ + + for (i = 0; i < HC_RETRY_MAX; i++) { status = hyperv_hypercall_post_message( msg_pa + offsetof(struct vmbus_msg, msg_req)); if (status == HYPERCALL_STATUS_SUCCESS) - break; + return 0; if (msg->msg_flags & MSGF_NOSLEEP) { - delay(delays[i]); + DELAY(wait_ms * 1000); s = splnet(); hyperv_intr(); splx(s); } else - tsleep(wchan, PRIBIO, wchan, - uimax(1, mstohz(delays[i] / 1000))); + tsleep(wchan, PRIBIO, wchan, uimax(1, mstohz(wait_ms))); + + if (wait_ms < HC_WAIT_MAX) + wait_ms *= 2; } - if (status != HYPERCALL_STATUS_SUCCESS) { - device_printf(sc->sc_dev, - "posting vmbus message failed with %d\n", status); - if (!(msg->msg_flags & MSGF_NOQUEUE)) { - mutex_enter(&sc->sc_req_lock); - TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); - mutex_exit(&sc->sc_req_lock); - } - return EIO; + +#undef HC_RETRY_MAX +#undef HC_WAIT_MAX + + device_printf(sc->sc_dev, + "posting vmbus message failed with %d\n", status); + + if (!(msg->msg_flags & MSGF_NOQUEUE)) { + mutex_enter(&sc->sc_req_lock); + TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); + mutex_exit(&sc->sc_req_lock); } - return 0; + return EIO; } static int @@ -664,7 +676,7 @@ vmbus_reply(struct vmbus_softc *sc, struct vmbus_msg *msg) hyperv_intr(); splx(s); } else - tsleep(msg, PRIBIO, "hvreply", 1); + tsleep(msg, PRIBIO, "hvreply", uimax(1, mstohz(1))); } mutex_enter(&sc->sc_rsp_lock); @@ -705,7 +717,8 @@ vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *revents, continue; pending = atomic_swap_ulong(&revents[row], 0); - chanid_base = row * LONG_BIT; + pending &= ~sc->sc_evtmask[row]; + chanid_base = row * VMBUS_EVTFLAG_LEN; while ((chanid_ofs = ffsl(pending)) != 0) { chanid_ofs--; /* NOTE: ffs is 1-based */ @@ -716,12 +729,12 @@ vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *revents, if (chanid == 0) continue; - ch = vmbus_channel_lookup(sc, chanid); - if (ch == NULL) { - device_printf(sc->sc_dev, - "unhandled event on %d\n", chanid); + ch = sc->sc_chanmap[chanid]; + if (__predict_false(ch == NULL)) { + /* Channel is closed. */ continue; } + __insn_barrier(); if (ch->ch_state != VMBUS_CHANSTATE_OPENED) { device_printf(sc->sc_dev, "channel %d is not active\n", chanid); @@ -987,7 +1000,7 @@ vmbus_channel_scan(struct vmbus_softc *sc) hdr.chm_type = VMBUS_CHANMSG_CHREQUEST; if (vmbus_cmd(sc, &hdr, sizeof(hdr), &rsp, sizeof(rsp), - HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK))) { + HCF_NOREPLY | HCF_NOSLEEP)) { DPRINTF("%s: CHREQUEST failed\n", device_xname(sc->sc_dev)); return -1; } @@ -1013,7 +1026,7 @@ vmbus_channel_alloc(struct vmbus_softc *sc) ch = kmem_zalloc(sizeof(*ch), KM_SLEEP); ch->ch_monprm = hyperv_dma_alloc(sc->sc_dmat, &ch->ch_monprm_dma, - sizeof(*ch->ch_monprm), 8, 0, 1, HYPERV_DMA_SLEEPOK); + sizeof(*ch->ch_monprm), 8, 0, 1); if (ch->ch_monprm == NULL) { device_printf(sc->sc_dev, "monprm alloc failed\n"); kmem_free(ch, sizeof(*ch)); @@ -1022,7 +1035,10 @@ vmbus_channel_alloc(struct vmbus_softc *sc) ch->ch_refs = 1; ch->ch_sc = sc; + mutex_init(&ch->ch_event_lock, MUTEX_DEFAULT, IPL_NET); + cv_init(&ch->ch_event_cv, "hvevwait"); mutex_init(&ch->ch_subchannel_lock, MUTEX_DEFAULT, IPL_NET); + cv_init(&ch->ch_subchannel_cv, "hvsubch"); TAILQ_INIT(&ch->ch_subchannels); ch->ch_state = VMBUS_CHANSTATE_CLOSED; @@ -1043,7 +1059,10 @@ vmbus_channel_free(struct vmbus_channel *ch) ch->ch_id, ch->ch_refs); hyperv_dma_free(sc->sc_dmat, &ch->ch_monprm_dma); + mutex_destroy(&ch->ch_event_lock); + cv_destroy(&ch->ch_event_cv); mutex_destroy(&ch->ch_subchannel_lock); + cv_destroy(&ch->ch_subchannel_cv); /* XXX ch_evcnt */ if (ch->ch_taskq != NULL) softint_disestablish(ch->ch_taskq); @@ -1055,7 +1074,7 @@ vmbus_channel_add(struct vmbus_channel *nch) { struct vmbus_softc *sc = nch->ch_sc; struct vmbus_channel *ch; - u_int refs __diagused; + int refs __diagused; if (nch->ch_id == 0) { device_printf(sc->sc_dev, "got channel 0 offer, discard\n"); @@ -1105,8 +1124,8 @@ vmbus_channel_add(struct vmbus_channel *nch) mutex_enter(&ch->ch_subchannel_lock); TAILQ_INSERT_TAIL(&ch->ch_subchannels, nch, ch_subentry); ch->ch_subchannel_count++; + cv_signal(&ch->ch_subchannel_cv); mutex_exit(&ch->ch_subchannel_lock); - wakeup(ch); done: mutex_enter(&sc->sc_channel_lock); @@ -1133,6 +1152,10 @@ vmbus_channel_cpu_set(struct vmbus_channel *ch, int cpu) ch->ch_cpuid = cpu; ch->ch_vcpu = hyperv_get_vcpuid(cpu); + + aprint_debug_dev(ch->ch_dev != NULL ? ch->ch_dev : sc->sc_dev, + "channel %u assigned to cpu%u [vcpu%u]\n", + ch->ch_id, ch->ch_cpuid, ch->ch_vcpu); } void @@ -1270,7 +1293,7 @@ vmbus_channel_release(struct vmbus_channel *ch) cmd.chm_chanid = ch->ch_id; rv = vmbus_cmd(sc, &cmd, sizeof(cmd), NULL, 0, - HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK)); + HCF_NOREPLY | HCF_SLEEPOK); if (rv) { DPRINTF("%s: CHFREE failed with %d\n", device_xname(sc->sc_dev), rv); @@ -1279,19 +1302,21 @@ vmbus_channel_release(struct vmbus_channel *ch) } struct vmbus_channel ** -vmbus_subchannel_get(struct vmbus_channel *prich, int cnt) +vmbus_subchannel_get(struct vmbus_channel *prich, int subchan_cnt) { struct vmbus_softc *sc = prich->ch_sc; struct vmbus_channel **ret, *ch; int i, s; - KASSERTMSG(cnt > 0, "invalid sub-channel count %d", cnt); + KASSERTMSG(subchan_cnt > 0, + "invalid sub-channel count %d", subchan_cnt); - ret = kmem_zalloc(sizeof(struct vmbus_channel *) * cnt, KM_SLEEP); + ret = kmem_zalloc(sizeof(struct vmbus_channel *) * subchan_cnt, + KM_SLEEP); mutex_enter(&prich->ch_subchannel_lock); - while (prich->ch_subchannel_count < cnt) { + while (prich->ch_subchannel_count < subchan_cnt) { if (cold) { mutex_exit(&prich->ch_subchannel_lock); delay(1000); @@ -1312,12 +1337,12 @@ vmbus_subchannel_get(struct vmbus_channel *prich, int cnt) TAILQ_FOREACH(ch, &prich->ch_subchannels, ch_subentry) { ret[i] = ch; /* XXX inc refs */ - if (++i == cnt) + if (++i == subchan_cnt) break; } - KASSERTMSG(i == cnt, "invalid subchan count %d, should be %d", - prich->ch_subchannel_count, cnt); + KASSERTMSG(i == subchan_cnt, "invalid subchan count %d, should be %d", + prich->ch_subchannel_count, subchan_cnt); mutex_exit(&prich->ch_subchannel_lock); @@ -1325,19 +1350,41 @@ vmbus_subchannel_get(struct vmbus_channel *prich, int cnt) } void -vmbus_subchannel_put(struct vmbus_channel **subch, int cnt) +vmbus_subchannel_rel(struct vmbus_channel **subch, int cnt) { kmem_free(subch, sizeof(struct vmbus_channel *) * cnt); } +void +vmbus_subchannel_drain(struct vmbus_channel *prich) +{ + int s; + + mutex_enter(&prich->ch_subchannel_lock); + while (prich->ch_subchannel_count > 0) { + if (cold) { + mutex_exit(&prich->ch_subchannel_lock); + delay(1000); + s = splnet(); + hyperv_intr(); + splx(s); + mutex_enter(&prich->ch_subchannel_lock); + } else { + cv_wait(&prich->ch_subchannel_cv, + &prich->ch_subchannel_lock); + } + } + mutex_exit(&prich->ch_subchannel_lock); +} + static struct vmbus_channel * -vmbus_channel_lookup(struct vmbus_softc *sc, uint32_t relid) +vmbus_channel_lookup(struct vmbus_softc *sc, uint32_t chanid) { - struct vmbus_channel *ch; + struct vmbus_channel *ch = NULL; TAILQ_FOREACH(ch, &sc->sc_channels, ch_entry) { - if (ch->ch_id == relid) + if (ch->ch_id == chanid) return ch; } return NULL; @@ -1352,7 +1399,7 @@ vmbus_channel_ring_create(struct vmbus_channel *ch, uint32_t buflen) ch->ch_ring_size = 2 * buflen; /* page aligned memory */ ch->ch_ring = hyperv_dma_alloc(sc->sc_dmat, &ch->ch_ring_dma, - ch->ch_ring_size, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); + ch->ch_ring_size, PAGE_SIZE, 0, 1); if (ch->ch_ring == NULL) { device_printf(sc->sc_dev, "failed to allocate channel ring\n"); @@ -1414,6 +1461,9 @@ vmbus_channel_open(struct vmbus_channel *ch, size_t buflen, void *udata, return rv; } + __insn_barrier(); + sc->sc_chanmap[ch->ch_id] = ch; + memset(&cmd, 0, sizeof(cmd)); cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHOPEN; cmd.chm_openid = ch->ch_id; @@ -1430,9 +1480,9 @@ vmbus_channel_open(struct vmbus_channel *ch, size_t buflen, void *udata, ch->ch_ctx = arg; ch->ch_state = VMBUS_CHANSTATE_OPENED; - rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), - cold ? HCF_NOSLEEP : HCF_SLEEPOK); + rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), HCF_NOSLEEP); if (rv) { + sc->sc_chanmap[ch->ch_id] = NULL; vmbus_channel_ring_destroy(ch); DPRINTF("%s: CHOPEN failed with %d\n", device_xname(sc->sc_dev), rv); @@ -1468,13 +1518,15 @@ vmbus_channel_close_internal(struct vmbus_channel *ch) struct vmbus_chanmsg_chclose cmd; int rv; + sc->sc_chanmap[ch->ch_id] = NULL; + memset(&cmd, 0, sizeof(cmd)); cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHCLOSE; cmd.chm_chanid = ch->ch_id; ch->ch_state = VMBUS_CHANSTATE_CLOSING; rv = vmbus_cmd(sc, &cmd, sizeof(cmd), NULL, 0, - HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK)); + HCF_NOREPLY | HCF_NOSLEEP); if (rv) { DPRINTF("%s: CHCLOSE failed with %d\n", device_xname(sc->sc_dev), rv); @@ -1513,7 +1565,7 @@ vmbus_channel_close(struct vmbus_channel *ch) (void) rv; /* XXX */ vmbus_channel_detach(ch); } - vmbus_subchannel_put(subch, cnt); + vmbus_subchannel_rel(subch, cnt); } return vmbus_channel_close_internal(ch); @@ -1618,6 +1670,13 @@ vmbus_ring_avail(struct vmbus_ring_data *rd, uint32_t *towrite, *toread = r; } +static bool +vmbus_ring_is_empty(struct vmbus_ring_data *rd) +{ + + return rd->rd_ring->br_rindex == rd->rd_ring->br_windex; +} + static int vmbus_ring_write(struct vmbus_ring_data *wrd, struct iovec *iov, int iov_cnt, int *needsig) @@ -1886,25 +1945,29 @@ vmbus_ring_unmask(struct vmbus_ring_data *rd) membar_sync(); } -static void +void vmbus_channel_pause(struct vmbus_channel *ch) { + atomic_or_ulong(&ch->ch_sc->sc_evtmask[ch->ch_id / VMBUS_EVTFLAG_LEN], + __BIT(ch->ch_id % VMBUS_EVTFLAG_LEN)); vmbus_ring_mask(&ch->ch_rrd); } -static uint32_t +uint32_t vmbus_channel_unpause(struct vmbus_channel *ch) { uint32_t avail; + atomic_and_ulong(&ch->ch_sc->sc_evtmask[ch->ch_id / VMBUS_EVTFLAG_LEN], + ~__BIT(ch->ch_id % VMBUS_EVTFLAG_LEN)); vmbus_ring_unmask(&ch->ch_rrd); vmbus_ring_avail(&ch->ch_rrd, NULL, &avail); return avail; } -static uint32_t +uint32_t vmbus_channel_ready(struct vmbus_channel *ch) { uint32_t avail; @@ -1914,6 +1977,20 @@ vmbus_channel_ready(struct vmbus_channel *ch) return avail; } +bool +vmbus_channel_tx_empty(struct vmbus_channel *ch) +{ + + return vmbus_ring_is_empty(&ch->ch_wrd); +} + +bool +vmbus_channel_rx_empty(struct vmbus_channel *ch) +{ + + return vmbus_ring_is_empty(&ch->ch_rrd); +} + /* How many PFNs can be referenced by the header */ #define VMBUS_NPFNHDR ((VMBUS_MSG_DSIZE_MAX - \ sizeof(struct vmbus_chanmsg_gpadl_conn)) / sizeof(uint64_t)) @@ -1926,10 +2003,6 @@ int vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, uint32_t buflen, uint32_t *handle) { - const int prflags = cold ? PR_NOWAIT : PR_WAITOK; - const int kmemflags = cold ? KM_NOSLEEP : KM_SLEEP; - const int msgflags = cold ? MSGF_NOSLEEP : 0; - const int hcflags = cold ? HCF_NOSLEEP : HCF_SLEEPOK; struct vmbus_softc *sc = ch->ch_sc; struct vmbus_chanmsg_gpadl_conn *hdr; struct vmbus_chanmsg_gpadl_subconn *cmd; @@ -1948,16 +2021,10 @@ vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, KASSERT((buflen & PAGE_MASK) == 0); KASSERT(buflen == (uint32_t)dma->map->dm_mapsize); - msg = pool_cache_get_paddr(sc->sc_msgpool, prflags, &pa); - if (msg == NULL) - return ENOMEM; + msg = pool_cache_get_paddr(sc->sc_msgpool, PR_WAITOK, &pa); /* Prepare array of frame addresses */ - frames = kmem_zalloc(total * sizeof(*frames), kmemflags); - if (frames == NULL) { - pool_cache_put_paddr(sc->sc_msgpool, msg, pa); - return ENOMEM; - } + frames = kmem_zalloc(total * sizeof(*frames), KM_SLEEP); for (i = 0, j = 0; i < dma->map->dm_nsegs && j < total; i++) { bus_dma_segment_t *seg = &dma->map->dm_segs[i]; bus_addr_t addr = seg->ds_addr; @@ -1977,7 +2044,7 @@ vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, hdr = (struct vmbus_chanmsg_gpadl_conn *)msg->msg_req.hc_data; msg->msg_rsp = &rsp; msg->msg_rsplen = sizeof(rsp); - msg->msg_flags = msgflags; + msg->msg_flags = MSGF_NOSLEEP; left = total - inhdr; @@ -1985,12 +2052,7 @@ vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, if (left > 0) { ncmds = howmany(left, VMBUS_NPFNBODY); bodylen = ncmds * VMBUS_MSG_DSIZE_MAX; - body = kmem_zalloc(bodylen, kmemflags); - if (body == NULL) { - kmem_free(frames, total * sizeof(*frames)); - pool_cache_put_paddr(sc->sc_msgpool, msg, pa); - return ENOMEM; - } + body = kmem_zalloc(bodylen, KM_SLEEP); } *handle = atomic_inc_32_nv(&sc->sc_handle); @@ -2035,7 +2097,8 @@ vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, cmdlen += last * sizeof(uint64_t); else cmdlen += VMBUS_NPFNBODY * sizeof(uint64_t); - rv = vmbus_cmd(sc, cmd, cmdlen, NULL, 0, HCF_NOREPLY | hcflags); + rv = vmbus_cmd(sc, cmd, cmdlen, NULL, 0, + HCF_NOREPLY | HCF_NOSLEEP); if (rv != 0) { DPRINTF("%s: GPADL_SUBCONN (iteration %d/%d) failed " "with %d\n", device_xname(sc->sc_dev), i, ncmds, @@ -2075,8 +2138,7 @@ vmbus_handle_free(struct vmbus_channel *ch, uint32_t handle) cmd.chm_chanid = ch->ch_id; cmd.chm_gpadl = handle; - rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), - cold ? HCF_NOSLEEP : HCF_SLEEPOK); + rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), HCF_NOSLEEP); if (rv) { DPRINTF("%s: GPADL_DISCONN failed with %d\n", device_xname(sc->sc_dev), rv); diff --git a/sys/dev/hyperv/vmbusvar.h b/sys/dev/hyperv/vmbusvar.h index d7f264d8f4d..44d1d665e53 100644 --- a/sys/dev/hyperv/vmbusvar.h +++ b/sys/dev/hyperv/vmbusvar.h @@ -1,4 +1,4 @@ -/* $NetBSD: vmbusvar.h,v 1.6 2020/07/14 00:45:53 yamaguchi Exp $ */ +/* $NetBSD: vmbusvar.h,v 1.7 2022/05/20 13:55:17 nonaka Exp $ */ /* $OpenBSD: hypervvar.h,v 1.13 2017/06/23 19:05:42 mikeb Exp $ */ /* @@ -121,6 +121,9 @@ struct vmbus_channel { struct evcnt ch_evcnt; void *ch_taskq; + kmutex_t ch_event_lock; + kcondvar_t ch_event_cv; + uint32_t ch_flags; #define CHF_BATCHED __BIT(0) #define CHF_MONITOR __BIT(1) @@ -135,6 +138,7 @@ struct vmbus_channel { TAILQ_ENTRY(vmbus_channel) ch_entry; kmutex_t ch_subchannel_lock; + kcondvar_t ch_subchannel_cv; struct vmbus_channels ch_subchannels; u_int ch_subchannel_count; TAILQ_ENTRY(vmbus_channel) ch_subentry; @@ -175,6 +179,8 @@ struct vmbus_softc { u_long *sc_wevents; /* Write events */ u_long *sc_revents; /* Read events */ + struct vmbus_channel * volatile *sc_chanmap; + volatile u_long sc_evtmask[VMBUS_EVTFLAGS_MAX]; struct vmbus_mnf *sc_monitor[2]; struct vmbus_percpu_data sc_percpu[MAXCPUS]; @@ -286,9 +292,15 @@ int vmbus_channel_recv(struct vmbus_channel *, void *, uint32_t, uint32_t *, void vmbus_channel_cpu_set(struct vmbus_channel *, int); void vmbus_channel_cpu_rr(struct vmbus_channel *); bool vmbus_channel_is_revoked(struct vmbus_channel *); +bool vmbus_channel_tx_empty(struct vmbus_channel *); +bool vmbus_channel_rx_empty(struct vmbus_channel *); +void vmbus_channel_pause(struct vmbus_channel *); +uint32_t vmbus_channel_unpause(struct vmbus_channel *); +uint32_t vmbus_channel_ready(struct vmbus_channel *); struct vmbus_channel ** vmbus_subchannel_get(struct vmbus_channel *, int); -void vmbus_subchannel_put(struct vmbus_channel **, int); +void vmbus_subchannel_rel(struct vmbus_channel **, int); +void vmbus_subchannel_drain(struct vmbus_channel *); #endif /* _VMBUSVAR_H_ */ |
