diff options
| author | msaitoh <msaitoh@NetBSD.org> | 2015-04-14 07:17:06 +0000 |
|---|---|---|
| committer | msaitoh <msaitoh@NetBSD.org> | 2015-04-14 07:17:06 +0000 |
| commit | c97895046e93cccda687e5f34bb4312c8b496c1a (patch) | |
| tree | 309190ccf976cb9da7a1380ee09277a680665f02 /sys/dev | |
| parent | 564b3f30a9f3f3b9063a47babfca3e31afb84d74 (diff) | |
Sync ixg(4) up to FreeBSD r243716:
- A lot of bugfixes. Some of them are realted to multi queue and those
have not affected in NetBSD because we have not used it yet.
- Show 1000Base-SX correctly.
- Fix if_baudrate from 1G to 10G.
- Improve performance.
Diffstat (limited to 'sys/dev')
| -rw-r--r-- | sys/dev/pci/ixgbe/ixgbe.c | 622 | ||||
| -rw-r--r-- | sys/dev/pci/ixgbe/ixgbe.h | 43 | ||||
| -rw-r--r-- | sys/dev/pci/ixgbe/ixgbe_82599.c | 7 | ||||
| -rw-r--r-- | sys/dev/pci/ixgbe/ixgbe_osdep.h | 28 | ||||
| -rw-r--r-- | sys/dev/pci/ixgbe/ixv.c | 8 |
5 files changed, 320 insertions, 388 deletions
diff --git a/sys/dev/pci/ixgbe/ixgbe.c b/sys/dev/pci/ixgbe/ixgbe.c index 7f36ff53d1c..d6e3fbb1856 100644 --- a/sys/dev/pci/ixgbe/ixgbe.c +++ b/sys/dev/pci/ixgbe/ixgbe.c @@ -58,8 +58,8 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 238149 2012-07-05 20:51:44Z jfv $*/ -/*$NetBSD: ixgbe.c,v 1.25 2015/04/02 09:26:55 msaitoh Exp $*/ +/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 243716 2012-11-30 22:33:21Z jfv $*/ +/*$NetBSD: ixgbe.c,v 1.26 2015/04/14 07:17:06 msaitoh Exp $*/ #include "opt_inet.h" #include "opt_inet6.h" @@ -74,7 +74,7 @@ int ixgbe_display_debug_stats = 0; /********************************************************************* * Driver version *********************************************************************/ -char ixgbe_driver_version[] = "2.4.8"; +char ixgbe_driver_version[] = "2.5.0 - 2"; /********************************************************************* * PCI Device ID Table @@ -134,13 +134,15 @@ static int ixgbe_detach(device_t, int); #if 0 static int ixgbe_shutdown(device_t); #endif -static void ixgbe_start(struct ifnet *); -static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); #if __FreeBSD_version >= 800000 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); static int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *, struct mbuf *); static void ixgbe_qflush(struct ifnet *); +static void ixgbe_deferred_mq_start(void *, int); +#else +static void ixgbe_start(struct ifnet *); +static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); #endif static int ixgbe_ioctl(struct ifnet *, u_long, void *); static void ixgbe_ifstop(struct ifnet *, int); @@ -216,6 +218,9 @@ static __inline void ixgbe_rx_discard(struct rx_ring *, int); static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, struct mbuf *, u32); +static void ixgbe_enable_rx_drop(struct adapter *); +static void ixgbe_disable_rx_drop(struct adapter *); + /* Support for pluggable optic modules */ static bool ixgbe_sfp_probe(struct adapter *); static void ixgbe_setup_optics(struct adapter *); @@ -296,15 +301,6 @@ static int ixgbe_smart_speed = ixgbe_smart_speed_on; static int ixgbe_enable_msix = 1; TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); -/* - * Header split: this causes the hardware to DMA - * the header into a separate mbuf from the payload, - * it can be a performance win in some workloads, but - * in others it actually hurts, its off by default. - */ -static int ixgbe_header_split = FALSE; -TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); - #if defined(NETBSD_MSI_OR_MSIX) /* * Number of Queues, can be set to 0, @@ -328,6 +324,20 @@ TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); static int ixgbe_rxd = PERFORM_RXD; TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); +/* +** HW RSC control: +** this feature only works with +** IPv4, and only on 82599 and later. +** Also this will cause IP forwarding to +** fail and that can't be controlled by +** the stack as LRO can. For all these +** reasons I've deemed it best to leave +** this off and not bother with a tuneable +** interface, this would need to be compiled +** to enable. +*/ +static bool ixgbe_rsc_enable = FALSE; + /* Keep running tab on them for sanity check */ static int ixgbe_total_ports; @@ -697,10 +707,10 @@ static int ixgbe_detach(device_t dev, int flags) { struct adapter *adapter = device_private(dev); - struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; struct ixgbe_hw_stats *stats = &adapter->stats; struct ix_queue *que = adapter->queues; + struct tx_ring *txr = adapter->tx_rings; u32 ctrl_ext; INIT_DEBUGOUT("ixgbe_detach: begin"); @@ -719,7 +729,10 @@ ixgbe_detach(device_t dev, int flags) ixgbe_stop(adapter); IXGBE_CORE_UNLOCK(adapter); - for (int i = 0; i < adapter->num_queues; i++, que++) { + for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { +#if __FreeBSD_version >= 800000 + softint_disestablish(txr->txq_si); +#endif softint_disestablish(que->que_si); } @@ -764,6 +777,8 @@ ixgbe_detach(device_t dev, int flags) evcnt_detach(&adapter->tso_err); evcnt_detach(&adapter->tso_tx); evcnt_detach(&adapter->link_irq); + + txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { evcnt_detach(&txr->no_desc_avail); evcnt_detach(&txr->total_packets); @@ -790,7 +805,6 @@ ixgbe_detach(device_t dev, int flags) evcnt_detach(&rxr->rx_bytes); evcnt_detach(&rxr->no_jmbuf); evcnt_detach(&rxr->rx_discarded); - evcnt_detach(&rxr->rx_split_packets); evcnt_detach(&rxr->rx_irq); } evcnt_detach(&stats->ipcs); @@ -883,6 +897,7 @@ ixgbe_shutdown(device_t dev) #endif +#if __FreeBSD_version < 800000 /********************************************************************* * Transmit entry point * @@ -902,25 +917,20 @@ ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) IXGBE_TX_LOCK_ASSERT(txr); - if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != - IFF_RUNNING) + if ((ifp->if_flags & IFF_RUNNING) == 0) return; if (!adapter->link_active) return; while (!IFQ_IS_EMPTY(&ifp->if_snd)) { - if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) { - txr->queue_status |= IXGBE_QUEUE_DEPLETED; + if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) break; - } IFQ_POLL(&ifp->if_snd, m_head); if (m_head == NULL) break; if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { - if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) - txr->queue_status |= IXGBE_QUEUE_DEPLETED; break; } IFQ_DEQUEUE(&ifp->if_snd, m_head); @@ -970,7 +980,7 @@ ixgbe_start(struct ifnet *ifp) return; } -#if __FreeBSD_version >= 800000 +#else /* ** Multiqueue Transmit driver ** @@ -992,13 +1002,12 @@ ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) txr = &adapter->tx_rings[i]; que = &adapter->queues[i]; - if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) && - IXGBE_TX_TRYLOCK(txr)) { + if (IXGBE_TX_TRYLOCK(txr)) { err = ixgbe_mq_start_locked(ifp, txr, m); IXGBE_TX_UNLOCK(txr); } else { err = drbr_enqueue(ifp, txr->br, m); - softint_schedule(que->que_si); + softint_schedule(txr->txq_si); } return (err); @@ -1012,7 +1021,6 @@ ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) int enqueued, err = 0; if (((ifp->if_flags & IFF_RUNNING) == 0) || - (txr->queue_status == IXGBE_QUEUE_DEPLETED) || adapter->link_active == 0) { if (m != NULL) err = drbr_enqueue(ifp, txr->br, m); @@ -1037,23 +1045,18 @@ ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) break; } enqueued++; - drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); /* Send a copy of the frame to the BPF listener */ bpf_mtap(ifp, next); if ((ifp->if_flags & IFF_RUNNING) == 0) break; if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) ixgbe_txeof(txr); - if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { - ifp->if_flags |= IFF_OACTIVE; - break; - } next = drbr_dequeue(ifp, txr->br); } if (enqueued > 0) { /* Set watchdog on */ - txr->queue_status |= IXGBE_QUEUE_WORKING; + txr->queue_status = IXGBE_QUEUE_WORKING; getmicrotime(&txr->watchdog_time); } @@ -1064,6 +1067,22 @@ ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) } /* + * Called from a taskqueue to drain queued transmit packets. + */ +static void +ixgbe_deferred_mq_start(void *arg, int pending) +{ + struct tx_ring *txr = arg; + struct adapter *adapter = txr->adapter; + struct ifnet *ifp = adapter->ifp; + + IXGBE_TX_LOCK(txr); + if (!drbr_empty(ifp, txr->br)) + ixgbe_mq_start_locked(ifp, txr, NULL); + IXGBE_TX_UNLOCK(txr); +} + +/* ** Flush all ring buffers */ static void @@ -1248,7 +1267,7 @@ ixgbe_init_locked(struct adapter *adapter) /* ** Determine the correct mbuf pool - ** for doing jumbo/headersplit + ** for doing jumbo frames */ if (adapter->max_frame_size <= 2048) adapter->rx_mbuf_sz = MCLBYTES; @@ -1459,7 +1478,6 @@ ixgbe_init_locked(struct adapter *adapter) /* Now inform the stack we're ready */ ifp->if_flags |= IFF_RUNNING; - ifp->if_flags &= ~IFF_OACTIVE; return; } @@ -1571,7 +1589,6 @@ ixgbe_handle_que(void *context) /* Reenable this interrupt */ ixgbe_enable_queue(adapter, que->msix); - return; } @@ -1835,7 +1852,7 @@ ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) ifmr->ifm_active |= IFM_100_TX | IFM_FDX; break; case IXGBE_LINK_SPEED_1GB_FULL: - ifmr->ifm_active |= IFM_1000_T | IFM_FDX; + ifmr->ifm_active |= adapter->optics | IFM_FDX; break; case IXGBE_LINK_SPEED_10GB_FULL: ifmr->ifm_active |= adapter->optics | IFM_FDX; @@ -2157,13 +2174,11 @@ ixgbe_local_timer1(void *arg) { struct adapter *adapter = arg; device_t dev = adapter->dev; - struct ifnet *ifp = adapter->ifp; struct ix_queue *que = adapter->queues; struct tx_ring *txr = adapter->tx_rings; - int hung, busy, paused; + int hung = 0, paused = 0; KASSERT(mutex_owned(&adapter->core_mtx)); - hung = busy = paused = 0; /* Check for pluggable optics */ if (adapter->sfp_probe) @@ -2182,27 +2197,18 @@ ixgbe_local_timer1(void *arg) /* ** Check the TX queues status - ** - central locked handling of OACTIVE ** - watchdog only if all queues show hung */ for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { - if ((txr->queue_status & IXGBE_QUEUE_HUNG) && + if ((txr->queue_status == IXGBE_QUEUE_HUNG) && (paused == 0)) ++hung; - if (txr->queue_status & IXGBE_QUEUE_DEPLETED) - ++busy; - if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0) + else if (txr->queue_status == IXGBE_QUEUE_WORKING) softint_schedule(que->que_si); } /* Only truely watchdog if all queues show hung */ if (hung == adapter->num_queues) goto watchdog; - /* Only turn off the stack flow when ALL are depleted */ - if (busy == adapter->num_queues) - ifp->if_flags |= IFF_OACTIVE; - else if ((ifp->if_flags & IFF_OACTIVE) && - (busy < adapter->num_queues)) - ifp->if_flags &= ~IFF_OACTIVE; out: ixgbe_rearm_queues(adapter, adapter->que_mask); @@ -2241,7 +2247,6 @@ static void ixgbe_update_link_status(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; device_t dev = adapter->dev; @@ -2262,9 +2267,6 @@ ixgbe_update_link_status(struct adapter *adapter) device_printf(dev,"Link is Down\n"); if_link_state_change(ifp, LINK_STATE_DOWN); adapter->link_active = FALSE; - for (int i = 0; i < adapter->num_queues; - i++, txr++) - txr->queue_status = IXGBE_QUEUE_IDLE; } } @@ -2305,7 +2307,6 @@ ixgbe_stop(void *arg) /* Let the stack know...*/ ifp->if_flags &= ~IFF_RUNNING; - ifp->if_flags |= IFF_OACTIVE; ixgbe_reset_hw(hw); hw->adapter_stopped = FALSE; @@ -2373,16 +2374,22 @@ ixgbe_setup_optics(struct adapter *adapter) int layer; layer = ixgbe_get_supported_physical_layer(hw); + if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { adapter->optics = IFM_10G_T; return; - } + } if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { adapter->optics = IFM_1000_T; return; } + if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { + adapter->optics = IFM_1000_SX; + return; + } + if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR | IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) { adapter->optics = IFM_10G_LR; @@ -2420,6 +2427,9 @@ ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) { device_t dev = adapter->dev; struct ix_queue *que = adapter->queues; +#if __FreeBSD_version >= 800000 + struct tx_ring *txr = adapter->tx_rings; +#endif char intrbuf[PCI_INTRSTR_LEN]; #if 0 int rid = 0; @@ -2443,6 +2453,10 @@ ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) * Try allocating a fast interrupt and the associated deferred * processing contexts. */ +#if __FreeBSD_version >= 800000 + txr->txq_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start, + txr); +#endif que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); /* Tasklets for Link, SFP and Multispeed Fiber */ @@ -2502,9 +2516,10 @@ ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) #else device_t dev = adapter->dev; struct ix_queue *que = adapter->queues; + struct tx_ring *txr = adapter->tx_rings; int error, rid, vector = 0; - for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { + for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { rid = vector + 1; que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); @@ -2535,7 +2550,12 @@ ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) if (adapter->num_queues > 1) bus_bind_intr(dev, que->res, i); - que->que_si = softint_establish(ixgbe_handle_que, que); +#if __FreeBSD_version >= 800000 + txr->txq_si = softint_establish(SOFTINT_NET, + ixgbe_deferred_mq_start, txr); +#endif + que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, + que); if (que->que_si == NULL) { aprint_error_dev(dev, "could not establish software interrupt\n"); @@ -2792,18 +2812,19 @@ ixgbe_setup_interface(device_t dev, struct adapter *adapter) ifp = adapter->ifp = &ec->ec_if; strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); - ifp->if_baudrate = 1000000000; + ifp->if_baudrate = IF_Gbps(10); ifp->if_init = ixgbe_init; ifp->if_stop = ixgbe_ifstop; ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = ixgbe_ioctl; - ifp->if_start = ixgbe_start; #if __FreeBSD_version >= 800000 ifp->if_transmit = ixgbe_mq_start; ifp->if_qflush = ixgbe_qflush; +#else + ifp->if_start = ixgbe_start; + IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); #endif - ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; if_attach(ifp); ether_ifattach(ifp, adapter->hw.mac.addr); @@ -2822,13 +2843,11 @@ ixgbe_setup_interface(device_t dev, struct adapter *adapter) ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; + ifp->if_capabilities |= IFCAP_LRO; ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU; ec->ec_capenable = ec->ec_capabilities; - /* Don't enable LRO by default */ - ifp->if_capabilities |= IFCAP_LRO; - /* ** Don't turn this on by default, if vlans are ** created on another pseudo device (eg. lagg) @@ -3902,10 +3921,6 @@ ixgbe_txeof(struct tx_ring *txr) if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) txr->queue_status = IXGBE_QUEUE_HUNG; - /* With a minimum free clear the depleted state bit. */ - if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) - txr->queue_status &= ~IXGBE_QUEUE_DEPLETED; - if (txr->tx_avail == adapter->num_tx_desc) { txr->queue_status = IXGBE_QUEUE_IDLE; return false; @@ -3928,7 +3943,7 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) { struct adapter *adapter = rxr->adapter; struct ixgbe_rx_buf *rxbuf; - struct mbuf *mh, *mp; + struct mbuf *mp; int i, j, error; bool refreshed = false; @@ -3939,36 +3954,7 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) while (j != limit) { rxbuf = &rxr->rx_buffers[i]; - if (rxr->hdr_split == FALSE) - goto no_split; - - if (rxbuf->m_head == NULL) { - mh = m_gethdr(M_DONTWAIT, MT_DATA); - if (mh == NULL) - goto update; - } else - mh = rxbuf->m_head; - - mh->m_pkthdr.len = mh->m_len = MHLEN; - mh->m_len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, - rxbuf->hmap, mh, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(mh); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = mh; - ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.hdr_addr = - htole64(rxbuf->hmap->dm_segs[0].ds_addr); - -no_split: - if (rxbuf->m_pack == NULL) { + if (rxbuf->buf == NULL) { mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); if (mp == NULL) { @@ -3976,24 +3962,32 @@ no_split: goto update; } } else - mp = rxbuf->m_pack; + mp = rxbuf->buf; mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, - rxbuf->pmap, mp, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: payload dmamap load" - " failure - %d\n", error); - m_free(mp); - rxbuf->m_pack = NULL; - goto update; + /* If we're dealing with an mbuf that was copied rather + * than replaced, there's no need to go through busdma. + */ + if ((rxbuf->flags & IXGBE_RX_COPY) == 0) { + /* Get the memory mapping */ + error = bus_dmamap_load_mbuf(rxr->tag->dt_dmat, + rxbuf->map, mp, BUS_DMA_NOWAIT); + if (error != 0) { + printf("Refresh mbufs: payload dmamap load" + " failure - %d\n", error); + m_free(mp); + rxbuf->buf = NULL; + goto update; + } + rxbuf->buf = mp; + bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map, + 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); + rxbuf->addr = rxr->rx_base[i].read.pkt_addr = + htole64(rxbuf->map->dm_segs[0].ds_addr); + } else { + rxr->rx_base[i].read.pkt_addr = rxbuf->addr; + rxbuf->flags &= ~IXGBE_RX_COPY; } - rxbuf->m_pack = mp; - bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, - 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.pkt_addr = - htole64(rxbuf->pmap->dm_segs[0].ds_addr); refreshed = true; /* Next is precalculated */ @@ -4036,38 +4030,21 @@ ixgbe_allocate_receive_buffers(struct rx_ring *rxr) if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 1, 0, /* alignment, bounds */ - MSIZE, /* maxsize */ - 1, /* nsegments */ - MSIZE, /* maxsegsize */ - 0, /* flags */ - &rxr->htag))) { - aprint_error_dev(dev, "Unable to create RX DMA tag\n"); - goto fail; - } - - if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ - 1, 0, /* alignment, bounds */ MJUM16BYTES, /* maxsize */ 1, /* nsegments */ MJUM16BYTES, /* maxsegsize */ 0, /* flags */ - &rxr->ptag))) { + &rxr->tag))) { aprint_error_dev(dev, "Unable to create RX DMA tag\n"); goto fail; } for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { rxbuf = &rxr->rx_buffers[i]; - error = ixgbe_dmamap_create(rxr->htag, - BUS_DMA_NOWAIT, &rxbuf->hmap); - if (error) { - aprint_error_dev(dev, "Unable to create RX head map\n"); - goto fail; - } - error = ixgbe_dmamap_create(rxr->ptag, - BUS_DMA_NOWAIT, &rxbuf->pmap); + error = ixgbe_dmamap_create(rxr->tag, + BUS_DMA_NOWAIT, &rxbuf->map); if (error) { - aprint_error_dev(dev, "Unable to create RX pkt map\n"); + aprint_error_dev(dev, "Unable to create RX dma map\n"); goto fail; } } @@ -4097,6 +4074,10 @@ ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) * for an RX ring, this is toggled by the LRO capability * even though it is transparent to the stack. * + * NOTE: since this HW feature only works with IPV4 and + * our testing has shown soft LRO to be as effective + * I have decided to disable this by default. + * **********************************************************************/ static void ixgbe_setup_hw_rsc(struct rx_ring *rxr) @@ -4105,6 +4086,13 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr) struct ixgbe_hw *hw = &adapter->hw; u32 rscctrl, rdrxctl; + /* If turning LRO/RSC off we need to disable it */ + if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) { + rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); + rscctrl &= ~IXGBE_RSCCTL_RSCEN; + return; + } + rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; #ifdef DEV_NETMAP /* crcstrip is optional in netmap */ @@ -4154,23 +4142,15 @@ ixgbe_free_receive_ring(struct rx_ring *rxr) adapter = rxr->adapter; for (i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, + if (rxbuf->buf != NULL) { + bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map, + 0, rxbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); - ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); + ixgbe_dmamap_unload(rxr->tag, rxbuf->map); + rxbuf->buf->m_flags |= M_PKTHDR; + m_freem(rxbuf->buf); + rxbuf->buf = NULL; } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, - 0, rxbuf->m_pack->m_pkthdr.len, - BUS_DMASYNC_POSTREAD); - ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; } } @@ -4184,9 +4164,9 @@ static int ixgbe_setup_receive_ring(struct rx_ring *rxr) { struct adapter *adapter; - struct ifnet *ifp; struct ixgbe_rx_buf *rxbuf; #ifdef LRO + struct ifnet *ifp; struct lro_ctrl *lro = &rxr->lro; #endif /* LRO */ int rsize, error = 0; @@ -4196,7 +4176,9 @@ ixgbe_setup_receive_ring(struct rx_ring *rxr) #endif /* DEV_NETMAP */ adapter = rxr->adapter; +#ifdef LRO ifp = adapter->ifp; +#endif /* LRO */ /* Clear the ring contents */ IXGBE_RX_LOCK(rxr); @@ -4216,18 +4198,14 @@ ixgbe_setup_receive_ring(struct rx_ring *rxr) /* Now reinitialize our supply of jumbo mbufs. The number * or size of jumbo mbufs may have changed. */ - ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, + ixgbe_jcl_reinit(&adapter->jcl_head, rxr->tag->dt_dmat, 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); IXGBE_RX_LOCK(rxr); - /* Configure header split? */ - if (ixgbe_header_split) - rxr->hdr_split = TRUE; - /* Now replenish the mbufs */ for (int j = 0; j != adapter->num_rx_desc; ++j) { - struct mbuf *mh, *mp; + struct mbuf *mp; rxbuf = &rxr->rx_buffers[j]; #ifdef DEV_NETMAP @@ -4244,60 +4222,30 @@ ixgbe_setup_receive_ring(struct rx_ring *rxr) void *addr; addr = PNMB(slot + sj, &paddr); - netmap_load_map(rxr->ptag, rxbuf->pmap, addr); + netmap_load_map(rxr->tag, rxbuf->map, addr); /* Update descriptor */ rxr->rx_base[j].read.pkt_addr = htole64(paddr); continue; } #endif /* DEV_NETMAP */ - /* - ** Don't allocate mbufs if not - ** doing header split, its wasteful - */ - if (rxr->hdr_split == FALSE) - goto skip_head; - - /* First the header */ - rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - m_adj(rxbuf->m_head, ETHER_ALIGN); - mh = rxbuf->m_head; - mh->m_len = mh->m_pkthdr.len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, - rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); - if (error != 0) /* Nothing elegant to do here */ - goto fail; - bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, - 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].read.hdr_addr = - htole64(rxbuf->hmap->dm_segs[0].ds_addr); - -skip_head: - /* Now the payload cluster */ - rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, + rxbuf->buf = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_pack == NULL) { + if (rxbuf->buf == NULL) { error = ENOBUFS; goto fail; } - mp = rxbuf->m_pack; + mp = rxbuf->buf; mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; /* Get the memory mapping */ - error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, - rxbuf->pmap, mp, BUS_DMA_NOWAIT); + error = bus_dmamap_load_mbuf(rxr->tag->dt_dmat, + rxbuf->map, mp, BUS_DMA_NOWAIT); if (error != 0) goto fail; - bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, + bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map, 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); /* Update descriptor */ rxr->rx_base[j].read.pkt_addr = - htole64(rxbuf->pmap->dm_segs[0].ds_addr); + htole64(rxbuf->map->dm_segs[0].ds_addr); } @@ -4305,7 +4253,7 @@ skip_head: rxr->next_to_check = 0; rxr->next_to_refresh = 0; rxr->lro_enabled = FALSE; - rxr->rx_split_packets.ev_count = 0; + rxr->rx_copies.ev_count = 0; rxr->rx_bytes.ev_count = 0; rxr->discard = FALSE; rxr->vtag_strip = FALSE; @@ -4315,12 +4263,8 @@ skip_head: /* ** Now set up the LRO interface: - ** 82598 uses software LRO, the - ** 82599 and X540 use a hardware assist. */ - if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && - (ifp->if_capenable & IFCAP_RXCSUM) && - (ifp->if_capenable & IFCAP_LRO)) + if (ixgbe_rsc_enable) ixgbe_setup_hw_rsc(rxr); #ifdef LRO else if (ifp->if_capenable & IFCAP_LRO) { @@ -4443,14 +4387,7 @@ ixgbe_initialize_receive_units(struct adapter *adapter) srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; srrctl |= bufsz; - if (rxr->hdr_split) { - /* Use a standard mbuf for the header */ - srrctl |= ((IXGBE_RX_HDR << - IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) - & IXGBE_SRRCTL_BSIZEHDR_MASK); - srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; - } else - srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; + srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); /* Setup the HW Rx Head and Tail Descriptor Pointers */ @@ -4561,30 +4498,18 @@ ixgbe_free_receive_buffers(struct rx_ring *rxr) if (rxr->rx_buffers != NULL) { for (int i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, + if (rxbuf->buf != NULL) { + bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map, + 0, rxbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); - ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, - 0, rxbuf->m_pack->m_pkthdr.len, - BUS_DMASYNC_POSTREAD); - ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; - if (rxbuf->hmap != NULL) { - ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); - rxbuf->hmap = NULL; + ixgbe_dmamap_unload(rxr->tag, rxbuf->map); + rxbuf->buf->m_flags |= M_PKTHDR; + m_freem(rxbuf->buf); } - if (rxbuf->pmap != NULL) { - ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); - rxbuf->pmap = NULL; + rxbuf->buf = NULL; + if (rxbuf->map != NULL) { + ixgbe_dmamap_destroy(rxr->tag, rxbuf->map); + rxbuf->map = NULL; } } if (rxr->rx_buffers != NULL) { @@ -4593,13 +4518,9 @@ ixgbe_free_receive_buffers(struct rx_ring *rxr) } } - if (rxr->htag != NULL) { - ixgbe_dma_tag_destroy(rxr->htag); - rxr->htag = NULL; - } - if (rxr->ptag != NULL) { - ixgbe_dma_tag_destroy(rxr->ptag); - rxr->ptag = NULL; + if (rxr->tag != NULL) { + ixgbe_dma_tag_destroy(rxr->tag); + rxr->tag = NULL; } return; @@ -4671,16 +4592,11 @@ ixgbe_rx_discard(struct rx_ring *rxr, int i) ** the normal refresh path to get new buffers ** and mapping. */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; + if (rbuf->buf) { + m_free(rbuf->buf); + rbuf->buf = NULL; } - if (rbuf->m_pack) { - m_free(rbuf->m_pack); - rbuf->m_pack = NULL; - } - return; } @@ -4733,9 +4649,9 @@ ixgbe_rxeof(struct ix_queue *que, int count) } #endif /* DEV_NETMAP */ for (i = rxr->next_to_check; count != 0;) { - struct mbuf *sendmp, *mh, *mp; + struct mbuf *sendmp, *mp; u32 rsc, ptype; - u16 hlen, plen, hdr; + u16 len; u16 vtag = 0; bool eop; @@ -4757,19 +4673,13 @@ ixgbe_rxeof(struct ix_queue *que, int count) rsc = 0; cur->wb.upper.status_error = 0; rbuf = &rxr->rx_buffers[i]; - mh = rbuf->m_head; - mp = rbuf->m_pack; + mp = rbuf->buf; - plen = le16toh(cur->wb.upper.length); + len = le16toh(cur->wb.upper.length); ptype = le32toh(cur->wb.lower.lo_dword.data) & IXGBE_RXDADV_PKTTYPE_MASK; - hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); - /* Process vlan info */ - if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) - vtag = le16toh(cur->wb.upper.vlan); - /* Make sure bad packets are discarded */ if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || (rxr->discard)) { @@ -4817,126 +4727,72 @@ ixgbe_rxeof(struct ix_queue *que, int count) prefetch(nbuf); } /* - ** The header mbuf is ONLY used when header - ** split is enabled, otherwise we get normal - ** behavior, ie, both header and payload - ** are DMA'd into the payload buffer. - ** ** Rather than using the fmp/lmp global pointers ** we now keep the head of a packet chain in the ** buffer struct and pass this along from one ** descriptor to the next, until we get EOP. */ - if (rxr->hdr_split && (rbuf->fmp == NULL)) { - /* This must be an initial descriptor */ - hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> - IXGBE_RXDADV_HDRBUFLEN_SHIFT; - if (hlen > IXGBE_RX_HDR) - hlen = IXGBE_RX_HDR; - mh->m_len = hlen; - mh->m_flags |= M_PKTHDR; - mh->m_next = NULL; - mh->m_pkthdr.len = mh->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_head = NULL; - /* - ** Check the payload length, this - ** could be zero if its a small - ** packet. - */ - if (plen > 0) { - mp->m_len = plen; - mp->m_next = NULL; - mp->m_flags &= ~M_PKTHDR; - mh->m_next = mp; - mh->m_pkthdr.len += mp->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_pack = NULL; - rxr->rx_split_packets.ev_count++; - } - /* - ** Now create the forward - ** chain so when complete - ** we wont have to. - */ - if (eop == 0) { - /* stash the chain head */ - nbuf->fmp = mh; - /* Make forward chain */ - if (plen) - mp->m_next = nbuf->m_pack; - else - mh->m_next = nbuf->m_pack; - } else { - /* Singlet, prepare to send */ - sendmp = mh; - /* If hardware handled vtag */ - if (vtag) { - /* XXX Do something reasonable on - * error. - */ -#if 0 - printf("%s.%d: VLAN_INPUT_TAG\n", - __func__, __LINE__); - Debugger(); -#endif - VLAN_INPUT_TAG(ifp, sendmp, vtag, - printf("%s: could not apply VLAN " - "tag", __func__)); - } - } + mp->m_len = len; + /* + ** See if there is a stored head + ** that determines what we are + */ + sendmp = rbuf->fmp; + + if (sendmp != NULL) { /* secondary frag */ + rbuf->buf = rbuf->fmp = NULL; + mp->m_flags &= ~M_PKTHDR; + sendmp->m_pkthdr.len += mp->m_len; } else { /* - ** Either no header split, or a - ** secondary piece of a fragmented - ** split packet. - */ - mp->m_len = plen; - /* - ** See if there is a stored head - ** that determines what we are - */ - sendmp = rbuf->fmp; - rbuf->m_pack = rbuf->fmp = NULL; - - if (sendmp != NULL) { /* secondary frag */ - mp->m_flags &= ~M_PKTHDR; - sendmp->m_pkthdr.len += mp->m_len; - } else { - /* first desc of a non-ps chain */ - sendmp = mp; - sendmp->m_flags |= M_PKTHDR; - sendmp->m_pkthdr.len = mp->m_len; - if (staterr & IXGBE_RXD_STAT_VP) { - /* XXX Do something reasonable on - * error. - */ -#if 0 - printf("%s.%d: VLAN_INPUT_TAG\n", - __func__, __LINE__); - Debugger(); -#endif - VLAN_INPUT_TAG(ifp, sendmp, vtag, - printf("%s: could not apply VLAN " - "tag", __func__)); + * Optimize. This might be a small packet, + * maybe just a TCP ACK. Do a fast copy that + * is cache aligned into a new mbuf, and + * leave the old mbuf+cluster for re-use. + */ + if (eop && len <= IXGBE_RX_COPY_LEN) { + sendmp = m_gethdr(M_DONTWAIT, MT_DATA); + if (sendmp != NULL) { + sendmp->m_data += + IXGBE_RX_COPY_ALIGN; + ixgbe_bcopy(mp->m_data, + sendmp->m_data, len); + sendmp->m_len = len; + rxr->rx_copies.ev_count++; + rbuf->flags |= IXGBE_RX_COPY; } - } - /* Pass the head pointer on */ - if (eop == 0) { - nbuf->fmp = sendmp; - sendmp = NULL; - mp->m_next = nbuf->m_pack; } + if (sendmp == NULL) { + rbuf->buf = rbuf->fmp = NULL; + sendmp = mp; + } + + /* first desc of a non-ps chain */ + sendmp->m_flags |= M_PKTHDR; + sendmp->m_pkthdr.len = mp->m_len; } ++processed; - /* Sending this frame? */ - if (eop) { + /* Pass the head pointer on */ + if (eop == 0) { + nbuf->fmp = sendmp; + sendmp = NULL; + mp->m_next = nbuf->buf; + } else { /* Sending this frame */ sendmp->m_pkthdr.rcvif = ifp; ifp->if_ipackets++; rxr->rx_packets.ev_count++; /* capture data for AIM */ rxr->bytes += sendmp->m_pkthdr.len; rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; + /* Process vlan info */ + if ((rxr->vtag_strip) && + (staterr & IXGBE_RXD_STAT_VP)) + vtag = le16toh(cur->wb.upper.vlan); + if (vtag) { + VLAN_INPUT_TAG(ifp, sendmp, vtag, + printf("%s: could not apply VLAN " + "tag", __func__)); + } if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { ixgbe_rx_checksum(staterr, sendmp, ptype, &adapter->stats); @@ -5143,7 +4999,7 @@ ixgbe_setup_vlan_hw_support(struct adapter *adapter) /* On 82599 the VLAN enable is per/queue in RXDCTL */ if (hw->mac.type != ixgbe_mac_82598EB) { ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); - ctrl |= IXGBE_RXDCTL_VME; + ctrl |= IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); } rxr->vtag_strip = TRUE; @@ -5588,6 +5444,7 @@ ixgbe_update_stats_counters(struct adapter *adapter) ifp->if_ibytes = adapter->stats.gorc.ev_count; ifp->if_obytes = adapter->stats.gotc.ev_count; ifp->if_imcasts = adapter->stats.mprc.ev_count; + ifp->if_omcasts = adapter->stats.mptc.ev_count; ifp->if_collisions = 0; /* Rx Errors */ @@ -5909,12 +5766,12 @@ ixgbe_add_hw_stats(struct adapter *adapter) NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); + evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, + NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, "Rx discarded"); - evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, - NULL, adapter->queues[i].evnamebuf, "Rx split packets"); evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); #ifdef LRO @@ -6083,10 +5940,14 @@ ixgbe_set_flowcntl(SYSCTLFN_ARGS) case ixgbe_fc_tx_pause: case ixgbe_fc_full: adapter->hw.fc.requested_mode = adapter->fc; + if (adapter->num_queues > 1) + ixgbe_disable_rx_drop(adapter); break; case ixgbe_fc_none: default: adapter->hw.fc.requested_mode = ixgbe_fc_none; + if (adapter->num_queues > 1) + ixgbe_enable_rx_drop(adapter); } /* Don't autoneg if forcing a value */ adapter->hw.fc.disable_fc_autoneg = TRUE; @@ -6209,3 +6070,34 @@ ixgbe_set_thermal_test(SYSCTLFN_ARGS) return (0); } + +/* +** Enable the hardware to drop packets when the buffer is +** full. This is useful when multiqueue,so that no single +** queue being full stalls the entire RX engine. We only +** enable this when Multiqueue AND when Flow Control is +** disabled. +*/ +static void +ixgbe_enable_rx_drop(struct adapter *adapter) +{ + struct ixgbe_hw *hw = &adapter->hw; + + for (int i = 0; i < adapter->num_queues; i++) { + u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); + srrctl |= IXGBE_SRRCTL_DROP_EN; + IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); + } +} + +static void +ixgbe_disable_rx_drop(struct adapter *adapter) +{ + struct ixgbe_hw *hw = &adapter->hw; + + for (int i = 0; i < adapter->num_queues; i++) { + u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); + srrctl &= ~IXGBE_SRRCTL_DROP_EN; + IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); + } +} diff --git a/sys/dev/pci/ixgbe/ixgbe.h b/sys/dev/pci/ixgbe/ixgbe.h index 52d030ee6f5..da3fb25d939 100644 --- a/sys/dev/pci/ixgbe/ixgbe.h +++ b/sys/dev/pci/ixgbe/ixgbe.h @@ -58,8 +58,8 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.h 234620 2012-04-23 22:05:09Z bz $*/ -/*$NetBSD: ixgbe.h,v 1.5 2015/04/02 09:26:55 msaitoh Exp $*/ +/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.h 243716 2012-11-30 22:33:21Z jfv $*/ +/*$NetBSD: ixgbe.h,v 1.6 2015/04/14 07:17:06 msaitoh Exp $*/ #ifndef _IXGBE_H_ @@ -173,6 +173,19 @@ #define IXGBE_FC_HI 0x20000 #define IXGBE_FC_LO 0x10000 +/* + * Used for optimizing small rx mbufs. Effort is made to keep the copy + * small and aligned for the CPU L1 cache. + * + * MHLEN is typically 168 bytes, giving us 8-byte alignment. Getting + * 32 byte alignment needed for the fast bcopy results in 8 bytes being + * wasted. Getting 64 byte alignment, which _should_ be ideal for + * modern Intel CPUs, results in 40 bytes wasted and a significant drop + * in observed efficiency of the optimization, 97.9% -> 81.8%. + */ +#define IXGBE_RX_COPY_LEN 160 +#define IXGBE_RX_COPY_ALIGN (MHLEN - IXGBE_RX_COPY_LEN) + /* Keep older OS drivers building... */ #if !defined(SYSCTL_ADD_UQUAD) #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD @@ -204,10 +217,6 @@ #define IXGBE_VFTA_SIZE 128 #define IXGBE_BR_SIZE 4096 #define IXGBE_QUEUE_MIN_FREE 32 -#define IXGBE_QUEUE_IDLE 1 -#define IXGBE_QUEUE_WORKING 2 -#define IXGBE_QUEUE_HUNG 4 -#define IXGBE_QUEUE_DEPLETED 8 /* Offload bits in mbuf flag */ #define M_CSUM_OFFLOAD \ @@ -246,11 +255,12 @@ struct ixgbe_tx_buf { }; struct ixgbe_rx_buf { - struct mbuf *m_head; - struct mbuf *m_pack; + struct mbuf *buf; struct mbuf *fmp; - bus_dmamap_t hmap; - bus_dmamap_t pmap; + bus_dmamap_t map; + u_int flags; +#define IXGBE_RX_COPY 0x01 + uint64_t addr; }; /* @@ -291,7 +301,11 @@ struct tx_ring { struct adapter *adapter; kmutex_t tx_mtx; u32 me; - int queue_status; + enum { + IXGBE_QUEUE_IDLE, + IXGBE_QUEUE_WORKING, + IXGBE_QUEUE_HUNG, + } queue_status; struct timeval watchdog_time; union ixgbe_adv_tx_desc *tx_base; struct ixgbe_dma_alloc txdma; @@ -304,6 +318,7 @@ struct tx_ring { char mtx_name[16]; #if __FreeBSD_version >= 800000 struct buf_ring *br; + struct task txq_task; #endif #ifdef IXGBE_FDIR u16 atr_sample; @@ -330,7 +345,6 @@ struct rx_ring { struct lro_ctrl lro; #endif /* LRO */ bool lro_enabled; - bool hdr_split; bool hw_rsc; bool discard; bool vtag_strip; @@ -338,15 +352,14 @@ struct rx_ring { u32 next_to_check; char mtx_name[16]; struct ixgbe_rx_buf *rx_buffers; - ixgbe_dma_tag_t *htag; - ixgbe_dma_tag_t *ptag; + ixgbe_dma_tag_t *tag; u32 bytes; /* Used for AIM calc */ u32 packets; /* Soft stats */ struct evcnt rx_irq; - struct evcnt rx_split_packets; + struct evcnt rx_copies; struct evcnt rx_packets; struct evcnt rx_bytes; struct evcnt rx_discarded; diff --git a/sys/dev/pci/ixgbe/ixgbe_82599.c b/sys/dev/pci/ixgbe/ixgbe_82599.c index 04625542f2e..35d4967ac19 100644 --- a/sys/dev/pci/ixgbe/ixgbe_82599.c +++ b/sys/dev/pci/ixgbe/ixgbe_82599.c @@ -30,8 +30,8 @@ POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_82599.c 238149 2012-07-05 20:51:44Z jfv $*/ -/*$NetBSD: ixgbe_82599.c,v 1.7 2015/04/02 09:26:55 msaitoh Exp $*/ +/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_82599.c 240155 2012-09-06 02:07:58Z kevlo $*/ +/*$NetBSD: ixgbe_82599.c,v 1.8 2015/04/14 07:17:06 msaitoh Exp $*/ #include "ixgbe_type.h" #include "ixgbe_82599.h" @@ -870,12 +870,13 @@ s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw, link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII) { /* Set KX4/KX/KR support according to speed requested */ autoc &= ~(IXGBE_AUTOC_KX4_KX_SUPP_MASK | IXGBE_AUTOC_KR_SUPP); - if (speed & IXGBE_LINK_SPEED_10GB_FULL) + if (speed & IXGBE_LINK_SPEED_10GB_FULL) { if (orig_autoc & IXGBE_AUTOC_KX4_SUPP) autoc |= IXGBE_AUTOC_KX4_SUPP; if ((orig_autoc & IXGBE_AUTOC_KR_SUPP) && (hw->phy.smart_speed_active == FALSE)) autoc |= IXGBE_AUTOC_KR_SUPP; + } if (speed & IXGBE_LINK_SPEED_1GB_FULL) autoc |= IXGBE_AUTOC_KX_SUPP; } else if ((pma_pmd_1g == IXGBE_AUTOC_1G_SFI) && diff --git a/sys/dev/pci/ixgbe/ixgbe_osdep.h b/sys/dev/pci/ixgbe/ixgbe_osdep.h index 9a60bfcf74c..6c5eece734c 100644 --- a/sys/dev/pci/ixgbe/ixgbe_osdep.h +++ b/sys/dev/pci/ixgbe/ixgbe_osdep.h @@ -30,8 +30,8 @@ POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_osdep.h 238149 2012-07-05 20:51:44Z jfv $*/ -/*$NetBSD: ixgbe_osdep.h,v 1.5 2015/04/02 09:26:55 msaitoh Exp $*/ +/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_osdep.h 240466 2012-09-13 14:40:24Z eadler $*/ +/*$NetBSD: ixgbe_osdep.h,v 1.6 2015/04/14 07:17:06 msaitoh Exp $*/ #ifndef _IXGBE_OS_H_ #define _IXGBE_OS_H_ @@ -67,12 +67,17 @@ #define DEBUGOUT1(S,A) printf(S "\n",A) #define DEBUGOUT2(S,A,B) printf(S "\n",A,B) #define DEBUGOUT3(S,A,B,C) printf(S "\n",A,B,C) + #define DEBUGOUT4(S,A,B,C,D) printf(S "\n",A,B,C,D) + #define DEBUGOUT5(S,A,B,C,D,E) printf(S "\n",A,B,C,D,E) + #define DEBUGOUT6(S,A,B,C,D,E,F) printf(S "\n",A,B,C,D,E,F) #define DEBUGOUT7(S,A,B,C,D,E,F,G) printf(S "\n",A,B,C,D,E,F,G) #else #define DEBUGOUT(S) do { } while (/*CONSTCOND*/false) #define DEBUGOUT1(S,A) do { } while (/*CONSTCOND*/false) #define DEBUGOUT2(S,A,B) do { } while (/*CONSTCOND*/false) #define DEBUGOUT3(S,A,B,C) do { } while (/*CONSTCOND*/false) + #define DEBUGOUT4(S,A,B,C,D) do { } while (/*CONSTCOND*/false) + #define DEBUGOUT5(S,A,B,C,D,E) do { } while (/*CONSTCOND*/false) #define DEBUGOUT6(S,A,B,C,D,E,F) \ do { } while (/*CONSTCOND*/false) #define DEBUGOUT7(S,A,B,C,D,E,F,G) \ @@ -132,6 +137,25 @@ void prefetch(void *x) #define prefetch(x) #endif +/* + * Optimized bcopy thanks to Luigi Rizzo's investigative work. Assumes + * non-overlapping regions and 32-byte padding on both src and dst. + */ +static __inline int +ixgbe_bcopy(void *_src, void *_dst, int l) +{ + uint64_t *src = _src; + uint64_t *dst = _dst; + + for (; l > 0; l -= 32) { + *dst++ = *src++; + *dst++ = *src++; + *dst++ = *src++; + *dst++ = *src++; + } + return (0); +} + struct ixgbe_osdep { struct ethercom ec; diff --git a/sys/dev/pci/ixgbe/ixv.c b/sys/dev/pci/ixgbe/ixv.c index 73f666c5fe8..764b571d14e 100644 --- a/sys/dev/pci/ixgbe/ixv.c +++ b/sys/dev/pci/ixgbe/ixv.c @@ -30,8 +30,8 @@ POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -/*$FreeBSD: head/sys/dev/ixgbe/ixv.c 238149 2012-07-05 20:51:44Z jfv $*/ -/*$NetBSD: ixv.c,v 1.6 2015/04/02 09:26:55 msaitoh Exp $*/ +/*$FreeBSD: head/sys/dev/ixgbe/ixv.c 241917 2012-10-22 22:29:48Z eadler $*/ +/*$NetBSD: ixv.c,v 1.7 2015/04/14 07:17:06 msaitoh Exp $*/ #include "opt_inet.h" #include "opt_inet6.h" @@ -706,7 +706,9 @@ ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) break; } enqueued++; - drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); + ifp->if_obytes += next->m_pkthdr.len; + if (next->m_flags & M_MCAST) + ifp->if_omcasts++; /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, next); if ((ifp->if_flags & IFF_RUNNING) == 0) |
