Index: conf/NOTES =================================================================== --- conf/NOTES (revision 227552) +++ conf/NOTES (working copy) @@ -799,6 +799,12 @@ # option. DHCP requires bpf. device bpf +# The `netmap' device implements memory-mapped access to network +# devices from userspace, enabling wire-speed packet capture and +# generation even at 10Gbit/s. Requires support in the device +# driver. Supported drivers are ixgbe, e1000, re. +device netmap + # The `disc' device implements a minimal network interface, # which throws away all packets sent and never receives any. It is # included for testing and benchmarking purposes. Index: conf/files =================================================================== --- conf/files (revision 227552) +++ conf/files (working copy) @@ -1507,6 +1507,7 @@ dev/my/if_my.c optional my dev/ncv/ncr53c500.c optional ncv dev/ncv/ncr53c500_pccard.c optional ncv pccard +dev/netmap/netmap.c optional netmap dev/nge/if_nge.c optional nge dev/nxge/if_nxge.c optional nxge dev/nxge/xgehal/xgehal-device.c optional nxge Index: conf/options =================================================================== --- conf/options (revision 227552) +++ conf/options (working copy) @@ -689,6 +689,7 @@ # various 'device presence' options. DEV_BPF opt_bpf.h +DEV_NETMAP opt_global.h DEV_MCA opt_mca.h DEV_CARP opt_carp.h DEV_SPLASH opt_splash.h Index: dev/e1000/if_igb.c =================================================================== --- dev/e1000/if_igb.c (revision 227552) +++ dev/e1000/if_igb.c (working copy) @@ -369,6 +369,9 @@ &igb_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); +#ifdef DEV_NETMAP +#include +#endif /* DEV_NETMAP */ /********************************************************************* * Device identification routine * @@ -664,6 +667,9 @@ adapter->led_dev = led_create(igb_led_func, adapter, device_get_nameunit(dev)); +#ifdef DEV_NETMAP + igb_netmap_attach(adapter); +#endif /* DEV_NETMAP */ INIT_DEBUGOUT("igb_attach: end"); return (0); @@ -742,6 +748,9 @@ callout_drain(&adapter->timer); +#ifdef DEV_NETMAP + netmap_detach(adapter->ifp); +#endif /* DEV_NETMAP */ igb_free_pci_resources(adapter); bus_generic_detach(dev); if_free(ifp); @@ -3212,6 +3221,10 @@ struct adapter *adapter = txr->adapter; struct igb_tx_buffer *txbuf; int i; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), + NR_TX, txr->me, 0); +#endif /* Clear the old descriptor contents */ IGB_TX_LOCK(txr); @@ -3231,6 +3244,13 @@ m_freem(txbuf->m_head); txbuf->m_head = NULL; } +#ifdef DEV_NETMAP + if (slot) { + netmap_load_map(txr->txtag, txbuf->map, + NMB(slot), adapter->rx_mbuf_sz); + slot++; + } +#endif /* DEV_NETMAP */ /* clear the watch index */ txbuf->next_eop = -1; } @@ -3626,6 +3646,19 @@ IGB_TX_LOCK_ASSERT(txr); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_adapter *na = NA(ifp); + + selwakeuppri(&na->tx_rings[txr->me].si, PI_NET); + IGB_TX_UNLOCK(txr); + IGB_CORE_LOCK(adapter); + selwakeuppri(&na->tx_rings[na->num_queues + 1].si, PI_NET); + IGB_CORE_UNLOCK(adapter); + IGB_TX_LOCK(txr); // the caller is supposed to own the lock + return FALSE; + } +#endif /* DEV_NETMAP */ if (txr->tx_avail == adapter->num_tx_desc) { txr->queue_status = IGB_QUEUE_IDLE; return FALSE; @@ -3949,6 +3982,10 @@ bus_dma_segment_t pseg[1], hseg[1]; struct lro_ctrl *lro = &rxr->lro; int rsize, nsegs, error = 0; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(rxr->adapter->ifp), + NR_RX, rxr->me, 0); +#endif adapter = rxr->adapter; dev = adapter->dev; @@ -3974,6 +4011,18 @@ struct mbuf *mh, *mp; rxbuf = &rxr->rx_buffers[j]; +#ifdef DEV_NETMAP + if (slot) { + netmap_load_map(rxr->ptag, + rxbuf->pmap, NMB(slot), + adapter->rx_mbuf_sz); + /* Update descriptor */ + rxr->rx_base[j].read.pkt_addr = + htole64(vtophys(NMB(slot))); + slot++; + continue; + } +#endif /* DEV_NETMAP */ if (rxr->hdr_split == FALSE) goto skip_head; @@ -4436,6 +4485,19 @@ bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_adapter *na = NA(ifp); + + selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET); + IGB_RX_UNLOCK(rxr); + IGB_CORE_LOCK(adapter); + selwakeuppri(&na->rx_rings[na->num_queues + 1].si, PI_NET); + IGB_CORE_UNLOCK(adapter); + return (0); + } +#endif /* DEV_NETMAP */ + /* Main clean loop */ for (i = rxr->next_to_check; count != 0;) { struct mbuf *sendmp, *mh, *mp; Index: dev/e1000/if_lem.c =================================================================== --- dev/e1000/if_lem.c (revision 227552) +++ dev/e1000/if_lem.c (working copy) @@ -316,6 +316,10 @@ /* Global used in WOL setup with multiport cards */ static int global_quad_port_a = 0; +#ifdef DEV_NETMAP +#include +#endif /* DEV_NETMAP */ + /********************************************************************* * Device identification routine * @@ -646,6 +650,9 @@ adapter->led_dev = led_create(lem_led_func, adapter, device_get_nameunit(dev)); +#ifdef DEV_NETMAP + lem_netmap_attach(adapter); +#endif /* DEV_NETMAP */ INIT_DEBUGOUT("lem_attach: end"); return (0); @@ -724,6 +731,9 @@ callout_drain(&adapter->timer); callout_drain(&adapter->tx_fifo_timer); +#ifdef DEV_NETMAP + netmap_detach(ifp); +#endif /* DEV_NETMAP */ lem_free_pci_resources(adapter); bus_generic_detach(dev); if_free(ifp); @@ -2637,6 +2647,9 @@ lem_setup_transmit_structures(struct adapter *adapter) { struct em_buffer *tx_buffer; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), NR_TX, 0, 0); +#endif /* Clear the old ring contents */ bzero(adapter->tx_desc_base, @@ -2650,6 +2663,15 @@ bus_dmamap_unload(adapter->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; +#ifdef DEV_NETMAP + if (slot) { + /* reload the map for netmap mode */ + netmap_load_map(adapter->txtag, + tx_buffer->map, NMB(slot), + NA(adapter->ifp)->buff_size); + slot++; + } +#endif /* DEV_NETMAP */ tx_buffer->next_eop = -1; } @@ -2951,6 +2973,12 @@ EM_TX_LOCK_ASSERT(adapter); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + selwakeuppri(&NA(ifp)->tx_rings[0].si, PI_NET); + return; + } +#endif /* DEV_NETMAP */ if (adapter->num_tx_desc_avail == adapter->num_tx_desc) return; @@ -3181,6 +3209,9 @@ { struct em_buffer *rx_buffer; int i, error; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), NR_RX, 0, 0); +#endif /* Reset descriptor ring */ bzero(adapter->rx_desc_base, @@ -3200,6 +3231,18 @@ /* Allocate new ones. */ for (i = 0; i < adapter->num_rx_desc; i++) { +#ifdef DEV_NETMAP + if (slot) { + netmap_load_map(adapter->rxtag, + rx_buffer->map, NMB(slot), + NA(adapter->ifp)->buff_size); + /* Update descriptor */ + adapter->rx_desc_base[i].buffer_addr = + htole64(vtophys(NMB(slot))); + slot++; + continue; + } +#endif /* DEV_NETMAP */ error = lem_get_buf(adapter, i); if (error) return (error); @@ -3407,6 +3450,14 @@ bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_POSTREAD); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + selwakeuppri(&NA(ifp)->rx_rings[0].si, PI_NET); + EM_RX_UNLOCK(adapter); + return (0); + } +#endif /* DEV_NETMAP */ + if (!((current_desc->status) & E1000_RXD_STAT_DD)) { if (done != NULL) *done = rx_sent; Index: dev/e1000/if_em.c =================================================================== --- dev/e1000/if_em.c (revision 227552) +++ dev/e1000/if_em.c (working copy) @@ -399,6 +399,10 @@ /* Global used in WOL setup with multiport cards */ static int global_quad_port_a = 0; +#ifdef DEV_NETMAP +#include +#endif /* DEV_NETMAP */ + /********************************************************************* * Device identification routine * @@ -714,6 +718,9 @@ adapter->led_dev = led_create(em_led_func, adapter, device_get_nameunit(dev)); +#ifdef DEV_NETMAP + em_netmap_attach(adapter); +#endif /* DEV_NETMAP */ INIT_DEBUGOUT("em_attach: end"); @@ -785,6 +792,10 @@ ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); +#ifdef DEV_NETMAP + netmap_detach(ifp); +#endif /* DEV_NETMAP */ + em_free_pci_resources(adapter); bus_generic_detach(dev); if_free(ifp); @@ -3213,6 +3224,10 @@ struct adapter *adapter = txr->adapter; struct em_buffer *txbuf; int i; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), + NR_TX, txr->me, 0); +#endif /* Clear the old descriptor contents */ EM_TX_LOCK(txr); @@ -3232,6 +3247,16 @@ m_freem(txbuf->m_head); txbuf->m_head = NULL; } +#ifdef DEV_NETMAP + if (slot) { + /* reload the map for netmap mode */ + netmap_load_map(txr->txtag, + txbuf->map, NMB(slot), + adapter->rx_mbuf_sz); + slot++; + } +#endif /* DEV_NETMAP */ + /* clear the watch index */ txbuf->next_eop = -1; } @@ -3682,6 +3707,12 @@ struct ifnet *ifp = adapter->ifp; EM_TX_LOCK_ASSERT(txr); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + selwakeuppri(&NA(ifp)->tx_rings[txr->me].si, PI_NET); + return (FALSE); + } +#endif /* DEV_NETMAP */ /* No work, make sure watchdog is off */ if (txr->tx_avail == adapter->num_tx_desc) { @@ -3978,6 +4009,33 @@ if (++j == adapter->num_rx_desc) j = 0; } +#ifdef DEV_NETMAP + { + /* slot is NULL if we are not in netmap mode */ + struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), + NR_RX, rxr->me, rxr->next_to_check); + /* + * we need to restore all buffer addresses in the ring as they might + * be in the wrong state if we are exiting from netmap mode. + */ + for (j = 0; j != adapter->num_rx_desc; ++j) { + void *addr; + rxbuf = &rxr->rx_buffers[j]; + if (rxbuf->m_head == NULL && !slot) + continue; + addr = slot ? NMB(slot) : rxbuf->m_head->m_data; + // XXX load or reload ? + netmap_load_map(rxr->rxtag, rxbuf->map, addr, adapter->rx_mbuf_sz); + /* Update descriptor */ + rxr->rx_base[j].buffer_addr = htole64(vtophys(addr)); + bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); + if (slot) + slot++; + } + /* Setup our descriptor indices */ + NA(adapter->ifp)->rx_rings[rxr->me].nr_hwcur = rxr->next_to_check; + } +#endif /* DEV_NETMAP */ fail: rxr->next_to_refresh = i; @@ -4247,6 +4305,14 @@ EM_RX_LOCK(rxr); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + selwakeuppri(&NA(ifp)->rx_rings[rxr->me].si, PI_NET); + EM_RX_UNLOCK(rxr); + return (0); + } +#endif /* DEV_NETMAP */ + for (i = rxr->next_to_check, processed = 0; count != 0;) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) Index: dev/re/if_re.c =================================================================== --- dev/re/if_re.c (revision 227552) +++ dev/re/if_re.c (working copy) @@ -291,6 +291,10 @@ static void re_setwol (struct rl_softc *); static void re_clrwol (struct rl_softc *); +#ifdef DEV_NETMAP +#include +#endif /* !DEV_NETMAP */ + #ifdef RE_DIAG static int re_diag (struct rl_softc *); #endif @@ -1583,6 +1587,9 @@ */ ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); +#ifdef DEV_NETMAP + re_netmap_attach(sc); +#endif /* DEV_NETMAP */ #ifdef RE_DIAG /* * Perform hardware diagnostic on the original RTL8169. @@ -1778,6 +1785,9 @@ bus_dma_tag_destroy(sc->rl_ldata.rl_stag); } +#ifdef DEV_NETMAP + netmap_detach(ifp); +#endif /* DEV_NETMAP */ if (sc->rl_parent_tag) bus_dma_tag_destroy(sc->rl_parent_tag); @@ -1952,6 +1962,9 @@ sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc)); for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) sc->rl_ldata.rl_tx_desc[i].tx_m = NULL; +#ifdef DEV_NETMAP + re_netmap_tx_init(sc); +#endif /* DEV_NETMAP */ /* Set EOR. */ desc = &sc->rl_ldata.rl_tx_list[sc->rl_ldata.rl_tx_desc_cnt - 1]; desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOR); @@ -1979,6 +1992,9 @@ if ((error = re_newbuf(sc, i)) != 0) return (error); } +#ifdef DEV_NETMAP + re_netmap_rx_init(sc); +#endif /* DEV_NETMAP */ /* Flush the RX descriptors */ @@ -2035,6 +2051,12 @@ RL_LOCK_ASSERT(sc); ifp = sc->rl_ifp; +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + selwakeuppri(&NA(ifp)->rx_rings->si, PI_NET); + return 0; + } +#endif /* DEV_NETMAP */ if (ifp->if_mtu > RL_MTU && (sc->rl_flags & RL_FLAG_JUMBOV2) != 0) jumbo = 1; else @@ -2276,6 +2298,12 @@ return; ifp = sc->rl_ifp; +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + selwakeuppri(&NA(ifp)->tx_rings[0].si, PI_NET); + return; + } +#endif /* DEV_NETMAP */ /* Invalidate the TX descriptor list */ bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, sc->rl_ldata.rl_tx_list_map, @@ -2794,6 +2822,20 @@ sc = ifp->if_softc; +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_kring *kring = &NA(ifp)->tx_rings[0]; + if (sc->rl_ldata.rl_tx_prodidx != kring->nr_hwcur) { + /* kick the tx unit */ + CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START); +#ifdef RE_TX_MODERATION + CSR_WRITE_4(sc, RL_TIMERCNT, 1); +#endif + sc->rl_watchdog_timer = 5; + } + return; + } +#endif /* DEV_NETMAP */ if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING || (sc->rl_flags & RL_FLAG_LINK) == 0) return; Index: dev/ixgbe/ixgbe.c =================================================================== --- dev/ixgbe/ixgbe.c (revision 227552) +++ dev/ixgbe/ixgbe.c (working copy) @@ -313,6 +313,10 @@ static int fdir_pballoc = 1; #endif +#ifdef DEV_NETMAP +#include +#endif /* DEV_NETMAP */ + /********************************************************************* * Device identification routine * @@ -578,6 +582,9 @@ ixgbe_add_hw_stats(adapter); +#ifdef DEV_NETMAP + ixgbe_netmap_attach(adapter); +#endif /* DEV_NETMAP */ INIT_DEBUGOUT("ixgbe_attach: end"); return (0); err_late: @@ -652,6 +659,9 @@ ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); +#ifdef DEV_NETMAP + netmap_detach(adapter->ifp); +#endif /* DEV_NETMAP */ ixgbe_free_pci_resources(adapter); bus_generic_detach(dev); if_free(adapter->ifp); @@ -1719,6 +1729,7 @@ if (++i == adapter->num_tx_desc) i = 0; + // XXX should we sync each buffer ? txbuf->m_head = NULL; txbuf->eop_index = -1; } @@ -2813,6 +2824,10 @@ struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *txbuf; int i; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), + NR_TX, txr->me, 0); +#endif /* Clear the old ring contents */ IXGBE_TX_LOCK(txr); @@ -2832,6 +2847,13 @@ m_freem(txbuf->m_head); txbuf->m_head = NULL; } +#ifdef DEV_NETMAP + if (slot) { + netmap_load_map(txr->txtag, txbuf->map, + NMB(slot), adapter->rx_mbuf_sz); + slot++; + } +#endif /* DEV_NETMAP */ /* Clear the EOP index */ txbuf->eop_index = -1; } @@ -3310,6 +3332,20 @@ mtx_assert(&txr->tx_mtx, MA_OWNED); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_adapter *na = NA(ifp); + + selwakeuppri(&na->tx_rings[txr->me].si, PI_NET); + IXGBE_TX_UNLOCK(txr); + IXGBE_CORE_LOCK(adapter); + selwakeuppri(&na->tx_rings[na->num_queues + 1].si, PI_NET); + IXGBE_CORE_UNLOCK(adapter); + IXGBE_TX_LOCK(txr); // the caller is supposed to own the lock + return (FALSE); + } +#endif /* DEV_NETMAP */ + if (txr->tx_avail == adapter->num_tx_desc) { txr->queue_status = IXGBE_QUEUE_IDLE; return FALSE; @@ -3698,6 +3734,10 @@ bus_dma_segment_t pseg[1], hseg[1]; struct lro_ctrl *lro = &rxr->lro; int rsize, nsegs, error = 0; +#ifdef DEV_NETMAP + struct netmap_slot *slot = netmap_reset(NA(rxr->adapter->ifp), + NR_RX, rxr->me, 0); +#endif /* DEV_NETMAP */ adapter = rxr->adapter; ifp = adapter->ifp; @@ -3721,6 +3761,18 @@ struct mbuf *mh, *mp; rxbuf = &rxr->rx_buffers[j]; +#ifdef DEV_NETMAP + if (slot) { + netmap_load_map(rxr->ptag, + rxbuf->pmap, NMB(slot), + adapter->rx_mbuf_sz); + /* Update descriptor */ + rxr->rx_base[j].read.pkt_addr = + htole64(vtophys(NMB(slot))); + slot++; + continue; + } +#endif /* DEV_NETMAP */ /* ** Don't allocate mbufs if not ** doing header split, its wasteful @@ -4148,6 +4200,18 @@ IXGBE_RX_LOCK(rxr); +#ifdef DEV_NETMAP + if (ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_adapter *na = NA(ifp); + + selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET); + IXGBE_RX_UNLOCK(rxr); + IXGBE_CORE_LOCK(adapter); + selwakeuppri(&na->rx_rings[na->num_queues + 1].si, PI_NET); + IXGBE_CORE_UNLOCK(adapter); + return (0); + } +#endif /* DEV_NETMAP */ for (i = rxr->next_to_check; count != 0;) { struct mbuf *sendmp, *mh, *mp; u32 rsc, ptype;