From da3bc07171dff957906cbe2ad5abb443eccf57c4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 18 Jan 2009 21:50:16 -0800 Subject: sfc: Replace LRO with GRO This patch makes sfc invoke the GRO hooks instead of LRO. As GRO has a compatible external interface to LRO this is a very straightforward replacement. Everything should appear identical to the user except that the offload is now controlled by the GRO ethtool option instead of LRO. I've kept the lro module parameter as is since that's for compatibility only. I have eliminated efx_rx_mk_skb as the GRO layer can take care of all packets regardless of whether GRO is enabled or not. So the only case where we don't call GRO is if the packet checksum is absent. This is to keep the behaviour changes of the patch to a minimum. Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- drivers/net/sfc/rx.c | 207 ++++----------------------------------------------- 1 file changed, 16 insertions(+), 191 deletions(-) (limited to 'drivers/net/sfc/rx.c') diff --git a/drivers/net/sfc/rx.c b/drivers/net/sfc/rx.c index b8ba4bbad889..a0345b380979 100644 --- a/drivers/net/sfc/rx.c +++ b/drivers/net/sfc/rx.c @@ -99,109 +99,6 @@ static inline unsigned int efx_rx_buf_size(struct efx_nic *efx) } -/************************************************************************** - * - * Linux generic LRO handling - * - ************************************************************************** - */ - -static int efx_lro_get_skb_hdr(struct sk_buff *skb, void **ip_hdr, - void **tcpudp_hdr, u64 *hdr_flags, void *priv) -{ - struct efx_channel *channel = priv; - struct iphdr *iph; - struct tcphdr *th; - - iph = (struct iphdr *)skb->data; - if (skb->protocol != htons(ETH_P_IP) || iph->protocol != IPPROTO_TCP) - goto fail; - - th = (struct tcphdr *)(skb->data + iph->ihl * 4); - - *tcpudp_hdr = th; - *ip_hdr = iph; - *hdr_flags = LRO_IPV4 | LRO_TCP; - - channel->rx_alloc_level += RX_ALLOC_FACTOR_LRO; - return 0; -fail: - channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB; - return -1; -} - -static int efx_get_frag_hdr(struct skb_frag_struct *frag, void **mac_hdr, - void **ip_hdr, void **tcpudp_hdr, u64 *hdr_flags, - void *priv) -{ - struct efx_channel *channel = priv; - struct ethhdr *eh; - struct iphdr *iph; - - /* We support EtherII and VLAN encapsulated IPv4 */ - eh = page_address(frag->page) + frag->page_offset; - *mac_hdr = eh; - - if (eh->h_proto == htons(ETH_P_IP)) { - iph = (struct iphdr *)(eh + 1); - } else { - struct vlan_ethhdr *veh = (struct vlan_ethhdr *)eh; - if (veh->h_vlan_encapsulated_proto != htons(ETH_P_IP)) - goto fail; - - iph = (struct iphdr *)(veh + 1); - } - *ip_hdr = iph; - - /* We can only do LRO over TCP */ - if (iph->protocol != IPPROTO_TCP) - goto fail; - - *hdr_flags = LRO_IPV4 | LRO_TCP; - *tcpudp_hdr = (struct tcphdr *)((u8 *) iph + iph->ihl * 4); - - channel->rx_alloc_level += RX_ALLOC_FACTOR_LRO; - return 0; - fail: - channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB; - return -1; -} - -int efx_lro_init(struct net_lro_mgr *lro_mgr, struct efx_nic *efx) -{ - size_t s = sizeof(struct net_lro_desc) * EFX_MAX_LRO_DESCRIPTORS; - struct net_lro_desc *lro_arr; - - /* Allocate the LRO descriptors structure */ - lro_arr = kzalloc(s, GFP_KERNEL); - if (lro_arr == NULL) - return -ENOMEM; - - lro_mgr->lro_arr = lro_arr; - lro_mgr->max_desc = EFX_MAX_LRO_DESCRIPTORS; - lro_mgr->max_aggr = EFX_MAX_LRO_AGGR; - lro_mgr->frag_align_pad = EFX_PAGE_SKB_ALIGN; - - lro_mgr->get_skb_header = efx_lro_get_skb_hdr; - lro_mgr->get_frag_header = efx_get_frag_hdr; - lro_mgr->dev = efx->net_dev; - - lro_mgr->features = LRO_F_NAPI; - - /* We can pass packets up with the checksum intact */ - lro_mgr->ip_summed = CHECKSUM_UNNECESSARY; - - lro_mgr->ip_summed_aggr = CHECKSUM_UNNECESSARY; - - return 0; -} - -void efx_lro_fini(struct net_lro_mgr *lro_mgr) -{ - kfree(lro_mgr->lro_arr); - lro_mgr->lro_arr = NULL; -} - /** * efx_init_rx_buffer_skb - create new RX buffer using skb-based allocation * @@ -549,77 +446,31 @@ static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, static void efx_rx_packet_lro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf) { - struct net_lro_mgr *lro_mgr = &channel->lro_mgr; - void *priv = channel; + struct napi_struct *napi = &channel->napi_str; /* Pass the skb/page into the LRO engine */ if (rx_buf->page) { - struct skb_frag_struct frags; + struct napi_gro_fraginfo info; - frags.page = rx_buf->page; - frags.page_offset = efx_rx_buf_offset(rx_buf); - frags.size = rx_buf->len; + info.frags[0].page = rx_buf->page; + info.frags[0].page_offset = efx_rx_buf_offset(rx_buf); + info.frags[0].size = rx_buf->len; + info.nr_frags = 1; + info.ip_summed = CHECKSUM_UNNECESSARY; + info.len = rx_buf->len; - lro_receive_frags(lro_mgr, &frags, rx_buf->len, - rx_buf->len, priv, 0); + napi_gro_frags(napi, &info); EFX_BUG_ON_PARANOID(rx_buf->skb); rx_buf->page = NULL; } else { EFX_BUG_ON_PARANOID(!rx_buf->skb); - lro_receive_skb(lro_mgr, rx_buf->skb, priv); + napi_gro_receive(napi, rx_buf->skb); rx_buf->skb = NULL; } } -/* Allocate and construct an SKB around a struct page.*/ -static struct sk_buff *efx_rx_mk_skb(struct efx_rx_buffer *rx_buf, - struct efx_nic *efx, - int hdr_len) -{ - struct sk_buff *skb; - - /* Allocate an SKB to store the headers */ - skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN); - if (unlikely(skb == NULL)) { - EFX_ERR_RL(efx, "RX out of memory for skb\n"); - return NULL; - } - - EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags); - EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len); - - skb->ip_summed = CHECKSUM_UNNECESSARY; - skb_reserve(skb, EFX_PAGE_SKB_ALIGN); - - skb->len = rx_buf->len; - skb->truesize = rx_buf->len + sizeof(struct sk_buff); - memcpy(skb->data, rx_buf->data, hdr_len); - skb->tail += hdr_len; - - /* Append the remaining page onto the frag list */ - if (unlikely(rx_buf->len > hdr_len)) { - struct skb_frag_struct *frag = skb_shinfo(skb)->frags; - frag->page = rx_buf->page; - frag->page_offset = efx_rx_buf_offset(rx_buf) + hdr_len; - frag->size = skb->len - hdr_len; - skb_shinfo(skb)->nr_frags = 1; - skb->data_len = frag->size; - } else { - __free_pages(rx_buf->page, efx->rx_buffer_order); - skb->data_len = 0; - } - - /* Ownership has transferred from the rx_buf to skb */ - rx_buf->page = NULL; - - /* Move past the ethernet header */ - skb->protocol = eth_type_trans(skb, efx->net_dev); - - return skb; -} - void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, unsigned int len, bool checksummed, bool discard) { @@ -687,7 +538,6 @@ void __efx_rx_packet(struct efx_channel *channel, { struct efx_nic *efx = channel->efx; struct sk_buff *skb; - bool lro = !!(efx->net_dev->features & NETIF_F_LRO); /* If we're in loopback test, then pass the packet directly to the * loopback layer, and free the rx_buf here @@ -709,41 +559,21 @@ void __efx_rx_packet(struct efx_channel *channel, efx->net_dev); } - /* Both our generic-LRO and SFC-SSR support skb and page based - * allocation, but neither support switching from one to the - * other on the fly. If we spot that the allocation mode has - * changed, then flush the LRO state. - */ - if (unlikely(channel->rx_alloc_pop_pages != (rx_buf->page != NULL))) { - efx_flush_lro(channel); - channel->rx_alloc_pop_pages = (rx_buf->page != NULL); - } - if (likely(checksummed && lro)) { + if (likely(checksummed || rx_buf->page)) { efx_rx_packet_lro(channel, rx_buf); goto done; } - /* Form an skb if required */ - if (rx_buf->page) { - int hdr_len = min(rx_buf->len, EFX_SKB_HEADERS); - skb = efx_rx_mk_skb(rx_buf, efx, hdr_len); - if (unlikely(skb == NULL)) { - efx_free_rx_buffer(efx, rx_buf); - goto done; - } - } else { - /* We now own the SKB */ - skb = rx_buf->skb; - rx_buf->skb = NULL; - } + /* We now own the SKB */ + skb = rx_buf->skb; + rx_buf->skb = NULL; EFX_BUG_ON_PARANOID(rx_buf->page); EFX_BUG_ON_PARANOID(rx_buf->skb); EFX_BUG_ON_PARANOID(!skb); /* Set the SKB flags */ - if (unlikely(!checksummed || !efx->rx_checksum_enabled)) - skb->ip_summed = CHECKSUM_NONE; + skb->ip_summed = CHECKSUM_NONE; /* Pass the packet up */ netif_receive_skb(skb); @@ -760,7 +590,7 @@ void efx_rx_strategy(struct efx_channel *channel) enum efx_rx_alloc_method method = rx_alloc_method; /* Only makes sense to use page based allocation if LRO is enabled */ - if (!(channel->efx->net_dev->features & NETIF_F_LRO)) { + if (!(channel->efx->net_dev->features & NETIF_F_GRO)) { method = RX_ALLOC_METHOD_SKB; } else if (method == RX_ALLOC_METHOD_AUTO) { /* Constrain the rx_alloc_level */ @@ -865,11 +695,6 @@ void efx_remove_rx_queue(struct efx_rx_queue *rx_queue) rx_queue->buffer = NULL; } -void efx_flush_lro(struct efx_channel *channel) -{ - lro_flush_all(&channel->lro_mgr); -} - module_param(rx_alloc_method, int, 0644); MODULE_PARM_DESC(rx_alloc_method, "Allocation method used for RX buffers"); -- cgit v1.2.3-58-ga151 From 0c8dfc830aadd978e461dad66c33741b71c6a0be Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 27 Jan 2009 16:22:32 -0800 Subject: net: Add skb_record_rx_queue() calls to multiqueue capable drivers. Signed-off-by: David S. Miller --- drivers/net/bnx2.c | 2 ++ drivers/net/bnx2x_main.c | 1 + drivers/net/cxgb3/sge.c | 1 + drivers/net/igb/igb_main.c | 1 + drivers/net/ixgbe/ixgbe_main.c | 1 + drivers/net/mlx4/en_rx.c | 1 + drivers/net/myri10ge/myri10ge.c | 1 + drivers/net/niu.c | 1 + drivers/net/qlge/qlge_main.c | 1 + drivers/net/s2io.c | 1 + drivers/net/sfc/rx.c | 2 ++ 11 files changed, 13 insertions(+) (limited to 'drivers/net/sfc/rx.c') diff --git a/drivers/net/bnx2.c b/drivers/net/bnx2.c index fe575b9a9b73..49e0e51a9dfc 100644 --- a/drivers/net/bnx2.c +++ b/drivers/net/bnx2.c @@ -3007,6 +3007,8 @@ bnx2_rx_int(struct bnx2 *bp, struct bnx2_napi *bnapi, int budget) skb->ip_summed = CHECKSUM_UNNECESSARY; } + skb_record_rx_queue(skb, bnapi - &bp->bnx2_napi[0]); + #ifdef BCM_VLAN if (hw_vlan) vlan_hwaccel_receive_skb(skb, bp->vlgrp, vtag); diff --git a/drivers/net/bnx2x_main.c b/drivers/net/bnx2x_main.c index 71f81c79d638..88da14c141f4 100644 --- a/drivers/net/bnx2x_main.c +++ b/drivers/net/bnx2x_main.c @@ -1325,6 +1325,7 @@ static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, skb->protocol = eth_type_trans(skb, bp->dev); skb->ip_summed = CHECKSUM_UNNECESSARY; + skb_record_rx_queue(skb, queue); { struct iphdr *iph; diff --git a/drivers/net/cxgb3/sge.c b/drivers/net/cxgb3/sge.c index 8299fb538f25..272a0168f3e9 100644 --- a/drivers/net/cxgb3/sge.c +++ b/drivers/net/cxgb3/sge.c @@ -1937,6 +1937,7 @@ static void rx_eth(struct adapter *adap, struct sge_rspq *rq, skb->ip_summed = CHECKSUM_UNNECESSARY; } else skb->ip_summed = CHECKSUM_NONE; + skb_record_rx_queue(skb, qs - &adap->sge.qs[0]); if (unlikely(p->vlan_valid)) { struct vlan_group *grp = pi->vlan_grp; diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c index e11043d90dbd..bd166803671d 100644 --- a/drivers/net/igb/igb_main.c +++ b/drivers/net/igb/igb_main.c @@ -3751,6 +3751,7 @@ static void igb_receive_skb(struct igb_ring *ring, u8 status, struct igb_adapter * adapter = ring->adapter; bool vlan_extracted = (adapter->vlgrp && (status & E1000_RXD_STAT_VP)); + skb_record_rx_queue(skb, ring->queue_index); if (skb->ip_summed == CHECKSUM_UNNECESSARY) { if (vlan_extracted) vlan_gro_receive(&ring->napi, adapter->vlgrp, diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c index 43980dc45e3e..fe4a4d17c4bc 100644 --- a/drivers/net/ixgbe/ixgbe_main.c +++ b/drivers/net/ixgbe/ixgbe_main.c @@ -414,6 +414,7 @@ static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector, bool is_vlan = (status & IXGBE_RXD_STAT_VP); u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); + skb_record_rx_queue(skb, q_vector - &adapter->q_vector[0]); if (skb->ip_summed == CHECKSUM_UNNECESSARY) { if (adapter->vlgrp && is_vlan && (tag != 0)) vlan_gro_receive(napi, adapter->vlgrp, tag, skb); diff --git a/drivers/net/mlx4/en_rx.c b/drivers/net/mlx4/en_rx.c index ac55ebd2f146..a4130e764991 100644 --- a/drivers/net/mlx4/en_rx.c +++ b/drivers/net/mlx4/en_rx.c @@ -768,6 +768,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud skb->ip_summed = ip_summed; skb->protocol = eth_type_trans(skb, dev); + skb_record_rx_queue(skb, cq->ring); /* Push it up the stack */ if (priv->vlgrp && (be32_to_cpu(cqe->vlan_my_qpn) & diff --git a/drivers/net/myri10ge/myri10ge.c b/drivers/net/myri10ge/myri10ge.c index 2dacb8852dc3..aea9fdaa3cd5 100644 --- a/drivers/net/myri10ge/myri10ge.c +++ b/drivers/net/myri10ge/myri10ge.c @@ -1324,6 +1324,7 @@ myri10ge_rx_done(struct myri10ge_slice_state *ss, struct myri10ge_rx_buf *rx, skb_shinfo(skb)->nr_frags = 0; } skb->protocol = eth_type_trans(skb, dev); + skb_record_rx_queue(skb, ss - &mgp->ss[0]); if (mgp->csum_flag) { if ((skb->protocol == htons(ETH_P_IP)) || diff --git a/drivers/net/niu.c b/drivers/net/niu.c index 4a5a089fa301..2346ca6bf5ba 100644 --- a/drivers/net/niu.c +++ b/drivers/net/niu.c @@ -3390,6 +3390,7 @@ static int niu_process_rx_pkt(struct niu *np, struct rx_ring_info *rp) rp->rx_bytes += skb->len; skb->protocol = eth_type_trans(skb, np->dev); + skb_record_rx_queue(skb, rp->rx_channel); netif_receive_skb(skb); return num_rcr; diff --git a/drivers/net/qlge/qlge_main.c b/drivers/net/qlge/qlge_main.c index 16eb9dd85286..4ab6e72ea95c 100644 --- a/drivers/net/qlge/qlge_main.c +++ b/drivers/net/qlge/qlge_main.c @@ -1436,6 +1436,7 @@ static void ql_process_mac_rx_intr(struct ql_adapter *qdev, qdev->stats.rx_packets++; qdev->stats.rx_bytes += skb->len; skb->protocol = eth_type_trans(skb, ndev); + skb_record_rx_queue(skb, rx_ring - &qdev->rx_ring[0]); if (qdev->vlgrp && (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V)) { QPRINTK(qdev, RX_STATUS, DEBUG, "Passing a VLAN packet upstream.\n"); diff --git a/drivers/net/s2io.c b/drivers/net/s2io.c index 2a96a10fd0cf..e0a353f4ec92 100644 --- a/drivers/net/s2io.c +++ b/drivers/net/s2io.c @@ -7542,6 +7542,7 @@ static int rx_osm_handler(struct ring_info *ring_data, struct RxD_t * rxdp) sp->mac_control.stats_info->sw_stat.mem_freed += skb->truesize; send_up: + skb_record_rx_queue(skb, ring_no); queue_rx_frame(skb, RXD_GET_VLAN_TAG(rxdp->Control_2)); aggregate: sp->mac_control.rings[ring_no].rx_bufs_left -= 1; diff --git a/drivers/net/sfc/rx.c b/drivers/net/sfc/rx.c index a0345b380979..66d7fe3db3e6 100644 --- a/drivers/net/sfc/rx.c +++ b/drivers/net/sfc/rx.c @@ -575,6 +575,8 @@ void __efx_rx_packet(struct efx_channel *channel, /* Set the SKB flags */ skb->ip_summed = CHECKSUM_NONE; + skb_record_rx_queue(skb, channel->channel); + /* Pass the packet up */ netif_receive_skb(skb); -- cgit v1.2.3-58-ga151