diff options
Diffstat (limited to 'drivers/net/ethernet/meta/fbnic/fbnic_txrx.c')
| -rw-r--r-- | drivers/net/ethernet/meta/fbnic/fbnic_txrx.c | 1913 |
1 files changed, 1913 insertions, 0 deletions
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c new file mode 100644 index 000000000000..0ed4c9fff5d8 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -0,0 +1,1913 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <net/netdev_queues.h> +#include <net/page_pool/helpers.h> + +#include "fbnic.h" +#include "fbnic_csr.h" +#include "fbnic_netdev.h" +#include "fbnic_txrx.h" + +struct fbnic_xmit_cb { + u32 bytecount; + u8 desc_count; + int hw_head; +}; + +#define FBNIC_XMIT_CB(__skb) ((struct fbnic_xmit_cb *)((__skb)->cb)) + +static u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring) +{ + unsigned long csr_base = (unsigned long)ring->doorbell; + + csr_base &= ~(FBNIC_QUEUE_STRIDE * sizeof(u32) - 1); + + return (u32 __iomem *)csr_base; +} + +static u32 fbnic_ring_rd32(struct fbnic_ring *ring, unsigned int csr) +{ + u32 __iomem *csr_base = fbnic_ring_csr_base(ring); + + return readl(csr_base + csr); +} + +static void fbnic_ring_wr32(struct fbnic_ring *ring, unsigned int csr, u32 val) +{ + u32 __iomem *csr_base = fbnic_ring_csr_base(ring); + + writel(val, csr_base + csr); +} + +static unsigned int fbnic_desc_unused(struct fbnic_ring *ring) +{ + return (ring->head - ring->tail - 1) & ring->size_mask; +} + +static unsigned int fbnic_desc_used(struct fbnic_ring *ring) +{ + return (ring->tail - ring->head) & ring->size_mask; +} + +static struct netdev_queue *txring_txq(const struct net_device *dev, + const struct fbnic_ring *ring) +{ + return netdev_get_tx_queue(dev, ring->q_idx); +} + +static int fbnic_maybe_stop_tx(const struct net_device *dev, + struct fbnic_ring *ring, + const unsigned int size) +{ + struct netdev_queue *txq = txring_txq(dev, ring); + int res; + + res = netif_txq_maybe_stop(txq, fbnic_desc_unused(ring), size, + FBNIC_TX_DESC_WAKEUP); + + return !res; +} + +static bool fbnic_tx_sent_queue(struct sk_buff *skb, struct fbnic_ring *ring) +{ + struct netdev_queue *dev_queue = txring_txq(skb->dev, ring); + unsigned int bytecount = FBNIC_XMIT_CB(skb)->bytecount; + bool xmit_more = netdev_xmit_more(); + + /* TBD: Request completion more often if xmit_more becomes large */ + + return __netdev_tx_sent_queue(dev_queue, bytecount, xmit_more); +} + +static void fbnic_unmap_single_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + unsigned int len; + dma_addr_t dma; + + dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); + len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); + + dma_unmap_single(dev, dma, len, DMA_TO_DEVICE); +} + +static void fbnic_unmap_page_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + unsigned int len; + dma_addr_t dma; + + dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); + len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); + + dma_unmap_page(dev, dma, len, DMA_TO_DEVICE); +} + +#define FBNIC_TWD_TYPE(_type) \ + cpu_to_le64(FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_##_type)) + +static bool +fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) +{ + unsigned int l2len, i3len; + + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) + return false; + + l2len = skb_mac_header_len(skb); + i3len = skb_checksum_start(skb) - skb_network_header(skb); + + *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_CSUM_OFFSET_MASK, + skb->csum_offset / 2)); + + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_CSO); + + *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) | + FIELD_PREP(FBNIC_TWD_L3_IHLEN_MASK, i3len / 2)); + return false; +} + +static void +fbnic_rx_csum(u64 rcd, struct sk_buff *skb, struct fbnic_ring *rcq) +{ + skb_checksum_none_assert(skb); + + if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) + return; + + if (FIELD_GET(FBNIC_RCD_META_L4_CSUM_UNNECESSARY, rcd)) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + u16 csum = FIELD_GET(FBNIC_RCD_META_L2_CSUM_MASK, rcd); + + skb->ip_summed = CHECKSUM_COMPLETE; + skb->csum = (__force __wsum)csum; + } +} + +static bool +fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) +{ + struct device *dev = skb->dev->dev.parent; + unsigned int tail = ring->tail, first; + unsigned int size, data_len; + skb_frag_t *frag; + dma_addr_t dma; + __le64 *twd; + + ring->tx_buf[tail] = skb; + + tail++; + tail &= ring->size_mask; + first = tail; + + size = skb_headlen(skb); + data_len = skb->data_len; + + if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) + goto dma_error; + + dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); + + for (frag = &skb_shinfo(skb)->frags[0];; frag++) { + twd = &ring->desc[tail]; + + if (dma_mapping_error(dev, dma)) + goto dma_error; + + *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | + FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | + FIELD_PREP(FBNIC_TWD_TYPE_MASK, + FBNIC_TWD_TYPE_AL)); + + tail++; + tail &= ring->size_mask; + + if (!data_len) + break; + + size = skb_frag_size(frag); + data_len -= size; + + if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) + goto dma_error; + + dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); + } + + *twd |= FBNIC_TWD_TYPE(LAST_AL); + + FBNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask; + + ring->tail = tail; + + /* Verify there is room for another packet */ + fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC); + + if (fbnic_tx_sent_queue(skb, ring)) { + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_COMPLETION); + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(tail, ring->doorbell); + } + + return false; +dma_error: + if (net_ratelimit()) + netdev_err(skb->dev, "TX DMA map failed\n"); + + while (tail != first) { + tail--; + tail &= ring->size_mask; + twd = &ring->desc[tail]; + if (tail == first) + fbnic_unmap_single_twd(dev, twd); + else + fbnic_unmap_page_twd(dev, twd); + } + + return true; +} + +#define FBNIC_MIN_FRAME_LEN 60 + +static netdev_tx_t +fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring) +{ + __le64 *meta = &ring->desc[ring->tail]; + u16 desc_needed; + + if (skb_put_padto(skb, FBNIC_MIN_FRAME_LEN)) + goto err_count; + + /* Need: 1 descriptor per page, + * + 1 desc for skb_head, + * + 2 desc for metadata and timestamp metadata + * + 7 desc gap to keep tail from touching head + * otherwise try next time + */ + desc_needed = skb_shinfo(skb)->nr_frags + 10; + if (fbnic_maybe_stop_tx(skb->dev, ring, desc_needed)) + return NETDEV_TX_BUSY; + + *meta = cpu_to_le64(FBNIC_TWD_FLAG_DEST_MAC); + + /* Write all members within DWORD to condense this into 2 4B writes */ + FBNIC_XMIT_CB(skb)->bytecount = skb->len; + FBNIC_XMIT_CB(skb)->desc_count = 0; + + if (fbnic_tx_offloads(ring, skb, meta)) + goto err_free; + + if (fbnic_tx_map(ring, skb, meta)) + goto err_free; + + return NETDEV_TX_OK; + +err_free: + dev_kfree_skb_any(skb); +err_count: + return NETDEV_TX_OK; +} + +netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev) +{ + struct fbnic_net *fbn = netdev_priv(dev); + unsigned int q_map = skb->queue_mapping; + + return fbnic_xmit_frame_ring(skb, fbn->tx[q_map]); +} + +netdev_features_t +fbnic_features_check(struct sk_buff *skb, struct net_device *dev, + netdev_features_t features) +{ + unsigned int l2len, l3len; + + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) + return features; + + l2len = skb_mac_header_len(skb); + l3len = skb_checksum_start(skb) - skb_network_header(skb); + + /* Check header lengths are multiple of 2. + * In case of 6in6 we support longer headers (IHLEN + OHLEN) + * but keep things simple for now, 512B is plenty. + */ + if ((l2len | l3len | skb->csum_offset) % 2 || + !FIELD_FIT(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) || + !FIELD_FIT(FBNIC_TWD_L3_IHLEN_MASK, l3len / 2) || + !FIELD_FIT(FBNIC_TWD_CSUM_OFFSET_MASK, skb->csum_offset / 2)) + return features & ~NETIF_F_CSUM_MASK; + + return features; +} + +static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, + struct fbnic_ring *ring, bool discard, + unsigned int hw_head) +{ + u64 total_bytes = 0, total_packets = 0; + unsigned int head = ring->head; + struct netdev_queue *txq; + unsigned int clean_desc; + + clean_desc = (hw_head - head) & ring->size_mask; + + while (clean_desc) { + struct sk_buff *skb = ring->tx_buf[head]; + unsigned int desc_cnt; + + desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; + if (desc_cnt > clean_desc) + break; + + ring->tx_buf[head] = NULL; + + clean_desc -= desc_cnt; + + while (!(ring->desc[head] & FBNIC_TWD_TYPE(AL))) { + head++; + head &= ring->size_mask; + desc_cnt--; + } + + fbnic_unmap_single_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + desc_cnt--; + + while (desc_cnt--) { + fbnic_unmap_page_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + } + + total_bytes += FBNIC_XMIT_CB(skb)->bytecount; + total_packets += 1; + + napi_consume_skb(skb, napi_budget); + } + + if (!total_bytes) + return; + + ring->head = head; + + txq = txring_txq(nv->napi.dev, ring); + + if (unlikely(discard)) { + netdev_tx_completed_queue(txq, total_packets, total_bytes); + return; + } + + netif_txq_completed_wake(txq, total_packets, total_bytes, + fbnic_desc_unused(ring), + FBNIC_TX_DESC_WAKEUP); +} + +static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, + struct page *page) +{ + struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; + + page_pool_fragment_page(page, PAGECNT_BIAS_MAX); + rx_buf->pagecnt_bias = PAGECNT_BIAS_MAX; + rx_buf->page = page; +} + +static struct page *fbnic_page_pool_get(struct fbnic_ring *ring, + unsigned int idx) +{ + struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; + + rx_buf->pagecnt_bias--; + + return rx_buf->page; +} + +static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, + struct fbnic_napi_vector *nv, int budget) +{ + struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; + struct page *page = rx_buf->page; + + if (!page_pool_unref_page(page, rx_buf->pagecnt_bias)) + page_pool_put_unrefed_page(nv->page_pool, page, -1, !!budget); + + rx_buf->page = NULL; +} + +static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, + struct fbnic_q_triad *qt, s32 head0) +{ + if (head0 >= 0) + fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); +} + +static void +fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, + int napi_budget) +{ + struct fbnic_ring *cmpl = &qt->cmpl; + __le64 *raw_tcd, done; + u32 head = cmpl->head; + s32 head0 = -1; + + done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(FBNIC_TCD_DONE); + raw_tcd = &cmpl->desc[head & cmpl->size_mask]; + + /* Walk the completion queue collecting the heads reported by NIC */ + while ((*raw_tcd & cpu_to_le64(FBNIC_TCD_DONE)) == done) { + u64 tcd; + + dma_rmb(); + + tcd = le64_to_cpu(*raw_tcd); + + switch (FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd)) { + case FBNIC_TCD_TYPE_0: + if (!(tcd & FBNIC_TCD_TWQ1)) + head0 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD0_MASK, + tcd); + /* Currently all err status bits are related to + * timestamps and as those have yet to be added + * they are skipped for now. + */ + break; + default: + break; + } + + raw_tcd++; + head++; + if (!(head & cmpl->size_mask)) { + done ^= cpu_to_le64(FBNIC_TCD_DONE); + raw_tcd = &cmpl->desc[0]; + } + } + + /* Record the current head/tail of the queue */ + if (cmpl->head != head) { + cmpl->head = head; + writel(head & cmpl->size_mask, cmpl->doorbell); + } + + /* Unmap and free processed buffers */ + fbnic_clean_twq(nv, napi_budget, qt, head0); +} + +static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, + struct fbnic_ring *ring, unsigned int hw_head) +{ + unsigned int head = ring->head; + + if (head == hw_head) + return; + + do { + fbnic_page_pool_drain(ring, head, nv, napi_budget); + + head++; + head &= ring->size_mask; + } while (head != hw_head); + + ring->head = head; +} + +static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page) +{ + __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; + dma_addr_t dma = page_pool_get_dma_addr(page); + u64 bd, i = FBNIC_BD_FRAG_COUNT; + + bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | + FIELD_PREP(FBNIC_BD_PAGE_ID_MASK, id); + + /* In the case that a page size is larger than 4K we will map a + * single page to multiple fragments. The fragments will be + * FBNIC_BD_FRAG_COUNT in size and the lower n bits will be use + * to indicate the individual fragment IDs. + */ + do { + *bdq_desc = cpu_to_le64(bd); + bd += FIELD_PREP(FBNIC_BD_DESC_ADDR_MASK, 1) | + FIELD_PREP(FBNIC_BD_DESC_ID_MASK, 1); + } while (--i); +} + +static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq) +{ + unsigned int count = fbnic_desc_unused(bdq); + unsigned int i = bdq->tail; + + if (!count) + return; + + do { + struct page *page; + + page = page_pool_dev_alloc_pages(nv->page_pool); + if (!page) + break; + + fbnic_page_pool_init(bdq, i, page); + fbnic_bd_prep(bdq, i, page); + + i++; + i &= bdq->size_mask; + + count--; + } while (count); + + if (bdq->tail != i) { + bdq->tail = i; + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(i, bdq->doorbell); + } +} + +static unsigned int fbnic_hdr_pg_start(unsigned int pg_off) +{ + /* The headroom of the first header may be larger than FBNIC_RX_HROOM + * due to alignment. So account for that by just making the page + * offset 0 if we are starting at the first header. + */ + if (ALIGN(FBNIC_RX_HROOM, 128) > FBNIC_RX_HROOM && + pg_off == ALIGN(FBNIC_RX_HROOM, 128)) + return 0; + + return pg_off - FBNIC_RX_HROOM; +} + +static unsigned int fbnic_hdr_pg_end(unsigned int pg_off, unsigned int len) +{ + /* Determine the end of the buffer by finding the start of the next + * and then subtracting the headroom from that frame. + */ + pg_off += len + FBNIC_RX_TROOM + FBNIC_RX_HROOM; + + return ALIGN(pg_off, 128) - FBNIC_RX_HROOM; +} + +static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, + struct fbnic_pkt_buff *pkt, + struct fbnic_q_triad *qt) +{ + unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); + struct page *page = fbnic_page_pool_get(&qt->sub0, hdr_pg_idx); + unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); + unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; + unsigned char *hdr_start; + + /* data_hard_start should always be NULL when this is called */ + WARN_ON_ONCE(pkt->buff.data_hard_start); + + /* Short-cut the end calculation if we know page is fully consumed */ + hdr_pg_end = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? + FBNIC_BD_FRAG_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); + hdr_pg_start = fbnic_hdr_pg_start(hdr_pg_off); + + headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; + frame_sz = hdr_pg_end - hdr_pg_start; + xdp_init_buff(&pkt->buff, frame_sz, NULL); + hdr_pg_start += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * + FBNIC_BD_FRAG_SIZE; + + /* Sync DMA buffer */ + dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), + hdr_pg_start, frame_sz, + DMA_BIDIRECTIONAL); + + /* Build frame around buffer */ + hdr_start = page_address(page) + hdr_pg_start; + + xdp_prepare_buff(&pkt->buff, hdr_start, headroom, + len - FBNIC_RX_PAD, true); + + pkt->data_truesize = 0; + pkt->data_len = 0; + pkt->nr_frags = 0; +} + +static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, + struct fbnic_pkt_buff *pkt, + struct fbnic_q_triad *qt) +{ + unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); + unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); + struct page *page = fbnic_page_pool_get(&qt->sub1, pg_idx); + struct skb_shared_info *shinfo; + unsigned int truesize; + + truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? + FBNIC_BD_FRAG_SIZE - pg_off : ALIGN(len, 128); + + pg_off += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * + FBNIC_BD_FRAG_SIZE; + + /* Sync DMA buffer */ + dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), + pg_off, truesize, DMA_BIDIRECTIONAL); + + /* Add page to xdp shared info */ + shinfo = xdp_get_shared_info_from_buff(&pkt->buff); + + /* We use gso_segs to store truesize */ + pkt->data_truesize += truesize; + + __skb_fill_page_desc_noacc(shinfo, pkt->nr_frags++, page, pg_off, len); + + /* Store data_len in gso_size */ + pkt->data_len += len; +} + +static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt, int budget) +{ + struct skb_shared_info *shinfo; + struct page *page; + int nr_frags; + + if (!pkt->buff.data_hard_start) + return; + + shinfo = xdp_get_shared_info_from_buff(&pkt->buff); + nr_frags = pkt->nr_frags; + + while (nr_frags--) { + page = skb_frag_page(&shinfo->frags[nr_frags]); + page_pool_put_full_page(nv->page_pool, page, !!budget); + } + + page = virt_to_page(pkt->buff.data_hard_start); + page_pool_put_full_page(nv->page_pool, page, !!budget); +} + +static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt) +{ + unsigned int nr_frags = pkt->nr_frags; + struct skb_shared_info *shinfo; + unsigned int truesize; + struct sk_buff *skb; + + truesize = xdp_data_hard_end(&pkt->buff) + FBNIC_RX_TROOM - + pkt->buff.data_hard_start; + + /* Build frame around buffer */ + skb = napi_build_skb(pkt->buff.data_hard_start, truesize); + if (unlikely(!skb)) + return NULL; + + /* Push data pointer to start of data, put tail to end of data */ + skb_reserve(skb, pkt->buff.data - pkt->buff.data_hard_start); + __skb_put(skb, pkt->buff.data_end - pkt->buff.data); + + /* Add tracking for metadata at the start of the frame */ + skb_metadata_set(skb, pkt->buff.data - pkt->buff.data_meta); + + /* Add Rx frags */ + if (nr_frags) { + /* Verify that shared info didn't move */ + shinfo = xdp_get_shared_info_from_buff(&pkt->buff); + WARN_ON(skb_shinfo(skb) != shinfo); + + skb->truesize += pkt->data_truesize; + skb->data_len += pkt->data_len; + shinfo->nr_frags = nr_frags; + skb->len += pkt->data_len; + } + + skb_mark_for_recycle(skb); + + /* Set MAC header specific fields */ + skb->protocol = eth_type_trans(skb, nv->napi.dev); + + return skb; +} + +static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd) +{ + return (FBNIC_RCD_META_L4_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L4 : + (FBNIC_RCD_META_L3_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L3 : + PKT_HASH_TYPE_L2; +} + +static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv, + u64 rcd, struct sk_buff *skb, + struct fbnic_q_triad *qt) +{ + struct net_device *netdev = nv->napi.dev; + struct fbnic_ring *rcq = &qt->cmpl; + + fbnic_rx_csum(rcd, skb, rcq); + + if (netdev->features & NETIF_F_RXHASH) + skb_set_hash(skb, + FIELD_GET(FBNIC_RCD_META_RSS_HASH_MASK, rcd), + fbnic_skb_hash_type(rcd)); + + skb_record_rx_queue(skb, rcq->q_idx); +} + +static bool fbnic_rcd_metadata_err(u64 rcd) +{ + return !!(FBNIC_RCD_META_UNCORRECTABLE_ERR_MASK & rcd); +} + +static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, + struct fbnic_q_triad *qt, int budget) +{ + struct fbnic_ring *rcq = &qt->cmpl; + struct fbnic_pkt_buff *pkt; + s32 head0 = -1, head1 = -1; + __le64 *raw_rcd, done; + u32 head = rcq->head; + u64 packets = 0; + + done = (head & (rcq->size_mask + 1)) ? cpu_to_le64(FBNIC_RCD_DONE) : 0; + raw_rcd = &rcq->desc[head & rcq->size_mask]; + pkt = rcq->pkt; + + /* Walk the completion queue collecting the heads reported by NIC */ + while (likely(packets < budget)) { + struct sk_buff *skb = ERR_PTR(-EINVAL); + u64 rcd; + + if ((*raw_rcd & cpu_to_le64(FBNIC_RCD_DONE)) == done) + break; + + dma_rmb(); + + rcd = le64_to_cpu(*raw_rcd); + + switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { + case FBNIC_RCD_TYPE_HDR_AL: + head0 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + fbnic_pkt_prepare(nv, rcd, pkt, qt); + + break; + case FBNIC_RCD_TYPE_PAY_AL: + head1 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + fbnic_add_rx_frag(nv, rcd, pkt, qt); + + break; + case FBNIC_RCD_TYPE_OPT_META: + /* Only type 0 is currently supported */ + if (FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd)) + break; + + /* We currently ignore the action table index */ + break; + case FBNIC_RCD_TYPE_META: + if (likely(!fbnic_rcd_metadata_err(rcd))) + skb = fbnic_build_skb(nv, pkt); + + /* Populate skb and invalidate XDP */ + if (!IS_ERR_OR_NULL(skb)) { + fbnic_populate_skb_fields(nv, rcd, skb, qt); + + packets++; + + napi_gro_receive(&nv->napi, skb); + } else { + fbnic_put_pkt_buff(nv, pkt, 1); + } + + pkt->buff.data_hard_start = NULL; + + break; + } + + raw_rcd++; + head++; + if (!(head & rcq->size_mask)) { + done ^= cpu_to_le64(FBNIC_RCD_DONE); + raw_rcd = &rcq->desc[0]; + } + } + + /* Unmap and free processed buffers */ + if (head0 >= 0) + fbnic_clean_bdq(nv, budget, &qt->sub0, head0); + fbnic_fill_bdq(nv, &qt->sub0); + + if (head1 >= 0) + fbnic_clean_bdq(nv, budget, &qt->sub1, head1); + fbnic_fill_bdq(nv, &qt->sub1); + + /* Record the current head/tail of the queue */ + if (rcq->head != head) { + rcq->head = head; + writel(head & rcq->size_mask, rcq->doorbell); + } + + return packets; +} + +static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + u32 v_idx = nv->v_idx; + + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(v_idx / 32), 1 << (v_idx % 32)); +} + +static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + u32 v_idx = nv->v_idx; + + fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(v_idx), + FBNIC_INTR_CQ_REARM_INTR_UNMASK); +} + +static int fbnic_poll(struct napi_struct *napi, int budget) +{ + struct fbnic_napi_vector *nv = container_of(napi, + struct fbnic_napi_vector, + napi); + int i, j, work_done = 0; + + for (i = 0; i < nv->txt_count; i++) + fbnic_clean_tcq(nv, &nv->qt[i], budget); + + for (j = 0; j < nv->rxt_count; j++, i++) + work_done += fbnic_clean_rcq(nv, &nv->qt[i], budget); + + if (work_done >= budget) + return budget; + + if (likely(napi_complete_done(napi, work_done))) + fbnic_nv_irq_rearm(nv); + + return 0; +} + +static irqreturn_t fbnic_msix_clean_rings(int __always_unused irq, void *data) +{ + struct fbnic_napi_vector *nv = data; + + napi_schedule_irqoff(&nv->napi); + + return IRQ_HANDLED; +} + +static void fbnic_remove_tx_ring(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + if (!(txr->flags & FBNIC_RING_F_STATS)) + return; + + /* Remove pointer to the Tx ring */ + WARN_ON(fbn->tx[txr->q_idx] && fbn->tx[txr->q_idx] != txr); + fbn->tx[txr->q_idx] = NULL; +} + +static void fbnic_remove_rx_ring(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + if (!(rxr->flags & FBNIC_RING_F_STATS)) + return; + + /* Remove pointer to the Rx ring */ + WARN_ON(fbn->rx[rxr->q_idx] && fbn->rx[rxr->q_idx] != rxr); + fbn->rx[rxr->q_idx] = NULL; +} + +static void fbnic_free_napi_vector(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + u32 v_idx = nv->v_idx; + int i, j; + + for (i = 0; i < nv->txt_count; i++) { + fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0); + fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl); + } + + for (j = 0; j < nv->rxt_count; j++, i++) { + fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0); + fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1); + fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl); + } + + fbnic_free_irq(fbd, v_idx, nv); + page_pool_destroy(nv->page_pool); + netif_napi_del(&nv->napi); + list_del(&nv->napis); + kfree(nv); +} + +void fbnic_free_napi_vectors(struct fbnic_net *fbn) +{ + struct fbnic_napi_vector *nv, *temp; + + list_for_each_entry_safe(nv, temp, &fbn->napis, napis) + fbnic_free_napi_vector(fbn, nv); +} + +static void fbnic_name_napi_vector(struct fbnic_napi_vector *nv) +{ + unsigned char *dev_name = nv->napi.dev->name; + + if (!nv->rxt_count) + snprintf(nv->name, sizeof(nv->name), "%s-Tx-%u", dev_name, + nv->v_idx - FBNIC_NON_NAPI_VECTORS); + else + snprintf(nv->name, sizeof(nv->name), "%s-TxRx-%u", dev_name, + nv->v_idx - FBNIC_NON_NAPI_VECTORS); +} + +#define FBNIC_PAGE_POOL_FLAGS \ + (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV) + +static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + struct page_pool_params pp_params = { + .order = 0, + .flags = FBNIC_PAGE_POOL_FLAGS, + .pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count, + .nid = NUMA_NO_NODE, + .dev = nv->dev, + .dma_dir = DMA_BIDIRECTIONAL, + .offset = 0, + .max_len = PAGE_SIZE + }; + struct page_pool *pp; + + /* Page pool cannot exceed a size of 32768. This doesn't limit the + * pages on the ring but the number we can have cached waiting on + * the next use. + * + * TBD: Can this be reduced further? Would a multiple of + * NAPI_POLL_WEIGHT possibly make more sense? The question is how + * may pages do we need to hold in reserve to get the best return + * without hogging too much system memory. + */ + if (pp_params.pool_size > 32768) + pp_params.pool_size = 32768; + + pp = page_pool_create(&pp_params); + if (IS_ERR(pp)) + return PTR_ERR(pp); + + nv->page_pool = pp; + + return 0; +} + +static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, + int q_idx, u8 flags) +{ + ring->doorbell = doorbell; + ring->q_idx = q_idx; + ring->flags = flags; +} + +static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, + unsigned int v_count, unsigned int v_idx, + unsigned int txq_count, unsigned int txq_idx, + unsigned int rxq_count, unsigned int rxq_idx) +{ + int txt_count = txq_count, rxt_count = rxq_count; + u32 __iomem *uc_addr = fbd->uc_addr0; + struct fbnic_napi_vector *nv; + struct fbnic_q_triad *qt; + int qt_count, err; + u32 __iomem *db; + + qt_count = txt_count + rxq_count; + if (!qt_count) + return -EINVAL; + + /* If MMIO has already failed there are no rings to initialize */ + if (!uc_addr) + return -EIO; + + /* Allocate NAPI vector and queue triads */ + nv = kzalloc(struct_size(nv, qt, qt_count), GFP_KERNEL); + if (!nv) + return -ENOMEM; + + /* Record queue triad counts */ + nv->txt_count = txt_count; + nv->rxt_count = rxt_count; + + /* Provide pointer back to fbnic and MSI-X vectors */ + nv->fbd = fbd; + nv->v_idx = v_idx; + + /* Record IRQ to NAPI struct */ + netif_napi_set_irq(&nv->napi, + pci_irq_vector(to_pci_dev(fbd->dev), nv->v_idx)); + + /* Tie napi to netdev */ + list_add(&nv->napis, &fbn->napis); + netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll); + + /* Tie nv back to PCIe dev */ + nv->dev = fbd->dev; + + /* Allocate page pool */ + if (rxq_count) { + err = fbnic_alloc_nv_page_pool(fbn, nv); + if (err) + goto napi_del; + } + + /* Initialize vector name */ + fbnic_name_napi_vector(nv); + + /* Request the IRQ for napi vector */ + err = fbnic_request_irq(fbd, v_idx, &fbnic_msix_clean_rings, + IRQF_SHARED, nv->name, nv); + if (err) + goto pp_destroy; + + /* Initialize queue triads */ + qt = nv->qt; + + while (txt_count) { + /* Configure Tx queue */ + db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ0_TAIL]; + + /* Assign Tx queue to netdev if applicable */ + if (txq_count > 0) { + u8 flags = FBNIC_RING_F_CTX | FBNIC_RING_F_STATS; + + fbnic_ring_init(&qt->sub0, db, txq_idx, flags); + fbn->tx[txq_idx] = &qt->sub0; + txq_count--; + } else { + fbnic_ring_init(&qt->sub0, db, 0, + FBNIC_RING_F_DISABLED); + } + + /* Configure Tx completion queue */ + db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TCQ_HEAD]; + fbnic_ring_init(&qt->cmpl, db, 0, 0); + + /* Update Tx queue index */ + txt_count--; + txq_idx += v_count; + + /* Move to next queue triad */ + qt++; + } + + while (rxt_count) { + /* Configure header queue */ + db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_HPQ_TAIL]; + fbnic_ring_init(&qt->sub0, db, 0, FBNIC_RING_F_CTX); + + /* Configure payload queue */ + db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_PPQ_TAIL]; + fbnic_ring_init(&qt->sub1, db, 0, FBNIC_RING_F_CTX); + + /* Configure Rx completion queue */ + db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_RCQ_HEAD]; + fbnic_ring_init(&qt->cmpl, db, rxq_idx, FBNIC_RING_F_STATS); + fbn->rx[rxq_idx] = &qt->cmpl; + + /* Update Rx queue index */ + rxt_count--; + rxq_idx += v_count; + + /* Move to next queue triad */ + qt++; + } + + return 0; + +pp_destroy: + page_pool_destroy(nv->page_pool); +napi_del: + netif_napi_del(&nv->napi); + list_del(&nv->napis); + kfree(nv); + return err; +} + +int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) +{ + unsigned int txq_idx = 0, rxq_idx = 0, v_idx = FBNIC_NON_NAPI_VECTORS; + unsigned int num_tx = fbn->num_tx_queues; + unsigned int num_rx = fbn->num_rx_queues; + unsigned int num_napi = fbn->num_napi; + struct fbnic_dev *fbd = fbn->fbd; + int err; + + /* Allocate 1 Tx queue per napi vector */ + if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { + while (num_tx) { + err = fbnic_alloc_napi_vector(fbd, fbn, + num_napi, v_idx, + 1, txq_idx, 0, 0); + if (err) + goto free_vectors; + + /* Update counts and index */ + num_tx--; + txq_idx++; + + v_idx++; + } + } + + /* Allocate Tx/Rx queue pairs per vector, or allocate remaining Rx */ + while (num_rx | num_tx) { + int tqpv = DIV_ROUND_UP(num_tx, num_napi - txq_idx); + int rqpv = DIV_ROUND_UP(num_rx, num_napi - rxq_idx); + + err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, + tqpv, txq_idx, rqpv, rxq_idx); + if (err) + goto free_vectors; + + /* Update counts and index */ + num_tx -= tqpv; + txq_idx++; + + num_rx -= rqpv; + rxq_idx++; + + v_idx++; + } + + return 0; + +free_vectors: + fbnic_free_napi_vectors(fbn); + + return -ENOMEM; +} + +static void fbnic_free_ring_resources(struct device *dev, + struct fbnic_ring *ring) +{ + kvfree(ring->buffer); + ring->buffer = NULL; + + /* If size is not set there are no descriptors present */ + if (!ring->size) + return; + + dma_free_coherent(dev, ring->size, ring->desc, ring->dma); + ring->size_mask = 0; + ring->size = 0; +} + +static int fbnic_alloc_tx_ring_desc(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + struct device *dev = fbn->netdev->dev.parent; + size_t size; + + /* Round size up to nearest 4K */ + size = ALIGN(array_size(sizeof(*txr->desc), fbn->txq_size), 4096); + + txr->desc = dma_alloc_coherent(dev, size, &txr->dma, + GFP_KERNEL | __GFP_NOWARN); + if (!txr->desc) + return -ENOMEM; + + /* txq_size should be a power of 2, so mask is just that -1 */ + txr->size_mask = fbn->txq_size - 1; + txr->size = size; + + return 0; +} + +static int fbnic_alloc_tx_ring_buffer(struct fbnic_ring *txr) +{ + size_t size = array_size(sizeof(*txr->tx_buf), txr->size_mask + 1); + + txr->tx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); + + return txr->tx_buf ? 0 : -ENOMEM; +} + +static int fbnic_alloc_tx_ring_resources(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + if (txr->flags & FBNIC_RING_F_DISABLED) + return 0; + + err = fbnic_alloc_tx_ring_desc(fbn, txr); + if (err) + return err; + + if (!(txr->flags & FBNIC_RING_F_CTX)) + return 0; + + err = fbnic_alloc_tx_ring_buffer(txr); + if (err) + goto free_desc; + + return 0; + +free_desc: + fbnic_free_ring_resources(dev, txr); + return err; +} + +static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + struct device *dev = fbn->netdev->dev.parent; + size_t desc_size = sizeof(*rxr->desc); + u32 rxq_size; + size_t size; + + switch (rxr->doorbell - fbnic_ring_csr_base(rxr)) { + case FBNIC_QUEUE_BDQ_HPQ_TAIL: + rxq_size = fbn->hpq_size / FBNIC_BD_FRAG_COUNT; + desc_size *= FBNIC_BD_FRAG_COUNT; + break; + case FBNIC_QUEUE_BDQ_PPQ_TAIL: + rxq_size = fbn->ppq_size / FBNIC_BD_FRAG_COUNT; + desc_size *= FBNIC_BD_FRAG_COUNT; + break; + case FBNIC_QUEUE_RCQ_HEAD: + rxq_size = fbn->rcq_size; + break; + default: + return -EINVAL; + } + + /* Round size up to nearest 4K */ + size = ALIGN(array_size(desc_size, rxq_size), 4096); + + rxr->desc = dma_alloc_coherent(dev, size, &rxr->dma, + GFP_KERNEL | __GFP_NOWARN); + if (!rxr->desc) + return -ENOMEM; + + /* rxq_size should be a power of 2, so mask is just that -1 */ + rxr->size_mask = rxq_size - 1; + rxr->size = size; + + return 0; +} + +static int fbnic_alloc_rx_ring_buffer(struct fbnic_ring *rxr) +{ + size_t size = array_size(sizeof(*rxr->rx_buf), rxr->size_mask + 1); + + if (rxr->flags & FBNIC_RING_F_CTX) + size = sizeof(*rxr->rx_buf) * (rxr->size_mask + 1); + else + size = sizeof(*rxr->pkt); + + rxr->rx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); + + return rxr->rx_buf ? 0 : -ENOMEM; +} + +static int fbnic_alloc_rx_ring_resources(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + err = fbnic_alloc_rx_ring_desc(fbn, rxr); + if (err) + return err; + + err = fbnic_alloc_rx_ring_buffer(rxr); + if (err) + goto free_desc; + + return 0; + +free_desc: + fbnic_free_ring_resources(dev, rxr); + return err; +} + +static void fbnic_free_qt_resources(struct fbnic_net *fbn, + struct fbnic_q_triad *qt) +{ + struct device *dev = fbn->netdev->dev.parent; + + fbnic_free_ring_resources(dev, &qt->cmpl); + fbnic_free_ring_resources(dev, &qt->sub1); + fbnic_free_ring_resources(dev, &qt->sub0); +} + +static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, + struct fbnic_q_triad *qt) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub0); + if (err) + return err; + + err = fbnic_alloc_tx_ring_resources(fbn, &qt->cmpl); + if (err) + goto free_sub1; + + return 0; + +free_sub1: + fbnic_free_ring_resources(dev, &qt->sub0); + return err; +} + +static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, + struct fbnic_q_triad *qt) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); + if (err) + return err; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1); + if (err) + goto free_sub0; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->cmpl); + if (err) + goto free_sub1; + + return 0; + +free_sub1: + fbnic_free_ring_resources(dev, &qt->sub1); +free_sub0: + fbnic_free_ring_resources(dev, &qt->sub0); + return err; +} + +static void fbnic_free_nv_resources(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + int i, j; + + /* Free Tx Resources */ + for (i = 0; i < nv->txt_count; i++) + fbnic_free_qt_resources(fbn, &nv->qt[i]); + + for (j = 0; j < nv->rxt_count; j++, i++) + fbnic_free_qt_resources(fbn, &nv->qt[i]); +} + +static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + int i, j, err; + + /* Allocate Tx Resources */ + for (i = 0; i < nv->txt_count; i++) { + err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]); + if (err) + goto free_resources; + } + + /* Allocate Rx Resources */ + for (j = 0; j < nv->rxt_count; j++, i++) { + err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]); + if (err) + goto free_resources; + } + + return 0; + +free_resources: + while (i--) + fbnic_free_qt_resources(fbn, &nv->qt[i]); + return err; +} + +void fbnic_free_resources(struct fbnic_net *fbn) +{ + struct fbnic_napi_vector *nv; + + list_for_each_entry(nv, &fbn->napis, napis) + fbnic_free_nv_resources(fbn, nv); +} + +int fbnic_alloc_resources(struct fbnic_net *fbn) +{ + struct fbnic_napi_vector *nv; + int err = -ENODEV; + + list_for_each_entry(nv, &fbn->napis, napis) { + err = fbnic_alloc_nv_resources(fbn, nv); + if (err) + goto free_resources; + } + + return 0; + +free_resources: + list_for_each_entry_continue_reverse(nv, &fbn->napis, napis) + fbnic_free_nv_resources(fbn, nv); + + return err; +} + +static void fbnic_disable_twq0(struct fbnic_ring *txr) +{ + u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ0_CTL); + + twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; + + fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ0_CTL, twq_ctl); +} + +static void fbnic_disable_tcq(struct fbnic_ring *txr) +{ + fbnic_ring_wr32(txr, FBNIC_QUEUE_TCQ_CTL, 0); + fbnic_ring_wr32(txr, FBNIC_QUEUE_TIM_MASK, FBNIC_QUEUE_TIM_MASK_MASK); +} + +static void fbnic_disable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) +{ + u32 bdq_ctl = fbnic_ring_rd32(hpq, FBNIC_QUEUE_BDQ_CTL); + + bdq_ctl &= ~FBNIC_QUEUE_BDQ_CTL_ENABLE; + + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); +} + +static void fbnic_disable_rcq(struct fbnic_ring *rxr) +{ + fbnic_ring_wr32(rxr, FBNIC_QUEUE_RCQ_CTL, 0); + fbnic_ring_wr32(rxr, FBNIC_QUEUE_RIM_MASK, FBNIC_QUEUE_RIM_MASK_MASK); +} + +void fbnic_napi_disable(struct fbnic_net *fbn) +{ + struct fbnic_napi_vector *nv; + + list_for_each_entry(nv, &fbn->napis, napis) { + napi_disable(&nv->napi); + + fbnic_nv_irq_disable(nv); + } +} + +void fbnic_disable(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_napi_vector *nv; + int i, j; + + list_for_each_entry(nv, &fbn->napis, napis) { + /* Disable Tx queue triads */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + fbnic_disable_twq0(&qt->sub0); + fbnic_disable_tcq(&qt->cmpl); + } + + /* Disable Rx queue triads */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + fbnic_disable_bdq(&qt->sub0, &qt->sub1); + fbnic_disable_rcq(&qt->cmpl); + } + } + + fbnic_wrfl(fbd); +} + +static void fbnic_tx_flush(struct fbnic_dev *fbd) +{ + netdev_warn(fbd->netdev, "triggering Tx flush\n"); + + fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, + FBNIC_TMI_DROP_CTRL_EN); +} + +static void fbnic_tx_flush_off(struct fbnic_dev *fbd) +{ + fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, 0); +} + +struct fbnic_idle_regs { + u32 reg_base; + u8 reg_cnt; +}; + +static bool fbnic_all_idle(struct fbnic_dev *fbd, + const struct fbnic_idle_regs *regs, + unsigned int nregs) +{ + unsigned int i, j; + + for (i = 0; i < nregs; i++) { + for (j = 0; j < regs[i].reg_cnt; j++) { + if (fbnic_rd32(fbd, regs[i].reg_base + j) != ~0U) + return false; + } + } + return true; +} + +static void fbnic_idle_dump(struct fbnic_dev *fbd, + const struct fbnic_idle_regs *regs, + unsigned int nregs, const char *dir, int err) +{ + unsigned int i, j; + + netdev_err(fbd->netdev, "error waiting for %s idle %d\n", dir, err); + for (i = 0; i < nregs; i++) + for (j = 0; j < regs[i].reg_cnt; j++) + netdev_err(fbd->netdev, "0x%04x: %08x\n", + regs[i].reg_base + j, + fbnic_rd32(fbd, regs[i].reg_base + j)); +} + +int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail) +{ + static const struct fbnic_idle_regs tx[] = { + { FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TWQ_IDLE_CNT, }, + { FBNIC_QM_TQS_IDLE(0), FBNIC_QM_TQS_IDLE_CNT, }, + { FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TDE_IDLE_CNT, }, + { FBNIC_QM_TCQ_IDLE(0), FBNIC_QM_TCQ_IDLE_CNT, }, + }, rx[] = { + { FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_HPQ_IDLE_CNT, }, + { FBNIC_QM_PPQ_IDLE(0), FBNIC_QM_PPQ_IDLE_CNT, }, + { FBNIC_QM_RCQ_IDLE(0), FBNIC_QM_RCQ_IDLE_CNT, }, + }; + bool idle; + int err; + + err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, + false, fbd, tx, ARRAY_SIZE(tx)); + if (err == -ETIMEDOUT) { + fbnic_tx_flush(fbd); + err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, + 2, 500000, false, + fbd, tx, ARRAY_SIZE(tx)); + fbnic_tx_flush_off(fbd); + } + if (err) { + fbnic_idle_dump(fbd, tx, ARRAY_SIZE(tx), "Tx", err); + if (may_fail) + return err; + } + + err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, + false, fbd, rx, ARRAY_SIZE(rx)); + if (err) + fbnic_idle_dump(fbd, rx, ARRAY_SIZE(rx), "Rx", err); + return err; +} + +void fbnic_flush(struct fbnic_net *fbn) +{ + struct fbnic_napi_vector *nv; + + list_for_each_entry(nv, &fbn->napis, napis) { + int i, j; + + /* Flush any processed Tx Queue Triads and drop the rest */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + struct netdev_queue *tx_queue; + + /* Clean the work queues of unprocessed work */ + fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); + + /* Reset completion queue descriptor ring */ + memset(qt->cmpl.desc, 0, qt->cmpl.size); + + /* Nothing else to do if Tx queue is disabled */ + if (qt->sub0.flags & FBNIC_RING_F_DISABLED) + continue; + + /* Reset BQL associated with Tx queue */ + tx_queue = netdev_get_tx_queue(nv->napi.dev, + qt->sub0.q_idx); + netdev_tx_reset_queue(tx_queue); + + /* Disassociate Tx queue from NAPI */ + netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, NULL); + } + + /* Flush any processed Rx Queue Triads and drop the rest */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + /* Clean the work queues of unprocessed work */ + fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail); + fbnic_clean_bdq(nv, 0, &qt->sub1, qt->sub1.tail); + + /* Reset completion queue descriptor ring */ + memset(qt->cmpl.desc, 0, qt->cmpl.size); + + fbnic_put_pkt_buff(nv, qt->cmpl.pkt, 0); + qt->cmpl.pkt->buff.data_hard_start = NULL; + + /* Disassociate Rx queue from NAPI */ + netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, + NETDEV_QUEUE_TYPE_RX, NULL); + } + } +} + +void fbnic_fill(struct fbnic_net *fbn) +{ + struct fbnic_napi_vector *nv; + + list_for_each_entry(nv, &fbn->napis, napis) { + int i, j; + + /* Configure NAPI mapping for Tx */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + /* Nothing to do if Tx queue is disabled */ + if (qt->sub0.flags & FBNIC_RING_F_DISABLED) + continue; + + /* Associate Tx queue with NAPI */ + netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, &nv->napi); + } + + /* Configure NAPI mapping and populate pages + * in the BDQ rings to use for Rx + */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + /* Associate Rx queue with NAPI */ + netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, + NETDEV_QUEUE_TYPE_RX, &nv->napi); + + /* Populate the header and payload BDQs */ + fbnic_fill_bdq(nv, &qt->sub0); + fbnic_fill_bdq(nv, &qt->sub1); + } + } +} + +static void fbnic_enable_twq0(struct fbnic_ring *twq) +{ + u32 log_size = fls(twq->size_mask); + + if (!twq->size_mask) + return; + + /* Reset head/tail */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); + twq->tail = 0; + twq->head = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAL, lower_32_bits(twq->dma)); + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAH, upper_32_bits(twq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_SIZE, log_size & 0xf); + + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); +} + +static void fbnic_enable_tcq(struct fbnic_napi_vector *nv, + struct fbnic_ring *tcq) +{ + u32 log_size = fls(tcq->size_mask); + + if (!tcq->size_mask) + return; + + /* Reset head/tail */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET); + tcq->tail = 0; + tcq->head = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAL, lower_32_bits(tcq->dma)); + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAH, upper_32_bits(tcq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_SIZE, log_size & 0xf); + + /* Store interrupt information for the completion queue */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_CTL, nv->v_idx); + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_THRESHOLD, tcq->size_mask / 2); + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_MASK, 0); + + /* Enable queue */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_ENABLE); +} + +static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) +{ + u32 bdq_ctl = FBNIC_QUEUE_BDQ_CTL_ENABLE; + u32 log_size; + + /* Reset head/tail */ + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET); + ppq->tail = 0; + ppq->head = 0; + hpq->tail = 0; + hpq->head = 0; + + log_size = fls(hpq->size_mask); + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAH, upper_32_bits(hpq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_SIZE, log_size & 0xf); + + if (!ppq->size_mask) + goto write_ctl; + + log_size = fls(ppq->size_mask); + + /* Add enabling of PPQ to BDQ control */ + bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAL, lower_32_bits(ppq->dma)); + fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAH, upper_32_bits(ppq->dma)); + fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_SIZE, log_size & 0xf); + +write_ctl: + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); +} + +static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, + struct fbnic_ring *rcq) +{ + u32 drop_mode, rcq_ctl; + + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; + + /* Specify packet layout */ + rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM); + + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); +} + +static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, + struct fbnic_ring *rcq) +{ + u32 log_size = fls(rcq->size_mask); + u32 rcq_ctl; + + fbnic_config_drop_mode_rcq(nv, rcq); + + rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, + FBNIC_RX_MAX_HDR) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, + FBNIC_RX_PAYLD_OFFSET) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_PG_CL_MASK, + FBNIC_RX_PAYLD_PG_CL); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL1, rcq_ctl); + + /* Reset head/tail */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET); + rcq->head = 0; + rcq->tail = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAL, lower_32_bits(rcq->dma)); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAH, upper_32_bits(rcq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_SIZE, log_size & 0xf); + + /* Store interrupt information for the completion queue */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_CTL, nv->v_idx); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, rcq->size_mask / 2); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_MASK, 0); + + /* Enable queue */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE); +} + +void fbnic_enable(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_napi_vector *nv; + int i, j; + + list_for_each_entry(nv, &fbn->napis, napis) { + /* Setup Tx Queue Triads */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + fbnic_enable_twq0(&qt->sub0); + fbnic_enable_tcq(nv, &qt->cmpl); + } + + /* Setup Rx Queue Triads */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + fbnic_enable_bdq(&qt->sub0, &qt->sub1); + fbnic_config_drop_mode_rcq(nv, &qt->cmpl); + fbnic_enable_rcq(nv, &qt->cmpl); + } + } + + fbnic_wrfl(fbd); +} + +static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + u32 val; + + val = FBNIC_INTR_CQ_REARM_INTR_UNMASK; + + fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(nv->v_idx), val); +} + +void fbnic_napi_enable(struct fbnic_net *fbn) +{ + u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_napi_vector *nv; + int i; + + list_for_each_entry(nv, &fbn->napis, napis) { + napi_enable(&nv->napi); + + fbnic_nv_irq_enable(nv); + + /* Record bit used for NAPI IRQs so we can + * set the mask appropriately + */ + irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); + } + + /* Force the first interrupt on the device to guarantee + * that any packets that may have been enqueued during the + * bringup are processed. + */ + for (i = 0; i < ARRAY_SIZE(irqs); i++) { + if (!irqs[i]) + continue; + fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); + } + + fbnic_wrfl(fbd); +} + +void fbnic_napi_depletion_check(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_napi_vector *nv; + int i, j; + + list_for_each_entry(nv, &fbn->napis, napis) { + /* Find RQs which are completely out of pages */ + for (i = nv->txt_count, j = 0; j < nv->rxt_count; j++, i++) { + /* Assume 4 pages is always enough to fit a packet + * and therefore generate a completion and an IRQ. + */ + if (fbnic_desc_used(&nv->qt[i].sub0) < 4 || + fbnic_desc_used(&nv->qt[i].sub1) < 4) + irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); + } + } + + for (i = 0; i < ARRAY_SIZE(irqs); i++) { + if (!irqs[i]) + continue; + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i), irqs[i]); + fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); + } + + fbnic_wrfl(fbd); +} |