diff options
Diffstat (limited to 'drivers/net/ethernet/tehuti/tn40.c')
| -rw-r--r-- | drivers/net/ethernet/tehuti/tn40.c | 1850 |
1 files changed, 1850 insertions, 0 deletions
diff --git a/drivers/net/ethernet/tehuti/tn40.c b/drivers/net/ethernet/tehuti/tn40.c new file mode 100644 index 000000000000..259bdac24cf2 --- /dev/null +++ b/drivers/net/ethernet/tehuti/tn40.c @@ -0,0 +1,1850 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (c) Tehuti Networks Ltd. */ + +#include <linux/bitfield.h> +#include <linux/ethtool.h> +#include <linux/firmware.h> +#include <linux/if_vlan.h> +#include <linux/iopoll.h> +#include <linux/netdevice.h> +#include <linux/pci.h> +#include <linux/phylink.h> +#include <linux/vmalloc.h> +#include <net/netdev_queues.h> +#include <net/page_pool/helpers.h> + +#include "tn40.h" + +#define TN40_SHORT_PACKET_SIZE 60 +#define TN40_FIRMWARE_NAME "tehuti/bdx.bin" + +static void tn40_enable_interrupts(struct tn40_priv *priv) +{ + tn40_write_reg(priv, TN40_REG_IMR, priv->isr_mask); +} + +static void tn40_disable_interrupts(struct tn40_priv *priv) +{ + tn40_write_reg(priv, TN40_REG_IMR, 0); +} + +static int tn40_fifo_alloc(struct tn40_priv *priv, struct tn40_fifo *f, + int fsz_type, + u16 reg_cfg0, u16 reg_cfg1, + u16 reg_rptr, u16 reg_wptr) +{ + u16 memsz = TN40_FIFO_SIZE * (1 << fsz_type); + u64 cfg_base; + + memset(f, 0, sizeof(struct tn40_fifo)); + /* 1K extra space is allocated at the end of the fifo to simplify + * processing of descriptors that wraps around fifo's end. + */ + f->va = dma_alloc_coherent(&priv->pdev->dev, + memsz + TN40_FIFO_EXTRA_SPACE, &f->da, + GFP_KERNEL); + if (!f->va) + return -ENOMEM; + + f->reg_cfg0 = reg_cfg0; + f->reg_cfg1 = reg_cfg1; + f->reg_rptr = reg_rptr; + f->reg_wptr = reg_wptr; + f->rptr = 0; + f->wptr = 0; + f->memsz = memsz; + f->size_mask = memsz - 1; + cfg_base = lower_32_bits((f->da & TN40_TX_RX_CFG0_BASE) | fsz_type); + tn40_write_reg(priv, reg_cfg0, cfg_base); + tn40_write_reg(priv, reg_cfg1, upper_32_bits(f->da)); + return 0; +} + +static void tn40_fifo_free(struct tn40_priv *priv, struct tn40_fifo *f) +{ + dma_free_coherent(&priv->pdev->dev, + f->memsz + TN40_FIFO_EXTRA_SPACE, f->va, f->da); +} + +static struct tn40_rxdb *tn40_rxdb_alloc(int nelem) +{ + size_t size = sizeof(struct tn40_rxdb) + (nelem * sizeof(int)) + + (nelem * sizeof(struct tn40_rx_map)); + struct tn40_rxdb *db; + int i; + + db = vzalloc(size); + if (db) { + db->stack = (int *)(db + 1); + db->elems = (void *)(db->stack + nelem); + db->nelem = nelem; + db->top = nelem; + /* make the first alloc close to db struct */ + for (i = 0; i < nelem; i++) + db->stack[i] = nelem - i - 1; + } + return db; +} + +static void tn40_rxdb_free(struct tn40_rxdb *db) +{ + vfree(db); +} + +static int tn40_rxdb_alloc_elem(struct tn40_rxdb *db) +{ + return db->stack[--db->top]; +} + +static void *tn40_rxdb_addr_elem(struct tn40_rxdb *db, unsigned int n) +{ + return db->elems + n; +} + +static int tn40_rxdb_available(struct tn40_rxdb *db) +{ + return db->top; +} + +static void tn40_rxdb_free_elem(struct tn40_rxdb *db, unsigned int n) +{ + db->stack[db->top++] = n; +} + +/** + * tn40_create_rx_ring - Initialize RX all related HW and SW resources + * @priv: NIC private structure + * + * create_rx_ring creates rxf and rxd fifos, updates the relevant HW registers, + * preallocates skbs for rx. It assumes that Rx is disabled in HW funcs are + * grouped for better cache usage + * + * RxD fifo is smaller then RxF fifo by design. Upon high load, RxD will be + * filled and packets will be dropped by the NIC without getting into the host + * or generating interrupts. In this situation the host has no chance of + * processing all the packets. Dropping packets by the NIC is cheaper, since it + * takes 0 CPU cycles. + * + * Return: 0 on success and negative value on error. + */ +static int tn40_create_rx_ring(struct tn40_priv *priv) +{ + struct page_pool_params pp = { + .dev = &priv->pdev->dev, + .napi = &priv->napi, + .dma_dir = DMA_FROM_DEVICE, + .netdev = priv->ndev, + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .max_len = PAGE_SIZE, + }; + int ret, pkt_size, nr; + + priv->page_pool = page_pool_create(&pp); + if (IS_ERR(priv->page_pool)) + return PTR_ERR(priv->page_pool); + + ret = tn40_fifo_alloc(priv, &priv->rxd_fifo0.m, priv->rxd_size, + TN40_REG_RXD_CFG0_0, TN40_REG_RXD_CFG1_0, + TN40_REG_RXD_RPTR_0, TN40_REG_RXD_WPTR_0); + if (ret) + goto err_destroy_page_pool; + + ret = tn40_fifo_alloc(priv, &priv->rxf_fifo0.m, priv->rxf_size, + TN40_REG_RXF_CFG0_0, TN40_REG_RXF_CFG1_0, + TN40_REG_RXF_RPTR_0, TN40_REG_RXF_WPTR_0); + if (ret) + goto err_free_rxd; + + pkt_size = priv->ndev->mtu + VLAN_ETH_HLEN; + priv->rxf_fifo0.m.pktsz = pkt_size; + nr = priv->rxf_fifo0.m.memsz / sizeof(struct tn40_rxf_desc); + priv->rxdb0 = tn40_rxdb_alloc(nr); + if (!priv->rxdb0) { + ret = -ENOMEM; + goto err_free_rxf; + } + return 0; +err_free_rxf: + tn40_fifo_free(priv, &priv->rxf_fifo0.m); +err_free_rxd: + tn40_fifo_free(priv, &priv->rxd_fifo0.m); +err_destroy_page_pool: + page_pool_destroy(priv->page_pool); + return ret; +} + +static void tn40_rx_free_buffers(struct tn40_priv *priv) +{ + struct tn40_rxdb *db = priv->rxdb0; + struct tn40_rx_map *dm; + u16 i; + + netdev_dbg(priv->ndev, "total =%d free =%d busy =%d\n", db->nelem, + tn40_rxdb_available(db), + db->nelem - tn40_rxdb_available(db)); + + for (i = 0; i < db->nelem; i++) { + dm = tn40_rxdb_addr_elem(db, i); + if (dm->page) + page_pool_put_full_page(priv->page_pool, dm->page, + false); + } +} + +static void tn40_destroy_rx_ring(struct tn40_priv *priv) +{ + if (priv->rxdb0) { + tn40_rx_free_buffers(priv); + tn40_rxdb_free(priv->rxdb0); + priv->rxdb0 = NULL; + } + tn40_fifo_free(priv, &priv->rxf_fifo0.m); + tn40_fifo_free(priv, &priv->rxd_fifo0.m); + page_pool_destroy(priv->page_pool); +} + +static void tn40_set_rx_desc(struct tn40_priv *priv, int idx, u64 dma) +{ + struct tn40_rxf_fifo *f = &priv->rxf_fifo0; + struct tn40_rxf_desc *rxfd; + int delta; + + rxfd = (struct tn40_rxf_desc *)(f->m.va + f->m.wptr); + rxfd->info = cpu_to_le32(0x10003); /* INFO =1 BC =3 */ + rxfd->va_lo = cpu_to_le32(idx); + rxfd->pa_lo = cpu_to_le32(lower_32_bits(dma)); + rxfd->pa_hi = cpu_to_le32(upper_32_bits(dma)); + rxfd->len = cpu_to_le32(f->m.pktsz); + f->m.wptr += sizeof(struct tn40_rxf_desc); + delta = f->m.wptr - f->m.memsz; + if (unlikely(delta >= 0)) { + f->m.wptr = delta; + if (delta > 0) { + memcpy(f->m.va, f->m.va + f->m.memsz, delta); + netdev_dbg(priv->ndev, + "wrapped rxd descriptor\n"); + } + } +} + +/** + * tn40_rx_alloc_buffers - Fill rxf fifo with buffers. + * + * @priv: NIC's private structure + * + * rx_alloc_buffers allocates buffers via the page pool API, builds rxf descs + * and pushes them (rxf descr) into the rxf fifo. The pages are stored in rxdb. + * To calculate the free space, we uses the cached values of RPTR and WPTR + * when needed. This function also updates RPTR and WPTR. + */ +static void tn40_rx_alloc_buffers(struct tn40_priv *priv) +{ + struct tn40_rxf_fifo *f = &priv->rxf_fifo0; + struct tn40_rxdb *db = priv->rxdb0; + struct tn40_rx_map *dm; + struct page *page; + int dno, i, idx; + + dno = tn40_rxdb_available(db) - 1; + for (i = dno; i > 0; i--) { + page = page_pool_dev_alloc_pages(priv->page_pool); + if (!page) + break; + + idx = tn40_rxdb_alloc_elem(db); + tn40_set_rx_desc(priv, idx, page_pool_get_dma_addr(page)); + dm = tn40_rxdb_addr_elem(db, idx); + dm->page = page; + } + if (i != dno) + tn40_write_reg(priv, f->m.reg_wptr, + f->m.wptr & TN40_TXF_WPTR_WR_PTR); + netdev_dbg(priv->ndev, "write_reg 0x%04x f->m.reg_wptr 0x%x\n", + f->m.reg_wptr, f->m.wptr & TN40_TXF_WPTR_WR_PTR); + netdev_dbg(priv->ndev, "read_reg 0x%04x f->m.reg_rptr=0x%x\n", + f->m.reg_rptr, tn40_read_reg(priv, f->m.reg_rptr)); + netdev_dbg(priv->ndev, "write_reg 0x%04x f->m.reg_wptr=0x%x\n", + f->m.reg_wptr, tn40_read_reg(priv, f->m.reg_wptr)); +} + +static void tn40_recycle_rx_buffer(struct tn40_priv *priv, + struct tn40_rxd_desc *rxdd) +{ + struct tn40_rxf_fifo *f = &priv->rxf_fifo0; + struct tn40_rx_map *dm; + int idx; + + idx = le32_to_cpu(rxdd->va_lo); + dm = tn40_rxdb_addr_elem(priv->rxdb0, idx); + tn40_set_rx_desc(priv, idx, page_pool_get_dma_addr(dm->page)); + + tn40_write_reg(priv, f->m.reg_wptr, f->m.wptr & TN40_TXF_WPTR_WR_PTR); +} + +static int tn40_rx_receive(struct tn40_priv *priv, int budget) +{ + struct tn40_rxd_fifo *f = &priv->rxd_fifo0; + u32 rxd_val1, rxd_err, pkt_id; + int tmp_len, size, done = 0; + struct tn40_rxdb *db = NULL; + struct tn40_rxd_desc *rxdd; + struct tn40_rx_map *dm; + struct sk_buff *skb; + u16 len, rxd_vlan; + int idx; + + f->m.wptr = tn40_read_reg(priv, f->m.reg_wptr) & TN40_TXF_WPTR_WR_PTR; + size = f->m.wptr - f->m.rptr; + if (size < 0) + size += f->m.memsz; /* Size is negative :-) */ + + while (size > 0) { + rxdd = (struct tn40_rxd_desc *)(f->m.va + f->m.rptr); + db = priv->rxdb0; + + /* We have a chicken and egg problem here. If the + * descriptor is wrapped we first need to copy the tail + * of the descriptor to the end of the buffer before + * extracting values from the descriptor. However in + * order to know if the descriptor is wrapped we need to + * obtain the length of the descriptor from (the + * wrapped) descriptor. Luckily the length is the first + * word of the descriptor. Descriptor lengths are + * multiples of 8 bytes so in case of a wrapped + * descriptor the first 8 bytes guaranteed to appear + * before the end of the buffer. We first obtain the + * length, we then copy the rest of the descriptor if + * needed and then extract the rest of the values from + * the descriptor. + * + * Do not change the order of operations as it will + * break the code!!! + */ + rxd_val1 = le32_to_cpu(rxdd->rxd_val1); + tmp_len = TN40_GET_RXD_BC(rxd_val1) << 3; + pkt_id = TN40_GET_RXD_PKT_ID(rxd_val1); + size -= tmp_len; + /* CHECK FOR A PARTIALLY ARRIVED DESCRIPTOR */ + if (size < 0) { + netdev_dbg(priv->ndev, + "%s partially arrived desc tmp_len %d\n", + __func__, tmp_len); + break; + } + /* make sure that the descriptor fully is arrived + * before reading the rest of the descriptor. + */ + rmb(); + + /* A special treatment is given to non-contiguous + * descriptors that start near the end, wraps around + * and continue at the beginning. The second part is + * copied right after the first, and then descriptor + * is interpreted as normal. The fifo has an extra + * space to allow such operations. + */ + + /* HAVE WE REACHED THE END OF THE QUEUE? */ + f->m.rptr += tmp_len; + tmp_len = f->m.rptr - f->m.memsz; + if (unlikely(tmp_len >= 0)) { + f->m.rptr = tmp_len; + if (tmp_len > 0) { + /* COPY PARTIAL DESCRIPTOR + * TO THE END OF THE QUEUE + */ + netdev_dbg(priv->ndev, + "wrapped desc rptr=%d tmp_len=%d\n", + f->m.rptr, tmp_len); + memcpy(f->m.va + f->m.memsz, f->m.va, tmp_len); + } + } + idx = le32_to_cpu(rxdd->va_lo); + dm = tn40_rxdb_addr_elem(db, idx); + prefetch(dm); + + len = le16_to_cpu(rxdd->len); + rxd_vlan = le16_to_cpu(rxdd->rxd_vlan); + /* CHECK FOR ERRORS */ + rxd_err = TN40_GET_RXD_ERR(rxd_val1); + if (unlikely(rxd_err)) { + u64_stats_update_begin(&priv->syncp); + priv->stats.rx_errors++; + u64_stats_update_end(&priv->syncp); + tn40_recycle_rx_buffer(priv, rxdd); + continue; + } + + skb = napi_build_skb(page_address(dm->page), PAGE_SIZE); + if (!skb) { + u64_stats_update_begin(&priv->syncp); + priv->stats.rx_dropped++; + priv->alloc_fail++; + u64_stats_update_end(&priv->syncp); + tn40_recycle_rx_buffer(priv, rxdd); + break; + } + skb_mark_for_recycle(skb); + skb_put(skb, len); + skb->protocol = eth_type_trans(skb, priv->ndev); + skb->ip_summed = + (pkt_id == 0) ? CHECKSUM_NONE : CHECKSUM_UNNECESSARY; + if (TN40_GET_RXD_VTAG(rxd_val1)) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + TN40_GET_RXD_VLAN_TCI(rxd_vlan)); + + dm->page = NULL; + tn40_rxdb_free_elem(db, idx); + + napi_gro_receive(&priv->napi, skb); + + u64_stats_update_begin(&priv->syncp); + priv->stats.rx_bytes += len; + u64_stats_update_end(&priv->syncp); + + if (unlikely(++done >= budget)) + break; + } + u64_stats_update_begin(&priv->syncp); + priv->stats.rx_packets += done; + u64_stats_update_end(&priv->syncp); + /* FIXME: Do something to minimize pci accesses */ + tn40_write_reg(priv, f->m.reg_rptr, f->m.rptr & TN40_TXF_WPTR_WR_PTR); + tn40_rx_alloc_buffers(priv); + return done; +} + +/* TX HW/SW interaction overview + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * There are 2 types of TX communication channels between driver and NIC. + * 1) TX Free Fifo - TXF - Holds ack descriptors for sent packets. + * 2) TX Data Fifo - TXD - Holds descriptors of full buffers. + * + * Currently the NIC supports TSO, checksumming and gather DMA + * UFO and IP fragmentation is on the way. + * + * RX SW Data Structures + * ~~~~~~~~~~~~~~~~~~~~~ + * TXDB is used to keep track of all skbs owned by SW and their DMA addresses. + * For TX case, ownership lasts from getting the packet via hard_xmit and + * until the HW acknowledges sending the packet by TXF descriptors. + * TXDB is implemented as a cyclic buffer. + * + * FIFO objects keep info about the fifo's size and location, relevant HW + * registers, usage and skb db. Each RXD and RXF fifo has their own fifo + * structure. Implemented as simple struct. + * + * TX SW Execution Flow + * ~~~~~~~~~~~~~~~~~~~~ + * OS calls the driver's hard_xmit method with a packet to send. The driver + * creates DMA mappings, builds TXD descriptors and kicks the HW by updating + * TXD WPTR. + * + * When a packet is sent, The HW write a TXF descriptor and the SW + * frees the original skb. To prevent TXD fifo overflow without + * reading HW registers every time, the SW deploys "tx level" + * technique. Upon startup, the tx level is initialized to TXD fifo + * length. For every sent packet, the SW gets its TXD descriptor size + * (from a pre-calculated array) and subtracts it from tx level. The + * size is also stored in txdb. When a TXF ack arrives, the SW fetched + * the size of the original TXD descriptor from the txdb and adds it + * to the tx level. When the Tx level drops below some predefined + * threshold, the driver stops the TX queue. When the TX level rises + * above that level, the tx queue is enabled again. + * + * This technique avoids excessive reading of RPTR and WPTR registers. + * As our benchmarks shows, it adds 1.5 Gbit/sec to NIC's throughput. + */ +static void tn40_do_tx_db_ptr_next(struct tn40_txdb *db, + struct tn40_tx_map **pptr) +{ + ++*pptr; + if (unlikely(*pptr == db->end)) + *pptr = db->start; +} + +static void tn40_tx_db_inc_rptr(struct tn40_txdb *db) +{ + tn40_do_tx_db_ptr_next(db, &db->rptr); +} + +static void tn40_tx_db_inc_wptr(struct tn40_txdb *db) +{ + tn40_do_tx_db_ptr_next(db, &db->wptr); +} + +static int tn40_tx_db_init(struct tn40_txdb *d, int sz_type) +{ + int memsz = TN40_FIFO_SIZE * (1 << (sz_type + 1)); + + d->start = vzalloc(memsz); + if (!d->start) + return -ENOMEM; + /* In order to differentiate between an empty db state and a full db + * state at least one element should always be empty in order to + * avoid rptr == wptr, which means that the db is empty. + */ + d->size = memsz / sizeof(struct tn40_tx_map) - 1; + d->end = d->start + d->size + 1; /* just after last element */ + + /* All dbs are created empty */ + d->rptr = d->start; + d->wptr = d->start; + return 0; +} + +static void tn40_tx_db_close(struct tn40_txdb *d) +{ + if (d->start) { + vfree(d->start); + d->start = NULL; + } +} + +/* Sizes of tx desc (including padding if needed) as function of the SKB's + * frag number + * 7 - is number of lwords in txd with one phys buffer + * 3 - is number of lwords used for every additional phys buffer + * for (i = 0; i < TN40_MAX_PBL; i++) { + * lwords = 7 + (i * 3); + * if (lwords & 1) + * lwords++; pad it with 1 lword + * tn40_txd_sizes[i].bytes = lwords << 2; + * tn40_txd_sizes[i].qwords = lwords >> 1; + * } + */ +static struct { + u16 bytes; + u16 qwords; /* qword = 64 bit */ +} tn40_txd_sizes[] = { + {0x20, 0x04}, + {0x28, 0x05}, + {0x38, 0x07}, + {0x40, 0x08}, + {0x50, 0x0a}, + {0x58, 0x0b}, + {0x68, 0x0d}, + {0x70, 0x0e}, + {0x80, 0x10}, + {0x88, 0x11}, + {0x98, 0x13}, + {0xa0, 0x14}, + {0xb0, 0x16}, + {0xb8, 0x17}, + {0xc8, 0x19}, + {0xd0, 0x1a}, + {0xe0, 0x1c}, + {0xe8, 0x1d}, + {0xf8, 0x1f}, +}; + +static void tn40_pbl_set(struct tn40_pbl *pbl, dma_addr_t dma, int len) +{ + pbl->len = cpu_to_le32(len); + pbl->pa_lo = cpu_to_le32(lower_32_bits(dma)); + pbl->pa_hi = cpu_to_le32(upper_32_bits(dma)); +} + +static void tn40_txdb_set(struct tn40_txdb *db, dma_addr_t dma, int len) +{ + db->wptr->len = len; + db->wptr->addr.dma = dma; +} + +struct tn40_mapping_info { + dma_addr_t dma; + size_t size; +}; + +/** + * tn40_tx_map_skb - create and store DMA mappings for skb's data blocks + * @priv: NIC private structure + * @skb: socket buffer to map + * @txdd: pointer to tx descriptor to be updated + * @pkt_len: pointer to unsigned long value + * + * This function creates DMA mappings for skb's data blocks and writes them to + * PBL of a new tx descriptor. It also stores them in the tx db, so they could + * be unmapped after the data has been sent. It is the responsibility of the + * caller to make sure that there is enough space in the txdb. The last + * element holds a pointer to skb itself and is marked with a zero length. + * + * Return: 0 on success and negative value on error. + */ +static int tn40_tx_map_skb(struct tn40_priv *priv, struct sk_buff *skb, + struct tn40_txd_desc *txdd, unsigned int *pkt_len) +{ + struct tn40_mapping_info info[TN40_MAX_PBL]; + int nr_frags = skb_shinfo(skb)->nr_frags; + struct tn40_pbl *pbl = &txdd->pbl[0]; + struct tn40_txdb *db = &priv->txdb; + unsigned int size; + int i, len, ret; + dma_addr_t dma; + + netdev_dbg(priv->ndev, "TX skb %p skbLen %d dataLen %d frags %d\n", skb, + skb->len, skb->data_len, nr_frags); + if (nr_frags > TN40_MAX_PBL - 1) { + ret = skb_linearize(skb); + if (ret) + return ret; + nr_frags = skb_shinfo(skb)->nr_frags; + } + /* initial skb */ + len = skb->len - skb->data_len; + dma = dma_map_single(&priv->pdev->dev, skb->data, len, + DMA_TO_DEVICE); + ret = dma_mapping_error(&priv->pdev->dev, dma); + if (ret) + return ret; + + tn40_txdb_set(db, dma, len); + tn40_pbl_set(pbl++, db->wptr->addr.dma, db->wptr->len); + *pkt_len = db->wptr->len; + + for (i = 0; i < nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + size = skb_frag_size(frag); + dma = skb_frag_dma_map(&priv->pdev->dev, frag, 0, + size, DMA_TO_DEVICE); + + ret = dma_mapping_error(&priv->pdev->dev, dma); + if (ret) + goto mapping_error; + info[i].dma = dma; + info[i].size = size; + } + + for (i = 0; i < nr_frags; i++) { + tn40_tx_db_inc_wptr(db); + tn40_txdb_set(db, info[i].dma, info[i].size); + tn40_pbl_set(pbl++, db->wptr->addr.dma, db->wptr->len); + *pkt_len += db->wptr->len; + } + + /* SHORT_PKT_FIX */ + if (skb->len < TN40_SHORT_PACKET_SIZE) + ++nr_frags; + + /* Add skb clean up info. */ + tn40_tx_db_inc_wptr(db); + db->wptr->len = -tn40_txd_sizes[nr_frags].bytes; + db->wptr->addr.skb = skb; + tn40_tx_db_inc_wptr(db); + + return 0; + mapping_error: + dma_unmap_page(&priv->pdev->dev, db->wptr->addr.dma, db->wptr->len, + DMA_TO_DEVICE); + for (; i > 0; i--) + dma_unmap_page(&priv->pdev->dev, info[i - 1].dma, + info[i - 1].size, DMA_TO_DEVICE); + return -ENOMEM; +} + +static int tn40_create_tx_ring(struct tn40_priv *priv) +{ + int ret; + + ret = tn40_fifo_alloc(priv, &priv->txd_fifo0.m, priv->txd_size, + TN40_REG_TXD_CFG0_0, TN40_REG_TXD_CFG1_0, + TN40_REG_TXD_RPTR_0, TN40_REG_TXD_WPTR_0); + if (ret) + return ret; + + ret = tn40_fifo_alloc(priv, &priv->txf_fifo0.m, priv->txf_size, + TN40_REG_TXF_CFG0_0, TN40_REG_TXF_CFG1_0, + TN40_REG_TXF_RPTR_0, TN40_REG_TXF_WPTR_0); + if (ret) + goto err_free_txd; + + /* The TX db has to keep mappings for all packets sent (on + * TxD) and not yet reclaimed (on TxF). + */ + ret = tn40_tx_db_init(&priv->txdb, max(priv->txd_size, priv->txf_size)); + if (ret) + goto err_free_txf; + + /* SHORT_PKT_FIX */ + priv->b0_len = 64; + priv->b0_va = dma_alloc_coherent(&priv->pdev->dev, priv->b0_len, + &priv->b0_dma, GFP_KERNEL); + if (!priv->b0_va) + goto err_free_db; + + priv->tx_level = TN40_MAX_TX_LEVEL; + priv->tx_update_mark = priv->tx_level - 1024; + return 0; +err_free_db: + tn40_tx_db_close(&priv->txdb); +err_free_txf: + tn40_fifo_free(priv, &priv->txf_fifo0.m); +err_free_txd: + tn40_fifo_free(priv, &priv->txd_fifo0.m); + return -ENOMEM; +} + +/** + * tn40_tx_space - Calculate the available space in the TX fifo. + * @priv: NIC private structure + * + * Return: available space in TX fifo in bytes + */ +static int tn40_tx_space(struct tn40_priv *priv) +{ + struct tn40_txd_fifo *f = &priv->txd_fifo0; + int fsize; + + f->m.rptr = tn40_read_reg(priv, f->m.reg_rptr) & TN40_TXF_WPTR_WR_PTR; + fsize = f->m.rptr - f->m.wptr; + if (fsize <= 0) + fsize = f->m.memsz + fsize; + return fsize; +} + +#define TN40_TXD_FULL_CHECKSUM 7 + +static netdev_tx_t tn40_start_xmit(struct sk_buff *skb, struct net_device *ndev) +{ + struct tn40_priv *priv = netdev_priv(ndev); + struct tn40_txd_fifo *f = &priv->txd_fifo0; + int txd_checksum = TN40_TXD_FULL_CHECKSUM; + struct tn40_txd_desc *txdd; + int nr_frags, len, err; + unsigned int pkt_len; + int txd_vlan_id = 0; + int txd_lgsnd = 0; + int txd_vtag = 0; + int txd_mss = 0; + + /* Build tx descriptor */ + txdd = (struct tn40_txd_desc *)(f->m.va + f->m.wptr); + err = tn40_tx_map_skb(priv, skb, txdd, &pkt_len); + if (err) { + u64_stats_update_begin(&priv->syncp); + priv->stats.tx_dropped++; + u64_stats_update_end(&priv->syncp); + dev_kfree_skb(skb); + return NETDEV_TX_OK; + } + nr_frags = skb_shinfo(skb)->nr_frags; + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) + txd_checksum = 0; + + if (skb_shinfo(skb)->gso_size) { + txd_mss = skb_shinfo(skb)->gso_size; + txd_lgsnd = 1; + netdev_dbg(priv->ndev, "skb %p pkt len %d gso size = %d\n", skb, + pkt_len, txd_mss); + } + if (skb_vlan_tag_present(skb)) { + /* Don't cut VLAN ID to 12 bits */ + txd_vlan_id = skb_vlan_tag_get(skb); + txd_vtag = 1; + } + txdd->va_hi = 0; + txdd->va_lo = 0; + txdd->length = cpu_to_le16(pkt_len); + txdd->mss = cpu_to_le16(txd_mss); + txdd->txd_val1 = + cpu_to_le32(TN40_TXD_W1_VAL + (tn40_txd_sizes[nr_frags].qwords, txd_checksum, + txd_vtag, txd_lgsnd, txd_vlan_id)); + netdev_dbg(priv->ndev, "=== w1 qwords[%d] %d =====\n", nr_frags, + tn40_txd_sizes[nr_frags].qwords); + netdev_dbg(priv->ndev, "=== TxD desc =====================\n"); + netdev_dbg(priv->ndev, "=== w1: 0x%x ================\n", + txdd->txd_val1); + netdev_dbg(priv->ndev, "=== w2: mss 0x%x len 0x%x\n", txdd->mss, + txdd->length); + /* SHORT_PKT_FIX */ + if (pkt_len < TN40_SHORT_PACKET_SIZE) { + struct tn40_pbl *pbl = &txdd->pbl[++nr_frags]; + + txdd->length = cpu_to_le16(TN40_SHORT_PACKET_SIZE); + txdd->txd_val1 = + cpu_to_le32(TN40_TXD_W1_VAL + (tn40_txd_sizes[nr_frags].qwords, + txd_checksum, txd_vtag, txd_lgsnd, + txd_vlan_id)); + pbl->len = cpu_to_le32(TN40_SHORT_PACKET_SIZE - pkt_len); + pbl->pa_lo = cpu_to_le32(lower_32_bits(priv->b0_dma)); + pbl->pa_hi = cpu_to_le32(upper_32_bits(priv->b0_dma)); + netdev_dbg(priv->ndev, "=== SHORT_PKT_FIX ==============\n"); + netdev_dbg(priv->ndev, "=== nr_frags : %d ==============\n", + nr_frags); + } + + /* Increment TXD write pointer. In case of fifo wrapping copy + * reminder of the descriptor to the beginning. + */ + f->m.wptr += tn40_txd_sizes[nr_frags].bytes; + len = f->m.wptr - f->m.memsz; + if (unlikely(len >= 0)) { + f->m.wptr = len; + if (len > 0) + memcpy(f->m.va, f->m.va + f->m.memsz, len); + } + /* Force memory writes to complete before letting the HW know + * there are new descriptors to fetch. + */ + wmb(); + + priv->tx_level -= tn40_txd_sizes[nr_frags].bytes; + if (priv->tx_level > priv->tx_update_mark) { + tn40_write_reg(priv, f->m.reg_wptr, + f->m.wptr & TN40_TXF_WPTR_WR_PTR); + } else { + if (priv->tx_noupd++ > TN40_NO_UPD_PACKETS) { + priv->tx_noupd = 0; + tn40_write_reg(priv, f->m.reg_wptr, + f->m.wptr & TN40_TXF_WPTR_WR_PTR); + } + } + + u64_stats_update_begin(&priv->syncp); + priv->stats.tx_packets++; + priv->stats.tx_bytes += pkt_len; + u64_stats_update_end(&priv->syncp); + if (priv->tx_level < TN40_MIN_TX_LEVEL) { + netdev_dbg(priv->ndev, "TX Q STOP level %d\n", priv->tx_level); + netif_stop_queue(ndev); + } + + return NETDEV_TX_OK; +} + +static void tn40_tx_cleanup(struct tn40_priv *priv) +{ + struct tn40_txf_fifo *f = &priv->txf_fifo0; + struct tn40_txdb *db = &priv->txdb; + int tx_level = 0; + + f->m.wptr = tn40_read_reg(priv, f->m.reg_wptr) & TN40_TXF_WPTR_MASK; + + netif_tx_lock(priv->ndev); + while (f->m.wptr != f->m.rptr) { + f->m.rptr += TN40_TXF_DESC_SZ; + f->m.rptr &= f->m.size_mask; + /* Unmap all fragments */ + /* First has to come tx_maps containing DMA */ + do { + dma_addr_t addr = db->rptr->addr.dma; + size_t size = db->rptr->len; + + netif_tx_unlock(priv->ndev); + dma_unmap_page(&priv->pdev->dev, addr, + size, DMA_TO_DEVICE); + netif_tx_lock(priv->ndev); + tn40_tx_db_inc_rptr(db); + } while (db->rptr->len > 0); + tx_level -= db->rptr->len; /* '-' Because the len is negative */ + + /* Now should come skb pointer - free it */ + dev_kfree_skb_any(db->rptr->addr.skb); + netdev_dbg(priv->ndev, "dev_kfree_skb_any %p %d\n", + db->rptr->addr.skb, -db->rptr->len); + tn40_tx_db_inc_rptr(db); + } + + /* Let the HW know which TXF descriptors were cleaned */ + tn40_write_reg(priv, f->m.reg_rptr, f->m.rptr & TN40_TXF_WPTR_WR_PTR); + + /* We reclaimed resources, so in case the Q is stopped by xmit + * callback, we resume the transmission and use tx_lock to + * synchronize with xmit. + */ + priv->tx_level += tx_level; + if (priv->tx_noupd) { + priv->tx_noupd = 0; + tn40_write_reg(priv, priv->txd_fifo0.m.reg_wptr, + priv->txd_fifo0.m.wptr & TN40_TXF_WPTR_WR_PTR); + } + if (unlikely(netif_queue_stopped(priv->ndev) && + netif_carrier_ok(priv->ndev) && + (priv->tx_level >= TN40_MAX_TX_LEVEL / 2))) { + netdev_dbg(priv->ndev, "TX Q WAKE level %d\n", priv->tx_level); + netif_wake_queue(priv->ndev); + } + netif_tx_unlock(priv->ndev); +} + +static void tn40_tx_free_skbs(struct tn40_priv *priv) +{ + struct tn40_txdb *db = &priv->txdb; + + while (db->rptr != db->wptr) { + if (likely(db->rptr->len)) + dma_unmap_page(&priv->pdev->dev, db->rptr->addr.dma, + db->rptr->len, DMA_TO_DEVICE); + else + dev_kfree_skb(db->rptr->addr.skb); + tn40_tx_db_inc_rptr(db); + } +} + +static void tn40_destroy_tx_ring(struct tn40_priv *priv) +{ + tn40_tx_free_skbs(priv); + tn40_fifo_free(priv, &priv->txd_fifo0.m); + tn40_fifo_free(priv, &priv->txf_fifo0.m); + tn40_tx_db_close(&priv->txdb); + /* SHORT_PKT_FIX */ + if (priv->b0_len) { + dma_free_coherent(&priv->pdev->dev, priv->b0_len, priv->b0_va, + priv->b0_dma); + priv->b0_len = 0; + } +} + +/** + * tn40_tx_push_desc - Push a descriptor to TxD fifo. + * + * @priv: NIC private structure + * @data: desc's data + * @size: desc's size + * + * This function pushes desc to TxD fifo and overlaps it if needed. + * + * This function does not check for available space, nor does it check + * that the data size is smaller than the fifo size. Checking for + * space is the responsibility of the caller. + */ +static void tn40_tx_push_desc(struct tn40_priv *priv, void *data, int size) +{ + struct tn40_txd_fifo *f = &priv->txd_fifo0; + int i = f->m.memsz - f->m.wptr; + + if (size == 0) + return; + + if (i > size) { + memcpy(f->m.va + f->m.wptr, data, size); + f->m.wptr += size; + } else { + memcpy(f->m.va + f->m.wptr, data, i); + f->m.wptr = size - i; + memcpy(f->m.va, data + i, f->m.wptr); + } + tn40_write_reg(priv, f->m.reg_wptr, f->m.wptr & TN40_TXF_WPTR_WR_PTR); +} + +/** + * tn40_tx_push_desc_safe - push descriptor to TxD fifo in a safe way. + * + * @priv: NIC private structure + * @data: descriptor data + * @size: descriptor size + * + * This function does check for available space and, if necessary, + * waits for the NIC to read existing data before writing new data. + */ +static void tn40_tx_push_desc_safe(struct tn40_priv *priv, void *data, int size) +{ + int timer = 0; + + while (size > 0) { + /* We subtract 8 because when the fifo is full rptr == + * wptr, which also means that fifo is empty, we can + * understand the difference, but could the HW do the + * same ??? + */ + int avail = tn40_tx_space(priv) - 8; + + if (avail <= 0) { + if (timer++ > 300) /* Prevent endless loop */ + break; + /* Give the HW a chance to clean the fifo */ + usleep_range(50, 60); + continue; + } + avail = min(avail, size); + netdev_dbg(priv->ndev, + "about to push %d bytes starting %p size %d\n", + avail, data, size); + tn40_tx_push_desc(priv, data, avail); + size -= avail; + data += avail; + } +} + +int tn40_set_link_speed(struct tn40_priv *priv, u32 speed) +{ + u32 val; + int i; + + netdev_dbg(priv->ndev, "speed %d\n", speed); + switch (speed) { + case SPEED_10000: + case SPEED_5000: + case SPEED_2500: + netdev_dbg(priv->ndev, "link_speed %d\n", speed); + + tn40_write_reg(priv, 0x1010, 0x217); /*ETHSD.REFCLK_CONF */ + tn40_write_reg(priv, 0x104c, 0x4c); /*ETHSD.L0_RX_PCNT */ + tn40_write_reg(priv, 0x1050, 0x4c); /*ETHSD.L1_RX_PCNT */ + tn40_write_reg(priv, 0x1054, 0x4c); /*ETHSD.L2_RX_PCNT */ + tn40_write_reg(priv, 0x1058, 0x4c); /*ETHSD.L3_RX_PCNT */ + tn40_write_reg(priv, 0x102c, 0x434); /*ETHSD.L0_TX_PCNT */ + tn40_write_reg(priv, 0x1030, 0x434); /*ETHSD.L1_TX_PCNT */ + tn40_write_reg(priv, 0x1034, 0x434); /*ETHSD.L2_TX_PCNT */ + tn40_write_reg(priv, 0x1038, 0x434); /*ETHSD.L3_TX_PCNT */ + tn40_write_reg(priv, 0x6300, 0x0400); /*MAC.PCS_CTRL */ + + tn40_write_reg(priv, 0x1018, 0x00); /*Mike2 */ + udelay(5); + tn40_write_reg(priv, 0x1018, 0x04); /*Mike2 */ + udelay(5); + tn40_write_reg(priv, 0x1018, 0x06); /*Mike2 */ + udelay(5); + /*MikeFix1 */ + /*L0: 0x103c , L1: 0x1040 , L2: 0x1044 , L3: 0x1048 =0x81644 */ + tn40_write_reg(priv, 0x103c, 0x81644); /*ETHSD.L0_TX_DCNT */ + tn40_write_reg(priv, 0x1040, 0x81644); /*ETHSD.L1_TX_DCNT */ + tn40_write_reg(priv, 0x1044, 0x81644); /*ETHSD.L2_TX_DCNT */ + tn40_write_reg(priv, 0x1048, 0x81644); /*ETHSD.L3_TX_DCNT */ + tn40_write_reg(priv, 0x1014, 0x043); /*ETHSD.INIT_STAT */ + for (i = 1000; i; i--) { + usleep_range(50, 60); + /*ETHSD.INIT_STAT */ + val = tn40_read_reg(priv, 0x1014); + if (val & (1 << 9)) { + /*ETHSD.INIT_STAT */ + tn40_write_reg(priv, 0x1014, 0x3); + /*ETHSD.INIT_STAT */ + val = tn40_read_reg(priv, 0x1014); + + break; + } + } + if (!i) + netdev_err(priv->ndev, "MAC init timeout!\n"); + + tn40_write_reg(priv, 0x6350, 0x0); /*MAC.PCS_IF_MODE */ + tn40_write_reg(priv, TN40_REG_CTRLST, 0xC13); /*0x93//0x13 */ + tn40_write_reg(priv, 0x111c, 0x7ff); /*MAC.MAC_RST_CNT */ + usleep_range(2000, 2100); + + tn40_write_reg(priv, 0x111c, 0x0); /*MAC.MAC_RST_CNT */ + break; + + case SPEED_1000: + case SPEED_100: + tn40_write_reg(priv, 0x1010, 0x613); /*ETHSD.REFCLK_CONF */ + tn40_write_reg(priv, 0x104c, 0x4d); /*ETHSD.L0_RX_PCNT */ + tn40_write_reg(priv, 0x1050, 0x0); /*ETHSD.L1_RX_PCNT */ + tn40_write_reg(priv, 0x1054, 0x0); /*ETHSD.L2_RX_PCNT */ + tn40_write_reg(priv, 0x1058, 0x0); /*ETHSD.L3_RX_PCNT */ + tn40_write_reg(priv, 0x102c, 0x35); /*ETHSD.L0_TX_PCNT */ + tn40_write_reg(priv, 0x1030, 0x0); /*ETHSD.L1_TX_PCNT */ + tn40_write_reg(priv, 0x1034, 0x0); /*ETHSD.L2_TX_PCNT */ + tn40_write_reg(priv, 0x1038, 0x0); /*ETHSD.L3_TX_PCNT */ + tn40_write_reg(priv, 0x6300, 0x01140); /*MAC.PCS_CTRL */ + + tn40_write_reg(priv, 0x1014, 0x043); /*ETHSD.INIT_STAT */ + for (i = 1000; i; i--) { + usleep_range(50, 60); + val = tn40_read_reg(priv, 0x1014); /*ETHSD.INIT_STAT */ + if (val & (1 << 9)) { + /*ETHSD.INIT_STAT */ + tn40_write_reg(priv, 0x1014, 0x3); + /*ETHSD.INIT_STAT */ + val = tn40_read_reg(priv, 0x1014); + + break; + } + } + if (!i) + netdev_err(priv->ndev, "MAC init timeout!\n"); + + tn40_write_reg(priv, 0x6350, 0x2b); /*MAC.PCS_IF_MODE 1g */ + tn40_write_reg(priv, 0x6310, 0x9801); /*MAC.PCS_DEV_AB */ + + tn40_write_reg(priv, 0x6314, 0x1); /*MAC.PCS_PART_AB */ + tn40_write_reg(priv, 0x6348, 0xc8); /*MAC.PCS_LINK_LO */ + tn40_write_reg(priv, 0x634c, 0xc8); /*MAC.PCS_LINK_HI */ + usleep_range(50, 60); + tn40_write_reg(priv, TN40_REG_CTRLST, 0xC13); /*0x93//0x13 */ + tn40_write_reg(priv, 0x111c, 0x7ff); /*MAC.MAC_RST_CNT */ + usleep_range(2000, 2100); + + tn40_write_reg(priv, 0x111c, 0x0); /*MAC.MAC_RST_CNT */ + tn40_write_reg(priv, 0x6300, 0x1140); /*MAC.PCS_CTRL */ + break; + + case 0: /* Link down */ + tn40_write_reg(priv, 0x104c, 0x0); /*ETHSD.L0_RX_PCNT */ + tn40_write_reg(priv, 0x1050, 0x0); /*ETHSD.L1_RX_PCNT */ + tn40_write_reg(priv, 0x1054, 0x0); /*ETHSD.L2_RX_PCNT */ + tn40_write_reg(priv, 0x1058, 0x0); /*ETHSD.L3_RX_PCNT */ + tn40_write_reg(priv, 0x102c, 0x0); /*ETHSD.L0_TX_PCNT */ + tn40_write_reg(priv, 0x1030, 0x0); /*ETHSD.L1_TX_PCNT */ + tn40_write_reg(priv, 0x1034, 0x0); /*ETHSD.L2_TX_PCNT */ + tn40_write_reg(priv, 0x1038, 0x0); /*ETHSD.L3_TX_PCNT */ + + tn40_write_reg(priv, TN40_REG_CTRLST, 0x800); + tn40_write_reg(priv, 0x111c, 0x7ff); /*MAC.MAC_RST_CNT */ + usleep_range(2000, 2100); + + tn40_write_reg(priv, 0x111c, 0x0); /*MAC.MAC_RST_CNT */ + break; + + default: + netdev_err(priv->ndev, + "Link speed was not identified yet (%d)\n", speed); + speed = 0; + break; + } + return speed; +} + +static void tn40_link_changed(struct tn40_priv *priv) +{ + u32 link = tn40_read_reg(priv, + TN40_REG_MAC_LNK_STAT) & TN40_MAC_LINK_STAT; + + netdev_dbg(priv->ndev, "link changed %u\n", link); +} + +static void tn40_isr_extra(struct tn40_priv *priv, u32 isr) +{ + if (isr & (TN40_IR_LNKCHG0 | TN40_IR_LNKCHG1 | TN40_IR_TMR0)) { + netdev_dbg(priv->ndev, "isr = 0x%x\n", isr); + tn40_link_changed(priv); + } +} + +static irqreturn_t tn40_isr_napi(int irq, void *dev) +{ + struct tn40_priv *priv = netdev_priv((struct net_device *)dev); + u32 isr; + + isr = tn40_read_reg(priv, TN40_REG_ISR_MSK0); + + if (unlikely(!isr)) { + tn40_enable_interrupts(priv); + return IRQ_NONE; /* Not our interrupt */ + } + + if (isr & TN40_IR_EXTRA) + tn40_isr_extra(priv, isr); + + if (isr & (TN40_IR_RX_DESC_0 | TN40_IR_TX_FREE_0 | TN40_IR_TMR1)) { + if (likely(napi_schedule_prep(&priv->napi))) { + __napi_schedule(&priv->napi); + return IRQ_HANDLED; + } + /* We get here if an interrupt has slept into the + * small time window between these lines in + * tn40_poll: tn40_enable_interrupts(priv); return 0; + * + * Currently interrupts are disabled (since we read + * the ISR register) and we have failed to register + * the next poll. So we read the regs to trigger the + * chip and allow further interrupts. + */ + tn40_read_reg(priv, TN40_REG_TXF_WPTR_0); + tn40_read_reg(priv, TN40_REG_RXD_WPTR_0); + } + + tn40_enable_interrupts(priv); + return IRQ_HANDLED; +} + +static int tn40_poll(struct napi_struct *napi, int budget) +{ + struct tn40_priv *priv = container_of(napi, struct tn40_priv, napi); + int work_done; + + tn40_tx_cleanup(priv); + + if (!budget) + return 0; + + work_done = tn40_rx_receive(priv, budget); + if (work_done == budget) + return budget; + + if (napi_complete_done(napi, work_done)) + tn40_enable_interrupts(priv); + return work_done; +} + +static int tn40_fw_load(struct tn40_priv *priv) +{ + const struct firmware *fw = NULL; + int master, ret; + u32 val; + + ret = request_firmware(&fw, TN40_FIRMWARE_NAME, &priv->pdev->dev); + if (ret) + return ret; + + master = tn40_read_reg(priv, TN40_REG_INIT_SEMAPHORE); + if (!tn40_read_reg(priv, TN40_REG_INIT_STATUS) && master) { + netdev_dbg(priv->ndev, "Loading FW...\n"); + tn40_tx_push_desc_safe(priv, (void *)fw->data, fw->size); + msleep(100); + } + ret = read_poll_timeout(tn40_read_reg, val, val, 2000, 400000, false, + priv, TN40_REG_INIT_STATUS); + if (master) + tn40_write_reg(priv, TN40_REG_INIT_SEMAPHORE, 1); + + if (ret) { + netdev_err(priv->ndev, "firmware loading failed\n"); + netdev_dbg(priv->ndev, "VPC: 0x%x VIC: 0x%x STATUS: 0x%xd\n", + tn40_read_reg(priv, TN40_REG_VPC), + tn40_read_reg(priv, TN40_REG_VIC), + tn40_read_reg(priv, TN40_REG_INIT_STATUS)); + ret = -EIO; + } else { + netdev_dbg(priv->ndev, "firmware loading success\n"); + } + release_firmware(fw); + return ret; +} + +static void tn40_restore_mac(struct net_device *ndev, struct tn40_priv *priv) +{ + u32 val; + + netdev_dbg(priv->ndev, "mac0 =%x mac1 =%x mac2 =%x\n", + tn40_read_reg(priv, TN40_REG_UNC_MAC0_A), + tn40_read_reg(priv, TN40_REG_UNC_MAC1_A), + tn40_read_reg(priv, TN40_REG_UNC_MAC2_A)); + + val = (ndev->dev_addr[0] << 8) | (ndev->dev_addr[1]); + tn40_write_reg(priv, TN40_REG_UNC_MAC2_A, val); + val = (ndev->dev_addr[2] << 8) | (ndev->dev_addr[3]); + tn40_write_reg(priv, TN40_REG_UNC_MAC1_A, val); + val = (ndev->dev_addr[4] << 8) | (ndev->dev_addr[5]); + tn40_write_reg(priv, TN40_REG_UNC_MAC0_A, val); + + /* More then IP MAC address */ + tn40_write_reg(priv, TN40_REG_MAC_ADDR_0, + (ndev->dev_addr[3] << 24) | (ndev->dev_addr[2] << 16) | + (ndev->dev_addr[1] << 8) | (ndev->dev_addr[0])); + tn40_write_reg(priv, TN40_REG_MAC_ADDR_1, + (ndev->dev_addr[5] << 8) | (ndev->dev_addr[4])); + + netdev_dbg(priv->ndev, "mac0 =%x mac1 =%x mac2 =%x\n", + tn40_read_reg(priv, TN40_REG_UNC_MAC0_A), + tn40_read_reg(priv, TN40_REG_UNC_MAC1_A), + tn40_read_reg(priv, TN40_REG_UNC_MAC2_A)); +} + +static void tn40_hw_start(struct tn40_priv *priv) +{ + tn40_write_reg(priv, TN40_REG_FRM_LENGTH, 0X3FE0); + tn40_write_reg(priv, TN40_REG_GMAC_RXF_A, 0X10fd); + /*MikeFix1 */ + /*L0: 0x103c , L1: 0x1040 , L2: 0x1044 , L3: 0x1048 =0x81644 */ + tn40_write_reg(priv, 0x103c, 0x81644); /*ETHSD.L0_TX_DCNT */ + tn40_write_reg(priv, 0x1040, 0x81644); /*ETHSD.L1_TX_DCNT */ + tn40_write_reg(priv, 0x1044, 0x81644); /*ETHSD.L2_TX_DCNT */ + tn40_write_reg(priv, 0x1048, 0x81644); /*ETHSD.L3_TX_DCNT */ + tn40_write_reg(priv, TN40_REG_RX_FIFO_SECTION, 0x10); + tn40_write_reg(priv, TN40_REG_TX_FIFO_SECTION, 0xE00010); + tn40_write_reg(priv, TN40_REG_RX_FULLNESS, 0); + tn40_write_reg(priv, TN40_REG_TX_FULLNESS, 0); + + tn40_write_reg(priv, TN40_REG_VGLB, 0); + tn40_write_reg(priv, TN40_REG_MAX_FRAME_A, + priv->rxf_fifo0.m.pktsz & TN40_MAX_FRAME_AB_VAL); + tn40_write_reg(priv, TN40_REG_RDINTCM0, priv->rdintcm); + tn40_write_reg(priv, TN40_REG_RDINTCM2, 0); + + /* old val = 0x300064 */ + tn40_write_reg(priv, TN40_REG_TDINTCM0, priv->tdintcm); + + /* Enable timer interrupt once in 2 secs. */ + tn40_restore_mac(priv->ndev, priv); + + /* Pause frame */ + tn40_write_reg(priv, 0x12E0, 0x28); + tn40_write_reg(priv, TN40_REG_PAUSE_QUANT, 0xFFFF); + tn40_write_reg(priv, 0x6064, 0xF); + + tn40_write_reg(priv, TN40_REG_GMAC_RXF_A, + TN40_GMAC_RX_FILTER_OSEN | TN40_GMAC_RX_FILTER_TXFC | + TN40_GMAC_RX_FILTER_AM | TN40_GMAC_RX_FILTER_AB); + + tn40_enable_interrupts(priv); +} + +static int tn40_hw_reset(struct tn40_priv *priv) +{ + u32 val; + + /* Reset sequences: read, write 1, read, write 0 */ + val = tn40_read_reg(priv, TN40_REG_CLKPLL); + tn40_write_reg(priv, TN40_REG_CLKPLL, (val | TN40_CLKPLL_SFTRST) + 0x8); + usleep_range(50, 60); + val = tn40_read_reg(priv, TN40_REG_CLKPLL); + tn40_write_reg(priv, TN40_REG_CLKPLL, val & ~TN40_CLKPLL_SFTRST); + + /* Check that the PLLs are locked and reset ended */ + val = read_poll_timeout(tn40_read_reg, val, + (val & TN40_CLKPLL_LKD) == TN40_CLKPLL_LKD, + 10000, 700000, false, priv, TN40_REG_CLKPLL); + if (val) + return -EIO; + + usleep_range(50, 60); + /* Do any PCI-E read transaction */ + tn40_read_reg(priv, TN40_REG_RXD_CFG0_0); + return 0; +} + +static void tn40_sw_reset(struct tn40_priv *priv) +{ + int i, ret; + u32 val; + + /* 1. load MAC (obsolete) */ + /* 2. disable Rx (and Tx) */ + tn40_write_reg(priv, TN40_REG_GMAC_RXF_A, 0); + msleep(100); + /* 3. Disable port */ + tn40_write_reg(priv, TN40_REG_DIS_PORT, 1); + /* 4. Disable queue */ + tn40_write_reg(priv, TN40_REG_DIS_QU, 1); + /* 5. Wait until hw is disabled */ + ret = read_poll_timeout(tn40_read_reg, val, val & 1, 10000, 500000, + false, priv, TN40_REG_RST_PORT); + if (ret) + netdev_err(priv->ndev, "SW reset timeout. continuing anyway\n"); + + /* 6. Disable interrupts */ + tn40_write_reg(priv, TN40_REG_RDINTCM0, 0); + tn40_write_reg(priv, TN40_REG_TDINTCM0, 0); + tn40_write_reg(priv, TN40_REG_IMR, 0); + tn40_read_reg(priv, TN40_REG_ISR); + + /* 7. Reset queue */ + tn40_write_reg(priv, TN40_REG_RST_QU, 1); + /* 8. Reset port */ + tn40_write_reg(priv, TN40_REG_RST_PORT, 1); + /* 9. Zero all read and write pointers */ + for (i = TN40_REG_TXD_WPTR_0; i <= TN40_REG_TXF_RPTR_3; i += 0x10) + tn40_write_reg(priv, i, 0); + /* 10. Unset port disable */ + tn40_write_reg(priv, TN40_REG_DIS_PORT, 0); + /* 11. Unset queue disable */ + tn40_write_reg(priv, TN40_REG_DIS_QU, 0); + /* 12. Unset queue reset */ + tn40_write_reg(priv, TN40_REG_RST_QU, 0); + /* 13. Unset port reset */ + tn40_write_reg(priv, TN40_REG_RST_PORT, 0); + /* 14. Enable Rx */ + /* Skipped. will be done later */ +} + +static int tn40_start(struct tn40_priv *priv) +{ + int ret; + + ret = tn40_create_tx_ring(priv); + if (ret) { + netdev_err(priv->ndev, "failed to tx init %d\n", ret); + return ret; + } + + ret = tn40_create_rx_ring(priv); + if (ret) { + netdev_err(priv->ndev, "failed to rx init %d\n", ret); + goto err_tx_ring; + } + + tn40_rx_alloc_buffers(priv); + if (tn40_rxdb_available(priv->rxdb0) != 1) { + ret = -ENOMEM; + netdev_err(priv->ndev, "failed to allocate rx buffers\n"); + goto err_rx_ring; + } + + ret = request_irq(priv->pdev->irq, &tn40_isr_napi, IRQF_SHARED, + priv->ndev->name, priv->ndev); + if (ret) { + netdev_err(priv->ndev, "failed to request irq %d\n", ret); + goto err_rx_ring; + } + + tn40_hw_start(priv); + return 0; +err_rx_ring: + tn40_destroy_rx_ring(priv); +err_tx_ring: + tn40_destroy_tx_ring(priv); + return ret; +} + +static void tn40_stop(struct tn40_priv *priv) +{ + tn40_disable_interrupts(priv); + free_irq(priv->pdev->irq, priv->ndev); + tn40_sw_reset(priv); + tn40_destroy_tx_ring(priv); + tn40_destroy_rx_ring(priv); +} + +static int tn40_close(struct net_device *ndev) +{ + struct tn40_priv *priv = netdev_priv(ndev); + + phylink_stop(priv->phylink); + phylink_disconnect_phy(priv->phylink); + + napi_disable(&priv->napi); + netif_napi_del(&priv->napi); + tn40_stop(priv); + return 0; +} + +static int tn40_open(struct net_device *dev) +{ + struct tn40_priv *priv = netdev_priv(dev); + int ret; + + ret = phylink_connect_phy(priv->phylink, priv->phydev); + if (ret) { + netdev_err(dev, "failed to connect to phy %d\n", ret); + return ret; + } + tn40_sw_reset(priv); + ret = tn40_start(priv); + if (ret) { + phylink_disconnect_phy(priv->phylink); + netdev_err(dev, "failed to start %d\n", ret); + return ret; + } + napi_enable(&priv->napi); + phylink_start(priv->phylink); + netif_start_queue(priv->ndev); + return 0; +} + +static void __tn40_vlan_rx_vid(struct net_device *ndev, uint16_t vid, + int enable) +{ + struct tn40_priv *priv = netdev_priv(ndev); + u32 reg, bit, val; + + netdev_dbg(priv->ndev, "vid =%d value =%d\n", (int)vid, enable); + reg = TN40_REG_VLAN_0 + (vid / 32) * 4; + bit = 1 << vid % 32; + val = tn40_read_reg(priv, reg); + netdev_dbg(priv->ndev, "reg =%x, val =%x, bit =%d\n", reg, val, bit); + if (enable) + val |= bit; + else + val &= ~bit; + netdev_dbg(priv->ndev, "new val %x\n", val); + tn40_write_reg(priv, reg, val); +} + +static int tn40_vlan_rx_add_vid(struct net_device *ndev, + __always_unused __be16 proto, u16 vid) +{ + __tn40_vlan_rx_vid(ndev, vid, 1); + return 0; +} + +static int tn40_vlan_rx_kill_vid(struct net_device *ndev, + __always_unused __be16 proto, u16 vid) +{ + __tn40_vlan_rx_vid(ndev, vid, 0); + return 0; +} + +static void tn40_setmulti(struct net_device *ndev) +{ + u32 rxf_val = TN40_GMAC_RX_FILTER_AM | TN40_GMAC_RX_FILTER_AB | + TN40_GMAC_RX_FILTER_OSEN | TN40_GMAC_RX_FILTER_TXFC; + struct tn40_priv *priv = netdev_priv(ndev); + int i; + + /* IMF - imperfect (hash) rx multicast filter */ + /* PMF - perfect rx multicast filter */ + + /* FIXME: RXE(OFF) */ + if (ndev->flags & IFF_PROMISC) { + rxf_val |= TN40_GMAC_RX_FILTER_PRM; + } else if (ndev->flags & IFF_ALLMULTI) { + /* set IMF to accept all multicast frames */ + for (i = 0; i < TN40_MAC_MCST_HASH_NUM; i++) + tn40_write_reg(priv, + TN40_REG_RX_MCST_HASH0 + i * 4, ~0); + } else if (netdev_mc_count(ndev)) { + struct netdev_hw_addr *mclist; + u32 reg, val; + u8 hash; + + /* Set IMF to deny all multicast frames */ + for (i = 0; i < TN40_MAC_MCST_HASH_NUM; i++) + tn40_write_reg(priv, + TN40_REG_RX_MCST_HASH0 + i * 4, 0); + + /* Set PMF to deny all multicast frames */ + for (i = 0; i < TN40_MAC_MCST_NUM; i++) { + tn40_write_reg(priv, + TN40_REG_RX_MAC_MCST0 + i * 8, 0); + tn40_write_reg(priv, + TN40_REG_RX_MAC_MCST1 + i * 8, 0); + } + /* Use PMF to accept first MAC_MCST_NUM (15) addresses */ + + /* TBD: Sort the addresses and write them in ascending + * order into RX_MAC_MCST regs. we skip this phase now + * and accept ALL multicast frames through IMF. Accept + * the rest of addresses throw IMF. + */ + netdev_for_each_mc_addr(mclist, ndev) { + hash = 0; + for (i = 0; i < ETH_ALEN; i++) + hash ^= mclist->addr[i]; + + reg = TN40_REG_RX_MCST_HASH0 + ((hash >> 5) << 2); + val = tn40_read_reg(priv, reg); + val |= (1 << (hash % 32)); + tn40_write_reg(priv, reg, val); + } + } else { + rxf_val |= TN40_GMAC_RX_FILTER_AB; + } + tn40_write_reg(priv, TN40_REG_GMAC_RXF_A, rxf_val); + /* Enable RX */ + /* FIXME: RXE(ON) */ +} + +static int tn40_set_mac(struct net_device *ndev, void *p) +{ + struct tn40_priv *priv = netdev_priv(ndev); + struct sockaddr *addr = p; + + eth_hw_addr_set(ndev, addr->sa_data); + tn40_restore_mac(ndev, priv); + return 0; +} + +static void tn40_mac_init(struct tn40_priv *priv) +{ + u8 addr[ETH_ALEN]; + u64 val; + + val = (u64)tn40_read_reg(priv, TN40_REG_UNC_MAC0_A); + val |= (u64)tn40_read_reg(priv, TN40_REG_UNC_MAC1_A) << 16; + val |= (u64)tn40_read_reg(priv, TN40_REG_UNC_MAC2_A) << 32; + + u64_to_ether_addr(val, addr); + eth_hw_addr_set(priv->ndev, addr); +} + +static void tn40_get_stats(struct net_device *ndev, + struct rtnl_link_stats64 *stats) +{ + struct tn40_priv *priv = netdev_priv(ndev); + unsigned int start; + + do { + start = u64_stats_fetch_begin(&priv->syncp); + stats->tx_packets = priv->stats.tx_packets; + stats->tx_bytes = priv->stats.tx_bytes; + stats->tx_dropped = priv->stats.tx_dropped; + + stats->rx_packets = priv->stats.rx_packets; + stats->rx_bytes = priv->stats.rx_bytes; + stats->rx_dropped = priv->stats.rx_dropped; + stats->rx_errors = priv->stats.rx_errors; + } while (u64_stats_fetch_retry(&priv->syncp, start)); +} + +static const struct net_device_ops tn40_netdev_ops = { + .ndo_open = tn40_open, + .ndo_stop = tn40_close, + .ndo_start_xmit = tn40_start_xmit, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_rx_mode = tn40_setmulti, + .ndo_get_stats64 = tn40_get_stats, + .ndo_set_mac_address = tn40_set_mac, + .ndo_vlan_rx_add_vid = tn40_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = tn40_vlan_rx_kill_vid, +}; + +static int tn40_ethtool_get_link_ksettings(struct net_device *ndev, + struct ethtool_link_ksettings *cmd) +{ + struct tn40_priv *priv = netdev_priv(ndev); + + return phylink_ethtool_ksettings_get(priv->phylink, cmd); +} + +static const struct ethtool_ops tn40_ethtool_ops = { + .get_link = ethtool_op_get_link, + .get_link_ksettings = tn40_ethtool_get_link_ksettings, +}; + +static void tn40_get_queue_stats_rx(struct net_device *ndev, int idx, + struct netdev_queue_stats_rx *stats) +{ + struct tn40_priv *priv = netdev_priv(ndev); + unsigned int start; + + do { + start = u64_stats_fetch_begin(&priv->syncp); + + stats->packets = priv->stats.rx_packets; + stats->bytes = priv->stats.rx_bytes; + stats->alloc_fail = priv->alloc_fail; + } while (u64_stats_fetch_retry(&priv->syncp, start)); +} + +static void tn40_get_queue_stats_tx(struct net_device *ndev, int idx, + struct netdev_queue_stats_tx *stats) +{ + struct tn40_priv *priv = netdev_priv(ndev); + unsigned int start; + + do { + start = u64_stats_fetch_begin(&priv->syncp); + + stats->packets = priv->stats.tx_packets; + stats->bytes = priv->stats.tx_bytes; + } while (u64_stats_fetch_retry(&priv->syncp, start)); +} + +static void tn40_get_base_stats(struct net_device *ndev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx) +{ + rx->packets = 0; + rx->bytes = 0; + rx->alloc_fail = 0; + + tx->packets = 0; + tx->bytes = 0; +} + +static const struct netdev_stat_ops tn40_stat_ops = { + .get_queue_stats_rx = tn40_get_queue_stats_rx, + .get_queue_stats_tx = tn40_get_queue_stats_tx, + .get_base_stats = tn40_get_base_stats, +}; + +static int tn40_priv_init(struct tn40_priv *priv) +{ + int ret; + + tn40_set_link_speed(priv, 0); + + /* Set GPIO[9:0] to output 0 */ + tn40_write_reg(priv, 0x51E0, 0x30010006); /* GPIO_OE_ WR CMD */ + tn40_write_reg(priv, 0x51F0, 0x0); /* GPIO_OE_ DATA */ + tn40_write_reg(priv, TN40_REG_MDIO_CMD_STAT, 0x3ec8); + + /* we use tx descriptors to load a firmware. */ + ret = tn40_create_tx_ring(priv); + if (ret) + return ret; + ret = tn40_fw_load(priv); + tn40_destroy_tx_ring(priv); + return ret; +} + +static struct net_device *tn40_netdev_alloc(struct pci_dev *pdev) +{ + struct net_device *ndev; + + ndev = devm_alloc_etherdev(&pdev->dev, sizeof(struct tn40_priv)); + if (!ndev) + return NULL; + ndev->netdev_ops = &tn40_netdev_ops; + ndev->ethtool_ops = &tn40_ethtool_ops; + ndev->stat_ops = &tn40_stat_ops; + ndev->tx_queue_len = TN40_NDEV_TXQ_LEN; + ndev->mem_start = pci_resource_start(pdev, 0); + ndev->mem_end = pci_resource_end(pdev, 0); + ndev->min_mtu = ETH_ZLEN; + ndev->max_mtu = TN40_MAX_MTU; + + ndev->features = NETIF_F_IP_CSUM | + NETIF_F_SG | + NETIF_F_FRAGLIST | + NETIF_F_TSO | NETIF_F_GRO | + NETIF_F_RXCSUM | + NETIF_F_RXHASH | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_FILTER; + ndev->vlan_features = NETIF_F_IP_CSUM | + NETIF_F_SG | + NETIF_F_TSO | NETIF_F_GRO | NETIF_F_RXHASH; + + if (dma_get_mask(&pdev->dev) == DMA_BIT_MASK(64)) { + ndev->features |= NETIF_F_HIGHDMA; + ndev->vlan_features |= NETIF_F_HIGHDMA; + } + ndev->hw_features |= ndev->features; + + SET_NETDEV_DEV(ndev, &pdev->dev); + netif_stop_queue(ndev); + return ndev; +} + +static int tn40_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct net_device *ndev; + struct tn40_priv *priv; + unsigned int nvec = 1; + void __iomem *regs; + int ret; + + ret = pci_enable_device(pdev); + if (ret) + return ret; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(&pdev->dev, "failed to set DMA mask.\n"); + goto err_disable_device; + } + + ret = pci_request_regions(pdev, TN40_DRV_NAME); + if (ret) { + dev_err(&pdev->dev, "failed to request PCI regions.\n"); + goto err_disable_device; + } + + pci_set_master(pdev); + + regs = pci_iomap(pdev, 0, TN40_REGS_SIZE); + if (!regs) { + ret = -EIO; + dev_err(&pdev->dev, "failed to map PCI bar.\n"); + goto err_free_regions; + } + + ndev = tn40_netdev_alloc(pdev); + if (!ndev) { + ret = -ENOMEM; + dev_err(&pdev->dev, "failed to allocate netdev.\n"); + goto err_iounmap; + } + + priv = netdev_priv(ndev); + pci_set_drvdata(pdev, priv); + netif_napi_add(ndev, &priv->napi, tn40_poll); + + priv->regs = regs; + priv->pdev = pdev; + priv->ndev = ndev; + /* Initialize fifo sizes. */ + priv->txd_size = 3; + priv->txf_size = 3; + priv->rxd_size = 3; + priv->rxf_size = 3; + /* Initialize the initial coalescing registers. */ + priv->rdintcm = TN40_INT_REG_VAL(0x20, 1, 4, 12); + priv->tdintcm = TN40_INT_REG_VAL(0x20, 1, 0, 12); + + ret = tn40_hw_reset(priv); + if (ret) { + dev_err(&pdev->dev, "failed to reset HW.\n"); + goto err_unset_drvdata; + } + + ret = pci_alloc_irq_vectors(pdev, 1, nvec, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(&pdev->dev, "failed to allocate irq.\n"); + goto err_unset_drvdata; + } + + ret = tn40_mdiobus_init(priv); + if (ret) { + dev_err(&pdev->dev, "failed to initialize mdio bus.\n"); + goto err_free_irq; + } + + priv->stats_flag = + ((tn40_read_reg(priv, TN40_FPGA_VER) & 0xFFF) != 308); + u64_stats_init(&priv->syncp); + + priv->isr_mask = TN40_IR_RX_FREE_0 | TN40_IR_LNKCHG0 | TN40_IR_PSE | + TN40_IR_TMR0 | TN40_IR_RX_DESC_0 | TN40_IR_TX_FREE_0 | + TN40_IR_TMR1; + + tn40_mac_init(priv); + ret = tn40_phy_register(priv); + if (ret) { + dev_err(&pdev->dev, "failed to set up PHY.\n"); + goto err_free_irq; + } + + ret = tn40_priv_init(priv); + if (ret) { + dev_err(&pdev->dev, "failed to initialize tn40_priv.\n"); + goto err_unregister_phydev; + } + + ret = register_netdev(ndev); + if (ret) { + dev_err(&pdev->dev, "failed to register netdev.\n"); + goto err_unregister_phydev; + } + return 0; +err_unregister_phydev: + tn40_phy_unregister(priv); +err_free_irq: + pci_free_irq_vectors(pdev); +err_unset_drvdata: + pci_set_drvdata(pdev, NULL); +err_iounmap: + iounmap(regs); +err_free_regions: + pci_release_regions(pdev); +err_disable_device: + pci_disable_device(pdev); + return ret; +} + +static void tn40_remove(struct pci_dev *pdev) +{ + struct tn40_priv *priv = pci_get_drvdata(pdev); + struct net_device *ndev = priv->ndev; + + unregister_netdev(ndev); + + tn40_phy_unregister(priv); + pci_free_irq_vectors(priv->pdev); + pci_set_drvdata(pdev, NULL); + iounmap(priv->regs); + pci_release_regions(pdev); + pci_disable_device(pdev); +} + +static const struct pci_device_id tn40_id_table[] = { + { PCI_DEVICE_SUB(PCI_VENDOR_ID_TEHUTI, 0x4022, + PCI_VENDOR_ID_TEHUTI, 0x3015) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_TEHUTI, 0x4022, + PCI_VENDOR_ID_DLINK, 0x4d00) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_TEHUTI, 0x4022, + PCI_VENDOR_ID_ASUSTEK, 0x8709) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_TEHUTI, 0x4022, + PCI_VENDOR_ID_EDIMAX, 0x8103) }, + { } +}; + +static struct pci_driver tn40_driver = { + .name = TN40_DRV_NAME, + .id_table = tn40_id_table, + .probe = tn40_probe, + .remove = tn40_remove, +}; + +module_pci_driver(tn40_driver); + +MODULE_DEVICE_TABLE(pci, tn40_id_table); +MODULE_LICENSE("GPL"); +MODULE_FIRMWARE(TN40_FIRMWARE_NAME); +MODULE_DESCRIPTION("Tehuti Network TN40xx Driver"); |