// SPDX-License-Identifier: GPL-2.0+ // Copyright (c) 2016-2017 Hisilicon Limited. #include <linux/dma-mapping.h>#include <linux/etherdevice.h>#include <linux/interrupt.h>#ifdef CONFIG_RFS_ACCEL#include <linux/cpu_rmap.h>#endif #include <linux/if_vlan.h>#include <linux/irq.h>#include <linux/ip.h>#include <linux/ipv6.h>#include <linux/iommu.h>#include <linux/module.h>#include <linux/pci.h>#include <linux/skbuff.h>#include <linux/sctp.h>#include <net/gre.h>#include <net/gro.h>#include <net/ip6_checksum.h>#include <net/page_pool/helpers.h>#include <net/pkt_cls.h>#include <net/pkt_sched.h>#include <net/tcp.h>#include <net/vxlan.h>#include <net/geneve.h>#include "hnae3.h" #include "hns3_enet.h" /* All hns3 tracepoints are defined by the include below, which * must be included exactly once across the whole kernel with * CREATE_TRACE_POINTS defined #define CREATE_TRACE_POINTS#include "hns3_trace.h" #define hns3_set_field(origin, shift, val) ((origin) |= (val) << (shift))#define hns3_tx_bd_count(S) DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)#define hns3_rl_err(fmt, ...) \do { \if (net_ratelimit()) \## __VA_ARGS__); \while (0)static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);static const char hns3_driver_name[] = "hns3" ;static const char hns3_driver_string[] ="Hisilicon Ethernet Network Driver for Hip08 Family" ;static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation." ;static struct hnae3_client client;static int debug = -1;int , 0);" Network interface message level setting" );static unsigned int tx_sgl = 1;"Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping" );static bool page_pool_enabled = true ;bool , 0400);#define HNS3_SGL_SIZE(nfrag) (sizeof (struct scatterlist) * (nfrag) + \sizeof (struct sg_table))#define HNS3_MAX_SGL_SIZE ALIGN(HNS3_SGL_SIZE(HNS3_MAX_TSO_BD_NUM), \#define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \#define HNS3_INNER_VLAN_TAG 1#define HNS3_OUTER_VLAN_TAG 2#define HNS3_MIN_TX_LEN 33U#define HNS3_MIN_TUN_PKT_LEN 65U/* hns3_pci_tbl - PCI Device ID Table * * Last entry must be all 0s * * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, * Class, Class Mask, private data (not used) } static const struct pci_device_id hns3_pci_tbl[] = {/* required last entry */ #define HNS3_RX_PTYPE_ENTRY(ptype, l, s, t, h) \## s, \## t, \#define HNS3_RX_PTYPE_UNUSED_ENTRY(ptype) \static const struct hns3_rx_ptype hns3_rx_ptype_tbl[] = {#define HNS3_INVALID_PTYPE \static irqreturn_t hns3_irq_handle(int irq, void *vector)struct hns3_enet_tqp_vector *tqp_vector = vector;return IRQ_HANDLED;static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)struct hns3_enet_tqp_vector *tqp_vectors;unsigned int i;for (i = 0; i < priv->vector_num; i++) {if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)continue ;/* clear the affinity mask */ /* release the irq resource */ static int hns3_nic_init_irq(struct hns3_nic_priv *priv)struct hns3_enet_tqp_vector *tqp_vectors;int txrx_int_idx = 0;int rx_int_idx = 0;int tx_int_idx = 0;unsigned int i;int ret;for (i = 0; i < priv->vector_num; i++) {if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)continue ;if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {"%s-%s-%s-%d" , hns3_driver_name,"TxRx" , txrx_int_idx++);else if (tqp_vectors->rx_group.ring) {"%s-%s-%s-%d" , hns3_driver_name,"Rx" , rx_int_idx++);else if (tqp_vectors->tx_group.ring) {"%s-%s-%s-%d" , hns3_driver_name,"Tx" , tx_int_idx++);else {/* Skip this unused q_vector */ continue ;'\0' ;if (ret) {"request irq(%d) fail\n" ,return ret;return 0;static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,static void hns3_irq_enable(struct hns3_enet_tqp_vector *tqp_vector)static void hns3_irq_disable(struct hns3_enet_tqp_vector *tqp_vector)void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,/* this defines the configuration for RL (Interrupt Rate Limiter). * Rl defines rate of interrupts i.e. number of interrupts-per-second * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&/* According to the hardware, the range of rl_reg is * 0-59 and the unit is 4. void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,if (tqp_vector->rx_group.coal.unit_1us)else void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,if (tqp_vector->tx_group.coal.unit_1us)else void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,struct hns3_nic_priv *priv)struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle);struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal;struct hns3_enet_coalesce *prx_coal = &priv->rx_coal;/* device version above V3(include V3), GL can configure 1us * unit, so uses 1us unit. if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {if (ae_dev->dev_specs.int_ql_max) {static void struct hns3_enet_tqp_vector *tqp_vector,struct hns3_nic_priv *priv)struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;struct hnae3_handle *h = priv->ae_handle;if (tx_coal->ql_enable)if (rx_coal->ql_enable)static int hns3_nic_set_real_num_queue(struct net_device *netdev)struct hnae3_handle *h = hns3_get_handle(netdev);struct hnae3_knic_private_info *kinfo = &h->kinfo;struct hnae3_tc_info *tc_info = &kinfo->tc_info;unsigned int queue_size = kinfo->num_tqps;int i, ret;if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) {else {if (ret) {"netdev_set_num_tc fail, ret=%d!\n" , ret);return ret;for (i = 0; i < tc_info->num_tc; i++)if (ret) {"netif_set_real_num_tx_queues fail, ret=%d!\n" , ret);return ret;if (ret) {"netif_set_real_num_rx_queues fail, ret=%d!\n" , ret);return ret;return 0;struct hnae3_handle *h)return min_t(u16, rss_size, max_rss_size);static void hns3_tqp_enable(struct hnae3_queue *tqp)static void hns3_tqp_disable(struct hnae3_queue *tqp)static void hns3_free_rx_cpu_rmap(struct net_device *netdev)#ifdef CONFIG_RFS_ACCEL#endif static int hns3_set_rx_cpu_rmap(struct net_device *netdev)#ifdef CONFIG_RFS_ACCELstruct hns3_nic_priv *priv = netdev_priv(netdev);struct hns3_enet_tqp_vector *tqp_vector;int i, ret;if (!netdev->rx_cpu_rmap) {if (!netdev->rx_cpu_rmap)return -ENOMEM;for (i = 0; i < priv->vector_num; i++) {if (ret) {return ret;#endif return 0;static void hns3_enable_irqs_and_tqps(struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);struct hnae3_handle *h = priv->ae_handle;for (i = 0; i < priv->vector_num; i++)for (i = 0; i < priv->vector_num; i++)for (i = 0; i < h->kinfo.num_tqps; i++)static void hns3_disable_irqs_and_tqps(struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);struct hnae3_handle *h = priv->ae_handle;for (i = 0; i < h->kinfo.num_tqps; i++)for (i = 0; i < priv->vector_num; i++)for (i = 0; i < priv->vector_num; i++)static int hns3_nic_net_up(struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);struct hnae3_handle *h = priv->ae_handle;int ret;if (ret)return ret;/* start the ae_dev */ if (ret) {return ret;static void hns3_config_xps(struct hns3_nic_priv *priv)int i;for (i = 0; i < priv->vector_num; i++) {struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;while (ring) {int ret;if (ret)"set xps queue failed: %d" , ret);static int hns3_nic_net_open(struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);struct hnae3_handle *h = hns3_get_handle(netdev);struct hnae3_knic_private_info *kinfo;int i, ret;if (hns3_nic_resetting(netdev))return -EBUSY;if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {"net open repeatedly!\n" );return 0;if (ret)return ret;if (ret) {"net up fail, ret=%d!\n" , ret);return ret;for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)if (h->ae_algo->ops->set_timer_task)true );"net open\n" );return 0;static void hns3_reset_tx_queue(struct hnae3_handle *h)struct net_device *ndev = h->kinfo.netdev;struct hns3_nic_priv *priv = netdev_priv(ndev);struct netdev_queue *dev_queue;for (i = 0; i < h->kinfo.num_tqps; i++) {static void hns3_nic_net_down(struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);const struct hnae3_ae_ops *ops;/* stop ae_dev */ if (ops->stop)/* delay ring buffer clearing to hns3_reset_notify_uninit_enet * during reset process, because driver may not be able * to disable the ring through firmware when downing the netdev. if (!hns3_nic_resetting(netdev))false );static int hns3_nic_net_stop(struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);struct hnae3_handle *h = hns3_get_handle(netdev);if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))return 0;"net stop\n" );if (h->ae_algo->ops->set_timer_task)false );return 0;static int hns3_nic_uc_sync(struct net_device *netdev,const unsigned char *addr)struct hnae3_handle *h = hns3_get_handle(netdev);if (h->ae_algo->ops->add_uc_addr)return h->ae_algo->ops->add_uc_addr(h, addr);return 0;static int hns3_nic_uc_unsync(struct net_device *netdev,const unsigned char *addr)struct hnae3_handle *h = hns3_get_handle(netdev);/* need ignore the request of removing device address, because * we store the device address and other addresses of uc list * in the function's mac filter list. if (ether_addr_equal(addr, netdev->dev_addr))return 0;if (h->ae_algo->ops->rm_uc_addr)return h->ae_algo->ops->rm_uc_addr(h, addr);return 0;static int hns3_nic_mc_sync(struct net_device *netdev,const unsigned char *addr)struct hnae3_handle *h = hns3_get_handle(netdev);if (h->ae_algo->ops->add_mc_addr)return h->ae_algo->ops->add_mc_addr(h, addr);return 0;static int hns3_nic_mc_unsync(struct net_device *netdev,const unsigned char *addr)struct hnae3_handle *h = hns3_get_handle(netdev);if (h->ae_algo->ops->rm_mc_addr)return h->ae_algo->ops->rm_mc_addr(h, addr);return 0;static u8 hns3_get_netdev_flags(struct net_device *netdev)if (netdev->flags & IFF_PROMISC)else if (netdev->flags & IFF_ALLMULTI)return flags;static void hns3_nic_set_rx_mode(struct net_device *netdev)struct hnae3_handle *h = hns3_get_handle(netdev);/* User mode Promisc mode enable and vlan filtering is disabled to * let all packets in. void hns3_request_update_promisc_mode(struct hnae3_handle *handle)const struct hnae3_ae_ops *ops = hns3_get_ops(handle);if (ops->request_update_promisc_mode)static u32 hns3_tx_spare_space(struct hns3_enet_ring *ring)struct hns3_tx_spare *tx_spare = ring->tx_spare;/* This smp_load_acquire() pairs with smp_store_release() in * hns3_tx_spare_update() called in tx desc cleaning process. if (ntc > ntu)return ntc - ntu - 1;/* The free tx buffer is divided into two part, so pick the * larger one. return max(ntc, tx_spare->len - ntu) - 1;static void hns3_tx_spare_update(struct hns3_enet_ring *ring)struct hns3_tx_spare *tx_spare = ring->tx_spare;if (!tx_spare ||return ;/* This smp_store_release() pairs with smp_load_acquire() in * hns3_tx_spare_space() called in xmit process. static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring,struct sk_buff *skb,if (len > ring->tx_copybreak)return false ;if (ALIGN(len, dma_get_cache_alignment()) > space) {return false ;return true ;static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring,struct sk_buff *skb,if (skb->len <= ring->tx_copybreak || !tx_sgl ||return false ;if (space < HNS3_MAX_SGL_SIZE) {return false ;return true ;static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring)struct net_device *netdev = ring_to_netdev(ring);struct hns3_nic_priv *priv = netdev_priv(netdev);struct hns3_tx_spare *tx_spare;struct page *page;int order;if (!alloc_size)return ;if (order > MAX_PAGE_ORDER) {if (net_ratelimit())"failed to allocate tx spare buffer, exceed to max order\n" );return ;sizeof (*tx_spare),if (!tx_spare) {/* The driver still work without the tx spare buffer */ "failed to allocate hns3_tx_spare\n" );goto devm_kzalloc_error;if (!page) {"failed to allocate tx spare pages\n" );goto alloc_pages_error;if (dma_mapping_error(ring_to_dev(ring), dma)) {"failed to map pages for tx spare\n" );goto dma_mapping_error;return ;/* Use hns3_tx_spare_space() to make sure there is enough buffer * before calling below function to allocate tx buffer. static void *hns3_tx_spare_alloc(struct hns3_enet_ring *ring,unsigned int size, dma_addr_t *dma,struct hns3_tx_spare *tx_spare = ring->tx_spare;/* Tx spare buffer wraps back here because the end of * freed tx buffer is not enough. if (ntu + size > tx_spare->len) {if (tx_spare->next_to_use == tx_spare->len)return tx_spare->buf + ntu;static void hns3_tx_spare_rollback(struct hns3_enet_ring *ring, u32 len)struct hns3_tx_spare *tx_spare = ring->tx_spare;if (len > tx_spare->next_to_use) {else {static void hns3_tx_spare_reclaim_cb(struct hns3_enet_ring *ring,struct hns3_desc_cb *cb)struct hns3_tx_spare *tx_spare = ring->tx_spare;if (tx_spare->next_to_clean >= tx_spare->len) {if (tx_spare->next_to_clean) {/* This tx spare buffer is only really reclaimed after calling * hns3_tx_spare_update(), so it is still safe to use the info in * the tx buffer to do the dma sync or sg unmapping after * tx_spare->next_to_clean is moved forword. if (cb->type & (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) {else {struct sg_table *sgt = tx_spare->buf + ntc;static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs,union l3_hdr_info l3;union l4_hdr_info l4;int ret;if (!skb_is_gso(skb))return 0;if (unlikely(ret < 0))return ret;/* Software should clear the IPv4's checksum field when tso is * needed. if (l3.v4->version == 4)/* tunnel packet */ if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |/* reset l3&l4 pointers from outer to inner headers */ /* Software should clear the IPv4's checksum field when * tso is needed. if (l3.v4->version == 4)/* normal or tunnel packet */ /* remove payload length from inner pseudo checksum when tso */ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {sizeof (*l4.udp) + l4_offset;else {/* find the txbd field values */ /* offload outer UDP header checksum */ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)/* get MSS for TSO */ return 0;static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,union l3_hdr_info l3;unsigned char *l4_hdr;unsigned char *exthdr;/* find outer header point */ if (skb->protocol == htons(ETH_P_IPV6)) {sizeof (*l3.v6);if (l4_hdr != exthdr)else if (skb->protocol == htons(ETH_P_IP)) {else {return -EINVAL;/* tunnel packet */ if (!skb->encapsulation) {return 0;/* find inner header point */ if (l3.v6->version == 6) {sizeof (*l3.v6);if (l4_hdr != exthdr)else if (l3.v4->version == 4) {return 0;/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL * and it is udp packet, which has a dest port as the IANA assigned. * the hardware is expected to do the checksum offload, but the * hardware will not do the checksum offload when udp dest port is * 4789, 4790 or 6081. static bool hns3_tunnel_csum_bug(struct sk_buff *skb)struct hns3_nic_priv *priv = netdev_priv(skb->dev);struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle);union l4_hdr_info l4;/* device version above V3(include V3), the hardware can * do this checksum offload. if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)return false ;if (!(!skb->encapsulation &&return false ;return true ;static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,unsigned char *il2_hdr;union l3_hdr_info l3;union l4_hdr_info l4;/* compute OL2 header size, defined in 2 Bytes */ /* compute OL3 header size, defined in 4 Bytes */ /* compute OL4 header size, defined in 4 Bytes */ /* define outer network header type */ if (skb->protocol == htons(ETH_P_IP)) {if (skb_is_gso(skb))else else if (skb->protocol == htons(ETH_P_IPV6)) {if (ol4_proto == IPPROTO_UDP)else if (ol4_proto == IPPROTO_GRE)static void hns3_set_l3_type(struct sk_buff *skb, union l3_hdr_info l3,if (l3.v4->version == 4) {/* the stack computes the IP header already, the only time we * need the hardware to recompute it is in the case of TSO. if (skb_is_gso(skb))else if (l3.v6->version == 6) {static int hns3_set_l4_csum_length(struct sk_buff *skb, union l4_hdr_info l4,/* compute inner(/normal) L4 header size, defined in 4 Bytes */ switch (l4_proto) {case IPPROTO_TCP:break ;case IPPROTO_UDP:if (hns3_tunnel_csum_bug(skb)) {int ret = skb_put_padto(skb, HNS3_MIN_TUN_PKT_LEN);return ret ? ret : skb_checksum_help(skb);sizeof (struct udphdr) >> 2));break ;case IPPROTO_SCTP:sizeof (struct sctphdr) >> 2));break ;default :/* drop the skb tunnel packet if hardware don't support, * because hardware can't calculate csum when TSO. if (skb_is_gso(skb))return -EDOM;/* the stack computes the IP header already, * driver calculate l4 checksum when not TSO. return skb_checksum_help(skb);return 0;static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,unsigned char *l2_hdr = skb->data;union l4_hdr_info l4;union l3_hdr_info l3;/* handle encapsulation skb */ if (skb->encapsulation) {/* If this is a not UDP/GRE encapsulation skb */ if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {/* drop the skb tunnel packet if hardware don't support, * because hardware can't calculate csum when TSO. if (skb_is_gso(skb))return -EDOM;/* the stack computes the IP header already, * driver calculate l4 checksum when not TSO. return skb_checksum_help(skb);/* switch to inner header */ /* compute inner(/normal) L2 header size, defined in 2 Bytes */ /* compute inner(/normal) L3 header size, defined in 4 Bytes */ return hns3_set_l4_csum_length(skb, l4, l4_proto, type_cs_vlan_tso);static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,struct sk_buff *skb)struct hnae3_handle *handle = tx_ring->tqp->handle;struct hnae3_ae_dev *ae_dev;struct vlan_ethhdr *vhdr;int rc;if (!(skb->protocol == htons(ETH_P_8021Q) ||return 0;/* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert * VLAN enabled, only one VLAN header is allowed in skb, otherwise it * will cause RAS error. if (unlikely(skb_vlan_tagged_multi(skb) &&return -EINVAL;if (skb->protocol == htons(ETH_P_8021Q) &&/* When HW VLAN acceleration is turned off, and the stack * sets the protocol to 802.1q, the driver just need to * set the protocol to the encapsulated ethertype. return 0;if (skb_vlan_tag_present(skb)) {/* Based on hw strategy, use out_vtag in two layer tag case, * and use inner_vtag in one tag case. if (skb->protocol == htons(ETH_P_8021Q) &&else return rc;if (unlikely(rc < 0))return rc;return 0;/* check if the hardware is capable of checksum offloading */ static bool hns3_check_hw_tx_csum(struct sk_buff *skb)struct hns3_nic_priv *priv = netdev_priv(skb->dev);/* Kindly note, due to backward compatibility of the TX descriptor, * HW checksum of the non-IP packets and GSO packets is handled at * different place in the following code if (skb_csum_is_sctp(skb) || skb_is_gso(skb) ||return false ;return true ;struct hns3_desc_param {static void hns3_init_desc_data(struct sk_buff *skb, struct hns3_desc_param *pa)static int hns3_handle_vlan_info(struct hns3_enet_ring *ring,struct sk_buff *skb,struct hns3_desc_param *param)int ret;if (unlikely(ret < 0)) {return ret;else if (ret == HNS3_INNER_VLAN_TAG) {else if (ret == HNS3_OUTER_VLAN_TAG) {return 0;static int hns3_handle_csum_partial(struct hns3_enet_ring *ring,struct sk_buff *skb,struct hns3_desc_cb *desc_cb,struct hns3_desc_param *param)int ret;if (hns3_check_hw_tx_csum(skb)) {/* set checksum start and offset, defined in 2 Bytes */ return 0;if (unlikely(ret < 0)) {return ret;if (unlikely(ret < 0)) {return ret;if (unlikely(ret < 0)) {return ret;return 0;static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,struct sk_buff *skb, struct hns3_desc *desc,struct hns3_desc_cb *desc_cb)struct hns3_desc_param param;int ret;if (unlikely(ret < 0))return ret;if (skb->ip_summed == CHECKSUM_PARTIAL) {if (ret)return ret;/* Set txbd */ return 0;static int hns3_fill_desc(struct hns3_enet_ring *ring, dma_addr_t dma,unsigned int size)#define HNS3_LIKELY_BD_NUM 1struct hns3_desc *desc = &ring->desc[ring->next_to_use];unsigned int frag_buf_num, k;int sizeoflast;if (likely(size <= HNS3_MAX_BD_SIZE)) {return HNS3_LIKELY_BD_NUM;/* When frag size is bigger than hardware limit, split this frag */ for (k = 0; k < frag_buf_num; k++) {/* now, fill the descriptor */ /* move ring pointer to next */ return frag_buf_num;static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv,unsigned int type)struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];struct device *dev = ring_to_dev(ring);unsigned int size;if (type & (DESC_TYPE_FRAGLIST_SKB | DESC_TYPE_SKB)) {struct sk_buff *skb = (struct sk_buff *)priv;if (!size)return 0;else if (type & DESC_TYPE_BOUNCE_HEAD) {/* Head data has been filled in hns3_handle_tx_bounce(), * just return 0 here. return 0;else {if (!size)return 0;if (unlikely(dma_mapping_error(dev, dma))) {return -ENOMEM;return hns3_fill_desc(ring, dma, size);static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,unsigned int bd_num)unsigned int size;int i;while (size > HNS3_MAX_BD_SIZE) {if (bd_num > HNS3_MAX_TSO_BD_NUM)return bd_num;if (size) {if (bd_num > HNS3_MAX_TSO_BD_NUM)return bd_num;for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {if (!size)continue ;while (size > HNS3_MAX_BD_SIZE) {if (bd_num > HNS3_MAX_TSO_BD_NUM)return bd_num;if (bd_num > HNS3_MAX_TSO_BD_NUM)return bd_num;return bd_num;static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size,unsigned int bd_num,unsigned int recursion_level)#define HNS3_MAX_RECURSION_LEVEL 24struct sk_buff *frag_skb;/* If the total len is within the max bd limit */ if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level &&return skb_shinfo(skb)->nr_frags + 1U;if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL))return UINT_MAX;if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)return bd_num;if (bd_num > HNS3_MAX_TSO_BD_NUM)return bd_num;return bd_num;static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)if (!skb->encapsulation)return skb_tcp_all_headers(skb);return skb_inner_tcp_all_headers(skb);/* HW need every continuous max_non_tso_bd_num buffer data to be larger * than MSS, we simplify it by ensuring skb_headlen + the first continuous * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss, * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger * than MSS except the last max_non_tso_bd_num - 1 frags. static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,unsigned int bd_num, u8 max_non_tso_bd_num)unsigned int tot_len = 0;unsigned int i;for (i = 0; i < max_non_tso_bd_num - 1U; i++)/* ensure the first max_non_tso_bd_num frags is greater than * mss + header if (tot_len + bd_size[max_non_tso_bd_num - 1U] <return true ;/* ensure every continuous max_non_tso_bd_num - 1 buffer is greater * than mss except the last one. for (i = 0; i < bd_num - max_non_tso_bd_num; i++) {if (tot_len < skb_shinfo(skb)->gso_size)return true ;return false ;void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)for (i = 0; i < MAX_SKB_FRAGS; i++)static int hns3_skb_linearize(struct hns3_enet_ring *ring,struct sk_buff *skb,unsigned int bd_num)/* 'bd_num == UINT_MAX' means the skb' fraglist has a * recursion level of over HNS3_MAX_RECURSION_LEVEL. if (bd_num == UINT_MAX) {return -ENOMEM;/* The skb->len has exceeded the hw limitation, linearization * will not help. if (skb->len > HNS3_MAX_TSO_SIZE ||return -ENOMEM;if (__skb_linearize(skb)) {return -ENOMEM;return 0;static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,struct net_device *netdev,struct sk_buff *skb)struct hns3_nic_priv *priv = netdev_priv(netdev);unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];unsigned int bd_num;if (unlikely(bd_num > max_non_tso_bd_num)) {if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&goto out;if (hns3_skb_linearize(ring, skb, bd_num))return -ENOMEM;if (likely(ring_space(ring) >= bd_num))return bd_num;/* Memory barrier before checking ring_space */ /* Start queue in case hns3_clean_tx_ring has just made room * available and has not seen the queue stopped state performed * by netif_stop_subqueue above. if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&return bd_num;return -EBUSY;static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)struct device *dev = ring_to_dev(ring);unsigned int i;for (i = 0; i < ring->desc_num; i++) {struct hns3_desc *desc = &ring->desc[ring->next_to_use];struct hns3_desc_cb *desc_cb;sizeof (*desc));/* check if this is where we started */ if (ring->next_to_use == next_to_use_orig)break ;/* rollback one */ if (!desc_cb->dma)continue ;/* unmap the descriptor dma address */ if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))else if (desc_cb->type &else if (desc_cb->length)static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,struct sk_buff *skb, unsigned int type)struct sk_buff *frag_skb;int i, ret, bd_num = 0;if (unlikely(ret < 0))return ret;for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {if (unlikely(ret < 0))return ret;if (unlikely(ret < 0))return ret;return bd_num;static void hns3_tx_push_bd(struct hns3_enet_ring *ring, int num)#define HNS3_BYTES_PER_64BIT 8struct hns3_desc desc[HNS3_MAX_PUSH_BD_NUM] = {};int offset = 0;/* make sure everything is visible to device before * excuting tx push or updating doorbell do {int idx = (ring->next_to_use - num + ring->desc_num) %sizeof (struct hns3_desc));while (--num);sizeof (struct hns3_desc) * HNS3_MAX_PUSH_BD_NUM) /static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring)#define HNS3_MEM_DOORBELL_OFFSET 64/* make sure everything is visible to device before * excuting tx push or updating doorbell static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,bool doorbell)struct net_device *netdev = ring_to_netdev(ring);struct hns3_nic_priv *priv = netdev_priv(netdev);/* when tx push is enabled, the packet whose number of BD below * HNS3_MAX_PUSH_BD_NUM can be pushed directly. if (test_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state) && num &&/* This smp_store_release() pairs with smp_load_acquire() in * hns3_nic_reclaim_desc(). Ensure that the BD valid bit * is updated. return ;if (!doorbell) {return ;/* This smp_store_release() pairs with smp_load_acquire() in * hns3_nic_reclaim_desc(). Ensure that the BD valid bit is updated. if (ring->tqp->mem_base)else static void hns3_tsyn(struct net_device *netdev, struct sk_buff *skb,struct hns3_desc *desc)struct hnae3_handle *h = hns3_get_handle(netdev);if (!(h->ae_algo->ops->set_tx_hwts_info &&return ;static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring,struct sk_buff *skb)struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];unsigned int type = DESC_TYPE_BOUNCE_HEAD;unsigned int size = skb_headlen(skb);int bd_num = 0;void *buf;int ret;if (skb->len <= ring->tx_copybreak) {/* hns3_can_use_tx_bounce() is called to ensure the below * function can always return the tx buffer. if (unlikely(ret < 0)) {return ret;if (type == DESC_TYPE_BOUNCE_HEAD) {if (unlikely(ret < 0))return ret;return bd_num;static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring,struct sk_buff *skb)struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];struct sg_table *sgt;int bd_num = 0;int nents;if (skb_has_frag_list(skb))/* hns3_can_use_tx_sgl() is called to ensure the below * function can always return the tx buffer. /* scatterlist follows by the sg table */ struct scatterlist *)(sgt + 1);if (unlikely(nents < 0)) {return -ENOMEM;if (unlikely(!sgt->nents)) {return -ENOMEM;for (i = 0; i < sgt->nents; i++)return bd_num;static int hns3_handle_desc_filling(struct hns3_enet_ring *ring,struct sk_buff *skb)if (!ring->tx_spare)goto out;if (hns3_can_use_tx_sgl(ring, skb, space))return hns3_handle_tx_sgl(ring, skb);if (hns3_can_use_tx_bounce(ring, skb, space))return hns3_handle_tx_bounce(ring, skb);return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);static int hns3_handle_skb_desc(struct hns3_enet_ring *ring,struct sk_buff *skb,struct hns3_desc_cb *desc_cb,int next_to_use_head)int ret;if (unlikely(ret < 0))goto fill_err;/* 'ret < 0' means filling error, 'ret == 0' means skb->len is * zero, which is unlikely, and 'ret > 0' means how many tx desc * need to be notified to the hw. if (likely(ret > 0))return ret;return ret;struct sk_buff *skb, struct net_device *netdev)struct hns3_nic_priv *priv = netdev_priv(netdev);struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];struct netdev_queue *dev_queue;int pre_ntu, ret;bool doorbell;/* Hardware can only handle short frames above 32 bytes */ if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {return NETDEV_TX_OK;/* Prefetch the data used later */ if (unlikely(ret <= 0)) {if (ret == -EBUSY) {true );return NETDEV_TX_BUSY;"xmit error: %d!\n" , ret);goto out_err_tx_ok;if (unlikely(ret <= 0))
Messung V0.5 C=97 H=93 G=94
¤ Dauer der Verarbeitung: 0.23 Sekunden
¤ *© Formatika GbR, Deutschland
