// SPDX-License-Identifier: GPL-2.0-or-later /* A network driver using virtio. * * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation //#define DEBUG #include <linux/netdevice.h>#include <linux/etherdevice.h>#include <linux/ethtool.h>#include <linux/module.h>#include <linux/virtio.h>#include <linux/virtio_net.h>#include <linux/bpf.h>#include <linux/bpf_trace.h>#include <linux/scatterlist.h>#include <linux/if_vlan.h>#include <linux/slab.h>#include <linux/cpu.h>#include <linux/average.h>#include <linux/filter.h>#include <linux/kernel.h>#include <linux/dim.h>#include <net/route.h>#include <net/xdp.h>#include <net/net_failover.h>#include <net/netdev_rx_queue.h>#include <net/netdev_queues.h>#include <net/xdp_sock_drv.h>static int napi_weight = NAPI_POLL_WEIGHT;int , 0444);static bool csum = true , gso = true , napi_tx = true ;bool , 0444);bool , 0444);bool , 0644);#define VIRTIO_OFFLOAD_MAP_MIN 46#define VIRTIO_OFFLOAD_MAP_MAX 47#define VIRTIO_FEATURES_MAP_MIN 65#define VIRTIO_O2F_DELTA (VIRTIO_FEATURES_MAP_MIN - \static bool virtio_is_mapped_offload(unsigned int obit)return obit >= VIRTIO_OFFLOAD_MAP_MIN &&static unsigned int virtio_offload_to_feature(unsigned int obit)return virtio_is_mapped_offload(obit) ? obit + VIRTIO_O2F_DELTA : obit;/* FIXME: MTU in config. */ #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)#define GOOD_COPY_LEN 128#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)/* Separating two types of XDP xmit */ #define VIRTIO_XDP_TX BIT(0)#define VIRTIO_XDP_REDIR BIT(1)/* RX packet size EWMA. The average packet size is used to determine the packet * buffer size when refilling RX rings. As the entire RX ring may be refilled * at once, the weight is chosen so that the EWMA will be insensitive to short- * term, transient changes in packet size. #define VIRTNET_DRIVER_VERSION "1.0.0" static const unsigned long guest_offloads[] = {#define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \struct virtnet_stat_desc {char desc[ETH_GSTRING_LEN];struct virtnet_sq_free_stats {struct virtnet_sq_stats {struct u64_stats_sync syncp;struct virtnet_rq_stats {struct u64_stats_sync syncp;#define VIRTNET_SQ_STAT(name, m) {name, offsetof(struct virtnet_sq_stats, m), -1}#define VIRTNET_RQ_STAT(name, m) {name, offsetof(struct virtnet_rq_stats, m), -1}#define VIRTNET_SQ_STAT_QSTAT(name, m) \struct virtnet_sq_stats, m), \struct netdev_queue_stats_tx, m), \#define VIRTNET_RQ_STAT_QSTAT(name, m) \struct virtnet_rq_stats, m), \struct netdev_queue_stats_rx, m), \static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {"xdp_tx" , xdp_tx),"xdp_tx_drops" , xdp_tx_drops),"kicks" , kicks),"tx_timeouts" , tx_timeouts),static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {"drops" , drops),"xdp_packets" , xdp_packets),"xdp_tx" , xdp_tx),"xdp_redirects" , xdp_redirects),"xdp_drops" , xdp_drops),"kicks" , kicks),static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = {"packets" , packets),"bytes" , bytes),"stop" , stop),"wake" , wake),static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = {"packets" , packets),"bytes" , bytes),#define VIRTNET_STATS_DESC_CQ(name) \#name , offsetof(struct virtio_net_stats_cvq, name), -1}#define VIRTNET_STATS_DESC_RX(class , name) \#name , offsetof(struct virtio_net_stats_rx_ ## class , rx_ ## name), -1}#define VIRTNET_STATS_DESC_TX(class , name) \#name , offsetof(struct virtio_net_stats_tx_ ## class , tx_ ## name), -1}static const struct virtnet_stat_desc virtnet_stats_cvq_desc[] = {static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc[] = {static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc[] = {static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc[] = {static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc[] = {static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc[] = {static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc[] = {#define VIRTNET_STATS_DESC_RX_QSTAT(class , name, qstat_field) \#name , \struct virtio_net_stats_rx_ ## class , rx_ ## name), \struct netdev_queue_stats_rx, qstat_field), \#define VIRTNET_STATS_DESC_TX_QSTAT(class , name, qstat_field) \#name , \struct virtio_net_stats_tx_ ## class , tx_ ## name), \struct netdev_queue_stats_tx, qstat_field), \static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_tx_csum_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_rx_gso_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc_qstat[] = {static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc_qstat[] = {#define VIRTNET_Q_TYPE_RX 0#define VIRTNET_Q_TYPE_TX 1#define VIRTNET_Q_TYPE_CQ 2struct virtnet_interrupt_coalesce {/* The dma information of pages allocated at a time. */ struct virtnet_rq_dma {/* Internal representation of a send virtqueue */ struct send_queue {/* Virtqueue associated with this send _queue */ struct virtqueue *vq;/* TX: fragments + linear part + virtio header */ struct scatterlist sg[MAX_SKB_FRAGS + 2];/* Name of the send queue: output.$index */ char name[16];struct virtnet_sq_stats stats;struct virtnet_interrupt_coalesce intr_coal;struct napi_struct napi;/* Record whether sq is in reset state. */ bool reset;struct xsk_buff_pool *xsk_pool;/* Internal representation of a receive virtqueue */ struct receive_queue {/* Virtqueue associated with this receive_queue */ struct virtqueue *vq;struct napi_struct napi;struct bpf_prog __rcu *xdp_prog;struct virtnet_rq_stats stats;/* The number of rx notifications */ /* Is dynamic interrupt moderation enabled? */ bool dim_enabled;/* Used to protect dim_enabled and inter_coal */ struct mutex dim_lock;/* Dynamic Interrupt Moderation */ struct dim dim;struct virtnet_interrupt_coalesce intr_coal;/* Chain pages by the private ptr. */ struct page *pages;/* Average packet length for mergeable receive buffers. */ struct ewma_pkt_len mrg_avg_pkt_len;/* Page frag for packet buffer allocation. */ struct page_frag alloc_frag;/* RX: fragments + linear part + virtio header */ struct scatterlist sg[MAX_SKB_FRAGS + 2];/* Min single buffer size for mergeable buffers case. */ unsigned int min_buf_len;/* Name of this receive queue: input.$index */ char name[16];struct xdp_rxq_info xdp_rxq;/* Record the last dma info to free after new pages is allocated. */ struct virtnet_rq_dma *last_dma;struct xsk_buff_pool *xsk_pool;/* xdp rxq used by xsk */ struct xdp_rxq_info xsk_rxq_info;struct xdp_buff **xsk_buffs;#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40/* Control VQ buffers: protected by the rtnl lock */ struct control_buf {struct virtio_net_ctrl_hdr hdr;struct virtnet_info {struct virtio_device *vdev;struct virtqueue *cvq;struct net_device *dev;struct send_queue *sq;struct receive_queue *rq;unsigned int status;/* Max # of queue pairs supported by the device */ /* # of queue pairs currently used by the driver */ /* # of XDP queue pairs currently used by the driver */ /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ bool xdp_enabled;/* I like... big packets and I cannot lie! */ bool big_packets;/* number of sg entries allocated for big packets */ unsigned int big_packets_num_skbfrags;/* Host will merge rx buffers for big packets (shake it! shake it!) */ bool mergeable_rx_bufs;/* Host supports rss and/or hash report */ bool has_rss;bool has_rss_hash_report;struct virtio_net_rss_config_hdr *rss_hdr;struct virtio_net_rss_config_trailer rss_trailer;/* Has control virtqueue */ bool has_cvq;/* Lock to protect the control VQ */ struct mutex cvq_lock;/* Host can handle any s/g split between our header and packet data */ bool any_header_sg;/* Packet virtio header size */ /* Work struct for delayed refilling if we run low on memory. */ struct delayed_work refill;/* UDP tunnel support */ bool tx_tnl;bool rx_tnl;bool rx_tnl_csum;/* Is delayed refill enabled? */ bool refill_enabled;/* The lock to synchronize the access to refill_enabled */ /* Work struct for config space updates */ struct work_struct config_work;/* Work struct for setting rx mode */ struct work_struct rx_mode_work;/* OK to queue work setting RX mode? */ bool rx_mode_work_enabled;/* Does the affinity hint is set for virtqueues? */ bool affinity_hint_set;/* CPU hotplug instances for online & dead */ struct hlist_node node;struct hlist_node node_dead;struct control_buf *ctrl;/* Ethtool settings */ /* Is rx dynamic interrupt moderation enabled? */ bool rx_dim_enabled;/* Interrupt coalescing settings */ struct virtnet_interrupt_coalesce intr_coal_tx;struct virtnet_interrupt_coalesce intr_coal_rx;unsigned long guest_offloads;unsigned long guest_offloads_capable;/* failover when STANDBY feature enabled */ struct failover *failover;struct padded_vnet_hdr {struct virtio_net_hdr_v1_hash hdr;/* * hdr is in a separate sg buffer, and data sg buffer shares same page * with this header sg. This padding makes next sg 16 byte aligned * after the header. char padding[12];struct virtio_net_common_hdr {union {struct virtio_net_hdr hdr;struct virtio_net_hdr_mrg_rxbuf mrg_hdr;struct virtio_net_hdr_v1_hash hash_v1_hdr;struct virtio_net_hdr_v1_hash_tunnel tnl_hdr;static struct virtio_net_common_hdr xsk_hdr;static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);static void virtnet_sq_free_unused_buf_done(struct virtqueue *vq);static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,struct net_device *dev,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats);static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,struct sk_buff *skb, u8 flags);static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,struct sk_buff *curr_skb,struct page *page, void *buf,int len, int truesize);static void virtnet_xsk_completed(struct send_queue *sq, int num);enum virtnet_xmit_type {static size_t virtnet_rss_hdr_size(const struct virtnet_info *vi)return struct_size(vi->rss_hdr, indirection_table, indir_table_size);static size_t virtnet_rss_trailer_size(const struct virtnet_info *vi)return struct_size(&vi->rss_trailer, hash_key_data, vi->rss_key_size);/* We use the last two bits of the pointer to distinguish the xmit type. */ #define VIRTNET_XMIT_TYPE_MASK (BIT(0) | BIT(1))#define VIRTIO_XSK_FLAG_OFFSET 2static enum virtnet_xmit_type virtnet_xmit_ptr_unpack(void **ptr)unsigned long p = (unsigned long )*ptr;void *)(p & ~VIRTNET_XMIT_TYPE_MASK);return p & VIRTNET_XMIT_TYPE_MASK;static void *virtnet_xmit_ptr_pack(void *ptr, enum virtnet_xmit_type type)return (void *)((unsigned long )ptr | type);static int virtnet_add_outbuf(struct send_queue *sq, int num, void *data,enum virtnet_xmit_type type)return virtqueue_add_outbuf(sq->vq, sq->sg, num,static u32 virtnet_ptr_to_xsk_buff_len(void *ptr)return ((unsigned long )ptr) >> VIRTIO_XSK_FLAG_OFFSET;static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)static void __free_old_xmit(struct send_queue *sq, struct netdev_queue *txq,bool in_napi, struct virtnet_sq_free_stats *stats)struct xdp_frame *frame;struct sk_buff *skb;unsigned int len;void *ptr;while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {switch (virtnet_xmit_ptr_unpack(&ptr)) {case VIRTNET_XMIT_TYPE_SKB:"Sent skb %p\n" , skb);break ;case VIRTNET_XMIT_TYPE_SKB_ORPHAN:break ;case VIRTNET_XMIT_TYPE_XDP:break ;case VIRTNET_XMIT_TYPE_XSK:break ;static void virtnet_free_old_xmit(struct send_queue *sq,struct netdev_queue *txq,bool in_napi,struct virtnet_sq_free_stats *stats)if (stats->xsk)/* Converting between virtqueue no. and kernel tx/rx queue no. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq static int vq2txq(struct virtqueue *vq)return (vq->index - 1) / 2;static int txq2vq(int txq)return txq * 2 + 1;static int vq2rxq(struct virtqueue *vq)return vq->index / 2;static int rxq2vq(int rxq)return rxq * 2;static int vq_type(struct virtnet_info *vi, int qid)if (qid == vi->max_queue_pairs * 2)return VIRTNET_Q_TYPE_CQ;if (qid % 2)return VIRTNET_Q_TYPE_TX;return VIRTNET_Q_TYPE_RX;static inline struct virtio_net_common_hdr *struct sk_buff *skb)return (struct virtio_net_common_hdr *)skb->cb;/* * private is used to chain pages for big packets, put the whole * most recent used list in the beginning for reuse static void give_pages(struct receive_queue *rq, struct page *page)struct page *end;/* Find end of list, sew whole thing into vi->rq.pages. */ for (end = page; end->private ; end = (struct page *)end->private );private = (unsigned long )rq->pages;static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)struct page *p = rq->pages;if (p) {struct page *)p->private ;/* clear private here, it is used to chain pages */ private = 0;else return p;static void virtnet_rq_free_buf(struct virtnet_info *vi,struct receive_queue *rq, void *buf)if (vi->mergeable_rx_bufs)else if (vi->big_packets)else static void enable_delayed_refill(struct virtnet_info *vi)true ;static void disable_delayed_refill(struct virtnet_info *vi)false ;static void enable_rx_mode_work(struct virtnet_info *vi)true ;static void disable_rx_mode_work(struct virtnet_info *vi)false ;static void virtqueue_napi_schedule(struct napi_struct *napi,struct virtqueue *vq)if (napi_schedule_prep(napi)) {static bool virtqueue_napi_complete(struct napi_struct *napi,struct virtqueue *vq, int processed)int opaque;if (napi_complete_done(napi, processed)) {if (unlikely(virtqueue_poll(vq, opaque)))else return true ;else {return false ;static void skb_xmit_done(struct virtqueue *vq)struct virtnet_info *vi = vq->vdev->priv;struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;/* Suppress further interrupts. */ if (napi->weight)else /* We were probably waiting for more output buffers. */ #define MRG_CTX_HEADER_SHIFT 22static void *mergeable_len_to_ctx(unsigned int truesize,unsigned int headroom)return (void *)(unsigned long )((headroom << MRG_CTX_HEADER_SHIFT) | truesize);static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)return (unsigned long )mrg_ctx >> MRG_CTX_HEADER_SHIFT;static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)return (unsigned long )mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);static int check_mergeable_len(struct net_device *dev, void *mrg_ctx,unsigned int len)unsigned int headroom, tailroom, room, truesize;sizeof (struct skb_shared_info) : 0;if (len > truesize - room) {"%s: rx error: len %u exceeds truesize %lu\n" ,unsigned long )(truesize - room));return -1;return 0;static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen,unsigned int headroom,unsigned int len)struct sk_buff *skb;if (unlikely(!skb))return NULL;return skb;/* Called from bottom half context */ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq,struct page *page, unsigned int offset,unsigned int len, unsigned int truesize,unsigned int headroom)struct sk_buff *skb;struct virtio_net_common_hdr *hdr;unsigned int copy, hdr_len, hdr_padded_len;struct page *page_to_free = NULL;int tailroom, shinfo_size;char *p, *hdr_p, *buf;if (vi->mergeable_rx_bufs)else sizeof (struct padded_vnet_hdr);sizeof (struct skb_shared_info));if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {if (unlikely(!skb))return NULL;struct page *)page->private ;if (page)goto ok;/* copy small packet so we can reuse these pages for small data */ if (unlikely(!skb))return NULL;/* Copy all frame if it fits skb->head, otherwise * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. if (len <= skb_tailroom(skb))else if (vi->mergeable_rx_bufs) {if (len)else goto ok;while (len) {unsigned int frag_size = min((unsigned )PAGE_SIZE - offset, len);struct page *)page->private ;if (page)if (page_to_free)return skb;static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)struct virtnet_info *vi = rq->vq->vdev->priv;struct page *page = virt_to_head_page(buf);struct virtnet_rq_dma *dma;void *head;int offset;if (dma->need_sync && len) {sizeof (*dma));if (dma->ref)return ;static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)struct virtnet_info *vi = rq->vq->vdev->priv;void *buf;if (buf)return buf;static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)struct virtnet_info *vi = rq->vq->vdev->priv;struct virtnet_rq_dma *dma;void *head;sizeof (*dma) + offset;static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)struct page_frag *alloc_frag = &rq->alloc_frag;struct virtnet_info *vi = rq->vq->vdev->priv;struct virtnet_rq_dma *dma;void *buf, *head;/* new pages */ if (!alloc_frag->offset) {if (rq->last_dma) {/* Now, the new page is allocated, the last dma * will not be used. So the dma can be unmapped * if the ref is 0. sizeof (*dma);if (virtqueue_dma_mapping_error(rq->vq, addr))return NULL;/* Add a reference to dma to prevent the entire dma from * being released during error handling. This reference * will be freed after the pages are no longer used. sizeof (*dma);return buf;static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf)struct virtnet_info *vi = vq->vdev->priv;struct receive_queue *rq;int i = vq2rxq(vq);if (rq->xsk_pool) {struct xdp_buff *)buf);return ;if (!vi->big_packets || vi->mergeable_rx_bufs)static void free_old_xmit(struct send_queue *sq, struct netdev_queue *txq,bool in_napi)struct virtnet_sq_free_stats stats = {0};/* Avoid overhead when no packets have been processed * happens when called speculatively from start_xmit. if (!stats.packets && !stats.napi_packets)return ;static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))return false ;else if (q < vi->curr_queue_pairs)return true ;else return false ;static bool tx_may_stop(struct virtnet_info *vi,struct net_device *dev,struct send_queue *sq)int qnum;/* If running out of space, stop queue to avoid getting packets that we * are then unable to transmit. * An alternative would be to force queuing layer to requeue the skb by * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be * returned in a normal path of operation: it means that driver is not * maintaining the TX queue stop/start state properly, and causes * the stack to do a non-trivial amount of useless work. * Since most packets only take 1 or 2 ring slots, stopping the queue * early means 16 slots are typically wasted. if (sq->vq->num_free < MAX_SKB_FRAGS + 2) {struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);return true ;return false ;static void check_sq_full_and_disable(struct virtnet_info *vi,struct net_device *dev,struct send_queue *sq)bool use_napi = sq->napi.weight;int qnum;if (tx_may_stop(vi, dev, sq)) {struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);if (use_napi) {if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {/* More just got used, free them then recheck. */ false );if (sq->vq->num_free >= MAX_SKB_FRAGS + 2) {/* Note that @len is the length of received data without virtio header */ static struct xdp_buff *buf_to_xdp(struct virtnet_info *vi,struct receive_queue *rq, void *buf,bool first_buf)struct xdp_buff *xdp;struct xdp_buff *)buf;/* In virtnet_add_recvbuf_xsk, we use part of XDP_PACKET_HEADROOM for * virtio header and ask the vhost to fill data from * hard_start + XDP_PACKET_HEADROOM - vi->hdr_len * The first buffer has virtio header so the remaining region for frame * data is * xsk_pool_get_rx_frame_size() * While other buffers than the first one do not have virtio header, so * the maximum frame data's length can be * xsk_pool_get_rx_frame_size() + vi->hdr_len if (!first_buf)if (unlikely(len > bufsize)) {"%s: rx error: len %u exceeds truesize %u\n" ,return NULL;if (first_buf) {else {return xdp;static struct sk_buff *xsk_construct_skb(struct receive_queue *rq,struct xdp_buff *xdp)unsigned int metasize = xdp->data - xdp->data_meta;struct sk_buff *skb;unsigned int size;if (unlikely(!skb)) {return NULL;if (metasize) {return skb;static struct sk_buff *virtnet_receive_xsk_small(struct net_device *dev, struct virtnet_info *vi,struct receive_queue *rq, struct xdp_buff *xdp,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)struct bpf_prog *prog;if (prog)switch (ret) {case XDP_PASS:return xsk_construct_skb(rq, xdp);case XDP_TX:case XDP_REDIRECT:return NULL;default :/* drop packet */ return NULL;static void xsk_drop_follow_bufs(struct net_device *dev,struct receive_queue *rq,struct virtnet_rq_stats *stats)struct xdp_buff *xdp;while (num_buf-- > 1) {if (unlikely(!xdp)) {"%s: rx error: %d buffers missing\n" ,break ;static int xsk_append_merge_buffer(struct virtnet_info *vi,struct receive_queue *rq,struct sk_buff *head_skb,struct virtio_net_hdr_mrg_rxbuf *hdr,struct virtnet_rq_stats *stats)struct sk_buff *curr_skb;struct xdp_buff *xdp;struct page *page;void *buf;while (--num_buf) {if (unlikely(!buf)) {"%s: rx error: %d buffers out of %d missing\n" ,return -EINVAL;false );if (!xdp)goto err;if (!buf) {goto err;if (!curr_skb) {goto err;return 0;return -EINVAL;static struct sk_buff *virtnet_receive_xsk_merge(struct net_device *dev, struct virtnet_info *vi,struct receive_queue *rq, struct xdp_buff *xdp,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)struct virtio_net_hdr_mrg_rxbuf *hdr;struct bpf_prog *prog;struct sk_buff *skb;if (prog) {/* TODO: support multi buffer. */ if (num_buf == 1)else switch (ret) {case XDP_PASS:if (!skb)goto drop_bufs;if (xsk_append_merge_buffer(vi, rq, skb, num_buf, hdr, stats)) {goto drop;return skb;case XDP_TX:case XDP_REDIRECT:return NULL;default :/* drop packet */ return NULL;static void virtnet_receive_xsk_buf(struct virtnet_info *vi, struct receive_queue *rq,void *buf, u32 len,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)struct net_device *dev = vi->dev;struct sk_buff *skb = NULL;struct xdp_buff *xdp;true );if (!xdp)return ;if (unlikely(len < ETH_HLEN)) {"%s: short packet %i\n" , dev->name, len);return ;struct virtio_net_common_hdr *)(xdp->data - vi->hdr_len))->hdr.flags;if (!vi->mergeable_rx_bufs)else if (skb)static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue *rq,struct xsk_buff_pool *pool, gfp_t gfp)struct xdp_buff **xsk_buffs;int err = 0;int num;if (!num)return -ENOMEM;for (i = 0; i < num; ++i) {/* Use the part of XDP_PACKET_HEADROOM as the virtnet hdr space. * We assume XDP_PACKET_HEADROOM is larger than hdr->len. * (see function virtnet_xsk_pool_enable) if (err)goto err;return num;for (; i < num; ++i)return err;static void *virtnet_xsk_to_ptr(u32 len)unsigned long p;return virtnet_xmit_ptr_pack((void *)p, VIRTNET_XMIT_TYPE_XSK);static int virtnet_xsk_xmit_one(struct send_queue *sq,struct xsk_buff_pool *pool,struct xdp_desc *desc)struct virtnet_info *vi;return virtqueue_add_outbuf_premapped(sq->vq, sq->sg, 2,static int virtnet_xsk_xmit_batch(struct send_queue *sq,struct xsk_buff_pool *pool,unsigned int budget,struct xdp_desc *descs = pool->tx_descs;bool kick = false ;int err;if (!nb_pkts)return 0;for (i = 0; i < nb_pkts; i++) {if (unlikely(err)) {break ;true ;if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))return i;static bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,int budget)struct virtnet_info *vi = sq->vq->vdev->priv;struct virtnet_sq_free_stats stats = {};struct net_device *dev = vi->dev;int sent;/* Avoid to wakeup napi meanless, so call __free_old_xmit instead of * free_old_xmit(). true , &stats);if (stats.xsk)if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq))if (sent) {struct netdev_queue *txq;if (xsk_uses_need_wakeup(pool))return sent;static void xsk_wakeup(struct send_queue *sq)if (napi_if_scheduled_mark_missed(&sq->napi))return ;static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag)struct virtnet_info *vi = netdev_priv(dev);struct send_queue *sq;if (!netif_running(dev))return -ENETDOWN;if (qid >= vi->curr_queue_pairs)return -EINVAL;return 0;static void virtnet_xsk_completed(struct send_queue *sq, int num)/* If this is called by rx poll, start_xmit and xdp xmit we should * wakeup the tx napi to consume the xsk tx queue, because the tx * interrupt may not be triggered. static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,struct send_queue *sq,struct xdp_frame *xdpf)struct virtio_net_hdr_mrg_rxbuf *hdr;struct skb_shared_info *shinfo;int err, i;if (unlikely(xdpf->headroom < vi->hdr_len))return -EOVERFLOW;if (unlikely(xdp_frame_has_frags(xdpf))) {/* In wrapping function virtnet_xdp_xmit(), we need to free * up the pending old buffers, where we need to calculate the * position of skb_shared_info in xdp_get_frame_len() and * xdp_return_frame(), which will involve to xdpf->data and * xdpf->headroom. Therefore, we need to update the value of * headroom synchronously here. /* Zero header and leave csum up to XDP layers */ for (i = 0; i < nr_frags; i++) {if (unlikely(err))return -ENOSPC; /* Caller handle free/refcnt */ return 0;/* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on * the current cpu, so it does not need to be locked. * * Here we use marco instead of inline functions because we have to deal with * three issues at the same time: 1. the choice of sq. 2. judge and execute the * lock/unlock of txq 3. make sparse happy. It is difficult for two inline * functions to perfectly solve these three problems at the same time. #define virtnet_xdp_get_sq(vi) ({ \int cpu = smp_processor_id(); \struct netdev_queue *txq; \unsigned int qp; \if (v->curr_queue_pairs > nr_cpu_ids) { \else { \#define virtnet_xdp_put_sq(vi, q) { \struct netdev_queue *txq; \if (v->curr_queue_pairs > nr_cpu_ids) \else \static int virtnet_xdp_xmit(struct net_device *dev,int n, struct xdp_frame **frames, u32 flags)struct virtnet_info *vi = netdev_priv(dev);struct virtnet_sq_free_stats stats = {0};struct receive_queue *rq = vi->rq;struct bpf_prog *xdp_prog;struct send_queue *sq;int nxmit = 0;int kicks = 0;int ret;int i;/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this * indicate XDP resources have been successfully allocated. if (!xdp_prog)return -ENXIO;if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {goto out;/* Free up any pending old buffers before queueing new ones. */ false , &stats);for (i = 0; i < n; i++) {struct xdp_frame *xdpf = frames[i];if (__virtnet_xdp_xmit_one(vi, sq, xdpf))break ;if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq))if (flags & XDP_XMIT_FLUSH) {if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))return ret;static void put_xdp_frags(struct xdp_buff *xdp)struct skb_shared_info *shinfo;struct page *xdp_page;int i;if (xdp_buff_has_frags(xdp)) {for (i = 0; i < shinfo->nr_frags; i++) {static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,struct net_device *dev,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)struct xdp_frame *xdpf;int err;switch (act) {case XDP_PASS:return act;case XDP_TX:if (unlikely(!xdpf)) {"convert buff to frame failed for xdp\n" );return XDP_DROP;if (unlikely(!err)) {else if (unlikely(err < 0)) {return XDP_DROP;return act;case XDP_REDIRECT:if (err)return XDP_DROP;return act;default :case XDP_ABORTED:case XDP_DROP:return XDP_DROP;static unsigned int virtnet_get_headroom(struct virtnet_info *vi)return vi->xdp_enabled ? XDP_PACKET_HEADROOM : 0;/* We copy the packet for XDP in the following cases: * * 1) Packet is scattered across multiple rx buffers. * 2) Headroom space is insufficient. * * This is inefficient but it's a temporary condition that * we hit right after XDP is enabled and until queue is refilled * with large buffers with sufficient headroom - so it should affect * at most queue size packets. * Afterwards, the conditions to enable * XDP should preclude the underlying device from sending packets * across multiple buffers (num_buf > 1), and we make sure buffers * have enough headroom. static struct page *xdp_linearize_page(struct net_device *dev,struct receive_queue *rq,int *num_buf,struct page *p,int offset,int page_off,unsigned int *len)int tailroom = SKB_DATA_ALIGN(sizeof (struct skb_shared_info));struct page *page;if (page_off + *len + tailroom > PAGE_SIZE)return NULL;if (!page)return NULL;/* Only mergeable mode can go inside this while loop. In small mode, * *num_buf == 1, so it cannot go inside. while (--*num_buf) {unsigned int buflen;void *buf;void *ctx;int off;if (unlikely(!buf))goto err_buf;if (check_mergeable_len(dev, ctx, buflen)) {goto err_buf;/* guard against a misconfigured or uncooperative backend that * is sending packet larger than the MTU. if ((page_off + buflen + tailroom) > PAGE_SIZE) {goto err_buf;/* Headroom does not contribute to packet length */ return page;return NULL;static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,unsigned int xdp_headroom,void *buf,unsigned int len)unsigned int header_offset;unsigned int headroom;unsigned int buflen;struct sk_buff *skb;sizeof (struct skb_shared_info));if (unlikely(!skb))return NULL;return skb;static struct sk_buff *receive_small_xdp(struct net_device *dev,struct virtnet_info *vi,struct receive_queue *rq,struct bpf_prog *xdp_prog,void *buf,unsigned int xdp_headroom,unsigned int len,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;unsigned int headroom = vi->hdr_len + header_offset;struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;struct page *page = virt_to_head_page(buf);struct page *xdp_page;unsigned int buflen;struct xdp_buff xdp;struct sk_buff *skb;unsigned int metasize = 0;if (unlikely(hdr->hdr.gso_type))goto err_xdp;/* Partially checksummed packets must be dropped. */ if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))goto err_xdp;sizeof (struct skb_shared_info));if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {int offset = buf - page_address(page) + header_offset;unsigned int tlen = len + vi->hdr_len;int num_buf = 1;sizeof (struct skb_shared_info));if (!xdp_page)goto err_xdp;true );switch (act) {case XDP_PASS:/* Recalculate length in case bpf program changed it */ break ;case XDP_TX:case XDP_REDIRECT:goto xdp_xmit;default :goto err_xdp;if (unlikely(!skb))goto err;if (metasize)return skb;return NULL;static struct sk_buff *receive_small(struct net_device *dev,struct virtnet_info *vi,struct receive_queue *rq,void *buf, void *ctx,unsigned int len,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)unsigned int xdp_headroom = (unsigned long )ctx;struct page *page = virt_to_head_page(buf);struct sk_buff *skb;/* We passed the address of virtnet header to virtio-core, * so truncate the padding. if (unlikely(len > GOOD_PACKET_LEN)) {"%s: rx error: len %u exceeds max size %d\n" ,goto err;if (unlikely(vi->xdp_enabled)) {struct bpf_prog *xdp_prog;if (xdp_prog) {return skb;if (likely(skb))return skb;return NULL;static struct sk_buff *receive_big(struct net_device *dev,struct virtnet_info *vi,struct receive_queue *rq,void *buf,unsigned int len,struct virtnet_rq_stats *stats)struct page *page = buf;struct sk_buff *skb;/* Make sure that len does not exceed the size allocated in * add_recvbuf_big. if (unlikely(len > (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE)) {"%s: rx error: len %u exceeds allocated size %lu\n" ,goto err;if (unlikely(!skb))goto err;return skb;return NULL;static void mergeable_buf_free(struct receive_queue *rq, int num_buf,struct net_device *dev,struct virtnet_rq_stats *stats)struct page *page;void *buf;int len;while (num_buf-- > 1) {if (unlikely(!buf)) {"%s: rx error: %d buffers missing\n" ,break ;/* Why not use xdp_build_skb_from_frame() ? * XDP core assumes that xdp frags are PAGE_SIZE in length, while in * virtio-net there are 2 points that do not match its requirements: * 1. The size of the prefilled buffer is not fixed before xdp is set. * 2. xdp_build_skb_from_frame() does more checks that we don't need, * like eth_type_trans() (which virtio-net does in receive_buf()). static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,struct virtnet_info *vi,struct xdp_buff *xdp,unsigned int xdp_frags_truesz)struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);unsigned int headroom, data_len;struct sk_buff *skb;int metasize;if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {"Error building skb as missing reserved tailroom for xdp" );return NULL;if (unlikely(xdp_buff_has_frags(xdp)))if (unlikely(!skb))return NULL;if (metasize)if (unlikely(xdp_buff_has_frags(xdp)))return skb;/* TODO: build xdp in big mode */ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,struct virtnet_info *vi,struct receive_queue *rq,struct xdp_buff *xdp,void *buf,unsigned int len,unsigned int frame_sz,int *num_buf,unsigned int *xdp_frags_truesize,struct virtnet_rq_stats *stats)struct virtio_net_hdr_mrg_rxbuf *hdr = buf;struct skb_shared_info *shinfo;unsigned int xdp_frags_truesz = 0;unsigned int truesize;struct page *page;int offset;void *ctx;true );if (!*num_buf)return 0;if (*num_buf > 1) {/* If we want to build multi-buffer xdp, we need * to specify that the flags of xdp_buff have the * XDP_FLAGS_HAS_FRAG bit. if (!xdp_buff_has_frags(xdp))if (*num_buf > MAX_SKB_FRAGS + 1)return -EINVAL;while (--*num_buf > 0) {if (unlikely(!buf)) {"%s: rx error: %d buffers out of %d missing\n" ,goto err;if (check_mergeable_len(dev, ctx, len)) {goto err;if (page_is_pfmemalloc(page))return 0;return -EINVAL;static void *mergeable_xdp_get_buf(struct virtnet_info *vi,struct receive_queue *rq,struct bpf_prog *xdp_prog,void *ctx,unsigned int *frame_sz,int *num_buf,struct page **page,int offset,unsigned int *len,struct virtio_net_hdr_mrg_rxbuf *hdr)unsigned int truesize = mergeable_ctx_to_truesize(ctx);unsigned int headroom = mergeable_ctx_to_headroom(ctx);struct page *xdp_page;unsigned int xdp_room;/* Transient failure which in theory could occur if * in-flight packets from before XDP was enabled reach * the receive path after XDP is loaded. if (unlikely(hdr->hdr.gso_type))return NULL;/* Partially checksummed packets must be dropped. */ if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))return NULL;/* Now XDP core assumes frag size is PAGE_SIZE, but buffers * with headroom may add hole in truesize, which * make their length exceed PAGE_SIZE. So we disabled the * hole mechanism for xdp. See add_recvbuf_mergeable(). if (likely(headroom >= virtnet_get_headroom(vi) &&return page_address(*page) + offset;/* This happens when headroom is not enough because * of the buffer was prefilled before XDP is set. * This should only happen for the first several packets. * In fact, vq reset can be used here to help us clean up * the prefilled buffers, but many existing devices do not * support it, and we don't want to bother users who are * using xdp normally. if (!xdp_prog->aux->xdp_has_frags) {/* linearize data for XDP */ if (!xdp_page)return NULL;else {sizeof (struct skb_shared_info));if (*len + xdp_room > PAGE_SIZE)return NULL;if (!xdp_page)return NULL;return page_address(*page) + XDP_PACKET_HEADROOM;static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,struct virtnet_info *vi,struct receive_queue *rq,struct bpf_prog *xdp_prog,void *buf,void *ctx,unsigned int len,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)struct virtio_net_hdr_mrg_rxbuf *hdr = buf;int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);struct page *page = virt_to_head_page(buf);int offset = buf - page_address(page);unsigned int xdp_frags_truesz = 0;struct sk_buff *head_skb;unsigned int frame_sz;struct xdp_buff xdp;void *data;int err;if (unlikely(!data))goto err_xdp;if (unlikely(err))goto err_xdp;switch (act) {case XDP_PASS:if (unlikely(!head_skb))break ;return head_skb;case XDP_TX:case XDP_REDIRECT:return NULL;default :break ;return NULL;static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,struct sk_buff *curr_skb,struct page *page, void *buf,int len, int truesize)int num_skb_frags;int offset;if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);if (unlikely(!nskb))return NULL;if (curr_skb == head_skb)else if (curr_skb != head_skb) {if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {else {return curr_skb;static struct sk_buff *receive_mergeable(struct net_device *dev,struct virtnet_info *vi,struct receive_queue *rq,void *buf,void *ctx,unsigned int len,unsigned int *xdp_xmit,struct virtnet_rq_stats *stats)struct virtio_net_hdr_mrg_rxbuf *hdr = buf;int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);struct page *page = virt_to_head_page(buf);int offset = buf - page_address(page);struct sk_buff *head_skb, *curr_skb;unsigned int truesize = mergeable_ctx_to_truesize(ctx);unsigned int headroom = mergeable_ctx_to_headroom(ctx);if (check_mergeable_len(dev, ctx, len))goto err_skb;if (unlikely(vi->xdp_enabled)) {struct bpf_prog *xdp_prog;if (xdp_prog) {return head_skb;if (unlikely(!curr_skb))goto err_skb;while (--num_buf) {if (unlikely(!buf)) {"%s: rx error: %d buffers out of %d missing\n" ,goto err_buf;if (check_mergeable_len(dev, ctx, len))goto err_skb;if (!curr_skb)goto err_skb;return head_skb;return NULL;static inline u32const struct virtio_net_hdr_v1_hash *hdr_hash)return __le16_to_cpu(hdr_hash->hash_value_lo) |static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,struct sk_buff *skb)enum pkt_hash_types rss_hash_type;if (!hdr_hash || !skb)return ;switch (__le16_to_cpu(hdr_hash->hash_report)) {case VIRTIO_NET_HASH_REPORT_TCPv4:case VIRTIO_NET_HASH_REPORT_UDPv4:case VIRTIO_NET_HASH_REPORT_TCPv6:case VIRTIO_NET_HASH_REPORT_UDPv6:case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
Messung V0.5 C=96 H=96 G=95
¤ Dauer der Verarbeitung: 0.22 Sekunden
¤ *© Formatika GbR, Deutschland
