/* Driver can alloc up to 2 segments for the header and 2 for the payload. */ #define GVE_TX_MAX_IOVEC 4 /* 1 for management, 1 for rx, 1 for tx */ #define GVE_MIN_MSIX 3
/* Numbers of gve tx/rx stats in stats report. */ #define GVE_TX_STATS_REPORT_NUM 6 #define GVE_RX_STATS_REPORT_NUM 2
/* Interval to schedule a stats report update, 20000ms. */ #define GVE_STATS_REPORT_TIMER_PERIOD 20000
/* Numbers of NIC tx/rx stats in stats report. */ #define NIC_TX_STATS_REPORT_NUM 0 #define NIC_RX_STATS_REPORT_NUM 4
/* If number of free/recyclable buffers are less than this threshold; driver * allocs and uses a non-qpl page on the receive path of DQO QPL to free * up buffers. * Value is set big enough to post at least 3 64K LRO packet via 2K buffer to NIC.
*/ #define GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD 96
#define GVE_DQO_RX_HWTSTAMP_VALID 0x1
/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ struct gve_rx_desc_queue { struct gve_rx_desc *desc_ring; /* the descriptor ring */
dma_addr_t bus; /* the bus for the desc_ring */
u8 seqno; /* the next expected seqno for this desc*/
};
/* The page info for a single slot in the RX data queue */ struct gve_rx_slot_page_info { /* netmem is used for DQO RDA mode * page is used in all other modes
*/ union { struct page *page;
netmem_ref netmem;
}; void *page_address;
u32 page_offset; /* offset to write to in page */ unsignedint buf_size; int pagecnt_bias; /* expected pagecnt if only the driver has a ref */
u16 pad; /* adjustment for rx padding */
u8 can_flip; /* tracks if the networking stack is using the page */
};
/* A list of pages registered with the device during setup and used by a queue * as buffers
*/ struct gve_queue_page_list {
u32 id; /* unique id */
u32 num_entries; struct page **pages; /* list of num_entries pages */
dma_addr_t *page_buses; /* the dma addrs of the pages */
};
/* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */ struct gve_rx_data_queue { union gve_rx_data_slot *data_ring; /* read by NIC */
dma_addr_t data_bus; /* dma mapping of the slots */ struct gve_rx_slot_page_info *page_info; /* page info of the buffers */ struct gve_queue_page_list *qpl; /* qpl assigned to this queue */
u8 raw_addressing; /* use raw_addressing? */
};
struct gve_priv;
/* RX buffer queue for posting buffers to HW. * Each RX (completion) queue has a corresponding buffer queue.
*/ struct gve_rx_buf_queue_dqo { struct gve_rx_desc_dqo *desc_ring;
dma_addr_t bus;
u32 head; /* Pointer to start cleaning buffers at. */
u32 tail; /* Last posted buffer index + 1 */
u32 mask; /* Mask for indices to the size of the ring */
};
/* RX completion queue to receive packets from HW. */ struct gve_rx_compl_queue_dqo { struct gve_rx_compl_desc_dqo *desc_ring;
dma_addr_t bus;
/* Number of slots which did not have a buffer posted yet. We should not * post more buffers than the queue size to avoid HW overrunning the * queue.
*/ int num_free_slots;
/* HW uses a "generation bit" to notify SW of new descriptors. When a * descriptor's generation bit is different from the current generation, * that descriptor is ready to be consumed by SW.
*/
u8 cur_gen_bit;
/* Pointer into desc_ring where the next completion descriptor will be * received.
*/
u32 head;
u32 mask; /* Mask for indices to the size of the ring */
};
/* Stores state for tracking buffers posted to HW */ struct gve_rx_buf_state_dqo { /* The page posted to HW. */ struct gve_rx_slot_page_info page_info;
/* XSK buffer */ struct xdp_buff *xsk_buff;
/* The DMA address corresponding to `page_info`. */
dma_addr_t addr;
/* Last offset into the page when it only had a single reference, at * which point every other offset is free to be reused.
*/
u32 last_single_ref_offset;
/* Linked list index to next element in the list, or -1 if none */
s16 next;
};
/* `head` and `tail` are indices into an array, or -1 if empty. */ struct gve_index_list {
s16 head;
s16 tail;
};
/* A single received packet split across multiple buffers may be * reconstructed using the information in this structure.
*/ struct gve_rx_ctx { /* head and tail of skb chain for the current packet or NULL if none */ struct sk_buff *skb_head; struct sk_buff *skb_tail;
u32 total_size;
u8 frag_cnt; bool drop_pkt;
};
/* Linked list of gve_rx_buf_state_dqo. Index into * buf_states, or -1 if empty.
*/
s16 free_buf_states;
/* Linked list of gve_rx_buf_state_dqo. Indexes into * buf_states, or -1 if empty. * * This list contains buf_states which are pointing to * valid buffers. * * We use a FIFO here in order to increase the * probability that buffers can be reused by increasing * the time between usages.
*/ struct gve_index_list recycled_buf_states;
/* Linked list of gve_rx_buf_state_dqo. Indexes into * buf_states, or -1 if empty. * * This list contains buf_states which have buffers * which cannot be reused yet.
*/ struct gve_index_list used_buf_states;
/* qpl assigned to this queue */ struct gve_queue_page_list *qpl;
/* index into queue page list */
u32 next_qpl_page_idx;
/* track number of used buffers */
u16 used_buf_states_cnt;
/* Address info of the buffers for header-split */ struct gve_header_buf hdr_bufs;
struct page_pool *page_pool;
} dqo;
};
u64 rbytes; /* free-running bytes received */
u64 rx_hsplit_bytes; /* free-running header bytes received */
u64 rpackets; /* free-running packets received */
u32 cnt; /* free-running total number of completed packets */
u32 fill_cnt; /* free-running total number of descs and buffs posted */
u32 mask; /* masks the cnt and fill_cnt to the size of the ring */
u64 rx_hsplit_pkt; /* free-running packets with headers split */
u64 rx_copybreak_pkt; /* free-running count of copybreak packets */
u64 rx_copied_pkt; /* free-running total number of copied packets */
u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */
u64 rx_buf_alloc_fail; /* free-running count of buffer alloc fails */
u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */ /* free-running count of unsplit packets due to header buffer overflow or hdr_len is 0 */
u64 rx_hsplit_unsplit_pkt;
u64 rx_cont_packet_cnt; /* free-running multi-fragment packets received */
u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */
u64 rx_frag_copy_cnt; /* free-running count of rx segments copied */
u64 rx_frag_alloc_cnt; /* free-running count of rx page allocations */
u64 xdp_tx_errors;
u64 xdp_redirect_errors;
u64 xdp_alloc_fails;
u64 xdp_actions[GVE_XDP_ACTIONS];
u32 q_num; /* queue index */
u32 ntfy_id; /* notification block index */ struct gve_queue_resources *q_resources; /* head and tail pointer idx */
dma_addr_t q_resources_bus; /* dma address for the queue resources */ struct u64_stats_sync statss; /* sync stats for 32bit archs */
struct gve_rx_ctx ctx; /* Info for packet currently being processed in this ring. */
/* A TX desc ring entry */ union gve_tx_desc { struct gve_tx_pkt_desc pkt; /* first desc for a packet */ struct gve_tx_mtd_desc mtd; /* optional metadata descriptor */ struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
};
/* Tracks the memory in the fifo occupied by a segment of a packet */ struct gve_tx_iovec {
u32 iov_offset; /* offset into this segment */
u32 iov_len; /* length */
u32 iov_padding; /* padding associated with this segment */
};
/* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc * ring entry but only used for a pkt_desc not a seg_desc
*/ struct gve_tx_buffer_state { union { struct sk_buff *skb; /* skb for this pkt */ struct xdp_frame *xdp_frame; /* xdp_frame */
}; struct {
u16 size; /* size of xmitted xdp pkt */
u8 is_xsk; /* xsk buff */
} xdp; union { struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */ struct {
DEFINE_DMA_UNMAP_ADDR(dma);
DEFINE_DMA_UNMAP_LEN(len);
};
};
};
/* A TX buffer - each queue has one */ struct gve_tx_fifo { void *base; /* address of base of FIFO */
u32 size; /* total size */
atomic_t available; /* how much space is still available */
u32 head; /* offset to write at */ struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */
};
/* TX descriptor for DQO format */ union gve_tx_desc_dqo { struct gve_tx_pkt_desc_dqo pkt; struct gve_tx_tso_context_desc_dqo tso_ctx; struct gve_tx_general_context_desc_dqo general_ctx;
};
enum gve_packet_state { /* Packet is in free list, available to be allocated. * This should always be zero since state is not explicitly initialized.
*/
GVE_PACKET_STATE_UNALLOCATED, /* Packet is expecting a regular data completion or miss completion */
GVE_PACKET_STATE_PENDING_DATA_COMPL, /* Packet has received a miss completion and is expecting a * re-injection completion.
*/
GVE_PACKET_STATE_PENDING_REINJECT_COMPL, /* No valid completion received within the specified timeout. */
GVE_PACKET_STATE_TIMED_OUT_COMPL, /* XSK pending packet has received a packet/reinjection completion, or * has timed out. At this point, the pending packet can be counted by * xsk_tx_complete and freed.
*/
GVE_PACKET_STATE_XSK_COMPLETE,
};
/* 0th element corresponds to the linear portion of `skb`, should be * unmapped with `dma_unmap_single`. * * All others correspond to `skb`'s frags and should be unmapped with * `dma_unmap_page`.
*/ union { struct {
DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]);
DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]);
};
s16 tx_qpl_buf_ids[GVE_MAX_TX_BUFS_PER_PKT];
};
u16 num_bufs;
/* Linked list index to next element in the list, or -1 if none */
s16 next;
/* Linked list index to prev element in the list, or -1 if none. * Used for tracking either outstanding miss completions or prematurely * freed packets.
*/
s16 prev;
/* Identifies the current state of the packet as defined in * `enum gve_packet_state`.
*/
u8 state : 3;
/* gve_tx_pending_packet_dqo_type */
u8 type : 2;
/* If packet is an outstanding miss completion, then the packet is * freed if the corresponding re-injection completion is not received * before kernel jiffies exceeds timeout_jiffies.
*/ unsignedlong timeout_jiffies;
};
/* Contains datapath state used to represent a TX queue. */ struct gve_tx_ring { /* Cacheline 0 -- Accessed & dirtied during transmit */ union { /* GQI fields */ struct { struct gve_tx_fifo tx_fifo;
u32 req; /* driver tracked head pointer */
u32 done; /* driver tracked tail pointer */
};
/* Linked list of gve_tx_pending_packet_dqo. Index into * pending_packets, or -1 if empty. * * This is a consumer list owned by the TX path. When it * runs out, the producer list is stolen from the * completion handling path * (dqo_compl.free_pending_packets).
*/
s16 free_pending_packets;
/* Cached value of `dqo_compl.hw_tx_head` */
u32 head;
u32 tail; /* Last posted buffer index + 1 */
/* Index of the last descriptor with "report event" bit * set.
*/
u32 last_re_idx;
/* free running number of packet buf descriptors posted */
u16 posted_packet_desc_cnt; /* free running number of packet buf descriptors completed */
u16 completed_packet_desc_cnt;
/* QPL fields */ struct { /* Linked list of gve_tx_buf_dqo. Index into * tx_qpl_buf_next, or -1 if empty. * * This is a consumer list owned by the TX path. When it * runs out, the producer list is stolen from the * completion handling path * (dqo_compl.free_tx_qpl_buf_head).
*/
s16 free_tx_qpl_buf_head;
/* Free running count of the number of QPL tx buffers * allocated
*/
u32 alloc_tx_qpl_buf_cnt;
/* Cached value of `dqo_compl.free_tx_qpl_buf_cnt` */
u32 free_tx_qpl_buf_cnt;
};
atomic_t xsk_reorder_queue_tail;
} dqo_tx;
};
/* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */ union { /* GQI fields */ struct { /* Spinlock for when cleanup in progress */
spinlock_t clean_lock; /* Spinlock for XDP tx traffic */
spinlock_t xdp_lock;
};
/* DQO fields. */ struct {
u32 head; /* Last read on compl_desc */
/* Tracks the current gen bit of compl_q */
u8 cur_gen_bit;
/* Linked list of gve_tx_pending_packet_dqo. Index into * pending_packets, or -1 if empty. * * This is the producer list, owned by the completion * handling path. When the consumer list * (dqo_tx.free_pending_packets) is runs out, this list * will be stolen.
*/
atomic_t free_pending_packets;
/* Last TX ring index fetched by HW */
atomic_t hw_tx_head;
/* List to track pending packets which received a miss * completion but not a corresponding reinjection.
*/ struct gve_index_list miss_completions;
/* List to track pending packets that were completed * before receiving a valid completion because they * reached a specified timeout.
*/ struct gve_index_list timed_out_completions;
/* QPL fields */ struct { /* Linked list of gve_tx_buf_dqo. Index into * tx_qpl_buf_next, or -1 if empty. * * This is the producer list, owned by the completion * handling path. When the consumer list * (dqo_tx.free_tx_qpl_buf_head) is runs out, this list * will be stolen.
*/
atomic_t free_tx_qpl_buf_head;
/* Free running count of the number of tx buffers * freed
*/
atomic_t free_tx_qpl_buf_cnt;
};
} dqo_compl;
} ____cacheline_aligned;
u64 pkt_done; /* free-running - total packets completed */
u64 bytes_done; /* free-running - total bytes completed */
u64 dropped_pkt; /* free-running - total packets dropped */
u64 dma_mapping_error; /* count of dma mapping errors */
/* Cacheline 2 -- Read-mostly fields */ union { /* GQI fields */ struct { union gve_tx_desc *desc;
/* Maps 1:1 to a desc */ struct gve_tx_buffer_state *info;
};
u32 complq_mask; /* complq size is complq_mask + 1 */
/* QPL fields */ struct { /* qpl assigned to this queue */ struct gve_queue_page_list *qpl;
/* Each QPL page is divided into TX bounce buffers * of size GVE_TX_BUF_SIZE_DQO. tx_qpl_buf_next is * an array to manage linked lists of TX buffers. * An entry j at index i implies that j'th buffer * is next on the list after i
*/
s16 *tx_qpl_buf_next;
u32 num_tx_qpl_bufs;
};
} dqo;
} ____cacheline_aligned; struct netdev_queue *netdev_txq; struct gve_queue_resources *q_resources; /* head and tail pointer idx */ struct device *dev;
u32 mask; /* masks req and done down to queue size */
u8 raw_addressing; /* use raw_addressing? */
/* Slow-path fields */
u32 q_num ____cacheline_aligned; /* queue idx */
u32 stop_queue; /* count of queue stops */
u32 wake_queue; /* count of queue wakes */
u32 queue_timeout; /* count of queue timeouts */
u32 ntfy_id; /* notification block index */
u32 last_kick_msec; /* Last time the queue was kicked */
dma_addr_t bus; /* dma address of the descr ring */
dma_addr_t q_resources_bus; /* dma address of the queue resources */
dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */ struct u64_stats_sync statss; /* sync stats for 32bit archs */ struct xsk_buff_pool *xsk_pool;
u64 xdp_xsk_sent;
u64 xdp_xmit;
u64 xdp_xmit_errors;
} ____cacheline_aligned;
/* Wraps the info for one irq including the napi struct and the queues * associated with that irq.
*/ struct gve_notify_block {
__be32 *irq_db_index; /* pointer to idx into Bar2 */ char name[IFNAMSIZ + 16]; /* name registered with the kernel */ struct napi_struct napi; /* kernel napi struct for this block */ struct gve_priv *priv; struct gve_tx_ring *tx; /* tx rings on this block */ struct gve_rx_ring *rx; /* rx rings on this block */
u32 irq;
};
/* Tracks allowed and current rx queue settings */ struct gve_rx_queue_config {
u16 max_queues;
u16 num_queues;
u16 packet_buffer_size;
};
/* Tracks allowed and current tx queue settings */ struct gve_tx_queue_config {
u16 max_queues;
u16 num_queues; /* number of TX queues, excluding XDP queues */
u16 num_xdp_queues;
};
/* Tracks the available and used qpl IDs */ struct gve_qpl_config {
u32 qpl_map_size; /* map memory size */ unsignedlong *qpl_id_map; /* bitmap of used qpl ids */
};
/* Allocated resources are returned here */ struct gve_rx_ring *rx;
};
/* GVE_QUEUE_FORMAT_UNSPECIFIED must be zero since 0 is the default value * when the entire configure_device_resources command is zeroed out and the * queue_format is not specified.
*/ enum gve_queue_format {
GVE_QUEUE_FORMAT_UNSPECIFIED = 0x0,
GVE_GQI_RDA_FORMAT = 0x1,
GVE_GQI_QPL_FORMAT = 0x2,
GVE_DQO_RDA_FORMAT = 0x3,
GVE_DQO_QPL_FORMAT = 0x4,
};
struct gve_flow_rules_cache { bool rules_cache_synced; /* False if the driver's rules_cache is outdated */ struct gve_adminq_queried_flow_rule *rules_cache;
__be32 *rule_ids_cache; /* The total number of queried rules that stored in the caches */
u32 rules_cache_num;
u32 rule_ids_cache_num;
};
u16 num_event_counters;
u16 tx_desc_cnt; /* num desc per ring */
u16 rx_desc_cnt; /* num desc per ring */
u16 max_tx_desc_cnt;
u16 max_rx_desc_cnt;
u16 min_tx_desc_cnt;
u16 min_rx_desc_cnt; bool modify_ring_size_enabled; bool default_min_ring_size;
u16 tx_pages_per_qpl; /* Suggested number of pages per qpl for TX queues by NIC */
u64 max_registered_pages;
u64 num_registered_pages; /* num pages registered with NIC */ struct bpf_prog *xdp_prog; /* XDP BPF program */
u32 rx_copybreak; /* copy packets smaller than this */
u16 default_num_queues; /* default num queues to set up */
struct gve_tx_queue_config tx_cfg; struct gve_rx_queue_config rx_cfg; unsignedlong *xsk_pools; /* bitmap of RX queues with XSK pools */
u32 num_ntfy_blks; /* split between TX and RX so must be even */ int numa_node;
struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
__be32 __iomem *db_bar2; /* "array" of doorbells */
u32 msg_enable; /* level for netif* netdev print macros */ struct pci_dev *pdev;
/* metrics */
u32 tx_timeo_cnt;
/* Admin queue - see gve_adminq.h*/ union gve_adminq_command *adminq;
dma_addr_t adminq_bus_addr; struct dma_pool *adminq_pool; struct mutex adminq_lock; /* Protects adminq command execution */
u32 adminq_mask; /* masks prod_cnt to adminq size */
u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
u32 adminq_cmd_fail; /* free-running count of AQ cmds failed */
u32 adminq_timeouts; /* free-running count of AQ cmds timeouts */ /* free-running count of per AQ cmd executed */
u32 adminq_describe_device_cnt;
u32 adminq_cfg_device_resources_cnt;
u32 adminq_register_page_list_cnt;
u32 adminq_unregister_page_list_cnt;
u32 adminq_create_tx_queue_cnt;
u32 adminq_create_rx_queue_cnt;
u32 adminq_destroy_tx_queue_cnt;
u32 adminq_destroy_rx_queue_cnt;
u32 adminq_dcfg_device_resources_cnt;
u32 adminq_set_driver_parameter_cnt;
u32 adminq_report_stats_cnt;
u32 adminq_report_link_speed_cnt;
u32 adminq_report_nic_timestamp_cnt;
u32 adminq_get_ptype_map_cnt;
u32 adminq_verify_driver_compatibility_cnt;
u32 adminq_query_flow_rules_cnt;
u32 adminq_cfg_flow_rule_cnt;
u32 adminq_cfg_rss_cnt;
u32 adminq_query_rss_cnt;
/* Global stats */
u32 interface_up_cnt; /* count of times interface turned up since last reset */
u32 interface_down_cnt; /* count of times interface turned down since last reset */
u32 reset_cnt; /* count of reset */
u32 page_alloc_fail; /* count of page alloc fails */
u32 dma_mapping_error; /* count of dma mapping errors */
u32 stats_report_trigger_cnt; /* count of device-requested stats-reports since last reset */
u32 suspend_cnt; /* count of times suspended */
u32 resume_cnt; /* count of times resumed */ struct workqueue_struct *gve_wq; struct work_struct service_task; struct work_struct stats_report_task; unsignedlong service_task_flags; unsignedlong state_flags;
struct gve_stats_report *stats_report;
u64 stats_report_len;
dma_addr_t stats_report_bus; /* dma address for the stats report */ unsignedlong ethtool_flags;
u16 header_buf_size; /* device configured, header-split supported if non-zero */ bool header_split_enabled; /* True if the header split is enabled by the user */
/* Returns the address of the ntfy_blocks irq doorbell
*/ staticinline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv, struct gve_notify_block *block)
{ return &priv->db_bar2[be32_to_cpu(*block->irq_db_index)];
}
/* Returns the index into ntfy_blocks of the given tx ring's block
*/ staticinline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
{ return queue_idx;
}
/* Returns the index into ntfy_blocks of the given rx ring's block
*/ staticinline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
{ return (priv->num_ntfy_blks / 2) + queue_idx;
}
staticinline u32 gve_get_rx_pages_per_qpl_dqo(u32 rx_desc_cnt)
{ /* For DQO, page count should be more than ring size for * out-of-order completions. Set it to two times of ring size.
*/ return 2 * rx_desc_cnt;
}
/* Returns the correct dma direction for tx and rx qpls */ staticinlineenum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv, int id)
{ if (id < gve_rx_start_qpl_id(&priv->tx_cfg)) return DMA_TO_DEVICE; else return DMA_FROM_DEVICE;
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
