/
phprus
/
github_dpdk
Обзор
Документация
Войти
/
phprus
/
github_dpdk
Код
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
main
drivers/net/intel/common/tx.h
459 строк
15 KB
Bruce Richardson
net/intel: optimize for fast-free hint
05 июн 2026, 17:06
05 июн 2026, 17:06
bfe0249
Код
Авторство
О чём код?
/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2024 Intel Corporation */ #ifndef _COMMON_INTEL_TX_H_ #define _COMMON_INTEL_TX_H_ #include <stdint.h> #include <rte_mbuf.h> #include <rte_ethdev.h> #include <rte_vect.h> /* Common TX Descriptor QW1 Field Definitions */ #define CI_TXD_QW1_DTYPE_S 0 #define CI_TXD_QW1_DTYPE_M (0xFUL << CI_TXD_QW1_DTYPE_S) #define CI_TXD_QW1_CMD_S 4 #define CI_TXD_QW1_CMD_M (0xFFFUL << CI_TXD_QW1_CMD_S) #define CI_TXD_QW1_OFFSET_S 16 #define CI_TXD_QW1_OFFSET_M (0x3FFFFULL << CI_TXD_QW1_OFFSET_S) #define CI_TXD_QW1_TX_BUF_SZ_S 34 #define CI_TXD_QW1_TX_BUF_SZ_M (0x3FFFULL << CI_TXD_QW1_TX_BUF_SZ_S) #define CI_TXD_QW1_L2TAG1_S 48 #define CI_TXD_QW1_L2TAG1_M (0xFFFFULL << CI_TXD_QW1_L2TAG1_S) /* Common Descriptor Types */ #define CI_TX_DESC_DTYPE_DATA 0x0 #define CI_TX_DESC_DTYPE_CTX 0x1 #define CI_TX_DESC_DTYPE_DESC_DONE 0xF /* Common TX Descriptor Command Flags */ #define CI_TX_DESC_CMD_EOP 0x0001 #define CI_TX_DESC_CMD_RS 0x0002 #define CI_TX_DESC_CMD_ICRC 0x0004 #define CI_TX_DESC_CMD_IL2TAG1 0x0008 #define CI_TX_DESC_CMD_DUMMY 0x0010 #define CI_TX_DESC_CMD_IIPT_IPV6 0x0020 #define CI_TX_DESC_CMD_IIPT_IPV4 0x0040 #define CI_TX_DESC_CMD_IIPT_IPV4_CSUM 0x0060 #define CI_TX_DESC_CMD_L4T_EOFT_TCP 0x0100 #define CI_TX_DESC_CMD_L4T_EOFT_SCTP 0x0200 #define CI_TX_DESC_CMD_L4T_EOFT_UDP 0x0300 /* Common TX Context Descriptor Commands */ #define CI_TX_CTX_DESC_TSO 0x01 #define CI_TX_CTX_DESC_TSYN 0x02 #define CI_TX_CTX_DESC_IL2TAG2 0x04 /** * L2TAG1 Field Source Selection * Specifies which mbuf VLAN field to use for the L2TAG1 field in data descriptors. * Context descriptor VLAN handling (L2TAG2) is managed by driver-specific callbacks. */ enum ci_tx_l2tag1_field { /** For VLAN (not QinQ), use L2Tag1 field in data desc */ CI_VLAN_IN_L2TAG1, /** For VLAN (not QinQ), use L2Tag2 field in ctx desc. * NOTE: When set, drivers must set the VLAN tag in the context * descriptor callback function, rather than relying on the * common Tx code to insert it. */ CI_VLAN_IN_L2TAG2, }; /* Common TX Descriptor Length Field Shifts */ #define CI_TX_DESC_LEN_MACLEN_S 0 /* 7 BITS */ #define CI_TX_DESC_LEN_IPLEN_S 7 /* 7 BITS */ #define CI_TX_DESC_LEN_L4_LEN_S 14 /* 4 BITS */ /* Common maximum data per TX descriptor */ #define CI_MAX_DATA_PER_TXD (CI_TXD_QW1_TX_BUF_SZ_M >> CI_TXD_QW1_TX_BUF_SZ_S) /* Common TX maximum burst size for chunked transmission in simple paths */ #define CI_TX_MAX_BURST 32 /* Common TX maximum free buffer size for batched bulk freeing */ #define CI_TX_MAX_FREE_BUF_SZ 64 /* Common TX descriptor command flags for simple transmit */ #define CI_TX_DESC_CMD_DEFAULT (CI_TX_DESC_CMD_ICRC | CI_TX_DESC_CMD_EOP) /* Checksum offload mask to identify packets requesting offload */ #define CI_TX_CKSUM_OFFLOAD_MASK (RTE_MBUF_F_TX_IP_CKSUM | \ RTE_MBUF_F_TX_L4_MASK | \ RTE_MBUF_F_TX_TCP_SEG | \ RTE_MBUF_F_TX_UDP_SEG | \ RTE_MBUF_F_TX_OUTER_IP_CKSUM | \ RTE_MBUF_F_TX_OUTER_UDP_CKSUM) /** * Common TX offload union for Intel drivers. * Supports both basic offloads (l2_len, l3_len, l4_len, tso_segsz) and * extended offloads (outer_l2_len, outer_l3_len) for tunneling support. */ union ci_tx_offload { uint64_t data; struct { uint64_t l2_len:7; /**< L2 (MAC) Header Length. */ uint64_t l3_len:9; /**< L3 (IP) Header Length. */ uint64_t l4_len:8; /**< L4 Header Length. */ uint64_t tso_segsz:16; /**< TCP TSO segment size */ uint64_t outer_l2_len:8; /**< outer L2 Header Length */ uint64_t outer_l3_len:16; /**< outer L3 Header Length */ }; }; /* * Structure of a 16-byte Tx descriptor common across i40e, ice, iavf and idpf drivers */ struct ci_tx_desc { uint64_t buffer_addr; /* Address of descriptor's data buf */ uint64_t cmd_type_offset_bsz; }; /* forward declaration of the common intel (ci) queue structure */ struct ci_tx_queue; /** * Structure associated with each descriptor of the TX ring of a TX queue. */ struct ci_tx_entry { struct rte_mbuf *mbuf; /* mbuf associated with TX desc, if any. */ uint16_t next_id; /* Index of next descriptor in ring. */ uint16_t first_id; /* Split-queue: first sw_id of packet at EOP entry. */ }; /** * Structure associated with each descriptor of the TX ring of a TX queue in vector Tx. */ struct ci_tx_entry_vec { struct rte_mbuf *mbuf; /* mbuf associated with TX desc, if any. */ }; typedef void (*ice_tx_release_mbufs_t)(struct ci_tx_queue *txq); struct ci_tx_queue { union { /* TX ring virtual address */ volatile struct ci_tx_desc *ci_tx_ring; volatile union ixgbe_adv_tx_desc *ixgbe_tx_ring; }; volatile uint8_t *qtx_tail; /* register address of tail */ union { struct ci_tx_entry *sw_ring; /* virtual address of SW ring */ struct ci_tx_entry_vec *sw_ring_vec; }; /* Scalar TX path: Array tracking last_id at each RS threshold boundary */ uint16_t *rs_last_id; uint16_t nb_tx_desc; /* number of TX descriptors */ uint16_t tx_tail; /* current value of tail register */ /* index to last TX descriptor to have been cleaned */ uint16_t last_desc_cleaned; /* Total number of TX descriptors ready to be allocated. */ uint16_t nb_tx_free; /* Start freeing TX buffers if there are less free descriptors than * this value. */ uint16_t tx_free_thresh; /* Number of TX descriptors to use before RS bit is set. */ uint16_t tx_rs_thresh; /* Scalar TX path: log2 of tx_rs_thresh for efficient bit operations */ uint8_t log2_rs_thresh; uint16_t port_id; /* Device port identifier. */ uint16_t queue_id; /* TX queue index. */ uint16_t reg_idx; uint16_t tx_next_dd; uint16_t tx_next_rs; /* Mempool pointer for fast release of mbufs. * NULL if disabled, UINTPTR_MAX if enabled and not yet known. * Set at first use (if enabled and not yet known). */ struct rte_mempool *fast_free_mp; uint64_t offloads; uint64_t mbuf_errors; rte_iova_t tx_ring_dma; /* TX ring DMA address */ bool tx_deferred_start; /* don't start this queue in dev start */ bool q_set; /* indicate if tx queue has been configured */ bool use_vec_entry; /* use sw_ring_vec (true for vector and simple paths) */ union { /* the VSI this queue belongs to */ struct i40e_vsi *i40e_vsi; struct iavf_vsi *iavf_vsi; struct ice_vsi *ice_vsi; }; const struct rte_memzone *mz; union { struct { /* ICE driver specific values */ struct ice_txtime *tsq; /* Tx Time based queue */ uint32_t q_teid; /* TX schedule node id. */ }; struct { /* I40E driver specific values */ uint8_t dcb_tc; }; struct { /* iavf driver specific values */ uint16_t ipsec_crypto_pkt_md_offset; #define IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1 BIT(0) #define IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2 BIT(1) uint8_t vlan_flag; uint8_t tc; bool use_ctx; /* with ctx info, each pkt needs two descriptors */ uint8_t lldp_mode; /* ptype or dynfield */ }; struct { /* ixgbe specific values */ const struct ixgbe_txq_ops *ops; struct ixgbe_advctx_info *ctx_cache; uint8_t ctx_curr; uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold reg. */ uint8_t using_ipsec; /**< indicates that IPsec TX feature is in use */ uint8_t is_vf; /**< indicates that this is a VF queue */ uint8_t vf_ctx_initialized; /**< VF context descriptors initialized */ }; struct { /* idpf specific values */ struct idpf_flex_tx_sched_desc *desc_ring; struct idpf_complq *complq; uint32_t latch_idx; /* Tx timestamp latch index */ uint16_t sw_nb_desc; uint16_t sw_tail; uint16_t rs_compl_count; }; }; }; struct ci_tx_path_features { uint32_t tx_offloads; enum rte_vect_max_simd simd_width; bool simple_tx; bool ctx_desc; bool disabled; bool single_queue; }; struct ci_tx_path_info { eth_tx_burst_t pkt_burst; const char *info; struct ci_tx_path_features features; eth_tx_prep_t pkt_prep; }; static __rte_always_inline void ci_tx_backlog_entry(struct ci_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { for (uint16_t i = 0; i < (int)nb_pkts; ++i) txep[i].mbuf = tx_pkts[i]; } static __rte_always_inline void ci_tx_backlog_entry_vec(struct ci_tx_entry_vec *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { for (uint16_t i = 0; i < nb_pkts; ++i) txep[i].mbuf = tx_pkts[i]; } #define IETH_VPMD_TX_MAX_FREE_BUF 64 typedef int (*ci_desc_done_fn)(struct ci_tx_queue *txq, uint16_t idx); static __rte_always_inline int ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done, bool ctx_descs) { int nb_free = 0; struct rte_mbuf *free[IETH_VPMD_TX_MAX_FREE_BUF]; struct rte_mbuf *m; /* check DD bits on threshold descriptor */ if (!desc_done(txq, txq->tx_next_dd)) return 0; const uint32_t n = txq->tx_rs_thresh >> ctx_descs; /* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh - 1) */ struct ci_tx_entry_vec *txep = txq->sw_ring_vec; txep += (txq->tx_next_dd >> ctx_descs) - (n - 1); /* is fast-free enabled? */ struct rte_mempool *mp = likely(txq->fast_free_mp != (void *)UINTPTR_MAX) ? txq->fast_free_mp : (txq->fast_free_mp = txep[0].mbuf->pool); if (mp != NULL && (n & 31) == 0) { static_assert(sizeof(*txep) == sizeof(struct rte_mbuf *), "txep array is not similar to an array of rte_mbuf pointers"); rte_mbuf_raw_free_bulk(mp, (void *)txep, n); goto done; } m = rte_pktmbuf_prefree_seg(txep[0].mbuf); if (likely(m)) { free[0] = m; nb_free = 1; for (uint32_t i = 1; i < n; i++) { m = rte_pktmbuf_prefree_seg(txep[i].mbuf); if (likely(m)) { if (likely(m->pool == free[0]->pool)) { free[nb_free++] = m; } else { rte_mbuf_raw_free_bulk(free[0]->pool, free, nb_free); free[0] = m; nb_free = 1; } } } rte_mbuf_raw_free_bulk(free[0]->pool, free, nb_free); } else { for (uint32_t i = 1; i < n; i++) { m = rte_pktmbuf_prefree_seg(txep[i].mbuf); if (m) rte_mempool_put(m->pool, m); } } done: /* buffers were freed, update counters */ txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); if (txq->tx_next_dd >= txq->nb_tx_desc) txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); return txq->tx_rs_thresh; } static inline void ci_txq_release_all_mbufs(struct ci_tx_queue *txq, bool use_ctx) { if (unlikely(!txq || !txq->sw_ring)) return; if (!txq->use_vec_entry) { /* Free mbufs from (last_desc_cleaned + 1) to (tx_tail - 1). */ const uint16_t start = (txq->last_desc_cleaned + 1) % txq->nb_tx_desc; const uint16_t nb_desc = txq->nb_tx_desc; const uint16_t end = txq->tx_tail; uint16_t i = start; if (end < i) { for (; i < nb_desc; i++) if (txq->sw_ring[i].mbuf != NULL) rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); i = 0; } for (; i < end; i++) if (txq->sw_ring[i].mbuf != NULL) rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); memset(txq->sw_ring, 0, sizeof(txq->sw_ring[0]) * nb_desc); return; } /** * Vector and simple paths use sw_ring_vec (ci_tx_entry_vec). * vPMD tx will not set sw_ring's mbuf to NULL after free, * so determining buffers to free is a little more complex. */ const uint16_t start = (txq->tx_next_dd - txq->tx_rs_thresh + 1) >> use_ctx; const uint16_t nb_desc = txq->nb_tx_desc >> use_ctx; const uint16_t end = txq->tx_tail >> use_ctx; uint16_t i = start; if (end < i) { for (; i < nb_desc; i++) rte_pktmbuf_free_seg(txq->sw_ring_vec[i].mbuf); i = 0; } for (; i < end; i++) rte_pktmbuf_free_seg(txq->sw_ring_vec[i].mbuf); memset(txq->sw_ring_vec, 0, sizeof(txq->sw_ring_vec[0]) * nb_desc); } /** * Select the best matching Tx path based on features * * @param req_features * The requested features for the Tx path * @param infos * Array of information about the available Tx paths * @param num_paths * Number of available paths in the infos array * @param default_path * Index of the default path to use if no suitable path is found * * @return * The packet burst function index that best matches the requested features, * or default_path if no suitable path is found */ static inline int ci_tx_path_select(const struct ci_tx_path_features *req_features, const struct ci_tx_path_info *infos, size_t num_paths, int default_path) { int idx = default_path; const struct ci_tx_path_features *chosen_path_features = NULL; for (unsigned int i = 0; i < num_paths; i++) { const struct ci_tx_path_features *path_features = &infos[i].features; /* Do not select a path with a NULL pkt_burst function. */ if (infos[i].pkt_burst == NULL) continue; /* Do not select a disabled tx path. */ if (path_features->disabled) continue; /* Do not use a simple tx path if not requested. */ if (path_features->simple_tx && !req_features->simple_tx) continue; /* If a context descriptor is requested, ensure the path supports it. */ if (!path_features->ctx_desc && req_features->ctx_desc) continue; /* If requested, ensure the path supports single queue TX. */ if (path_features->single_queue != req_features->single_queue) continue; /* Ensure the path supports the requested TX offloads. */ if ((path_features->tx_offloads & req_features->tx_offloads) != req_features->tx_offloads) continue; /* Ensure the path's SIMD width is compatible with the requested width. */ if (path_features->simd_width > req_features->simd_width) continue; /* Do not select the path if it is less suitable than the chosen path. */ if (chosen_path_features != NULL) { /* Do not select paths with lower SIMD width than the chosen path. */ if (path_features->simd_width < chosen_path_features->simd_width) continue; /* Do not select paths with more offloads enabled than the chosen path if * the SIMD widths are the same. */ if (path_features->simd_width == chosen_path_features->simd_width && rte_popcount32(path_features->tx_offloads) > rte_popcount32(chosen_path_features->tx_offloads)) continue; /* Don't use a context descriptor unless necessary */ if (path_features->ctx_desc && !chosen_path_features->ctx_desc) continue; } /* Finally, select the path since it has met all the requirements. */ idx = i; chosen_path_features = &infos[idx].features; } return idx; } /* include the scalar functions at the end, so they can use the common definitions. * This is done so drivers can use all functions just by including tx.h */ #include "tx_scalar.h" #endif /* _COMMON_INTEL_TX_H_ */