80 FatalError(
"Error creating thread %s: you do not have "
81 "support for DPDK enabled, on Linux host please recompile "
93 static struct timeval machine_start_time = { 0, 0 };
98 typedef struct DPDKThreadVars_ {
106 uint16_t capture_dpdk_packets;
107 uint16_t capture_dpdk_rx_errs;
108 uint16_t capture_dpdk_imissed;
109 uint16_t capture_dpdk_rx_no_mbufs;
110 uint16_t capture_dpdk_ierrors;
111 uint16_t capture_dpdk_tx_errs;
116 uint16_t out_port_id;
124 struct rte_mempool *pkt_mempool;
125 struct rte_mbuf *received_mbufs[BURST_SIZE];
129 static void ReceiveDPDKThreadExitStats(
ThreadVars *,
void *);
137 static uint64_t CyclesToMicroseconds(uint64_t cycles);
138 static uint64_t CyclesToSeconds(uint64_t cycles);
139 static void DPDKFreeMbufArray(
struct rte_mbuf **mbuf_array, uint16_t mbuf_cnt, uint16_t
offset);
140 static uint64_t DPDKGetSeconds(
void);
142 static void DPDKFreeMbufArray(
struct rte_mbuf **mbuf_array, uint16_t mbuf_cnt, uint16_t
offset)
144 for (
int i =
offset; i < mbuf_cnt; i++) {
145 rte_pktmbuf_free(mbuf_array[i]);
149 static uint64_t CyclesToMicroseconds(
const uint64_t cycles)
151 const uint64_t ticks_per_us = rte_get_tsc_hz() / 1000000;
152 return cycles / ticks_per_us;
155 static uint64_t CyclesToSeconds(
const uint64_t cycles)
157 const uint64_t ticks_per_s = rte_get_tsc_hz();
158 return cycles / ticks_per_s;
161 static void CyclesAddToTimeval(
162 const uint64_t cycles,
struct timeval *orig_tv,
struct timeval *new_tv)
164 uint64_t usec = CyclesToMicroseconds(cycles) + orig_tv->tv_usec;
165 new_tv->tv_sec = orig_tv->tv_sec + usec / 1000000;
166 new_tv->tv_usec = (usec % 1000000);
171 gettimeofday(&machine_start_time, NULL);
172 machine_start_time.tv_sec -= DPDKGetSeconds();
181 static SCTime_t DPDKSetTimevalReal(
struct timeval *machine_start_tv)
183 struct timeval real_tv;
184 CyclesAddToTimeval(rte_get_tsc_cycles(), machine_start_tv, &real_tv);
189 static uint64_t DPDKGetSeconds(
void)
191 return CyclesToSeconds(rte_get_tsc_cycles());
194 static void DevicePostStartPMDSpecificActions(DPDKThreadVars *ptv,
const char *driver_name)
198 if (strcmp(driver_name,
"net_i40e") == 0)
199 i40eDeviceSetRSS(ptv->port_id, ptv->threads);
202 static void DevicePreStopPMDSpecificActions(DPDKThreadVars *ptv,
const char *driver_name)
206 if (strcmp(driver_name,
"net_i40e") == 0) {
208 struct rte_flow_error flush_error = { 0 };
209 retval = rte_flow_flush(ptv->port_id, &flush_error);
211 SCLogError(
"Unable to flush rte_flow rules: %s Flush error msg: %s",
212 rte_strerror(-retval), flush_error.message);
221 static int GetNumaNode(
void)
226 #if defined(__linux__)
227 cpu = sched_getcpu();
228 node = numa_node_of_cpu(cpu);
230 SCLogWarning(
"NUMA node retrieval is not supported on this OS.");
268 static inline void DPDKDumpCounters(DPDKThreadVars *ptv)
270 struct rte_eth_stats eth_stats;
271 int retval = rte_eth_stats_get(ptv->port_id, ð_stats);
273 SCLogError(
"Failed to get stats for port id %d: %s", ptv->port_id, rte_strerror(-retval));
280 if (ptv->queue_id == 0) {
282 ptv->pkts + eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
284 eth_stats.ipackets + eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
286 eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
287 StatsSetUI64(ptv->tv, ptv->capture_dpdk_imissed, eth_stats.imissed);
288 StatsSetUI64(ptv->tv, ptv->capture_dpdk_rx_no_mbufs, eth_stats.rx_nombuf);
289 StatsSetUI64(ptv->tv, ptv->capture_dpdk_ierrors, eth_stats.ierrors);
290 StatsSetUI64(ptv->tv, ptv->capture_dpdk_tx_errs, eth_stats.oerrors);
292 ptv->livedev->drop, eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
294 StatsSetUI64(ptv->tv, ptv->capture_dpdk_packets, ptv->pkts);
298 static void DPDKReleasePacket(
Packet *p)
307 #
if defined(RTE_LIBRTE_I40E_PMD) || defined(RTE_LIBRTE_IXGBE_PMD) || defined(RTE_LIBRTE_ICE_PMD)
313 rte_eth_tx_burst(p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id, &p->dpdk_v.mbuf, 1);
320 retval = rte_eth_tx_burst(
321 p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id, &p->dpdk_v.mbuf, 1);
323 SCLogDebug(
"Unable to transmit the packet on port %u queue %u",
324 p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id);
325 rte_pktmbuf_free(p->dpdk_v.mbuf);
326 p->dpdk_v.mbuf = NULL;
330 rte_pktmbuf_free(p->dpdk_v.mbuf);
331 p->dpdk_v.mbuf = NULL;
345 time_t last_dump = 0;
348 DPDKThreadVars *ptv = (DPDKThreadVars *)data;
361 DPDKDumpCounters(ptv);
365 nb_rx = rte_eth_rx_burst(ptv->port_id, ptv->queue_id, ptv->received_mbufs, BURST_SIZE);
370 ptv->pkts += (uint64_t)nb_rx;
371 for (uint16_t i = 0; i < nb_rx; i++) {
382 p->
ts = DPDKSetTimevalReal(&machine_start_time);
383 p->dpdk_v.mbuf = ptv->received_mbufs[i];
385 p->dpdk_v.copy_mode = ptv->copy_mode;
386 p->dpdk_v.out_port_id = ptv->out_port_id;
387 p->dpdk_v.out_queue_id = ptv->queue_id;
393 uint64_t ol_flags = ptv->received_mbufs[i]->ol_flags;
394 if ((ol_flags & RTE_MBUF_F_RX_IP_CKSUM_MASK) == RTE_MBUF_F_RX_IP_CKSUM_GOOD &&
395 (ol_flags & RTE_MBUF_F_RX_L4_CKSUM_MASK) == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
396 SCLogDebug(
"HW detected GOOD IP and L4 chsum, ignoring validation");
399 if ((ol_flags & RTE_MBUF_F_RX_IP_CKSUM_MASK) == RTE_MBUF_F_RX_IP_CKSUM_BAD) {
404 if ((ol_flags & RTE_MBUF_F_RX_L4_CKSUM_MASK) == RTE_MBUF_F_RX_L4_CKSUM_BAD) {
411 PacketSetData(p, rte_pktmbuf_mtod(p->dpdk_v.mbuf, uint8_t *),
412 rte_pktmbuf_pkt_len(p->dpdk_v.mbuf));
413 if (TmThreadsSlotProcessPkt(ptv->tv, ptv->slot, p) !=
TM_ECODE_OK) {
415 DPDKFreeMbufArray(ptv->received_mbufs, nb_rx - i - 1, i + 1);
421 current_time = DPDKGetSeconds();
422 if (current_time != last_dump) {
423 DPDKDumpCounters(ptv);
424 last_dump = current_time;
443 int retval, thread_numa;
444 DPDKThreadVars *ptv = NULL;
447 if (initdata == NULL) {
448 SCLogError(
"DPDK configuration is NULL in thread initialization");
452 ptv =
SCCalloc(1,
sizeof(DPDKThreadVars));
470 ptv->copy_mode = dpdk_config->copy_mode;
471 ptv->checksum_mode = dpdk_config->checksum_mode;
473 ptv->threads = dpdk_config->threads;
474 ptv->port_id = dpdk_config->port_id;
475 ptv->out_port_id = dpdk_config->out_port_id;
477 ptv->pkt_mempool = dpdk_config->pkt_mempool;
478 dpdk_config->pkt_mempool = NULL;
480 thread_numa = GetNumaNode();
481 if (thread_numa >= 0 && thread_numa != rte_eth_dev_socket_id(ptv->port_id)) {
483 SCLogPerf(
"%s: NIC is on NUMA %d, thread on NUMA %d", dpdk_config->iface,
484 rte_eth_dev_socket_id(ptv->port_id), thread_numa);
488 ptv->queue_id = queue_id;
491 if (queue_id == dpdk_config->threads - 1) {
492 retval = rte_eth_dev_start(ptv->port_id);
494 SCLogError(
"Error (%s) during device startup of %s", rte_strerror(-retval),
499 struct rte_eth_dev_info dev_info;
500 retval = rte_eth_dev_info_get(ptv->port_id, &dev_info);
502 SCLogError(
"Error (%s) when getting device info of %s", rte_strerror(-retval),
508 DevicePostStartPMDSpecificActions(ptv, dev_info.driver_name);
510 uint16_t inconsist_numa_cnt =
SC_ATOMIC_GET(dpdk_config->inconsitent_numa_cnt);
511 if (inconsist_numa_cnt > 0) {
512 SCLogWarning(
"%s: NIC is on NUMA %d, %u threads on different NUMA node(s)",
513 dpdk_config->iface, rte_eth_dev_socket_id(ptv->port_id), inconsist_numa_cnt);
518 dpdk_config->DerefFunc(dpdk_config);
522 if (dpdk_config != NULL)
523 dpdk_config->DerefFunc(dpdk_config);
529 static void PrintDPDKPortXstats(uint32_t port_id,
const char *port_name)
531 struct rte_eth_xstat *xstats;
532 struct rte_eth_xstat_name *xstats_names;
534 int32_t
len = rte_eth_xstats_get(port_id, NULL, 0);
536 FatalError(
"Error (%s) getting count of rte_eth_xstats failed on port %s",
537 rte_strerror(-
len), port_name);
541 FatalError(
"Failed to allocate memory for the rte_eth_xstat structure");
543 int32_t ret = rte_eth_xstats_get(port_id, xstats,
len);
544 if (ret < 0 || ret >
len) {
546 FatalError(
"Error (%s) getting rte_eth_xstats failed on port %s", rte_strerror(-ret),
549 xstats_names =
SCCalloc(
len,
sizeof(*xstats_names));
550 if (xstats_names == NULL) {
552 FatalError(
"Failed to allocate memory for the rte_eth_xstat_name array");
554 ret = rte_eth_xstats_get_names(port_id, xstats_names,
len);
555 if (ret < 0 || ret >
len) {
558 FatalError(
"Error (%s) getting names of rte_eth_xstats failed on port %s",
559 rte_strerror(-ret), port_name);
561 for (int32_t i = 0; i <
len; i++) {
562 if (xstats[i].value > 0)
563 SCLogPerf(
"Port %u (%s) - %s: %" PRIu64, port_id, port_name, xstats_names[i].name,
576 static void ReceiveDPDKThreadExitStats(
ThreadVars *
tv,
void *data)
580 DPDKThreadVars *ptv = (DPDKThreadVars *)data;
582 if (ptv->queue_id == 0) {
583 struct rte_eth_stats eth_stats;
584 char port_name[RTE_ETH_NAME_MAX_LEN];
586 retval = rte_eth_dev_get_name_by_port(ptv->port_id, port_name);
588 SCLogError(
"Failed to convert port id %d to the interface name: %s", ptv->port_id,
593 PrintDPDKPortXstats(ptv->port_id, port_name);
595 retval = rte_eth_stats_get(ptv->port_id, ð_stats);
597 SCLogError(
"Failed to get stats for interface %s: %s", port_name, strerror(-retval));
600 SCLogPerf(
"Total RX stats of %s: packets %" PRIu64
" bytes: %" PRIu64
" missed: %" PRIu64
601 " errors: %" PRIu64
" nombufs: %" PRIu64,
602 port_name, eth_stats.ipackets, eth_stats.ibytes, eth_stats.imissed,
603 eth_stats.ierrors, eth_stats.rx_nombuf);
605 SCLogPerf(
"Total TX stats of %s: packets %" PRIu64
" bytes: %" PRIu64
607 port_name, eth_stats.opackets, eth_stats.obytes, eth_stats.oerrors);
610 DPDKDumpCounters(ptv);
622 DPDKThreadVars *ptv = (DPDKThreadVars *)data;
625 if (ptv->queue_id == 0) {
626 struct rte_eth_dev_info dev_info;
627 char iface[RTE_ETH_NAME_MAX_LEN];
628 retval = rte_eth_dev_get_name_by_port(ptv->port_id, iface);
630 SCLogError(
"Error (err=%d) when getting device name (port %d)", retval, ptv->port_id);
633 retval = rte_eth_dev_info_get(ptv->port_id, &dev_info);
635 SCLogError(
"Error (err=%d) during getting device info (port %s)", retval, iface);
639 DevicePreStopPMDSpecificActions(ptv, dev_info.driver_name);
642 rte_eth_dev_stop(ptv->port_id);
644 rte_eth_dev_stop(ptv->out_port_id);
647 if (ptv->queue_id == 0 && ptv->pkt_mempool != NULL) {
648 rte_mempool_free(ptv->pkt_mempool);
649 ptv->pkt_mempool = NULL;