78 "Error creating thread %s: you do not have "
79 "support for DPDK enabled, on Linux host please recompile "
95 typedef struct DPDKThreadVars_ {
103 uint16_t capture_dpdk_packets;
104 uint16_t capture_dpdk_rx_errs;
105 uint16_t capture_dpdk_imissed;
106 uint16_t capture_dpdk_rx_no_mbufs;
107 uint16_t capture_dpdk_ierrors;
108 uint16_t capture_dpdk_tx_errs;
113 uint16_t out_port_id;
121 struct rte_mempool *pkt_mempool;
122 struct rte_mbuf *received_mbufs[BURST_SIZE];
123 struct timeval machine_start_time;
127 static void ReceiveDPDKThreadExitStats(
ThreadVars *,
void *);
135 static uint64_t CyclesToMicroseconds(uint64_t cycles);
136 static uint64_t CyclesToSeconds(uint64_t cycles);
137 static void DPDKFreeMbufArray(
struct rte_mbuf **mbuf_array, uint16_t mbuf_cnt, uint16_t
offset);
138 static uint64_t DPDKGetSeconds(
void);
140 static void DPDKFreeMbufArray(
struct rte_mbuf **mbuf_array, uint16_t mbuf_cnt, uint16_t
offset)
142 for (
int i =
offset; i < mbuf_cnt; i++) {
143 rte_pktmbuf_free(mbuf_array[i]);
147 static uint64_t CyclesToMicroseconds(
const uint64_t cycles)
149 const uint64_t ticks_per_us = rte_get_tsc_hz() / 1000000;
150 return cycles / ticks_per_us;
153 static uint64_t CyclesToSeconds(
const uint64_t cycles)
155 const uint64_t ticks_per_s = rte_get_tsc_hz();
156 return cycles / ticks_per_s;
159 static void CyclesAddToTimeval(
160 const uint64_t cycles,
struct timeval *orig_tv,
struct timeval *new_tv)
162 uint64_t usec = CyclesToMicroseconds(cycles) + orig_tv->tv_usec;
163 new_tv->tv_sec = orig_tv->tv_sec + usec / 1000000;
164 new_tv->tv_usec = (usec % 1000000);
167 static void DPDKSetTimevalOfMachineStart(
struct timeval *
tv)
169 gettimeofday(
tv, NULL);
170 tv->tv_sec -= DPDKGetSeconds();
179 static void DPDKSetTimevalReal(
struct timeval *machine_start_tv,
struct timeval *real_tv)
181 CyclesAddToTimeval(rte_get_tsc_cycles(), machine_start_tv, real_tv);
185 static uint64_t DPDKGetSeconds()
187 return CyclesToSeconds(rte_get_tsc_cycles());
190 static void DevicePostStartPMDSpecificActions(DPDKThreadVars *ptv,
const char *driver_name)
194 if (strcmp(driver_name,
"net_i40e") == 0)
195 i40eDeviceSetRSS(ptv->port_id, ptv->threads);
198 static void DevicePreStopPMDSpecificActions(DPDKThreadVars *ptv,
const char *driver_name)
202 if (strcmp(driver_name,
"net_i40e") == 0) {
204 struct rte_flow_error flush_error = { 0 };
205 retval = rte_flow_flush(ptv->port_id, &flush_error);
208 rte_strerror(-retval), flush_error.message);
217 static int GetNumaNode(
void)
222 #if defined(__linux__)
223 cpu = sched_getcpu();
224 node = numa_node_of_cpu(cpu);
264 static inline void DPDKDumpCounters(DPDKThreadVars *ptv)
266 struct rte_eth_stats eth_stats;
267 int retval = rte_eth_stats_get(ptv->port_id, ð_stats);
270 rte_strerror(-retval));
277 if (ptv->queue_id == 0) {
279 ptv->pkts + eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
281 eth_stats.ipackets + eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
283 eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
284 StatsSetUI64(ptv->tv, ptv->capture_dpdk_imissed, eth_stats.imissed);
285 StatsSetUI64(ptv->tv, ptv->capture_dpdk_rx_no_mbufs, eth_stats.rx_nombuf);
286 StatsSetUI64(ptv->tv, ptv->capture_dpdk_ierrors, eth_stats.ierrors);
287 StatsSetUI64(ptv->tv, ptv->capture_dpdk_tx_errs, eth_stats.oerrors);
289 ptv->livedev->drop, eth_stats.imissed + eth_stats.ierrors + eth_stats.rx_nombuf);
291 StatsSetUI64(ptv->tv, ptv->capture_dpdk_packets, ptv->pkts);
295 static void DPDKReleasePacket(
Packet *p)
304 #
if defined(RTE_LIBRTE_I40E_PMD) || defined(RTE_LIBRTE_IXGBE_PMD) || defined(RTE_LIBRTE_ICE_PMD)
310 rte_eth_tx_burst(p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id, &p->dpdk_v.mbuf, 1);
317 retval = rte_eth_tx_burst(
318 p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id, &p->dpdk_v.mbuf, 1);
320 SCLogDebug(
"Unable to transmit the packet on port %u queue %u",
321 p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id);
322 rte_pktmbuf_free(p->dpdk_v.mbuf);
323 p->dpdk_v.mbuf = NULL;
327 rte_pktmbuf_free(p->dpdk_v.mbuf);
328 p->dpdk_v.mbuf = NULL;
342 time_t last_dump = 0;
345 DPDKThreadVars *ptv = (DPDKThreadVars *)data;
354 DPDKDumpCounters(ptv);
358 nb_rx = rte_eth_rx_burst(ptv->port_id, ptv->queue_id, ptv->received_mbufs, BURST_SIZE);
363 ptv->pkts += (uint64_t)nb_rx;
364 for (uint16_t i = 0; i < nb_rx; i++) {
375 DPDKSetTimevalReal(&ptv->machine_start_time, &p->
ts);
376 p->dpdk_v.mbuf = ptv->received_mbufs[i];
378 p->dpdk_v.copy_mode = ptv->copy_mode;
379 p->dpdk_v.out_port_id = ptv->out_port_id;
380 p->dpdk_v.out_queue_id = ptv->queue_id;
382 PacketSetData(p, rte_pktmbuf_mtod(p->dpdk_v.mbuf, uint8_t *),
383 rte_pktmbuf_pkt_len(p->dpdk_v.mbuf));
384 if (TmThreadsSlotProcessPkt(ptv->tv, ptv->slot, p) !=
TM_ECODE_OK) {
386 DPDKFreeMbufArray(ptv->received_mbufs, nb_rx - i - 1, i + 1);
392 current_time = DPDKGetSeconds();
393 if (current_time != last_dump) {
394 DPDKDumpCounters(ptv);
395 last_dump = current_time;
414 int retval, thread_numa;
415 DPDKThreadVars *ptv = NULL;
418 if (initdata == NULL) {
423 ptv =
SCCalloc(1,
sizeof(DPDKThreadVars));
433 DPDKSetTimevalOfMachineStart(&ptv->machine_start_time);
442 ptv->copy_mode = dpdk_config->copy_mode;
443 ptv->checksum_mode = dpdk_config->checksum_mode;
445 ptv->threads = dpdk_config->threads;
446 ptv->port_id = dpdk_config->port_id;
447 ptv->out_port_id = dpdk_config->out_port_id;
449 ptv->queue_id = queue_id;
451 ptv->pkt_mempool = dpdk_config->pkt_mempool;
452 dpdk_config->pkt_mempool = NULL;
455 if (queue_id == dpdk_config->threads - 1) {
456 retval = rte_eth_dev_start(ptv->port_id);
459 rte_strerror(-retval), dpdk_config->iface);
463 struct rte_eth_dev_info dev_info;
464 retval = rte_eth_dev_info_get(ptv->port_id, &dev_info);
467 rte_strerror(-retval), dpdk_config->iface);
472 DevicePostStartPMDSpecificActions(ptv, dev_info.driver_name);
475 thread_numa = GetNumaNode();
476 if (thread_numa >= 0 && thread_numa != rte_eth_dev_socket_id(ptv->port_id)) {
478 "NIC on NUMA %d but thread on NUMA %d. Decreased performance expected",
479 rte_eth_dev_socket_id(ptv->port_id), thread_numa);
483 dpdk_config->DerefFunc(dpdk_config);
487 if (dpdk_config != NULL)
488 dpdk_config->DerefFunc(dpdk_config);
499 static void ReceiveDPDKThreadExitStats(
ThreadVars *
tv,
void *data)
503 DPDKThreadVars *ptv = (DPDKThreadVars *)data;
505 if (ptv->queue_id == 0) {
506 struct rte_eth_stats eth_stats;
507 char port_name[RTE_ETH_NAME_MAX_LEN];
509 retval = rte_eth_dev_get_name_by_port(ptv->port_id, port_name);
512 ptv->port_id, strerror(-retval));
515 retval = rte_eth_stats_get(ptv->port_id, ð_stats);
521 SCLogPerf(
"Total RX stats of %s: packets %" PRIu64
" bytes: %" PRIu64
" missed: %" PRIu64
522 " errors: %" PRIu64
" nombufs: %" PRIu64,
523 port_name, eth_stats.ipackets, eth_stats.ibytes, eth_stats.imissed,
524 eth_stats.ierrors, eth_stats.rx_nombuf);
526 SCLogPerf(
"Total TX stats of %s: packets %" PRIu64
" bytes: %" PRIu64
528 port_name, eth_stats.opackets, eth_stats.obytes, eth_stats.oerrors);
531 DPDKDumpCounters(ptv);
543 DPDKThreadVars *ptv = (DPDKThreadVars *)data;
546 if (ptv->queue_id == 0) {
547 struct rte_eth_dev_info dev_info;
548 char iface[RTE_ETH_NAME_MAX_LEN];
549 retval = rte_eth_dev_get_name_by_port(ptv->port_id, iface);
552 retval, ptv->port_id);
555 retval = rte_eth_dev_info_get(ptv->port_id, &dev_info);
562 DevicePreStopPMDSpecificActions(ptv, dev_info.driver_name);
565 rte_eth_dev_stop(ptv->port_id);
567 rte_eth_dev_stop(ptv->out_port_id);
570 if (ptv->queue_id == 0 && ptv->pkt_mempool != NULL) {
571 rte_mempool_free(ptv->pkt_mempool);
572 ptv->pkt_mempool = NULL;