33 #define SC_PCAP_DONT_INCLUDE_PCAP_H 1
67 #include <sys/ioctl.h>
70 #if HAVE_LINUX_SOCKIOS_H
71 #include <linux/sockios.h>
74 #ifdef HAVE_PACKET_EBPF
75 #define PCAP_DONT_INCLUDE_PCAP_BPF_H 1
76 #include <bpf/libbpf.h>
81 struct bpf_insn *bf_insns;
89 #ifdef HAVE_PCAP_PCAP_H
90 #include <pcap/pcap.h>
95 #if HAVE_LINUX_IF_ETHER_H
96 #include <linux/if_ether.h>
99 #if HAVE_LINUX_IF_PACKET_H
100 #include <linux/if_packet.h>
103 #if HAVE_LINUX_IF_ARP_H
104 #include <linux/if_arp.h>
107 #if HAVE_LINUX_FILTER_H
108 #include <linux/filter.h>
112 #include <sys/mman.h>
115 #ifdef HAVE_HW_TIMESTAMPING
116 #include <linux/net_tstamp.h>
123 #ifndef HAVE_AF_PACKET
157 SCLogError(
"Error creating thread %s: you do not have "
158 "support for AF_PACKET enabled, on Linux host please recompile "
159 "with --enable-af-packet",
166 #define AFP_IFACE_NAME_LENGTH 48
168 #define AFP_STATE_DOWN 0
169 #define AFP_STATE_UP 1
171 #define AFP_RECONNECT_TIMEOUT 500000
172 #define AFP_DOWN_COUNTER_INTERVAL 40
174 #define POLL_TIMEOUT 100
177 #ifndef TP_STATUS_KERNEL
178 #define TP_STATUS_KERNEL 0
180 #ifndef TP_STATUS_USER
181 #define TP_STATUS_USER BIT_U32(0)
183 #ifndef TP_STATUS_COPY
184 #define TP_STATUS_COPY BIT_U32(1)
186 #ifndef TP_STATUS_LOSING
187 #define TP_STATUS_LOSING BIT_U32(2)
189 #ifndef TP_STATUS_CSUMNOTREADY
190 #define TP_STATUS_CSUMNOTREADY BIT_U32(3)
192 #ifndef TP_STATUS_VLAN_VALID
193 #define TP_STATUS_VLAN_VALID BIT_U32(4)
195 #ifndef TP_STATUS_BLK_TMO
196 #define TP_STATUS_BLK_TMO BIT_U32(5)
198 #ifndef TP_STATUS_VLAN_TPID_VALID
199 #define TP_STATUS_VLAN_TPID_VALID BIT_U32(6)
201 #ifndef TP_STATUS_CSUM_VALID
202 #define TP_STATUS_CSUM_VALID BIT_U32(7)
205 #ifndef TP_STATUS_TS_SOFTWARE
206 #define TP_STATUS_TS_SOFTWARE BIT_U32(29)
208 #ifndef TP_STATUS_TS_SYS_HARDWARE
209 #define TP_STATUS_TS_SYS_HARDWARE BIT_U32(30)
211 #ifndef TP_STATUS_TS_RAW_HARDWARE
212 #define TP_STATUS_TS_RAW_HARDWARE BIT_U32(31)
215 #ifndef TP_STATUS_USER_BUSY
227 #define TP_STATUS_USER_BUSY \
228 (uint32_t)((uint32_t)TP_STATUS_TS_SOFTWARE | (uint32_t)TP_STATUS_TS_SYS_HARDWARE | \
229 (uint32_t)TP_STATUS_TS_RAW_HARDWARE)
231 #define FRAME_BUSY(tp_status) \
232 (((uint32_t)(tp_status) & (uint32_t)TP_STATUS_USER_BUSY) == (uint32_t)TP_STATUS_USER_BUSY)
248 struct tpacket2_hdr *
h2;
249 #ifdef HAVE_TPACKET_V3
250 struct tpacket3_hdr *h3;
255 #ifdef HAVE_PACKET_EBPF
256 static int AFPBypassCallback(
Packet *p);
257 static int AFPXDPBypassCallback(
Packet *p);
279 #ifdef HAVE_PACKET_EBPF
340 struct tpacket_req
v2;
341 #ifdef HAVE_TPACKET_V3
342 struct tpacket_req3 v3;
354 #ifdef HAVE_PACKET_EBPF
358 struct ebpf_timeout_config ebpf_t_config;
364 static void ReceiveAFPThreadExitStats(
ThreadVars *,
void *);
373 static int AFPGetIfnumByDev(
int fd,
const char *ifname,
int verbose);
374 static int AFPGetDevFlags(
int fd,
const char *ifname);
375 static int AFPDerefSocket(
AFPPeer* peer);
376 static int AFPRefSocket(
AFPPeer* peer);
427 if (ptv->
mpeer == NULL) {
438 static void AFPPeerClean(
AFPPeer *peer)
470 #define AFP_PEERS_MAX_TRY 4
471 #define AFP_PEERS_WAIT 20000
532 if (out_iface == NULL)
534 if (iface->
mtu != out_iface->
mtu) {
535 SCLogWarning(
"MTU on %s (%d) and %s (%d) are not equal, transmission of packets "
536 "bigger than %d will fail.",
550 static int AFPPeersListWaitTurn(
AFPPeer *peer)
561 static void AFPPeersListReachedInc(
void)
575 static int AFPPeersListStarted(
void)
612 static int AFPCreateSocket(
AFPThreadVars *ptv,
char *devname,
int verbose,
const bool peer_update);
616 #ifdef PACKET_STATISTICS
617 struct tpacket_stats kstats;
618 socklen_t
len =
sizeof (
struct tpacket_stats);
619 if (getsockopt(ptv->
socket, SOL_PACKET, PACKET_STATISTICS,
620 &kstats, &
len) > -1) {
621 SCLogDebug(
"(%s) Kernel: Packets %" PRIu32
", dropped %" PRIu32
"",
623 kstats.tp_packets, kstats.tp_drops);
651 struct sockaddr_ll socket_address;
660 if (!PacketIsEthernet(p)) {
665 const EthernetHdr *ethh = PacketGetEthernet(p);
667 socket_address.sll_ifindex =
SC_ATOMIC_GET(p->afp_v.peer->if_idx);
669 socket_address.sll_halen = ETH_ALEN;
671 memcpy(socket_address.sll_addr, ethh, 6);
679 sizeof(
struct sockaddr_ll)) < 0) {
681 SCLogWarning(
"%s: sending packet failed on socket %d: %s", p->afp_v.peer->iface, socket,
689 static void AFPReleaseDataFromRing(
Packet *p)
696 AFPWritePacket(p, TPACKET_V2);
699 BUG_ON(p->afp_v.relptr == NULL);
702 h.
raw = p->afp_v.relptr;
705 (void)AFPDerefSocket(p->afp_v.mpeer);
710 #ifdef HAVE_TPACKET_V3
711 static void AFPReleasePacketV3(
Packet *p)
718 AFPWritePacket(p, TPACKET_V3);
724 static void AFPReleasePacket(
Packet *p)
726 AFPReleaseDataFromRing(p);
745 #ifdef HAVE_PACKET_EBPF
748 p->afp_v.v4_map_fd = ptv->v4_map_fd;
749 p->afp_v.v6_map_fd = ptv->v6_map_fd;
750 p->afp_v.nr_cpus = ptv->ebpf_t_config.cpus_count;
754 p->afp_v.v4_map_fd = ptv->v4_map_fd;
755 p->afp_v.v6_map_fd = ptv->v6_map_fd;
756 p->afp_v.nr_cpus = ptv->ebpf_t_config.cpus_count;
764 static void AFPReadFromRingSetupPacket(
777 AFPReadApplyBypass(ptv, p);
779 if (h.
h2->tp_len > h.
h2->tp_snaplen) {
780 SCLogDebug(
"Packet length (%d) > snaplen (%d), truncating", h.
h2->tp_len, h.
h2->tp_snaplen);
786 p->
vlan_id[0] = h.
h2->tp_vlan_tci & 0x0fff;
788 p->afp_v.vlan_tci = h.
h2->tp_vlan_tci;
794 p->afp_v.relptr = h.
raw;
796 p->afp_v.mpeer = ptv->
mpeer;
797 AFPRefSocket(ptv->
mpeer);
799 p->afp_v.mpeer = NULL;
824 static inline int AFPReadFromRingWaitForPacket(
AFPThreadVars *ptv)
827 struct timeval start_time;
828 gettimeofday(&start_time, NULL);
829 uint64_t busy_loop_iter = 0;
840 const unsigned int tp_status = h.
h2->tp_status;
844 struct timeval cur_time;
845 memset(&cur_time, 0,
sizeof(cur_time));
846 uint64_t milliseconds =
847 ((cur_time.tv_sec - start_time.tv_sec) * 1000) +
848 (((1000000 + cur_time.tv_usec - start_time.tv_usec) / 1000) - 1000);
849 if (milliseconds > 1000) {
856 if (busy_loop_iter) {
873 static inline bool AFPShouldIgnoreFrame(
AFPThreadVars *ptv,
const struct sockaddr_ll *sll)
875 if (
unlikely(sll->sll_pkttype > 31))
893 bool emergency_flush =
false;
898 if (AFPReadFromRingWaitForPacket(ptv) !=
AFP_READ_OK)
910 const unsigned int tp_status = h.
h2->tp_status;
926 const struct sockaddr_ll *sll =
927 (
const struct sockaddr_ll *)((uint8_t *)h.
h2 +
928 TPACKET_ALIGN(
sizeof(
struct tpacket2_hdr)));
929 if (
unlikely(AFPShouldIgnoreFrame(ptv, sll)))
934 return AFPSuriFailure(ptv, h);
936 AFPReadFromRingSetupPacket(ptv, h, tp_status, p);
939 return AFPSuriFailure(ptv, h);
949 if (emergency_flush) {
950 AFPDumpCounters(ptv);
955 #ifdef HAVE_TPACKET_V3
956 static inline void AFPFlushBlock(
struct tpacket_block_desc *pbd)
961 static inline int AFPParsePacketV3(
AFPThreadVars *ptv,
struct tpacket_block_desc *pbd,
struct tpacket3_hdr *ppd)
969 AFPReadApplyBypass(ptv, p);
977 p->
vlan_id[0] = ppd->hv1.tp_vlan_tci & 0x0fff;
979 p->afp_v.vlan_tci = (uint16_t)ppd->hv1.tp_vlan_tci;
982 (void)
PacketSetData(p, (
unsigned char *)ppd + ppd->tp_mac, ppd->tp_snaplen);
985 p->afp_v.relptr = NULL;
986 p->afp_v.mpeer = NULL;
991 p->
ts = (
SCTime_t){ .
secs = ppd->tp_sec, .usecs = ppd->tp_nsec / 1000 };
992 SCLogDebug(
"pktlen: %" PRIu32
" (pkt %p, pkt data %p)",
1018 static inline int AFPWalkBlock(
AFPThreadVars *ptv,
struct tpacket_block_desc *pbd)
1020 const int num_pkts = pbd->hdr.bh1.num_pkts;
1021 uint8_t *ppd = (uint8_t *)pbd + pbd->hdr.bh1.offset_to_first_pkt;
1023 for (
int i = 0; i < num_pkts; ++i) {
1024 const struct sockaddr_ll *sll =
1025 (
const struct sockaddr_ll *)(ppd + TPACKET_ALIGN(
sizeof(
struct tpacket3_hdr)));
1026 if (
unlikely(AFPShouldIgnoreFrame(ptv, sll))) {
1027 ppd = ppd + ((
struct tpacket3_hdr *)ppd)->tp_next_offset;
1030 int ret = AFPParsePacketV3(ptv, pbd, (
struct tpacket3_hdr *)ppd);
1043 ppd = ppd + ((
struct tpacket3_hdr *)ppd)->tp_next_offset;
1061 #ifdef HAVE_TPACKET_V3
1069 struct tpacket_block_desc *pbd =
1077 int ret = AFPWalkBlock(ptv, pbd);
1099 static int AFPRefSocket(
AFPPeer* peer)
1114 static int AFPDerefSocket(
AFPPeer* peer)
1127 if (ptv->
mpeer != NULL)
1131 #ifdef HAVE_TPACKET_V3
1152 static void AFPSwitchState(
AFPThreadVars *ptv, uint8_t state)
1169 static int AFPReadAndDiscardFromRing(
AFPThreadVars *ptv,
struct timeval *synctv,
1170 uint64_t *discarded_pkts)
1176 #ifdef HAVE_TPACKET_V3
1179 struct tpacket_block_desc *pbd =
1181 *discarded_pkts += pbd->hdr.bh1.num_pkts;
1182 struct tpacket3_hdr *ppd =
1183 (
struct tpacket3_hdr *)((uint8_t *)pbd + pbd->hdr.bh1.offset_to_first_pkt);
1184 if (((time_t)ppd->tp_sec > synctv->tv_sec) ||
1185 ((time_t)ppd->tp_sec == synctv->tv_sec &&
1186 (suseconds_t) (ppd->tp_nsec / 1000) > (suseconds_t)synctv->tv_usec)) {
1199 if (h.
raw == NULL) {
1205 if (((time_t)h.
h2->tp_sec > synctv->tv_sec) ||
1206 ((time_t)h.
h2->tp_sec == synctv->tv_sec &&
1207 (suseconds_t) (h.
h2->tp_nsec / 1000) > synctv->tv_usec)) {
1211 (*discarded_pkts)++;
1230 static int AFPSynchronizeStart(
AFPThreadVars *ptv, uint64_t *discarded_pkts)
1232 struct timeval synctv;
1236 fds.events = POLLIN;
1239 synctv.tv_sec = 0xffffffff;
1240 synctv.tv_usec = 0xffffffff;
1245 (fds.revents & (POLLHUP|POLLRDHUP|POLLERR|POLLNVAL))) {
1247 fds.revents & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL));
1250 if (AFPPeersListStarted() && synctv.tv_sec == (time_t) 0xffffffff) {
1251 gettimeofday(&synctv, NULL);
1253 r = AFPReadAndDiscardFromRing(ptv, &synctv, discarded_pkts);
1263 }
else if (r == 0 && AFPPeersListStarted()) {
1290 AFPCloseSocket(ptv);
1292 int afp_activate_r = AFPCreateSocket(ptv, ptv->
iface, 0,
false);
1293 if (afp_activate_r != 0) {
1297 return afp_activate_r;
1315 time_t last_dump = 0;
1316 time_t current_time;
1318 uint64_t discarded_pkts = 0;
1323 AFPReadFunc = AFPReadFromRingV3;
1325 AFPReadFunc = AFPReadFromRing;
1330 while (AFPPeersListWaitTurn(ptv->
mpeer)) {
1336 r = AFPCreateSocket(ptv, ptv->
iface, 1,
true);
1344 "%s: failed to init socket for interface, retrying soon", ptv->
iface);
1350 AFPSynchronizeStart(ptv, &discarded_pkts);
1353 #ifdef PACKET_STATISTICS
1354 struct tpacket_stats kstats;
1355 socklen_t
len =
sizeof (
struct tpacket_stats);
1356 if (getsockopt(ptv->
socket, SOL_PACKET, PACKET_STATISTICS,
1357 &kstats, &
len) > -1) {
1359 SCLogDebug(
"(%s) Kernel socket startup: Packets %" PRIu32
1360 ", dropped %" PRIu32
"",
1362 kstats.tp_packets, kstats.tp_drops);
1363 pkts = kstats.tp_packets - discarded_pkts - kstats.tp_drops;
1371 fds.events = POLLIN;
1388 r = AFPTryReopen(ptv);
1408 (fds.revents & (POLLHUP|POLLRDHUP|POLLERR|POLLNVAL))) {
1410 if (fds.revents & (POLLHUP | POLLRDHUP)) {
1413 }
else if (fds.revents & POLLERR) {
1416 if (recv(ptv->
socket, &c,
sizeof c, MSG_PEEK) != -1)
1421 }
else if (fds.revents & POLLNVAL) {
1428 r = AFPReadFunc(ptv);
1432 current_time = time(NULL);
1433 if (current_time != last_dump) {
1434 AFPDumpCounters(ptv);
1435 last_dump = current_time;
1447 AFPDumpCounters(ptv);
1453 current_time = time(NULL);
1454 if (current_time != last_dump) {
1455 AFPDumpCounters(ptv);
1456 last_dump = current_time;
1459 TmThreadsCaptureHandleTimeout(
tv, NULL);
1461 }
else if ((r < 0) && (errno != EINTR)) {
1470 AFPDumpCounters(ptv);
1475 static int AFPGetDevFlags(
int fd,
const char *ifname)
1479 memset(&ifr, 0,
sizeof(ifr));
1480 strlcpy(ifr.ifr_name, ifname,
sizeof(ifr.ifr_name));
1482 if (ioctl(fd, SIOCGIFFLAGS, &ifr) == -1) {
1483 SCLogError(
"%s: failed to get interface flags: %s", ifname, strerror(errno));
1487 return ifr.ifr_flags;
1491 static int AFPGetIfnumByDev(
int fd,
const char *ifname,
int verbose)
1495 memset(&ifr, 0,
sizeof(ifr));
1496 strlcpy(ifr.ifr_name, ifname,
sizeof(ifr.ifr_name));
1498 if (ioctl(fd, SIOCGIFINDEX, &ifr) == -1) {
1500 SCLogError(
"%s: failed to find interface: %s", ifname, strerror(errno));
1504 return ifr.ifr_ifindex;
1507 static int AFPGetDevLinktype(
int fd,
const char *ifname)
1511 memset(&ifr, 0,
sizeof(ifr));
1512 strlcpy(ifr.ifr_name, ifname,
sizeof(ifr.ifr_name));
1514 if (ioctl(fd, SIOCGIFHWADDR, &ifr) == -1) {
1515 SCLogError(
"%s: failed to find interface type: %s", ifname, strerror(errno));
1519 switch (ifr.ifr_hwaddr.sa_family) {
1520 case ARPHRD_LOOPBACK:
1526 return ifr.ifr_hwaddr.sa_family;
1534 int fd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
1536 SCLogError(
"%s: failed to create AF_PACKET socket: %s", ifname, strerror(errno));
1540 ltype = AFPGetDevLinktype(fd, ifname);
1548 static int AFPComputeRingParams(
AFPThreadVars *ptv,
int order)
1574 int tp_hdrlen =
sizeof(
struct tpacket_hdr);
1580 SCLogWarning(
"%s: unable to get MTU, setting snaplen default of 1514", ptv->
iface);
1585 ptv->
req.
v2.tp_frame_size = TPACKET_ALIGN(snaplen +TPACKET_ALIGN(TPACKET_ALIGN(tp_hdrlen) +
sizeof(
struct sockaddr_ll) + ETH_HLEN) - ETH_HLEN);
1586 ptv->
req.
v2.tp_block_size = getpagesize() << order;
1587 int frames_per_block = ptv->
req.
v2.tp_block_size / ptv->
req.
v2.tp_frame_size;
1588 if (frames_per_block == 0) {
1593 ptv->
req.
v2.tp_block_nr = ptv->
req.
v2.tp_frame_nr / frames_per_block + 1;
1595 ptv->
req.
v2.tp_frame_nr = ptv->
req.
v2.tp_block_nr * frames_per_block;
1596 SCLogPerf(
"%s: rx ring: block_size=%d block_nr=%d frame_size=%d frame_nr=%d", ptv->
iface,
1597 ptv->
req.
v2.tp_block_size, ptv->
req.
v2.tp_block_nr, ptv->
req.
v2.tp_frame_size,
1598 ptv->
req.
v2.tp_frame_nr);
1602 #ifdef HAVE_TPACKET_V3
1606 ptv->
req.v3.tp_frame_size = 2048;
1607 int frames_per_block = 0;
1608 int tp_hdrlen =
sizeof(
struct tpacket3_hdr);
1614 SCLogWarning(
"%s: unable to get MTU, setting snaplen default of 1514", ptv->
iface);
1619 ptv->
req.v3.tp_frame_size = TPACKET_ALIGN(snaplen +TPACKET_ALIGN(TPACKET_ALIGN(tp_hdrlen) +
sizeof(
struct sockaddr_ll) + ETH_HLEN) - ETH_HLEN);
1620 frames_per_block = ptv->
req.v3.tp_block_size / ptv->
req.v3.tp_frame_size;
1622 if (frames_per_block == 0) {
1623 SCLogError(
"%s: block size is too small, it should be at least %d", ptv->
iface,
1624 ptv->
req.v3.tp_frame_size);
1627 ptv->
req.v3.tp_block_nr = ptv->
ring_size / frames_per_block + 1;
1629 ptv->
req.v3.tp_frame_nr = ptv->
req.v3.tp_block_nr * frames_per_block;
1631 ptv->
req.v3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH;
1632 SCLogPerf(
"%s: rx ring params: block_size=%d block_nr=%d frame_size=%d frame_nr=%d (mem: %d)",
1633 ptv->
iface, ptv->
req.v3.tp_block_size, ptv->
req.v3.tp_block_nr,
1634 ptv->
req.v3.tp_frame_size, ptv->
req.v3.tp_frame_nr,
1635 ptv->
req.v3.tp_block_size * ptv->
req.v3.tp_block_nr);
1643 unsigned int len =
sizeof(val), i;
1647 #ifdef HAVE_TPACKET_V3
1655 if (getsockopt(ptv->
socket, SOL_PACKET, PACKET_HDRLEN, &val, &
len) < 0) {
1656 if (errno == ENOPROTOOPT) {
1658 SCLogError(
"%s: kernel too old for TPACKET_V3 (need 3.2+)", devname);
1660 SCLogError(
"%s: kernel too old (need 2.6.27+)", devname);
1663 SCLogError(
"%s: failed to retrieve packet header len", devname);
1668 #ifdef HAVE_TPACKET_V3
1673 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_VERSION, &val,
1675 SCLogError(
"%s: failed to activate TPACKET_V2/TPACKET_V3 on packet socket: %s", devname,
1680 #ifdef HAVE_HW_TIMESTAMPING
1681 int req = SOF_TIMESTAMPING_RAW_HARDWARE;
1682 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_TIMESTAMP, (
void *) &req,
1684 SCLogWarning(
"%s: failed to activate hardware timestamping on packet socket: %s", devname,
1692 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_RESERVE, (
void *)&reserve,
sizeof(reserve)) <
1694 SCLogError(
"%s: failed to activate reserve on packet socket: %s", devname, strerror(errno));
1699 #ifdef HAVE_TPACKET_V3
1701 if (AFPComputeRingParamsV3(ptv) != 1) {
1704 r = setsockopt(ptv->
socket, SOL_PACKET, PACKET_RX_RING,
1705 (
void *) &ptv->
req.v3,
sizeof(ptv->
req.v3));
1707 SCLogError(
"%s: failed to allocate RX Ring: %s", devname, strerror(errno));
1713 if (AFPComputeRingParams(ptv, order) != 1) {
1714 SCLogError(
"%s: ring parameters are incorrect. Please file a bug report", devname);
1718 r = setsockopt(ptv->
socket, SOL_PACKET, PACKET_RX_RING,
1719 (
void *) &ptv->
req,
sizeof(ptv->
req));
1722 if (errno == ENOMEM) {
1723 SCLogWarning(
"%s: memory issue with ring parameters. Retrying", devname);
1726 SCLogError(
"%s: failed to setup RX Ring: %s", devname, strerror(errno));
1733 SCLogError(
"%s: failed to setup RX Ring (order 0 failed)", devname);
1736 #ifdef HAVE_TPACKET_V3
1741 #ifdef HAVE_TPACKET_V3
1747 #ifdef HAVE_TPACKET_V3
1750 mmap_flag = MAP_SHARED;
1752 mmap_flag |= MAP_LOCKED;
1754 mmap_flag, ptv->
socket, 0);
1756 SCLogError(
"%s: failed to mmap: %s", devname, strerror(errno));
1759 #ifdef HAVE_TPACKET_V3
1763 SCLogError(
"%s: failed to alloc ring: %s", devname, strerror(errno));
1766 for (i = 0; i < ptv->
req.v3.tp_block_nr; ++i) {
1768 ptv->
ring.
v3[i].iov_len = ptv->
req.v3.tp_block_size;
1774 if (ptv->
ring.
v2 == NULL) {
1775 SCLogError(
"%s: failed to alloc ring: %s", devname, strerror(errno));
1780 for (i = 0; i < ptv->
req.
v2.tp_block_nr; ++i) {
1781 void *base = &(ptv->
ring_buf[i * ptv->
req.
v2.tp_block_size]);
1785 base += ptv->
req.
v2.tp_frame_size;
1789 #ifdef HAVE_TPACKET_V3
1811 #ifdef HAVE_PACKET_FANOUT
1812 int fd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
1817 uint32_t option = (mode << 16) | cluster_id;
1818 int r = setsockopt(fd, SOL_PACKET,
PACKET_FANOUT,(
void *)&option,
sizeof(option));
1822 SCLogError(
"fanout not supported by kernel: "
1823 "Kernel too old or cluster-id %d already in use.",
1833 #ifdef HAVE_PACKET_EBPF
1837 int pfd = ptv->ebpf_lb_fd;
1839 SCLogError(
"Fanout file descriptor is invalid");
1843 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_FANOUT_DATA, &pfd,
sizeof(pfd))) {
1854 int pfd = ptv->ebpf_filter_fd;
1856 SCLogError(
"Filter file descriptor is invalid");
1860 if (setsockopt(ptv->
socket, SOL_SOCKET, SO_ATTACH_BPF, &pfd,
sizeof(pfd))) {
1861 SCLogError(
"Error setting ebpf: %s", strerror(errno));
1864 SCLogInfo(
"Activated eBPF filter on socket");
1871 static int AFPCreateSocket(
AFPThreadVars *ptv,
char *devname,
int verbose,
const bool peer_update)
1875 struct packet_mreq sock_params;
1876 struct sockaddr_ll bind_address;
1880 ptv->
socket = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
1882 SCLogError(
"%s: failed to create socket: %s", devname, strerror(errno));
1886 if_idx = AFPGetIfnumByDev(ptv->
socket, devname, verbose);
1892 memset(&bind_address, 0,
sizeof(bind_address));
1893 bind_address.sll_family = AF_PACKET;
1894 bind_address.sll_protocol = htons(ETH_P_ALL);
1895 bind_address.sll_ifindex = if_idx;
1896 if (bind_address.sll_ifindex == -1) {
1903 int if_flags = AFPGetDevFlags(ptv->
socket, ptv->
iface);
1904 if (if_flags == -1) {
1910 }
else if ((if_flags & (IFF_UP | IFF_RUNNING)) == 0) {
1919 if (if_flags & IFF_LOOPBACK)
1924 memset(&sock_params, 0,
sizeof(sock_params));
1925 sock_params.mr_type = PACKET_MR_PROMISC;
1926 sock_params.mr_ifindex = bind_address.sll_ifindex;
1927 r = setsockopt(ptv->
socket, SOL_PACKET, PACKET_ADD_MEMBERSHIP,(
void *)&sock_params,
sizeof(sock_params));
1929 SCLogError(
"%s: failed to set promisc mode: %s", devname, strerror(errno));
1936 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_AUXDATA, &val,
1937 sizeof(val)) == -1 && errno != ENOPROTOOPT) {
1939 "%s: 'kernel' checksum mode not supported, falling back to full mode", devname);
1950 if (setsockopt(ptv->
socket, SOL_SOCKET, SO_RCVBUF,
1959 r = bind(ptv->
socket, (
struct sockaddr *)&bind_address,
sizeof(bind_address));
1962 if (errno == ENETDOWN) {
1963 SCLogWarning(
"%s: failed to bind socket, iface is down", devname);
1965 SCLogWarning(
"%s: failed to bind socket: %s", devname, strerror(errno));
1973 #ifdef HAVE_PACKET_FANOUT
1978 uint32_t option = (mode << 16) | (
id & 0xffff);
1981 SCLogError(
"%s: failed to set fanout mode: %s", devname, strerror(errno));
1987 #ifdef HAVE_PACKET_EBPF
1989 r = SockFanoutSeteBPF(ptv);
1991 SCLogError(
"%s: failed to set eBPF: %s", devname, strerror(errno));
1998 AFPPeersListReachedInc();
2000 ret = AFPSetupRing(ptv, devname);
2004 SCLogDebug(
"Using interface '%s' via socket %d", (
char *)devname, ptv->
socket);
2008 TmEcode rc = AFPSetBPFFilter(ptv);
2039 struct bpf_program filter;
2040 struct sock_fprog fcode;
2043 #ifdef HAVE_PACKET_EBPF
2044 if (ptv->ebpf_filter_fd != -1) {
2045 return SetEbpfFilter(ptv);
2054 char errbuf[PCAP_ERRBUF_SIZE];
2062 sizeof(errbuf)) == -1) {
2067 if (filter.bf_len > USHRT_MAX) {
2070 fcode.len = (
unsigned short)filter.bf_len;
2071 fcode.filter = (
struct sock_filter*)filter.bf_insns;
2073 rc = setsockopt(ptv->
socket, SOL_SOCKET, SO_ATTACH_FILTER, &fcode,
sizeof(fcode));
2077 SCLogError(
"%s: failed to attach filter: %s", ptv->
iface, strerror(errno));
2084 #ifdef HAVE_PACKET_EBPF
2092 static int AFPInsertHalfFlow(
int mapd,
void *key,
unsigned int nr_cpus)
2094 BPF_DECLARE_PERCPU(
struct pair, value, nr_cpus);
2103 for (i = 0; i < nr_cpus; i++) {
2104 BPF_PERCPU(value, i).packets = 0;
2105 BPF_PERCPU(value, i).bytes = 0;
2107 if (bpf_map_update_elem(mapd, key, value, BPF_NOEXIST) != 0) {
2120 SCLogError(
"Can't update eBPF map: %s (%d)", strerror(errno), errno);
2127 static int AFPSetFlowStorage(
Packet *p,
int map_fd,
void *key0,
void* key1,
2138 EBPFBypassData *eb =
SCCalloc(1,
sizeof(EBPFBypassData));
2140 EBPFDeleteKey(map_fd, key0);
2141 EBPFDeleteKey(map_fd, key1);
2150 eb->cpus_count = p->afp_v.nr_cpus;
2155 EBPFDeleteKey(map_fd, key0);
2156 EBPFDeleteKey(map_fd, key1);
2182 static int AFPBypassCallback(
Packet *p)
2184 SCLogDebug(
"Calling af_packet callback function");
2186 if (!(PacketIsTCP(p) || PacketIsUDP(p))) {
2193 if (p->
flow == NULL) {
2199 if (PacketIsTunnel(p)) {
2202 if (PacketIsIPv4(p)) {
2204 if (p->afp_v.v4_map_fd == -1) {
2207 struct flowv4_keys *keys[2];
2208 keys[0] =
SCCalloc(1,
sizeof(
struct flowv4_keys));
2209 if (keys[0] == NULL) {
2214 keys[0]->port16[0] = p->
sp;
2215 keys[0]->port16[1] = p->
dp;
2216 keys[0]->vlan0 = p->
vlan_id[0];
2217 keys[0]->vlan1 = p->
vlan_id[1];
2218 keys[0]->vlan2 = p->
vlan_id[2];
2220 if (p->
proto == IPPROTO_TCP) {
2221 keys[0]->ip_proto = 1;
2223 keys[0]->ip_proto = 0;
2225 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[0],
2226 p->afp_v.nr_cpus) == 0) {
2231 keys[1]=
SCCalloc(1,
sizeof(
struct flowv4_keys));
2232 if (keys[1] == NULL) {
2233 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2240 keys[1]->port16[0] = p->
dp;
2241 keys[1]->port16[1] = p->
sp;
2242 keys[1]->vlan0 = p->
vlan_id[0];
2243 keys[1]->vlan1 = p->
vlan_id[1];
2244 keys[1]->vlan2 = p->
vlan_id[2];
2246 keys[1]->ip_proto = keys[0]->ip_proto;
2247 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[1],
2248 p->afp_v.nr_cpus) == 0) {
2249 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2255 EBPFUpdateFlow(p->
flow, p, NULL);
2256 return AFPSetFlowStorage(p, p->afp_v.v4_map_fd, keys[0], keys[1], AF_INET);
2259 if (PacketIsIPv6(p) && ((p->
proto == IPPROTO_TCP) || (p->
proto == IPPROTO_UDP))) {
2261 if (p->afp_v.v6_map_fd == -1) {
2265 struct flowv6_keys *keys[2];
2266 keys[0] =
SCCalloc(1,
sizeof(
struct flowv6_keys));
2267 if (keys[0] == NULL) {
2271 for (i = 0; i < 4; i++) {
2275 keys[0]->port16[0] = p->
sp;
2276 keys[0]->port16[1] = p->
dp;
2277 keys[0]->vlan0 = p->
vlan_id[0];
2278 keys[0]->vlan1 = p->
vlan_id[1];
2279 keys[0]->vlan2 = p->
vlan_id[2];
2281 if (p->
proto == IPPROTO_TCP) {
2282 keys[0]->ip_proto = 1;
2284 keys[0]->ip_proto = 0;
2286 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[0],
2287 p->afp_v.nr_cpus) == 0) {
2292 keys[1]=
SCCalloc(1,
sizeof(
struct flowv6_keys));
2293 if (keys[1] == NULL) {
2294 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2299 for (i = 0; i < 4; i++) {
2303 keys[1]->port16[0] = p->
dp;
2304 keys[1]->port16[1] = p->
sp;
2305 keys[1]->vlan0 = p->
vlan_id[0];
2306 keys[1]->vlan1 = p->
vlan_id[1];
2307 keys[1]->vlan2 = p->
vlan_id[2];
2309 keys[1]->ip_proto = keys[0]->ip_proto;
2310 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[1],
2311 p->afp_v.nr_cpus) == 0) {
2312 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2319 EBPFUpdateFlow(p->
flow, p, NULL);
2320 return AFPSetFlowStorage(p, p->afp_v.v6_map_fd, keys[0], keys[1], AF_INET6);
2336 static int AFPXDPBypassCallback(
Packet *p)
2338 SCLogDebug(
"Calling af_packet callback function");
2340 if (!(PacketIsTCP(p) || PacketIsUDP(p))) {
2347 if (p->
flow == NULL) {
2353 if (PacketIsTunnel(p)) {
2356 if (PacketIsIPv4(p)) {
2357 struct flowv4_keys *keys[2];
2358 keys[0]=
SCCalloc(1,
sizeof(
struct flowv4_keys));
2359 if (keys[0] == NULL) {
2363 if (p->afp_v.v4_map_fd == -1) {
2367 keys[0]->src = p->
src.addr_data32[0];
2368 keys[0]->dst = p->
dst.addr_data32[0];
2371 keys[0]->port16[0] = htons(p->
sp);
2372 keys[0]->port16[1] = htons(p->
dp);
2373 keys[0]->vlan0 = p->
vlan_id[0];
2374 keys[0]->vlan1 = p->
vlan_id[1];
2375 keys[0]->vlan2 = p->
vlan_id[2];
2376 if (p->
proto == IPPROTO_TCP) {
2377 keys[0]->ip_proto = 1;
2379 keys[0]->ip_proto = 0;
2381 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[0],
2382 p->afp_v.nr_cpus) == 0) {
2387 keys[1]=
SCCalloc(1,
sizeof(
struct flowv4_keys));
2388 if (keys[1] == NULL) {
2389 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2394 keys[1]->src = p->
dst.addr_data32[0];
2395 keys[1]->dst = p->
src.addr_data32[0];
2396 keys[1]->port16[0] = htons(p->
dp);
2397 keys[1]->port16[1] = htons(p->
sp);
2398 keys[1]->vlan0 = p->
vlan_id[0];
2399 keys[1]->vlan1 = p->
vlan_id[1];
2400 keys[1]->vlan2 = p->
vlan_id[2];
2401 keys[1]->ip_proto = keys[0]->ip_proto;
2402 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[1],
2403 p->afp_v.nr_cpus) == 0) {
2404 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2410 return AFPSetFlowStorage(p, p->afp_v.v4_map_fd, keys[0], keys[1], AF_INET);
2413 if (PacketIsIPv6(p) && ((p->
proto == IPPROTO_TCP) || (p->
proto == IPPROTO_UDP))) {
2415 if (p->afp_v.v6_map_fd == -1) {
2419 struct flowv6_keys *keys[2];
2420 keys[0] =
SCCalloc(1,
sizeof(
struct flowv6_keys));
2421 if (keys[0] == NULL) {
2425 for (i = 0; i < 4; i++) {
2429 keys[0]->port16[0] = htons(p->
sp);
2430 keys[0]->port16[1] = htons(p->
dp);
2431 keys[0]->vlan0 = p->
vlan_id[0];
2432 keys[0]->vlan1 = p->
vlan_id[1];
2433 keys[0]->vlan2 = p->
vlan_id[2];
2434 if (p->
proto == IPPROTO_TCP) {
2435 keys[0]->ip_proto = 1;
2437 keys[0]->ip_proto = 0;
2439 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[0],
2440 p->afp_v.nr_cpus) == 0) {
2445 keys[1]=
SCCalloc(1,
sizeof(
struct flowv6_keys));
2446 if (keys[1] == NULL) {
2447 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2452 for (i = 0; i < 4; i++) {
2456 keys[1]->port16[0] = htons(p->
dp);
2457 keys[1]->port16[1] = htons(p->
sp);
2458 keys[1]->vlan0 = p->
vlan_id[0];
2459 keys[1]->vlan1 = p->
vlan_id[1];
2460 keys[1]->vlan2 = p->
vlan_id[2];
2461 keys[1]->ip_proto = keys[0]->ip_proto;
2462 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[1],
2463 p->afp_v.nr_cpus) == 0) {
2464 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2470 return AFPSetFlowStorage(p, p->afp_v.v6_map_fd, keys[0], keys[1], AF_INET6);
2475 bool g_flowv4_ok =
true;
2476 bool g_flowv6_ok =
true;
2494 if (initdata == NULL) {
2527 #ifdef HAVE_PACKET_FANOUT
2542 #ifdef HAVE_PACKET_EBPF
2545 ptv->xdp_mode = afpconfig->
xdp_mode;
2549 ptv->v4_map_fd = EBPFGetMapFDByName(ptv->
iface,
"flow_table_v4");
2550 if (ptv->v4_map_fd == -1) {
2552 SCLogError(
"Can't find eBPF map fd for '%s'",
"flow_table_v4");
2556 ptv->v6_map_fd = EBPFGetMapFDByName(ptv->
iface,
"flow_table_v6");
2557 if (ptv->v6_map_fd == -1) {
2559 SCLogError(
"Can't find eBPF map fd for '%s'",
"flow_table_v6");
2560 g_flowv6_ok =
false;
2564 ptv->ebpf_t_config = afpconfig->ebpf_t_config;
2567 #ifdef PACKET_STATISTICS
2591 SCLogWarning(
"Enabling a BPF filter in IPS mode result"
2592 " in dropping all non matching packets.");
2603 *data = (
void *)ptv;
2622 void ReceiveAFPThreadExitStats(
ThreadVars *
tv,
void *data)
2627 #ifdef PACKET_STATISTICS
2628 AFPDumpCounters(ptv);
2646 #ifdef HAVE_PACKET_XDP
2647 if ((ptv->ebpf_t_config.
flags & EBPF_XDP_CODE) &&
2648 (!(ptv->ebpf_t_config.
flags & EBPF_PINNED_MAPS))) {
2649 EBPFSetupXDP(ptv->
iface, -1, ptv->xdp_mode);
2675 static void UpdateRawDataForVLANHdr(
Packet *p)
2677 if (p->afp_v.vlan_tci != 0) {
2683 *(uint16_t *)(pstart + 2 * ETH_ALEN) = htons(0x8100);
2684 *(uint16_t *)(pstart + 2 * ETH_ALEN + 2) = htons(p->afp_v.vlan_tci);
2707 const bool afp_vlan_hdr = p->
vlan_idx != 0;
2720 UpdateRawDataForVLANHdr(p);
2737 *data = (
void *)
dtv;