33 #define SC_PCAP_DONT_INCLUDE_PCAP_H 1
67 #include <sys/ioctl.h>
70 #if HAVE_LINUX_SOCKIOS_H
71 #include <linux/sockios.h>
74 #ifdef HAVE_PACKET_EBPF
75 #define PCAP_DONT_INCLUDE_PCAP_BPF_H 1
76 #include <bpf/libbpf.h>
81 struct bpf_insn *bf_insns;
89 #ifdef HAVE_PCAP_PCAP_H
90 #include <pcap/pcap.h>
95 #if HAVE_LINUX_IF_ETHER_H
96 #include <linux/if_ether.h>
99 #if HAVE_LINUX_IF_PACKET_H
100 #include <linux/if_packet.h>
103 #if HAVE_LINUX_IF_ARP_H
104 #include <linux/if_arp.h>
107 #if HAVE_LINUX_FILTER_H
108 #include <linux/filter.h>
112 #include <sys/mman.h>
115 #ifdef HAVE_HW_TIMESTAMPING
116 #include <linux/net_tstamp.h>
123 #ifndef HAVE_AF_PACKET
157 SCLogError(
"Error creating thread %s: you do not have "
158 "support for AF_PACKET enabled, on Linux host please recompile "
159 "with --enable-af-packet",
166 #define AFP_IFACE_NAME_LENGTH 48
168 #define AFP_STATE_DOWN 0
169 #define AFP_STATE_UP 1
171 #define AFP_RECONNECT_TIMEOUT 500000
172 #define AFP_DOWN_COUNTER_INTERVAL 40
174 #define POLL_TIMEOUT 100
177 #ifndef TP_STATUS_KERNEL
178 #define TP_STATUS_KERNEL 0
180 #ifndef TP_STATUS_USER
181 #define TP_STATUS_USER BIT_U32(0)
183 #ifndef TP_STATUS_COPY
184 #define TP_STATUS_COPY BIT_U32(1)
186 #ifndef TP_STATUS_LOSING
187 #define TP_STATUS_LOSING BIT_U32(2)
189 #ifndef TP_STATUS_CSUMNOTREADY
190 #define TP_STATUS_CSUMNOTREADY BIT_U32(3)
192 #ifndef TP_STATUS_VLAN_VALID
193 #define TP_STATUS_VLAN_VALID BIT_U32(4)
195 #ifndef TP_STATUS_BLK_TMO
196 #define TP_STATUS_BLK_TMO BIT_U32(5)
198 #ifndef TP_STATUS_VLAN_TPID_VALID
199 #define TP_STATUS_VLAN_TPID_VALID BIT_U32(6)
201 #ifndef TP_STATUS_CSUM_VALID
202 #define TP_STATUS_CSUM_VALID BIT_U32(7)
205 #ifndef TP_STATUS_TS_SOFTWARE
206 #define TP_STATUS_TS_SOFTWARE BIT_U32(29)
208 #ifndef TP_STATUS_TS_SYS_HARDWARE
209 #define TP_STATUS_TS_SYS_HARDWARE BIT_U32(30)
211 #ifndef TP_STATUS_TS_RAW_HARDWARE
212 #define TP_STATUS_TS_RAW_HARDWARE BIT_U32(31)
215 #ifndef TP_STATUS_USER_BUSY
227 #define TP_STATUS_USER_BUSY \
228 (uint32_t)((uint32_t)TP_STATUS_TS_SOFTWARE | (uint32_t)TP_STATUS_TS_SYS_HARDWARE | \
229 (uint32_t)TP_STATUS_TS_RAW_HARDWARE)
231 #define FRAME_BUSY(tp_status) \
232 (((uint32_t)(tp_status) & (uint32_t)TP_STATUS_USER_BUSY) == (uint32_t)TP_STATUS_USER_BUSY)
248 struct tpacket2_hdr *
h2;
249 struct tpacket3_hdr *
h3;
253 #ifdef HAVE_PACKET_EBPF
254 static int AFPBypassCallback(
Packet *p);
255 static int AFPXDPBypassCallback(
Packet *p);
277 #ifdef HAVE_PACKET_EBPF
339 struct tpacket_req
v2;
340 struct tpacket_req3
v3;
352 #ifdef HAVE_PACKET_EBPF
356 struct ebpf_timeout_config ebpf_t_config;
362 static void ReceiveAFPThreadExitStats(
ThreadVars *,
void *);
371 static int AFPGetIfnumByDev(
int fd,
const char *ifname,
int verbose);
372 static int AFPGetDevFlags(
int fd,
const char *ifname);
373 static int AFPDerefSocket(
AFPPeer* peer);
374 static int AFPRefSocket(
AFPPeer* peer);
425 if (ptv->
mpeer == NULL) {
436 static void AFPPeerClean(
AFPPeer *peer)
468 #define AFP_PEERS_MAX_TRY 4
469 #define AFP_PEERS_WAIT 20000
530 if (out_iface == NULL)
532 if (iface->
mtu != out_iface->
mtu) {
533 SCLogWarning(
"MTU on %s (%d) and %s (%d) are not equal, transmission of packets "
534 "bigger than %d will fail.",
548 static int AFPPeersListWaitTurn(
AFPPeer *peer)
559 static void AFPPeersListReachedInc(
void)
573 static int AFPPeersListStarted(
void)
610 static int AFPCreateSocket(
AFPThreadVars *ptv,
char *devname,
int verbose,
const bool peer_update);
614 #ifdef PACKET_STATISTICS
615 struct tpacket_stats kstats;
616 socklen_t
len =
sizeof (
struct tpacket_stats);
617 if (getsockopt(ptv->
socket, SOL_PACKET, PACKET_STATISTICS,
618 &kstats, &
len) > -1) {
619 SCLogDebug(
"(%s) Kernel: Packets %" PRIu32
", dropped %" PRIu32
"",
621 kstats.tp_packets, kstats.tp_drops);
649 struct sockaddr_ll socket_address;
658 if (!PacketIsEthernet(p)) {
663 const EthernetHdr *ethh = PacketGetEthernet(p);
665 socket_address.sll_ifindex =
SC_ATOMIC_GET(p->afp_v.peer->if_idx);
667 socket_address.sll_halen = ETH_ALEN;
669 memcpy(socket_address.sll_addr, ethh, 6);
677 sizeof(
struct sockaddr_ll)) < 0) {
679 SCLogWarning(
"%s: sending packet failed on socket %d: %s", p->afp_v.peer->iface, socket,
687 static void AFPReleaseDataFromRing(
Packet *p)
694 AFPWritePacket(p, TPACKET_V2);
697 BUG_ON(p->afp_v.relptr == NULL);
700 h.
raw = p->afp_v.relptr;
703 (void)AFPDerefSocket(p->afp_v.mpeer);
708 static void AFPReleasePacketV3(
Packet *p)
715 AFPWritePacket(p, TPACKET_V3);
720 static void AFPReleasePacket(
Packet *p)
722 AFPReleaseDataFromRing(p);
741 #ifdef HAVE_PACKET_EBPF
744 p->afp_v.v4_map_fd = ptv->v4_map_fd;
745 p->afp_v.v6_map_fd = ptv->v6_map_fd;
746 p->afp_v.nr_cpus = ptv->ebpf_t_config.cpus_count;
750 p->afp_v.v4_map_fd = ptv->v4_map_fd;
751 p->afp_v.v6_map_fd = ptv->v6_map_fd;
752 p->afp_v.nr_cpus = ptv->ebpf_t_config.cpus_count;
760 static void AFPReadFromRingSetupPacket(
773 AFPReadApplyBypass(ptv, p);
775 if (h.
h2->tp_len > h.
h2->tp_snaplen) {
776 SCLogDebug(
"Packet length (%d) > snaplen (%d), truncating", h.
h2->tp_len, h.
h2->tp_snaplen);
783 p->
vlan_id[0] = h.
h2->tp_vlan_tci & 0x0fff;
785 p->afp_v.vlan_tci = h.
h2->tp_vlan_tci;
791 p->afp_v.relptr = h.
raw;
793 p->afp_v.mpeer = ptv->
mpeer;
794 AFPRefSocket(ptv->
mpeer);
796 p->afp_v.mpeer = NULL;
821 static inline int AFPReadFromRingWaitForPacket(
AFPThreadVars *ptv)
824 struct timeval start_time;
825 gettimeofday(&start_time, NULL);
826 uint64_t busy_loop_iter = 0;
837 const unsigned int tp_status = h.
h2->tp_status;
841 struct timeval cur_time;
842 memset(&cur_time, 0,
sizeof(cur_time));
843 uint64_t milliseconds =
844 ((cur_time.tv_sec - start_time.tv_sec) * 1000) +
845 (((1000000 + cur_time.tv_usec - start_time.tv_usec) / 1000) - 1000);
846 if (milliseconds > 1000) {
853 if (busy_loop_iter) {
870 static inline bool AFPShouldIgnoreFrame(
AFPThreadVars *ptv,
const struct sockaddr_ll *sll)
872 if (
unlikely(sll->sll_pkttype > 31))
890 bool emergency_flush =
false;
895 if (AFPReadFromRingWaitForPacket(ptv) !=
AFP_READ_OK)
907 const unsigned int tp_status = h.
h2->tp_status;
923 const struct sockaddr_ll *sll =
924 (
const struct sockaddr_ll *)((uint8_t *)h.
h2 +
925 TPACKET_ALIGN(
sizeof(
struct tpacket2_hdr)));
926 if (
unlikely(AFPShouldIgnoreFrame(ptv, sll)))
931 return AFPSuriFailure(ptv, h);
933 AFPReadFromRingSetupPacket(ptv, h, tp_status, p);
936 return AFPSuriFailure(ptv, h);
946 if (emergency_flush) {
947 AFPDumpCounters(ptv);
952 static inline void AFPFlushBlock(
struct tpacket_block_desc *pbd)
957 static inline int AFPParsePacketV3(
AFPThreadVars *ptv,
struct tpacket_block_desc *pbd,
struct tpacket3_hdr *ppd)
965 AFPReadApplyBypass(ptv, p);
973 p->
vlan_id[0] = ppd->hv1.tp_vlan_tci & 0x0fff;
975 p->afp_v.vlan_tci = (uint16_t)ppd->hv1.tp_vlan_tci;
978 if (ppd->tp_len > ppd->tp_snaplen) {
979 SCLogDebug(
"Packet length (%d) > snaplen (%d), truncating", ppd->tp_len, ppd->tp_snaplen);
983 (void)
PacketSetData(p, (
unsigned char *)ppd + ppd->tp_mac, ppd->tp_snaplen);
986 p->afp_v.relptr = NULL;
987 p->afp_v.mpeer = NULL;
992 p->
ts = (
SCTime_t){ .
secs = ppd->tp_sec, .usecs = ppd->tp_nsec / 1000 };
993 SCLogDebug(
"pktlen: %" PRIu32
" (pkt %p, pkt data %p)",
1019 static inline int AFPWalkBlock(
AFPThreadVars *ptv,
struct tpacket_block_desc *pbd)
1021 const int num_pkts = pbd->hdr.bh1.num_pkts;
1022 uint8_t *ppd = (uint8_t *)pbd + pbd->hdr.bh1.offset_to_first_pkt;
1024 for (
int i = 0; i < num_pkts; ++i) {
1025 const struct sockaddr_ll *sll =
1026 (
const struct sockaddr_ll *)(ppd + TPACKET_ALIGN(
sizeof(
struct tpacket3_hdr)));
1027 if (
unlikely(AFPShouldIgnoreFrame(ptv, sll))) {
1028 ppd = ppd + ((
struct tpacket3_hdr *)ppd)->tp_next_offset;
1031 int ret = AFPParsePacketV3(ptv, pbd, (
struct tpacket3_hdr *)ppd);
1044 ppd = ppd + ((
struct tpacket3_hdr *)ppd)->tp_next_offset;
1068 struct tpacket_block_desc *pbd =
1076 int ret = AFPWalkBlock(ptv, pbd);
1097 static int AFPRefSocket(
AFPPeer* peer)
1112 static int AFPDerefSocket(
AFPPeer* peer)
1125 if (ptv->
mpeer != NULL)
1148 static void AFPSwitchState(
AFPThreadVars *ptv, uint8_t state)
1165 static int AFPReadAndDiscardFromRing(
AFPThreadVars *ptv,
struct timeval *synctv,
1166 uint64_t *discarded_pkts)
1174 struct tpacket_block_desc *pbd =
1176 *discarded_pkts += pbd->hdr.bh1.num_pkts;
1177 struct tpacket3_hdr *ppd =
1178 (
struct tpacket3_hdr *)((uint8_t *)pbd + pbd->hdr.bh1.offset_to_first_pkt);
1179 if (((time_t)ppd->tp_sec > synctv->tv_sec) ||
1180 ((time_t)ppd->tp_sec == synctv->tv_sec &&
1181 (suseconds_t) (ppd->tp_nsec / 1000) > (suseconds_t)synctv->tv_usec)) {
1192 if (h.
raw == NULL) {
1198 if (((time_t)h.
h2->tp_sec > synctv->tv_sec) ||
1199 ((time_t)h.
h2->tp_sec == synctv->tv_sec &&
1200 (suseconds_t) (h.
h2->tp_nsec / 1000) > synctv->tv_usec)) {
1204 (*discarded_pkts)++;
1223 static int AFPSynchronizeStart(
AFPThreadVars *ptv, uint64_t *discarded_pkts)
1225 struct timeval synctv;
1229 fds.events = POLLIN;
1232 synctv.tv_sec = 0xffffffff;
1233 synctv.tv_usec = 0xffffffff;
1238 (fds.revents & (POLLHUP|POLLRDHUP|POLLERR|POLLNVAL))) {
1240 fds.revents & (POLLHUP | POLLRDHUP | POLLERR | POLLNVAL));
1243 if (AFPPeersListStarted() && synctv.tv_sec == (time_t) 0xffffffff) {
1244 gettimeofday(&synctv, NULL);
1246 r = AFPReadAndDiscardFromRing(ptv, &synctv, discarded_pkts);
1256 }
else if (r == 0 && AFPPeersListStarted()) {
1283 AFPCloseSocket(ptv);
1285 int afp_activate_r = AFPCreateSocket(ptv, ptv->
iface, 0,
false);
1286 if (afp_activate_r != 0) {
1290 return afp_activate_r;
1308 time_t last_dump = 0;
1309 time_t current_time;
1311 uint64_t discarded_pkts = 0;
1316 AFPReadFunc = AFPReadFromRingV3;
1318 AFPReadFunc = AFPReadFromRing;
1323 while (AFPPeersListWaitTurn(ptv->
mpeer)) {
1329 r = AFPCreateSocket(ptv, ptv->
iface, 1,
true);
1337 "%s: failed to init socket for interface, retrying soon", ptv->
iface);
1343 AFPSynchronizeStart(ptv, &discarded_pkts);
1346 #ifdef PACKET_STATISTICS
1347 struct tpacket_stats kstats;
1348 socklen_t
len =
sizeof (
struct tpacket_stats);
1349 if (getsockopt(ptv->
socket, SOL_PACKET, PACKET_STATISTICS,
1350 &kstats, &
len) > -1) {
1352 SCLogDebug(
"(%s) Kernel socket startup: Packets %" PRIu32
1353 ", dropped %" PRIu32
"",
1355 kstats.tp_packets, kstats.tp_drops);
1356 pkts = kstats.tp_packets - discarded_pkts - kstats.tp_drops;
1364 fds.events = POLLIN;
1381 r = AFPTryReopen(ptv);
1401 (fds.revents & (POLLHUP|POLLRDHUP|POLLERR|POLLNVAL))) {
1403 if (fds.revents & (POLLHUP | POLLRDHUP)) {
1406 }
else if (fds.revents & POLLERR) {
1409 if (recv(ptv->
socket, &c,
sizeof c, MSG_PEEK) != -1)
1414 }
else if (fds.revents & POLLNVAL) {
1421 r = AFPReadFunc(ptv);
1425 current_time = time(NULL);
1426 if (current_time != last_dump) {
1427 AFPDumpCounters(ptv);
1428 last_dump = current_time;
1440 AFPDumpCounters(ptv);
1446 current_time = time(NULL);
1447 if (current_time != last_dump) {
1448 AFPDumpCounters(ptv);
1449 last_dump = current_time;
1452 TmThreadsCaptureHandleTimeout(
tv, NULL);
1454 }
else if ((r < 0) && (errno != EINTR)) {
1463 AFPDumpCounters(ptv);
1468 static int AFPGetDevFlags(
int fd,
const char *ifname)
1472 memset(&ifr, 0,
sizeof(ifr));
1473 strlcpy(ifr.ifr_name, ifname,
sizeof(ifr.ifr_name));
1475 if (ioctl(fd, SIOCGIFFLAGS, &ifr) == -1) {
1476 SCLogError(
"%s: failed to get interface flags: %s", ifname, strerror(errno));
1480 return ifr.ifr_flags;
1484 static int AFPGetIfnumByDev(
int fd,
const char *ifname,
int verbose)
1488 memset(&ifr, 0,
sizeof(ifr));
1489 strlcpy(ifr.ifr_name, ifname,
sizeof(ifr.ifr_name));
1491 if (ioctl(fd, SIOCGIFINDEX, &ifr) == -1) {
1493 SCLogError(
"%s: failed to find interface: %s", ifname, strerror(errno));
1497 return ifr.ifr_ifindex;
1500 static int AFPGetDevLinktype(
int fd,
const char *ifname)
1504 memset(&ifr, 0,
sizeof(ifr));
1505 strlcpy(ifr.ifr_name, ifname,
sizeof(ifr.ifr_name));
1507 if (ioctl(fd, SIOCGIFHWADDR, &ifr) == -1) {
1508 SCLogError(
"%s: failed to find interface type: %s", ifname, strerror(errno));
1512 switch (ifr.ifr_hwaddr.sa_family) {
1513 case ARPHRD_LOOPBACK:
1519 return ifr.ifr_hwaddr.sa_family;
1527 int fd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
1529 SCLogError(
"%s: failed to create AF_PACKET socket: %s", ifname, strerror(errno));
1533 ltype = AFPGetDevLinktype(fd, ifname);
1541 static int AFPComputeRingParams(
AFPThreadVars *ptv,
int order)
1567 int tp_hdrlen =
sizeof(
struct tpacket_hdr);
1578 SCLogWarning(
"%s: unable to get MTU, setting snaplen default of 1514", ptv->
iface);
1585 ptv->
req.
v2.tp_frame_size = TPACKET_ALIGN(snaplen +TPACKET_ALIGN(TPACKET_ALIGN(tp_hdrlen) +
sizeof(
struct sockaddr_ll) + ETH_HLEN) - ETH_HLEN);
1586 ptv->
req.
v2.tp_block_size = getpagesize() << order;
1587 int frames_per_block = ptv->
req.
v2.tp_block_size / ptv->
req.
v2.tp_frame_size;
1588 if (frames_per_block == 0) {
1593 ptv->
req.
v2.tp_block_nr = ptv->
req.
v2.tp_frame_nr / frames_per_block + 1;
1595 ptv->
req.
v2.tp_frame_nr = ptv->
req.
v2.tp_block_nr * frames_per_block;
1596 SCLogPerf(
"%s: rx ring: block_size=%d block_nr=%d frame_size=%d frame_nr=%d", ptv->
iface,
1597 ptv->
req.
v2.tp_block_size, ptv->
req.
v2.tp_block_nr, ptv->
req.
v2.tp_frame_size,
1598 ptv->
req.
v2.tp_frame_nr);
1602 static int AFPComputeRingParamsWithBlockSize(
AFPThreadVars *ptv,
unsigned int block_size)
1628 int tp_hdrlen =
sizeof(
struct tpacket_hdr);
1639 SCLogWarning(
"%s: unable to get MTU, setting snaplen default of 1514", ptv->
iface);
1646 ptv->
req.
v2.tp_frame_size = TPACKET_ALIGN(
1648 TPACKET_ALIGN(TPACKET_ALIGN(tp_hdrlen) +
sizeof(
struct sockaddr_ll) + ETH_HLEN) -
1650 ptv->
req.
v2.tp_block_size = block_size;
1651 int frames_per_block = ptv->
req.
v2.tp_block_size / ptv->
req.
v2.tp_frame_size;
1652 if (frames_per_block == 0) {
1657 ptv->
req.
v2.tp_block_nr = ptv->
req.
v2.tp_frame_nr / frames_per_block + 1;
1659 ptv->
req.
v2.tp_frame_nr = ptv->
req.
v2.tp_block_nr * frames_per_block;
1660 SCLogPerf(
"%s: rx ring: block_size=%d block_nr=%d frame_size=%d frame_nr=%d", ptv->
iface,
1661 ptv->
req.
v2.tp_block_size, ptv->
req.
v2.tp_block_nr, ptv->
req.
v2.tp_frame_size,
1662 ptv->
req.
v2.tp_frame_nr);
1669 ptv->
req.
v3.tp_frame_size = 2048;
1670 int frames_per_block = 0;
1671 int tp_hdrlen =
sizeof(
struct tpacket3_hdr);
1677 SCLogWarning(
"%s: unable to get MTU, setting snaplen default of 1514", ptv->
iface);
1683 ptv->
req.
v3.tp_frame_size = TPACKET_ALIGN(snaplen +TPACKET_ALIGN(TPACKET_ALIGN(tp_hdrlen) +
sizeof(
struct sockaddr_ll) + ETH_HLEN) - ETH_HLEN);
1684 frames_per_block = ptv->
req.
v3.tp_block_size / ptv->
req.
v3.tp_frame_size;
1686 if (frames_per_block == 0) {
1687 SCLogError(
"%s: block size is too small, it should be at least %d", ptv->
iface,
1688 ptv->
req.
v3.tp_frame_size);
1693 ptv->
req.
v3.tp_frame_nr = ptv->
req.
v3.tp_block_nr * frames_per_block;
1695 ptv->
req.
v3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH;
1696 SCLogPerf(
"%s: rx ring params: block_size=%d block_nr=%d frame_size=%d frame_nr=%d (mem: %d)",
1698 ptv->
req.
v3.tp_frame_size, ptv->
req.
v3.tp_frame_nr,
1699 ptv->
req.
v3.tp_block_size * ptv->
req.
v3.tp_block_nr);
1706 unsigned int len =
sizeof(val), i;
1715 if (getsockopt(ptv->
socket, SOL_PACKET, PACKET_HDRLEN, &val, &
len) < 0) {
1716 if (errno == ENOPROTOOPT) {
1718 SCLogError(
"%s: kernel too old for TPACKET_V3 (need 3.2+)", devname);
1720 SCLogError(
"%s: kernel too old (need 2.6.27+)", devname);
1723 SCLogError(
"%s: failed to retrieve packet header len", devname);
1731 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_VERSION, &val,
1733 SCLogError(
"%s: failed to activate TPACKET_V2/TPACKET_V3 on packet socket: %s", devname,
1738 #ifdef HAVE_HW_TIMESTAMPING
1739 int req = SOF_TIMESTAMPING_RAW_HARDWARE;
1740 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_TIMESTAMP, (
void *) &req,
1742 SCLogWarning(
"%s: failed to activate hardware timestamping on packet socket: %s", devname,
1750 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_RESERVE, (
void *)&reserve,
sizeof(reserve)) <
1752 SCLogError(
"%s: failed to activate reserve on packet socket: %s", devname, strerror(errno));
1758 if (AFPComputeRingParamsV3(ptv) != 1) {
1761 r = setsockopt(ptv->
socket, SOL_PACKET, PACKET_RX_RING,
1764 SCLogError(
"%s: failed to allocate RX Ring: %s", devname, strerror(errno));
1770 if (AFPComputeRingParamsWithBlockSize(ptv, ptv->
v2_block_size) != 1) {
1771 SCLogError(
"%s: ring parameters are incorrect. Please file a bug report", devname);
1776 ptv->
socket, SOL_PACKET, PACKET_RX_RING, (
void *)&ptv->
req,
sizeof(ptv->
req));
1779 if (errno == ENOMEM) {
1780 SCLogError(
"%s: memory issue with ring parameters", devname);
1783 SCLogError(
"%s: failed to setup RX Ring: %s", devname, strerror(errno));
1789 if (AFPComputeRingParams(ptv, order) != 1) {
1791 "%s: ring parameters are incorrect. Please file a bug report", devname);
1795 r = setsockopt(ptv->
socket, SOL_PACKET, PACKET_RX_RING, (
void *)&ptv->
req,
1799 if (errno == ENOMEM) {
1800 SCLogWarning(
"%s: memory issue with ring parameters. Retrying", devname);
1803 SCLogError(
"%s: failed to setup RX Ring: %s", devname, strerror(errno));
1810 SCLogError(
"%s: failed to setup RX Ring (order 0 failed)", devname);
1822 mmap_flag = MAP_SHARED;
1824 mmap_flag |= MAP_LOCKED;
1826 mmap_flag, ptv->
socket, 0);
1828 SCLogError(
"%s: failed to mmap: %s", devname, strerror(errno));
1834 SCLogError(
"%s: failed to alloc ring: %s", devname, strerror(errno));
1837 for (i = 0; i < ptv->
req.
v3.tp_block_nr; ++i) {
1839 ptv->
ring.
v3[i].iov_len = ptv->
req.
v3.tp_block_size;
1844 if (ptv->
ring.
v2 == NULL) {
1845 SCLogError(
"%s: failed to alloc ring: %s", devname, strerror(errno));
1850 for (i = 0; i < ptv->
req.
v2.tp_block_nr; ++i) {
1851 void *base = &(ptv->
ring_buf[i * ptv->
req.
v2.tp_block_size]);
1855 base += ptv->
req.
v2.tp_frame_size;
1879 #ifdef HAVE_PACKET_FANOUT
1880 int fd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
1885 uint32_t option = (mode << 16) | cluster_id;
1886 int r = setsockopt(fd, SOL_PACKET,
PACKET_FANOUT,(
void *)&option,
sizeof(option));
1890 SCLogError(
"fanout not supported by kernel: "
1891 "Kernel too old or cluster-id %d already in use.",
1901 #ifdef HAVE_PACKET_EBPF
1905 int pfd = ptv->ebpf_lb_fd;
1907 SCLogError(
"Fanout file descriptor is invalid");
1911 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_FANOUT_DATA, &pfd,
sizeof(pfd))) {
1922 int pfd = ptv->ebpf_filter_fd;
1924 SCLogError(
"Filter file descriptor is invalid");
1928 if (setsockopt(ptv->
socket, SOL_SOCKET, SO_ATTACH_BPF, &pfd,
sizeof(pfd))) {
1929 SCLogError(
"Error setting ebpf: %s", strerror(errno));
1932 SCLogInfo(
"Activated eBPF filter on socket");
1939 static int AFPCreateSocket(
AFPThreadVars *ptv,
char *devname,
int verbose,
const bool peer_update)
1943 struct packet_mreq sock_params;
1944 struct sockaddr_ll bind_address;
1948 ptv->
socket = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
1950 SCLogError(
"%s: failed to create socket: %s", devname, strerror(errno));
1954 if_idx = AFPGetIfnumByDev(ptv->
socket, devname, verbose);
1960 memset(&bind_address, 0,
sizeof(bind_address));
1961 bind_address.sll_family = AF_PACKET;
1962 bind_address.sll_protocol = htons(ETH_P_ALL);
1963 bind_address.sll_ifindex = if_idx;
1964 if (bind_address.sll_ifindex == -1) {
1971 int if_flags = AFPGetDevFlags(ptv->
socket, ptv->
iface);
1972 if (if_flags == -1) {
1978 }
else if ((if_flags & (IFF_UP | IFF_RUNNING)) == 0) {
1987 if (if_flags & IFF_LOOPBACK)
1992 memset(&sock_params, 0,
sizeof(sock_params));
1993 sock_params.mr_type = PACKET_MR_PROMISC;
1994 sock_params.mr_ifindex = bind_address.sll_ifindex;
1995 r = setsockopt(ptv->
socket, SOL_PACKET, PACKET_ADD_MEMBERSHIP,(
void *)&sock_params,
sizeof(sock_params));
1997 SCLogError(
"%s: failed to set promisc mode: %s", devname, strerror(errno));
2004 if (setsockopt(ptv->
socket, SOL_PACKET, PACKET_AUXDATA, &val,
2005 sizeof(val)) == -1 && errno != ENOPROTOOPT) {
2007 "%s: 'kernel' checksum mode not supported, falling back to full mode", devname);
2018 if (setsockopt(ptv->
socket, SOL_SOCKET, SO_RCVBUF,
2027 r = bind(ptv->
socket, (
struct sockaddr *)&bind_address,
sizeof(bind_address));
2030 if (errno == ENETDOWN) {
2031 SCLogWarning(
"%s: failed to bind socket, iface is down", devname);
2033 SCLogWarning(
"%s: failed to bind socket: %s", devname, strerror(errno));
2041 #ifdef HAVE_PACKET_FANOUT
2046 uint32_t option = (mode << 16) | (
id & 0xffff);
2049 SCLogError(
"%s: failed to set fanout mode: %s", devname, strerror(errno));
2055 #ifdef HAVE_PACKET_EBPF
2057 r = SockFanoutSeteBPF(ptv);
2059 SCLogError(
"%s: failed to set eBPF: %s", devname, strerror(errno));
2066 AFPPeersListReachedInc();
2068 ret = AFPSetupRing(ptv, devname);
2072 SCLogDebug(
"Using interface '%s' via socket %d", (
char *)devname, ptv->
socket);
2076 TmEcode rc = AFPSetBPFFilter(ptv);
2107 struct bpf_program filter;
2108 struct sock_fprog fcode;
2111 #ifdef HAVE_PACKET_EBPF
2112 if (ptv->ebpf_filter_fd != -1) {
2113 return SetEbpfFilter(ptv);
2122 char errbuf[PCAP_ERRBUF_SIZE];
2129 errbuf,
sizeof(errbuf)) == -1) {
2134 if (filter.bf_len > USHRT_MAX) {
2137 fcode.len = (
unsigned short)filter.bf_len;
2138 fcode.filter = (
struct sock_filter*)filter.bf_insns;
2140 rc = setsockopt(ptv->
socket, SOL_SOCKET, SO_ATTACH_FILTER, &fcode,
sizeof(fcode));
2144 SCLogError(
"%s: failed to attach filter: %s", ptv->
iface, strerror(errno));
2151 #ifdef HAVE_PACKET_EBPF
2159 static int AFPInsertHalfFlow(
int mapd,
void *key,
unsigned int nr_cpus)
2161 BPF_DECLARE_PERCPU(
struct pair, value, nr_cpus);
2170 for (i = 0; i < nr_cpus; i++) {
2171 BPF_PERCPU(value, i).packets = 0;
2172 BPF_PERCPU(value, i).bytes = 0;
2174 if (bpf_map_update_elem(mapd, key, value, BPF_NOEXIST) != 0) {
2187 SCLogError(
"Can't update eBPF map: %s (%d)", strerror(errno), errno);
2194 static int AFPSetFlowStorage(
Packet *p,
int map_fd,
void *key0,
void* key1,
2205 EBPFBypassData *eb =
SCCalloc(1,
sizeof(EBPFBypassData));
2207 EBPFDeleteKey(map_fd, key0);
2208 EBPFDeleteKey(map_fd, key1);
2217 eb->cpus_count = p->afp_v.nr_cpus;
2222 EBPFDeleteKey(map_fd, key0);
2223 EBPFDeleteKey(map_fd, key1);
2249 static int AFPBypassCallback(
Packet *p)
2251 SCLogDebug(
"Calling af_packet callback function");
2253 if (!(PacketIsTCP(p) || PacketIsUDP(p))) {
2260 if (p->
flow == NULL) {
2266 if (PacketIsTunnel(p)) {
2269 if (PacketIsIPv4(p)) {
2271 if (p->afp_v.v4_map_fd == -1) {
2274 struct flowv4_keys *keys[2];
2275 keys[0] =
SCCalloc(1,
sizeof(
struct flowv4_keys));
2276 if (keys[0] == NULL) {
2281 keys[0]->port16[0] = p->
sp;
2282 keys[0]->port16[1] = p->
dp;
2283 keys[0]->vlan0 = p->
vlan_id[0];
2284 keys[0]->vlan1 = p->
vlan_id[1];
2285 keys[0]->vlan2 = p->
vlan_id[2];
2287 if (p->
proto == IPPROTO_TCP) {
2288 keys[0]->ip_proto = 1;
2290 keys[0]->ip_proto = 0;
2292 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[0],
2293 p->afp_v.nr_cpus) == 0) {
2298 keys[1]=
SCCalloc(1,
sizeof(
struct flowv4_keys));
2299 if (keys[1] == NULL) {
2300 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2307 keys[1]->port16[0] = p->
dp;
2308 keys[1]->port16[1] = p->
sp;
2309 keys[1]->vlan0 = p->
vlan_id[0];
2310 keys[1]->vlan1 = p->
vlan_id[1];
2311 keys[1]->vlan2 = p->
vlan_id[2];
2313 keys[1]->ip_proto = keys[0]->ip_proto;
2314 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[1],
2315 p->afp_v.nr_cpus) == 0) {
2316 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2322 EBPFUpdateFlow(p->
flow, p, NULL);
2323 return AFPSetFlowStorage(p, p->afp_v.v4_map_fd, keys[0], keys[1], AF_INET);
2326 if (PacketIsIPv6(p) && ((p->
proto == IPPROTO_TCP) || (p->
proto == IPPROTO_UDP))) {
2328 if (p->afp_v.v6_map_fd == -1) {
2332 struct flowv6_keys *keys[2];
2333 keys[0] =
SCCalloc(1,
sizeof(
struct flowv6_keys));
2334 if (keys[0] == NULL) {
2338 for (i = 0; i < 4; i++) {
2342 keys[0]->port16[0] = p->
sp;
2343 keys[0]->port16[1] = p->
dp;
2344 keys[0]->vlan0 = p->
vlan_id[0];
2345 keys[0]->vlan1 = p->
vlan_id[1];
2346 keys[0]->vlan2 = p->
vlan_id[2];
2348 if (p->
proto == IPPROTO_TCP) {
2349 keys[0]->ip_proto = 1;
2351 keys[0]->ip_proto = 0;
2353 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[0],
2354 p->afp_v.nr_cpus) == 0) {
2359 keys[1]=
SCCalloc(1,
sizeof(
struct flowv6_keys));
2360 if (keys[1] == NULL) {
2361 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2366 for (i = 0; i < 4; i++) {
2370 keys[1]->port16[0] = p->
dp;
2371 keys[1]->port16[1] = p->
sp;
2372 keys[1]->vlan0 = p->
vlan_id[0];
2373 keys[1]->vlan1 = p->
vlan_id[1];
2374 keys[1]->vlan2 = p->
vlan_id[2];
2376 keys[1]->ip_proto = keys[0]->ip_proto;
2377 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[1],
2378 p->afp_v.nr_cpus) == 0) {
2379 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2386 EBPFUpdateFlow(p->
flow, p, NULL);
2387 return AFPSetFlowStorage(p, p->afp_v.v6_map_fd, keys[0], keys[1], AF_INET6);
2403 static int AFPXDPBypassCallback(
Packet *p)
2405 SCLogDebug(
"Calling af_packet callback function");
2407 if (!(PacketIsTCP(p) || PacketIsUDP(p))) {
2414 if (p->
flow == NULL) {
2420 if (PacketIsTunnel(p)) {
2423 if (PacketIsIPv4(p)) {
2424 struct flowv4_keys *keys[2];
2425 keys[0]=
SCCalloc(1,
sizeof(
struct flowv4_keys));
2426 if (keys[0] == NULL) {
2430 if (p->afp_v.v4_map_fd == -1) {
2434 keys[0]->src = p->
src.addr_data32[0];
2435 keys[0]->dst = p->
dst.addr_data32[0];
2438 keys[0]->port16[0] = htons(p->
sp);
2439 keys[0]->port16[1] = htons(p->
dp);
2440 keys[0]->vlan0 = p->
vlan_id[0];
2441 keys[0]->vlan1 = p->
vlan_id[1];
2442 keys[0]->vlan2 = p->
vlan_id[2];
2443 if (p->
proto == IPPROTO_TCP) {
2444 keys[0]->ip_proto = 1;
2446 keys[0]->ip_proto = 0;
2448 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[0],
2449 p->afp_v.nr_cpus) == 0) {
2454 keys[1]=
SCCalloc(1,
sizeof(
struct flowv4_keys));
2455 if (keys[1] == NULL) {
2456 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2461 keys[1]->src = p->
dst.addr_data32[0];
2462 keys[1]->dst = p->
src.addr_data32[0];
2463 keys[1]->port16[0] = htons(p->
dp);
2464 keys[1]->port16[1] = htons(p->
sp);
2465 keys[1]->vlan0 = p->
vlan_id[0];
2466 keys[1]->vlan1 = p->
vlan_id[1];
2467 keys[1]->vlan2 = p->
vlan_id[2];
2468 keys[1]->ip_proto = keys[0]->ip_proto;
2469 if (AFPInsertHalfFlow(p->afp_v.v4_map_fd, keys[1],
2470 p->afp_v.nr_cpus) == 0) {
2471 EBPFDeleteKey(p->afp_v.v4_map_fd, keys[0]);
2477 return AFPSetFlowStorage(p, p->afp_v.v4_map_fd, keys[0], keys[1], AF_INET);
2480 if (PacketIsIPv6(p) && ((p->
proto == IPPROTO_TCP) || (p->
proto == IPPROTO_UDP))) {
2482 if (p->afp_v.v6_map_fd == -1) {
2486 struct flowv6_keys *keys[2];
2487 keys[0] =
SCCalloc(1,
sizeof(
struct flowv6_keys));
2488 if (keys[0] == NULL) {
2492 for (i = 0; i < 4; i++) {
2496 keys[0]->port16[0] = htons(p->
sp);
2497 keys[0]->port16[1] = htons(p->
dp);
2498 keys[0]->vlan0 = p->
vlan_id[0];
2499 keys[0]->vlan1 = p->
vlan_id[1];
2500 keys[0]->vlan2 = p->
vlan_id[2];
2501 if (p->
proto == IPPROTO_TCP) {
2502 keys[0]->ip_proto = 1;
2504 keys[0]->ip_proto = 0;
2506 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[0],
2507 p->afp_v.nr_cpus) == 0) {
2512 keys[1]=
SCCalloc(1,
sizeof(
struct flowv6_keys));
2513 if (keys[1] == NULL) {
2514 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2519 for (i = 0; i < 4; i++) {
2523 keys[1]->port16[0] = htons(p->
dp);
2524 keys[1]->port16[1] = htons(p->
sp);
2525 keys[1]->vlan0 = p->
vlan_id[0];
2526 keys[1]->vlan1 = p->
vlan_id[1];
2527 keys[1]->vlan2 = p->
vlan_id[2];
2528 keys[1]->ip_proto = keys[0]->ip_proto;
2529 if (AFPInsertHalfFlow(p->afp_v.v6_map_fd, keys[1],
2530 p->afp_v.nr_cpus) == 0) {
2531 EBPFDeleteKey(p->afp_v.v6_map_fd, keys[0]);
2537 return AFPSetFlowStorage(p, p->afp_v.v6_map_fd, keys[0], keys[1], AF_INET6);
2542 bool g_flowv4_ok =
true;
2543 bool g_flowv6_ok =
true;
2561 if (initdata == NULL) {
2595 #ifdef HAVE_PACKET_FANOUT
2610 #ifdef HAVE_PACKET_EBPF
2613 ptv->xdp_mode = afpconfig->
xdp_mode;
2617 ptv->v4_map_fd = EBPFGetMapFDByName(ptv->
iface,
"flow_table_v4");
2618 if (ptv->v4_map_fd == -1) {
2620 SCLogError(
"Can't find eBPF map fd for '%s'",
"flow_table_v4");
2624 ptv->v6_map_fd = EBPFGetMapFDByName(ptv->
iface,
"flow_table_v6");
2625 if (ptv->v6_map_fd == -1) {
2627 SCLogError(
"Can't find eBPF map fd for '%s'",
"flow_table_v6");
2628 g_flowv6_ok =
false;
2632 ptv->ebpf_t_config = afpconfig->ebpf_t_config;
2635 #ifdef PACKET_STATISTICS
2659 SCLogWarning(
"Enabling a BPF filter in IPS mode result"
2660 " in dropping all non matching packets.");
2671 *data = (
void *)ptv;
2690 void ReceiveAFPThreadExitStats(
ThreadVars *
tv,
void *data)
2695 #ifdef PACKET_STATISTICS
2696 AFPDumpCounters(ptv);
2714 #ifdef HAVE_PACKET_XDP
2715 if ((ptv->ebpf_t_config.
flags & EBPF_XDP_CODE) &&
2716 (!(ptv->ebpf_t_config.
flags & EBPF_PINNED_MAPS))) {
2717 EBPFSetupXDP(ptv->
iface, -1, ptv->xdp_mode);
2743 static void UpdateRawDataForVLANHdr(
Packet *p)
2745 if (p->afp_v.vlan_tci != 0) {
2751 *(uint16_t *)(pstart + 2 * ETH_ALEN) = htons(0x8100);
2752 *(uint16_t *)(pstart + 2 * ETH_ALEN + 2) = htons(p->afp_v.vlan_tci);
2775 const bool afp_vlan_hdr = p->
vlan_idx != 0;
2788 UpdateRawDataForVLANHdr(p);
2805 *data = (
void *)
dtv;