suricata
source-napatech.c
Go to the documentation of this file.
1 /* Copyright (C) 2012-2017 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21 - * \author nPulse Technologies, LLC.
22 - * \author Matt Keeler <mk@npulsetech.com>
23  * *
24  * Support for NAPATECH adapter with the 3GD Driver/API.
25  * Requires libntapi from Napatech A/S.
26  *
27  */
28 #include "suricata-common.h"
29 #include "suricata.h"
30 #include "threadvars.h"
31 #include "util-optimize.h"
32 #include "tm-queuehandlers.h"
33 #include "tm-threads.h"
34 #include "tm-modules.h"
35 #include "util-privs.h"
36 #include "tmqh-packetpool.h"
37 #include "util-napatech.h"
38 #include "source-napatech.h"
39 
40 #ifndef HAVE_NAPATECH
41 
42 TmEcode NoNapatechSupportExit(ThreadVars *, const void *, void **);
43 
45  tmm_modules[TMM_RECEIVENAPATECH].name = "NapatechStream";
46  tmm_modules[TMM_RECEIVENAPATECH].ThreadInit = NoNapatechSupportExit;
52 }
53 
55  tmm_modules[TMM_DECODENAPATECH].name = "NapatechDecode";
56  tmm_modules[TMM_DECODENAPATECH].ThreadInit = NoNapatechSupportExit;
63 }
64 
65 TmEcode NoNapatechSupportExit(ThreadVars *tv, const void *initdata, void **data) {
67  "Error creating thread %s: you do not have support for Napatech adapter "
68  "enabled please recompile with --enable-napatech", tv->name);
69  exit(EXIT_FAILURE);
70 }
71 
72 #else /* Implied we do have NAPATECH support */
73 
74 
75 #include <nt.h>
76 
77 #define MAX_STREAMS 256
78 
79 extern int max_pending_packets;
80 
81 typedef struct NapatechThreadVars_ {
83  NtNetStreamRx_t rx_stream;
84  uint16_t stream_id;
85  int hba;
88 
89 
90 TmEcode NapatechStreamThreadInit(ThreadVars *, const void *, void **);
92 TmEcode NapatechPacketLoopZC(ThreadVars *tv, void *data, void *slot);
93 
94 TmEcode NapatechDecodeThreadInit(ThreadVars *, const void *, void **);
97 
98 /* These are used as the threads are exiting to get a comprehensive count of
99  * all the packets received and dropped.
100  */
101 SC_ATOMIC_DECLARE(uint64_t, total_packets);
102 SC_ATOMIC_DECLARE(uint64_t, total_drops);
103 SC_ATOMIC_DECLARE(uint16_t, total_tallied);
104 
105 /**
106  * \brief Register the Napatech receiver (reader) module.
107  */
109 {
110  tmm_modules[TMM_RECEIVENAPATECH].name = "NapatechStream";
120 
121  SC_ATOMIC_INIT(total_packets);
122  SC_ATOMIC_INIT(total_drops);
123  SC_ATOMIC_INIT(total_tallied);
124 }
125 
126 /**
127  * \brief Register the Napatech decoder module.
128  */
130 {
131  tmm_modules[TMM_DECODENAPATECH].name = "NapatechDecode";
139 }
140 
141 /*
142  *-----------------------------------------------------------------------------
143  *-----------------------------------------------------------------------------
144  * Statistics code
145  *-----------------------------------------------------------------------------
146 */
147 
148 /**
149  * \brief Initialize the Napatech receiver thread, generate a single
150  * NapatechThreadVar structure for each thread, this will
151  * contain a NtNetStreamRx_t stream handle which is used when the
152  * thread executes to acquire the packets.
153  *
154  * \param tv Thread variable to ThreadVars
155  * \param initdata Initial data to the adapter passed from the user,
156  * this is processed by the user.
157  *
158  * For now, we assume that we have only a single name for the NAPATECH
159  * adapter.
160  *
161  * \param data data pointer gets populated with
162  *
163  */
164 TmEcode NapatechStreamThreadInit(ThreadVars *tv, const void *initdata, void **data)
165 {
166  SCEnter();
167  struct NapatechStreamDevConf *conf = (struct NapatechStreamDevConf *) initdata;
168  uint16_t stream_id = conf->stream_id;
169  *data = NULL;
170 
171  NapatechThreadVars *ntv = SCCalloc(1, sizeof (NapatechThreadVars));
172  if (unlikely(ntv == NULL)) {
173  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate memory for NAPATECH thread vars.");
174  exit(EXIT_FAILURE);
175  }
176 
177  memset(ntv, 0, sizeof (NapatechThreadVars));
178  ntv->stream_id = stream_id;
179  ntv->tv = tv;
180  ntv->hba = conf->hba;
181  SCLogDebug("Started processing packets from NAPATECH Stream: %lu", ntv->stream_id);
182 
183  *data = (void *) ntv;
185 }
186 
187 static PacketQueue packets_to_release[MAX_STREAMS];
188 
189 static void NapatechReleasePacket(struct Packet_ *p)
190 {
192  PacketEnqueue(&packets_to_release[p->ntpv.stream_id], p);
193 }
194 
196 {
197  int32_t status;
198  char error_buffer[100];
199  uint64_t pkt_ts;
200  NtNetBuf_t packet_buffer;
201  NapatechThreadVars *ntv = (NapatechThreadVars *) data;
202  uint64_t hba_pkt_drops = 0;
203  uint64_t hba_byte_drops = 0;
204  uint16_t hba_pkt = 0;
205 
206  /* This just keeps the startup output more orderly. */
207  usleep(200000 * ntv->stream_id);
208 
209  if (ntv->hba > 0) {
210  char *s_hbad_pkt = SCCalloc(1, 32);
211  if (unlikely(s_hbad_pkt == NULL)) {
212  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate memory for NAPATECH stream counter.");
213  exit(EXIT_FAILURE);
214  }
215  snprintf(s_hbad_pkt, 32, "nt%d.hba_drop", ntv->stream_id);
216  hba_pkt = StatsRegisterCounter(s_hbad_pkt, tv);
217  StatsSetupPrivate(tv);
218  StatsSetUI64(tv, hba_pkt, 0);
219  }
220  SCLogDebug("Opening NAPATECH Stream: %lu for processing", ntv->stream_id);
221 
222  if ((status = NT_NetRxOpen(&(ntv->rx_stream), "SuricataStream", NT_NET_INTERFACE_PACKET, ntv->stream_id, ntv->hba)) != NT_SUCCESS) {
223  NT_ExplainError(status, error_buffer, sizeof (error_buffer));
224  SCLogError(SC_ERR_NAPATECH_OPEN_FAILED, "Failed to open NAPATECH Stream: %u - %s", ntv->stream_id, error_buffer);
225  SCFree(ntv);
227  }
228 
229 #if defined(__linux__)
230  SCLogInfo("Napatech Packet Loop Started - cpu: %3d, stream: %3u (numa: %u)",
231  sched_getcpu(), ntv->stream_id, NapatechGetNumaNode(ntv->stream_id));
232 #else
233  SCLogInfo("Napatech Packet Loop Started - stream: %lu ", ntv->stream_id);
234 #endif
235 
236  TmSlot *s = (TmSlot *) slot;
237  ntv->slot = s->slot_next;
238 
239  while (!(suricata_ctl_flags & SURICATA_STOP)) {
240  /* make sure we have at least one packet in the packet pool, to prevent
241  * us from alloc'ing packets at line rate */
242  PacketPoolWait();
243 
244  /* Napatech returns packets 1 at a time */
245  status = NT_NetRxGet(ntv->rx_stream, &packet_buffer, 1000);
246  if (unlikely(status == NT_STATUS_TIMEOUT || status == NT_STATUS_TRYAGAIN)) {
247  continue;
248  } else if (unlikely(status != NT_SUCCESS)) {
249  NT_ExplainError(status, error_buffer, sizeof (error_buffer) - 1);
250 
251  SCLogInfo("Failed to read from Napatech Stream%d: %s",
252  ntv->stream_id, error_buffer);
254  }
255 
257  if (unlikely(p == NULL)) {
258  NT_NetRxRelease(ntv->rx_stream, packet_buffer);
260  }
261 
262  pkt_ts = NT_NET_GET_PKT_TIMESTAMP(packet_buffer);
263 
264  /*
265  * Handle the different timestamp forms that the napatech cards could use
266  * - NT_TIMESTAMP_TYPE_NATIVE is not supported due to having an base of 0 as opposed to NATIVE_UNIX which has a base of 1/1/1970
267  */
268  switch (NT_NET_GET_PKT_TIMESTAMP_TYPE(packet_buffer)) {
269  case NT_TIMESTAMP_TYPE_NATIVE_UNIX:
270  p->ts.tv_sec = pkt_ts / 100000000;
271  p->ts.tv_usec = ((pkt_ts % 100000000) / 100) + (pkt_ts % 100) > 50 ? 1 : 0;
272  break;
273  case NT_TIMESTAMP_TYPE_PCAP:
274  p->ts.tv_sec = pkt_ts >> 32;
275  p->ts.tv_usec = pkt_ts & 0xFFFFFFFF;
276  break;
277  case NT_TIMESTAMP_TYPE_PCAP_NANOTIME:
278  p->ts.tv_sec = pkt_ts >> 32;
279  p->ts.tv_usec = ((pkt_ts & 0xFFFFFFFF) / 1000) + (pkt_ts % 1000) > 500 ? 1 : 0;
280  break;
281  case NT_TIMESTAMP_TYPE_NATIVE_NDIS:
282  /* number of seconds between 1/1/1601 and 1/1/1970 */
283  p->ts.tv_sec = (pkt_ts / 100000000) - 11644473600;
284  p->ts.tv_usec = ((pkt_ts % 100000000) / 100) + (pkt_ts % 100) > 50 ? 1 : 0;
285  break;
286  default:
288  "Packet from Napatech Stream: %u does not have a supported timestamp format",
289  ntv->stream_id);
290  NT_NetRxRelease(ntv->rx_stream, packet_buffer);
292  }
293 
294  if (unlikely(ntv->hba > 0)) {
295  NtNetRx_t stat_cmd;
296  stat_cmd.cmd = NT_NETRX_READ_CMD_STREAM_DROP;
297  // Update drop counter
298  if (unlikely((status = NT_NetRxRead(ntv->rx_stream, &stat_cmd)) != NT_SUCCESS)) {
299  NT_ExplainError(status, error_buffer, sizeof (error_buffer));
300  SCLogInfo("Couldn't retrieve drop statistics from the RX stream: %u - %s",
301  ntv->stream_id, error_buffer);
302  } else {
303  hba_pkt_drops = stat_cmd.u.streamDrop.pktsDropped;
304 
305  StatsSetUI64(tv, hba_pkt, hba_pkt_drops);
306  }
308  }
309 
310  p->ReleasePacket = NapatechReleasePacket;
311  p->ntpv.nt_packet_buf = packet_buffer;
312  p->ntpv.stream_id = ntv->stream_id;
314 
315  if (unlikely(PacketSetData(p, (uint8_t *)NT_NET_GET_PKT_L2_PTR(packet_buffer), NT_NET_GET_PKT_WIRE_LENGTH(packet_buffer)))) {
316  TmqhOutputPacketpool(ntv->tv, p);
317  NT_NetRxRelease(ntv->rx_stream, packet_buffer);
319  }
320 
321  if (unlikely(TmThreadsSlotProcessPkt(ntv->tv, ntv->slot, p) != TM_ECODE_OK)) {
322  TmqhOutputPacketpool(ntv->tv, p);
323  NT_NetRxRelease(ntv->rx_stream, packet_buffer);
325  }
326 
327  /* Release any packets that were returned by the callback function */
328  Packet *rel_pkt = PacketDequeue(&packets_to_release[ntv->stream_id]);
329  while (rel_pkt != NULL) {
330  NT_NetRxRelease(ntv->rx_stream, rel_pkt->ntpv.nt_packet_buf);
331  rel_pkt = PacketDequeue(&packets_to_release[ntv->stream_id]);
332  }
334  }
335 
336  if (unlikely(ntv->hba > 0)) {
337  SCLogInfo("Host Buffer Allowance Drops - pkts: %ld, bytes: %ld", hba_pkt_drops, hba_byte_drops);
338  }
339 
341 }
342 
343 
344 /**
345  * \brief Print some stats to the log at program exit.
346  *
347  * \param tv Pointer to ThreadVars.
348  * \param data Pointer to data, ErfFileThreadVars.
349  */
351 {
352  NapatechThreadVars *ntv = (NapatechThreadVars *) data;
354 
355  double percent = 0;
356  if (stat.current_drops > 0)
357  percent = (((double) stat.current_drops)
358  / (stat.current_packets + stat.current_drops)) * 100;
359 
360  SCLogInfo("nt%lu - pkts: %lu; drop: %lu (%5.2f%%); bytes: %lu",
361  (uint64_t) ntv->stream_id, stat.current_packets,
362  stat.current_drops, percent, stat.current_bytes);
363 
364  SC_ATOMIC_ADD(total_packets, stat.current_packets);
365  SC_ATOMIC_ADD(total_drops, stat.current_drops);
366  SC_ATOMIC_ADD(total_tallied, 1);
367 
368  if (SC_ATOMIC_GET(total_tallied) == GetNumConfiguredStreams()) {
369  if (SC_ATOMIC_GET(total_drops) > 0)
370  percent = (((double) SC_ATOMIC_GET(total_drops)) / (SC_ATOMIC_GET(total_packets)
371  + SC_ATOMIC_GET(total_drops))) * 100;
372 
373  SCLogInfo(" ");
374  SCLogInfo("--- Total Packets: %ld Total Dropped: %ld (%5.2f%%)",
375  SC_ATOMIC_GET(total_packets), SC_ATOMIC_GET(total_drops), percent);
376  }
377 }
378 
379 /**
380  * \brief Deinitializes the NAPATECH card.
381  * \param tv pointer to ThreadVars
382  * \param data pointer that gets cast into PcapThreadVars for ptv
383  */
385 {
386  SCEnter();
387  NapatechThreadVars *ntv = (NapatechThreadVars *) data;
388  SCLogDebug("Closing Napatech Stream: %d", ntv->stream_id);
389  NT_NetRxClose(ntv->rx_stream);
391 }
392 
393 /** Decode Napatech */
394 
395 /**
396  * \brief This function passes off to link type decoders.
397  *
398  * NapatechDecode reads packets from the PacketQueue and passes
399  * them off to the proper link type decoder.
400  *
401  * \param t pointer to ThreadVars
402  * \param p pointer to the current packet
403  * \param data pointer that gets cast into PcapThreadVars for ptv
404  * \param pq pointer to the current PacketQueue
405  */
407  PacketQueue *postpq)
408 {
409  SCEnter();
410 
411  DecodeThreadVars *dtv = (DecodeThreadVars *) data;
412 
413  /* XXX HACK: flow timeout can call us for injected pseudo packets
414  * see bug: https://redmine.openinfosecfoundation.org/issues/1107 */
415  if (p->flags & PKT_PSEUDO_STREAM_END)
416  return TM_ECODE_OK;
417 
418  // update counters
419  DecodeUpdatePacketCounters(tv, dtv, p);
420 
421  switch (p->datalink) {
422  case LINKTYPE_ETHERNET:
423  DecodeEthernet(tv, dtv, p, GET_PKT_DATA(p), GET_PKT_LEN(p), pq);
424  break;
425  default:
427  "Error: datalink type %" PRId32 " not yet supported in module NapatechDecode",
428  p->datalink);
429  break;
430  }
431 
432  PacketDecodeFinalize(tv, dtv, p);
434 }
435 
436 TmEcode NapatechDecodeThreadInit(ThreadVars *tv, const void *initdata, void **data)
437 {
438  SCEnter();
439  DecodeThreadVars *dtv = NULL;
440  dtv = DecodeThreadVarsAlloc(tv);
441  if (dtv == NULL)
443 
445  *data = (void *) dtv;
447 }
448 
450 {
451  if (data != NULL)
452  DecodeThreadVarsFree(tv, data);
454 
455 #endif /* HAVE_NAPATECH */
#define TM_FLAG_DECODE_TM
Definition: tm-modules.h:32
DecodeThreadVars * DecodeThreadVarsAlloc(ThreadVars *tv)
Alloc and setup DecodeThreadVars.
Definition: decode.c:570
#define SCLogDebug(...)
Definition: util-debug.h:335
uint8_t cap_flags
Definition: tm-modules.h:67
NapatechCurrentStats NapatechGetCurrentStats(uint16_t id)
Definition: util-napatech.c:84
uint8_t flags
Definition: tm-modules.h:70
TmEcode NapatechStreamThreadInit(ThreadVars *, const void *, void **)
Initialize the Napatech receiver thread, generate a single NapatechThreadVar structure for each threa...
void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p)
Finalize decoding of a packet.
Definition: decode.c:114
#define unlikely(expr)
Definition: util-optimize.h:35
TmEcode(* Func)(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *)
Definition: tm-modules.h:52
void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
Definition: decode.c:431
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:108
volatile uint8_t suricata_ctl_flags
Definition: suricata.c:201
#define SC_CAP_NET_ADMIN
Definition: util-privs.h:31
Packet * PacketGetFromQueueOrAlloc(void)
Get a packet. We try to get a packet from the packetpool first, but if that is empty we alloc a packe...
Definition: decode.c:176
void TmModuleNapatechStreamRegister(void)
Register the Napatech receiver (reader) module.
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
void(* ReleasePacket)(struct Packet_ *)
Definition: decode.h:487
void PacketFreeOrRelease(Packet *p)
Return a packet to where it was allocated.
Definition: decode.c:161
void TmModuleNapatechDecodeRegister(void)
Register the Napatech decoder module.
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
uint16_t StatsRegisterCounter(const char *name, struct ThreadVars_ *tv)
Registers a normal, unqualified counter.
Definition: counters.c:939
SC_ATOMIC_DECLARE(uint64_t, total_packets)
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:82
#define SURICATA_STOP
Definition: suricata.h:95
#define SCCalloc(nm, a)
Definition: util-mem.h:205
void StatsSetUI64(ThreadVars *tv, uint16_t id, uint64_t x)
Sets a value of type double to the local counter.
Definition: counters.c:185
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
NtNetBuf_t nt_packet_buf
Definition: util-napatech.h:34
int max_pending_packets
Definition: suricata.c:215
int datalink
Definition: decode.h:579
TmEcode NapatechStreamThreadDeinit(ThreadVars *tv, void *data)
Deinitializes the NAPATECH card.
int DecodeEthernet(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, uint8_t *pkt, uint32_t len, PacketQueue *pq)
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
Structure to hold thread specific data for all decode modules.
Definition: decode.h:642
void(* RegisterTests)(void)
Definition: tm-modules.h:65
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: tm-modules.h:49
#define SCEnter(...)
Definition: util-debug.h:337
uint16_t GetNumConfiguredStreams(void)
TmEcode NapatechPacketLoopZC(ThreadVars *tv, void *data, void *slot)
struct TmSlot_ * slot_next
Definition: tm-threads.h:72
NtNetStreamRx_t rx_stream
#define MAX_STREAMS
uint16_t NapatechGetNumaNode(uint16_t stream_id)
Definition: util-napatech.c:35
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1102
#define SCReturnInt(x)
Definition: util-debug.h:341
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-modules.h:48
#define SC_CAP_NET_RAW
Definition: util-privs.h:32
TmEcode NapatechDecodeThreadInit(ThreadVars *, const void *, void **)
void PacketPoolWait(void)
void NapatechStreamThreadExitStats(ThreadVars *, void *)
Print some stats to the log at program exit.
struct NapatechThreadVars_ NapatechThreadVars
const char * name
Definition: tm-modules.h:44
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
int PacketSetData(Packet *p, uint8_t *pktdata, uint32_t pktlen)
Set data for Packet and set length when zeo copy is used.
Definition: decode.c:616
#define SCFree(a)
Definition: util-mem.h:236
TmModule tmm_modules[TMM_SIZE]
Definition: tm-modules.h:73
#define StatsSyncCountersIfSignalled(tv)
Definition: counters.h:136
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:47
int StatsSetupPrivate(ThreadVars *tv)
Definition: counters.c:1201
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:193
#define GET_PKT_DATA(p)
Definition: decode.h:224
void DecodeUpdatePacketCounters(ThreadVars *tv, const DecodeThreadVars *dtv, const Packet *p)
Definition: decode.c:536
Packet * PacketDequeue(PacketQueue *q)
Definition: packet-queue.c:167
char name[16]
Definition: threadvars.h:59
NapatechPacketVars ntpv
Definition: decode.h:610
TmEcode NapatechDecode(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *)
This function passes off to link type decoders.
#define LINKTYPE_ETHERNET
Definition: decode.h:1081
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:450
#define GET_PKT_LEN(p)
Definition: decode.h:223
uint32_t flags
Definition: decode.h:442
void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv)
Definition: decode.c:596
void PacketEnqueue(PacketQueue *q, Packet *p)
Definition: packet-queue.c:139
TmEcode NapatechDecodeThreadDeinit(ThreadVars *tv, void *data)