suricata
source-pcap-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * File based pcap packet acquisition support
24  */
25 
26 #include "suricata-common.h"
27 #include "source-pcap-file.h"
30 #include "flow-manager.h"
31 #include "util-checksum.h"
32 
33 extern int max_pending_packets;
35 
36 /**
37  * Union determining whether the behavior of the thread is file or directory
38  */
39 typedef union PcapFileBehaviorVar_
40 {
44 
45 /**
46  * Data specific to the thread
47  */
48 typedef struct PcapFileThreadVars_
49 {
52 
55 
56 static TmEcode ReceivePcapFileLoop(ThreadVars *, void *, void *);
57 static TmEcode ReceivePcapFileThreadInit(ThreadVars *, const void *, void **);
58 static void ReceivePcapFileThreadExitStats(ThreadVars *, void *);
59 static TmEcode ReceivePcapFileThreadDeinit(ThreadVars *, void *);
60 
61 static TmEcode DecodePcapFile(ThreadVars *, Packet *, void *, PacketQueue *,
62  PacketQueue *);
63 static TmEcode DecodePcapFileThreadInit(ThreadVars *, const void *, void **);
64 static TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data);
65 
66 static void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv,
68 static void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv);
69 static void CleanupPcapFileThreadVars(PcapFileThreadVars *tv);
70 static TmEcode PcapFileExit(TmEcode status);
71 
72 void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv)
73 {
75  if (tv->is_directory == 0) {
76  tv->behavior.file = NULL;
77  }
78 }
79 
80 void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv, PcapFileDirectoryVars *ptv)
81 {
83  if (tv->is_directory == 1) {
84  tv->behavior.directory = NULL;
85  }
86 }
87 
88 void CleanupPcapFileThreadVars(PcapFileThreadVars *tv)
89 {
90  if(tv != NULL) {
91  if (tv->is_directory == 0) {
92  if (tv->behavior.file != NULL) {
93  CleanupPcapFileFromThreadVars(tv, tv->behavior.file);
94  }
95  tv->behavior.file = NULL;
96  } else {
97  if (tv->behavior.directory != NULL) {
98  CleanupPcapDirectoryFromThreadVars(tv, tv->behavior.directory);
99  }
100  tv->behavior.directory = NULL;
101  }
102  if (tv->shared.bpf_string != NULL) {
103  SCFree(tv->shared.bpf_string);
104  tv->shared.bpf_string = NULL;
105  }
106  SCFree(tv);
107  }
108 }
109 
110 /**
111  * Pcap File Functionality
112  */
114 {
115  tmm_modules[TMM_RECEIVEPCAPFILE].name = "ReceivePcapFile";
116  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadInit = ReceivePcapFileThreadInit;
118  tmm_modules[TMM_RECEIVEPCAPFILE].PktAcqLoop = ReceivePcapFileLoop;
120  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadExitPrintStats = ReceivePcapFileThreadExitStats;
121  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadDeinit = ReceivePcapFileThreadDeinit;
125 }
126 
128 {
129  tmm_modules[TMM_DECODEPCAPFILE].name = "DecodePcapFile";
130  tmm_modules[TMM_DECODEPCAPFILE].ThreadInit = DecodePcapFileThreadInit;
131  tmm_modules[TMM_DECODEPCAPFILE].Func = DecodePcapFile;
133  tmm_modules[TMM_DECODEPCAPFILE].ThreadDeinit = DecodePcapFileThreadDeinit;
137 }
138 
140 {
141  memset(&pcap_g, 0x00, sizeof(pcap_g));
142  SC_ATOMIC_INIT(pcap_g.invalid_checksums);
143 }
144 
145 TmEcode PcapFileExit(TmEcode status)
146 {
149  } else {
150  EngineStop();
151  SCReturnInt(status);
152  }
153 }
154 
155 TmEcode ReceivePcapFileLoop(ThreadVars *tv, void *data, void *slot)
156 {
157  SCEnter();
158 
159  TmEcode status = TM_ECODE_OK;
160  PcapFileThreadVars *ptv = (PcapFileThreadVars *) data;
161  TmSlot *s = (TmSlot *)slot;
162 
163  ptv->shared.slot = s->slot_next;
165 
166  if(ptv->is_directory == 0) {
167  SCLogInfo("Starting file run for %s", ptv->behavior.file->filename);
168  status = PcapFileDispatch(ptv->behavior.file);
169  if (!RunModeUnixSocketIsActive()) {
170  EngineStop();
171  } else {
173  }
174  CleanupPcapFileFromThreadVars(ptv, ptv->behavior.file);
175  } else {
176  SCLogInfo("Starting directory run for %s", ptv->behavior.directory->filename);
178  CleanupPcapDirectoryFromThreadVars(ptv, ptv->behavior.directory);
179  }
180 
181  SCLogDebug("Pcap file loop complete with status %u", status);
182 
183  status = PcapFileExit(status);
184 
185  SCReturnInt(status);
186 }
187 
188 TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
189 {
190  SCEnter();
191 
192  TmEcode status = TM_ECODE_OK;
193  const char *tmpstring = NULL;
194  const char *tmp_bpf_string = NULL;
195 
196  if (initdata == NULL) {
197  SCLogError(SC_ERR_INVALID_ARGUMENT, "error: initdata == NULL");
198 
199  status = PcapFileExit(TM_ECODE_FAILED);
200  SCReturnInt(status);
201  }
202 
204  if (unlikely(ptv == NULL)) {
205  status = PcapFileExit(TM_ECODE_FAILED);
206  SCReturnInt(status);
207  }
208  memset(ptv, 0, sizeof(PcapFileThreadVars));
209  memset(&ptv->shared.last_processed, 0, sizeof(struct timespec));
210 
211  intmax_t tenant = 0;
212  if (ConfGetInt("pcap-file.tenant-id", &tenant) == 1) {
213  if (tenant > 0 && tenant < UINT_MAX) {
214  ptv->shared.tenant_id = (uint32_t)tenant;
215  SCLogInfo("tenant %u", ptv->shared.tenant_id);
216  } else {
217  SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant out of range");
218  }
219  }
220 
221  if (ConfGet("bpf-filter", &(tmp_bpf_string)) != 1) {
222  SCLogDebug("could not get bpf or none specified");
223  } else {
224  ptv->shared.bpf_string = SCStrdup(tmp_bpf_string);
225  if (unlikely(ptv->shared.bpf_string == NULL)) {
226  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate bpf_string");
227  CleanupPcapFileThreadVars(ptv);
228 
229  status = PcapFileExit(TM_ECODE_FAILED);
230  SCReturnInt(status);
231  }
232  }
233 
234  int should_delete = 0;
235  ptv->shared.should_delete = false;
236  if (ConfGetBool("pcap-file.delete-when-done", &should_delete) == 1) {
237  ptv->shared.should_delete = should_delete == 1;
238  }
239 
240  DIR *directory = NULL;
241  SCLogInfo("Checking file or directory %s", (char*)initdata);
242  if(PcapDetermineDirectoryOrFile((char *)initdata, &directory) == TM_ECODE_FAILED) {
243  CleanupPcapFileThreadVars(ptv);
244  status = PcapFileExit(TM_ECODE_FAILED);
245  SCReturnInt(status);
246  }
247 
248  if(directory == NULL) {
249  SCLogInfo("Argument %s was a file", (char *)initdata);
251  if (unlikely(pv == NULL)) {
252  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate file vars");
253  CleanupPcapFileThreadVars(ptv);
254  status = PcapFileExit(TM_ECODE_FAILED);
255  SCReturnInt(status);
256  }
257  memset(pv, 0, sizeof(PcapFileFileVars));
258 
259  pv->filename = SCStrdup((char *)initdata);
260  if (unlikely(pv->filename == NULL)) {
261  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
263  CleanupPcapFileThreadVars(ptv);
264  status = PcapFileExit(TM_ECODE_FAILED);
265  SCReturnInt(status);
266  }
267 
268  status = InitPcapFile(pv);
269  if(status == TM_ECODE_OK) {
270  pv->shared = &ptv->shared;
271 
272  ptv->is_directory = 0;
273  ptv->behavior.file = pv;
274  } else {
276  "Failed to init pcap file %s, skipping", (char *)initdata);
278  CleanupPcapFileThreadVars(ptv);
279 
280  status = PcapFileExit(status);
281  SCReturnInt(status);
282  }
283  } else {
284  SCLogInfo("Argument %s was a directory", (char *)initdata);
286  if (unlikely(pv == NULL)) {
287  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate directory vars");
288  closedir(directory);
289  CleanupPcapFileThreadVars(ptv);
290  status = PcapFileExit(TM_ECODE_FAILED);
291  SCReturnInt(status);
292  }
293  memset(pv, 0, sizeof(PcapFileDirectoryVars));
294 
295  pv->filename = SCStrdup((char*)initdata);
296  if (unlikely(pv->filename == NULL)) {
297  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
299  CleanupPcapFileThreadVars(ptv);
300  status = PcapFileExit(TM_ECODE_FAILED);
301  SCReturnInt(status);
302  }
303 
304  int should_loop = 0;
305  pv->should_loop = false;
306  if (ConfGetBool("pcap-file.continuous", &should_loop) == 1) {
307  pv->should_loop = should_loop == 1;
308  }
309 
310  pv->delay = 30;
311  intmax_t delay = 0;
312  if (ConfGetInt("pcap-file.delay", &delay) == 1) {
313  if (delay > 0 && delay < UINT_MAX) {
314  pv->delay = (time_t)delay;
315  SCLogDebug("delay %lu", pv->delay);
316  } else {
317  SCLogError(SC_ERR_INVALID_ARGUMENT, "delay out of range");
318  }
319  }
320 
321  pv->poll_interval = 5;
322  intmax_t poll_interval = 0;
323  if (ConfGetInt("pcap-file.poll-interval", &poll_interval) == 1) {
324  if (poll_interval > 0 && poll_interval < UINT_MAX) {
325  pv->poll_interval = (time_t)poll_interval;
326  SCLogDebug("poll-interval %lu", pv->delay);
327  } else {
328  SCLogError(SC_ERR_INVALID_ARGUMENT, "poll-interval out of range");
329  }
330  }
331 
332  pv->shared = &ptv->shared;
333  pv->directory = directory;
334  TAILQ_INIT(&pv->directory_content);
335 
336  ptv->is_directory = 1;
337  ptv->behavior.directory = pv;
338  }
339 
340  if (ConfGet("pcap-file.checksum-checks", &tmpstring) != 1) {
342  } else {
343  if (strcmp(tmpstring, "auto") == 0) {
345  } else if (ConfValIsTrue(tmpstring)){
347  } else if (ConfValIsFalse(tmpstring)) {
349  }
350  }
351  pcap_g.checksum_mode = pcap_g.conf_checksum_mode;
352 
353  ptv->shared.tv = tv;
354  *data = (void *)ptv;
355 
357 }
358 
359 void ReceivePcapFileThreadExitStats(ThreadVars *tv, void *data)
360 {
361  SCEnter();
362  PcapFileThreadVars *ptv = (PcapFileThreadVars *)data;
363 
365  pcap_g.cnt < CHECKSUM_SAMPLE_COUNT &&
366  SC_ATOMIC_GET(pcap_g.invalid_checksums)) {
367  uint64_t chrate = pcap_g.cnt / SC_ATOMIC_GET(pcap_g.invalid_checksums);
368  if (chrate < CHECKSUM_INVALID_RATIO)
370  "1/%" PRIu64 "th of packets have an invalid checksum,"
371  " consider setting pcap-file.checksum-checks variable to no"
372  " or use '-k none' option on command line.",
373  chrate);
374  else
375  SCLogInfo("1/%" PRIu64 "th of packets have an invalid checksum",
376  chrate);
377  }
378  SCLogNotice(
379  "Pcap-file module read %" PRIu64 " files, %" PRIu64 " packets, %" PRIu64 " bytes",
380  ptv->shared.files,
381  ptv->shared.pkts,
382  ptv->shared.bytes
383  );
384 }
385 
386 TmEcode ReceivePcapFileThreadDeinit(ThreadVars *tv, void *data)
387 {
388  SCEnter();
389  PcapFileThreadVars *ptv = (PcapFileThreadVars *)data;
390  CleanupPcapFileThreadVars(ptv);
392 }
393 
394 static double prev_signaled_ts = 0;
395 
396 TmEcode DecodePcapFile(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
397 {
398  SCEnter();
399  DecodeThreadVars *dtv = (DecodeThreadVars *)data;
400 
401  /* XXX HACK: flow timeout can call us for injected pseudo packets
402  * see bug: https://redmine.openinfosecfoundation.org/issues/1107 */
403  if (p->flags & PKT_PSEUDO_STREAM_END)
404  return TM_ECODE_OK;
405 
406  /* update counters */
407  DecodeUpdatePacketCounters(tv, dtv, p);
408 
409  double curr_ts = p->ts.tv_sec + p->ts.tv_usec / 1000.0;
410  if (curr_ts < prev_signaled_ts || (curr_ts - prev_signaled_ts) > 60.0) {
411  prev_signaled_ts = curr_ts;
413  }
414 
415  Decoder decoder;
416  if(ValidateLinkType(p->datalink, &decoder) == TM_ECODE_OK) {
417 
418  /* call the decoder */
419  decoder(tv, dtv, p, GET_PKT_DATA(p), GET_PKT_LEN(p), pq);
420 
421 #ifdef DEBUG
423 #endif
424 
425  PacketDecodeFinalize(tv, dtv, p);
426 
428  } else {
430  }
431 }
432 
433 TmEcode DecodePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
434 {
435  SCEnter();
436  DecodeThreadVars *dtv = NULL;
437  dtv = DecodeThreadVarsAlloc(tv);
438 
439  if (dtv == NULL)
441 
443 
444  *data = (void *)dtv;
445 
447 }
448 
449 TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data)
450 {
451  if (data != NULL)
452  DecodeThreadVarsFree(tv, data);
454 }
455 
457 {
458  (void) SC_ATOMIC_ADD(pcap_g.invalid_checksums, 1);
459 }
460 
461 /* eof */
TmEcode InitPcapFile(PcapFileFileVars *pfv)
ChecksumValidationMode checksum_mode
union PcapFileBehaviorVar_ PcapFileBehaviorVar
#define TM_FLAG_DECODE_TM
Definition: tm-modules.h:32
PcapFileGlobalVars pcap_g
void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
DecodeThreadVars * DecodeThreadVarsAlloc(ThreadVars *tv)
Alloc and setup DecodeThreadVars.
Definition: decode.c:570
#define SCLogDebug(...)
Definition: util-debug.h:335
uint8_t cap_flags
Definition: tm-modules.h:67
#define BUG_ON(x)
uint8_t flags
Definition: tm-modules.h:70
void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p)
Finalize decoding of a packet.
Definition: decode.c:114
PcapFileSharedVars * shared
#define unlikely(expr)
Definition: util-optimize.h:35
struct timespec last_processed
TmEcode(* Func)(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *)
Definition: tm-modules.h:52
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
Definition: decode.c:431
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:108
TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
#define CHECKSUM_SAMPLE_COUNT
Definition: util-checksum.h:33
struct PcapFileThreadVars_ PcapFileThreadVars
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
void TmModuleReceivePcapFileRegister(void)
ChecksumValidationMode conf_checksum_mode
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:331
#define FlowWakeupFlowManagerThread()
Definition: flow-manager.h:34
PcapFileFileVars * file
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
int(* Decoder)(ThreadVars *, DecodeThreadVars *, Packet *, uint8_t *, uint32_t, PacketQueue *)
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
void TmModuleDecodePcapFileRegister(void)
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:82
int max_pending_packets
Definition: suricata.c:215
int datalink
Definition: decode.h:579
#define TAILQ_INIT(head)
Definition: queue.h:370
int RunModeUnixSocketIsActive(void)
PcapFileSharedVars shared
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
Structure to hold thread specific data for all decode modules.
Definition: decode.h:642
void(* RegisterTests)(void)
Definition: tm-modules.h:65
TmEcode PcapFileDispatch(PcapFileFileVars *ptv)
Main PCAP file reading Loop function.
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: tm-modules.h:49
#define SCEnter(...)
Definition: util-debug.h:337
struct TmSlot_ * slot_next
Definition: tm-threads.h:72
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:591
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1102
TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
PcapFileBehaviorVar behavior
void PcapIncreaseInvalidChecksum()
#define SCReturnInt(x)
Definition: util-debug.h:341
uint8_t pkt_src
Definition: decode.h:551
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-modules.h:48
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
const char * name
Definition: tm-modules.h:44
#define SCMalloc(a)
Definition: util-mem.h:174
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:236
PcapFileDirectoryVars * directory
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:269
TmModule tmm_modules[TMM_SIZE]
Definition: tm-modules.h:73
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
#define CHECKSUM_INVALID_RATIO
Definition: util-checksum.h:34
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:47
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:193
#define GET_PKT_DATA(p)
Definition: decode.h:224
void DecodeUpdatePacketCounters(ThreadVars *tv, const DecodeThreadVars *dtv, const Packet *p)
Definition: decode.c:536
#define SCStrdup(a)
Definition: util-mem.h:220
void EngineStop(void)
make sure threads can stop the engine by calling this function. Purpose: pcap file mode needs to be a...
Definition: suricata.c:416
void PcapFileGlobalInit()
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:450
TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed)
#define GET_PKT_LEN(p)
Definition: decode.h:223
uint32_t flags
Definition: decode.h:442
TmEcode ValidateLinkType(int datalink, Decoder *decoder)
void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv)
void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv)
Definition: decode.c:596