suricata
source-pcap-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * File based pcap packet acquisition support
24  */
25 
26 #include "suricata-common.h"
27 #include "source-pcap-file.h"
30 #include "flow-manager.h"
31 #include "util-checksum.h"
32 
33 extern int max_pending_packets;
35 
36 /**
37  * Union determining whether the behavior of the thread is file or directory
38  */
39 typedef union PcapFileBehaviorVar_
40 {
44 
45 /**
46  * Data specific to the thread
47  */
48 typedef struct PcapFileThreadVars_
49 {
52 
55 
56 static TmEcode ReceivePcapFileLoop(ThreadVars *, void *, void *);
57 static TmEcode ReceivePcapFileThreadInit(ThreadVars *, const void *, void **);
58 static void ReceivePcapFileThreadExitStats(ThreadVars *, void *);
59 static TmEcode ReceivePcapFileThreadDeinit(ThreadVars *, void *);
60 
61 static TmEcode DecodePcapFile(ThreadVars *, Packet *, void *, PacketQueue *,
62  PacketQueue *);
63 static TmEcode DecodePcapFileThreadInit(ThreadVars *, const void *, void **);
64 static TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data);
65 
66 static void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv,
68 static void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv);
69 static void CleanupPcapFileThreadVars(PcapFileThreadVars *tv);
70 static TmEcode PcapFileExit(TmEcode status, struct timespec *last_processed);
71 
72 void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv)
73 {
75  if (tv->is_directory == 0) {
76  tv->behavior.file = NULL;
77  }
78 }
79 
80 void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv, PcapFileDirectoryVars *ptv)
81 {
83  if (tv->is_directory == 1) {
84  tv->behavior.directory = NULL;
85  }
86 }
87 
88 void CleanupPcapFileThreadVars(PcapFileThreadVars *tv)
89 {
90  if(tv != NULL) {
91  if (tv->is_directory == 0) {
92  if (tv->behavior.file != NULL) {
93  CleanupPcapFileFromThreadVars(tv, tv->behavior.file);
94  }
95  tv->behavior.file = NULL;
96  } else {
97  if (tv->behavior.directory != NULL) {
98  CleanupPcapDirectoryFromThreadVars(tv, tv->behavior.directory);
99  }
100  tv->behavior.directory = NULL;
101  }
102  if (tv->shared.bpf_string != NULL) {
103  SCFree(tv->shared.bpf_string);
104  tv->shared.bpf_string = NULL;
105  }
106  SCFree(tv);
107  }
108 }
109 
110 /**
111  * Pcap File Functionality
112  */
114 {
115  tmm_modules[TMM_RECEIVEPCAPFILE].name = "ReceivePcapFile";
116  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadInit = ReceivePcapFileThreadInit;
118  tmm_modules[TMM_RECEIVEPCAPFILE].PktAcqLoop = ReceivePcapFileLoop;
120  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadExitPrintStats = ReceivePcapFileThreadExitStats;
121  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadDeinit = ReceivePcapFileThreadDeinit;
125 }
126 
128 {
129  tmm_modules[TMM_DECODEPCAPFILE].name = "DecodePcapFile";
130  tmm_modules[TMM_DECODEPCAPFILE].ThreadInit = DecodePcapFileThreadInit;
131  tmm_modules[TMM_DECODEPCAPFILE].Func = DecodePcapFile;
133  tmm_modules[TMM_DECODEPCAPFILE].ThreadDeinit = DecodePcapFileThreadDeinit;
137 }
138 
140 {
141  memset(&pcap_g, 0x00, sizeof(pcap_g));
142  SC_ATOMIC_INIT(pcap_g.invalid_checksums);
143 }
144 
145 TmEcode PcapFileExit(TmEcode status, struct timespec *last_processed)
146 {
148  status = UnixSocketPcapFile(status, last_processed);
149  SCReturnInt(status);
150  } else {
151  EngineStop();
152  SCReturnInt(status);
153  }
154 }
155 
156 TmEcode ReceivePcapFileLoop(ThreadVars *tv, void *data, void *slot)
157 {
158  SCEnter();
159 
160  if(unlikely(data == NULL)) {
161  SCLogError(SC_ERR_INVALID_ARGUMENT, "pcap file reader thread failed to initialize");
162 
163  PcapFileExit(TM_ECODE_FAILED, NULL);
164 
166  }
167 
168  TmEcode status = TM_ECODE_OK;
169  PcapFileThreadVars *ptv = (PcapFileThreadVars *) data;
170  TmSlot *s = (TmSlot *)slot;
171 
172  ptv->shared.slot = s->slot_next;
174 
175  if(ptv->is_directory == 0) {
176  SCLogInfo("Starting file run for %s", ptv->behavior.file->filename);
177  status = PcapFileDispatch(ptv->behavior.file);
178  CleanupPcapFileFromThreadVars(ptv, ptv->behavior.file);
179  } else {
180  SCLogInfo("Starting directory run for %s", ptv->behavior.directory->filename);
182  CleanupPcapDirectoryFromThreadVars(ptv, ptv->behavior.directory);
183  }
184 
185  SCLogDebug("Pcap file loop complete with status %u", status);
186 
187  status = PcapFileExit(status, &ptv->shared.last_processed);
188  SCReturnInt(status);
189 }
190 
191 TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
192 {
193  SCEnter();
194 
195  TmEcode status = TM_ECODE_OK;
196  const char *tmpstring = NULL;
197  const char *tmp_bpf_string = NULL;
198 
199  if (initdata == NULL) {
200  SCLogError(SC_ERR_INVALID_ARGUMENT, "error: initdata == NULL");
201 
203  }
204 
206  if (unlikely(ptv == NULL)) {
208  }
209  memset(ptv, 0, sizeof(PcapFileThreadVars));
210  memset(&ptv->shared.last_processed, 0, sizeof(struct timespec));
211 
212  intmax_t tenant = 0;
213  if (ConfGetInt("pcap-file.tenant-id", &tenant) == 1) {
214  if (tenant > 0 && tenant < UINT_MAX) {
215  ptv->shared.tenant_id = (uint32_t)tenant;
216  SCLogInfo("tenant %u", ptv->shared.tenant_id);
217  } else {
218  SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant out of range");
219  }
220  }
221 
222  if (ConfGet("bpf-filter", &(tmp_bpf_string)) != 1) {
223  SCLogDebug("could not get bpf or none specified");
224  } else {
225  ptv->shared.bpf_string = SCStrdup(tmp_bpf_string);
226  if (unlikely(ptv->shared.bpf_string == NULL)) {
227  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate bpf_string");
228 
229  CleanupPcapFileThreadVars(ptv);
230 
232  }
233  }
234 
235  int should_delete = 0;
236  ptv->shared.should_delete = false;
237  if (ConfGetBool("pcap-file.delete-when-done", &should_delete) == 1) {
238  ptv->shared.should_delete = should_delete == 1;
239  }
240 
241  DIR *directory = NULL;
242  SCLogDebug("checking file or directory %s", (char*)initdata);
243  if(PcapDetermineDirectoryOrFile((char *)initdata, &directory) == TM_ECODE_FAILED) {
244  CleanupPcapFileThreadVars(ptv);
246  }
247 
248  if(directory == NULL) {
249  SCLogDebug("argument %s was a file", (char *)initdata);
251  if (unlikely(pv == NULL)) {
252  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate file vars");
253  CleanupPcapFileThreadVars(ptv);
255  }
256  memset(pv, 0, sizeof(PcapFileFileVars));
257 
258  pv->filename = SCStrdup((char *)initdata);
259  if (unlikely(pv->filename == NULL)) {
260  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
262  CleanupPcapFileThreadVars(ptv);
264  }
265 
266  pv->shared = &ptv->shared;
267  status = InitPcapFile(pv);
268  if(status == TM_ECODE_OK) {
269  ptv->is_directory = 0;
270  ptv->behavior.file = pv;
271  } else {
273  "Failed to init pcap file %s, skipping", pv->filename);
275  CleanupPcapFileThreadVars(ptv);
277  }
278  } else {
279  SCLogInfo("Argument %s was a directory", (char *)initdata);
281  if (unlikely(pv == NULL)) {
282  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate directory vars");
283  closedir(directory);
284  CleanupPcapFileThreadVars(ptv);
286  }
287  memset(pv, 0, sizeof(PcapFileDirectoryVars));
288 
289  pv->filename = SCStrdup((char*)initdata);
290  if (unlikely(pv->filename == NULL)) {
291  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
293  CleanupPcapFileThreadVars(ptv);
295  }
296 
297  int should_loop = 0;
298  pv->should_loop = false;
299  if (ConfGetBool("pcap-file.continuous", &should_loop) == 1) {
300  pv->should_loop = should_loop == 1;
301  }
302 
303  pv->delay = 30;
304  intmax_t delay = 0;
305  if (ConfGetInt("pcap-file.delay", &delay) == 1) {
306  if (delay > 0 && delay < UINT_MAX) {
307  pv->delay = (time_t)delay;
308  SCLogDebug("delay %lu", pv->delay);
309  } else {
310  SCLogError(SC_ERR_INVALID_ARGUMENT, "delay out of range");
311  }
312  }
313 
314  pv->poll_interval = 5;
315  intmax_t poll_interval = 0;
316  if (ConfGetInt("pcap-file.poll-interval", &poll_interval) == 1) {
317  if (poll_interval > 0 && poll_interval < UINT_MAX) {
318  pv->poll_interval = (time_t)poll_interval;
319  SCLogDebug("poll-interval %lu", pv->delay);
320  } else {
321  SCLogError(SC_ERR_INVALID_ARGUMENT, "poll-interval out of range");
322  }
323  }
324 
325  pv->shared = &ptv->shared;
326  pv->directory = directory;
327  TAILQ_INIT(&pv->directory_content);
328 
329  ptv->is_directory = 1;
330  ptv->behavior.directory = pv;
331  }
332 
333  if (ConfGet("pcap-file.checksum-checks", &tmpstring) != 1) {
335  } else {
336  if (strcmp(tmpstring, "auto") == 0) {
338  } else if (ConfValIsTrue(tmpstring)){
340  } else if (ConfValIsFalse(tmpstring)) {
342  }
343  }
344  pcap_g.checksum_mode = pcap_g.conf_checksum_mode;
345 
346  ptv->shared.tv = tv;
347  *data = (void *)ptv;
348 
350 }
351 
352 void ReceivePcapFileThreadExitStats(ThreadVars *tv, void *data)
353 {
354  SCEnter();
355  if(data != NULL) {
356  PcapFileThreadVars *ptv = (PcapFileThreadVars *)data;
357 
359  pcap_g.cnt < CHECKSUM_SAMPLE_COUNT &&
360  SC_ATOMIC_GET(pcap_g.invalid_checksums)) {
361  uint64_t chrate = pcap_g.cnt / SC_ATOMIC_GET(pcap_g.invalid_checksums);
362  if (chrate < CHECKSUM_INVALID_RATIO)
364  "1/%" PRIu64 "th of packets have an invalid checksum,"
365  " consider setting pcap-file.checksum-checks variable to no"
366  " or use '-k none' option on command line.",
367  chrate);
368  else
369  SCLogInfo("1/%" PRIu64 "th of packets have an invalid checksum",
370  chrate);
371  }
372  SCLogNotice(
373  "Pcap-file module read %" PRIu64 " files, %" PRIu64 " packets, %" PRIu64 " bytes",
374  ptv->shared.files,
375  ptv->shared.pkts,
376  ptv->shared.bytes
377  );
378  }
379 }
380 
381 TmEcode ReceivePcapFileThreadDeinit(ThreadVars *tv, void *data)
382 {
383  SCEnter();
384  if(data != NULL) {
385  PcapFileThreadVars *ptv = (PcapFileThreadVars *) data;
386  CleanupPcapFileThreadVars(ptv);
387  }
389 }
390 
391 static double prev_signaled_ts = 0;
392 
393 TmEcode DecodePcapFile(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
394 {
395  SCEnter();
396  DecodeThreadVars *dtv = (DecodeThreadVars *)data;
397 
398  /* XXX HACK: flow timeout can call us for injected pseudo packets
399  * see bug: https://redmine.openinfosecfoundation.org/issues/1107 */
400  if (p->flags & PKT_PSEUDO_STREAM_END)
401  return TM_ECODE_OK;
402 
403  /* update counters */
404  DecodeUpdatePacketCounters(tv, dtv, p);
405 
406  double curr_ts = p->ts.tv_sec + p->ts.tv_usec / 1000.0;
407  if (curr_ts < prev_signaled_ts || (curr_ts - prev_signaled_ts) > 60.0) {
408  prev_signaled_ts = curr_ts;
410  }
411 
412  DecoderFunc decoder;
413  if(ValidateLinkType(p->datalink, &decoder) == TM_ECODE_OK) {
414 
415  /* call the decoder */
416  decoder(tv, dtv, p, GET_PKT_DATA(p), GET_PKT_LEN(p), pq);
417 
418 #ifdef DEBUG
420 #endif
421 
422  PacketDecodeFinalize(tv, dtv, p);
423 
425  } else {
427  }
428 }
429 
430 TmEcode DecodePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
431 {
432  SCEnter();
433  DecodeThreadVars *dtv = NULL;
434  dtv = DecodeThreadVarsAlloc(tv);
435 
436  if (dtv == NULL)
438 
440 
441  *data = (void *)dtv;
442 
444 }
445 
446 TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data)
447 {
448  if (data != NULL)
449  DecodeThreadVarsFree(tv, data);
451 }
452 
454 {
455  (void) SC_ATOMIC_ADD(pcap_g.invalid_checksums, 1);
456 }
457 
458 /* eof */
TmEcode InitPcapFile(PcapFileFileVars *pfv)
ChecksumValidationMode checksum_mode
union PcapFileBehaviorVar_ PcapFileBehaviorVar
#define TM_FLAG_DECODE_TM
Definition: tm-modules.h:32
PcapFileGlobalVars pcap_g
void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
DecodeThreadVars * DecodeThreadVarsAlloc(ThreadVars *tv)
Alloc and setup DecodeThreadVars.
Definition: decode.c:614
#define SCLogDebug(...)
Definition: util-debug.h:335
uint8_t cap_flags
Definition: tm-modules.h:67
int(* DecoderFunc)(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *pkt, uint32_t len, PacketQueue *pq)
Definition: decode.h:956
#define BUG_ON(x)
uint8_t flags
Definition: tm-modules.h:70
void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p)
Finalize decoding of a packet.
Definition: decode.c:115
PcapFileSharedVars * shared
#define unlikely(expr)
Definition: util-optimize.h:35
struct timespec last_processed
TmEcode(* Func)(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *)
Definition: tm-modules.h:52
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
Definition: decode.c:474
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
#define CHECKSUM_SAMPLE_COUNT
Definition: util-checksum.h:33
struct PcapFileThreadVars_ PcapFileThreadVars
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
void TmModuleReceivePcapFileRegister(void)
ChecksumValidationMode conf_checksum_mode
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:331
#define FlowWakeupFlowManagerThread()
Definition: flow-manager.h:34
PcapFileFileVars * file
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
void TmModuleDecodePcapFileRegister(void)
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
int max_pending_packets
Definition: suricata.c:215
int datalink
Definition: decode.h:575
#define TAILQ_INIT(head)
Definition: queue.h:370
int RunModeUnixSocketIsActive(void)
PcapFileSharedVars shared
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
Structure to hold thread specific data for all decode modules.
Definition: decode.h:633
void(* RegisterTests)(void)
Definition: tm-modules.h:65
TmEcode PcapFileDispatch(PcapFileFileVars *ptv)
Main PCAP file reading Loop function.
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: tm-modules.h:49
#define SCEnter(...)
Definition: util-debug.h:337
struct TmSlot_ * slot_next
Definition: tm-threads.h:87
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:591
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1095
TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
PcapFileBehaviorVar behavior
void PcapIncreaseInvalidChecksum()
#define SCReturnInt(x)
Definition: util-debug.h:341
uint8_t pkt_src
Definition: decode.h:547
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-modules.h:48
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
const char * name
Definition: tm-modules.h:44
#define SCMalloc(a)
Definition: util-mem.h:222
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:322
PcapFileDirectoryVars * directory
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:269
TmModule tmm_modules[TMM_SIZE]
Definition: tm-modules.h:73
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
TmEcode ValidateLinkType(int datalink, DecoderFunc *decoder)
#define CHECKSUM_INVALID_RATIO
Definition: util-checksum.h:34
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:47
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
#define GET_PKT_DATA(p)
Definition: decode.h:226
void DecodeUpdatePacketCounters(ThreadVars *tv, const DecodeThreadVars *dtv, const Packet *p)
Definition: decode.c:580
#define SCStrdup(a)
Definition: util-mem.h:268
void EngineStop(void)
make sure threads can stop the engine by calling this function. Purpose: pcap file mode needs to be a...
Definition: suricata.c:431
void PcapFileGlobalInit()
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:452
TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed)
#define GET_PKT_LEN(p)
Definition: decode.h:225
uint32_t flags
Definition: decode.h:444
void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv)
void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv)
Definition: decode.c:633