suricata
source-pcap-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * File based pcap packet acquisition support
24  */
25 
26 #include "suricata-common.h"
27 #include "source-pcap-file.h"
30 #include "flow-manager.h"
31 #include "util-checksum.h"
32 
33 extern int max_pending_packets;
35 
36 /**
37  * Union determining whether the behavior of the thread is file or directory
38  */
39 typedef union PcapFileBehaviorVar_
40 {
44 
45 /**
46  * Data specific to the thread
47  */
48 typedef struct PcapFileThreadVars_
49 {
52 
55 
56 static TmEcode ReceivePcapFileLoop(ThreadVars *, void *, void *);
57 static TmEcode ReceivePcapFileThreadInit(ThreadVars *, const void *, void **);
58 static void ReceivePcapFileThreadExitStats(ThreadVars *, void *);
59 static TmEcode ReceivePcapFileThreadDeinit(ThreadVars *, void *);
60 
61 static TmEcode DecodePcapFile(ThreadVars *, Packet *, void *, PacketQueue *,
62  PacketQueue *);
63 static TmEcode DecodePcapFileThreadInit(ThreadVars *, const void *, void **);
64 static TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data);
65 
66 static void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv,
68 static void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv);
69 static void CleanupPcapFileThreadVars(PcapFileThreadVars *tv);
70 static TmEcode PcapFileExit(TmEcode status, struct timespec *last_processed);
71 
72 void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv)
73 {
75  if (tv->is_directory == 0) {
76  tv->behavior.file = NULL;
77  }
78 }
79 
80 void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv, PcapFileDirectoryVars *ptv)
81 {
83  if (tv->is_directory == 1) {
84  tv->behavior.directory = NULL;
85  }
86 }
87 
88 void CleanupPcapFileThreadVars(PcapFileThreadVars *tv)
89 {
90  if(tv != NULL) {
91  if (tv->is_directory == 0) {
92  if (tv->behavior.file != NULL) {
93  CleanupPcapFileFromThreadVars(tv, tv->behavior.file);
94  }
95  tv->behavior.file = NULL;
96  } else {
97  if (tv->behavior.directory != NULL) {
98  CleanupPcapDirectoryFromThreadVars(tv, tv->behavior.directory);
99  }
100  tv->behavior.directory = NULL;
101  }
102  if (tv->shared.bpf_string != NULL) {
103  SCFree(tv->shared.bpf_string);
104  tv->shared.bpf_string = NULL;
105  }
106  SCFree(tv);
107  }
108 }
109 
110 /**
111  * Pcap File Functionality
112  */
114 {
115  tmm_modules[TMM_RECEIVEPCAPFILE].name = "ReceivePcapFile";
116  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadInit = ReceivePcapFileThreadInit;
118  tmm_modules[TMM_RECEIVEPCAPFILE].PktAcqLoop = ReceivePcapFileLoop;
120  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadExitPrintStats = ReceivePcapFileThreadExitStats;
121  tmm_modules[TMM_RECEIVEPCAPFILE].ThreadDeinit = ReceivePcapFileThreadDeinit;
125 }
126 
128 {
129  tmm_modules[TMM_DECODEPCAPFILE].name = "DecodePcapFile";
130  tmm_modules[TMM_DECODEPCAPFILE].ThreadInit = DecodePcapFileThreadInit;
131  tmm_modules[TMM_DECODEPCAPFILE].Func = DecodePcapFile;
133  tmm_modules[TMM_DECODEPCAPFILE].ThreadDeinit = DecodePcapFileThreadDeinit;
137 }
138 
140 {
141  memset(&pcap_g, 0x00, sizeof(pcap_g));
142  SC_ATOMIC_INIT(pcap_g.invalid_checksums);
143 }
144 
145 TmEcode PcapFileExit(TmEcode status, struct timespec *last_processed)
146 {
148  status = UnixSocketPcapFile(status, last_processed);
149  SCReturnInt(status);
150  } else {
151  EngineStop();
152  SCReturnInt(status);
153  }
154 }
155 
156 TmEcode ReceivePcapFileLoop(ThreadVars *tv, void *data, void *slot)
157 {
158  SCEnter();
159 
160  if(unlikely(data == NULL)) {
161  SCLogError(SC_ERR_INVALID_ARGUMENT, "pcap file reader thread failed to initialize");
162 
163  PcapFileExit(TM_ECODE_FAILED, NULL);
164 
166  }
167 
168  TmEcode status = TM_ECODE_OK;
169  PcapFileThreadVars *ptv = (PcapFileThreadVars *) data;
170  TmSlot *s = (TmSlot *)slot;
171 
172  ptv->shared.slot = s->slot_next;
174 
175  if(ptv->is_directory == 0) {
176  SCLogInfo("Starting file run for %s", ptv->behavior.file->filename);
177  status = PcapFileDispatch(ptv->behavior.file);
178  CleanupPcapFileFromThreadVars(ptv, ptv->behavior.file);
179  } else {
180  SCLogInfo("Starting directory run for %s", ptv->behavior.directory->filename);
182  CleanupPcapDirectoryFromThreadVars(ptv, ptv->behavior.directory);
183  }
184 
185  SCLogDebug("Pcap file loop complete with status %u", status);
186 
187  status = PcapFileExit(status, &ptv->shared.last_processed);
188  SCReturnInt(status);
189 }
190 
191 TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
192 {
193  SCEnter();
194 
195  TmEcode status = TM_ECODE_OK;
196  const char *tmpstring = NULL;
197  const char *tmp_bpf_string = NULL;
198 
199  if (initdata == NULL) {
200  SCLogError(SC_ERR_INVALID_ARGUMENT, "error: initdata == NULL");
201 
203  }
204 
206  if (unlikely(ptv == NULL)) {
208  }
209  memset(ptv, 0, sizeof(PcapFileThreadVars));
210  memset(&ptv->shared.last_processed, 0, sizeof(struct timespec));
211 
212  intmax_t tenant = 0;
213  if (ConfGetInt("pcap-file.tenant-id", &tenant) == 1) {
214  if (tenant > 0 && tenant < UINT_MAX) {
215  ptv->shared.tenant_id = (uint32_t)tenant;
216  SCLogInfo("tenant %u", ptv->shared.tenant_id);
217  } else {
218  SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant out of range");
219  }
220  }
221 
222  if (ConfGet("bpf-filter", &(tmp_bpf_string)) != 1) {
223  SCLogDebug("could not get bpf or none specified");
224  } else {
225  ptv->shared.bpf_string = SCStrdup(tmp_bpf_string);
226  if (unlikely(ptv->shared.bpf_string == NULL)) {
227  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate bpf_string");
228 
229  CleanupPcapFileThreadVars(ptv);
230 
232  }
233  }
234 
235  int should_delete = 0;
236  ptv->shared.should_delete = false;
237  if (ConfGetBool("pcap-file.delete-when-done", &should_delete) == 1) {
238  ptv->shared.should_delete = should_delete == 1;
239  }
240 
241  DIR *directory = NULL;
242  SCLogDebug("checking file or directory %s", (char*)initdata);
243  if(PcapDetermineDirectoryOrFile((char *)initdata, &directory) == TM_ECODE_FAILED) {
244  CleanupPcapFileThreadVars(ptv);
246  }
247 
248  if(directory == NULL) {
249  SCLogDebug("argument %s was a file", (char *)initdata);
251  if (unlikely(pv == NULL)) {
252  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate file vars");
253  CleanupPcapFileThreadVars(ptv);
255  }
256  memset(pv, 0, sizeof(PcapFileFileVars));
257 
258  pv->filename = SCStrdup((char *)initdata);
259  if (unlikely(pv->filename == NULL)) {
260  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
262  CleanupPcapFileThreadVars(ptv);
264  }
265 
266  status = InitPcapFile(pv);
267  if(status == TM_ECODE_OK) {
268  pv->shared = &ptv->shared;
269 
270  ptv->is_directory = 0;
271  ptv->behavior.file = pv;
272  } else {
274  "Failed to init pcap file %s, skipping", pv->filename);
276  CleanupPcapFileThreadVars(ptv);
278  }
279  } else {
280  SCLogInfo("Argument %s was a directory", (char *)initdata);
282  if (unlikely(pv == NULL)) {
283  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate directory vars");
284  closedir(directory);
285  CleanupPcapFileThreadVars(ptv);
287  }
288  memset(pv, 0, sizeof(PcapFileDirectoryVars));
289 
290  pv->filename = SCStrdup((char*)initdata);
291  if (unlikely(pv->filename == NULL)) {
292  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
294  CleanupPcapFileThreadVars(ptv);
296  }
297 
298  int should_loop = 0;
299  pv->should_loop = false;
300  if (ConfGetBool("pcap-file.continuous", &should_loop) == 1) {
301  pv->should_loop = should_loop == 1;
302  }
303 
304  pv->delay = 30;
305  intmax_t delay = 0;
306  if (ConfGetInt("pcap-file.delay", &delay) == 1) {
307  if (delay > 0 && delay < UINT_MAX) {
308  pv->delay = (time_t)delay;
309  SCLogDebug("delay %lu", pv->delay);
310  } else {
311  SCLogError(SC_ERR_INVALID_ARGUMENT, "delay out of range");
312  }
313  }
314 
315  pv->poll_interval = 5;
316  intmax_t poll_interval = 0;
317  if (ConfGetInt("pcap-file.poll-interval", &poll_interval) == 1) {
318  if (poll_interval > 0 && poll_interval < UINT_MAX) {
319  pv->poll_interval = (time_t)poll_interval;
320  SCLogDebug("poll-interval %lu", pv->delay);
321  } else {
322  SCLogError(SC_ERR_INVALID_ARGUMENT, "poll-interval out of range");
323  }
324  }
325 
326  pv->shared = &ptv->shared;
327  pv->directory = directory;
328  TAILQ_INIT(&pv->directory_content);
329 
330  ptv->is_directory = 1;
331  ptv->behavior.directory = pv;
332  }
333 
334  if (ConfGet("pcap-file.checksum-checks", &tmpstring) != 1) {
336  } else {
337  if (strcmp(tmpstring, "auto") == 0) {
339  } else if (ConfValIsTrue(tmpstring)){
341  } else if (ConfValIsFalse(tmpstring)) {
343  }
344  }
345  pcap_g.checksum_mode = pcap_g.conf_checksum_mode;
346 
347  ptv->shared.tv = tv;
348  *data = (void *)ptv;
349 
351 }
352 
353 void ReceivePcapFileThreadExitStats(ThreadVars *tv, void *data)
354 {
355  SCEnter();
356  if(data != NULL) {
357  PcapFileThreadVars *ptv = (PcapFileThreadVars *)data;
358 
360  pcap_g.cnt < CHECKSUM_SAMPLE_COUNT &&
361  SC_ATOMIC_GET(pcap_g.invalid_checksums)) {
362  uint64_t chrate = pcap_g.cnt / SC_ATOMIC_GET(pcap_g.invalid_checksums);
363  if (chrate < CHECKSUM_INVALID_RATIO)
365  "1/%" PRIu64 "th of packets have an invalid checksum,"
366  " consider setting pcap-file.checksum-checks variable to no"
367  " or use '-k none' option on command line.",
368  chrate);
369  else
370  SCLogInfo("1/%" PRIu64 "th of packets have an invalid checksum",
371  chrate);
372  }
373  SCLogNotice(
374  "Pcap-file module read %" PRIu64 " files, %" PRIu64 " packets, %" PRIu64 " bytes",
375  ptv->shared.files,
376  ptv->shared.pkts,
377  ptv->shared.bytes
378  );
379  }
380 }
381 
382 TmEcode ReceivePcapFileThreadDeinit(ThreadVars *tv, void *data)
383 {
384  SCEnter();
385  if(data != NULL) {
386  PcapFileThreadVars *ptv = (PcapFileThreadVars *) data;
387  CleanupPcapFileThreadVars(ptv);
388  }
390 }
391 
392 static double prev_signaled_ts = 0;
393 
394 TmEcode DecodePcapFile(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
395 {
396  SCEnter();
397  DecodeThreadVars *dtv = (DecodeThreadVars *)data;
398 
399  /* XXX HACK: flow timeout can call us for injected pseudo packets
400  * see bug: https://redmine.openinfosecfoundation.org/issues/1107 */
401  if (p->flags & PKT_PSEUDO_STREAM_END)
402  return TM_ECODE_OK;
403 
404  /* update counters */
405  DecodeUpdatePacketCounters(tv, dtv, p);
406 
407  double curr_ts = p->ts.tv_sec + p->ts.tv_usec / 1000.0;
408  if (curr_ts < prev_signaled_ts || (curr_ts - prev_signaled_ts) > 60.0) {
409  prev_signaled_ts = curr_ts;
411  }
412 
413  Decoder decoder;
414  if(ValidateLinkType(p->datalink, &decoder) == TM_ECODE_OK) {
415 
416  /* call the decoder */
417  decoder(tv, dtv, p, GET_PKT_DATA(p), GET_PKT_LEN(p), pq);
418 
419 #ifdef DEBUG
421 #endif
422 
423  PacketDecodeFinalize(tv, dtv, p);
424 
426  } else {
428  }
429 }
430 
431 TmEcode DecodePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
432 {
433  SCEnter();
434  DecodeThreadVars *dtv = NULL;
435  dtv = DecodeThreadVarsAlloc(tv);
436 
437  if (dtv == NULL)
439 
441 
442  *data = (void *)dtv;
443 
445 }
446 
447 TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data)
448 {
449  if (data != NULL)
450  DecodeThreadVarsFree(tv, data);
452 }
453 
455 {
456  (void) SC_ATOMIC_ADD(pcap_g.invalid_checksums, 1);
457 }
458 
459 /* eof */
TmEcode InitPcapFile(PcapFileFileVars *pfv)
ChecksumValidationMode checksum_mode
union PcapFileBehaviorVar_ PcapFileBehaviorVar
#define TM_FLAG_DECODE_TM
Definition: tm-modules.h:32
PcapFileGlobalVars pcap_g
void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
DecodeThreadVars * DecodeThreadVarsAlloc(ThreadVars *tv)
Alloc and setup DecodeThreadVars.
Definition: decode.c:592
#define SCLogDebug(...)
Definition: util-debug.h:335
uint8_t cap_flags
Definition: tm-modules.h:67
#define BUG_ON(x)
uint8_t flags
Definition: tm-modules.h:70
void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p)
Finalize decoding of a packet.
Definition: decode.c:114
PcapFileSharedVars * shared
#define unlikely(expr)
Definition: util-optimize.h:35
struct timespec last_processed
TmEcode(* Func)(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *)
Definition: tm-modules.h:52
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
Definition: decode.c:453
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
#define CHECKSUM_SAMPLE_COUNT
Definition: util-checksum.h:33
struct PcapFileThreadVars_ PcapFileThreadVars
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
void TmModuleReceivePcapFileRegister(void)
ChecksumValidationMode conf_checksum_mode
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:331
#define FlowWakeupFlowManagerThread()
Definition: flow-manager.h:34
PcapFileFileVars * file
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
int(* Decoder)(ThreadVars *, DecodeThreadVars *, Packet *, uint8_t *, uint32_t, PacketQueue *)
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
void TmModuleDecodePcapFileRegister(void)
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
int max_pending_packets
Definition: suricata.c:215
int datalink
Definition: decode.h:574
#define TAILQ_INIT(head)
Definition: queue.h:370
int RunModeUnixSocketIsActive(void)
PcapFileSharedVars shared
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
Structure to hold thread specific data for all decode modules.
Definition: decode.h:632
void(* RegisterTests)(void)
Definition: tm-modules.h:65
TmEcode PcapFileDispatch(PcapFileFileVars *ptv)
Main PCAP file reading Loop function.
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: tm-modules.h:49
#define SCEnter(...)
Definition: util-debug.h:337
struct TmSlot_ * slot_next
Definition: tm-threads.h:87
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:591
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
PcapFileBehaviorVar behavior
void PcapIncreaseInvalidChecksum()
#define SCReturnInt(x)
Definition: util-debug.h:341
uint8_t pkt_src
Definition: decode.h:546
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-modules.h:48
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
const char * name
Definition: tm-modules.h:44
#define SCMalloc(a)
Definition: util-mem.h:166
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:228
PcapFileDirectoryVars * directory
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:269
TmModule tmm_modules[TMM_SIZE]
Definition: tm-modules.h:73
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
#define CHECKSUM_INVALID_RATIO
Definition: util-checksum.h:34
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:47
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
#define GET_PKT_DATA(p)
Definition: decode.h:223
void DecodeUpdatePacketCounters(ThreadVars *tv, const DecodeThreadVars *dtv, const Packet *p)
Definition: decode.c:558
#define SCStrdup(a)
Definition: util-mem.h:212
void EngineStop(void)
make sure threads can stop the engine by calling this function. Purpose: pcap file mode needs to be a...
Definition: suricata.c:422
void PcapFileGlobalInit()
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:449
TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed)
#define GET_PKT_LEN(p)
Definition: decode.h:222
uint32_t flags
Definition: decode.h:441
TmEcode ValidateLinkType(int datalink, Decoder *decoder)
void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv)
void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv)
Definition: decode.c:618