suricata
log-pcap.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author William Metcalf <William.Metcalf@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Pcap packet logging module.
25  */
26 
27 #include "suricata-common.h"
28 #ifdef HAVE_LIBLZ4
29 #include <lz4frame.h>
30 #include "util-fmemopen.h"
31 #endif /* HAVE_LIBLZ4 */
32 
33 #if defined(HAVE_DIRENT_H) && defined(HAVE_FNMATCH_H)
34 #define INIT_RING_BUFFER
35 #include <dirent.h>
36 #include <fnmatch.h>
37 #endif
38 
39 #include "log-pcap.h"
40 
41 #include "threads.h"
42 #include "threadvars.h"
43 #include "decode.h"
44 #include "stream.h"
45 #include "stream-tcp-reassemble.h"
46 
47 #include "output.h"
48 
49 #include "util-buffer.h"
50 #include "util-byte.h"
51 #include "util-conf.h"
52 #include "util-cpu.h"
53 #include "util-datalink.h"
54 #include "util-misc.h"
55 #include "util-path.h"
56 #include "util-time.h"
57 
58 #define DEFAULT_LOG_FILENAME "pcaplog"
59 #define MODULE_NAME "PcapLog"
60 #define MIN_LIMIT 4 * 1024 * 1024
61 #define DEFAULT_LIMIT 100 * 1024 * 1024
62 #define DEFAULT_FILE_LIMIT 0
63 
64 #define LOGMODE_NORMAL 0
65 #define LOGMODE_MULTI 1
66 
72 
73 #define RING_BUFFER_MODE_DISABLED 0
74 #define RING_BUFFER_MODE_ENABLED 1
75 
76 #define TS_FORMAT_SEC 0
77 #define TS_FORMAT_USEC 1
78 
79 #define USE_STREAM_DEPTH_DISABLED 0
80 #define USE_STREAM_DEPTH_ENABLED 1
81 
82 #define HONOR_PASS_RULES_DISABLED 0
83 #define HONOR_PASS_RULES_ENABLED 1
84 
85 #define PCAP_SNAPLEN 262144
86 #define PCAP_BUFFER_TIMEOUT 1000000 // microseconds
87 #define PCAP_PKTHDR_SIZE 16
88 
89 /* Defined since libpcap 1.1.0. */
90 #ifndef PCAP_NETMASK_UNKNOWN
91 #define PCAP_NETMASK_UNKNOWN 0xffffffff
92 #endif
93 
94 SC_ATOMIC_DECLARE(uint32_t, thread_cnt);
95 
96 typedef struct PcapFileName_ {
97  char *filename;
98  char *dirname;
99 
100  /* Like a struct timeval, but with fixed size. This is only used when
101  * seeding the ring buffer on start. */
102  struct {
103  uint64_t secs;
104  uint32_t usecs;
105  };
106 
107  TAILQ_ENTRY(PcapFileName_) next; /**< Pointer to next Pcap File for tailq. */
109 
110 thread_local char *pcap_file_thread = NULL;
111 
112 typedef struct PcapLogProfileData_ {
113  uint64_t total;
114  uint64_t cnt;
116 
117 #define MAX_TOKS 9
118 #define MAX_FILENAMELEN 513
119 
123 };
124 
125 typedef struct PcapLogCompressionData_ {
127  uint8_t *buffer;
128  uint64_t buffer_size;
129 #ifdef HAVE_LIBLZ4
130  LZ4F_compressionContext_t lz4f_context;
131  LZ4F_preferences_t lz4f_prefs;
132 #endif /* HAVE_LIBLZ4 */
133  FILE *file;
134  uint8_t *pcap_buf;
135  uint64_t pcap_buf_size;
137  uint64_t bytes_in_block;
139 
140 /**
141  * PcapLog thread vars
142  *
143  * Used for storing file options.
144  */
145 typedef struct PcapLogData_ {
146  int use_stream_depth; /**< use stream depth i.e. ignore packets that reach limit */
147  int honor_pass_rules; /**< don't log if pass rules have matched */
148  char *bpf_filter; /**< bpf filter to apply to output */
150  uint64_t pkt_cnt; /**< total number of packets */
151  struct pcap_pkthdr *h; /**< pcap header struct */
152  char *filename; /**< current filename */
153  int mode; /**< normal or multi */
154  int prev_day; /**< last day, for finding out when */
155  uint64_t size_current; /**< file current size */
156  uint64_t size_limit; /**< file size limit */
157  pcap_t *pcap_dead_handle; /**< pcap_dumper_t needs a handle */
158  pcap_dumper_t *pcap_dumper; /**< actually writes the packets */
159  struct bpf_program *bpfp; /**< compiled bpf program */
160  uint64_t profile_data_size; /**< track in bytes how many bytes we wrote */
161  uint32_t file_cnt; /**< count of pcap files we currently have */
162  uint32_t max_files; /**< maximum files to use in ring buffer mode */
163  bool is_private; /**< true if ctx is thread local */
165  conditional; /**< log all packets or just packets and flows with alerts */
166 
174 
175  TAILQ_HEAD(, PcapFileName_) pcap_file_list;
176 
177  uint32_t thread_number; /**< thread number, first thread is 1, second 2, etc */
178  int use_ringbuffer; /**< ring buffer mode enabled or disabled */
179  int timestamp_format; /**< timestamp format sec or usec */
180  char *prefix; /**< filename prefix */
181  const char *suffix; /**< filename suffix */
182  char dir[PATH_MAX]; /**< pcap log directory */
183  int reported;
184  int threads; /**< number of threads (only set in the global) */
185  char *filename_parts[MAX_TOKS];
186  int filename_part_cnt;
187  struct timeval last_pcap_dump;
188  int fopen_err; /**< set to the last fopen error */
189  bool pcap_open_err; /**< true if the last pcap open errored */
190 
191  PcapLogCompressionData compression;
193 
194 typedef struct PcapLogThreadData_ {
197  uint16_t counter_written; /**< Counter for number of packets written */
198  uint16_t counter_filtered_bpf; /**< Counter for number of packets filtered out and not writen */
200 
201 /* Pattern for extracting timestamp from pcap log files. */
202 static const char timestamp_pattern[] = ".*?(\\d+)(\\.(\\d+))?";
203 static pcre2_code *pcre_timestamp_code = NULL;
204 static pcre2_match_data *pcre_timestamp_match = NULL;
205 
206 /* global pcap data for when we're using multi mode. At exit we'll
207  * merge counters into this one and then report counters. */
208 static PcapLogData *g_pcap_data = NULL;
209 
210 static int PcapLogOpenFileCtx(PcapLogData *);
211 static int PcapLog(ThreadVars *, void *, const Packet *);
212 static TmEcode PcapLogDataInit(ThreadVars *, const void *, void **);
213 static TmEcode PcapLogDataDeinit(ThreadVars *, void *);
214 static void PcapLogFileDeInitCtx(OutputCtx *);
215 static OutputInitResult PcapLogInitCtx(SCConfNode *);
216 static void PcapLogProfilingDump(PcapLogData *);
217 static bool PcapLogCondition(ThreadVars *, void *, const Packet *);
218 
219 void PcapLogRegister(void)
220 {
221  OutputPacketLoggerFunctions output_logger_functions = {
222  .LogFunc = PcapLog,
223  .FlushFunc = NULL,
224  .ConditionFunc = PcapLogCondition,
225  .ThreadInitFunc = PcapLogDataInit,
226  .ThreadDeinitFunc = PcapLogDataDeinit,
227  .ThreadExitPrintStatsFunc = NULL,
228  };
230  LOGGER_PCAP, MODULE_NAME, "pcap-log", PcapLogInitCtx, &output_logger_functions);
232  SC_ATOMIC_INIT(thread_cnt);
233  SC_ATOMIC_SET(thread_cnt, 1); /* first id is 1 */
234 }
235 
236 #define PCAPLOG_PROFILE_START \
237  uint64_t pcaplog_profile_ticks = UtilCpuGetTicks()
238 
239 #define PCAPLOG_PROFILE_END(prof) \
240  (prof).total += (UtilCpuGetTicks() - pcaplog_profile_ticks); \
241  (prof).cnt++
242 
243 static bool PcapLogCondition(ThreadVars *tv, void *thread_data, const Packet *p)
244 {
245  PcapLogThreadData *ptd = (PcapLogThreadData *)thread_data;
246 
247  /* Log alerted flow or tagged flow */
248  switch (ptd->pcap_log->conditional) {
249  case LOGMODE_COND_ALL:
250  break;
251  case LOGMODE_COND_ALERTS:
252  return (p->alerts.cnt || (p->flow && FlowHasAlerts(p->flow)));
253  case LOGMODE_COND_TAG:
254  return (p->flags & (PKT_HAS_TAG | PKT_FIRST_TAG));
255  }
256 
257  if (p->flags & PKT_PSEUDO_STREAM_END) {
258  return false;
259  }
260 
261  if (p->ttype == PacketTunnelChild) {
262  return false;
263  }
264  return true;
265 }
266 
267 /**
268  * \brief Function to close pcaplog file
269  *
270  * \param t Thread Variable containing input/output queue, cpu affinity etc.
271  * \param pl PcapLog thread variable.
272  */
273 static int PcapLogCloseFile(ThreadVars *t, PcapLogData *pl)
274 {
275  if (pl != NULL) {
277 
278  if (pl->pcap_dumper != NULL) {
279  pcap_dump_close(pl->pcap_dumper);
280 #ifdef HAVE_LIBLZ4
281  PcapLogCompressionData *comp = &pl->compression;
283  /* pcap_dump_close() has closed its output ``file'',
284  * so we need to call fmemopen again. */
285 
286  comp->pcap_buf_wrapper = SCFmemopen(comp->pcap_buf,
287  comp->pcap_buf_size, "w");
288  if (comp->pcap_buf_wrapper == NULL) {
289  SCLogError("SCFmemopen failed: %s", strerror(errno));
290  return TM_ECODE_FAILED;
291  }
292  }
293 #endif /* HAVE_LIBLZ4 */
294  }
295  pl->size_current = 0;
296  pl->pcap_dumper = NULL;
297 
298  if (pl->pcap_dead_handle != NULL)
299  pcap_close(pl->pcap_dead_handle);
300  pl->pcap_dead_handle = NULL;
301 
302 #ifdef HAVE_LIBLZ4
303  PcapLogCompressionData *comp = &pl->compression;
305  /* pcap_dump_close did not write any data because we call
306  * pcap_dump_flush() after every write when writing
307  * compressed output. */
308  uint64_t bytes_written = LZ4F_compressEnd(comp->lz4f_context,
309  comp->buffer, comp->buffer_size, NULL);
310  if (LZ4F_isError(bytes_written)) {
311  SCLogError("LZ4F_compressEnd: %s", LZ4F_getErrorName(bytes_written));
312  return TM_ECODE_FAILED;
313  }
314  if (fwrite(comp->buffer, 1, bytes_written, comp->file) < bytes_written) {
315  SCLogError("fwrite failed: %s", strerror(errno));
316  return TM_ECODE_FAILED;
317  }
318  fclose(comp->file);
319  comp->bytes_in_block = 0;
320  }
321 #endif /* HAVE_LIBLZ4 */
322 
324  }
325 
326  return 0;
327 }
328 
329 static void PcapFileNameFree(PcapFileName *pf)
330 {
331  if (pf != NULL) {
332  if (pf->filename != NULL) {
333  SCFree(pf->filename);
334  }
335  if (pf->dirname != NULL) {
336  SCFree(pf->dirname);
337  }
338  SCFree(pf);
339  }
340 }
341 
342 /**
343  * \brief Function to rotate pcaplog file
344  *
345  * \param t Thread Variable containing input/output queue, cpu affinity etc.
346  * \param pl PcapLog thread variable.
347  *
348  * \retval 0 on success
349  * \retval -1 on failure
350  */
351 static int PcapLogRotateFile(ThreadVars *t, PcapLogData *pl)
352 {
353  PcapFileName *pf;
354 
356 
357  if (PcapLogCloseFile(t,pl) < 0) {
358  SCLogDebug("PcapLogCloseFile failed");
359  return -1;
360  }
361 
362  if (pl->use_ringbuffer == RING_BUFFER_MODE_ENABLED && pl->file_cnt >= pl->max_files) {
363  pf = TAILQ_FIRST(&pl->pcap_file_list);
364  SCLogDebug("Removing pcap file %s", pf->filename);
365 
366  if (remove(pf->filename) != 0) {
367  // VJ remove can fail because file is already gone
368  // SCLogWarning("failed to remove log file %s: %s",
369  // pf->filename, strerror( errno ));
370  }
371 
372  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
373  PcapFileNameFree(pf);
374  pl->file_cnt--;
375  }
376 
377  if (PcapLogOpenFileCtx(pl) < 0) {
378  SCLogError("opening new pcap log file failed");
379  return -1;
380  }
381  pl->file_cnt++;
382  SCLogDebug("file_cnt %u", pl->file_cnt);
383 
385  return 0;
386 }
387 
388 static int PcapLogOpenHandles(PcapLogData *pl, const Packet *p)
389 {
391 
392  int datalink = p->datalink;
393  if (p->ttype == PacketTunnelChild) {
394  Packet *real_p = p->root;
395  datalink = real_p->datalink;
396  }
397  if (pl->pcap_dead_handle == NULL) {
398  SCLogDebug("Setting pcap-log link type to %u", datalink);
399  if ((pl->pcap_dead_handle = pcap_open_dead(datalink, PCAP_SNAPLEN)) == NULL) {
400  SCLogDebug("Error opening dead pcap handle");
401  return TM_ECODE_FAILED;
402  }
403 
404  if (pl->bpfp == NULL && pl->bpf_filter) {
405  struct bpf_program bpfp;
406  if (pcap_compile(pl->pcap_dead_handle, &bpfp, pl->bpf_filter, 0,
407  PCAP_NETMASK_UNKNOWN) == PCAP_ERROR) {
408  FatalError("Failed to compile BPF filter, aborting: %s: %s", pl->bpf_filter,
409  pcap_geterr(pl->pcap_dead_handle));
410  } else {
411  pl->bpfp = SCCalloc(1, sizeof(*pl->bpfp));
412  if (pl->bpfp == NULL) {
413  FatalError("Failed to allocate memory for BPF filter, aborting");
414  }
415  *pl->bpfp = bpfp;
416  }
417  }
418  }
419 
420  if (pl->pcap_dumper == NULL) {
421  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_NONE) {
422  if ((pl->pcap_dumper = pcap_dump_open(pl->pcap_dead_handle,
423  pl->filename)) == NULL) {
424  if (!pl->pcap_open_err) {
425  SCLogError("Error opening dump file %s", pcap_geterr(pl->pcap_dead_handle));
426  pl->pcap_open_err = true;
427  }
428  return TM_ECODE_FAILED;
429  } else {
430  pl->pcap_open_err = false;
431  }
432  }
433 #ifdef HAVE_LIBLZ4
434  else if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
435  PcapLogCompressionData *comp = &pl->compression;
436 
437  comp->file = fopen(pl->filename, "w");
438  if (comp->file == NULL) {
439  if (errno != pl->fopen_err) {
440  SCLogError("Error opening file for compressed output: %s", strerror(errno));
441  pl->fopen_err = errno;
442  }
443  return TM_ECODE_FAILED;
444  } else {
445  pl->fopen_err = 0;
446  }
447 
448  if ((pl->pcap_dumper = pcap_dump_fopen(pl->pcap_dead_handle, comp->pcap_buf_wrapper)) ==
449  NULL) {
450  if (!pl->pcap_open_err) {
451  SCLogError("Error opening dump file %s", pcap_geterr(pl->pcap_dead_handle));
452  pl->pcap_open_err = true;
453  }
454  fclose(comp->file);
455  comp->file = NULL;
456  return TM_ECODE_FAILED;
457  } else {
458  pl->pcap_open_err = false;
459  }
460 
461  uint64_t bytes_written = LZ4F_compressBegin(comp->lz4f_context,
462  comp->buffer, comp->buffer_size, NULL);
463  if (LZ4F_isError(bytes_written)) {
464  SCLogError("LZ4F_compressBegin: %s", LZ4F_getErrorName(bytes_written));
465  return TM_ECODE_FAILED;
466  }
467  if (fwrite(comp->buffer, 1, bytes_written, comp->file) < bytes_written) {
468  SCLogError("fwrite failed: %s", strerror(errno));
469  return TM_ECODE_FAILED;
470  }
471  }
472 #endif /* HAVE_LIBLZ4 */
473  }
474 
476  return TM_ECODE_OK;
477 }
478 
479 /** \internal
480  * \brief lock wrapper for main PcapLog() function
481  * NOTE: only meant for use in main PcapLog() function.
482  */
483 static void PcapLogLock(PcapLogData *pl)
484 {
485  if (!(pl->is_private)) {
487  SCMutexLock(&pl->plog_lock);
489  }
490 }
491 
492 /** \internal
493  * \brief unlock wrapper for main PcapLog() function
494  * NOTE: only meant for use in main PcapLog() function.
495  */
496 static void PcapLogUnlock(PcapLogData *pl)
497 {
498  if (!(pl->is_private)) {
500  SCMutexUnlock(&pl->plog_lock);
502  }
503 }
504 
505 static inline int PcapWrite(
506  ThreadVars *tv, PcapLogThreadData *td, const uint8_t *data, const size_t len)
507 {
508  struct timeval current_dump;
509  gettimeofday(&current_dump, NULL);
510  PcapLogData *pl = td->pcap_log;
511 
512  if (pl->bpfp) {
513  if (pcap_offline_filter(pl->bpfp, pl->h, data) == 0) {
514  SCLogDebug("Packet doesn't match filter, will not be logged.");
516  return TM_ECODE_OK;
517  }
518  }
519 
521 
522  pcap_dump((u_char *)pl->pcap_dumper, pl->h, data);
523  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_NONE) {
524  pl->size_current += len;
525  }
526 #ifdef HAVE_LIBLZ4
527  else if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
528  PcapLogCompressionData *comp = &pl->compression;
529  pcap_dump_flush(pl->pcap_dumper);
530  long in_size = ftell(comp->pcap_buf_wrapper);
531  if (in_size < 0) {
532  SCLogError("ftell failed with: %s", strerror(errno));
533  return TM_ECODE_FAILED;
534  }
535  uint64_t out_size = LZ4F_compressUpdate(comp->lz4f_context, comp->buffer, comp->buffer_size,
536  comp->pcap_buf, (uint64_t)in_size, NULL);
537  if (LZ4F_isError(len)) {
538  SCLogError("LZ4F_compressUpdate: %s", LZ4F_getErrorName(len));
539  return TM_ECODE_FAILED;
540  }
541  if (fseek(comp->pcap_buf_wrapper, 0, SEEK_SET) != 0) {
542  SCLogError("fseek failed: %s", strerror(errno));
543  return TM_ECODE_FAILED;
544  }
545  if (fwrite(comp->buffer, 1, out_size, comp->file) < out_size) {
546  SCLogError("fwrite failed: %s", strerror(errno));
547  return TM_ECODE_FAILED;
548  }
549  if (out_size > 0) {
550  pl->size_current += out_size;
551  comp->bytes_in_block = len;
552  } else {
553  comp->bytes_in_block += len;
554  }
555  }
556 #endif /* HAVE_LIBLZ4 */
557  if (TimeDifferenceMicros(pl->last_pcap_dump, current_dump) >= PCAP_BUFFER_TIMEOUT) {
558  pcap_dump_flush(pl->pcap_dumper);
559  }
560  pl->last_pcap_dump = current_dump;
561  return TM_ECODE_OK;
562 }
563 
567 };
568 
569 static int PcapLogSegmentCallback(
570  const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
571 {
572  struct PcapLogCallbackContext *pctx = (struct PcapLogCallbackContext *)data;
573 
574  if (seg->pcap_hdr_storage->pktlen) {
575  struct timeval tv;
577  pctx->td->pcap_log->h->ts.tv_sec = tv.tv_sec;
578  pctx->td->pcap_log->h->ts.tv_usec = tv.tv_usec;
579  pctx->td->pcap_log->h->len = seg->pcap_hdr_storage->pktlen + buflen;
580  pctx->td->pcap_log->h->caplen = seg->pcap_hdr_storage->pktlen + buflen;
581  MemBufferReset(pctx->td->buf);
583  pctx->td->buf, seg->pcap_hdr_storage->pkt_hdr, seg->pcap_hdr_storage->pktlen);
584  MemBufferWriteRaw(pctx->td->buf, buf, buflen);
585 
586  PcapWrite(pctx->tv, pctx->td, (uint8_t *)pctx->td->buf->buffer, pctx->td->pcap_log->h->len);
587  }
588  return 1;
589 }
590 
591 static void PcapLogDumpSegments(ThreadVars *tv, PcapLogThreadData *td, const Packet *p)
592 {
593  uint8_t flag = STREAM_DUMP_HEADERS;
594 
595  /* Loop on segment from this side */
596  struct PcapLogCallbackContext data = { tv, td };
597  StreamSegmentForSession(p, flag, PcapLogSegmentCallback, (void *)&data);
598 }
599 
600 /**
601  * \brief Pcap logging main function
602  *
603  * \param t threadvar
604  * \param p packet
605  * \param thread_data thread module specific data
606  *
607  * \retval TM_ECODE_OK on succes
608  * \retval TM_ECODE_FAILED on serious error
609  */
610 static int PcapLog(ThreadVars *tv, void *thread_data, const Packet *p)
611 {
612  size_t len;
613  int ret = 0;
614  Packet *rp = NULL;
615 
616  PcapLogThreadData *td = (PcapLogThreadData *)thread_data;
617  PcapLogData *pl = td->pcap_log;
618 
621  return TM_ECODE_OK;
622  }
623 
624  PcapLogLock(pl);
625 
626  pl->pkt_cnt++;
627  pl->h->ts.tv_sec = SCTIME_SECS(p->ts);
628  pl->h->ts.tv_usec = SCTIME_USECS(p->ts);
629  if (p->ttype == PacketTunnelChild) {
630  rp = p->root;
631  pl->h->caplen = GET_PKT_LEN(rp);
632  pl->h->len = GET_PKT_LEN(rp);
634  } else {
635  pl->h->caplen = GET_PKT_LEN(p);
636  pl->h->len = GET_PKT_LEN(p);
638  }
639 
640  if (pl->filename == NULL) {
641  ret = PcapLogOpenFileCtx(pl);
642  if (ret < 0) {
643  PcapLogUnlock(pl);
644  return TM_ECODE_FAILED;
645  }
646  SCLogDebug("Opening PCAP log file %s", pl->filename);
647  }
648 
649  PcapLogCompressionData *comp = &pl->compression;
651  if ((pl->size_current + len) > pl->size_limit) {
652  if (PcapLogRotateFile(tv, pl) < 0) {
653  PcapLogUnlock(pl);
654  SCLogDebug("rotation of pcap failed");
655  return TM_ECODE_FAILED;
656  }
657  }
658  }
659 #ifdef HAVE_LIBLZ4
660  else if (comp->format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
661  /* When writing compressed pcap logs, we have no way of knowing
662  * for sure whether adding this packet would cause the current
663  * file to exceed the size limit. Thus, we record the number of
664  * bytes that have been fed into lz4 since the last write, and
665  * act as if they would be written uncompressed. */
666 
667  if ((pl->size_current + comp->bytes_in_block + len) > pl->size_limit) {
668  if (PcapLogRotateFile(tv, pl) < 0) {
669  PcapLogUnlock(pl);
670  SCLogDebug("rotation of pcap failed");
671  return TM_ECODE_FAILED;
672  }
673  }
674  }
675 #endif /* HAVE_LIBLZ4 */
676 
677  /* XXX pcap handles, nfq, pfring, can only have one link type ipfw? we do
678  * this here as we don't know the link type until we get our first packet */
679  if (pl->pcap_dead_handle == NULL || pl->pcap_dumper == NULL) {
680  if (PcapLogOpenHandles(pl, p) != TM_ECODE_OK) {
681  PcapLogUnlock(pl);
682  return TM_ECODE_FAILED;
683  }
684  }
685 
687 
688  /* if we are using alerted logging and if packet is first one with alert in flow
689  * then we need to dump in the pcap the stream acked by the packet */
691  if (PacketIsTCP(p)) {
692  /* dump fake packets for all segments we have on acked by packet */
693  PcapLogDumpSegments(tv, td, p);
694 
695  if (p->flags & PKT_PSEUDO_STREAM_END) {
696  PcapLogUnlock(pl);
697  return TM_ECODE_OK;
698  }
699 
700  /* PcapLogDumpSegment has written over the PcapLogData variables so need to update */
701  pl->h->ts.tv_sec = SCTIME_SECS(p->ts);
702  pl->h->ts.tv_usec = SCTIME_USECS(p->ts);
703  if (p->ttype == PacketTunnelChild) {
704  rp = p->root;
705  pl->h->caplen = GET_PKT_LEN(rp);
706  pl->h->len = GET_PKT_LEN(rp);
708  } else {
709  pl->h->caplen = GET_PKT_LEN(p);
710  pl->h->len = GET_PKT_LEN(p);
712  }
713  }
714  }
715 
716  if (p->ttype == PacketTunnelChild) {
717  rp = p->root;
718  ret = PcapWrite(tv, td, GET_PKT_DATA(rp), len);
719  } else {
720  ret = PcapWrite(tv, td, GET_PKT_DATA(p), len);
721  }
722  if (ret != TM_ECODE_OK) {
724  PcapLogUnlock(pl);
725  return ret;
726  }
727 
729  pl->profile_data_size += len;
730 
731  SCLogDebug("pl->size_current %"PRIu64", pl->size_limit %"PRIu64,
732  pl->size_current, pl->size_limit);
733 
734  PcapLogUnlock(pl);
735  return TM_ECODE_OK;
736 }
737 
738 static PcapLogData *PcapLogDataCopy(const PcapLogData *pl)
739 {
740  BUG_ON(pl->mode != LOGMODE_MULTI);
741  PcapLogData *copy = SCCalloc(1, sizeof(*copy));
742  if (unlikely(copy == NULL)) {
743  return NULL;
744  }
745 
746  copy->h = SCCalloc(1, sizeof(*copy->h));
747  if (unlikely(copy->h == NULL)) {
748  SCFree(copy);
749  return NULL;
750  }
751 
752  copy->prefix = SCStrdup(pl->prefix);
753  if (unlikely(copy->prefix == NULL)) {
754  SCFree(copy->h);
755  SCFree(copy);
756  return NULL;
757  }
758 
759  copy->suffix = pl->suffix;
760 
761  /* settings TODO move to global cfg struct */
762  copy->is_private = true;
763  copy->mode = pl->mode;
764  copy->max_files = pl->max_files;
765  copy->use_ringbuffer = pl->use_ringbuffer;
766  copy->timestamp_format = pl->timestamp_format;
768  copy->size_limit = pl->size_limit;
769  copy->conditional = pl->conditional;
770  copy->bpf_filter = pl->bpf_filter;
771 
772  const PcapLogCompressionData *comp = &pl->compression;
773  PcapLogCompressionData *copy_comp = &copy->compression;
774  copy_comp->format = comp->format;
775 #ifdef HAVE_LIBLZ4
777  /* We need to allocate a new compression context and buffers for
778  * the copy. First copy the things that can simply be copied. */
779 
780  copy_comp->buffer_size = comp->buffer_size;
781  copy_comp->pcap_buf_size = comp->pcap_buf_size;
782  copy_comp->lz4f_prefs = comp->lz4f_prefs;
783 
784  /* Allocate the buffers. */
785 
786  copy_comp->buffer = SCMalloc(copy_comp->buffer_size);
787  if (copy_comp->buffer == NULL) {
788  SCLogError("SCMalloc failed: %s", strerror(errno));
789  SCFree(copy->prefix);
790  SCFree(copy->h);
791  SCFree(copy);
792  return NULL;
793  }
794  copy_comp->pcap_buf = SCMalloc(copy_comp->pcap_buf_size);
795  if (copy_comp->pcap_buf == NULL) {
796  SCLogError("SCMalloc failed: %s", strerror(errno));
797  SCFree(copy_comp->buffer);
798  SCFree(copy->prefix);
799  SCFree(copy->h);
800  SCFree(copy);
801  return NULL;
802  }
803  copy_comp->pcap_buf_wrapper = SCFmemopen(copy_comp->pcap_buf,
804  copy_comp->pcap_buf_size, "w");
805  if (copy_comp->pcap_buf_wrapper == NULL) {
806  SCLogError("SCFmemopen failed: %s", strerror(errno));
807  SCFree(copy_comp->buffer);
808  SCFree(copy_comp->pcap_buf);
809  SCFree(copy->prefix);
810  SCFree(copy->h);
811  SCFree(copy);
812  return NULL;
813  }
814 
815  /* Initialize a new compression context. */
816 
817  LZ4F_errorCode_t errcode =
818  LZ4F_createCompressionContext(&copy_comp->lz4f_context, 1);
819  if (LZ4F_isError(errcode)) {
820  SCLogError("LZ4F_createCompressionContext failed: %s", LZ4F_getErrorName(errcode));
821  fclose(copy_comp->pcap_buf_wrapper);
822  SCFree(copy_comp->buffer);
823  SCFree(copy_comp->pcap_buf);
824  SCFree(copy->prefix);
825  SCFree(copy->h);
826  SCFree(copy);
827  return NULL;
828  }
829 
830  /* Initialize the rest. */
831 
832  copy_comp->file = NULL;
833  copy_comp->bytes_in_block = 0;
834  }
835 #endif /* HAVE_LIBLZ4 */
836 
837  TAILQ_INIT(&copy->pcap_file_list);
838  SCMutexInit(&copy->plog_lock, NULL);
839 
840  strlcpy(copy->dir, pl->dir, sizeof(copy->dir));
841 
842  for (int i = 0; i < pl->filename_part_cnt && i < MAX_TOKS; i++)
843  copy->filename_parts[i] = pl->filename_parts[i];
844  copy->filename_part_cnt = pl->filename_part_cnt;
845 
846  /* set thread number, first thread is 1 */
847  copy->thread_number = SC_ATOMIC_ADD(thread_cnt, 1);
848 
849  SCLogDebug("copied, returning %p", copy);
850  return copy;
851 }
852 
853 #ifdef INIT_RING_BUFFER
854 static int PcapLogGetTimeOfFile(const char *filename, uint64_t *secs,
855  uint32_t *usecs)
856 {
857  char buf[PATH_MAX];
858  size_t copylen;
859 
860  int n = pcre2_match(pcre_timestamp_code, (PCRE2_SPTR8)filename, strlen(filename), 0, 0,
861  pcre_timestamp_match, NULL);
862  if (n != 2 && n != 4) {
863  /* No match. */
864  return 0;
865  }
866 
867  if (n >= 2) {
868  /* Extract seconds. */
869  copylen = sizeof(buf);
870  if (pcre2_substring_copy_bynumber(pcre_timestamp_match, 1, (PCRE2_UCHAR8 *)buf, &copylen) <
871  0) {
872  return 0;
873  }
874  if (StringParseUint64(secs, 10, 0, buf) < 0) {
875  return 0;
876  }
877  }
878  if (n == 4) {
879  /* Extract microseconds. */
880  copylen = sizeof(buf);
881  if (pcre2_substring_copy_bynumber(pcre_timestamp_match, 3, (PCRE2_UCHAR8 *)buf, &copylen) <
882  0) {
883  return 0;
884  }
885  if (StringParseUint32(usecs, 10, 0, buf) < 0) {
886  return 0;
887  }
888  }
889 
890  return 1;
891 }
892 
893 static TmEcode PcapLogInitRingBuffer(PcapLogData *pl)
894 {
895  char pattern[PATH_MAX];
896 
897  SCLogInfo("Initializing PCAP ring buffer for %s/%s.",
898  pl->dir, pl->prefix);
899 
900  strlcpy(pattern, pl->dir, PATH_MAX);
901  if (pattern[strlen(pattern) - 1] != '/') {
902  strlcat(pattern, "/", PATH_MAX);
903  }
904  if (pl->mode == LOGMODE_MULTI) {
905  for (int i = 0; i < pl->filename_part_cnt; i++) {
906  char *part = pl->filename_parts[i];
907  if (part == NULL || strlen(part) == 0) {
908  continue;
909  }
910  if (part[0] != '%' || strlen(part) < 2) {
911  strlcat(pattern, part, PATH_MAX);
912  continue;
913  }
914  switch (part[1]) {
915  case 'i':
916  SCLogError("Thread ID not allowed in ring buffer mode.");
917  return TM_ECODE_FAILED;
918  case 'n': {
919  char tmp[PATH_MAX];
920  snprintf(tmp, PATH_MAX, "%"PRIu32, pl->thread_number);
921  strlcat(pattern, tmp, PATH_MAX);
922  break;
923  }
924  case 't':
925  strlcat(pattern, "*", PATH_MAX);
926  break;
927  default:
928  SCLogError("Unsupported format character: %%%s", part);
929  return TM_ECODE_FAILED;
930  }
931  }
932  } else {
933  strlcat(pattern, pl->prefix, PATH_MAX);
934  strlcat(pattern, ".*", PATH_MAX);
935  }
936  strlcat(pattern, pl->suffix, PATH_MAX);
937 
938  char *basename = strrchr(pattern, '/');
939  *basename++ = '\0';
940 
941  /* Pattern is now just the directory name. */
942  DIR *dir = opendir(pattern);
943  if (dir == NULL) {
944  SCLogWarning("Failed to open directory %s: %s", pattern, strerror(errno));
945  return TM_ECODE_FAILED;
946  }
947 
948  for (;;) {
949  struct dirent *entry = readdir(dir);
950  if (entry == NULL) {
951  break;
952  }
953  if (fnmatch(basename, entry->d_name, 0) != 0) {
954  continue;
955  }
956 
957  uint64_t secs = 0;
958  uint32_t usecs = 0;
959 
960  if (!PcapLogGetTimeOfFile(entry->d_name, &secs, &usecs)) {
961  /* Failed to get time stamp out of file name. Not necessarily a
962  * failure as the file might just not be a pcap log file. */
963  continue;
964  }
965 
966  PcapFileName *pf = SCCalloc(sizeof(*pf), 1);
967  if (unlikely(pf == NULL)) {
968  goto fail;
969  }
970  char path[PATH_MAX];
971  if (PathMerge(path, sizeof(path), pattern, entry->d_name) < 0)
972  goto fail;
973 
974  if ((pf->filename = SCStrdup(path)) == NULL) {
975  goto fail;
976  }
977  if ((pf->dirname = SCStrdup(pattern)) == NULL) {
978  goto fail;
979  }
980  pf->secs = secs;
981  pf->usecs = usecs;
982 
983  if (TAILQ_EMPTY(&pl->pcap_file_list)) {
984  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
985  } else {
986  /* Ordered insert. */
987  PcapFileName *it = NULL;
988  TAILQ_FOREACH(it, &pl->pcap_file_list, next) {
989  if (pf->secs < it->secs) {
990  break;
991  } else if (pf->secs == it->secs && pf->usecs < it->usecs) {
992  break;
993  }
994  }
995  if (it == NULL) {
996  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
997  } else {
998  TAILQ_INSERT_BEFORE(it, pf, next);
999  }
1000  }
1001  pl->file_cnt++;
1002  continue;
1003 
1004  fail:
1005  if (pf != NULL) {
1006  if (pf->filename != NULL) {
1007  SCFree(pf->filename);
1008  }
1009  if (pf->dirname != NULL) {
1010  SCFree(pf->dirname);
1011  }
1012  SCFree(pf);
1013  }
1014  break;
1015  }
1016 
1017  if (pl->file_cnt > pl->max_files) {
1018  PcapFileName *pf = TAILQ_FIRST(&pl->pcap_file_list);
1019  while (pf != NULL && pl->file_cnt > pl->max_files) {
1020  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
1021 
1022  SCLogDebug("Removing PCAP file %s", pf->filename);
1023  if (remove(pf->filename) != 0) {
1024  SCLogWarning("Failed to remove PCAP file %s: %s", pf->filename, strerror(errno));
1025  }
1026  PcapFileNameFree(pf);
1027  pl->file_cnt--;
1028 
1029  pf = TAILQ_FIRST(&pl->pcap_file_list);
1030  }
1031  }
1032 
1033  closedir(dir);
1034 
1035  /* For some reason file count is initialized at one, instead of 0. */
1036  SCLogNotice("Ring buffer initialized with %d files.", pl->file_cnt - 1);
1037 
1038  return TM_ECODE_OK;
1039 }
1040 #endif /* INIT_RING_BUFFER */
1041 
1042 static TmEcode PcapLogDataInit(ThreadVars *t, const void *initdata, void **data)
1043 {
1044  if (initdata == NULL) {
1045  SCLogDebug("Error getting context for LogPcap. \"initdata\" argument NULL");
1046  return TM_ECODE_FAILED;
1047  }
1048 
1049  PcapLogData *pl = ((OutputCtx *)initdata)->data;
1050 
1051  PcapLogThreadData *td = SCCalloc(1, sizeof(*td));
1052  if (unlikely(td == NULL))
1053  return TM_ECODE_FAILED;
1054 
1055  td->counter_written = StatsRegisterCounter("pcap_log.written", t);
1056  td->counter_filtered_bpf = StatsRegisterCounter("pcap_log.filtered_bpf", t);
1057 
1058  if (pl->mode == LOGMODE_MULTI)
1059  td->pcap_log = PcapLogDataCopy(pl);
1060  else
1061  td->pcap_log = pl;
1062  BUG_ON(td->pcap_log == NULL);
1063 
1064  if (DatalinkHasMultipleValues()) {
1065  if (pl->mode != LOGMODE_MULTI) {
1066  FatalError("Pcap logging with multiple link type is not supported.");
1067  } else {
1068  /* In multi mode, only pcap conditional is not supported as a flow timeout
1069  * will trigger packet logging with potentially invalid datalink. In regular
1070  * pcap logging, the logging should be done in the same thread if we
1071  * have a proper load balancing. So no mix of datalink should occur. But we need a
1072  * proper load balancing so this needs at least a warning.
1073  */
1074  switch (pl->conditional) {
1075  case LOGMODE_COND_ALERTS:
1076  case LOGMODE_COND_TAG:
1077  FatalError("Can't have multiple link types in pcap conditional mode.");
1078  break;
1079  default:
1080  SCLogWarning("Using multiple link types can result in invalid pcap output");
1081  }
1082  }
1083  }
1084 
1085  PcapLogLock(td->pcap_log);
1086 
1087  /** Use the Output Context (file pointer and mutex) */
1088  td->pcap_log->pkt_cnt = 0;
1089  td->pcap_log->pcap_dead_handle = NULL;
1090  td->pcap_log->pcap_dumper = NULL;
1091  if (td->pcap_log->file_cnt < 1) {
1092  td->pcap_log->file_cnt = 1;
1093  }
1094 
1095  SCTime_t ts = TimeGet();
1096  struct tm local_tm;
1097  struct tm *tms = SCLocalTime(SCTIME_SECS(ts), &local_tm);
1098  td->pcap_log->prev_day = tms->tm_mday;
1099 
1100  PcapLogUnlock(td->pcap_log);
1101 
1102  /* count threads in the global structure */
1103  SCMutexLock(&pl->plog_lock);
1104  pl->threads++;
1105  SCMutexUnlock(&pl->plog_lock);
1106 
1107  *data = (void *)td;
1108 
1111  } else {
1112  td->buf = NULL;
1113  }
1114 
1115  if (pl->max_files && (pl->mode == LOGMODE_MULTI || pl->threads == 1)) {
1116 #ifdef INIT_RING_BUFFER
1117  if (PcapLogInitRingBuffer(td->pcap_log) == TM_ECODE_FAILED) {
1118  return TM_ECODE_FAILED;
1119  }
1120 #else
1121  SCLogInfo("Unable to initialize ring buffer on this platform.");
1122 #endif /* INIT_RING_BUFFER */
1123  }
1124 
1125  /* Don't early initialize output files if in a PCAP file (offline)
1126  * mode. */
1127  if (!IsRunModeOffline(SCRunmodeGet())) {
1128  if (pl->mode == LOGMODE_MULTI) {
1129  PcapLogOpenFileCtx(td->pcap_log);
1130  } else {
1131  if (pl->filename == NULL) {
1132  PcapLogOpenFileCtx(pl);
1133  }
1134  }
1135  }
1136 
1137  return TM_ECODE_OK;
1138 }
1139 
1140 static void StatsMerge(PcapLogData *dst, PcapLogData *src)
1141 {
1142  dst->profile_open.total += src->profile_open.total;
1143  dst->profile_open.cnt += src->profile_open.cnt;
1144 
1145  dst->profile_close.total += src->profile_close.total;
1146  dst->profile_close.cnt += src->profile_close.cnt;
1147 
1148  dst->profile_write.total += src->profile_write.total;
1149  dst->profile_write.cnt += src->profile_write.cnt;
1150 
1151  dst->profile_rotate.total += src->profile_rotate.total;
1152  dst->profile_rotate.cnt += src->profile_rotate.cnt;
1153 
1154  dst->profile_handles.total += src->profile_handles.total;
1155  dst->profile_handles.cnt += src->profile_handles.cnt;
1156 
1157  dst->profile_lock.total += src->profile_lock.total;
1158  dst->profile_lock.cnt += src->profile_lock.cnt;
1159 
1160  dst->profile_unlock.total += src->profile_unlock.total;
1161  dst->profile_unlock.cnt += src->profile_unlock.cnt;
1162 
1163  dst->profile_data_size += src->profile_data_size;
1164 }
1165 
1166 static void PcapLogDataFree(PcapLogData *pl)
1167 {
1168 
1169  PcapFileName *pf;
1170  while ((pf = TAILQ_FIRST(&pl->pcap_file_list)) != NULL) {
1171  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
1172  PcapFileNameFree(pf);
1173  }
1174  if (pl == g_pcap_data) {
1175  for (int i = 0; i < MAX_TOKS; i++) {
1176  if (pl->filename_parts[i] != NULL) {
1177  SCFree(pl->filename_parts[i]);
1178  }
1179  }
1180  }
1181  SCFree(pl->h);
1182  SCFree(pl->filename);
1183  SCFree(pl->prefix);
1184 
1185  if (pl->pcap_dead_handle) {
1186  pcap_close(pl->pcap_dead_handle);
1187  }
1188 
1189  if (pl->bpfp) {
1190  pcap_freecode(pl->bpfp);
1191  SCFree(pl->bpfp);
1192  }
1193 
1194 #ifdef HAVE_LIBLZ4
1195  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
1196  SCFree(pl->compression.buffer);
1197  fclose(pl->compression.pcap_buf_wrapper);
1198  SCFree(pl->compression.pcap_buf);
1199  LZ4F_errorCode_t errcode =
1200  LZ4F_freeCompressionContext(pl->compression.lz4f_context);
1201  if (LZ4F_isError(errcode)) {
1202  SCLogWarning("Error freeing lz4 context.");
1203  }
1204  }
1205 #endif /* HAVE_LIBLZ4 */
1206  SCFree(pl);
1207 }
1208 
1209 /**
1210  * \brief Thread deinit function.
1211  *
1212  * \param t Thread Variable containing input/output queue, cpu affinity etc.
1213  * \param data PcapLog thread data.
1214  * \retval TM_ECODE_OK on success
1215  * \retval TM_ECODE_FAILED on failure
1216  */
1217 static TmEcode PcapLogDataDeinit(ThreadVars *t, void *thread_data)
1218 {
1219  PcapLogThreadData *td = (PcapLogThreadData *)thread_data;
1220  PcapLogData *pl = td->pcap_log;
1221 
1222  if (pl->pcap_dumper != NULL) {
1223  if (PcapLogCloseFile(t,pl) < 0) {
1224  SCLogDebug("PcapLogCloseFile failed");
1225  }
1226  }
1227 
1228  if (pl->mode == LOGMODE_MULTI) {
1229  SCMutexLock(&g_pcap_data->plog_lock);
1230  StatsMerge(g_pcap_data, pl);
1231  g_pcap_data->reported++;
1232  if (g_pcap_data->threads == g_pcap_data->reported)
1233  PcapLogProfilingDump(g_pcap_data);
1234  SCMutexUnlock(&g_pcap_data->plog_lock);
1235  } else {
1236  if (pl->reported == 0) {
1237  PcapLogProfilingDump(pl);
1238  pl->reported = 1;
1239  }
1240  }
1241 
1242  if (pl != g_pcap_data) {
1243  PcapLogDataFree(pl);
1244  }
1245 
1246  if (td->buf)
1247  MemBufferFree(td->buf);
1248 
1249  SCFree(td);
1250  return TM_ECODE_OK;
1251 }
1252 
1253 
1254 static int ParseFilename(PcapLogData *pl, const char *filename)
1255 {
1256  char *toks[MAX_TOKS] = { NULL };
1257  int tok = 0;
1258  char str[MAX_FILENAMELEN] = "";
1259  int s = 0;
1260  char *p = NULL;
1261  size_t filename_len = 0;
1262 
1263  if (filename) {
1264  filename_len = strlen(filename);
1265  if (filename_len > (MAX_FILENAMELEN-1)) {
1266  SCLogError("invalid filename option. Max filename-length: %d", MAX_FILENAMELEN - 1);
1267  goto error;
1268  }
1269 
1270  for (int i = 0; i < (int)strlen(filename); i++) {
1271  if (tok >= MAX_TOKS) {
1272  SCLogError("invalid filename option. Max 2 %%-sign options");
1273  goto error;
1274  }
1275 
1276  str[s++] = filename[i];
1277 
1278  if (filename[i] == '%') {
1279  str[s-1] = '\0';
1280  SCLogDebug("filename with %%-sign: %s", str);
1281 
1282  p = SCStrdup(str);
1283  if (p == NULL)
1284  goto error;
1285  toks[tok++] = p;
1286 
1287  s = 0;
1288 
1289  if (i+1 < (int)strlen(filename)) {
1290  if (tok >= MAX_TOKS) {
1291  SCLogError("invalid filename option. Max 2 %%-sign options");
1292  goto error;
1293  }
1294 
1295  if (filename[i+1] != 'n' && filename[i+1] != 't' && filename[i+1] != 'i') {
1296  SCLogError(
1297  "invalid filename option. Valid %%-sign options: %%n, %%i and %%t");
1298  goto error;
1299  }
1300  str[0] = '%';
1301  str[1] = filename[i+1];
1302  str[2] = '\0';
1303  p = SCStrdup(str);
1304  if (p == NULL)
1305  goto error;
1306  toks[tok++] = p;
1307  i++;
1308  }
1309  }
1310  }
1311 
1312  if ((tok == 0) && (pl->mode == LOGMODE_MULTI)) {
1313  SCLogError("Invalid filename for multimode. Need at least one %%-sign option");
1314  goto error;
1315  }
1316 
1317  if (s) {
1318  if (tok >= MAX_TOKS) {
1319  SCLogError("invalid filename option. Max 3 %%-sign options");
1320  goto error;
1321 
1322  }
1323  str[s++] = '\0';
1324  p = SCStrdup(str);
1325  if (p == NULL)
1326  goto error;
1327  toks[tok++] = p;
1328  }
1329 
1330  /* finally, store tokens in the pl */
1331  for (int i = 0; i < tok; i++) {
1332  if (toks[i] == NULL)
1333  goto error;
1334 
1335  SCLogDebug("toks[%d] %s", i, toks[i]);
1336  pl->filename_parts[i] = toks[i];
1337  }
1338  pl->filename_part_cnt = tok;
1339  }
1340  return 0;
1341 error:
1342  for (int x = 0; x < MAX_TOKS; x++) {
1343  if (toks[x] != NULL)
1344  SCFree(toks[x]);
1345  }
1346  return -1;
1347 }
1348 
1349 /** \brief Fill in pcap logging struct from the provided ConfNode.
1350  * \param conf The configuration node for this output.
1351  * \retval output_ctx
1352  * */
1353 static OutputInitResult PcapLogInitCtx(SCConfNode *conf)
1354 {
1355  OutputInitResult result = { NULL, false };
1356  int en;
1357  PCRE2_SIZE eo = 0;
1358 
1359  if (g_pcap_data) {
1360  FatalError("A pcap-log instance is already active, only one can be enabled.");
1361  }
1362 
1363  PcapLogData *pl = SCCalloc(1, sizeof(PcapLogData));
1364  if (unlikely(pl == NULL)) {
1365  FatalError("Failed to allocate Memory for PcapLogData");
1366  }
1367 
1368  pl->h = SCMalloc(sizeof(*pl->h));
1369  if (pl->h == NULL) {
1370  FatalError("Failed to allocate Memory for pcap header struct");
1371  }
1372 
1373  /* Set the defaults */
1374  pl->mode = LOGMODE_NORMAL;
1376  pl->use_ringbuffer = RING_BUFFER_MODE_DISABLED;
1377  pl->timestamp_format = TS_FORMAT_SEC;
1381 
1382  TAILQ_INIT(&pl->pcap_file_list);
1383 
1384  SCMutexInit(&pl->plog_lock, NULL);
1385 
1386  /* Initialize PCREs. */
1387  pcre_timestamp_code =
1388  pcre2_compile((PCRE2_SPTR8)timestamp_pattern, PCRE2_ZERO_TERMINATED, 0, &en, &eo, NULL);
1389  if (pcre_timestamp_code == NULL) {
1390  PCRE2_UCHAR errbuffer[256];
1391  pcre2_get_error_message(en, errbuffer, sizeof(errbuffer));
1392  FatalError(
1393  "Failed to compile \"%s\" at offset %d: %s", timestamp_pattern, (int)eo, errbuffer);
1394  }
1395  pcre_timestamp_match = pcre2_match_data_create_from_pattern(pcre_timestamp_code, NULL);
1396 
1397  /* conf params */
1398 
1399  const char *filename = NULL;
1400 
1401  if (conf != NULL) { /* To facilitate unit tests. */
1402  filename = SCConfNodeLookupChildValue(conf, "filename");
1403  }
1404 
1405  if (filename == NULL)
1406  filename = DEFAULT_LOG_FILENAME;
1407 
1408  if ((pl->prefix = SCStrdup(filename)) == NULL) {
1409  exit(EXIT_FAILURE);
1410  }
1411 
1412  pl->suffix = "";
1413 
1414  pl->size_limit = DEFAULT_LIMIT;
1415  if (conf != NULL) {
1416  const char *s_limit = NULL;
1417  s_limit = SCConfNodeLookupChildValue(conf, "limit");
1418  if (s_limit != NULL) {
1419  if (ParseSizeStringU64(s_limit, &pl->size_limit) < 0) {
1420  SCLogError("Failed to initialize pcap output, invalid limit: %s", s_limit);
1421  exit(EXIT_FAILURE);
1422  }
1423  if (pl->size_limit < 4096) {
1424  SCLogInfo("pcap-log \"limit\" value of %"PRIu64" assumed to be pre-1.2 "
1425  "style: setting limit to %"PRIu64"mb", pl->size_limit, pl->size_limit);
1426  uint64_t size = pl->size_limit * 1024 * 1024;
1427  pl->size_limit = size;
1428  } else if (pl->size_limit < MIN_LIMIT) {
1429  FatalError("Fail to initialize pcap-log output, limit less than "
1430  "allowed minimum of %d bytes.",
1431  MIN_LIMIT);
1432  }
1433  }
1434  }
1435 
1436  if (conf != NULL) {
1437  const char *s_mode = NULL;
1438  s_mode = SCConfNodeLookupChildValue(conf, "mode");
1439  if (s_mode != NULL) {
1440  if (strcasecmp(s_mode, "multi") == 0) {
1441  pl->mode = LOGMODE_MULTI;
1442  } else if (strcasecmp(s_mode, "normal") != 0) {
1443  FatalError("log-pcap: invalid mode \"%s\". Valid options: \"normal\""
1444  "or \"multi\" mode ",
1445  s_mode);
1446  }
1447  }
1448 
1449  const char *s_dir = NULL;
1450  s_dir = SCConfNodeLookupChildValue(conf, "dir");
1451  if (s_dir == NULL) {
1452  const char *log_dir = NULL;
1453  log_dir = SCConfigGetLogDirectory();
1454 
1455  strlcpy(pl->dir, log_dir, sizeof(pl->dir));
1456  SCLogInfo("Using log dir %s", pl->dir);
1457  } else {
1458  if (PathIsAbsolute(s_dir)) {
1459  strlcpy(pl->dir,
1460  s_dir, sizeof(pl->dir));
1461  } else {
1462  const char *log_dir = NULL;
1463  log_dir = SCConfigGetLogDirectory();
1464 
1465  snprintf(pl->dir, sizeof(pl->dir), "%s/%s",
1466  log_dir, s_dir);
1467  }
1468 
1469  struct stat stat_buf;
1470  if (stat(pl->dir, &stat_buf) != 0) {
1471  FatalError("The dir directory \"%s\" "
1472  "supplied doesn't exist. Shutting down the engine",
1473  pl->dir);
1474  }
1475  SCLogInfo("Using log dir %s", pl->dir);
1476  }
1477 
1478  const char *compression_str = SCConfNodeLookupChildValue(conf, "compression");
1479 
1480  PcapLogCompressionData *comp = &pl->compression;
1481  if (compression_str == NULL || strcmp(compression_str, "none") == 0) {
1483  comp->buffer = NULL;
1484  comp->buffer_size = 0;
1485  comp->file = NULL;
1486  comp->pcap_buf = NULL;
1487  comp->pcap_buf_size = 0;
1488  comp->pcap_buf_wrapper = NULL;
1489  } else if (strcmp(compression_str, "lz4") == 0) {
1490 #ifdef HAVE_LIBLZ4
1491  pl->compression.format = PCAP_LOG_COMPRESSION_FORMAT_LZ4;
1492 
1493  /* Use SCFmemopen so we can make pcap_dump write to a buffer. */
1494 
1495  comp->pcap_buf_size = sizeof(struct pcap_file_header) +
1496  sizeof(struct pcap_pkthdr) + PCAP_SNAPLEN;
1497  comp->pcap_buf = SCMalloc(comp->pcap_buf_size);
1498  if (comp->pcap_buf == NULL) {
1499  SCLogError("SCMalloc failed: %s", strerror(errno));
1500  exit(EXIT_FAILURE);
1501  }
1502  comp->pcap_buf_wrapper = SCFmemopen(comp->pcap_buf,
1503  comp->pcap_buf_size, "w");
1504  if (comp->pcap_buf_wrapper == NULL) {
1505  SCLogError("SCFmemopen failed: %s", strerror(errno));
1506  exit(EXIT_FAILURE);
1507  }
1508 
1509  /* Set lz4 preferences. */
1510 
1511  memset(&comp->lz4f_prefs, '\0', sizeof(comp->lz4f_prefs));
1512  comp->lz4f_prefs.frameInfo.blockSizeID = LZ4F_max4MB;
1513  comp->lz4f_prefs.frameInfo.blockMode = LZ4F_blockLinked;
1514  if (SCConfNodeChildValueIsTrue(conf, "lz4-checksum")) {
1515  comp->lz4f_prefs.frameInfo.contentChecksumFlag = 1;
1516  } else {
1517  comp->lz4f_prefs.frameInfo.contentChecksumFlag = 0;
1518  }
1519  intmax_t lvl = 0;
1520  if (SCConfGetChildValueInt(conf, "lz4-level", &lvl)) {
1521  if (lvl > 16) {
1522  lvl = 16;
1523  } else if (lvl < 0) {
1524  lvl = 0;
1525  }
1526  } else {
1527  lvl = 0;
1528  }
1529  comp->lz4f_prefs.compressionLevel = (int)lvl;
1530 
1531  /* Allocate resources for lz4. */
1532 
1533  LZ4F_errorCode_t errcode =
1534  LZ4F_createCompressionContext(&pl->compression.lz4f_context, 1);
1535 
1536  if (LZ4F_isError(errcode)) {
1537  SCLogError("LZ4F_createCompressionContext failed: %s", LZ4F_getErrorName(errcode));
1538  exit(EXIT_FAILURE);
1539  }
1540 
1541  /* Calculate the size of the lz4 output buffer. */
1542 
1543  comp->buffer_size = LZ4F_compressBound(comp->pcap_buf_size,
1544  &comp->lz4f_prefs);
1545 
1546  comp->buffer = SCMalloc(comp->buffer_size);
1547  if (unlikely(comp->buffer == NULL)) {
1548  FatalError("Failed to allocate memory for "
1549  "lz4 output buffer.");
1550  }
1551 
1552  comp->bytes_in_block = 0;
1553 
1554  /* Add the lz4 file extension to the log files. */
1555 
1556  pl->suffix = ".lz4";
1557 #else
1558  SCLogError("lz4 compression was selected "
1559  "in pcap-log, but suricata was not compiled with lz4 "
1560  "support.");
1561  PcapLogDataFree(pl);
1562  return result;
1563 #endif /* HAVE_LIBLZ4 */
1564  }
1565  else {
1566  SCLogError("Unsupported pcap-log "
1567  "compression format: %s",
1568  compression_str);
1569  PcapLogDataFree(pl);
1570  return result;
1571  }
1572 
1573  SCLogInfo("Selected pcap-log compression method: %s",
1574  compression_str ? compression_str : "none");
1575 
1576  const char *s_conditional = SCConfNodeLookupChildValue(conf, "conditional");
1577  if (s_conditional != NULL) {
1578  if (strcasecmp(s_conditional, "alerts") == 0) {
1581  } else if (strcasecmp(s_conditional, "tag") == 0) {
1584  } else if (strcasecmp(s_conditional, "all") != 0) {
1585  FatalError("log-pcap: invalid conditional \"%s\". Valid options: \"all\", "
1586  "\"alerts\", or \"tag\" mode ",
1587  s_conditional);
1588  }
1589  }
1590 
1591  SCLogInfo(
1592  "Selected pcap-log conditional logging: %s", s_conditional ? s_conditional : "all");
1593  }
1594 
1595  if (ParseFilename(pl, filename) != 0)
1596  exit(EXIT_FAILURE);
1597 
1598  SCLogInfo("using %s logging", (pl->mode == LOGMODE_MULTI ? "multi" : "normal"));
1599 
1600  uint32_t max_file_limit = DEFAULT_FILE_LIMIT;
1601  if (conf != NULL) {
1602  const char *max_number_of_files_s = NULL;
1603  max_number_of_files_s = SCConfNodeLookupChildValue(conf, "max-files");
1604  if (max_number_of_files_s != NULL) {
1605  if (StringParseUint32(&max_file_limit, 10, 0,
1606  max_number_of_files_s) == -1) {
1607  SCLogError("Failed to initialize "
1608  "pcap-log output, invalid number of files limit: %s",
1609  max_number_of_files_s);
1610  exit(EXIT_FAILURE);
1611  } else if (max_file_limit < 1) {
1612  FatalError("Failed to initialize pcap-log output, limit less than "
1613  "allowed minimum.");
1614  } else {
1615  pl->max_files = max_file_limit;
1616  pl->use_ringbuffer = RING_BUFFER_MODE_ENABLED;
1617  }
1618  }
1619  }
1620 
1621  const char *ts_format = NULL;
1622  if (conf != NULL) { /* To facilitate unit tests. */
1623  ts_format = SCConfNodeLookupChildValue(conf, "ts-format");
1624  }
1625  if (ts_format != NULL) {
1626  if (strcasecmp(ts_format, "usec") == 0) {
1627  pl->timestamp_format = TS_FORMAT_USEC;
1628  } else if (strcasecmp(ts_format, "sec") != 0) {
1629  SCLogError("log-pcap ts_format specified %s is invalid must be"
1630  " \"sec\" or \"usec\"",
1631  ts_format);
1632  exit(EXIT_FAILURE);
1633  }
1634  }
1635 
1636  const char *use_stream_depth = NULL;
1637  if (conf != NULL) { /* To facilitate unit tests. */
1638  use_stream_depth = SCConfNodeLookupChildValue(conf, "use-stream-depth");
1639  }
1640  if (use_stream_depth != NULL) {
1641  if (SCConfValIsFalse(use_stream_depth)) {
1643  } else if (SCConfValIsTrue(use_stream_depth)) {
1645  } else {
1646  FatalError("log-pcap use_stream_depth specified is invalid must be");
1647  }
1648  }
1649 
1650  const char *honor_pass_rules = NULL;
1651  if (conf != NULL) { /* To facilitate unit tests. */
1652  honor_pass_rules = SCConfNodeLookupChildValue(conf, "honor-pass-rules");
1653  }
1654  if (honor_pass_rules != NULL) {
1655  if (SCConfValIsFalse(honor_pass_rules)) {
1657  } else if (SCConfValIsTrue(honor_pass_rules)) {
1659  } else {
1660  FatalError("log-pcap honor-pass-rules specified is invalid");
1661  }
1662  }
1663 
1664  pl->bpf_filter = conf == NULL ? NULL : (char *)SCConfNodeLookupChildValue(conf, "bpf-filter");
1665 
1666  /* create the output ctx and send it back */
1667 
1668  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
1669  if (unlikely(output_ctx == NULL)) {
1670  FatalError("Failed to allocate memory for OutputCtx.");
1671  }
1672  output_ctx->data = pl;
1673  output_ctx->DeInit = PcapLogFileDeInitCtx;
1674  g_pcap_data = pl;
1675 
1676  result.ctx = output_ctx;
1677  result.ok = true;
1678  return result;
1679 }
1680 
1681 static void PcapLogFileDeInitCtx(OutputCtx *output_ctx)
1682 {
1683  if (output_ctx == NULL)
1684  return;
1685 
1686  PcapLogData *pl = output_ctx->data;
1687 
1688  PcapFileName *pf = NULL;
1689  TAILQ_FOREACH(pf, &pl->pcap_file_list, next) {
1690  SCLogDebug("PCAP files left at exit: %s\n", pf->filename);
1691  }
1692  PcapLogDataFree(pl);
1693  SCFree(output_ctx);
1694 
1695  pcre2_code_free(pcre_timestamp_code);
1696  pcre2_match_data_free(pcre_timestamp_match);
1697 }
1698 
1699 /**
1700  * \brief Read the config set the file pointer, open the file
1701  *
1702  * \param PcapLogData.
1703  *
1704  * \retval -1 if failure
1705  * \retval 0 if succesful
1706  */
1707 static int PcapLogOpenFileCtx(PcapLogData *pl)
1708 {
1709  char *path = NULL;
1710 
1712 
1713  if (pl->filename != NULL)
1714  path = pl->filename;
1715  else {
1716  path = SCMalloc(PATH_MAX);
1717  if (unlikely(path == NULL)) {
1718  return -1;
1719  }
1720  pl->filename = path;
1721  }
1722 
1723  /** get the time so we can have a filename with seconds since epoch */
1724  SCTime_t ts = TimeGet();
1725 
1726  /* Place to store the name of our PCAP file */
1727  PcapFileName *pf = SCCalloc(1, sizeof(PcapFileName));
1728  if (unlikely(pf == NULL)) {
1729  return -1;
1730  }
1731 
1732  char file[PATH_MAX] = "";
1733  if (pl->mode == LOGMODE_NORMAL) {
1734  int ret;
1735  /* create the filename to use */
1736  if (pl->timestamp_format == TS_FORMAT_SEC) {
1737  ret = snprintf(file, sizeof(file), "%s.%" PRIu32 "%s", pl->prefix,
1738  (uint32_t)SCTIME_SECS(ts), pl->suffix);
1739  } else {
1740  ret = snprintf(file, sizeof(file), "%s.%" PRIu32 ".%" PRIu32 "%s", pl->prefix,
1741  (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts), pl->suffix);
1742  }
1743  if (ret < 0 || (size_t)ret >= PATH_MAX) {
1744  SCLogError("failed to construct path");
1745  goto error;
1746  }
1747  } else if (pl->mode == LOGMODE_MULTI) {
1748  if (pl->filename_part_cnt > 0) {
1749  /* assemble filename from stored tokens */
1750 
1751  for (int i = 0; i < pl->filename_part_cnt; i++) {
1752  if (pl->filename_parts[i] == NULL ||strlen(pl->filename_parts[i]) == 0)
1753  continue;
1754 
1755  /* handle variables */
1756  if (pl->filename_parts[i][0] == '%') {
1757  char str[64] = "";
1758  if (strlen(pl->filename_parts[i]) < 2)
1759  continue;
1760 
1761  switch(pl->filename_parts[i][1]) {
1762  case 'n':
1763  snprintf(str, sizeof(str), "%u", pl->thread_number);
1764  break;
1765  case 'i':
1766  {
1767  long thread_id = SCGetThreadIdLong();
1768  snprintf(str, sizeof(str), "%"PRIu64, (uint64_t)thread_id);
1769  break;
1770  }
1771  case 't':
1772  /* create the filename to use */
1773  if (pl->timestamp_format == TS_FORMAT_SEC) {
1774  snprintf(str, sizeof(str), "%" PRIu32, (uint32_t)SCTIME_SECS(ts));
1775  } else {
1776  snprintf(str, sizeof(str), "%" PRIu32 ".%" PRIu32,
1777  (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts));
1778  }
1779  }
1780  strlcat(file, str, sizeof(file));
1781 
1782  /* copy the rest over */
1783  } else {
1784  strlcat(file, pl->filename_parts[i], sizeof(file));
1785  }
1786  }
1787  strlcat(file, pl->suffix, sizeof(file));
1788  } else {
1789  int ret;
1790  /* create the filename to use */
1791  if (pl->timestamp_format == TS_FORMAT_SEC) {
1792  ret = snprintf(file, sizeof(file), "%s.%u.%" PRIu32 "%s", pl->prefix,
1793  pl->thread_number, (uint32_t)SCTIME_SECS(ts), pl->suffix);
1794  } else {
1795  ret = snprintf(file, sizeof(file), "%s.%u.%" PRIu32 ".%" PRIu32 "%s", pl->prefix,
1796  pl->thread_number, (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts),
1797  pl->suffix);
1798  }
1799  if (ret < 0 || (size_t)ret >= PATH_MAX) {
1800  SCLogError("failed to construct path");
1801  goto error;
1802  }
1803  }
1804  SCLogDebug("multi-mode: filename %s", file);
1805  }
1806  if (PathMerge(path, PATH_MAX, pl->dir, file) < 0) {
1807  SCLogError("failed to construct path");
1808  goto error;
1809  }
1810 
1811  if ((pf->filename = SCStrdup(pl->filename)) == NULL) {
1812  SCLogError("Error allocating memory. For filename");
1813  goto error;
1814  }
1815  SCLogDebug("Opening pcap file log %s", pf->filename);
1816  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
1817 
1818  if (pl->mode == LOGMODE_MULTI || pl->mode == LOGMODE_NORMAL) {
1819  pcap_file_thread = pl->filename;
1820  }
1822  return 0;
1823 
1824 error:
1825  PcapFileNameFree(pf);
1826  return -1;
1827 }
1828 
1830 {
1831  /* return pcap filename per thread */
1832  if (pcap_file_thread != NULL) {
1833  return pcap_file_thread;
1834  }
1835  return NULL;
1836 }
1837 
1838 static int profiling_pcaplog_enabled = 0;
1839 static int profiling_pcaplog_output_to_file = 0;
1840 static char *profiling_pcaplog_file_name = NULL;
1841 static const char *profiling_pcaplog_file_mode = "a";
1842 
1843 static void FormatNumber(uint64_t num, char *str, size_t size)
1844 {
1845  if (num < 1000UL)
1846  snprintf(str, size, "%"PRIu64, num);
1847  else if (num < 1000000UL)
1848  snprintf(str, size, "%3.1fk", (float)num/1000UL);
1849  else if (num < 1000000000UL)
1850  snprintf(str, size, "%3.1fm", (float)num/1000000UL);
1851  else
1852  snprintf(str, size, "%3.1fb", (float)num/1000000000UL);
1853 }
1854 
1855 static void ProfileReportPair(FILE *fp, const char *name, PcapLogProfileData *p)
1856 {
1857  char ticks_str[32] = "n/a";
1858  char cnt_str[32] = "n/a";
1859  char avg_str[32] = "n/a";
1860 
1861  FormatNumber((uint64_t)p->cnt, cnt_str, sizeof(cnt_str));
1862  FormatNumber((uint64_t)p->total, ticks_str, sizeof(ticks_str));
1863  if (p->cnt && p->total)
1864  FormatNumber((uint64_t)(p->total/p->cnt), avg_str, sizeof(avg_str));
1865 
1866  fprintf(fp, "%-28s %-10s %-10s %-10s\n", name, cnt_str, avg_str, ticks_str);
1867 }
1868 
1869 static void ProfileReport(FILE *fp, PcapLogData *pl)
1870 {
1871  ProfileReportPair(fp, "open", &pl->profile_open);
1872  ProfileReportPair(fp, "close", &pl->profile_close);
1873  ProfileReportPair(fp, "write", &pl->profile_write);
1874  ProfileReportPair(fp, "rotate (incl open/close)", &pl->profile_rotate);
1875  ProfileReportPair(fp, "handles", &pl->profile_handles);
1876  ProfileReportPair(fp, "lock", &pl->profile_lock);
1877  ProfileReportPair(fp, "unlock", &pl->profile_unlock);
1878 }
1879 
1880 static void FormatBytes(uint64_t num, char *str, size_t size)
1881 {
1882  if (num < 1000UL)
1883  snprintf(str, size, "%"PRIu64, num);
1884  else if (num < 1048576UL)
1885  snprintf(str, size, "%3.1fKiB", (float)num/1000UL);
1886  else if (num < 1073741824UL)
1887  snprintf(str, size, "%3.1fMiB", (float)num/1000000UL);
1888  else
1889  snprintf(str, size, "%3.1fGiB", (float)num/1000000000UL);
1890 }
1891 
1892 static void PcapLogProfilingDump(PcapLogData *pl)
1893 {
1894  FILE *fp = NULL;
1895 
1896  if (profiling_pcaplog_enabled == 0)
1897  return;
1898 
1899  if (profiling_pcaplog_output_to_file == 1) {
1900  fp = fopen(profiling_pcaplog_file_name, profiling_pcaplog_file_mode);
1901  if (fp == NULL) {
1902  SCLogError("failed to open %s: %s", profiling_pcaplog_file_name, strerror(errno));
1903  return;
1904  }
1905  } else {
1906  fp = stdout;
1907  }
1908 
1909  /* counters */
1910  fprintf(fp, "\n\nOperation Cnt Avg ticks Total ticks\n");
1911  fprintf(fp, "---------------------------- ---------- ---------- -----------\n");
1912 
1913  ProfileReport(fp, pl);
1914  uint64_t total = pl->profile_write.total + pl->profile_rotate.total +
1917  pl->profile_unlock.total;
1918 
1919  /* overall stats */
1920  fprintf(fp, "\nOverall: %"PRIu64" bytes written, average %d bytes per write.\n",
1922  (int)(pl->profile_data_size / pl->profile_write.cnt) : 0);
1923  fprintf(fp, " PCAP data structure overhead: %"PRIuMAX" per write.\n",
1924  (uintmax_t)sizeof(struct pcap_pkthdr));
1925 
1926  /* print total bytes written */
1927  char bytes_str[32];
1928  FormatBytes(pl->profile_data_size, bytes_str, sizeof(bytes_str));
1929  fprintf(fp, " Size written: %s\n", bytes_str);
1930 
1931  /* ticks per MiB and GiB */
1932  uint64_t ticks_per_mib = 0, ticks_per_gib = 0;
1933  uint64_t mib = pl->profile_data_size/(1024*1024);
1934  if (mib)
1935  ticks_per_mib = total/mib;
1936  char ticks_per_mib_str[32] = "n/a";
1937  if (ticks_per_mib > 0)
1938  FormatNumber(ticks_per_mib, ticks_per_mib_str, sizeof(ticks_per_mib_str));
1939  fprintf(fp, " Ticks per MiB: %s\n", ticks_per_mib_str);
1940 
1941  uint64_t gib = pl->profile_data_size/(1024*1024*1024);
1942  if (gib)
1943  ticks_per_gib = total/gib;
1944  char ticks_per_gib_str[32] = "n/a";
1945  if (ticks_per_gib > 0)
1946  FormatNumber(ticks_per_gib, ticks_per_gib_str, sizeof(ticks_per_gib_str));
1947  fprintf(fp, " Ticks per GiB: %s\n", ticks_per_gib_str);
1948 
1949  if (fp != stdout)
1950  fclose(fp);
1951 }
1952 
1954 {
1955  SCConfNode *conf = SCConfGetNode("profiling.pcap-log");
1956  if (conf != NULL && SCConfNodeChildValueIsTrue(conf, "enabled")) {
1957  profiling_pcaplog_enabled = 1;
1958  SCLogInfo("pcap-log profiling enabled");
1959 
1960  const char *filename = SCConfNodeLookupChildValue(conf, "filename");
1961  if (filename != NULL) {
1962  const char *log_dir;
1963  log_dir = SCConfigGetLogDirectory();
1964 
1965  profiling_pcaplog_file_name = SCMalloc(PATH_MAX);
1966  if (unlikely(profiling_pcaplog_file_name == NULL)) {
1967  FatalError("can't duplicate file name");
1968  }
1969 
1970  snprintf(profiling_pcaplog_file_name, PATH_MAX, "%s/%s", log_dir, filename);
1971 
1972  const char *v = SCConfNodeLookupChildValue(conf, "append");
1973  if (v == NULL || SCConfValIsTrue(v)) {
1974  profiling_pcaplog_file_mode = "a";
1975  } else {
1976  profiling_pcaplog_file_mode = "w";
1977  }
1978 
1979  profiling_pcaplog_output_to_file = 1;
1980  SCLogInfo("pcap-log profiling output goes to %s (mode %s)",
1981  profiling_pcaplog_file_name, profiling_pcaplog_file_mode);
1982  }
1983  }
1984 }
util-byte.h
PcapLogCompressionData_::pcap_buf_size
uint64_t pcap_buf_size
Definition: log-pcap.c:135
PcapLogThreadData_::counter_written
uint16_t counter_written
Definition: log-pcap.c:197
PcapLogData
struct PcapLogData_ PcapLogData
PcapLogProfileData_
Definition: log-pcap.c:112
LOGMODE_COND_ALL
@ LOGMODE_COND_ALL
Definition: log-pcap.c:68
len
uint8_t len
Definition: app-layer-dnp3.h:2
ts
uint64_t ts
Definition: source-erf-file.c:55
SCConfValIsTrue
int SCConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:551
PcapLogData_::conditional
LogModeConditionalType conditional
Definition: log-pcap.c:165
PcapLogThreadData
struct PcapLogThreadData_ PcapLogThreadData
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:166
MemBuffer_::buffer
uint8_t buffer[]
Definition: util-buffer.h:30
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
util-fmemopen.h
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:262
IsRunModeOffline
bool IsRunModeOffline(enum SCRunModes run_mode_to_check)
Definition: runmodes.c:561
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
USE_STREAM_DEPTH_DISABLED
#define USE_STREAM_DEPTH_DISABLED
Definition: log-pcap.c:79
PcapFileName_::secs
uint64_t secs
Definition: log-pcap.c:103
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
PcapLogData_::pcap_dumper
pcap_dumper_t * pcap_dumper
Definition: log-pcap.c:158
TS_FORMAT_SEC
#define TS_FORMAT_SEC
Definition: log-pcap.c:76
PathMerge
int PathMerge(char *out_buf, size_t buf_size, const char *const dir, const char *const fname)
Definition: util-path.c:74
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:279
ParseSizeStringU64
int ParseSizeStringU64(const char *size, uint64_t *res)
Definition: util-misc.c:191
PcapFileName_::usecs
uint32_t usecs
Definition: log-pcap.c:104
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
PcapLogData_::filename
char * filename
Definition: log-pcap.c:152
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:287
HONOR_PASS_RULES_ENABLED
#define HONOR_PASS_RULES_ENABLED
Definition: log-pcap.c:83
Packet_::flags
uint32_t flags
Definition: decode.h:544
DEFAULT_LOG_FILENAME
#define DEFAULT_LOG_FILENAME
Definition: log-pcap.c:58
threads.h
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
RING_BUFFER_MODE_ENABLED
#define RING_BUFFER_MODE_ENABLED
Definition: log-pcap.c:74
PcapLogData_::mode
int mode
Definition: log-pcap.c:153
TAILQ_EMPTY
#define TAILQ_EMPTY(head)
Definition: queue.h:248
SCConfNodeChildValueIsTrue
int SCConfNodeChildValueIsTrue(const SCConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:868
PcapLogData_::pkt_cnt
uint64_t pkt_cnt
Definition: log-pcap.c:150
HONOR_PASS_RULES_DISABLED
#define HONOR_PASS_RULES_DISABLED
Definition: log-pcap.c:82
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:252
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
PcapLogData_::use_stream_depth
int use_stream_depth
Definition: log-pcap.c:146
PcapLogCompressionData_::format
enum PcapLogCompressionFormat format
Definition: log-pcap.c:126
SCConfValIsFalse
int SCConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:576
PcapLogProfileData
struct PcapLogProfileData_ PcapLogProfileData
stream-tcp-reassemble.h
PcapLogData_::profile_handles
PcapLogProfileData profile_handles
Definition: log-pcap.c:170
LOGMODE_MULTI
#define LOGMODE_MULTI
Definition: log-pcap.c:65
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
PcapLogRegister
void PcapLogRegister(void)
Definition: log-pcap.c:219
SCConfNodeLookupChildValue
const char * SCConfNodeLookupChildValue(const SCConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:824
SCFmemopen
#define SCFmemopen
Definition: util-fmemopen.h:52
PcapLogCompressionData_::buffer
uint8_t * buffer
Definition: log-pcap.c:127
DEFAULT_LIMIT
#define DEFAULT_LIMIT
Definition: log-pcap.c:61
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:82
PcapLogData_::pcap_dead_handle
pcap_t * pcap_dead_handle
Definition: log-pcap.c:157
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:620
PacketTunnelChild
@ PacketTunnelChild
Definition: decode.h:408
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
PcapLogData_::size_current
uint64_t size_current
Definition: log-pcap.c:155
PcapLogData_::bpf_filter
char * bpf_filter
Definition: log-pcap.c:148
PcapLogData_::profile_rotate
PcapLogProfileData profile_rotate
Definition: log-pcap.c:173
OutputCtx_::data
void * data
Definition: tm-modules.h:91
PcapLogCompressionData_
Definition: log-pcap.c:125
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:81
PCAPLOG_PROFILE_END
#define PCAPLOG_PROFILE_END(prof)
Definition: log-pcap.c:239
PcapLogGetFilename
char * PcapLogGetFilename(void)
Definition: log-pcap.c:1829
OutputCtx_
Definition: tm-modules.h:88
LogModeConditionalType_
LogModeConditionalType_
Definition: log-pcap.c:67
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
PcapLogCompressionData_::bytes_in_block
uint64_t bytes_in_block
Definition: log-pcap.c:137
TAILQ_ENTRY
#define TAILQ_ENTRY(type)
Definition: queue.h:239
Packet_::datalink
int datalink
Definition: decode.h:639
PcapLogData_::is_private
bool is_private
Definition: log-pcap.c:163
PcapLogData_::size_limit
uint64_t size_limit
Definition: log-pcap.c:156
PcapLogProfileData_::cnt
uint64_t cnt
Definition: log-pcap.c:114
PcapLogData_::max_files
uint32_t max_files
Definition: log-pcap.c:162
PcapLogCompressionData_::file
FILE * file
Definition: log-pcap.c:133
TAILQ_REMOVE
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:312
SCRunmodeGet
SCRunMode SCRunmodeGet(void)
Get the current run mode.
Definition: suricata.c:279
decode.h
TAILQ_FIRST
#define TAILQ_FIRST(head)
Definition: queue.h:250
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:47
PCAP_LOG_COMPRESSION_FORMAT_NONE
@ PCAP_LOG_COMPRESSION_FORMAT_NONE
Definition: log-pcap.c:121
strlcat
size_t strlcat(char *, const char *src, size_t siz)
Definition: util-strlcatu.c:45
util-cpu.h
PCAP_NETMASK_UNKNOWN
#define PCAP_NETMASK_UNKNOWN
Definition: log-pcap.c:91
Packet_::ts
SCTime_t ts
Definition: decode.h:555
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
TcpSegmentPcapHdrStorage_::ts
SCTime_t ts
Definition: stream-tcp-private.h:66
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1270
PCAP_LOG_COMPRESSION_FORMAT_LZ4
@ PCAP_LOG_COMPRESSION_FORMAT_LZ4
Definition: log-pcap.c:122
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
STREAM_DUMP_HEADERS
#define STREAM_DUMP_HEADERS
Definition: stream.h:34
GET_PKT_DATA
#define GET_PKT_DATA(p)
Definition: decode.h:209
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
PcapLogProfileData_::total
uint64_t total
Definition: log-pcap.c:113
PCAP_PKTHDR_SIZE
#define PCAP_PKTHDR_SIZE
Definition: log-pcap.c:87
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:316
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:48
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:259
PcapFileName_::dirname
char * dirname
Definition: log-pcap.c:98
PcapLogData_::file_cnt
uint32_t file_cnt
Definition: log-pcap.c:161
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
SCTIME_TO_TIMEVAL
#define SCTIME_TO_TIMEVAL(tv, t)
Definition: util-time.h:97
PCAP_OUTPUT_BUFFER_SIZE
#define PCAP_OUTPUT_BUFFER_SIZE
Definition: log-pcap.h:31
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
Packet_
Definition: decode.h:501
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:267
GET_PKT_LEN
#define GET_PKT_LEN(p)
Definition: decode.h:208
TimeGet
SCTime_t TimeGet(void)
Definition: util-time.c:152
SCConfGetChildValueInt
int SCConfGetChildValueInt(const SCConfNode *base, const char *name, intmax_t *val)
Definition: conf.c:449
SCConfigGetLogDirectory
const char * SCConfigGetLogDirectory(void)
Definition: util-conf.c:38
DEFAULT_FILE_LIMIT
#define DEFAULT_FILE_LIMIT
Definition: log-pcap.c:62
SCTime_t
Definition: util-time.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:80
RING_BUFFER_MODE_DISABLED
#define RING_BUFFER_MODE_DISABLED
Definition: log-pcap.c:73
PcapLogThreadData_::buf
MemBuffer * buf
Definition: log-pcap.c:196
name
const char * name
Definition: tm-threads.c:2163
PCAP_SNAPLEN
#define PCAP_SNAPLEN
Definition: log-pcap.c:85
PcapLogData_::plog_lock
SCMutex plog_lock
Definition: log-pcap.c:149
PcapLogData_::h
struct pcap_pkthdr * h
Definition: log-pcap.c:151
PcapLogCompressionData_::pcap_buf
uint8_t * pcap_buf
Definition: log-pcap.c:134
MemBuffer_
Definition: util-buffer.h:27
LOGMODE_COND_TAG
@ LOGMODE_COND_TAG
Definition: log-pcap.c:70
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:229
StreamSegmentForSession
int StreamSegmentForSession(const Packet *p, uint8_t flag, StreamSegmentCallback CallbackFunc, void *data)
Run callback for all segments on both directions of the session.
Definition: stream.c:64
PcapLogData_::prev_day
int prev_day
Definition: log-pcap.c:154
PKT_FIRST_TAG
#define PKT_FIRST_TAG
Definition: decode.h:1318
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
PCAPLOG_PROFILE_START
#define PCAPLOG_PROFILE_START
Definition: log-pcap.c:236
log-pcap.h
SCGetThreadIdLong
#define SCGetThreadIdLong(...)
Definition: threads.h:255
PcapLogData_::honor_pass_rules
int honor_pass_rules
Definition: log-pcap.c:147
pcap_file_thread
thread_local char * pcap_file_thread
Definition: log-pcap.c:110
PcapLogData_::profile_open
PcapLogProfileData profile_open
Definition: log-pcap.c:172
PcapLogData_::profile_lock
PcapLogProfileData profile_lock
Definition: log-pcap.c:167
OutputInitResult_
Definition: output.h:46
util-conf.h
Packet_::flow
struct Flow_ * flow
Definition: decode.h:546
StringParseUint64
int StringParseUint64(uint64_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:311
TimeDifferenceMicros
uint64_t TimeDifferenceMicros(struct timeval t0, struct timeval t1)
Definition: util-time.c:644
LOGGER_PCAP
@ LOGGER_PCAP
Definition: suricata-common.h:507
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:94
util-path.h
PcapLogCallbackContext::tv
ThreadVars * tv
Definition: log-pcap.c:565
LOGMODE_NORMAL
#define LOGMODE_NORMAL
Definition: log-pcap.c:64
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:86
PKT_STREAM_NOPCAPLOG
#define PKT_STREAM_NOPCAPLOG
Definition: decode.h:1279
PcapLogData_::profile_data_size
uint64_t profile_data_size
Definition: log-pcap.c:160
OutputPacketLoggerFunctions_::LogFunc
PacketLogger LogFunc
Definition: output.h:88
SCTIME_SECS
#define SCTIME_SECS(t)
Definition: util-time.h:57
OutputRegisterPacketModule
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, OutputPacketLoggerFunctions *output_module_functions)
Register a packet output module.
Definition: output.c:196
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
Packet_::ttype
enum PacketTunnelType ttype
Definition: decode.h:553
PcapLogProfileSetup
void PcapLogProfileSetup(void)
Definition: log-pcap.c:1953
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(...)
Definition: util-debug.h:514
PcapLogCallbackContext
Definition: log-pcap.c:564
PcapLogCompressionData_::buffer_size
uint64_t buffer_size
Definition: log-pcap.c:128
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
PcapLogThreadData_
Definition: log-pcap.c:194
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
PcapLogData_::bpfp
struct bpf_program * bpfp
Definition: log-pcap.c:159
LOGMODE_COND_ALERTS
@ LOGMODE_COND_ALERTS
Definition: log-pcap.c:69
PcapFileName_::filename
char * filename
Definition: log-pcap.c:97
Packet_::root
struct Packet_ * root
Definition: decode.h:653
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
str
#define str(s)
Definition: suricata-common.h:308
MAX_TOKS
#define MAX_TOKS
Definition: log-pcap.c:117
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:181
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:271
PcapLogCompressionData
struct PcapLogCompressionData_ PcapLogCompressionData
MODULE_NAME
#define MODULE_NAME
Definition: log-pcap.c:59
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint32_t, thread_cnt)
PcapLogCompressionData_::pcap_buf_wrapper
FILE * pcap_buf_wrapper
Definition: log-pcap.c:136
src
uint16_t src
Definition: app-layer-dnp3.h:5
util-buffer.h
PcapFileName_
Definition: log-pcap.c:96
TAILQ_HEAD
#define TAILQ_HEAD(name, type)
Definition: queue.h:230
PcapLogData_::profile_unlock
PcapLogProfileData profile_unlock
Definition: log-pcap.c:169
USE_STREAM_DEPTH_ENABLED
#define USE_STREAM_DEPTH_ENABLED
Definition: log-pcap.c:80
MemBufferWriteRaw
uint32_t MemBufferWriteRaw(MemBuffer *dst, const uint8_t *raw, const uint32_t raw_len)
Write a raw buffer to the MemBuffer dst.
Definition: util-buffer.c:115
PcapLogData_::profile_close
PcapLogProfileData profile_close
Definition: log-pcap.c:171
PcapLogThreadData_::counter_filtered_bpf
uint16_t counter_filtered_bpf
Definition: log-pcap.c:198
PcapLogCallbackContext::td
PcapLogThreadData * td
Definition: log-pcap.c:566
PKT_NOPACKET_INSPECTION
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:1249
FlowHasAlerts
int FlowHasAlerts(const Flow *f)
Check if flow has alerts.
Definition: flow.c:165
PcapFileName
struct PcapFileName_ PcapFileName
PCAP_BUFFER_TIMEOUT
#define PCAP_BUFFER_TIMEOUT
Definition: log-pcap.c:86
MIN_LIMIT
#define MIN_LIMIT
Definition: log-pcap.c:60
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
util-misc.h
PKT_HAS_TAG
#define PKT_HAS_TAG
Definition: decode.h:1260
PcapLogData_
Definition: log-pcap.c:145
PcapLogThreadData_::pcap_log
PcapLogData * pcap_log
Definition: log-pcap.c:195
PKT_FIRST_ALERTS
#define PKT_FIRST_ALERTS
Definition: decode.h:1317
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:247
TAILQ_INSERT_BEFORE
#define TAILQ_INSERT_BEFORE(listelm, elm, field)
Definition: queue.h:277
StatsRegisterCounter
uint16_t StatsRegisterCounter(const char *name, struct ThreadVars_ *tv)
Registers a normal, unqualified counter.
Definition: counters.c:968
PcapLogData_::profile_write
PcapLogProfileData profile_write
Definition: log-pcap.c:168
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCConfNode_
Definition: conf.h:37
TS_FORMAT_USEC
#define TS_FORMAT_USEC
Definition: log-pcap.c:77
OutputPacketLoggerFunctions_
Definition: output.h:87
SCMutex
#define SCMutex
Definition: threads-debug.h:114
MAX_FILENAMELEN
#define MAX_FILENAMELEN
Definition: log-pcap.c:118
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
output.h
LogModeConditionalType
enum LogModeConditionalType_ LogModeConditionalType
PcapLogCompressionFormat
PcapLogCompressionFormat
Definition: log-pcap.c:120
SCTIME_USECS
#define SCTIME_USECS(t)
Definition: util-time.h:56