suricata
log-pcap.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author William Metcalf <William.Metcalf@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Pcap packet logging module.
25  */
26 
27 #include "suricata-common.h"
28 #ifdef HAVE_LIBLZ4
29 #include <lz4frame.h>
30 #include "util-fmemopen.h"
31 #endif /* HAVE_LIBLZ4 */
32 
33 #if defined(HAVE_DIRENT_H) && defined(HAVE_FNMATCH_H)
34 #define INIT_RING_BUFFER
35 #include <dirent.h>
36 #include <fnmatch.h>
37 #endif
38 
39 #include "log-pcap.h"
40 
41 #include "threads.h"
42 #include "threadvars.h"
43 #include "decode.h"
44 #include "stream.h"
45 #include "stream-tcp-reassemble.h"
46 
47 #include "output.h"
48 
49 #include "util-buffer.h"
50 #include "util-byte.h"
51 #include "util-conf.h"
52 #include "util-cpu.h"
53 #include "util-datalink.h"
54 #include "util-misc.h"
55 #include "util-path.h"
56 
57 #define DEFAULT_LOG_FILENAME "pcaplog"
58 #define MODULE_NAME "PcapLog"
59 #define MIN_LIMIT 4 * 1024 * 1024
60 #define DEFAULT_LIMIT 100 * 1024 * 1024
61 #define DEFAULT_FILE_LIMIT 0
62 
63 #define LOGMODE_NORMAL 0
64 #define LOGMODE_MULTI 1
65 
71 
72 #define RING_BUFFER_MODE_DISABLED 0
73 #define RING_BUFFER_MODE_ENABLED 1
74 
75 #define TS_FORMAT_SEC 0
76 #define TS_FORMAT_USEC 1
77 
78 #define USE_STREAM_DEPTH_DISABLED 0
79 #define USE_STREAM_DEPTH_ENABLED 1
80 
81 #define HONOR_PASS_RULES_DISABLED 0
82 #define HONOR_PASS_RULES_ENABLED 1
83 
84 #define PCAP_SNAPLEN 262144
85 #define PCAP_BUFFER_TIMEOUT 1000000 // microseconds
86 #define PCAP_PKTHDR_SIZE 16
87 
88 /* Defined since libpcap 1.1.0. */
89 #ifndef PCAP_NETMASK_UNKNOWN
90 #define PCAP_NETMASK_UNKNOWN 0xffffffff
91 #endif
92 
93 SC_ATOMIC_DECLARE(uint32_t, thread_cnt);
94 
95 typedef struct PcapFileName_ {
96  char *filename;
97  char *dirname;
98 
99  /* Like a struct timeval, but with fixed size. This is only used when
100  * seeding the ring buffer on start. */
101  struct {
102  uint64_t secs;
103  uint32_t usecs;
104  };
105 
106  TAILQ_ENTRY(PcapFileName_) next; /**< Pointer to next Pcap File for tailq. */
108 
109 thread_local char *pcap_file_thread = NULL;
110 
111 typedef struct PcapLogProfileData_ {
112  uint64_t total;
113  uint64_t cnt;
115 
116 #define MAX_TOKS 9
117 #define MAX_FILENAMELEN 513
118 
122 };
123 
124 typedef struct PcapLogCompressionData_ {
126  uint8_t *buffer;
127  uint64_t buffer_size;
128 #ifdef HAVE_LIBLZ4
129  LZ4F_compressionContext_t lz4f_context;
130  LZ4F_preferences_t lz4f_prefs;
131  FILE *pcap_buf_wrapper;
132 #endif /* HAVE_LIBLZ4 */
133  FILE *file;
134  uint8_t *pcap_buf;
135  uint64_t pcap_buf_size;
136  uint64_t bytes_in_block;
138 
139 /**
140  * PcapLog thread vars
141  *
142  * Used for storing file options.
143  */
144 typedef struct PcapLogData_ {
145  int use_stream_depth; /**< use stream depth i.e. ignore packets that reach limit */
146  int honor_pass_rules; /**< don't log if pass rules have matched */
147  char *bpf_filter; /**< bpf filter to apply to output */
149  uint64_t pkt_cnt; /**< total number of packets */
150  struct pcap_pkthdr *h; /**< pcap header struct */
151  char *filename; /**< current filename */
152  int mode; /**< normal or multi */
153  int prev_day; /**< last day, for finding out when */
154  uint64_t size_current; /**< file current size */
155  uint64_t size_limit; /**< file size limit */
156  pcap_t *pcap_dead_handle; /**< pcap_dumper_t needs a handle */
157  pcap_dumper_t *pcap_dumper; /**< actually writes the packets */
158  struct bpf_program *bpfp; /**< compiled bpf program */
159  uint64_t profile_data_size; /**< track in bytes how many bytes we wrote */
160  uint32_t file_cnt; /**< count of pcap files we currently have */
161  uint32_t max_files; /**< maximum files to use in ring buffer mode */
162  bool is_private; /**< true if ctx is thread local */
164  conditional; /**< log all packets or just packets and flows with alerts */
165 
173 
174  TAILQ_HEAD(, PcapFileName_) pcap_file_list;
175 
176  uint32_t thread_number; /**< thread number, first thread is 1, second 2, etc */
177  int use_ringbuffer; /**< ring buffer mode enabled or disabled */
178  int timestamp_format; /**< timestamp format sec or usec */
179  char *prefix; /**< filename prefix */
180  const char *suffix; /**< filename suffix */
181  char dir[PATH_MAX]; /**< pcap log directory */
182  int reported;
183  int threads; /**< number of threads (only set in the global) */
184  char *filename_parts[MAX_TOKS];
185  int filename_part_cnt;
186  struct timeval last_pcap_dump;
187  int fopen_err; /**< set to the last fopen error */
188  bool pcap_open_err; /**< true if the last pcap open errored */
189 
190  PcapLogCompressionData compression;
192 
193 typedef struct PcapLogThreadData_ {
196  StatsCounterId counter_written; /**< Counter for number of packets written */
198  counter_filtered_bpf; /**< Counter for number of packets filtered out and not writen */
200 
201 /* Pattern for extracting timestamp from pcap log files. */
202 static const char timestamp_pattern[] = ".*?(\\d+)(\\.(\\d+))?";
203 static pcre2_code *pcre_timestamp_code = NULL;
204 static pcre2_match_data *pcre_timestamp_match = NULL;
205 
206 /* global pcap data for when we're using multi mode. At exit we'll
207  * merge counters into this one and then report counters. */
208 static PcapLogData *g_pcap_data = NULL;
209 
210 static int PcapLogOpenFileCtx(PcapLogData *);
211 static int PcapLog(ThreadVars *, void *, const Packet *);
212 static TmEcode PcapLogDataInit(ThreadVars *, const void *, void **);
213 static TmEcode PcapLogDataDeinit(ThreadVars *, void *);
214 static void PcapLogFileDeInitCtx(OutputCtx *);
215 static OutputInitResult PcapLogInitCtx(SCConfNode *);
216 static void PcapLogProfilingDump(PcapLogData *);
217 static bool PcapLogCondition(ThreadVars *, void *, const Packet *);
218 
219 void PcapLogRegister(void)
220 {
221  OutputPacketLoggerFunctions output_logger_functions = {
222  .LogFunc = PcapLog,
223  .ConditionFunc = PcapLogCondition,
224  .ThreadInitFunc = PcapLogDataInit,
225  .ThreadDeinitFunc = PcapLogDataDeinit,
226  .ThreadExitPrintStatsFunc = NULL,
227  };
229  LOGGER_PCAP, MODULE_NAME, "pcap-log", PcapLogInitCtx, &output_logger_functions);
231  SC_ATOMIC_INIT(thread_cnt);
232  SC_ATOMIC_SET(thread_cnt, 1); /* first id is 1 */
233 }
234 
235 #define PCAPLOG_PROFILE_START \
236  uint64_t pcaplog_profile_ticks = UtilCpuGetTicks()
237 
238 #define PCAPLOG_PROFILE_END(prof) \
239  (prof).total += (UtilCpuGetTicks() - pcaplog_profile_ticks); \
240  (prof).cnt++
241 
242 static bool PcapLogCondition(ThreadVars *tv, void *thread_data, const Packet *p)
243 {
244  PcapLogThreadData *ptd = (PcapLogThreadData *)thread_data;
245 
246  /* Log alerted flow or tagged flow */
247  switch (ptd->pcap_log->conditional) {
248  case LOGMODE_COND_ALL:
249  break;
250  case LOGMODE_COND_ALERTS:
251  return (p->alerts.cnt || (p->flow && FlowHasAlerts(p->flow)));
252  case LOGMODE_COND_TAG:
253  return (p->flags & (PKT_HAS_TAG | PKT_FIRST_TAG));
254  }
255 
256  if (p->flags & PKT_PSEUDO_STREAM_END) {
257  return false;
258  }
259 
260  return !PacketIsTunnelChild(p);
261 }
262 
263 /**
264  * \brief Function to close pcaplog file
265  *
266  * \param t Thread Variable containing input/output queue, cpu affinity etc.
267  * \param pl PcapLog thread variable.
268  */
269 static int PcapLogCloseFile(ThreadVars *t, PcapLogData *pl)
270 {
271  if (pl != NULL) {
273 
274  if (pl->pcap_dumper != NULL) {
275  pcap_dump_close(pl->pcap_dumper);
276 #ifdef HAVE_LIBLZ4
277  PcapLogCompressionData *comp = &pl->compression;
279  comp->pcap_buf_wrapper = NULL;
280  }
281 #endif /* HAVE_LIBLZ4 */
282  }
283  pl->size_current = 0;
284  pl->pcap_dumper = NULL;
285 
286  if (pl->pcap_dead_handle != NULL)
287  pcap_close(pl->pcap_dead_handle);
288  pl->pcap_dead_handle = NULL;
289 
290 #ifdef HAVE_LIBLZ4
291  PcapLogCompressionData *comp = &pl->compression;
293  /* pcap_dump_close did not write any data because we call
294  * pcap_dump_flush() after every write when writing
295  * compressed output. */
296  uint64_t bytes_written = LZ4F_compressEnd(comp->lz4f_context,
297  comp->buffer, comp->buffer_size, NULL);
298  if (LZ4F_isError(bytes_written)) {
299  SCLogError("LZ4F_compressEnd: %s", LZ4F_getErrorName(bytes_written));
300  return TM_ECODE_FAILED;
301  }
302  if (fwrite(comp->buffer, 1, bytes_written, comp->file) < bytes_written) {
303  SCLogError("fwrite failed: %s", strerror(errno));
304  return TM_ECODE_FAILED;
305  }
306  fclose(comp->file);
307  comp->bytes_in_block = 0;
308  }
309 #endif /* HAVE_LIBLZ4 */
310 
312  }
313 
314  return 0;
315 }
316 
317 static void PcapFileNameFree(PcapFileName *pf)
318 {
319  if (pf != NULL) {
320  if (pf->filename != NULL) {
321  SCFree(pf->filename);
322  }
323  if (pf->dirname != NULL) {
324  SCFree(pf->dirname);
325  }
326  SCFree(pf);
327  }
328 }
329 
330 /**
331  * \brief Function to rotate pcaplog file
332  *
333  * \param t Thread Variable containing input/output queue, cpu affinity etc.
334  * \param pl PcapLog thread variable.
335  *
336  * \retval 0 on success
337  * \retval -1 on failure
338  */
339 static int PcapLogRotateFile(ThreadVars *t, PcapLogData *pl)
340 {
341  PcapFileName *pf;
342 
344 
345  if (PcapLogCloseFile(t,pl) < 0) {
346  SCLogDebug("PcapLogCloseFile failed");
347  return -1;
348  }
349 
350  if (pl->use_ringbuffer == RING_BUFFER_MODE_ENABLED && pl->file_cnt >= pl->max_files) {
351  pf = TAILQ_FIRST(&pl->pcap_file_list);
352  SCLogDebug("Removing pcap file %s", pf->filename);
353 
354  if (remove(pf->filename) != 0) {
355  // VJ remove can fail because file is already gone
356  // SCLogWarning("failed to remove log file %s: %s",
357  // pf->filename, strerror( errno ));
358  }
359 
360  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
361  DEBUG_VALIDATE_BUG_ON(TAILQ_FIRST(&pl->pcap_file_list) == pf);
362  PcapFileNameFree(pf);
363  pl->file_cnt--;
364  }
365 
366  if (PcapLogOpenFileCtx(pl) < 0) {
367  SCLogError("opening new pcap log file failed");
368  return -1;
369  }
370  pl->file_cnt++;
371  SCLogDebug("file_cnt %u", pl->file_cnt);
372 
374  return 0;
375 }
376 
377 static int PcapLogOpenHandles(PcapLogData *pl, const Packet *p)
378 {
380 
381  int datalink = p->datalink;
382  if (PacketIsTunnelChild(p)) {
383  Packet *real_p = p->root;
384  datalink = real_p->datalink;
385  }
386  if (pl->pcap_dead_handle == NULL) {
387  SCLogDebug("Setting pcap-log link type to %u", datalink);
388  if ((pl->pcap_dead_handle = pcap_open_dead(datalink, PCAP_SNAPLEN)) == NULL) {
389  SCLogDebug("Error opening dead pcap handle");
390  return TM_ECODE_FAILED;
391  }
392 
393  if (pl->bpfp == NULL && pl->bpf_filter) {
394  struct bpf_program bpfp;
395  if (pcap_compile(pl->pcap_dead_handle, &bpfp, pl->bpf_filter, 0,
396  PCAP_NETMASK_UNKNOWN) == PCAP_ERROR) {
397  FatalError("Failed to compile BPF filter, aborting: %s: %s", pl->bpf_filter,
398  pcap_geterr(pl->pcap_dead_handle));
399  } else {
400  pl->bpfp = SCCalloc(1, sizeof(*pl->bpfp));
401  if (pl->bpfp == NULL) {
402  FatalError("Failed to allocate memory for BPF filter, aborting");
403  }
404  *pl->bpfp = bpfp;
405  }
406  }
407  }
408 
409  if (pl->pcap_dumper == NULL) {
410  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_NONE) {
411  if ((pl->pcap_dumper = pcap_dump_open(pl->pcap_dead_handle,
412  pl->filename)) == NULL) {
413  if (!pl->pcap_open_err) {
414  SCLogError("Error opening dump file %s", pcap_geterr(pl->pcap_dead_handle));
415  pl->pcap_open_err = true;
416  }
417  return TM_ECODE_FAILED;
418  } else {
419  pl->pcap_open_err = false;
420  }
421  }
422 #ifdef HAVE_LIBLZ4
423  else if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
424  PcapLogCompressionData *comp = &pl->compression;
425 
426  comp->file = fopen(pl->filename, "w");
427  if (comp->file == NULL) {
428  if (errno != pl->fopen_err) {
429  SCLogError("Error opening file for compressed output: %s", strerror(errno));
430  pl->fopen_err = errno;
431  }
432  return TM_ECODE_FAILED;
433  } else {
434  pl->fopen_err = 0;
435  }
436 
437  comp->pcap_buf_wrapper = SCFmemopen(comp->pcap_buf, comp->pcap_buf_size, "w");
438  if (comp->pcap_buf_wrapper == NULL) {
439  fclose(comp->file);
440  comp->file = NULL;
441  return TM_ECODE_FAILED;
442  }
443  if ((pl->pcap_dumper = pcap_dump_fopen(pl->pcap_dead_handle, comp->pcap_buf_wrapper)) ==
444  NULL) {
445  if (!pl->pcap_open_err) {
446  SCLogError("Error opening dump file %s", pcap_geterr(pl->pcap_dead_handle));
447  pl->pcap_open_err = true;
448  }
449  fclose(comp->file);
450  comp->file = NULL;
451  fclose(comp->pcap_buf_wrapper);
452  comp->pcap_buf_wrapper = NULL;
453  return TM_ECODE_FAILED;
454  } else {
455  pl->pcap_open_err = false;
456  }
457 
458  uint64_t bytes_written = LZ4F_compressBegin(comp->lz4f_context,
459  comp->buffer, comp->buffer_size, NULL);
460  if (LZ4F_isError(bytes_written)) {
461  SCLogError("LZ4F_compressBegin: %s", LZ4F_getErrorName(bytes_written));
462  return TM_ECODE_FAILED;
463  }
464  if (fwrite(comp->buffer, 1, bytes_written, comp->file) < bytes_written) {
465  SCLogError("fwrite failed: %s", strerror(errno));
466  return TM_ECODE_FAILED;
467  }
468  }
469 #endif /* HAVE_LIBLZ4 */
470  }
471 
473  return TM_ECODE_OK;
474 }
475 
476 /** \internal
477  * \brief lock wrapper for main PcapLog() function
478  * NOTE: only meant for use in main PcapLog() function.
479  */
480 static void PcapLogLock(PcapLogData *pl)
481 {
482  if (!(pl->is_private)) {
484  SCMutexLock(&pl->plog_lock);
486  }
487 }
488 
489 /** \internal
490  * \brief unlock wrapper for main PcapLog() function
491  * NOTE: only meant for use in main PcapLog() function.
492  */
493 static void PcapLogUnlock(PcapLogData *pl)
494 {
495  if (!(pl->is_private)) {
497  SCMutexUnlock(&pl->plog_lock);
499  }
500 }
501 
502 static inline int PcapWrite(
503  ThreadVars *tv, PcapLogThreadData *td, const uint8_t *data, const size_t len)
504 {
505  struct timeval current_dump;
506  gettimeofday(&current_dump, NULL);
507  PcapLogData *pl = td->pcap_log;
508 
509  if (pl->bpfp) {
510  if (pcap_offline_filter(pl->bpfp, pl->h, data) == 0) {
511  SCLogDebug("Packet doesn't match filter, will not be logged.");
513  return TM_ECODE_OK;
514  }
515  }
516 
518 
519  pcap_dump((u_char *)pl->pcap_dumper, pl->h, data);
520  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_NONE) {
521  pl->size_current += len;
522  }
523 #ifdef HAVE_LIBLZ4
524  else if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
525  PcapLogCompressionData *comp = &pl->compression;
526  pcap_dump_flush(pl->pcap_dumper);
527  long in_size = ftell(comp->pcap_buf_wrapper);
528  if (in_size < 0) {
529  SCLogError("ftell failed with: %s", strerror(errno));
530  return TM_ECODE_FAILED;
531  }
532  uint64_t out_size = LZ4F_compressUpdate(comp->lz4f_context, comp->buffer, comp->buffer_size,
533  comp->pcap_buf, (uint64_t)in_size, NULL);
534  if (LZ4F_isError(len)) {
535  SCLogError("LZ4F_compressUpdate: %s", LZ4F_getErrorName(len));
536  return TM_ECODE_FAILED;
537  }
538  if (fseek(comp->pcap_buf_wrapper, 0, SEEK_SET) != 0) {
539  SCLogError("fseek failed: %s", strerror(errno));
540  return TM_ECODE_FAILED;
541  }
542  if (fwrite(comp->buffer, 1, out_size, comp->file) < out_size) {
543  SCLogError("fwrite failed: %s", strerror(errno));
544  return TM_ECODE_FAILED;
545  }
546  if (out_size > 0) {
547  pl->size_current += out_size;
548  comp->bytes_in_block = len;
549  } else {
550  comp->bytes_in_block += len;
551  }
552  }
553 #endif /* HAVE_LIBLZ4 */
554  if (TimeDifferenceMicros(pl->last_pcap_dump, current_dump) >= PCAP_BUFFER_TIMEOUT) {
555  pcap_dump_flush(pl->pcap_dumper);
556  }
557  pl->last_pcap_dump = current_dump;
558  return TM_ECODE_OK;
559 }
560 
564 };
565 
566 static int PcapLogSegmentCallback(
567  const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
568 {
569  struct PcapLogCallbackContext *pctx = (struct PcapLogCallbackContext *)data;
570 
571  if (seg->pcap_hdr_storage->pktlen) {
572  struct timeval tv;
574  pctx->td->pcap_log->h->ts.tv_sec = tv.tv_sec;
575  pctx->td->pcap_log->h->ts.tv_usec = tv.tv_usec;
576 
577  /* Ensure the buffer can hold the full packet: headers + payload.
578  */
579  const uint32_t pktlen = seg->pcap_hdr_storage->pktlen;
580  const uint32_t total_len = pktlen + buflen;
581 
582  if (unlikely(total_len >= MEMBUFFER_SIZE(pctx->td->buf))) {
583  uint32_t expand_by = total_len + 1 - MEMBUFFER_SIZE(pctx->td->buf);
584  if (expand_by % 4096 != 0) {
585  expand_by = expand_by - (expand_by % 4096) + 4096;
586  }
587  if (unlikely(MemBufferExpand(&pctx->td->buf, expand_by) < 0)) {
588  SCLogWarning("Failed to expand pcap-log buffer for segment "
589  "of size %u",
590  total_len);
591  return 1;
592  }
593  }
594 
595  pctx->td->pcap_log->h->len = total_len;
596  pctx->td->pcap_log->h->caplen = total_len;
597  MemBufferReset(pctx->td->buf);
598  MemBufferWriteRaw(pctx->td->buf, seg->pcap_hdr_storage->pkt_hdr, pktlen);
599  MemBufferWriteRaw(pctx->td->buf, buf, buflen);
600 
601  PcapWrite(pctx->tv, pctx->td, (uint8_t *)pctx->td->buf->buffer, total_len);
602  }
603  return 1;
604 }
605 
606 static void PcapLogDumpSegments(ThreadVars *tv, PcapLogThreadData *td, const Packet *p)
607 {
608  uint8_t flag = STREAM_DUMP_HEADERS;
609 
610  /* Loop on segment from this side */
611  struct PcapLogCallbackContext data = { tv, td };
612  StreamSegmentForSession(p, flag, PcapLogSegmentCallback, (void *)&data);
613 }
614 
615 /**
616  * \brief Pcap logging main function
617  *
618  * \param t threadvar
619  * \param p packet
620  * \param thread_data thread module specific data
621  *
622  * \retval TM_ECODE_OK on succes
623  * \retval TM_ECODE_FAILED on serious error
624  */
625 static int PcapLog(ThreadVars *tv, void *thread_data, const Packet *p)
626 {
627  size_t len;
628  int ret = 0;
629  Packet *rp = NULL;
630 
631  PcapLogThreadData *td = (PcapLogThreadData *)thread_data;
632  PcapLogData *pl = td->pcap_log;
633 
636  return TM_ECODE_OK;
637  }
638 
639  PcapLogLock(pl);
640 
641  pl->pkt_cnt++;
642  pl->h->ts.tv_sec = SCTIME_SECS(p->ts);
643  pl->h->ts.tv_usec = SCTIME_USECS(p->ts);
644  if (PacketIsTunnelChild(p)) {
645  rp = p->root;
646  pl->h->caplen = GET_PKT_LEN(rp);
647  pl->h->len = GET_PKT_LEN(rp);
649  } else {
650  pl->h->caplen = GET_PKT_LEN(p);
651  pl->h->len = GET_PKT_LEN(p);
653  }
654 
655  if (pl->filename == NULL) {
656  ret = PcapLogOpenFileCtx(pl);
657  if (ret < 0) {
658  PcapLogUnlock(pl);
659  return TM_ECODE_FAILED;
660  }
661  SCLogDebug("Opening PCAP log file %s", pl->filename);
662  }
663 
664  PcapLogCompressionData *comp = &pl->compression;
666  if ((pl->size_current + len) > pl->size_limit) {
667  if (PcapLogRotateFile(tv, pl) < 0) {
668  PcapLogUnlock(pl);
669  SCLogDebug("rotation of pcap failed");
670  return TM_ECODE_FAILED;
671  }
672  }
673  }
674 #ifdef HAVE_LIBLZ4
675  else if (comp->format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
676  /* When writing compressed pcap logs, we have no way of knowing
677  * for sure whether adding this packet would cause the current
678  * file to exceed the size limit. Thus, we record the number of
679  * bytes that have been fed into lz4 since the last write, and
680  * act as if they would be written uncompressed. */
681 
682  if ((pl->size_current + comp->bytes_in_block + len) > pl->size_limit) {
683  if (PcapLogRotateFile(tv, pl) < 0) {
684  PcapLogUnlock(pl);
685  SCLogDebug("rotation of pcap failed");
686  return TM_ECODE_FAILED;
687  }
688  }
689  }
690 #endif /* HAVE_LIBLZ4 */
691 
692  /* XXX pcap handles, nfq, pfring, can only have one link type ipfw? we do
693  * this here as we don't know the link type until we get our first packet */
694  if (pl->pcap_dead_handle == NULL || pl->pcap_dumper == NULL) {
695  if (PcapLogOpenHandles(pl, p) != TM_ECODE_OK) {
696  PcapLogUnlock(pl);
697  return TM_ECODE_FAILED;
698  }
699  }
700 
702 
703  /* if we are using alerted logging and if packet is first one with alert in flow
704  * then we need to dump in the pcap the stream acked by the packet */
706  if (PacketIsTCP(p)) {
707  /* dump fake packets for all segments we have on acked by packet */
708  PcapLogDumpSegments(tv, td, p);
709 
710  if (p->flags & PKT_PSEUDO_STREAM_END) {
711  PcapLogUnlock(pl);
712  return TM_ECODE_OK;
713  }
714 
715  /* PcapLogDumpSegment has written over the PcapLogData variables so need to update */
716  pl->h->ts.tv_sec = SCTIME_SECS(p->ts);
717  pl->h->ts.tv_usec = SCTIME_USECS(p->ts);
718  if (PacketIsTunnelChild(p)) {
719  rp = p->root;
720  pl->h->caplen = GET_PKT_LEN(rp);
721  pl->h->len = GET_PKT_LEN(rp);
723  } else {
724  pl->h->caplen = GET_PKT_LEN(p);
725  pl->h->len = GET_PKT_LEN(p);
727  }
728  }
729  }
730 
731  if (PacketIsTunnelChild(p)) {
732  rp = p->root;
733  ret = PcapWrite(tv, td, GET_PKT_DATA(rp), len);
734  } else {
735  ret = PcapWrite(tv, td, GET_PKT_DATA(p), len);
736  }
737  if (ret != TM_ECODE_OK) {
739  PcapLogUnlock(pl);
740  return ret;
741  }
742 
744  pl->profile_data_size += len;
745 
746  SCLogDebug("pl->size_current %"PRIu64", pl->size_limit %"PRIu64,
747  pl->size_current, pl->size_limit);
748 
749  PcapLogUnlock(pl);
750  return TM_ECODE_OK;
751 }
752 
753 static PcapLogData *PcapLogDataCopy(const PcapLogData *pl)
754 {
755  BUG_ON(pl->mode != LOGMODE_MULTI);
756  PcapLogData *copy = SCCalloc(1, sizeof(*copy));
757  if (unlikely(copy == NULL)) {
758  return NULL;
759  }
760 
761  copy->h = SCCalloc(1, sizeof(*copy->h));
762  if (unlikely(copy->h == NULL)) {
763  SCFree(copy);
764  return NULL;
765  }
766 
767  copy->prefix = SCStrdup(pl->prefix);
768  if (unlikely(copy->prefix == NULL)) {
769  SCFree(copy->h);
770  SCFree(copy);
771  return NULL;
772  }
773 
774  copy->suffix = pl->suffix;
775 
776  /* settings TODO move to global cfg struct */
777  copy->is_private = true;
778  copy->mode = pl->mode;
779  copy->max_files = pl->max_files;
780  copy->use_ringbuffer = pl->use_ringbuffer;
781  copy->timestamp_format = pl->timestamp_format;
783  copy->size_limit = pl->size_limit;
784  copy->conditional = pl->conditional;
785  copy->bpf_filter = pl->bpf_filter;
786 
787  const PcapLogCompressionData *comp = &pl->compression;
788  PcapLogCompressionData *copy_comp = &copy->compression;
789  copy_comp->format = comp->format;
790 #ifdef HAVE_LIBLZ4
792  /* We need to allocate a new compression context and buffers for
793  * the copy. First copy the things that can simply be copied. */
794 
795  copy_comp->buffer_size = comp->buffer_size;
796  copy_comp->pcap_buf_size = comp->pcap_buf_size;
797  copy_comp->lz4f_prefs = comp->lz4f_prefs;
798 
799  /* Allocate the buffers. */
800 
801  copy_comp->buffer = SCMalloc(copy_comp->buffer_size);
802  if (copy_comp->buffer == NULL) {
803  SCLogError("SCMalloc failed: %s", strerror(errno));
804  SCFree(copy->prefix);
805  SCFree(copy->h);
806  SCFree(copy);
807  return NULL;
808  }
809  copy_comp->pcap_buf = SCMalloc(copy_comp->pcap_buf_size);
810  if (copy_comp->pcap_buf == NULL) {
811  SCLogError("SCMalloc failed: %s", strerror(errno));
812  SCFree(copy_comp->buffer);
813  SCFree(copy->prefix);
814  SCFree(copy->h);
815  SCFree(copy);
816  return NULL;
817  }
818  copy_comp->pcap_buf_wrapper = SCFmemopen(copy_comp->pcap_buf,
819  copy_comp->pcap_buf_size, "w");
820  if (copy_comp->pcap_buf_wrapper == NULL) {
821  SCLogError("SCFmemopen failed: %s", strerror(errno));
822  SCFree(copy_comp->buffer);
823  SCFree(copy_comp->pcap_buf);
824  SCFree(copy->prefix);
825  SCFree(copy->h);
826  SCFree(copy);
827  return NULL;
828  }
829 
830  /* Initialize a new compression context. */
831 
832  LZ4F_errorCode_t errcode =
833  LZ4F_createCompressionContext(&copy_comp->lz4f_context, 1);
834  if (LZ4F_isError(errcode)) {
835  SCLogError("LZ4F_createCompressionContext failed: %s", LZ4F_getErrorName(errcode));
836  fclose(copy_comp->pcap_buf_wrapper);
837  SCFree(copy_comp->buffer);
838  SCFree(copy_comp->pcap_buf);
839  SCFree(copy->prefix);
840  SCFree(copy->h);
841  SCFree(copy);
842  return NULL;
843  }
844 
845  /* Initialize the rest. */
846 
847  copy_comp->file = NULL;
848  copy_comp->bytes_in_block = 0;
849  }
850 #endif /* HAVE_LIBLZ4 */
851 
852  TAILQ_INIT(&copy->pcap_file_list);
853  SCMutexInit(&copy->plog_lock, NULL);
854 
855  strlcpy(copy->dir, pl->dir, sizeof(copy->dir));
856 
857  for (int i = 0; i < pl->filename_part_cnt && i < MAX_TOKS; i++)
858  copy->filename_parts[i] = pl->filename_parts[i];
859  copy->filename_part_cnt = pl->filename_part_cnt;
860 
861  /* set thread number, first thread is 1 */
862  copy->thread_number = SC_ATOMIC_ADD(thread_cnt, 1);
863 
864  SCLogDebug("copied, returning %p", copy);
865  return copy;
866 }
867 
868 #ifdef INIT_RING_BUFFER
869 static int PcapLogGetTimeOfFile(const char *filename, uint64_t *secs,
870  uint32_t *usecs)
871 {
872  char buf[PATH_MAX];
873  size_t copylen;
874 
875  int n = pcre2_match(pcre_timestamp_code, (PCRE2_SPTR8)filename, strlen(filename), 0, 0,
876  pcre_timestamp_match, NULL);
877  if (n != 2 && n != 4) {
878  /* No match. */
879  return 0;
880  }
881 
882  if (n >= 2) {
883  /* Extract seconds. */
884  copylen = sizeof(buf);
885  if (pcre2_substring_copy_bynumber(pcre_timestamp_match, 1, (PCRE2_UCHAR8 *)buf, &copylen) <
886  0) {
887  return 0;
888  }
889  if (StringParseUint64(secs, 10, 0, buf) < 0) {
890  return 0;
891  }
892  }
893  if (n == 4) {
894  /* Extract microseconds. */
895  copylen = sizeof(buf);
896  if (pcre2_substring_copy_bynumber(pcre_timestamp_match, 3, (PCRE2_UCHAR8 *)buf, &copylen) <
897  0) {
898  return 0;
899  }
900  if (StringParseUint32(usecs, 10, 0, buf) < 0) {
901  return 0;
902  }
903  }
904 
905  return 1;
906 }
907 
908 static TmEcode PcapLogInitRingBuffer(PcapLogData *pl)
909 {
910  char pattern[PATH_MAX];
911 
912  SCLogInfo("Initializing PCAP ring buffer for %s/%s.",
913  pl->dir, pl->prefix);
914 
915  strlcpy(pattern, pl->dir, PATH_MAX);
916  if (pattern[strlen(pattern) - 1] != '/') {
917  strlcat(pattern, "/", PATH_MAX);
918  }
919  if (pl->mode == LOGMODE_MULTI) {
920  for (int i = 0; i < pl->filename_part_cnt; i++) {
921  char *part = pl->filename_parts[i];
922  if (part == NULL || strlen(part) == 0) {
923  continue;
924  }
925  if (part[0] != '%' || strlen(part) < 2) {
926  strlcat(pattern, part, PATH_MAX);
927  continue;
928  }
929  switch (part[1]) {
930  case 'i':
931  SCLogError("Thread ID not allowed in ring buffer mode.");
932  return TM_ECODE_FAILED;
933  case 'n': {
934  char tmp[PATH_MAX];
935  snprintf(tmp, PATH_MAX, "%"PRIu32, pl->thread_number);
936  strlcat(pattern, tmp, PATH_MAX);
937  break;
938  }
939  case 't':
940  strlcat(pattern, "*", PATH_MAX);
941  break;
942  default:
943  SCLogError("Unsupported format character: %%%s", part);
944  return TM_ECODE_FAILED;
945  }
946  }
947  } else {
948  strlcat(pattern, pl->prefix, PATH_MAX);
949  strlcat(pattern, ".*", PATH_MAX);
950  }
951  strlcat(pattern, pl->suffix, PATH_MAX);
952 
953  char *basename = strrchr(pattern, '/');
954  *basename++ = '\0';
955 
956  /* Pattern is now just the directory name. */
957  DIR *dir = opendir(pattern);
958  if (dir == NULL) {
959  SCLogWarning("Failed to open directory %s: %s", pattern, strerror(errno));
960  return TM_ECODE_FAILED;
961  }
962 
963  for (;;) {
964  struct dirent *entry = readdir(dir);
965  if (entry == NULL) {
966  break;
967  }
968  if (fnmatch(basename, entry->d_name, 0) != 0) {
969  continue;
970  }
971 
972  uint64_t secs = 0;
973  uint32_t usecs = 0;
974 
975  if (!PcapLogGetTimeOfFile(entry->d_name, &secs, &usecs)) {
976  /* Failed to get time stamp out of file name. Not necessarily a
977  * failure as the file might just not be a pcap log file. */
978  continue;
979  }
980 
981  PcapFileName *pf = SCCalloc(sizeof(*pf), 1);
982  if (unlikely(pf == NULL)) {
983  goto fail;
984  }
985  char path[PATH_MAX];
986  if (PathMerge(path, sizeof(path), pattern, entry->d_name) < 0)
987  goto fail;
988 
989  if ((pf->filename = SCStrdup(path)) == NULL) {
990  goto fail;
991  }
992  if ((pf->dirname = SCStrdup(pattern)) == NULL) {
993  goto fail;
994  }
995  pf->secs = secs;
996  pf->usecs = usecs;
997 
998  if (TAILQ_EMPTY(&pl->pcap_file_list)) {
999  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
1000  } else {
1001  /* Ordered insert. */
1002  PcapFileName *it = NULL;
1003  TAILQ_FOREACH(it, &pl->pcap_file_list, next) {
1004  if (pf->secs < it->secs) {
1005  break;
1006  } else if (pf->secs == it->secs && pf->usecs < it->usecs) {
1007  break;
1008  }
1009  }
1010  if (it == NULL) {
1011  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
1012  } else {
1013  TAILQ_INSERT_BEFORE(it, pf, next);
1014  }
1015  }
1016  pl->file_cnt++;
1017  continue;
1018 
1019  fail:
1020  if (pf != NULL) {
1021  if (pf->filename != NULL) {
1022  SCFree(pf->filename);
1023  }
1024  if (pf->dirname != NULL) {
1025  SCFree(pf->dirname);
1026  }
1027  SCFree(pf);
1028  }
1029  break;
1030  }
1031 
1032  if (pl->file_cnt > pl->max_files) {
1033  PcapFileName *pf = TAILQ_FIRST(&pl->pcap_file_list);
1034  while (pf != NULL && pl->file_cnt > pl->max_files) {
1035  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
1036  DEBUG_VALIDATE_BUG_ON(TAILQ_FIRST(&pl->pcap_file_list) == pf);
1037 
1038  SCLogDebug("Removing PCAP file %s", pf->filename);
1039  if (remove(pf->filename) != 0) {
1040  SCLogWarning("Failed to remove PCAP file %s: %s", pf->filename, strerror(errno));
1041  }
1042  PcapFileNameFree(pf);
1043  pl->file_cnt--;
1044 
1045  pf = TAILQ_FIRST(&pl->pcap_file_list);
1046  }
1047  }
1048 
1049  closedir(dir);
1050 
1051  /* For some reason file count is initialized at one, instead of 0. */
1052  SCLogNotice("Ring buffer initialized with %d files.", pl->file_cnt - 1);
1053 
1054  return TM_ECODE_OK;
1055 }
1056 #endif /* INIT_RING_BUFFER */
1057 
1058 static TmEcode PcapLogDataInit(ThreadVars *t, const void *initdata, void **data)
1059 {
1060  if (initdata == NULL) {
1061  SCLogDebug("Error getting context for LogPcap. \"initdata\" argument NULL");
1062  return TM_ECODE_FAILED;
1063  }
1064 
1065  PcapLogData *pl = ((OutputCtx *)initdata)->data;
1066 
1067  PcapLogThreadData *td = SCCalloc(1, sizeof(*td));
1068  if (unlikely(td == NULL))
1069  return TM_ECODE_FAILED;
1070 
1071  td->counter_written = StatsRegisterCounter("pcap_log.written", &t->stats);
1072  td->counter_filtered_bpf = StatsRegisterCounter("pcap_log.filtered_bpf", &t->stats);
1073 
1074  if (pl->mode == LOGMODE_MULTI)
1075  td->pcap_log = PcapLogDataCopy(pl);
1076  else
1077  td->pcap_log = pl;
1078  BUG_ON(td->pcap_log == NULL);
1079 
1080  if (DatalinkHasMultipleValues()) {
1081  if (pl->mode != LOGMODE_MULTI) {
1082  FatalError("Pcap logging with multiple link type is not supported.");
1083  } else {
1084  /* In multi mode, only pcap conditional is not supported as a flow timeout
1085  * will trigger packet logging with potentially invalid datalink. In regular
1086  * pcap logging, the logging should be done in the same thread if we
1087  * have a proper load balancing. So no mix of datalink should occur. But we need a
1088  * proper load balancing so this needs at least a warning.
1089  */
1090  switch (pl->conditional) {
1091  case LOGMODE_COND_ALERTS:
1092  case LOGMODE_COND_TAG:
1093  FatalError("Can't have multiple link types in pcap conditional mode.");
1094  break;
1095  default:
1096  SCLogWarning("Using multiple link types can result in invalid pcap output");
1097  }
1098  }
1099  }
1100 
1101  PcapLogLock(td->pcap_log);
1102 
1103  /** Use the Output Context (file pointer and mutex) */
1104  td->pcap_log->pkt_cnt = 0;
1105  td->pcap_log->pcap_dead_handle = NULL;
1106  td->pcap_log->pcap_dumper = NULL;
1107  if (td->pcap_log->file_cnt < 1) {
1108  td->pcap_log->file_cnt = 1;
1109  }
1110 
1111  SCTime_t ts = TimeGet();
1112  struct tm local_tm;
1113  struct tm *tms = SCLocalTime(SCTIME_SECS(ts), &local_tm);
1114  td->pcap_log->prev_day = tms->tm_mday;
1115 
1116  PcapLogUnlock(td->pcap_log);
1117 
1118  /* count threads in the global structure */
1119  SCMutexLock(&pl->plog_lock);
1120  pl->threads++;
1121  SCMutexUnlock(&pl->plog_lock);
1122 
1123  *data = (void *)td;
1124 
1127  } else {
1128  td->buf = NULL;
1129  }
1130 
1131  if (pl->max_files && (pl->mode == LOGMODE_MULTI || pl->threads == 1)) {
1132 #ifdef INIT_RING_BUFFER
1133  if (PcapLogInitRingBuffer(td->pcap_log) == TM_ECODE_FAILED) {
1134  return TM_ECODE_FAILED;
1135  }
1136 #else
1137  SCLogInfo("Unable to initialize ring buffer on this platform.");
1138 #endif /* INIT_RING_BUFFER */
1139  }
1140 
1141  /* Don't early initialize output files if in a PCAP file (offline)
1142  * mode. */
1143  if (!IsRunModeOffline(SCRunmodeGet())) {
1144  if (pl->mode == LOGMODE_MULTI) {
1145  PcapLogOpenFileCtx(td->pcap_log);
1146  } else {
1147  if (pl->filename == NULL) {
1148  PcapLogOpenFileCtx(pl);
1149  }
1150  }
1151  }
1152 
1153  return TM_ECODE_OK;
1154 }
1155 
1156 static void StatsMerge(PcapLogData *dst, PcapLogData *src)
1157 {
1158  dst->profile_open.total += src->profile_open.total;
1159  dst->profile_open.cnt += src->profile_open.cnt;
1160 
1161  dst->profile_close.total += src->profile_close.total;
1162  dst->profile_close.cnt += src->profile_close.cnt;
1163 
1164  dst->profile_write.total += src->profile_write.total;
1165  dst->profile_write.cnt += src->profile_write.cnt;
1166 
1167  dst->profile_rotate.total += src->profile_rotate.total;
1168  dst->profile_rotate.cnt += src->profile_rotate.cnt;
1169 
1170  dst->profile_handles.total += src->profile_handles.total;
1171  dst->profile_handles.cnt += src->profile_handles.cnt;
1172 
1173  dst->profile_lock.total += src->profile_lock.total;
1174  dst->profile_lock.cnt += src->profile_lock.cnt;
1175 
1176  dst->profile_unlock.total += src->profile_unlock.total;
1177  dst->profile_unlock.cnt += src->profile_unlock.cnt;
1178 
1179  dst->profile_data_size += src->profile_data_size;
1180 }
1181 
1182 static void PcapLogDataFree(PcapLogData *pl)
1183 {
1184 
1185  PcapFileName *pf;
1186  while ((pf = TAILQ_FIRST(&pl->pcap_file_list)) != NULL) {
1187  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
1188  DEBUG_VALIDATE_BUG_ON(TAILQ_FIRST(&pl->pcap_file_list) == pf);
1189  PcapFileNameFree(pf);
1190  }
1191  if (pl == g_pcap_data) {
1192  for (int i = 0; i < MAX_TOKS; i++) {
1193  if (pl->filename_parts[i] != NULL) {
1194  SCFree(pl->filename_parts[i]);
1195  }
1196  }
1197  }
1198  SCFree(pl->h);
1199  SCFree(pl->filename);
1200  SCFree(pl->prefix);
1201 
1202  if (pl->pcap_dead_handle) {
1203  pcap_close(pl->pcap_dead_handle);
1204  }
1205 
1206  if (pl->bpfp) {
1207  pcap_freecode(pl->bpfp);
1208  SCFree(pl->bpfp);
1209  }
1210 
1211 #ifdef HAVE_LIBLZ4
1212  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
1213  SCFree(pl->compression.buffer);
1214  if (pl->compression.pcap_buf_wrapper)
1215  fclose(pl->compression.pcap_buf_wrapper);
1216  SCFree(pl->compression.pcap_buf);
1217  LZ4F_errorCode_t errcode =
1218  LZ4F_freeCompressionContext(pl->compression.lz4f_context);
1219  if (LZ4F_isError(errcode)) {
1220  SCLogWarning("Error freeing lz4 context.");
1221  }
1222  }
1223 #endif /* HAVE_LIBLZ4 */
1224  SCFree(pl);
1225 }
1226 
1227 /**
1228  * \brief Thread deinit function.
1229  *
1230  * \param t Thread Variable containing input/output queue, cpu affinity etc.
1231  * \param data PcapLog thread data.
1232  * \retval TM_ECODE_OK on success
1233  * \retval TM_ECODE_FAILED on failure
1234  */
1235 static TmEcode PcapLogDataDeinit(ThreadVars *t, void *thread_data)
1236 {
1237  PcapLogThreadData *td = (PcapLogThreadData *)thread_data;
1238  PcapLogData *pl = td->pcap_log;
1239 
1240  if (pl->pcap_dumper != NULL) {
1241  if (PcapLogCloseFile(t, pl) != TM_ECODE_OK) {
1242  SCLogDebug("PcapLogCloseFile failed");
1243  }
1244  }
1245 
1246  if (pl->mode == LOGMODE_MULTI) {
1247  SCMutexLock(&g_pcap_data->plog_lock);
1248  StatsMerge(g_pcap_data, pl);
1249  g_pcap_data->reported++;
1250  if (g_pcap_data->threads == g_pcap_data->reported)
1251  PcapLogProfilingDump(g_pcap_data);
1252  SCMutexUnlock(&g_pcap_data->plog_lock);
1253  } else {
1254  if (pl->reported == 0) {
1255  PcapLogProfilingDump(pl);
1256  pl->reported = 1;
1257  }
1258  }
1259 
1260  if (pl != g_pcap_data) {
1261  PcapLogDataFree(pl);
1262  }
1263 
1264  if (td->buf)
1265  MemBufferFree(td->buf);
1266 
1267  SCFree(td);
1268  return TM_ECODE_OK;
1269 }
1270 
1271 
1272 static int ParseFilename(PcapLogData *pl, const char *filename)
1273 {
1274  char *toks[MAX_TOKS] = { NULL };
1275  int tok = 0;
1276  char str[MAX_FILENAMELEN] = "";
1277  int s = 0;
1278  char *p = NULL;
1279  size_t filename_len = 0;
1280 
1281  if (filename) {
1282  filename_len = strlen(filename);
1283  if (filename_len > (MAX_FILENAMELEN-1)) {
1284  SCLogError("invalid filename option. Max filename-length: %d", MAX_FILENAMELEN - 1);
1285  goto error;
1286  }
1287 
1288  for (int i = 0; i < (int)strlen(filename); i++) {
1289  if (tok >= MAX_TOKS) {
1290  SCLogError("invalid filename option. Max 2 %%-sign options");
1291  goto error;
1292  }
1293 
1294  str[s++] = filename[i];
1295 
1296  if (filename[i] == '%') {
1297  str[s-1] = '\0';
1298  SCLogDebug("filename with %%-sign: %s", str);
1299 
1300  p = SCStrdup(str);
1301  if (p == NULL)
1302  goto error;
1303  toks[tok++] = p;
1304 
1305  s = 0;
1306 
1307  if (i+1 < (int)strlen(filename)) {
1308  if (tok >= MAX_TOKS) {
1309  SCLogError("invalid filename option. Max 2 %%-sign options");
1310  goto error;
1311  }
1312 
1313  if (filename[i+1] != 'n' && filename[i+1] != 't' && filename[i+1] != 'i') {
1314  SCLogError(
1315  "invalid filename option. Valid %%-sign options: %%n, %%i and %%t");
1316  goto error;
1317  }
1318  str[0] = '%';
1319  str[1] = filename[i+1];
1320  str[2] = '\0';
1321  p = SCStrdup(str);
1322  if (p == NULL)
1323  goto error;
1324  toks[tok++] = p;
1325  i++;
1326  }
1327  }
1328  }
1329 
1330  if ((tok == 0) && (pl->mode == LOGMODE_MULTI)) {
1331  SCLogError("Invalid filename for multimode. Need at least one %%-sign option");
1332  goto error;
1333  }
1334 
1335  if (s) {
1336  if (tok >= MAX_TOKS) {
1337  SCLogError("invalid filename option. Max 3 %%-sign options");
1338  goto error;
1339 
1340  }
1341  str[s++] = '\0';
1342  p = SCStrdup(str);
1343  if (p == NULL)
1344  goto error;
1345  toks[tok++] = p;
1346  }
1347 
1348  /* finally, store tokens in the pl */
1349  for (int i = 0; i < tok; i++) {
1350  if (toks[i] == NULL)
1351  goto error;
1352 
1353  SCLogDebug("toks[%d] %s", i, toks[i]);
1354  pl->filename_parts[i] = toks[i];
1355  }
1356  pl->filename_part_cnt = tok;
1357  }
1358  return 0;
1359 error:
1360  for (int x = 0; x < MAX_TOKS; x++) {
1361  if (toks[x] != NULL)
1362  SCFree(toks[x]);
1363  }
1364  return -1;
1365 }
1366 
1367 /** \brief Fill in pcap logging struct from the provided ConfNode.
1368  * \param conf The configuration node for this output.
1369  * \retval output_ctx
1370  * */
1371 static OutputInitResult PcapLogInitCtx(SCConfNode *conf)
1372 {
1373  OutputInitResult result = { NULL, false };
1374  int en;
1375  PCRE2_SIZE eo = 0;
1376 
1377  if (g_pcap_data) {
1378  FatalError("A pcap-log instance is already active, only one can be enabled.");
1379  }
1380 
1381  PcapLogData *pl = SCCalloc(1, sizeof(PcapLogData));
1382  if (unlikely(pl == NULL)) {
1383  FatalError("Failed to allocate Memory for PcapLogData");
1384  }
1385 
1386  pl->h = SCMalloc(sizeof(*pl->h));
1387  if (pl->h == NULL) {
1388  FatalError("Failed to allocate Memory for pcap header struct");
1389  }
1390 
1391  /* Set the defaults */
1392  pl->mode = LOGMODE_NORMAL;
1394  pl->use_ringbuffer = RING_BUFFER_MODE_DISABLED;
1395  pl->timestamp_format = TS_FORMAT_SEC;
1399 
1400  TAILQ_INIT(&pl->pcap_file_list);
1401 
1402  SCMutexInit(&pl->plog_lock, NULL);
1403 
1404  /* Initialize PCREs. */
1405  pcre_timestamp_code =
1406  pcre2_compile((PCRE2_SPTR8)timestamp_pattern, PCRE2_ZERO_TERMINATED, 0, &en, &eo, NULL);
1407  if (pcre_timestamp_code == NULL) {
1408  PCRE2_UCHAR errbuffer[256];
1409  pcre2_get_error_message(en, errbuffer, sizeof(errbuffer));
1410  FatalError(
1411  "Failed to compile \"%s\" at offset %d: %s", timestamp_pattern, (int)eo, errbuffer);
1412  }
1413  pcre_timestamp_match = pcre2_match_data_create_from_pattern(pcre_timestamp_code, NULL);
1414 
1415  /* conf params */
1416 
1417  const char *filename = NULL;
1418 
1419  if (conf != NULL) { /* To facilitate unit tests. */
1420  filename = SCConfNodeLookupChildValue(conf, "filename");
1421  }
1422 
1423  if (filename == NULL)
1424  filename = DEFAULT_LOG_FILENAME;
1425 
1426  if ((pl->prefix = SCStrdup(filename)) == NULL) {
1427  exit(EXIT_FAILURE);
1428  }
1429 
1430  pl->suffix = "";
1431 
1432  pl->size_limit = DEFAULT_LIMIT;
1433  if (conf != NULL) {
1434  const char *s_limit = NULL;
1435  s_limit = SCConfNodeLookupChildValue(conf, "limit");
1436  if (s_limit != NULL) {
1437  if (ParseSizeStringU64(s_limit, &pl->size_limit) < 0) {
1438  SCLogError("Failed to initialize pcap output, invalid limit: %s", s_limit);
1439  exit(EXIT_FAILURE);
1440  }
1441  if (pl->size_limit < 4096) {
1442  SCLogInfo("pcap-log \"limit\" value of %"PRIu64" assumed to be pre-1.2 "
1443  "style: setting limit to %"PRIu64"mb", pl->size_limit, pl->size_limit);
1444  uint64_t size = pl->size_limit * 1024 * 1024;
1445  pl->size_limit = size;
1446  } else if (pl->size_limit < MIN_LIMIT) {
1447  FatalError("Fail to initialize pcap-log output, limit less than "
1448  "allowed minimum of %d bytes.",
1449  MIN_LIMIT);
1450  }
1451  }
1452  }
1453 
1454  if (conf != NULL) {
1455  const char *s_mode = NULL;
1456  s_mode = SCConfNodeLookupChildValue(conf, "mode");
1457  if (s_mode != NULL) {
1458  if (strcasecmp(s_mode, "multi") == 0) {
1459  pl->mode = LOGMODE_MULTI;
1460  } else if (strcasecmp(s_mode, "normal") != 0) {
1461  FatalError("log-pcap: invalid mode \"%s\". Valid options: \"normal\""
1462  "or \"multi\" mode ",
1463  s_mode);
1464  }
1465  }
1466 
1467  const char *s_dir = NULL;
1468  s_dir = SCConfNodeLookupChildValue(conf, "dir");
1469  if (s_dir == NULL) {
1470  const char *log_dir = NULL;
1471  log_dir = SCConfigGetLogDirectory();
1472 
1473  strlcpy(pl->dir, log_dir, sizeof(pl->dir));
1474  SCLogInfo("Using log dir %s", pl->dir);
1475  } else {
1476  if (PathIsAbsolute(s_dir)) {
1477  strlcpy(pl->dir,
1478  s_dir, sizeof(pl->dir));
1479  } else {
1480  const char *log_dir = NULL;
1481  log_dir = SCConfigGetLogDirectory();
1482 
1483  snprintf(pl->dir, sizeof(pl->dir), "%s/%s",
1484  log_dir, s_dir);
1485  }
1486 
1487  struct stat stat_buf;
1488  if (stat(pl->dir, &stat_buf) != 0) {
1489  FatalError("The dir directory \"%s\" "
1490  "supplied doesn't exist. Shutting down the engine",
1491  pl->dir);
1492  }
1493  SCLogInfo("Using log dir %s", pl->dir);
1494  }
1495 
1496  const char *compression_str = SCConfNodeLookupChildValue(conf, "compression");
1497 
1498  PcapLogCompressionData *comp = &pl->compression;
1499  if (compression_str == NULL || strcmp(compression_str, "none") == 0) {
1501  comp->buffer = NULL;
1502  comp->buffer_size = 0;
1503  comp->file = NULL;
1504  comp->pcap_buf = NULL;
1505  comp->pcap_buf_size = 0;
1506 #ifdef HAVE_LIBLZ4
1507  comp->pcap_buf_wrapper = NULL;
1508 #endif
1509  } else if (strcmp(compression_str, "lz4") == 0) {
1510 #ifdef HAVE_LIBLZ4
1511  pl->compression.format = PCAP_LOG_COMPRESSION_FORMAT_LZ4;
1512 
1513  /* Use SCFmemopen so we can make pcap_dump write to a buffer. */
1514 
1515  comp->pcap_buf_size = sizeof(struct pcap_file_header) +
1516  sizeof(struct pcap_pkthdr) + PCAP_SNAPLEN;
1517  comp->pcap_buf = SCMalloc(comp->pcap_buf_size);
1518  if (comp->pcap_buf == NULL) {
1519  SCLogError("SCMalloc failed: %s", strerror(errno));
1520  exit(EXIT_FAILURE);
1521  }
1522  comp->pcap_buf_wrapper = SCFmemopen(comp->pcap_buf,
1523  comp->pcap_buf_size, "w");
1524  if (comp->pcap_buf_wrapper == NULL) {
1525  SCLogError("SCFmemopen failed: %s", strerror(errno));
1526  exit(EXIT_FAILURE);
1527  }
1528 
1529  /* Set lz4 preferences. */
1530 
1531  memset(&comp->lz4f_prefs, '\0', sizeof(comp->lz4f_prefs));
1532  comp->lz4f_prefs.frameInfo.blockSizeID = LZ4F_max4MB;
1533  comp->lz4f_prefs.frameInfo.blockMode = LZ4F_blockLinked;
1534  if (SCConfNodeChildValueIsTrue(conf, "lz4-checksum")) {
1535  comp->lz4f_prefs.frameInfo.contentChecksumFlag = 1;
1536  } else {
1537  comp->lz4f_prefs.frameInfo.contentChecksumFlag = 0;
1538  }
1539  intmax_t lvl = 0;
1540  if (SCConfGetChildValueInt(conf, "lz4-level", &lvl)) {
1541  if (lvl > 16) {
1542  lvl = 16;
1543  } else if (lvl < 0) {
1544  lvl = 0;
1545  }
1546  } else {
1547  lvl = 0;
1548  }
1549  comp->lz4f_prefs.compressionLevel = (int)lvl;
1550 
1551  /* Allocate resources for lz4. */
1552 
1553  LZ4F_errorCode_t errcode =
1554  LZ4F_createCompressionContext(&pl->compression.lz4f_context, 1);
1555 
1556  if (LZ4F_isError(errcode)) {
1557  SCLogError("LZ4F_createCompressionContext failed: %s", LZ4F_getErrorName(errcode));
1558  exit(EXIT_FAILURE);
1559  }
1560 
1561  /* Calculate the size of the lz4 output buffer. */
1562 
1563  comp->buffer_size = LZ4F_compressBound(comp->pcap_buf_size,
1564  &comp->lz4f_prefs);
1565 
1566  comp->buffer = SCMalloc(comp->buffer_size);
1567  if (unlikely(comp->buffer == NULL)) {
1568  FatalError("Failed to allocate memory for "
1569  "lz4 output buffer.");
1570  }
1571 
1572  comp->bytes_in_block = 0;
1573 
1574  /* Add the lz4 file extension to the log files. */
1575 
1576  pl->suffix = ".lz4";
1577 #else
1578  SCLogError("lz4 compression was selected "
1579  "in pcap-log, but suricata was not compiled with lz4 "
1580  "support.");
1581  PcapLogDataFree(pl);
1582  return result;
1583 #endif /* HAVE_LIBLZ4 */
1584  }
1585  else {
1586  SCLogError("Unsupported pcap-log "
1587  "compression format: %s",
1588  compression_str);
1589  PcapLogDataFree(pl);
1590  return result;
1591  }
1592 
1593  SCLogInfo("Selected pcap-log compression method: %s",
1594  compression_str ? compression_str : "none");
1595 
1596  const char *s_conditional = SCConfNodeLookupChildValue(conf, "conditional");
1597  if (s_conditional != NULL) {
1598  if (strcasecmp(s_conditional, "alerts") == 0) {
1601  } else if (strcasecmp(s_conditional, "tag") == 0) {
1604  } else if (strcasecmp(s_conditional, "all") != 0) {
1605  FatalError("log-pcap: invalid conditional \"%s\". Valid options: \"all\", "
1606  "\"alerts\", or \"tag\" mode ",
1607  s_conditional);
1608  }
1609  }
1610 
1611  SCLogInfo(
1612  "Selected pcap-log conditional logging: %s", s_conditional ? s_conditional : "all");
1613  }
1614 
1615  if (ParseFilename(pl, filename) != 0)
1616  exit(EXIT_FAILURE);
1617 
1618  SCLogInfo("using %s logging", (pl->mode == LOGMODE_MULTI ? "multi" : "normal"));
1619 
1620  uint32_t max_file_limit = DEFAULT_FILE_LIMIT;
1621  if (conf != NULL) {
1622  const char *max_number_of_files_s = NULL;
1623  max_number_of_files_s = SCConfNodeLookupChildValue(conf, "max-files");
1624  if (max_number_of_files_s != NULL) {
1625  if (StringParseUint32(&max_file_limit, 10, 0,
1626  max_number_of_files_s) == -1) {
1627  SCLogError("Failed to initialize "
1628  "pcap-log output, invalid number of files limit: %s",
1629  max_number_of_files_s);
1630  exit(EXIT_FAILURE);
1631  } else if (max_file_limit < 1) {
1632  FatalError("Failed to initialize pcap-log output, limit less than "
1633  "allowed minimum.");
1634  } else {
1635  pl->max_files = max_file_limit;
1636  pl->use_ringbuffer = RING_BUFFER_MODE_ENABLED;
1637  }
1638  }
1639  }
1640 
1641  const char *ts_format = NULL;
1642  if (conf != NULL) { /* To facilitate unit tests. */
1643  ts_format = SCConfNodeLookupChildValue(conf, "ts-format");
1644  }
1645  if (ts_format != NULL) {
1646  if (strcasecmp(ts_format, "usec") == 0) {
1647  pl->timestamp_format = TS_FORMAT_USEC;
1648  } else if (strcasecmp(ts_format, "sec") != 0) {
1649  SCLogError("log-pcap ts_format specified %s is invalid must be"
1650  " \"sec\" or \"usec\"",
1651  ts_format);
1652  exit(EXIT_FAILURE);
1653  }
1654  }
1655 
1656  const char *use_stream_depth = NULL;
1657  if (conf != NULL) { /* To facilitate unit tests. */
1658  use_stream_depth = SCConfNodeLookupChildValue(conf, "use-stream-depth");
1659  }
1660  if (use_stream_depth != NULL) {
1661  if (SCConfValIsFalse(use_stream_depth)) {
1663  } else if (SCConfValIsTrue(use_stream_depth)) {
1665  } else {
1666  FatalError("log-pcap use_stream_depth specified is invalid must be");
1667  }
1668  }
1669 
1670  const char *honor_pass_rules = NULL;
1671  if (conf != NULL) { /* To facilitate unit tests. */
1672  honor_pass_rules = SCConfNodeLookupChildValue(conf, "honor-pass-rules");
1673  }
1674  if (honor_pass_rules != NULL) {
1675  if (SCConfValIsFalse(honor_pass_rules)) {
1677  } else if (SCConfValIsTrue(honor_pass_rules)) {
1679  } else {
1680  FatalError("log-pcap honor-pass-rules specified is invalid");
1681  }
1682  }
1683 
1684  pl->bpf_filter = conf == NULL ? NULL : (char *)SCConfNodeLookupChildValue(conf, "bpf-filter");
1685 
1686  /* create the output ctx and send it back */
1687 
1688  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
1689  if (unlikely(output_ctx == NULL)) {
1690  FatalError("Failed to allocate memory for OutputCtx.");
1691  }
1692  output_ctx->data = pl;
1693  output_ctx->DeInit = PcapLogFileDeInitCtx;
1694  g_pcap_data = pl;
1695 
1696  result.ctx = output_ctx;
1697  result.ok = true;
1698  return result;
1699 }
1700 
1701 static void PcapLogFileDeInitCtx(OutputCtx *output_ctx)
1702 {
1703  if (output_ctx == NULL)
1704  return;
1705 
1706  PcapLogData *pl = output_ctx->data;
1707 
1708  PcapFileName *pf = NULL;
1709  TAILQ_FOREACH(pf, &pl->pcap_file_list, next) {
1710  SCLogDebug("PCAP files left at exit: %s\n", pf->filename);
1711  }
1712  PcapLogDataFree(pl);
1713  SCFree(output_ctx);
1714 
1715  pcre2_code_free(pcre_timestamp_code);
1716  pcre2_match_data_free(pcre_timestamp_match);
1717 }
1718 
1719 /**
1720  * \brief Read the config set the file pointer, open the file
1721  *
1722  * \param PcapLogData.
1723  *
1724  * \retval -1 if failure
1725  * \retval 0 if succesful
1726  */
1727 static int PcapLogOpenFileCtx(PcapLogData *pl)
1728 {
1729  char *path = NULL;
1730 
1732 
1733  if (pl->filename != NULL)
1734  path = pl->filename;
1735  else {
1736  path = SCMalloc(PATH_MAX);
1737  if (unlikely(path == NULL)) {
1738  return -1;
1739  }
1740  pl->filename = path;
1741  }
1742 
1743  /** get the time so we can have a filename with seconds since epoch */
1744  SCTime_t ts = TimeGet();
1745 
1746  /* Place to store the name of our PCAP file */
1747  PcapFileName *pf = SCCalloc(1, sizeof(PcapFileName));
1748  if (unlikely(pf == NULL)) {
1749  return -1;
1750  }
1751 
1752  char file[PATH_MAX] = "";
1753  if (pl->mode == LOGMODE_NORMAL) {
1754  int ret;
1755  /* create the filename to use */
1756  if (pl->timestamp_format == TS_FORMAT_SEC) {
1757  ret = snprintf(file, sizeof(file), "%s.%" PRIu32 "%s", pl->prefix,
1758  (uint32_t)SCTIME_SECS(ts), pl->suffix);
1759  } else {
1760  ret = snprintf(file, sizeof(file), "%s.%" PRIu32 ".%" PRIu32 "%s", pl->prefix,
1761  (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts), pl->suffix);
1762  }
1763  if (ret < 0 || (size_t)ret >= PATH_MAX) {
1764  SCLogError("failed to construct path");
1765  goto error;
1766  }
1767  } else if (pl->mode == LOGMODE_MULTI) {
1768  if (pl->filename_part_cnt > 0) {
1769  /* assemble filename from stored tokens */
1770 
1771  for (int i = 0; i < pl->filename_part_cnt; i++) {
1772  if (pl->filename_parts[i] == NULL ||strlen(pl->filename_parts[i]) == 0)
1773  continue;
1774 
1775  /* handle variables */
1776  if (pl->filename_parts[i][0] == '%') {
1777  char str[64] = "";
1778  if (strlen(pl->filename_parts[i]) < 2)
1779  continue;
1780 
1781  switch(pl->filename_parts[i][1]) {
1782  case 'n':
1783  snprintf(str, sizeof(str), "%u", pl->thread_number);
1784  break;
1785  case 'i':
1786  {
1787  long thread_id = SCGetThreadIdLong();
1788  snprintf(str, sizeof(str), "%"PRIu64, (uint64_t)thread_id);
1789  break;
1790  }
1791  case 't':
1792  /* create the filename to use */
1793  if (pl->timestamp_format == TS_FORMAT_SEC) {
1794  snprintf(str, sizeof(str), "%" PRIu32, (uint32_t)SCTIME_SECS(ts));
1795  } else {
1796  snprintf(str, sizeof(str), "%" PRIu32 ".%" PRIu32,
1797  (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts));
1798  }
1799  }
1800  strlcat(file, str, sizeof(file));
1801 
1802  /* copy the rest over */
1803  } else {
1804  strlcat(file, pl->filename_parts[i], sizeof(file));
1805  }
1806  }
1807  strlcat(file, pl->suffix, sizeof(file));
1808  } else {
1809  int ret;
1810  /* create the filename to use */
1811  if (pl->timestamp_format == TS_FORMAT_SEC) {
1812  ret = snprintf(file, sizeof(file), "%s.%u.%" PRIu32 "%s", pl->prefix,
1813  pl->thread_number, (uint32_t)SCTIME_SECS(ts), pl->suffix);
1814  } else {
1815  ret = snprintf(file, sizeof(file), "%s.%u.%" PRIu32 ".%" PRIu32 "%s", pl->prefix,
1816  pl->thread_number, (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts),
1817  pl->suffix);
1818  }
1819  if (ret < 0 || (size_t)ret >= PATH_MAX) {
1820  SCLogError("failed to construct path");
1821  goto error;
1822  }
1823  }
1824  SCLogDebug("multi-mode: filename %s", file);
1825  }
1826  if (PathMerge(path, PATH_MAX, pl->dir, file) < 0) {
1827  SCLogError("failed to construct path");
1828  goto error;
1829  }
1830 
1831  if ((pf->filename = SCStrdup(pl->filename)) == NULL) {
1832  SCLogError("Error allocating memory. For filename");
1833  goto error;
1834  }
1835  SCLogDebug("Opening pcap file log %s", pf->filename);
1836  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
1837 
1838  if (pl->mode == LOGMODE_MULTI || pl->mode == LOGMODE_NORMAL) {
1839  pcap_file_thread = pl->filename;
1840  }
1842  return 0;
1843 
1844 error:
1845  PcapFileNameFree(pf);
1846  return -1;
1847 }
1848 
1850 {
1851  /* return pcap filename per thread */
1852  if (pcap_file_thread != NULL) {
1853  return pcap_file_thread;
1854  }
1855  return NULL;
1856 }
1857 
1858 static int profiling_pcaplog_enabled = 0;
1859 static int profiling_pcaplog_output_to_file = 0;
1860 static char *profiling_pcaplog_file_name = NULL;
1861 static const char *profiling_pcaplog_file_mode = "a";
1862 
1863 static void FormatNumber(uint64_t num, char *str, size_t size)
1864 {
1865  if (num < 1000UL)
1866  snprintf(str, size, "%"PRIu64, num);
1867  else if (num < 1000000UL)
1868  snprintf(str, size, "%3.1fk", (float)num/1000UL);
1869  else if (num < 1000000000UL)
1870  snprintf(str, size, "%3.1fm", (float)num/1000000UL);
1871  else
1872  snprintf(str, size, "%3.1fb", (float)num/1000000000UL);
1873 }
1874 
1875 static void ProfileReportPair(FILE *fp, const char *name, const PcapLogProfileData *p)
1876 {
1877  char ticks_str[32] = "n/a";
1878  char cnt_str[32] = "n/a";
1879  char avg_str[32] = "n/a";
1880 
1881  FormatNumber((uint64_t)p->cnt, cnt_str, sizeof(cnt_str));
1882  FormatNumber((uint64_t)p->total, ticks_str, sizeof(ticks_str));
1883  if (p->cnt && p->total)
1884  FormatNumber((uint64_t)(p->total/p->cnt), avg_str, sizeof(avg_str));
1885 
1886  fprintf(fp, "%-28s %-10s %-10s %-10s\n", name, cnt_str, avg_str, ticks_str);
1887 }
1888 
1889 static void ProfileReport(FILE *fp, const PcapLogData *pl)
1890 {
1891  ProfileReportPair(fp, "open", &pl->profile_open);
1892  ProfileReportPair(fp, "close", &pl->profile_close);
1893  ProfileReportPair(fp, "write", &pl->profile_write);
1894  ProfileReportPair(fp, "rotate (incl open/close)", &pl->profile_rotate);
1895  ProfileReportPair(fp, "handles", &pl->profile_handles);
1896  ProfileReportPair(fp, "lock", &pl->profile_lock);
1897  ProfileReportPair(fp, "unlock", &pl->profile_unlock);
1898 }
1899 
1900 static void FormatBytes(uint64_t num, char *str, size_t size)
1901 {
1902  if (num < 1000UL)
1903  snprintf(str, size, "%"PRIu64, num);
1904  else if (num < 1048576UL)
1905  snprintf(str, size, "%3.1fKiB", (float)num/1000UL);
1906  else if (num < 1073741824UL)
1907  snprintf(str, size, "%3.1fMiB", (float)num/1000000UL);
1908  else
1909  snprintf(str, size, "%3.1fGiB", (float)num/1000000000UL);
1910 }
1911 
1912 static void DoDump(const PcapLogData *pl, FILE *fp)
1913 {
1914  /* counters */
1915  fprintf(fp, "\n\nOperation Cnt Avg ticks Total ticks\n");
1916  fprintf(fp, "---------------------------- ---------- ---------- -----------\n");
1917 
1918  ProfileReport(fp, pl);
1919  uint64_t total = pl->profile_write.total + pl->profile_rotate.total +
1922  pl->profile_unlock.total;
1923 
1924  /* overall stats */
1925  fprintf(fp, "\nOverall: %"PRIu64" bytes written, average %d bytes per write.\n",
1927  (int)(pl->profile_data_size / pl->profile_write.cnt) : 0);
1928  fprintf(fp, " PCAP data structure overhead: %"PRIuMAX" per write.\n",
1929  (uintmax_t)sizeof(struct pcap_pkthdr));
1930 
1931  /* print total bytes written */
1932  char bytes_str[32];
1933  FormatBytes(pl->profile_data_size, bytes_str, sizeof(bytes_str));
1934  fprintf(fp, " Size written: %s\n", bytes_str);
1935 
1936  /* ticks per MiB and GiB */
1937  uint64_t ticks_per_mib = 0, ticks_per_gib = 0;
1938  uint64_t mib = pl->profile_data_size/(1024*1024);
1939  if (mib)
1940  ticks_per_mib = total/mib;
1941  char ticks_per_mib_str[32] = "n/a";
1942  if (ticks_per_mib > 0)
1943  FormatNumber(ticks_per_mib, ticks_per_mib_str, sizeof(ticks_per_mib_str));
1944  fprintf(fp, " Ticks per MiB: %s\n", ticks_per_mib_str);
1945 
1946  uint64_t gib = pl->profile_data_size/(1024*1024*1024);
1947  if (gib)
1948  ticks_per_gib = total/gib;
1949  char ticks_per_gib_str[32] = "n/a";
1950  if (ticks_per_gib > 0)
1951  FormatNumber(ticks_per_gib, ticks_per_gib_str, sizeof(ticks_per_gib_str));
1952  fprintf(fp, " Ticks per GiB: %s\n", ticks_per_gib_str);
1953 }
1954 
1955 static void PcapLogProfilingDump(PcapLogData *pl)
1956 {
1957  if (profiling_pcaplog_enabled == 0)
1958  return;
1959 
1960  if (profiling_pcaplog_output_to_file == 1) {
1961  FILE *fp = fopen(profiling_pcaplog_file_name, profiling_pcaplog_file_mode);
1962  if (fp == NULL) {
1963  SCLogError("failed to open %s: %s", profiling_pcaplog_file_name, strerror(errno));
1964  return;
1965  }
1966  DoDump(pl, fp);
1967  fclose(fp);
1968  } else {
1969  DoDump(pl, stdout);
1970  }
1971 }
1972 
1974 {
1975  SCConfNode *conf = SCConfGetNode("profiling.pcap-log");
1976  if (conf != NULL && SCConfNodeChildValueIsTrue(conf, "enabled")) {
1977  profiling_pcaplog_enabled = 1;
1978  SCLogInfo("pcap-log profiling enabled");
1979 
1980  const char *filename = SCConfNodeLookupChildValue(conf, "filename");
1981  if (filename != NULL) {
1982  const char *log_dir;
1983  log_dir = SCConfigGetLogDirectory();
1984 
1985  profiling_pcaplog_file_name = SCMalloc(PATH_MAX);
1986  if (unlikely(profiling_pcaplog_file_name == NULL)) {
1987  FatalError("can't duplicate file name");
1988  }
1989 
1990  snprintf(profiling_pcaplog_file_name, PATH_MAX, "%s/%s", log_dir, filename);
1991 
1992  const char *v = SCConfNodeLookupChildValue(conf, "append");
1993  if (v == NULL || SCConfValIsTrue(v)) {
1994  profiling_pcaplog_file_mode = "a";
1995  } else {
1996  profiling_pcaplog_file_mode = "w";
1997  }
1998 
1999  profiling_pcaplog_output_to_file = 1;
2000  SCLogInfo("pcap-log profiling output goes to %s (mode %s)",
2001  profiling_pcaplog_file_name, profiling_pcaplog_file_mode);
2002  }
2003  }
2004 }
util-byte.h
PcapLogCompressionData_::pcap_buf_size
uint64_t pcap_buf_size
Definition: log-pcap.c:135
PcapLogData
struct PcapLogData_ PcapLogData
PcapLogProfileData_
Definition: log-pcap.c:111
LOGMODE_COND_ALL
@ LOGMODE_COND_ALL
Definition: log-pcap.c:67
len
uint8_t len
Definition: app-layer-dnp3.h:2
ts
uint64_t ts
Definition: source-erf-file.c:68
SCConfValIsTrue
int SCConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:577
PcapLogData_::conditional
LogModeConditionalType conditional
Definition: log-pcap.c:164
PcapLogThreadData
struct PcapLogThreadData_ PcapLogThreadData
MemBuffer_::buffer
uint8_t buffer[]
Definition: util-buffer.h:30
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
util-fmemopen.h
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:262
IsRunModeOffline
bool IsRunModeOffline(enum SCRunModes run_mode_to_check)
Definition: runmodes.c:556
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
USE_STREAM_DEPTH_DISABLED
#define USE_STREAM_DEPTH_DISABLED
Definition: log-pcap.c:78
PcapFileName_::secs
uint64_t secs
Definition: log-pcap.c:102
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
PcapLogData_::pcap_dumper
pcap_dumper_t * pcap_dumper
Definition: log-pcap.c:157
TS_FORMAT_SEC
#define TS_FORMAT_SEC
Definition: log-pcap.c:75
PathMerge
int PathMerge(char *out_buf, size_t buf_size, const char *const dir, const char *const fname)
Definition: util-path.c:74
MemBufferExpand
int MemBufferExpand(MemBuffer **buffer, uint32_t expand_by)
expand membuffer by size of 'expand_by'
Definition: util-buffer.c:60
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:282
ParseSizeStringU64
int ParseSizeStringU64(const char *size, uint64_t *res)
Definition: util-misc.c:190
StatsRegisterCounter
StatsCounterId StatsRegisterCounter(const char *name, StatsThreadContext *stats)
Registers a normal, unqualified counter.
Definition: counters.c:1038
PcapFileName_::usecs
uint32_t usecs
Definition: log-pcap.c:103
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
name
const char * name
Definition: detect-engine-proto.c:47
PcapLogData_::filename
char * filename
Definition: log-pcap.c:151
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:289
PcapLogThreadData_::counter_written
StatsCounterId counter_written
Definition: log-pcap.c:196
HONOR_PASS_RULES_ENABLED
#define HONOR_PASS_RULES_ENABLED
Definition: log-pcap.c:82
Packet_::flags
uint32_t flags
Definition: decode.h:562
DEFAULT_LOG_FILENAME
#define DEFAULT_LOG_FILENAME
Definition: log-pcap.c:57
threads.h
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
RING_BUFFER_MODE_ENABLED
#define RING_BUFFER_MODE_ENABLED
Definition: log-pcap.c:73
PcapLogData_::mode
int mode
Definition: log-pcap.c:152
TAILQ_EMPTY
#define TAILQ_EMPTY(head)
Definition: queue.h:248
SCConfNodeChildValueIsTrue
int SCConfNodeChildValueIsTrue(const SCConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:921
PcapLogData_::pkt_cnt
uint64_t pkt_cnt
Definition: log-pcap.c:149
HONOR_PASS_RULES_DISABLED
#define HONOR_PASS_RULES_DISABLED
Definition: log-pcap.c:81
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:252
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
PcapLogData_::use_stream_depth
int use_stream_depth
Definition: log-pcap.c:145
PcapLogCompressionData_::format
enum PcapLogCompressionFormat format
Definition: log-pcap.c:125
SCConfValIsFalse
int SCConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:602
PcapLogProfileData
struct PcapLogProfileData_ PcapLogProfileData
stream-tcp-reassemble.h
p
Packet * p
Definition: fuzz_dataset.c:30
PcapLogData_::profile_handles
PcapLogProfileData profile_handles
Definition: log-pcap.c:169
LOGMODE_MULTI
#define LOGMODE_MULTI
Definition: log-pcap.c:64
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
StatsCounterId
Definition: counters.h:30
PcapLogRegister
void PcapLogRegister(void)
Definition: log-pcap.c:219
SCConfNodeLookupChildValue
const char * SCConfNodeLookupChildValue(const SCConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:877
SCFmemopen
#define SCFmemopen
Definition: util-fmemopen.h:52
PcapLogCompressionData_::buffer
uint8_t * buffer
Definition: log-pcap.c:126
DEFAULT_LIMIT
#define DEFAULT_LIMIT
Definition: log-pcap.c:60
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:82
PcapLogData_::pcap_dead_handle
pcap_t * pcap_dead_handle
Definition: log-pcap.c:156
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:637
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
PcapLogData_::size_current
uint64_t size_current
Definition: log-pcap.c:154
PcapLogData_::bpf_filter
char * bpf_filter
Definition: log-pcap.c:147
PcapLogData_::profile_rotate
PcapLogProfileData profile_rotate
Definition: log-pcap.c:172
OutputCtx_::data
void * data
Definition: tm-modules.h:91
PcapLogCompressionData_
Definition: log-pcap.c:124
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:81
PCAPLOG_PROFILE_END
#define PCAPLOG_PROFILE_END(prof)
Definition: log-pcap.c:238
PcapLogGetFilename
char * PcapLogGetFilename(void)
Definition: log-pcap.c:1849
OutputCtx_
Definition: tm-modules.h:88
LogModeConditionalType_
LogModeConditionalType_
Definition: log-pcap.c:66
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
PcapLogCompressionData_::bytes_in_block
uint64_t bytes_in_block
Definition: log-pcap.c:136
TAILQ_ENTRY
#define TAILQ_ENTRY(type)
Definition: queue.h:239
Packet_::datalink
int datalink
Definition: decode.h:652
PcapLogData_::is_private
bool is_private
Definition: log-pcap.c:162
PcapLogData_::size_limit
uint64_t size_limit
Definition: log-pcap.c:155
PcapLogProfileData_::cnt
uint64_t cnt
Definition: log-pcap.c:113
PcapLogData_::max_files
uint32_t max_files
Definition: log-pcap.c:161
PcapLogCompressionData_::file
FILE * file
Definition: log-pcap.c:133
TAILQ_REMOVE
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:312
SCRunmodeGet
SCRunMode SCRunmodeGet(void)
Get the current run mode.
Definition: suricata.c:302
decode.h
TAILQ_FIRST
#define TAILQ_FIRST(head)
Definition: queue.h:250
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:47
PCAP_LOG_COMPRESSION_FORMAT_NONE
@ PCAP_LOG_COMPRESSION_FORMAT_NONE
Definition: log-pcap.c:120
strlcat
size_t strlcat(char *, const char *src, size_t siz)
Definition: util-strlcatu.c:45
util-cpu.h
PCAP_NETMASK_UNKNOWN
#define PCAP_NETMASK_UNKNOWN
Definition: log-pcap.c:90
Packet_::ts
SCTime_t ts
Definition: decode.h:570
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:120
TcpSegmentPcapHdrStorage_::ts
SCTime_t ts
Definition: stream-tcp-private.h:66
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1313
PCAP_LOG_COMPRESSION_FORMAT_LZ4
@ PCAP_LOG_COMPRESSION_FORMAT_LZ4
Definition: log-pcap.c:121
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
STREAM_DUMP_HEADERS
#define STREAM_DUMP_HEADERS
Definition: stream.h:34
GET_PKT_DATA
#define GET_PKT_DATA(p)
Definition: decode.h:210
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
PcapLogProfileData_::total
uint64_t total
Definition: log-pcap.c:112
StatsCounterIncr
void StatsCounterIncr(StatsThreadContext *stats, StatsCounterId id)
Increments the local counter.
Definition: counters.c:163
PCAP_PKTHDR_SIZE
#define PCAP_PKTHDR_SIZE
Definition: log-pcap.c:86
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:268
OutputInitResult_::ok
bool ok
Definition: output.h:48
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:262
PcapFileName_::dirname
char * dirname
Definition: log-pcap.c:97
PcapLogData_::file_cnt
uint32_t file_cnt
Definition: log-pcap.c:160
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:322
SCTIME_TO_TIMEVAL
#define SCTIME_TO_TIMEVAL(tv, t)
Definition: util-time.h:97
PCAP_OUTPUT_BUFFER_SIZE
#define PCAP_OUTPUT_BUFFER_SIZE
Definition: log-pcap.h:31
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
Packet_
Definition: decode.h:516
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:267
GET_PKT_LEN
#define GET_PKT_LEN(p)
Definition: decode.h:209
TimeGet
SCTime_t TimeGet(void)
Definition: util-time.c:152
SCConfGetChildValueInt
int SCConfGetChildValueInt(const SCConfNode *base, const char *name, intmax_t *val)
Definition: conf.c:475
SCConfigGetLogDirectory
const char * SCConfigGetLogDirectory(void)
Definition: util-conf.c:38
DEFAULT_FILE_LIMIT
#define DEFAULT_FILE_LIMIT
Definition: log-pcap.c:61
SCTime_t
Definition: util-time.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:80
RING_BUFFER_MODE_DISABLED
#define RING_BUFFER_MODE_DISABLED
Definition: log-pcap.c:72
PcapLogThreadData_::buf
MemBuffer * buf
Definition: log-pcap.c:195
PCAP_SNAPLEN
#define PCAP_SNAPLEN
Definition: log-pcap.c:84
PcapLogData_::plog_lock
SCMutex plog_lock
Definition: log-pcap.c:148
PcapLogData_::h
struct pcap_pkthdr * h
Definition: log-pcap.c:150
PcapLogCompressionData_::pcap_buf
uint8_t * pcap_buf
Definition: log-pcap.c:134
MemBuffer_
Definition: util-buffer.h:27
LOGMODE_COND_TAG
@ LOGMODE_COND_TAG
Definition: log-pcap.c:69
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:232
StreamSegmentForSession
int StreamSegmentForSession(const Packet *p, uint8_t flag, StreamSegmentCallback CallbackFunc, void *data)
Run callback for all segments on both directions of the session.
Definition: stream.c:64
PcapLogData_::prev_day
int prev_day
Definition: log-pcap.c:153
PKT_FIRST_TAG
#define PKT_FIRST_TAG
Definition: decode.h:1361
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
PCAPLOG_PROFILE_START
#define PCAPLOG_PROFILE_START
Definition: log-pcap.c:235
log-pcap.h
SCGetThreadIdLong
#define SCGetThreadIdLong(...)
Definition: threads.h:256
PcapLogData_::honor_pass_rules
int honor_pass_rules
Definition: log-pcap.c:146
pcap_file_thread
thread_local char * pcap_file_thread
Definition: log-pcap.c:109
PcapLogData_::profile_open
PcapLogProfileData profile_open
Definition: log-pcap.c:171
PcapLogData_::profile_lock
PcapLogProfileData profile_lock
Definition: log-pcap.c:166
OutputInitResult_
Definition: output.h:46
util-conf.h
Packet_::flow
struct Flow_ * flow
Definition: decode.h:564
StringParseUint64
int StringParseUint64(uint64_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:263
TimeDifferenceMicros
uint64_t TimeDifferenceMicros(struct timeval t0, struct timeval t1)
Definition: util-time.c:636
LOGGER_PCAP
@ LOGGER_PCAP
Definition: suricata-common.h:511
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:94
util-path.h
PcapLogCallbackContext::tv
ThreadVars * tv
Definition: log-pcap.c:562
LOGMODE_NORMAL
#define LOGMODE_NORMAL
Definition: log-pcap.c:63
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:86
PKT_STREAM_NOPCAPLOG
#define PKT_STREAM_NOPCAPLOG
Definition: decode.h:1322
PcapLogData_::profile_data_size
uint64_t profile_data_size
Definition: log-pcap.c:159
OutputPacketLoggerFunctions_::LogFunc
PacketLogger LogFunc
Definition: output.h:87
SCTIME_SECS
#define SCTIME_SECS(t)
Definition: util-time.h:57
OutputRegisterPacketModule
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, OutputPacketLoggerFunctions *output_module_functions)
Register a packet output module.
Definition: output.c:193
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
PcapLogProfileSetup
void PcapLogProfileSetup(void)
Definition: log-pcap.c:1973
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(...)
Definition: util-debug.h:517
PcapLogCallbackContext
Definition: log-pcap.c:561
PcapLogCompressionData_::buffer_size
uint64_t buffer_size
Definition: log-pcap.c:127
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:34
PcapLogThreadData_
Definition: log-pcap.c:193
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
PcapLogData_::bpfp
struct bpf_program * bpfp
Definition: log-pcap.c:158
LOGMODE_COND_ALERTS
@ LOGMODE_COND_ALERTS
Definition: log-pcap.c:68
PcapFileName_::filename
char * filename
Definition: log-pcap.c:96
Packet_::root
struct Packet_ * root
Definition: decode.h:666
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
str
#define str(s)
Definition: suricata-common.h:313
MAX_TOKS
#define MAX_TOKS
Definition: log-pcap.c:116
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:183
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:274
PcapLogCompressionData
struct PcapLogCompressionData_ PcapLogCompressionData
MODULE_NAME
#define MODULE_NAME
Definition: log-pcap.c:58
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint32_t, thread_cnt)
MEMBUFFER_SIZE
#define MEMBUFFER_SIZE(mem_buffer)
Get the MemBuffers current size.
Definition: util-buffer.h:61
src
uint16_t src
Definition: app-layer-dnp3.h:5
util-buffer.h
PcapFileName_
Definition: log-pcap.c:95
TAILQ_HEAD
#define TAILQ_HEAD(name, type)
Definition: queue.h:230
PcapLogData_::profile_unlock
PcapLogProfileData profile_unlock
Definition: log-pcap.c:168
USE_STREAM_DEPTH_ENABLED
#define USE_STREAM_DEPTH_ENABLED
Definition: log-pcap.c:79
PcapLogThreadData_::counter_filtered_bpf
StatsCounterId counter_filtered_bpf
Definition: log-pcap.c:198
MemBufferWriteRaw
uint32_t MemBufferWriteRaw(MemBuffer *dst, const uint8_t *raw, const uint32_t raw_len)
Write a raw buffer to the MemBuffer dst.
Definition: util-buffer.c:115
PcapLogData_::profile_close
PcapLogProfileData profile_close
Definition: log-pcap.c:170
PcapLogCallbackContext::td
PcapLogThreadData * td
Definition: log-pcap.c:563
PKT_NOPACKET_INSPECTION
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:1292
FlowHasAlerts
int FlowHasAlerts(const Flow *f)
Check if flow has alerts.
Definition: flow.c:162
PcapFileName
struct PcapFileName_ PcapFileName
PCAP_BUFFER_TIMEOUT
#define PCAP_BUFFER_TIMEOUT
Definition: log-pcap.c:85
MIN_LIMIT
#define MIN_LIMIT
Definition: log-pcap.c:59
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
util-misc.h
PKT_HAS_TAG
#define PKT_HAS_TAG
Definition: decode.h:1303
PcapLogData_
Definition: log-pcap.c:144
PcapLogThreadData_::pcap_log
PcapLogData * pcap_log
Definition: log-pcap.c:194
PKT_FIRST_ALERTS
#define PKT_FIRST_ALERTS
Definition: decode.h:1360
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:250
TAILQ_INSERT_BEFORE
#define TAILQ_INSERT_BEFORE(listelm, elm, field)
Definition: queue.h:277
PcapLogData_::profile_write
PcapLogProfileData profile_write
Definition: log-pcap.c:167
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
ThreadVars_::stats
StatsThreadContext stats
Definition: threadvars.h:120
SCConfNode_
Definition: conf.h:37
TS_FORMAT_USEC
#define TS_FORMAT_USEC
Definition: log-pcap.c:76
OutputPacketLoggerFunctions_
Definition: output.h:86
SCMutex
#define SCMutex
Definition: threads-debug.h:114
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:109
MAX_FILENAMELEN
#define MAX_FILENAMELEN
Definition: log-pcap.c:117
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
output.h
LogModeConditionalType
enum LogModeConditionalType_ LogModeConditionalType
PcapLogCompressionFormat
PcapLogCompressionFormat
Definition: log-pcap.c:119
SCTIME_USECS
#define SCTIME_USECS(t)
Definition: util-time.h:56