suricata
log-pcap.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author William Metcalf <William.Metcalf@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Pcap packet logging module.
25  */
26 
27 #include "suricata-common.h"
28 #ifdef HAVE_LIBLZ4
29 #include <lz4frame.h>
30 #include "util-fmemopen.h"
31 #endif /* HAVE_LIBLZ4 */
32 
33 #if defined(HAVE_DIRENT_H) && defined(HAVE_FNMATCH_H)
34 #define INIT_RING_BUFFER
35 #include <dirent.h>
36 #include <fnmatch.h>
37 #endif
38 
39 #include "log-pcap.h"
40 
41 #include "threads.h"
42 #include "threadvars.h"
43 #include "decode.h"
44 #include "stream.h"
45 #include "stream-tcp-reassemble.h"
46 
47 #include "output.h"
48 
49 #include "util-buffer.h"
50 #include "util-byte.h"
51 #include "util-conf.h"
52 #include "util-cpu.h"
53 #include "util-datalink.h"
54 #include "util-misc.h"
55 #include "util-path.h"
56 #include "util-profiling.h"
57 #include "util-time.h"
58 
59 #define DEFAULT_LOG_FILENAME "pcaplog"
60 #define MODULE_NAME "PcapLog"
61 #define MIN_LIMIT 4 * 1024 * 1024
62 #define DEFAULT_LIMIT 100 * 1024 * 1024
63 #define DEFAULT_FILE_LIMIT 0
64 
65 #define LOGMODE_NORMAL 0
66 #define LOGMODE_SGUIL 1
67 #define LOGMODE_MULTI 2
68 
74 
75 #define RING_BUFFER_MODE_DISABLED 0
76 #define RING_BUFFER_MODE_ENABLED 1
77 
78 #define TS_FORMAT_SEC 0
79 #define TS_FORMAT_USEC 1
80 
81 #define USE_STREAM_DEPTH_DISABLED 0
82 #define USE_STREAM_DEPTH_ENABLED 1
83 
84 #define HONOR_PASS_RULES_DISABLED 0
85 #define HONOR_PASS_RULES_ENABLED 1
86 
87 #define PCAP_SNAPLEN 262144
88 #define PCAP_BUFFER_TIMEOUT 1000000 // microseconds
89 
90 SC_ATOMIC_DECLARE(uint32_t, thread_cnt);
91 
92 typedef struct PcapFileName_ {
93  char *filename;
94  char *dirname;
95 
96  /* Like a struct timeval, but with fixed size. This is only used when
97  * seeding the ring buffer on start. */
98  struct {
99  uint64_t secs;
100  uint32_t usecs;
101  };
102 
103  TAILQ_ENTRY(PcapFileName_) next; /**< Pointer to next Pcap File for tailq. */
105 
106 thread_local char *pcap_file_thread = NULL;
107 
108 typedef struct PcapLogProfileData_ {
109  uint64_t total;
110  uint64_t cnt;
112 
113 #define MAX_TOKS 9
114 #define MAX_FILENAMELEN 513
115 
119 };
120 
121 typedef struct PcapLogCompressionData_ {
123  uint8_t *buffer;
124  uint64_t buffer_size;
125 #ifdef HAVE_LIBLZ4
126  LZ4F_compressionContext_t lz4f_context;
127  LZ4F_preferences_t lz4f_prefs;
128 #endif /* HAVE_LIBLZ4 */
129  FILE *file;
130  uint8_t *pcap_buf;
131  uint64_t pcap_buf_size;
133  uint64_t bytes_in_block;
135 
136 /**
137  * PcapLog thread vars
138  *
139  * Used for storing file options.
140  */
141 typedef struct PcapLogData_ {
142  int use_stream_depth; /**< use stream depth i.e. ignore packets that reach limit */
143  int honor_pass_rules; /**< don't log if pass rules have matched */
144  int is_private; /**< TRUE if ctx is thread local */
146  uint64_t pkt_cnt; /**< total number of packets */
147  struct pcap_pkthdr *h; /**< pcap header struct */
148  char *filename; /**< current filename */
149  int mode; /**< normal or sguil */
150  int prev_day; /**< last day, for finding out when */
151  uint64_t size_current; /**< file current size */
152  uint64_t size_limit; /**< file size limit */
153  pcap_t *pcap_dead_handle; /**< pcap_dumper_t needs a handle */
154  pcap_dumper_t *pcap_dumper; /**< actually writes the packets */
155  uint64_t profile_data_size; /**< track in bytes how many bytes we wrote */
156  uint32_t file_cnt; /**< count of pcap files we currently have */
157  uint32_t max_files; /**< maximum files to use in ring buffer mode */
159  conditional; /**< log all packets or just packets and flows with alerts */
160 
168 
169  TAILQ_HEAD(, PcapFileName_) pcap_file_list;
170 
171  uint32_t thread_number; /**< thread number, first thread is 1, second 2, etc */
172  int use_ringbuffer; /**< ring buffer mode enabled or disabled */
173  int timestamp_format; /**< timestamp format sec or usec */
174  char *prefix; /**< filename prefix */
175  const char *suffix; /**< filename suffix */
176  char dir[PATH_MAX]; /**< pcap log directory */
177  int reported;
178  int threads; /**< number of threads (only set in the global) */
179  char *filename_parts[MAX_TOKS];
180  int filename_part_cnt;
181  struct timeval last_pcap_dump;
182  int fopen_err; /**< set to the last fopen error */
183  bool pcap_open_err; /**< true if the last pcap open errored */
184 
185  PcapLogCompressionData compression;
187 
188 typedef struct PcapLogThreadData_ {
192 
193 /* Pattern for extracting timestamp from pcap log files. */
194 static const char timestamp_pattern[] = ".*?(\\d+)(\\.(\\d+))?";
195 static pcre2_code *pcre_timestamp_code = NULL;
196 static pcre2_match_data *pcre_timestamp_match = NULL;
197 
198 /* global pcap data for when we're using multi mode. At exit we'll
199  * merge counters into this one and then report counters. */
200 static PcapLogData *g_pcap_data = NULL;
201 
202 static int PcapLogOpenFileCtx(PcapLogData *);
203 static int PcapLog(ThreadVars *, void *, const Packet *);
204 static TmEcode PcapLogDataInit(ThreadVars *, const void *, void **);
205 static TmEcode PcapLogDataDeinit(ThreadVars *, void *);
206 static void PcapLogFileDeInitCtx(OutputCtx *);
207 static OutputInitResult PcapLogInitCtx(ConfNode *);
208 static void PcapLogProfilingDump(PcapLogData *);
209 static int PcapLogCondition(ThreadVars *, void *, const Packet *);
210 
211 void PcapLogRegister(void)
212 {
214  PcapLogInitCtx, PcapLog, PcapLogCondition, PcapLogDataInit,
215  PcapLogDataDeinit, NULL);
217  SC_ATOMIC_INIT(thread_cnt);
218  SC_ATOMIC_SET(thread_cnt, 1); /* first id is 1 */
219  return;
220 }
221 
222 #define PCAPLOG_PROFILE_START \
223  uint64_t pcaplog_profile_ticks = UtilCpuGetTicks()
224 
225 #define PCAPLOG_PROFILE_END(prof) \
226  (prof).total += (UtilCpuGetTicks() - pcaplog_profile_ticks); \
227  (prof).cnt++
228 
229 static int PcapLogCondition(ThreadVars *tv, void *thread_data, const Packet *p)
230 {
231  PcapLogThreadData *ptd = (PcapLogThreadData *)thread_data;
232 
233  /* Log alerted flow or tagged flow */
234  switch (ptd->pcap_log->conditional) {
235  case LOGMODE_COND_ALL:
236  break;
237  case LOGMODE_COND_ALERTS:
238  if (p->alerts.cnt || (p->flow && FlowHasAlerts(p->flow))) {
239  return TRUE;
240  } else {
241  return FALSE;
242  }
243  break;
244  case LOGMODE_COND_TAG:
245  if (p->flags & (PKT_HAS_TAG | PKT_FIRST_TAG)) {
246  return TRUE;
247  } else {
248  return FALSE;
249  }
250  break;
251  }
252 
253  if (p->flags & PKT_PSEUDO_STREAM_END) {
254  return FALSE;
255  }
256 
257  if (IS_TUNNEL_PKT(p) && !IS_TUNNEL_ROOT_PKT(p)) {
258  return FALSE;
259  }
260  return TRUE;
261 }
262 
263 /**
264  * \brief Function to close pcaplog file
265  *
266  * \param t Thread Variable containing input/output queue, cpu affinity etc.
267  * \param pl PcapLog thread variable.
268  */
269 static int PcapLogCloseFile(ThreadVars *t, PcapLogData *pl)
270 {
271  if (pl != NULL) {
273 
274  if (pl->pcap_dumper != NULL) {
275  pcap_dump_close(pl->pcap_dumper);
276 #ifdef HAVE_LIBLZ4
277  PcapLogCompressionData *comp = &pl->compression;
279  /* pcap_dump_close() has closed its output ``file'',
280  * so we need to call fmemopen again. */
281 
282  comp->pcap_buf_wrapper = SCFmemopen(comp->pcap_buf,
283  comp->pcap_buf_size, "w");
284  if (comp->pcap_buf_wrapper == NULL) {
285  SCLogError("SCFmemopen failed: %s", strerror(errno));
286  return TM_ECODE_FAILED;
287  }
288  }
289 #endif /* HAVE_LIBLZ4 */
290  }
291  pl->size_current = 0;
292  pl->pcap_dumper = NULL;
293 
294  if (pl->pcap_dead_handle != NULL)
295  pcap_close(pl->pcap_dead_handle);
296  pl->pcap_dead_handle = NULL;
297 
298 #ifdef HAVE_LIBLZ4
299  PcapLogCompressionData *comp = &pl->compression;
301  /* pcap_dump_close did not write any data because we call
302  * pcap_dump_flush() after every write when writing
303  * compressed output. */
304  uint64_t bytes_written = LZ4F_compressEnd(comp->lz4f_context,
305  comp->buffer, comp->buffer_size, NULL);
306  if (LZ4F_isError(bytes_written)) {
307  SCLogError("LZ4F_compressEnd: %s", LZ4F_getErrorName(bytes_written));
308  return TM_ECODE_FAILED;
309  }
310  if (fwrite(comp->buffer, 1, bytes_written, comp->file) < bytes_written) {
311  SCLogError("fwrite failed: %s", strerror(errno));
312  return TM_ECODE_FAILED;
313  }
314  fclose(comp->file);
315  comp->bytes_in_block = 0;
316  }
317 #endif /* HAVE_LIBLZ4 */
318 
320  }
321 
322  return 0;
323 }
324 
325 static void PcapFileNameFree(PcapFileName *pf)
326 {
327  if (pf != NULL) {
328  if (pf->filename != NULL) {
329  SCFree(pf->filename);
330  }
331  if (pf->dirname != NULL) {
332  SCFree(pf->dirname);
333  }
334  SCFree(pf);
335  }
336 
337  return;
338 }
339 
340 /**
341  * \brief Function to rotate pcaplog file
342  *
343  * \param t Thread Variable containing input/output queue, cpu affinity etc.
344  * \param pl PcapLog thread variable.
345  *
346  * \retval 0 on success
347  * \retval -1 on failure
348  */
349 static int PcapLogRotateFile(ThreadVars *t, PcapLogData *pl)
350 {
351  PcapFileName *pf;
352  PcapFileName *pfnext;
353 
355 
356  if (PcapLogCloseFile(t,pl) < 0) {
357  SCLogDebug("PcapLogCloseFile failed");
358  return -1;
359  }
360 
361  if (pl->use_ringbuffer == RING_BUFFER_MODE_ENABLED && pl->file_cnt >= pl->max_files) {
362  pf = TAILQ_FIRST(&pl->pcap_file_list);
363  SCLogDebug("Removing pcap file %s", pf->filename);
364 
365  if (remove(pf->filename) != 0) {
366  // VJ remove can fail because file is already gone
367  // SCLogWarning("failed to remove log file %s: %s",
368  // pf->filename, strerror( errno ));
369  }
370 
371  /* Remove directory if Sguil mode and no files left in sguil dir */
372  if (pl->mode == LOGMODE_SGUIL) {
373  pfnext = TAILQ_NEXT(pf,next);
374 
375  if (strcmp(pf->dirname, pfnext->dirname) == 0) {
376  SCLogDebug("Current entry dir %s and next entry %s "
377  "are equal: not removing dir",
378  pf->dirname, pfnext->dirname);
379  } else {
380  SCLogDebug("current entry %s and %s are "
381  "not equal: removing dir",
382  pf->dirname, pfnext->dirname);
383 
384  if (remove(pf->dirname) != 0) {
385  SCLogWarning("failed to remove sguil log %s: %s", pf->dirname, strerror(errno));
386  }
387  }
388  }
389 
390  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
391  PcapFileNameFree(pf);
392  pl->file_cnt--;
393  }
394 
395  if (PcapLogOpenFileCtx(pl) < 0) {
396  SCLogError("opening new pcap log file failed");
397  return -1;
398  }
399  pl->file_cnt++;
400  SCLogDebug("file_cnt %u", pl->file_cnt);
401 
403  return 0;
404 }
405 
406 static int PcapLogOpenHandles(PcapLogData *pl, const Packet *p)
407 {
409 
410  int datalink = p->datalink;
411  if (IS_TUNNEL_PKT(p) && !IS_TUNNEL_ROOT_PKT(p)) {
412  Packet *real_p = p->root;
413  datalink = real_p->datalink;
414  }
415  if (pl->pcap_dead_handle == NULL) {
416  SCLogDebug("Setting pcap-log link type to %u", datalink);
417  if ((pl->pcap_dead_handle = pcap_open_dead(datalink, PCAP_SNAPLEN)) == NULL) {
418  SCLogDebug("Error opening dead pcap handle");
419  return TM_ECODE_FAILED;
420  }
421  }
422 
423  if (pl->pcap_dumper == NULL) {
424  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_NONE) {
425  if ((pl->pcap_dumper = pcap_dump_open(pl->pcap_dead_handle,
426  pl->filename)) == NULL) {
427  if (!pl->pcap_open_err) {
428  SCLogError("Error opening dump file %s", pcap_geterr(pl->pcap_dead_handle));
429  pl->pcap_open_err = true;
430  }
431  return TM_ECODE_FAILED;
432  } else {
433  pl->pcap_open_err = false;
434  }
435  }
436 #ifdef HAVE_LIBLZ4
437  else if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
438  PcapLogCompressionData *comp = &pl->compression;
439 
440  comp->file = fopen(pl->filename, "w");
441  if (comp->file == NULL) {
442  if (errno != pl->fopen_err) {
443  SCLogError("Error opening file for compressed output: %s", strerror(errno));
444  pl->fopen_err = errno;
445  }
446  return TM_ECODE_FAILED;
447  } else {
448  pl->fopen_err = 0;
449  }
450 
451  if ((pl->pcap_dumper = pcap_dump_fopen(pl->pcap_dead_handle, comp->pcap_buf_wrapper)) ==
452  NULL) {
453  if (!pl->pcap_open_err) {
454  SCLogError("Error opening dump file %s", pcap_geterr(pl->pcap_dead_handle));
455  pl->pcap_open_err = true;
456  }
457  fclose(comp->file);
458  comp->file = NULL;
459  return TM_ECODE_FAILED;
460  } else {
461  pl->pcap_open_err = false;
462  }
463 
464  uint64_t bytes_written = LZ4F_compressBegin(comp->lz4f_context,
465  comp->buffer, comp->buffer_size, NULL);
466  if (LZ4F_isError(bytes_written)) {
467  SCLogError("LZ4F_compressBegin: %s", LZ4F_getErrorName(bytes_written));
468  return TM_ECODE_FAILED;
469  }
470  if (fwrite(comp->buffer, 1, bytes_written, comp->file) < bytes_written) {
471  SCLogError("fwrite failed: %s", strerror(errno));
472  return TM_ECODE_FAILED;
473  }
474  }
475 #endif /* HAVE_LIBLZ4 */
476  }
477 
479  return TM_ECODE_OK;
480 }
481 
482 /** \internal
483  * \brief lock wrapper for main PcapLog() function
484  * NOTE: only meant for use in main PcapLog() function.
485  */
486 static void PcapLogLock(PcapLogData *pl)
487 {
488  if (!(pl->is_private)) {
490  SCMutexLock(&pl->plog_lock);
492  }
493 }
494 
495 /** \internal
496  * \brief unlock wrapper for main PcapLog() function
497  * NOTE: only meant for use in main PcapLog() function.
498  */
499 static void PcapLogUnlock(PcapLogData *pl)
500 {
501  if (!(pl->is_private)) {
503  SCMutexUnlock(&pl->plog_lock);
505  }
506 }
507 
508 static inline int PcapWrite(
509  PcapLogData *pl, PcapLogCompressionData *comp, uint8_t *data, size_t len)
510 {
511  struct timeval current_dump;
512  gettimeofday(&current_dump, NULL);
513  pcap_dump((u_char *)pl->pcap_dumper, pl->h, data);
514  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_NONE) {
515  pl->size_current += len;
516  }
517 #ifdef HAVE_LIBLZ4
518  else if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
519  pcap_dump_flush(pl->pcap_dumper);
520  long in_size = ftell(comp->pcap_buf_wrapper);
521  if (in_size < 0) {
522  SCLogError("ftell failed with: %s", strerror(errno));
523  return TM_ECODE_FAILED;
524  }
525  uint64_t out_size = LZ4F_compressUpdate(comp->lz4f_context, comp->buffer, comp->buffer_size,
526  comp->pcap_buf, (uint64_t)in_size, NULL);
527  if (LZ4F_isError(len)) {
528  SCLogError("LZ4F_compressUpdate: %s", LZ4F_getErrorName(len));
529  return TM_ECODE_FAILED;
530  }
531  if (fseek(pl->compression.pcap_buf_wrapper, 0, SEEK_SET) != 0) {
532  SCLogError("fseek failed: %s", strerror(errno));
533  return TM_ECODE_FAILED;
534  }
535  if (fwrite(comp->buffer, 1, out_size, comp->file) < out_size) {
536  SCLogError("fwrite failed: %s", strerror(errno));
537  return TM_ECODE_FAILED;
538  }
539  if (out_size > 0) {
540  pl->size_current += out_size;
541  comp->bytes_in_block = len;
542  } else {
543  comp->bytes_in_block += len;
544  }
545  }
546 #endif /* HAVE_LIBLZ4 */
547  if (TimeDifferenceMicros(pl->last_pcap_dump, current_dump) >= PCAP_BUFFER_TIMEOUT) {
548  pcap_dump_flush(pl->pcap_dumper);
549  }
550  pl->last_pcap_dump = current_dump;
551  return TM_ECODE_OK;
552 }
553 
558 };
559 
560 static int PcapLogSegmentCallback(
561  const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
562 {
563  struct PcapLogCallbackContext *pctx = (struct PcapLogCallbackContext *)data;
564 
565  if (seg->pcap_hdr_storage->pktlen) {
566  pctx->pl->h->ts.tv_sec = seg->pcap_hdr_storage->ts.tv_sec;
567  pctx->pl->h->ts.tv_usec = seg->pcap_hdr_storage->ts.tv_usec;
568  pctx->pl->h->len = seg->pcap_hdr_storage->pktlen + buflen;
569  pctx->pl->h->caplen = seg->pcap_hdr_storage->pktlen + buflen;
570  MemBufferReset(pctx->buf);
572  MemBufferWriteRaw(pctx->buf, buf, buflen);
573 
574  PcapWrite(pctx->pl, pctx->connp, (uint8_t *)pctx->buf->buffer, pctx->pl->h->len);
575  }
576  return 1;
577 }
578 
579 static void PcapLogDumpSegments(
581 {
582  uint8_t flag;
583  flag = STREAM_DUMP_HEADERS;
584 
585  /* Loop on segment from this side */
586  struct PcapLogCallbackContext data = { td->pcap_log, connp, td->buf };
587  StreamSegmentForSession(p, flag, PcapLogSegmentCallback, (void *)&data);
588 }
589 
590 /**
591  * \brief Pcap logging main function
592  *
593  * \param t threadvar
594  * \param p packet
595  * \param thread_data thread module specific data
596  *
597  * \retval TM_ECODE_OK on succes
598  * \retval TM_ECODE_FAILED on serious error
599  */
600 static int PcapLog (ThreadVars *t, void *thread_data, const Packet *p)
601 {
602  size_t len;
603  int rotate = 0;
604  int ret = 0;
605  Packet *rp = NULL;
606 
607  PcapLogThreadData *td = (PcapLogThreadData *)thread_data;
608  PcapLogData *pl = td->pcap_log;
609 
612  return TM_ECODE_OK;
613  }
614 
615  PcapLogLock(pl);
616 
617  pl->pkt_cnt++;
618  pl->h->ts.tv_sec = SCTIME_SECS(p->ts);
619  pl->h->ts.tv_usec = SCTIME_USECS(p->ts);
620  if (IS_TUNNEL_PKT(p) && !IS_TUNNEL_ROOT_PKT(p)) {
621  rp = p->root;
622  pl->h->caplen = GET_PKT_LEN(rp);
623  pl->h->len = GET_PKT_LEN(rp);
624  len = sizeof(*pl->h) + GET_PKT_LEN(rp);
625  } else {
626  pl->h->caplen = GET_PKT_LEN(p);
627  pl->h->len = GET_PKT_LEN(p);
628  len = sizeof(*pl->h) + GET_PKT_LEN(p);
629  }
630 
631  if (pl->filename == NULL) {
632  ret = PcapLogOpenFileCtx(pl);
633  if (ret < 0) {
634  PcapLogUnlock(pl);
635  return TM_ECODE_FAILED;
636  }
637  SCLogDebug("Opening PCAP log file %s", pl->filename);
638  }
639 
640  if (pl->mode == LOGMODE_SGUIL) {
641  struct tm local_tm;
642  struct tm *tms = SCLocalTime(SCTIME_SECS(p->ts), &local_tm);
643  if (tms->tm_mday != pl->prev_day) {
644  rotate = 1;
645  pl->prev_day = tms->tm_mday;
646  }
647  }
648 
649  PcapLogCompressionData *comp = &pl->compression;
651  if ((pl->size_current + len) > pl->size_limit || rotate) {
652  if (PcapLogRotateFile(t,pl) < 0) {
653  PcapLogUnlock(pl);
654  SCLogDebug("rotation of pcap failed");
655  return TM_ECODE_FAILED;
656  }
657  }
658  }
659 #ifdef HAVE_LIBLZ4
660  else if (comp->format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
661  /* When writing compressed pcap logs, we have no way of knowing
662  * for sure whether adding this packet would cause the current
663  * file to exceed the size limit. Thus, we record the number of
664  * bytes that have been fed into lz4 since the last write, and
665  * act as if they would be written uncompressed. */
666 
667  if ((pl->size_current + comp->bytes_in_block + len) > pl->size_limit ||
668  rotate) {
669  if (PcapLogRotateFile(t,pl) < 0) {
670  PcapLogUnlock(pl);
671  SCLogDebug("rotation of pcap failed");
672  return TM_ECODE_FAILED;
673  }
674  }
675  }
676 #endif /* HAVE_LIBLZ4 */
677 
678  /* XXX pcap handles, nfq, pfring, can only have one link type ipfw? we do
679  * this here as we don't know the link type until we get our first packet */
680  if (pl->pcap_dead_handle == NULL || pl->pcap_dumper == NULL) {
681  if (PcapLogOpenHandles(pl, p) != TM_ECODE_OK) {
682  PcapLogUnlock(pl);
683  return TM_ECODE_FAILED;
684  }
685  }
686 
688 
689  /* if we are using alerted logging and if packet is first one with alert in flow
690  * then we need to dump in the pcap the stream acked by the packet */
691  if ((p->flags & PKT_FIRST_ALERTS) && (td->pcap_log->conditional != LOGMODE_COND_ALL)) {
692  if (PKT_IS_TCP(p)) {
693  /* dump fake packets for all segments we have on acked by packet */
694 #ifdef HAVE_LIBLZ4
695  PcapLogDumpSegments(td, comp, p);
696 #else
697  PcapLogDumpSegments(td, NULL, p);
698 #endif
699  if (p->flags & PKT_PSEUDO_STREAM_END) {
700  PcapLogUnlock(pl);
701  return TM_ECODE_OK;
702  }
703 
704  /* PcapLogDumpSegment has written over the PcapLogData variables so need to update */
705  pl->h->ts.tv_sec = SCTIME_SECS(p->ts);
706  pl->h->ts.tv_usec = SCTIME_USECS(p->ts);
707  if (IS_TUNNEL_PKT(p) && !IS_TUNNEL_ROOT_PKT(p)) {
708  rp = p->root;
709  pl->h->caplen = GET_PKT_LEN(rp);
710  pl->h->len = GET_PKT_LEN(rp);
711  len = sizeof(*pl->h) + GET_PKT_LEN(rp);
712  } else {
713  pl->h->caplen = GET_PKT_LEN(p);
714  pl->h->len = GET_PKT_LEN(p);
715  len = sizeof(*pl->h) + GET_PKT_LEN(p);
716  }
717  }
718  }
719 
720  if (IS_TUNNEL_PKT(p) && !IS_TUNNEL_ROOT_PKT(p)) {
721  rp = p->root;
722 #ifdef HAVE_LIBLZ4
723  ret = PcapWrite(pl, comp, GET_PKT_DATA(rp), len);
724 #else
725  ret = PcapWrite(pl, NULL, GET_PKT_DATA(rp), len);
726 #endif
727  } else {
728 #ifdef HAVE_LIBLZ4
729  ret = PcapWrite(pl, comp, GET_PKT_DATA(p), len);
730 #else
731  ret = PcapWrite(pl, NULL, GET_PKT_DATA(p), len);
732 #endif
733  }
734  if (ret != TM_ECODE_OK) {
736  PcapLogUnlock(pl);
737  return ret;
738  }
739 
741  pl->profile_data_size += len;
742 
743  SCLogDebug("pl->size_current %"PRIu64", pl->size_limit %"PRIu64,
744  pl->size_current, pl->size_limit);
745 
746  PcapLogUnlock(pl);
747  return TM_ECODE_OK;
748 }
749 
750 static PcapLogData *PcapLogDataCopy(const PcapLogData *pl)
751 {
752  BUG_ON(pl->mode != LOGMODE_MULTI);
753  PcapLogData *copy = SCCalloc(1, sizeof(*copy));
754  if (unlikely(copy == NULL)) {
755  return NULL;
756  }
757 
758  copy->h = SCCalloc(1, sizeof(*copy->h));
759  if (unlikely(copy->h == NULL)) {
760  SCFree(copy);
761  return NULL;
762  }
763 
764  copy->prefix = SCStrdup(pl->prefix);
765  if (unlikely(copy->prefix == NULL)) {
766  SCFree(copy->h);
767  SCFree(copy);
768  return NULL;
769  }
770 
771  copy->suffix = pl->suffix;
772 
773  /* settings TODO move to global cfg struct */
774  copy->is_private = TRUE;
775  copy->mode = pl->mode;
776  copy->max_files = pl->max_files;
777  copy->use_ringbuffer = pl->use_ringbuffer;
778  copy->timestamp_format = pl->timestamp_format;
780  copy->size_limit = pl->size_limit;
781  copy->conditional = pl->conditional;
782 
783  const PcapLogCompressionData *comp = &pl->compression;
784  PcapLogCompressionData *copy_comp = &copy->compression;
785  copy_comp->format = comp->format;
786 #ifdef HAVE_LIBLZ4
788  /* We need to allocate a new compression context and buffers for
789  * the copy. First copy the things that can simply be copied. */
790 
791  copy_comp->buffer_size = comp->buffer_size;
792  copy_comp->pcap_buf_size = comp->pcap_buf_size;
793  copy_comp->lz4f_prefs = comp->lz4f_prefs;
794 
795  /* Allocate the buffers. */
796 
797  copy_comp->buffer = SCMalloc(copy_comp->buffer_size);
798  if (copy_comp->buffer == NULL) {
799  SCLogError("SCMalloc failed: %s", strerror(errno));
800  SCFree(copy->h);
801  SCFree(copy);
802  return NULL;
803  }
804  copy_comp->pcap_buf = SCMalloc(copy_comp->pcap_buf_size);
805  if (copy_comp->pcap_buf == NULL) {
806  SCLogError("SCMalloc failed: %s", strerror(errno));
807  SCFree(copy_comp->buffer);
808  SCFree(copy->h);
809  SCFree(copy);
810  return NULL;
811  }
812  copy_comp->pcap_buf_wrapper = SCFmemopen(copy_comp->pcap_buf,
813  copy_comp->pcap_buf_size, "w");
814  if (copy_comp->pcap_buf_wrapper == NULL) {
815  SCLogError("SCFmemopen failed: %s", strerror(errno));
816  SCFree(copy_comp->buffer);
817  SCFree(copy_comp->pcap_buf);
818  SCFree(copy->h);
819  SCFree(copy);
820  return NULL;
821  }
822 
823  /* Initialize a new compression context. */
824 
825  LZ4F_errorCode_t errcode =
826  LZ4F_createCompressionContext(&copy_comp->lz4f_context, 1);
827  if (LZ4F_isError(errcode)) {
828  SCLogError("LZ4F_createCompressionContext failed: %s", LZ4F_getErrorName(errcode));
829  fclose(copy_comp->pcap_buf_wrapper);
830  SCFree(copy_comp->buffer);
831  SCFree(copy_comp->pcap_buf);
832  SCFree(copy->h);
833  SCFree(copy);
834  return NULL;
835  }
836 
837  /* Initialize the rest. */
838 
839  copy_comp->file = NULL;
840  copy_comp->bytes_in_block = 0;
841  }
842 #endif /* HAVE_LIBLZ4 */
843 
844  TAILQ_INIT(&copy->pcap_file_list);
845  SCMutexInit(&copy->plog_lock, NULL);
846 
847  strlcpy(copy->dir, pl->dir, sizeof(copy->dir));
848 
849  int i;
850  for (i = 0; i < pl->filename_part_cnt && i < MAX_TOKS; i++)
851  copy->filename_parts[i] = pl->filename_parts[i];
852  copy->filename_part_cnt = pl->filename_part_cnt;
853 
854  /* set thread number, first thread is 1 */
855  copy->thread_number = SC_ATOMIC_ADD(thread_cnt, 1);
856 
857  SCLogDebug("copied, returning %p", copy);
858  return copy;
859 }
860 
861 #ifdef INIT_RING_BUFFER
862 static int PcapLogGetTimeOfFile(const char *filename, uint64_t *secs,
863  uint32_t *usecs)
864 {
865  char buf[PATH_MAX];
866  size_t copylen;
867 
868  int n = pcre2_match(pcre_timestamp_code, (PCRE2_SPTR8)filename, strlen(filename), 0, 0,
869  pcre_timestamp_match, NULL);
870  if (n != 2 && n != 4) {
871  /* No match. */
872  return 0;
873  }
874 
875  if (n >= 2) {
876  /* Extract seconds. */
877  copylen = sizeof(buf);
878  if (pcre2_substring_copy_bynumber(pcre_timestamp_match, 1, (PCRE2_UCHAR8 *)buf, &copylen) <
879  0) {
880  return 0;
881  }
882  if (StringParseUint64(secs, 10, 0, buf) < 0) {
883  return 0;
884  }
885  }
886  if (n == 4) {
887  /* Extract microseconds. */
888  copylen = sizeof(buf);
889  if (pcre2_substring_copy_bynumber(pcre_timestamp_match, 3, (PCRE2_UCHAR8 *)buf, &copylen) <
890  0) {
891  return 0;
892  }
893  if (StringParseUint32(usecs, 10, 0, buf) < 0) {
894  return 0;
895  }
896  }
897 
898  return 1;
899 }
900 
901 static TmEcode PcapLogInitRingBuffer(PcapLogData *pl)
902 {
903  char pattern[PATH_MAX];
904 
905  SCLogInfo("Initializing PCAP ring buffer for %s/%s.",
906  pl->dir, pl->prefix);
907 
908  strlcpy(pattern, pl->dir, PATH_MAX);
909  if (pattern[strlen(pattern) - 1] != '/') {
910  strlcat(pattern, "/", PATH_MAX);
911  }
912  if (pl->mode == LOGMODE_MULTI) {
913  for (int i = 0; i < pl->filename_part_cnt; i++) {
914  char *part = pl->filename_parts[i];
915  if (part == NULL || strlen(part) == 0) {
916  continue;
917  }
918  if (part[0] != '%' || strlen(part) < 2) {
919  strlcat(pattern, part, PATH_MAX);
920  continue;
921  }
922  switch (part[1]) {
923  case 'i':
924  SCLogError("Thread ID not allowed in ring buffer mode.");
925  return TM_ECODE_FAILED;
926  case 'n': {
927  char tmp[PATH_MAX];
928  snprintf(tmp, PATH_MAX, "%"PRIu32, pl->thread_number);
929  strlcat(pattern, tmp, PATH_MAX);
930  break;
931  }
932  case 't':
933  strlcat(pattern, "*", PATH_MAX);
934  break;
935  default:
936  SCLogError("Unsupported format character: %%%s", part);
937  return TM_ECODE_FAILED;
938  }
939  }
940  } else {
941  strlcat(pattern, pl->prefix, PATH_MAX);
942  strlcat(pattern, ".*", PATH_MAX);
943  }
944  strlcat(pattern, pl->suffix, PATH_MAX);
945 
946  char *basename = strrchr(pattern, '/');
947  *basename++ = '\0';
948 
949  /* Pattern is now just the directory name. */
950  DIR *dir = opendir(pattern);
951  if (dir == NULL) {
952  SCLogWarning("Failed to open directory %s: %s", pattern, strerror(errno));
953  return TM_ECODE_FAILED;
954  }
955 
956  for (;;) {
957  struct dirent *entry = readdir(dir);
958  if (entry == NULL) {
959  break;
960  }
961  if (fnmatch(basename, entry->d_name, 0) != 0) {
962  continue;
963  }
964 
965  uint64_t secs = 0;
966  uint32_t usecs = 0;
967 
968  if (!PcapLogGetTimeOfFile(entry->d_name, &secs, &usecs)) {
969  /* Failed to get time stamp out of file name. Not necessarily a
970  * failure as the file might just not be a pcap log file. */
971  continue;
972  }
973 
974  PcapFileName *pf = SCCalloc(sizeof(*pf), 1);
975  if (unlikely(pf == NULL)) {
976  goto fail;
977  }
978  char path[PATH_MAX];
979  if (snprintf(path, PATH_MAX, "%s/%s", pattern, entry->d_name) == PATH_MAX)
980  goto fail;
981 
982  if ((pf->filename = SCStrdup(path)) == NULL) {
983  goto fail;
984  }
985  if ((pf->dirname = SCStrdup(pattern)) == NULL) {
986  goto fail;
987  }
988  pf->secs = secs;
989  pf->usecs = usecs;
990 
991  if (TAILQ_EMPTY(&pl->pcap_file_list)) {
992  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
993  } else {
994  /* Ordered insert. */
995  PcapFileName *it = NULL;
996  TAILQ_FOREACH(it, &pl->pcap_file_list, next) {
997  if (pf->secs < it->secs) {
998  break;
999  } else if (pf->secs == it->secs && pf->usecs < it->usecs) {
1000  break;
1001  }
1002  }
1003  if (it == NULL) {
1004  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
1005  } else {
1006  TAILQ_INSERT_BEFORE(it, pf, next);
1007  }
1008  }
1009  pl->file_cnt++;
1010  continue;
1011 
1012  fail:
1013  if (pf != NULL) {
1014  if (pf->filename != NULL) {
1015  SCFree(pf->filename);
1016  }
1017  if (pf->dirname != NULL) {
1018  SCFree(pf->dirname);
1019  }
1020  SCFree(pf);
1021  }
1022  break;
1023  }
1024 
1025  if (pl->file_cnt > pl->max_files) {
1026  PcapFileName *pf = TAILQ_FIRST(&pl->pcap_file_list);
1027  while (pf != NULL && pl->file_cnt > pl->max_files) {
1028  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
1029 
1030  SCLogDebug("Removing PCAP file %s", pf->filename);
1031  if (remove(pf->filename) != 0) {
1032  SCLogWarning("Failed to remove PCAP file %s: %s", pf->filename, strerror(errno));
1033  }
1034  PcapFileNameFree(pf);
1035  pl->file_cnt--;
1036 
1037  pf = TAILQ_FIRST(&pl->pcap_file_list);
1038  }
1039  }
1040 
1041  closedir(dir);
1042 
1043  /* For some reason file count is initialized at one, instead of 0. */
1044  SCLogNotice("Ring buffer initialized with %d files.", pl->file_cnt - 1);
1045 
1046  return TM_ECODE_OK;
1047 }
1048 #endif /* INIT_RING_BUFFER */
1049 
1050 static TmEcode PcapLogDataInit(ThreadVars *t, const void *initdata, void **data)
1051 {
1052  if (initdata == NULL) {
1053  SCLogDebug("Error getting context for LogPcap. \"initdata\" argument NULL");
1054  return TM_ECODE_FAILED;
1055  }
1056 
1057  PcapLogData *pl = ((OutputCtx *)initdata)->data;
1058 
1059  PcapLogThreadData *td = SCCalloc(1, sizeof(*td));
1060  if (unlikely(td == NULL))
1061  return TM_ECODE_FAILED;
1062 
1063  if (pl->mode == LOGMODE_MULTI)
1064  td->pcap_log = PcapLogDataCopy(pl);
1065  else
1066  td->pcap_log = pl;
1067  BUG_ON(td->pcap_log == NULL);
1068 
1069  if (DatalinkHasMultipleValues()) {
1070  if (pl->mode != LOGMODE_MULTI) {
1071  FatalError("Pcap logging with multiple link type is not supported.");
1072  } else {
1073  /* In multi mode, only pcap conditional is not supported as a flow timeout
1074  * will trigger packet logging with potentially invalid datalink. In regular
1075  * pcap logging, the logging should be done in the same thread if we
1076  * have a proper load balancing. So no mix of datalink should occur. But we need a
1077  * proper load balancing so this needs at least a warning.
1078  */
1079  switch (pl->conditional) {
1080  case LOGMODE_COND_ALERTS:
1081  case LOGMODE_COND_TAG:
1082  FatalError("Can't have multiple link types in pcap conditional mode.");
1083  break;
1084  default:
1085  SCLogWarning("Using multiple link types can result in invalid pcap output");
1086  }
1087  }
1088  }
1089 
1090  PcapLogLock(td->pcap_log);
1091 
1092  /** Use the Output Context (file pointer and mutex) */
1093  td->pcap_log->pkt_cnt = 0;
1094  td->pcap_log->pcap_dead_handle = NULL;
1095  td->pcap_log->pcap_dumper = NULL;
1096  if (td->pcap_log->file_cnt < 1) {
1097  td->pcap_log->file_cnt = 1;
1098  }
1099 
1100  SCTime_t ts = TimeGet();
1101  struct tm local_tm;
1102  struct tm *tms = SCLocalTime(SCTIME_SECS(ts), &local_tm);
1103  td->pcap_log->prev_day = tms->tm_mday;
1104 
1105  PcapLogUnlock(td->pcap_log);
1106 
1107  /* count threads in the global structure */
1108  SCMutexLock(&pl->plog_lock);
1109  pl->threads++;
1110  SCMutexUnlock(&pl->plog_lock);
1111 
1112  *data = (void *)td;
1113 
1116  } else {
1117  td->buf = NULL;
1118  }
1119 
1120  if (pl->max_files && (pl->mode == LOGMODE_MULTI || pl->threads == 1)) {
1121 #ifdef INIT_RING_BUFFER
1122  if (PcapLogInitRingBuffer(td->pcap_log) == TM_ECODE_FAILED) {
1123  return TM_ECODE_FAILED;
1124  }
1125 #else
1126  SCLogInfo("Unable to initialize ring buffer on this platform.");
1127 #endif /* INIT_RING_BUFFER */
1128  }
1129 
1130  /* Don't early initialize output files if in a PCAP file (offline)
1131  * mode. */
1133  if (pl->mode == LOGMODE_MULTI) {
1134  PcapLogOpenFileCtx(td->pcap_log);
1135  } else {
1136  if (pl->filename == NULL) {
1137  PcapLogOpenFileCtx(pl);
1138  }
1139  }
1140  }
1141 
1142  return TM_ECODE_OK;
1143 }
1144 
1145 static void StatsMerge(PcapLogData *dst, PcapLogData *src)
1146 {
1147  dst->profile_open.total += src->profile_open.total;
1148  dst->profile_open.cnt += src->profile_open.cnt;
1149 
1150  dst->profile_close.total += src->profile_close.total;
1151  dst->profile_close.cnt += src->profile_close.cnt;
1152 
1153  dst->profile_write.total += src->profile_write.total;
1154  dst->profile_write.cnt += src->profile_write.cnt;
1155 
1156  dst->profile_rotate.total += src->profile_rotate.total;
1157  dst->profile_rotate.cnt += src->profile_rotate.cnt;
1158 
1159  dst->profile_handles.total += src->profile_handles.total;
1160  dst->profile_handles.cnt += src->profile_handles.cnt;
1161 
1162  dst->profile_lock.total += src->profile_lock.total;
1163  dst->profile_lock.cnt += src->profile_lock.cnt;
1164 
1165  dst->profile_unlock.total += src->profile_unlock.total;
1166  dst->profile_unlock.cnt += src->profile_unlock.cnt;
1167 
1168  dst->profile_data_size += src->profile_data_size;
1169 }
1170 
1171 static void PcapLogDataFree(PcapLogData *pl)
1172 {
1173 
1174  PcapFileName *pf;
1175  while ((pf = TAILQ_FIRST(&pl->pcap_file_list)) != NULL) {
1176  TAILQ_REMOVE(&pl->pcap_file_list, pf, next);
1177  PcapFileNameFree(pf);
1178  }
1179  if (pl == g_pcap_data) {
1180  for (int i = 0; i < MAX_TOKS; i++) {
1181  if (pl->filename_parts[i] != NULL) {
1182  SCFree(pl->filename_parts[i]);
1183  }
1184  }
1185  }
1186  SCFree(pl->h);
1187  SCFree(pl->filename);
1188  SCFree(pl->prefix);
1189 
1190  if (pl->pcap_dead_handle) {
1191  pcap_close(pl->pcap_dead_handle);
1192  }
1193 
1194 #ifdef HAVE_LIBLZ4
1195  if (pl->compression.format == PCAP_LOG_COMPRESSION_FORMAT_LZ4) {
1196  SCFree(pl->compression.buffer);
1197  fclose(pl->compression.pcap_buf_wrapper);
1198  SCFree(pl->compression.pcap_buf);
1199  LZ4F_errorCode_t errcode =
1200  LZ4F_freeCompressionContext(pl->compression.lz4f_context);
1201  if (LZ4F_isError(errcode)) {
1202  SCLogWarning("Error freeing lz4 context.");
1203  }
1204  }
1205 #endif /* HAVE_LIBLZ4 */
1206  SCFree(pl);
1207 }
1208 
1209 /**
1210  * \brief Thread deinit function.
1211  *
1212  * \param t Thread Variable containing input/output queue, cpu affinity etc.
1213  * \param data PcapLog thread data.
1214  * \retval TM_ECODE_OK on success
1215  * \retval TM_ECODE_FAILED on failure
1216  */
1217 static TmEcode PcapLogDataDeinit(ThreadVars *t, void *thread_data)
1218 {
1219  PcapLogThreadData *td = (PcapLogThreadData *)thread_data;
1220  PcapLogData *pl = td->pcap_log;
1221 
1222  if (pl->pcap_dumper != NULL) {
1223  if (PcapLogCloseFile(t,pl) < 0) {
1224  SCLogDebug("PcapLogCloseFile failed");
1225  }
1226  }
1227 
1228  if (pl->mode == LOGMODE_MULTI) {
1229  SCMutexLock(&g_pcap_data->plog_lock);
1230  StatsMerge(g_pcap_data, pl);
1231  g_pcap_data->reported++;
1232  if (g_pcap_data->threads == g_pcap_data->reported)
1233  PcapLogProfilingDump(g_pcap_data);
1234  SCMutexUnlock(&g_pcap_data->plog_lock);
1235  } else {
1236  if (pl->reported == 0) {
1237  PcapLogProfilingDump(pl);
1238  pl->reported = 1;
1239  }
1240  }
1241 
1242  if (pl != g_pcap_data) {
1243  PcapLogDataFree(pl);
1244  }
1245 
1246  if (td->buf)
1247  MemBufferFree(td->buf);
1248 
1249  SCFree(td);
1250  return TM_ECODE_OK;
1251 }
1252 
1253 
1254 static int ParseFilename(PcapLogData *pl, const char *filename)
1255 {
1256  char *toks[MAX_TOKS] = { NULL };
1257  int tok = 0;
1258  char str[MAX_FILENAMELEN] = "";
1259  int s = 0;
1260  int i, x;
1261  char *p = NULL;
1262  size_t filename_len = 0;
1263 
1264  if (filename) {
1265  filename_len = strlen(filename);
1266  if (filename_len > (MAX_FILENAMELEN-1)) {
1267  SCLogError("invalid filename option. Max filename-length: %d", MAX_FILENAMELEN - 1);
1268  goto error;
1269  }
1270 
1271  for (i = 0; i < (int)strlen(filename); i++) {
1272  if (tok >= MAX_TOKS) {
1273  SCLogError("invalid filename option. Max 2 %%-sign options");
1274  goto error;
1275  }
1276 
1277  str[s++] = filename[i];
1278 
1279  if (filename[i] == '%') {
1280  str[s-1] = '\0';
1281  SCLogDebug("filename with %%-sign: %s", str);
1282 
1283  p = SCStrdup(str);
1284  if (p == NULL)
1285  goto error;
1286  toks[tok++] = p;
1287 
1288  s = 0;
1289 
1290  if (i+1 < (int)strlen(filename)) {
1291  if (tok >= MAX_TOKS) {
1292  SCLogError("invalid filename option. Max 2 %%-sign options");
1293  goto error;
1294  }
1295 
1296  if (filename[i+1] != 'n' && filename[i+1] != 't' && filename[i+1] != 'i') {
1297  SCLogError(
1298  "invalid filename option. Valid %%-sign options: %%n, %%i and %%t");
1299  goto error;
1300  }
1301  str[0] = '%';
1302  str[1] = filename[i+1];
1303  str[2] = '\0';
1304  p = SCStrdup(str);
1305  if (p == NULL)
1306  goto error;
1307  toks[tok++] = p;
1308  i++;
1309  }
1310  }
1311  }
1312 
1313  if ((tok == 0) && (pl->mode == LOGMODE_MULTI)) {
1314  SCLogError("Invalid filename for multimode. Need at least one %%-sign option");
1315  goto error;
1316  }
1317 
1318  if (s) {
1319  if (tok >= MAX_TOKS) {
1320  SCLogError("invalid filename option. Max 3 %%-sign options");
1321  goto error;
1322 
1323  }
1324  str[s++] = '\0';
1325  p = SCStrdup(str);
1326  if (p == NULL)
1327  goto error;
1328  toks[tok++] = p;
1329  }
1330 
1331  /* finally, store tokens in the pl */
1332  for (i = 0; i < tok; i++) {
1333  if (toks[i] == NULL)
1334  goto error;
1335 
1336  SCLogDebug("toks[%d] %s", i, toks[i]);
1337  pl->filename_parts[i] = toks[i];
1338  }
1339  pl->filename_part_cnt = tok;
1340  }
1341  return 0;
1342 error:
1343  for (x = 0; x < MAX_TOKS; x++) {
1344  if (toks[x] != NULL)
1345  SCFree(toks[x]);
1346  }
1347  return -1;
1348 }
1349 
1350 /** \brief Fill in pcap logging struct from the provided ConfNode.
1351  * \param conf The configuration node for this output.
1352  * \retval output_ctx
1353  * */
1354 static OutputInitResult PcapLogInitCtx(ConfNode *conf)
1355 {
1356  OutputInitResult result = { NULL, false };
1357  int en;
1358  PCRE2_SIZE eo = 0;
1359 
1360  PcapLogData *pl = SCMalloc(sizeof(PcapLogData));
1361  if (unlikely(pl == NULL)) {
1362  FatalError("Failed to allocate Memory for PcapLogData");
1363  }
1364  memset(pl, 0, sizeof(PcapLogData));
1365 
1366  pl->h = SCMalloc(sizeof(*pl->h));
1367  if (pl->h == NULL) {
1368  FatalError("Failed to allocate Memory for pcap header struct");
1369  }
1370 
1371  /* Set the defaults */
1372  pl->mode = LOGMODE_NORMAL;
1374  pl->use_ringbuffer = RING_BUFFER_MODE_DISABLED;
1375  pl->timestamp_format = TS_FORMAT_SEC;
1379 
1380  TAILQ_INIT(&pl->pcap_file_list);
1381 
1382  SCMutexInit(&pl->plog_lock, NULL);
1383 
1384  /* Initialize PCREs. */
1385  pcre_timestamp_code =
1386  pcre2_compile((PCRE2_SPTR8)timestamp_pattern, PCRE2_ZERO_TERMINATED, 0, &en, &eo, NULL);
1387  if (pcre_timestamp_code == NULL) {
1388  PCRE2_UCHAR errbuffer[256];
1389  pcre2_get_error_message(en, errbuffer, sizeof(errbuffer));
1390  FatalError(
1391  "Failed to compile \"%s\" at offset %d: %s", timestamp_pattern, (int)eo, errbuffer);
1392  }
1393  pcre_timestamp_match = pcre2_match_data_create_from_pattern(pcre_timestamp_code, NULL);
1394 
1395  /* conf params */
1396 
1397  const char *filename = NULL;
1398 
1399  if (conf != NULL) { /* To facilitate unit tests. */
1400  filename = ConfNodeLookupChildValue(conf, "filename");
1401  }
1402 
1403  if (filename == NULL)
1404  filename = DEFAULT_LOG_FILENAME;
1405 
1406  if ((pl->prefix = SCStrdup(filename)) == NULL) {
1407  exit(EXIT_FAILURE);
1408  }
1409 
1410  pl->suffix = "";
1411 
1412  pl->size_limit = DEFAULT_LIMIT;
1413  if (conf != NULL) {
1414  const char *s_limit = NULL;
1415  s_limit = ConfNodeLookupChildValue(conf, "limit");
1416  if (s_limit != NULL) {
1417  if (ParseSizeStringU64(s_limit, &pl->size_limit) < 0) {
1418  SCLogError("Failed to initialize pcap output, invalid limit: %s", s_limit);
1419  exit(EXIT_FAILURE);
1420  }
1421  if (pl->size_limit < 4096) {
1422  SCLogInfo("pcap-log \"limit\" value of %"PRIu64" assumed to be pre-1.2 "
1423  "style: setting limit to %"PRIu64"mb", pl->size_limit, pl->size_limit);
1424  uint64_t size = pl->size_limit * 1024 * 1024;
1425  pl->size_limit = size;
1426  } else if (pl->size_limit < MIN_LIMIT) {
1427  FatalError("Fail to initialize pcap-log output, limit less than "
1428  "allowed minimum of %d bytes.",
1429  MIN_LIMIT);
1430  }
1431  }
1432  }
1433 
1434  if (conf != NULL) {
1435  const char *s_mode = NULL;
1436  s_mode = ConfNodeLookupChildValue(conf, "mode");
1437  if (s_mode != NULL) {
1438  if (strcasecmp(s_mode, "sguil") == 0) {
1439  pl->mode = LOGMODE_SGUIL;
1440  } else if (strcasecmp(s_mode, "multi") == 0) {
1441  pl->mode = LOGMODE_MULTI;
1442  } else if (strcasecmp(s_mode, "normal") != 0) {
1443  SCLogError("log-pcap: invalid mode \"%s\". Valid options: \"normal\", "
1444  "\"sguil\", or \"multi\" mode ",
1445  s_mode);
1446  exit(EXIT_FAILURE);
1447  }
1448  }
1449 
1450  const char *s_dir = NULL;
1451  s_dir = ConfNodeLookupChildValue(conf, "dir");
1452  if (s_dir == NULL) {
1453  s_dir = ConfNodeLookupChildValue(conf, "sguil-base-dir");
1454  }
1455  if (s_dir == NULL) {
1456  if (pl->mode == LOGMODE_SGUIL) {
1457  FatalError("log-pcap \"sguil\" mode requires \"sguil-base-dir\" "
1458  "option to be set.");
1459  } else {
1460  const char *log_dir = NULL;
1461  log_dir = ConfigGetLogDirectory();
1462 
1463  strlcpy(pl->dir,
1464  log_dir, sizeof(pl->dir));
1465  SCLogInfo("Using log dir %s", pl->dir);
1466  }
1467  } else {
1468  if (PathIsAbsolute(s_dir)) {
1469  strlcpy(pl->dir,
1470  s_dir, sizeof(pl->dir));
1471  } else {
1472  const char *log_dir = NULL;
1473  log_dir = ConfigGetLogDirectory();
1474 
1475  snprintf(pl->dir, sizeof(pl->dir), "%s/%s",
1476  log_dir, s_dir);
1477  }
1478 
1479  struct stat stat_buf;
1480  if (stat(pl->dir, &stat_buf) != 0) {
1481  SCLogError("The sguil-base-dir directory \"%s\" "
1482  "supplied doesn't exist. Shutting down the engine",
1483  pl->dir);
1484  exit(EXIT_FAILURE);
1485  }
1486  SCLogInfo("Using log dir %s", pl->dir);
1487  }
1488 
1489  const char *compression_str = ConfNodeLookupChildValue(conf,
1490  "compression");
1491 
1492  PcapLogCompressionData *comp = &pl->compression;
1493  if (compression_str == NULL || strcmp(compression_str, "none") == 0) {
1495  comp->buffer = NULL;
1496  comp->buffer_size = 0;
1497  comp->file = NULL;
1498  comp->pcap_buf = NULL;
1499  comp->pcap_buf_size = 0;
1500  comp->pcap_buf_wrapper = NULL;
1501  } else if (strcmp(compression_str, "lz4") == 0) {
1502 #ifdef HAVE_LIBLZ4
1503  if (pl->mode == LOGMODE_SGUIL) {
1504  SCLogError("Compressed pcap "
1505  "logs are not possible in sguil mode");
1506  SCFree(pl->h);
1507  SCFree(pl);
1508  return result;
1509  }
1510  pl->compression.format = PCAP_LOG_COMPRESSION_FORMAT_LZ4;
1511 
1512  /* Use SCFmemopen so we can make pcap_dump write to a buffer. */
1513 
1514  comp->pcap_buf_size = sizeof(struct pcap_file_header) +
1515  sizeof(struct pcap_pkthdr) + PCAP_SNAPLEN;
1516  comp->pcap_buf = SCMalloc(comp->pcap_buf_size);
1517  if (comp->pcap_buf == NULL) {
1518  SCLogError("SCMalloc failed: %s", strerror(errno));
1519  exit(EXIT_FAILURE);
1520  }
1521  comp->pcap_buf_wrapper = SCFmemopen(comp->pcap_buf,
1522  comp->pcap_buf_size, "w");
1523  if (comp->pcap_buf_wrapper == NULL) {
1524  SCLogError("SCFmemopen failed: %s", strerror(errno));
1525  exit(EXIT_FAILURE);
1526  }
1527 
1528  /* Set lz4 preferences. */
1529 
1530  memset(&comp->lz4f_prefs, '\0', sizeof(comp->lz4f_prefs));
1531  comp->lz4f_prefs.frameInfo.blockSizeID = LZ4F_max4MB;
1532  comp->lz4f_prefs.frameInfo.blockMode = LZ4F_blockLinked;
1533  if (ConfNodeChildValueIsTrue(conf, "lz4-checksum")) {
1534  comp->lz4f_prefs.frameInfo.contentChecksumFlag = 1;
1535  }
1536  else {
1537  comp->lz4f_prefs.frameInfo.contentChecksumFlag = 0;
1538  }
1539  intmax_t lvl = 0;
1540  if (ConfGetChildValueInt(conf, "lz4-level", &lvl)) {
1541  if (lvl > 16) {
1542  lvl = 16;
1543  } else if (lvl < 0) {
1544  lvl = 0;
1545  }
1546  } else {
1547  lvl = 0;
1548  }
1549  comp->lz4f_prefs.compressionLevel = lvl;
1550 
1551  /* Allocate resources for lz4. */
1552 
1553  LZ4F_errorCode_t errcode =
1554  LZ4F_createCompressionContext(&pl->compression.lz4f_context, 1);
1555 
1556  if (LZ4F_isError(errcode)) {
1557  SCLogError("LZ4F_createCompressionContext failed: %s", LZ4F_getErrorName(errcode));
1558  exit(EXIT_FAILURE);
1559  }
1560 
1561  /* Calculate the size of the lz4 output buffer. */
1562 
1563  comp->buffer_size = LZ4F_compressBound(comp->pcap_buf_size,
1564  &comp->lz4f_prefs);
1565 
1566  comp->buffer = SCMalloc(comp->buffer_size);
1567  if (unlikely(comp->buffer == NULL)) {
1568  FatalError("Failed to allocate memory for "
1569  "lz4 output buffer.");
1570  }
1571 
1572  comp->bytes_in_block = 0;
1573 
1574  /* Add the lz4 file extension to the log files. */
1575 
1576  pl->suffix = ".lz4";
1577 #else
1578  SCLogError("lz4 compression was selected "
1579  "in pcap-log, but suricata was not compiled with lz4 "
1580  "support.");
1581  PcapLogDataFree(pl);
1582  return result;
1583 #endif /* HAVE_LIBLZ4 */
1584  }
1585  else {
1586  SCLogError("Unsupported pcap-log "
1587  "compression format: %s",
1588  compression_str);
1589  PcapLogDataFree(pl);
1590  return result;
1591  }
1592 
1593  SCLogInfo("Selected pcap-log compression method: %s",
1594  compression_str ? compression_str : "none");
1595 
1596  const char *s_conditional = ConfNodeLookupChildValue(conf, "conditional");
1597  if (s_conditional != NULL) {
1598  if (strcasecmp(s_conditional, "alerts") == 0) {
1601  } else if (strcasecmp(s_conditional, "tag") == 0) {
1604  } else if (strcasecmp(s_conditional, "all") != 0) {
1605  FatalError("log-pcap: invalid conditional \"%s\". Valid options: \"all\", "
1606  "\"alerts\", or \"tag\" mode ",
1607  s_conditional);
1608  }
1609  }
1610 
1611  SCLogInfo(
1612  "Selected pcap-log conditional logging: %s", s_conditional ? s_conditional : "all");
1613  }
1614 
1615  if (ParseFilename(pl, filename) != 0)
1616  exit(EXIT_FAILURE);
1617 
1618  SCLogInfo("using %s logging", pl->mode == LOGMODE_SGUIL ?
1619  "Sguil compatible" : (pl->mode == LOGMODE_MULTI ? "multi" : "normal"));
1620 
1621  uint32_t max_file_limit = DEFAULT_FILE_LIMIT;
1622  if (conf != NULL) {
1623  const char *max_number_of_files_s = NULL;
1624  max_number_of_files_s = ConfNodeLookupChildValue(conf, "max-files");
1625  if (max_number_of_files_s != NULL) {
1626  if (StringParseUint32(&max_file_limit, 10, 0,
1627  max_number_of_files_s) == -1) {
1628  SCLogError("Failed to initialize "
1629  "pcap-log output, invalid number of files limit: %s",
1630  max_number_of_files_s);
1631  exit(EXIT_FAILURE);
1632  } else if (max_file_limit < 1) {
1633  FatalError("Failed to initialize pcap-log output, limit less than "
1634  "allowed minimum.");
1635  } else {
1636  pl->max_files = max_file_limit;
1637  pl->use_ringbuffer = RING_BUFFER_MODE_ENABLED;
1638  }
1639  }
1640  }
1641 
1642  const char *ts_format = NULL;
1643  if (conf != NULL) { /* To facilitate unit tests. */
1644  ts_format = ConfNodeLookupChildValue(conf, "ts-format");
1645  }
1646  if (ts_format != NULL) {
1647  if (strcasecmp(ts_format, "usec") == 0) {
1648  pl->timestamp_format = TS_FORMAT_USEC;
1649  } else if (strcasecmp(ts_format, "sec") != 0) {
1650  SCLogError("log-pcap ts_format specified %s is invalid must be"
1651  " \"sec\" or \"usec\"",
1652  ts_format);
1653  exit(EXIT_FAILURE);
1654  }
1655  }
1656 
1657  const char *use_stream_depth = NULL;
1658  if (conf != NULL) { /* To facilitate unit tests. */
1659  use_stream_depth = ConfNodeLookupChildValue(conf, "use-stream-depth");
1660  }
1661  if (use_stream_depth != NULL) {
1662  if (ConfValIsFalse(use_stream_depth)) {
1664  } else if (ConfValIsTrue(use_stream_depth)) {
1666  } else {
1667  FatalError("log-pcap use_stream_depth specified is invalid must be");
1668  }
1669  }
1670 
1671  const char *honor_pass_rules = NULL;
1672  if (conf != NULL) { /* To facilitate unit tests. */
1673  honor_pass_rules = ConfNodeLookupChildValue(conf, "honor-pass-rules");
1674  }
1675  if (honor_pass_rules != NULL) {
1676  if (ConfValIsFalse(honor_pass_rules)) {
1678  } else if (ConfValIsTrue(honor_pass_rules)) {
1680  } else {
1681  FatalError("log-pcap honor-pass-rules specified is invalid");
1682  }
1683  }
1684 
1685  /* create the output ctx and send it back */
1686 
1687  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
1688  if (unlikely(output_ctx == NULL)) {
1689  FatalError("Failed to allocate memory for OutputCtx.");
1690  }
1691  output_ctx->data = pl;
1692  output_ctx->DeInit = PcapLogFileDeInitCtx;
1693  g_pcap_data = pl;
1694 
1695  result.ctx = output_ctx;
1696  result.ok = true;
1697  return result;
1698 }
1699 
1700 static void PcapLogFileDeInitCtx(OutputCtx *output_ctx)
1701 {
1702  if (output_ctx == NULL)
1703  return;
1704 
1705  PcapLogData *pl = output_ctx->data;
1706 
1707  PcapFileName *pf = NULL;
1708  TAILQ_FOREACH(pf, &pl->pcap_file_list, next) {
1709  SCLogDebug("PCAP files left at exit: %s\n", pf->filename);
1710  }
1711  PcapLogDataFree(pl);
1712  SCFree(output_ctx);
1713 
1714  pcre2_code_free(pcre_timestamp_code);
1715  pcre2_match_data_free(pcre_timestamp_match);
1716 
1717  return;
1718 }
1719 
1720 /**
1721  * \brief Read the config set the file pointer, open the file
1722  *
1723  * \param PcapLogData.
1724  *
1725  * \retval -1 if failure
1726  * \retval 0 if succesful
1727  */
1728 static int PcapLogOpenFileCtx(PcapLogData *pl)
1729 {
1730  char *filename = NULL;
1731 
1733 
1734  if (pl->filename != NULL)
1735  filename = pl->filename;
1736  else {
1737  filename = SCMalloc(PATH_MAX);
1738  if (unlikely(filename == NULL)) {
1739  return -1;
1740  }
1741  pl->filename = filename;
1742  }
1743 
1744  /** get the time so we can have a filename with seconds since epoch */
1745  SCTime_t ts = TimeGet();
1746 
1747  /* Place to store the name of our PCAP file */
1748  PcapFileName *pf = SCMalloc(sizeof(PcapFileName));
1749  if (unlikely(pf == NULL)) {
1750  return -1;
1751  }
1752  memset(pf, 0, sizeof(PcapFileName));
1753 
1754  if (pl->mode == LOGMODE_SGUIL) {
1755  struct tm local_tm;
1756  struct tm *tms = SCLocalTime(SCTIME_SECS(ts), &local_tm);
1757 
1758  char dirname[32], dirfull[PATH_MAX] = "";
1759 
1760  snprintf(dirname, sizeof(dirname), "%04d-%02d-%02d",
1761  tms->tm_year + 1900, tms->tm_mon + 1, tms->tm_mday);
1762 
1763  /* create the filename to use */
1764  int ret = snprintf(dirfull, sizeof(dirfull), "%s/%s", pl->dir, dirname);
1765  if (ret < 0 || (size_t)ret >= sizeof(dirfull)) {
1766  SCLogError("failed to construct path");
1767  goto error;
1768  }
1769 
1770  /* if mkdir fails file open will fail, so deal with errors there */
1771  (void)SCMkDir(dirfull, 0700);
1772 
1773  if ((pf->dirname = SCStrdup(dirfull)) == NULL) {
1774  SCLogError("Error allocating memory for "
1775  "directory name");
1776  goto error;
1777  }
1778 
1779  int written;
1780  if (pl->timestamp_format == TS_FORMAT_SEC) {
1781  written = snprintf(filename, PATH_MAX, "%s/%s.%" PRIu32 "%s", dirfull, pl->prefix,
1782  (uint32_t)SCTIME_SECS(ts), pl->suffix);
1783  } else {
1784  written = snprintf(filename, PATH_MAX, "%s/%s.%" PRIu32 ".%" PRIu32 "%s", dirfull,
1785  pl->prefix, (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts), pl->suffix);
1786  }
1787  if (written == PATH_MAX) {
1788  SCLogError("log-pcap path overflow");
1789  goto error;
1790  }
1791  } else if (pl->mode == LOGMODE_NORMAL) {
1792  int ret;
1793  /* create the filename to use */
1794  if (pl->timestamp_format == TS_FORMAT_SEC) {
1795  ret = snprintf(filename, PATH_MAX, "%s/%s.%" PRIu32 "%s", pl->dir, pl->prefix,
1796  (uint32_t)SCTIME_SECS(ts), pl->suffix);
1797  } else {
1798  ret = snprintf(filename, PATH_MAX, "%s/%s.%" PRIu32 ".%" PRIu32 "%s", pl->dir,
1799  pl->prefix, (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts), pl->suffix);
1800  }
1801  if (ret < 0 || (size_t)ret >= PATH_MAX) {
1802  SCLogError("failed to construct path");
1803  goto error;
1804  }
1805  } else if (pl->mode == LOGMODE_MULTI) {
1806  if (pl->filename_part_cnt > 0) {
1807  /* assemble filename from stored tokens */
1808 
1809  strlcpy(filename, pl->dir, PATH_MAX);
1810  strlcat(filename, "/", PATH_MAX);
1811 
1812  int i;
1813  for (i = 0; i < pl->filename_part_cnt; i++) {
1814  if (pl->filename_parts[i] == NULL ||strlen(pl->filename_parts[i]) == 0)
1815  continue;
1816 
1817  /* handle variables */
1818  if (pl->filename_parts[i][0] == '%') {
1819  char str[64] = "";
1820  if (strlen(pl->filename_parts[i]) < 2)
1821  continue;
1822 
1823  switch(pl->filename_parts[i][1]) {
1824  case 'n':
1825  snprintf(str, sizeof(str), "%u", pl->thread_number);
1826  break;
1827  case 'i':
1828  {
1829  long thread_id = SCGetThreadIdLong();
1830  snprintf(str, sizeof(str), "%"PRIu64, (uint64_t)thread_id);
1831  break;
1832  }
1833  case 't':
1834  /* create the filename to use */
1835  if (pl->timestamp_format == TS_FORMAT_SEC) {
1836  snprintf(str, sizeof(str), "%" PRIu32, (uint32_t)SCTIME_SECS(ts));
1837  } else {
1838  snprintf(str, sizeof(str), "%" PRIu32 ".%" PRIu32,
1839  (uint32_t)SCTIME_SECS(ts), (uint32_t)SCTIME_USECS(ts));
1840  }
1841  }
1842  strlcat(filename, str, PATH_MAX);
1843 
1844  /* copy the rest over */
1845  } else {
1846  strlcat(filename, pl->filename_parts[i], PATH_MAX);
1847  }
1848  }
1849  strlcat(filename, pl->suffix, PATH_MAX);
1850  } else {
1851  int ret;
1852  /* create the filename to use */
1853  if (pl->timestamp_format == TS_FORMAT_SEC) {
1854  ret = snprintf(filename, PATH_MAX, "%s/%s.%u.%" PRIu32 "%s", pl->dir, pl->prefix,
1855  pl->thread_number, (uint32_t)SCTIME_SECS(ts), pl->suffix);
1856  } else {
1857  ret = snprintf(filename, PATH_MAX, "%s/%s.%u.%" PRIu32 ".%" PRIu32 "%s", pl->dir,
1858  pl->prefix, pl->thread_number, (uint32_t)SCTIME_SECS(ts),
1859  (uint32_t)SCTIME_USECS(ts), pl->suffix);
1860  }
1861  if (ret < 0 || (size_t)ret >= PATH_MAX) {
1862  SCLogError("failed to construct path");
1863  goto error;
1864  }
1865  }
1866  SCLogDebug("multi-mode: filename %s", filename);
1867  }
1868 
1869  if ((pf->filename = SCStrdup(pl->filename)) == NULL) {
1870  SCLogError("Error allocating memory. For filename");
1871  goto error;
1872  }
1873  SCLogDebug("Opening pcap file log %s", pf->filename);
1874  TAILQ_INSERT_TAIL(&pl->pcap_file_list, pf, next);
1875 
1876  if (pl->mode == LOGMODE_MULTI || pl->mode == LOGMODE_NORMAL) {
1877  pcap_file_thread = pl->filename;
1878  }
1880  return 0;
1881 
1882 error:
1883  PcapFileNameFree(pf);
1884  return -1;
1885 }
1886 
1888 {
1889  /* return pcap filename per thread */
1890  if (pcap_file_thread != NULL) {
1891  return pcap_file_thread;
1892  }
1893  return NULL;
1894 }
1895 
1896 static int profiling_pcaplog_enabled = 0;
1897 static int profiling_pcaplog_output_to_file = 0;
1898 static char *profiling_pcaplog_file_name = NULL;
1899 static const char *profiling_pcaplog_file_mode = "a";
1900 
1901 static void FormatNumber(uint64_t num, char *str, size_t size)
1902 {
1903  if (num < 1000UL)
1904  snprintf(str, size, "%"PRIu64, num);
1905  else if (num < 1000000UL)
1906  snprintf(str, size, "%3.1fk", (float)num/1000UL);
1907  else if (num < 1000000000UL)
1908  snprintf(str, size, "%3.1fm", (float)num/1000000UL);
1909  else
1910  snprintf(str, size, "%3.1fb", (float)num/1000000000UL);
1911 }
1912 
1913 static void ProfileReportPair(FILE *fp, const char *name, PcapLogProfileData *p)
1914 {
1915  char ticks_str[32] = "n/a";
1916  char cnt_str[32] = "n/a";
1917  char avg_str[32] = "n/a";
1918 
1919  FormatNumber((uint64_t)p->cnt, cnt_str, sizeof(cnt_str));
1920  FormatNumber((uint64_t)p->total, ticks_str, sizeof(ticks_str));
1921  if (p->cnt && p->total)
1922  FormatNumber((uint64_t)(p->total/p->cnt), avg_str, sizeof(avg_str));
1923 
1924  fprintf(fp, "%-28s %-10s %-10s %-10s\n", name, cnt_str, avg_str, ticks_str);
1925 }
1926 
1927 static void ProfileReport(FILE *fp, PcapLogData *pl)
1928 {
1929  ProfileReportPair(fp, "open", &pl->profile_open);
1930  ProfileReportPair(fp, "close", &pl->profile_close);
1931  ProfileReportPair(fp, "write", &pl->profile_write);
1932  ProfileReportPair(fp, "rotate (incl open/close)", &pl->profile_rotate);
1933  ProfileReportPair(fp, "handles", &pl->profile_handles);
1934  ProfileReportPair(fp, "lock", &pl->profile_lock);
1935  ProfileReportPair(fp, "unlock", &pl->profile_unlock);
1936 }
1937 
1938 static void FormatBytes(uint64_t num, char *str, size_t size)
1939 {
1940  if (num < 1000UL)
1941  snprintf(str, size, "%"PRIu64, num);
1942  else if (num < 1048576UL)
1943  snprintf(str, size, "%3.1fKiB", (float)num/1000UL);
1944  else if (num < 1073741824UL)
1945  snprintf(str, size, "%3.1fMiB", (float)num/1000000UL);
1946  else
1947  snprintf(str, size, "%3.1fGiB", (float)num/1000000000UL);
1948 }
1949 
1950 static void PcapLogProfilingDump(PcapLogData *pl)
1951 {
1952  FILE *fp = NULL;
1953 
1954  if (profiling_pcaplog_enabled == 0)
1955  return;
1956 
1957  if (profiling_pcaplog_output_to_file == 1) {
1958  fp = fopen(profiling_pcaplog_file_name, profiling_pcaplog_file_mode);
1959  if (fp == NULL) {
1960  SCLogError("failed to open %s: %s", profiling_pcaplog_file_name, strerror(errno));
1961  return;
1962  }
1963  } else {
1964  fp = stdout;
1965  }
1966 
1967  /* counters */
1968  fprintf(fp, "\n\nOperation Cnt Avg ticks Total ticks\n");
1969  fprintf(fp, "---------------------------- ---------- ---------- -----------\n");
1970 
1971  ProfileReport(fp, pl);
1972  uint64_t total = pl->profile_write.total + pl->profile_rotate.total +
1975  pl->profile_unlock.total;
1976 
1977  /* overall stats */
1978  fprintf(fp, "\nOverall: %"PRIu64" bytes written, average %d bytes per write.\n",
1980  (int)(pl->profile_data_size / pl->profile_write.cnt) : 0);
1981  fprintf(fp, " PCAP data structure overhead: %"PRIuMAX" per write.\n",
1982  (uintmax_t)sizeof(struct pcap_pkthdr));
1983 
1984  /* print total bytes written */
1985  char bytes_str[32];
1986  FormatBytes(pl->profile_data_size, bytes_str, sizeof(bytes_str));
1987  fprintf(fp, " Size written: %s\n", bytes_str);
1988 
1989  /* ticks per MiB and GiB */
1990  uint64_t ticks_per_mib = 0, ticks_per_gib = 0;
1991  uint64_t mib = pl->profile_data_size/(1024*1024);
1992  if (mib)
1993  ticks_per_mib = total/mib;
1994  char ticks_per_mib_str[32] = "n/a";
1995  if (ticks_per_mib > 0)
1996  FormatNumber(ticks_per_mib, ticks_per_mib_str, sizeof(ticks_per_mib_str));
1997  fprintf(fp, " Ticks per MiB: %s\n", ticks_per_mib_str);
1998 
1999  uint64_t gib = pl->profile_data_size/(1024*1024*1024);
2000  if (gib)
2001  ticks_per_gib = total/gib;
2002  char ticks_per_gib_str[32] = "n/a";
2003  if (ticks_per_gib > 0)
2004  FormatNumber(ticks_per_gib, ticks_per_gib_str, sizeof(ticks_per_gib_str));
2005  fprintf(fp, " Ticks per GiB: %s\n", ticks_per_gib_str);
2006 
2007  if (fp != stdout)
2008  fclose(fp);
2009 }
2010 
2012 {
2013  ConfNode *conf = ConfGetNode("profiling.pcap-log");
2014  if (conf != NULL && ConfNodeChildValueIsTrue(conf, "enabled")) {
2015  profiling_pcaplog_enabled = 1;
2016  SCLogInfo("pcap-log profiling enabled");
2017 
2018  const char *filename = ConfNodeLookupChildValue(conf, "filename");
2019  if (filename != NULL) {
2020  const char *log_dir;
2021  log_dir = ConfigGetLogDirectory();
2022 
2023  profiling_pcaplog_file_name = SCMalloc(PATH_MAX);
2024  if (unlikely(profiling_pcaplog_file_name == NULL)) {
2025  FatalError("can't duplicate file name");
2026  }
2027 
2028  snprintf(profiling_pcaplog_file_name, PATH_MAX, "%s/%s", log_dir, filename);
2029 
2030  const char *v = ConfNodeLookupChildValue(conf, "append");
2031  if (v == NULL || ConfValIsTrue(v)) {
2032  profiling_pcaplog_file_mode = "a";
2033  } else {
2034  profiling_pcaplog_file_mode = "w";
2035  }
2036 
2037  profiling_pcaplog_output_to_file = 1;
2038  SCLogInfo("pcap-log profiling output goes to %s (mode %s)",
2039  profiling_pcaplog_file_name, profiling_pcaplog_file_mode);
2040  }
2041  }
2042 }
util-byte.h
PcapLogCompressionData_::pcap_buf_size
uint64_t pcap_buf_size
Definition: log-pcap.c:131
ConfGetChildValueInt
int ConfGetChildValueInt(const ConfNode *base, const char *name, intmax_t *val)
Definition: conf.c:434
PcapLogData
struct PcapLogData_ PcapLogData
PcapLogProfileData_
Definition: log-pcap.c:108
LOGMODE_COND_ALL
@ LOGMODE_COND_ALL
Definition: log-pcap.c:70
len
uint8_t len
Definition: app-layer-dnp3.h:2
ts
uint64_t ts
Definition: source-erf-file.c:55
PcapLogData_::conditional
LogModeConditionalType conditional
Definition: log-pcap.c:159
PcapLogThreadData
struct PcapLogThreadData_ PcapLogThreadData
ConfNodeChildValueIsTrue
int ConfNodeChildValueIsTrue(const ConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:859
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:89
util-fmemopen.h
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:262
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
USE_STREAM_DEPTH_DISABLED
#define USE_STREAM_DEPTH_DISABLED
Definition: log-pcap.c:81
PcapFileName_::secs
uint64_t secs
Definition: log-pcap.c:99
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
PcapLogData_::pcap_dumper
pcap_dumper_t * pcap_dumper
Definition: log-pcap.c:154
TS_FORMAT_SEC
#define TS_FORMAT_SEC
Definition: log-pcap.c:78
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
ParseSizeStringU64
int ParseSizeStringU64(const char *size, uint64_t *res)
Definition: util-misc.c:198
PcapFileName_::usecs
uint32_t usecs
Definition: log-pcap.c:100
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
PcapLogData_::filename
char * filename
Definition: log-pcap.c:148
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:289
HONOR_PASS_RULES_ENABLED
#define HONOR_PASS_RULES_ENABLED
Definition: log-pcap.c:85
Packet_::flags
uint32_t flags
Definition: decode.h:467
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:181
DEFAULT_LOG_FILENAME
#define DEFAULT_LOG_FILENAME
Definition: log-pcap.c:59
threads.h
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
RING_BUFFER_MODE_ENABLED
#define RING_BUFFER_MODE_ENABLED
Definition: log-pcap.c:76
PcapLogData_::mode
int mode
Definition: log-pcap.c:149
IS_TUNNEL_ROOT_PKT
#define IS_TUNNEL_ROOT_PKT(p)
Definition: decode.h:797
TAILQ_EMPTY
#define TAILQ_EMPTY(head)
Definition: queue.h:248
PcapLogData_::pkt_cnt
uint64_t pkt_cnt
Definition: log-pcap.c:146
HONOR_PASS_RULES_DISABLED
#define HONOR_PASS_RULES_DISABLED
Definition: log-pcap.c:84
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:252
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
PcapLogData_::use_stream_depth
int use_stream_depth
Definition: log-pcap.c:142
PcapLogCompressionData_::format
enum PcapLogCompressionFormat format
Definition: log-pcap.c:122
PcapLogProfileData
struct PcapLogProfileData_ PcapLogProfileData
stream-tcp-reassemble.h
PcapLogData_::profile_handles
PcapLogProfileData profile_handles
Definition: log-pcap.c:164
LOGMODE_MULTI
#define LOGMODE_MULTI
Definition: log-pcap.c:67
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
PcapLogRegister
void PcapLogRegister(void)
Definition: log-pcap.c:211
SCFmemopen
#define SCFmemopen
Definition: util-fmemopen.h:52
PcapLogCompressionData_::buffer
uint8_t * buffer
Definition: log-pcap.c:123
DEFAULT_LIMIT
#define DEFAULT_LIMIT
Definition: log-pcap.c:62
PcapLogCallbackContext::connp
PcapLogCompressionData * connp
Definition: log-pcap.c:556
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
IsRunModeOffline
bool IsRunModeOffline(enum RunModes run_mode_to_check)
Definition: runmodes.c:574
PcapLogData_::pcap_dead_handle
pcap_t * pcap_dead_handle
Definition: log-pcap.c:153
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:592
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
PcapLogData_::size_current
uint64_t size_current
Definition: log-pcap.c:151
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:537
PcapLogData_::profile_rotate
PcapLogProfileData profile_rotate
Definition: log-pcap.c:167
OutputCtx_::data
void * data
Definition: tm-modules.h:87
PcapLogCompressionData_
Definition: log-pcap.c:121
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
PCAPLOG_PROFILE_END
#define PCAPLOG_PROFILE_END(prof)
Definition: log-pcap.c:225
PcapLogGetFilename
char * PcapLogGetFilename(void)
Definition: log-pcap.c:1887
OutputCtx_
Definition: tm-modules.h:84
LogModeConditionalType_
LogModeConditionalType_
Definition: log-pcap.c:69
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
PcapLogCompressionData_::bytes_in_block
uint64_t bytes_in_block
Definition: log-pcap.c:133
TAILQ_ENTRY
#define TAILQ_ENTRY(type)
Definition: queue.h:239
Packet_::datalink
int datalink
Definition: decode.h:611
PcapLogData_::size_limit
uint64_t size_limit
Definition: log-pcap.c:152
PKT_IS_TCP
#define PKT_IS_TCP(p)
Definition: decode.h:247
PcapLogProfileData_::cnt
uint64_t cnt
Definition: log-pcap.c:110
PcapLogData_::max_files
uint32_t max_files
Definition: log-pcap.c:157
PcapLogCompressionData_::file
FILE * file
Definition: log-pcap.c:129
TAILQ_REMOVE
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:312
decode.h
TAILQ_FIRST
#define TAILQ_FIRST(head)
Definition: queue.h:250
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:47
PCAP_LOG_COMPRESSION_FORMAT_NONE
@ PCAP_LOG_COMPRESSION_FORMAT_NONE
Definition: log-pcap.c:117
strlcat
size_t strlcat(char *, const char *src, size_t siz)
Definition: util-strlcatu.c:45
util-cpu.h
Packet_::ts
SCTime_t ts
Definition: decode.h:475
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
MemBufferWriteRaw
#define MemBufferWriteRaw(dst, raw_buffer, raw_buffer_len)
Write a raw buffer to the MemBuffer dst.
Definition: util-buffer.h:133
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1002
PCAP_LOG_COMPRESSION_FORMAT_LZ4
@ PCAP_LOG_COMPRESSION_FORMAT_LZ4
Definition: log-pcap.c:118
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:94
STREAM_DUMP_HEADERS
#define STREAM_DUMP_HEADERS
Definition: stream.h:34
GET_PKT_DATA
#define GET_PKT_DATA(p)
Definition: decode.h:220
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
PcapLogProfileData_::total
uint64_t total
Definition: log-pcap.c:109
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:48
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
TRUE
#define TRUE
Definition: suricata-common.h:33
PcapFileName_::dirname
char * dirname
Definition: log-pcap.c:94
PcapLogData_::file_cnt
uint32_t file_cnt
Definition: log-pcap.c:156
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:295
util-profiling.h
FALSE
#define FALSE
Definition: suricata-common.h:34
PCAP_OUTPUT_BUFFER_SIZE
#define PCAP_OUTPUT_BUFFER_SIZE
Definition: log-pcap.h:31
stream.h
PcapLogCallbackContext::buf
MemBuffer * buf
Definition: log-pcap.c:557
TcpSegment
Definition: stream-tcp-private.h:72
Packet_
Definition: decode.h:430
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:267
GET_PKT_LEN
#define GET_PKT_LEN(p)
Definition: decode.h:219
TimeGet
SCTime_t TimeGet(void)
Definition: util-time.c:152
DEFAULT_FILE_LIMIT
#define DEFAULT_FILE_LIMIT
Definition: log-pcap.c:63
SCTime_t
Definition: util-time.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:83
RING_BUFFER_MODE_DISABLED
#define RING_BUFFER_MODE_DISABLED
Definition: log-pcap.c:75
PcapLogThreadData_::buf
MemBuffer * buf
Definition: log-pcap.c:190
PCAP_SNAPLEN
#define PCAP_SNAPLEN
Definition: log-pcap.c:87
PcapLogData_::plog_lock
SCMutex plog_lock
Definition: log-pcap.c:145
PcapLogData_::h
struct pcap_pkthdr * h
Definition: log-pcap.c:147
IS_TUNNEL_PKT
#define IS_TUNNEL_PKT(p)
Definition: decode.h:794
PcapLogCompressionData_::pcap_buf
uint8_t * pcap_buf
Definition: log-pcap.c:130
MemBuffer_
Definition: util-buffer.h:27
LOGMODE_COND_TAG
@ LOGMODE_COND_TAG
Definition: log-pcap.c:72
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
StreamSegmentForSession
int StreamSegmentForSession(const Packet *p, uint8_t flag, StreamSegmentCallback CallbackFunc, void *data)
Run callback for all segments on both directions of the session.
Definition: stream.c:64
PcapLogCallbackContext::pl
PcapLogData * pl
Definition: log-pcap.c:555
PcapLogData_::prev_day
int prev_day
Definition: log-pcap.c:150
PKT_FIRST_TAG
#define PKT_FIRST_TAG
Definition: decode.h:1050
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
PCAPLOG_PROFILE_START
#define PCAPLOG_PROFILE_START
Definition: log-pcap.c:222
log-pcap.h
SCGetThreadIdLong
#define SCGetThreadIdLong(...)
Definition: threads.h:255
PcapLogData_::honor_pass_rules
int honor_pass_rules
Definition: log-pcap.c:143
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
pcap_file_thread
thread_local char * pcap_file_thread
Definition: log-pcap.c:106
PcapLogData_::profile_open
PcapLogProfileData profile_open
Definition: log-pcap.c:166
PcapLogData_::profile_lock
PcapLogProfileData profile_lock
Definition: log-pcap.c:161
OutputInitResult_
Definition: output.h:46
util-conf.h
Packet_::flow
struct Flow_ * flow
Definition: decode.h:469
StringParseUint64
int StringParseUint64(uint64_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:308
TimeDifferenceMicros
uint64_t TimeDifferenceMicros(struct timeval t0, struct timeval t1)
Definition: util-time.c:644
LOGGER_PCAP
@ LOGGER_PCAP
Definition: suricata-common.h:484
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:90
util-path.h
TcpSegmentPcapHdrStorage_::ts
struct timeval ts
Definition: stream-tcp-private.h:66
LOGMODE_NORMAL
#define LOGMODE_NORMAL
Definition: log-pcap.c:65
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:87
PKT_STREAM_NOPCAPLOG
#define PKT_STREAM_NOPCAPLOG
Definition: decode.h:1010
PcapLogData_::profile_data_size
uint64_t profile_data_size
Definition: log-pcap.c:155
TAILQ_NEXT
#define TAILQ_NEXT(elm, field)
Definition: queue.h:307
SCTIME_SECS
#define SCTIME_SECS(t)
Definition: util-time.h:57
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
PcapLogProfileSetup
void PcapLogProfileSetup(void)
Definition: log-pcap.c:2011
SCMkDir
#define SCMkDir(a, b)
Definition: util-path.h:45
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(...)
Definition: util-debug.h:502
PcapLogCallbackContext
Definition: log-pcap.c:554
PcapLogCompressionData_::buffer_size
uint64_t buffer_size
Definition: log-pcap.c:124
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputRegisterPacketModule
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output module.
Definition: output.c:176
PcapLogThreadData_
Definition: log-pcap.c:188
RunmodeGetCurrent
int RunmodeGetCurrent(void)
Definition: suricata.c:260
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
ConfigGetLogDirectory
const char * ConfigGetLogDirectory(void)
Definition: util-conf.c:38
LOGMODE_COND_ALERTS
@ LOGMODE_COND_ALERTS
Definition: log-pcap.c:71
PcapFileName_::filename
char * filename
Definition: log-pcap.c:93
Packet_::root
struct Packet_ * root
Definition: decode.h:622
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
str
#define str(s)
Definition: suricata-common.h:286
MAX_TOKS
#define MAX_TOKS
Definition: log-pcap.c:113
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
PcapLogCompressionData
struct PcapLogCompressionData_ PcapLogCompressionData
MODULE_NAME
#define MODULE_NAME
Definition: log-pcap.c:60
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint32_t, thread_cnt)
PcapLogCompressionData_::pcap_buf_wrapper
FILE * pcap_buf_wrapper
Definition: log-pcap.c:132
src
uint16_t src
Definition: app-layer-dnp3.h:5
util-buffer.h
PcapFileName_
Definition: log-pcap.c:92
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:562
TAILQ_HEAD
#define TAILQ_HEAD(name, type)
Definition: queue.h:230
PcapLogData_::profile_unlock
PcapLogProfileData profile_unlock
Definition: log-pcap.c:163
USE_STREAM_DEPTH_ENABLED
#define USE_STREAM_DEPTH_ENABLED
Definition: log-pcap.c:82
PcapLogData_::profile_close
PcapLogProfileData profile_close
Definition: log-pcap.c:165
PKT_NOPACKET_INSPECTION
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:985
FlowHasAlerts
int FlowHasAlerts(const Flow *f)
Check if flow has alerts.
Definition: flow.c:184
PcapFileName
struct PcapFileName_ PcapFileName
PCAP_BUFFER_TIMEOUT
#define PCAP_BUFFER_TIMEOUT
Definition: log-pcap.c:88
MIN_LIMIT
#define MIN_LIMIT
Definition: log-pcap.c:61
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
MemBuffer_::buffer
uint8_t * buffer
Definition: util-buffer.h:28
util-misc.h
PKT_HAS_TAG
#define PKT_HAS_TAG
Definition: decode.h:993
PcapLogData_
Definition: log-pcap.c:141
PcapLogThreadData_::pcap_log
PcapLogData * pcap_log
Definition: log-pcap.c:189
PKT_FIRST_ALERTS
#define PKT_FIRST_ALERTS
Definition: decode.h:1049
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
TAILQ_INSERT_BEFORE
#define TAILQ_INSERT_BEFORE(listelm, elm, field)
Definition: queue.h:277
PcapLogData_::profile_write
PcapLogProfileData profile_write
Definition: log-pcap.c:162
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
TS_FORMAT_USEC
#define TS_FORMAT_USEC
Definition: log-pcap.c:79
SCMutex
#define SCMutex
Definition: threads-debug.h:114
PcapLogData_::is_private
int is_private
Definition: log-pcap.c:144
LOGMODE_SGUIL
#define LOGMODE_SGUIL
Definition: log-pcap.c:66
MAX_FILENAMELEN
#define MAX_FILENAMELEN
Definition: log-pcap.c:114
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
output.h
LogModeConditionalType
enum LogModeConditionalType_ LogModeConditionalType
PcapLogCompressionFormat
PcapLogCompressionFormat
Definition: log-pcap.c:116
SCTIME_USECS
#define SCTIME_USECS(t)
Definition: util-time.h:56
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:814