suricata
util-logopenfile.c
Go to the documentation of this file.
1 /* vi: set et ts=4: */
2 /* Copyright (C) 2007-2022 Open Information Security Foundation
3  *
4  * You can copy, redistribute or modify this Program under the terms of
5  * the GNU General Public License version 2 as published by the Free
6  * Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * version 2 along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16  * 02110-1301, USA.
17  */
18 
19 /**
20  * \file
21  *
22  * \author Mike Pomraning <mpomraning@qualys.com>
23  *
24  * File-like output for logging: regular files and sockets.
25  */
26 
27 #include "suricata-common.h" /* errno.h, string.h, etc. */
28 #include "util-logopenfile.h"
29 #include "suricata.h"
30 #include "conf.h" /* ConfNode, etc. */
31 #include "output.h" /* DEFAULT_LOG_* */
32 #include "util-byte.h"
33 #include "util-conf.h"
34 #include "util-path.h"
35 #include "util-time.h"
36 
37 #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H)
38 #define BUILD_WITH_UNIXSOCKET
39 #include <sys/types.h>
40 #include <sys/socket.h>
41 #include <sys/un.h>
42 #endif
43 
44 #ifdef HAVE_LIBHIREDIS
45 #include "util-log-redis.h"
46 #endif /* HAVE_LIBHIREDIS */
47 
48 #define LOGFILE_NAME_MAX 255
49 
50 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append,
51  ThreadLogFileHashEntry *entry);
52 
53 // Threaded eve.json identifier
54 static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint32_t, eve_file_id, 1);
55 
56 #ifdef BUILD_WITH_UNIXSOCKET
57 /** \brief connect to the indicated local stream socket, logging any errors
58  * \param path filesystem path to connect to
59  * \param log_err, non-zero if connect failure should be logged.
60  * \retval FILE* on success (fdopen'd wrapper of underlying socket)
61  * \retval NULL on error
62  */
63 static FILE *
64 SCLogOpenUnixSocketFp(const char *path, int sock_type, int log_err)
65 {
66  struct sockaddr_un saun;
67  int s = -1;
68  FILE * ret = NULL;
69 
70  memset(&saun, 0x00, sizeof(saun));
71 
72  s = socket(PF_UNIX, sock_type, 0);
73  if (s < 0) goto err;
74 
75  saun.sun_family = AF_UNIX;
76  strlcpy(saun.sun_path, path, sizeof(saun.sun_path));
77 
78  if (connect(s, (const struct sockaddr *)&saun, sizeof(saun)) < 0)
79  goto err;
80 
81  ret = fdopen(s, "w");
82  if (ret == NULL)
83  goto err;
84 
85  return ret;
86 
87 err:
88  if (log_err)
90  "Error connecting to socket \"%s\": %s (will keep trying)", path, strerror(errno));
91 
92  if (s >= 0)
93  close(s);
94 
95  return NULL;
96 }
97 
98 /**
99  * \brief Attempt to reconnect a disconnected (or never-connected) Unix domain socket.
100  * \retval 1 if it is now connected; otherwise 0
101  */
102 static int SCLogUnixSocketReconnect(LogFileCtx *log_ctx)
103 {
104  int disconnected = 0;
105  if (log_ctx->fp) {
106  SCLogWarning("Write error on Unix socket \"%s\": %s; reconnecting...", log_ctx->filename,
107  strerror(errno));
108  fclose(log_ctx->fp);
109  log_ctx->fp = NULL;
110  log_ctx->reconn_timer = 0;
111  disconnected = 1;
112  }
113 
114  struct timeval tv;
115  uint64_t now;
116  gettimeofday(&tv, NULL);
117  now = (uint64_t)tv.tv_sec * 1000;
118  now += tv.tv_usec / 1000; /* msec resolution */
119  if (log_ctx->reconn_timer != 0 &&
120  (now - log_ctx->reconn_timer) < LOGFILE_RECONN_MIN_TIME) {
121  /* Don't bother to try reconnecting too often. */
122  return 0;
123  }
124  log_ctx->reconn_timer = now;
125 
126  log_ctx->fp = SCLogOpenUnixSocketFp(log_ctx->filename, log_ctx->sock_type, 0);
127  if (log_ctx->fp) {
128  /* Connected at last (or reconnected) */
129  SCLogNotice("Reconnected socket \"%s\"", log_ctx->filename);
130  } else if (disconnected) {
131  SCLogWarning("Reconnect failed: %s (will keep trying)", strerror(errno));
132  }
133 
134  return log_ctx->fp ? 1 : 0;
135 }
136 
137 static int SCLogFileWriteSocket(const char *buffer, int buffer_len,
138  LogFileCtx *ctx)
139 {
140  int tries = 0;
141  int ret = 0;
142  bool reopen = false;
143  if (ctx->fp == NULL && ctx->is_sock) {
144  SCLogUnixSocketReconnect(ctx);
145  }
146 tryagain:
147  ret = -1;
148  reopen = 0;
149  errno = 0;
150  if (ctx->fp != NULL) {
151  int fd = fileno(ctx->fp);
152  ssize_t size = send(fd, buffer, buffer_len, ctx->send_flags);
153  if (size > -1) {
154  ret = 0;
155  } else {
156  if (errno == EAGAIN || errno == EWOULDBLOCK) {
157  SCLogDebug("Socket would block, dropping event.");
158  } else if (errno == EINTR) {
159  if (tries++ == 0) {
160  SCLogDebug("Interrupted system call, trying again.");
161  goto tryagain;
162  }
163  SCLogDebug("Too many interrupted system calls, "
164  "dropping event.");
165  } else {
166  /* Some other error. Assume badness and reopen. */
167  SCLogDebug("Send failed: %s", strerror(errno));
168  reopen = true;
169  }
170  }
171  }
172 
173  if (reopen && tries++ == 0) {
174  if (SCLogUnixSocketReconnect(ctx)) {
175  goto tryagain;
176  }
177  }
178 
179  if (ret == -1) {
180  ctx->dropped++;
181  }
182 
183  return ret;
184 }
185 #endif /* BUILD_WITH_UNIXSOCKET */
186 static inline void OutputWriteLock(pthread_mutex_t *m)
187 {
188  SCMutexLock(m);
189 
190 }
191 
192 /**
193  * \brief Write buffer to log file.
194  * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
195  * characters successfully written).
196  */
197 static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
198 {
199  int ret = 0;
200 
201  BUG_ON(log_ctx->is_sock);
202 
203  /* Check for rotation. */
204  if (log_ctx->rotation_flag) {
205  log_ctx->rotation_flag = 0;
206  SCConfLogReopen(log_ctx);
207  }
208 
209  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
210  time_t now = time(NULL);
211  if (now >= log_ctx->rotate_time) {
212  SCConfLogReopen(log_ctx);
213  log_ctx->rotate_time = now + log_ctx->rotate_interval;
214  }
215  }
216 
217  if (log_ctx->fp) {
218  SCClearErrUnlocked(log_ctx->fp);
219  if (1 != SCFwriteUnlocked(buffer, buffer_len, 1, log_ctx->fp)) {
220  /* Only the first error is logged */
221  if (!log_ctx->output_errors) {
222  SCLogError("%s error while writing to %s",
223  SCFerrorUnlocked(log_ctx->fp) ? strerror(errno) : "unknown error",
224  log_ctx->filename);
225  }
226  log_ctx->output_errors++;
227  } else {
228  SCFflushUnlocked(log_ctx->fp);
229  }
230  }
231 
232  return ret;
233 }
234 
235 /**
236  * \brief Write buffer to log file.
237  * \retval 0 on failure; otherwise, the return value of fwrite (number of
238  * characters successfully written).
239  */
240 static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
241 {
242  OutputWriteLock(&log_ctx->fp_mutex);
243  int ret = 0;
244 
245 #ifdef BUILD_WITH_UNIXSOCKET
246  if (log_ctx->is_sock) {
247  ret = SCLogFileWriteSocket(buffer, buffer_len, log_ctx);
248  } else
249 #endif
250  {
251 
252  /* Check for rotation. */
253  if (log_ctx->rotation_flag) {
254  log_ctx->rotation_flag = 0;
255  SCConfLogReopen(log_ctx);
256  }
257 
258  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
259  time_t now = time(NULL);
260  if (now >= log_ctx->rotate_time) {
261  SCConfLogReopen(log_ctx);
262  log_ctx->rotate_time = now + log_ctx->rotate_interval;
263  }
264  }
265 
266  if (log_ctx->fp) {
267  clearerr(log_ctx->fp);
268  if (1 != fwrite(buffer, buffer_len, 1, log_ctx->fp)) {
269  /* Only the first error is logged */
270  if (!log_ctx->output_errors) {
271  SCLogError("%s error while writing to %s",
272  ferror(log_ctx->fp) ? strerror(errno) : "unknown error",
273  log_ctx->filename);
274  }
275  log_ctx->output_errors++;
276  } else {
277  fflush(log_ctx->fp);
278  }
279  }
280  }
281 
282  SCMutexUnlock(&log_ctx->fp_mutex);
283 
284  return ret;
285 }
286 
287 /** \brief generate filename based on pattern
288  * \param pattern pattern to use
289  * \retval char* on success
290  * \retval NULL on error
291  */
292 static char *SCLogFilenameFromPattern(const char *pattern)
293 {
294  char *filename = SCMalloc(PATH_MAX);
295  if (filename == NULL) {
296  return NULL;
297  }
298 
299  int rc = SCTimeToStringPattern(time(NULL), pattern, filename, PATH_MAX);
300  if (rc != 0) {
301  SCFree(filename);
302  return NULL;
303  }
304 
305  return filename;
306 }
307 
308 static void SCLogFileCloseNoLock(LogFileCtx *log_ctx)
309 {
310  SCLogDebug("Closing %s", log_ctx->filename);
311  if (log_ctx->fp)
312  fclose(log_ctx->fp);
313 
314  if (log_ctx->output_errors) {
315  SCLogError("There were %" PRIu64 " output errors to %s", log_ctx->output_errors,
316  log_ctx->filename);
317  }
318 }
319 
320 static void SCLogFileClose(LogFileCtx *log_ctx)
321 {
322  SCMutexLock(&log_ctx->fp_mutex);
323  SCLogFileCloseNoLock(log_ctx);
324  SCMutexUnlock(&log_ctx->fp_mutex);
325 }
326 
327 static char ThreadLogFileHashCompareFunc(
328  void *data1, uint16_t datalen1, void *data2, uint16_t datalen2)
329 {
332 
333  if (p1 == NULL || p2 == NULL)
334  return 0;
335 
336  return p1->thread_id == p2->thread_id;
337 }
338 static uint32_t ThreadLogFileHashFunc(HashTable *ht, void *data, uint16_t datalen)
339 {
340  const ThreadLogFileHashEntry *ent = (ThreadLogFileHashEntry *)data;
341 
342  return ent->thread_id % ht->array_size;
343 }
344 
345 static void ThreadLogFileHashFreeFunc(void *data)
346 {
347  BUG_ON(data == NULL);
348  ThreadLogFileHashEntry *thread_ent = (ThreadLogFileHashEntry *)data;
349 
350  if (thread_ent) {
351  LogFileCtx *lf_ctx = thread_ent->ctx;
352  /* Free the leaf log file entries */
353  if (!lf_ctx->threaded) {
354  LogFileFreeCtx(lf_ctx);
355  }
356  SCFree(thread_ent);
357  }
358 }
359 
360 bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx)
361 {
362  parent_ctx->threads = SCCalloc(1, sizeof(LogThreadedFileCtx));
363  if (!parent_ctx->threads) {
364  SCLogError("Unable to allocate threads container");
365  return false;
366  }
367 
368  parent_ctx->threads->ht = HashTableInit(255, ThreadLogFileHashFunc,
369  ThreadLogFileHashCompareFunc, ThreadLogFileHashFreeFunc);
370  if (!parent_ctx->threads->ht) {
371  FatalError("Unable to initialize thread/entry hash table");
372  }
373 
374  parent_ctx->threads->append = SCStrdup(append == NULL ? DEFAULT_LOG_MODE_APPEND : append);
375  if (!parent_ctx->threads->append) {
376  SCLogError("Unable to allocate threads append setting");
377  goto error_exit;
378  }
379 
380  SCMutexInit(&parent_ctx->threads->mutex, NULL);
381  return true;
382 
383 error_exit:
384 
385  if (parent_ctx->threads->append) {
386  SCFree(parent_ctx->threads->append);
387  }
388  if (parent_ctx->threads->ht) {
389  HashTableFree(parent_ctx->threads->ht);
390  }
391  SCFree(parent_ctx->threads);
392  parent_ctx->threads = NULL;
393  return false;
394 }
395 
396 /** \brief open the indicated file, logging any errors
397  * \param path filesystem path to open
398  * \param append_setting open file with O_APPEND: "yes" or "no"
399  * \param mode permissions to set on file
400  * \retval FILE* on success
401  * \retval NULL on error
402  */
403 static FILE *
404 SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
405 {
406  FILE *ret = NULL;
407 
408  char *filename = SCLogFilenameFromPattern(path);
409  if (filename == NULL) {
410  return NULL;
411  }
412 
413  int rc = SCCreateDirectoryTree(filename, false);
414  if (rc < 0) {
415  SCFree(filename);
416  return NULL;
417  }
418 
419  if (ConfValIsTrue(append_setting)) {
420  ret = fopen(filename, "a");
421  } else {
422  ret = fopen(filename, "w");
423  }
424 
425  if (ret == NULL) {
426  SCLogError("Error opening file: \"%s\": %s", filename, strerror(errno));
427  } else {
428  if (mode != 0) {
429 #ifdef OS_WIN32
430  int r = _chmod(filename, (mode_t)mode);
431 #else
432  int r = fchmod(fileno(ret), (mode_t)mode);
433 #endif
434  if (r < 0) {
435  SCLogWarning("Could not chmod %s to %o: %s", filename, mode, strerror(errno));
436  }
437  }
438  }
439 
440  SCFree(filename);
441  return ret;
442 }
443 
444 /** \brief open a generic output "log file", which may be a regular file or a socket
445  * \param conf ConfNode structure for the output section in question
446  * \param log_ctx Log file context allocated by caller
447  * \param default_filename Default name of file to open, if not specified in ConfNode
448  * \param rotate Register the file for rotation in HUP.
449  * \retval 0 on success
450  * \retval -1 on error
451  */
452 int
454  LogFileCtx *log_ctx,
455  const char *default_filename,
456  int rotate)
457 {
458  char log_path[PATH_MAX];
459  const char *log_dir;
460  const char *filename, *filetype;
461 
462  // Arg check
463  if (conf == NULL || log_ctx == NULL || default_filename == NULL) {
464  SCLogError("SCConfLogOpenGeneric(conf %p, ctx %p, default %p) "
465  "missing an argument",
466  conf, log_ctx, default_filename);
467  return -1;
468  }
469  if (log_ctx->fp != NULL) {
470  SCLogError("SCConfLogOpenGeneric: previously initialized Log CTX "
471  "encountered");
472  return -1;
473  }
474 
475  // Resolve the given config
476  filename = ConfNodeLookupChildValue(conf, "filename");
477  if (filename == NULL)
478  filename = default_filename;
479 
480  log_dir = ConfigGetLogDirectory();
481 
482  if (PathIsAbsolute(filename)) {
483  snprintf(log_path, PATH_MAX, "%s", filename);
484  } else {
485  snprintf(log_path, PATH_MAX, "%s/%s", log_dir, filename);
486  }
487 
488  /* Rotate log file based on time */
489  const char *rotate_int = ConfNodeLookupChildValue(conf, "rotate-interval");
490  if (rotate_int != NULL) {
491  time_t now = time(NULL);
492  log_ctx->flags |= LOGFILE_ROTATE_INTERVAL;
493 
494  /* Use a specific time */
495  if (strcmp(rotate_int, "minute") == 0) {
496  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
497  log_ctx->rotate_interval = 60;
498  } else if (strcmp(rotate_int, "hour") == 0) {
499  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
500  log_ctx->rotate_interval = 3600;
501  } else if (strcmp(rotate_int, "day") == 0) {
502  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
503  log_ctx->rotate_interval = 86400;
504  }
505 
506  /* Use a timer */
507  else {
508  log_ctx->rotate_interval = SCParseTimeSizeString(rotate_int);
509  if (log_ctx->rotate_interval == 0) {
510  FatalError("invalid rotate-interval value");
511  }
512  log_ctx->rotate_time = now + log_ctx->rotate_interval;
513  }
514  }
515 
516  filetype = ConfNodeLookupChildValue(conf, "filetype");
517  if (filetype == NULL)
518  filetype = DEFAULT_LOG_FILETYPE;
519 
520  const char *filemode = ConfNodeLookupChildValue(conf, "filemode");
521  uint32_t mode = 0;
522  if (filemode != NULL && StringParseUint32(&mode, 8, (uint16_t)strlen(filemode), filemode) > 0) {
523  log_ctx->filemode = mode;
524  }
525 
526  const char *append = ConfNodeLookupChildValue(conf, "append");
527  if (append == NULL)
528  append = DEFAULT_LOG_MODE_APPEND;
529 
530  /* JSON flags */
531  log_ctx->json_flags = JSON_PRESERVE_ORDER|JSON_COMPACT|
532  JSON_ENSURE_ASCII|JSON_ESCAPE_SLASH;
533 
534  ConfNode *json_flags = ConfNodeLookupChild(conf, "json");
535 
536  if (json_flags != 0) {
537  const char *preserve_order = ConfNodeLookupChildValue(json_flags,
538  "preserve-order");
539  if (preserve_order != NULL && ConfValIsFalse(preserve_order))
540  log_ctx->json_flags &= ~(JSON_PRESERVE_ORDER);
541 
542  const char *compact = ConfNodeLookupChildValue(json_flags, "compact");
543  if (compact != NULL && ConfValIsFalse(compact))
544  log_ctx->json_flags &= ~(JSON_COMPACT);
545 
546  const char *ensure_ascii = ConfNodeLookupChildValue(json_flags,
547  "ensure-ascii");
548  if (ensure_ascii != NULL && ConfValIsFalse(ensure_ascii))
549  log_ctx->json_flags &= ~(JSON_ENSURE_ASCII);
550 
551  const char *escape_slash = ConfNodeLookupChildValue(json_flags,
552  "escape-slash");
553  if (escape_slash != NULL && ConfValIsFalse(escape_slash))
554  log_ctx->json_flags &= ~(JSON_ESCAPE_SLASH);
555  }
556 
557 #ifdef BUILD_WITH_UNIXSOCKET
558  if (log_ctx->threaded) {
559  if (strcasecmp(filetype, "unix_stream") == 0 || strcasecmp(filetype, "unix_dgram") == 0) {
560  FatalError("Socket file types do not support threaded output");
561  }
562  }
563 #endif
564  // Now, what have we been asked to open?
565  if (strcasecmp(filetype, "unix_stream") == 0) {
566 #ifdef BUILD_WITH_UNIXSOCKET
567  /* Don't bail. May be able to connect later. */
568  log_ctx->is_sock = 1;
569  log_ctx->sock_type = SOCK_STREAM;
570  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM, 1);
571 #else
572  return -1;
573 #endif
574  } else if (strcasecmp(filetype, "unix_dgram") == 0) {
575 #ifdef BUILD_WITH_UNIXSOCKET
576  /* Don't bail. May be able to connect later. */
577  log_ctx->is_sock = 1;
578  log_ctx->sock_type = SOCK_DGRAM;
579  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM, 1);
580 #else
581  return -1;
582 #endif
583  } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
584  strcasecmp(filetype, "file") == 0) {
585  log_ctx->is_regular = 1;
586  if (!log_ctx->threaded) {
587  log_ctx->fp = SCLogOpenFileFp(log_path, append, log_ctx->filemode);
588  if (log_ctx->fp == NULL)
589  return -1; // Error already logged by Open...Fp routine
590  } else {
591  if (!SCLogOpenThreadedFile(log_path, append, log_ctx)) {
592  return -1;
593  }
594  }
595  if (rotate) {
597  }
598  } else {
599  SCLogError("Invalid entry for "
600  "%s.filetype. Expected \"regular\" (default), \"unix_stream\", "
601  "or \"unix_dgram\"",
602  conf->name);
603  }
604  log_ctx->filename = SCStrdup(log_path);
605  if (unlikely(log_ctx->filename == NULL)) {
606  SCLogError("Failed to allocate memory for filename");
607  return -1;
608  }
609 
610 #ifdef BUILD_WITH_UNIXSOCKET
611  /* If a socket and running live, do non-blocking writes. */
612  if (log_ctx->is_sock && !IsRunModeOffline(RunmodeGetCurrent())) {
613  SCLogInfo("Setting logging socket of non-blocking in live mode.");
614  log_ctx->send_flags |= MSG_DONTWAIT;
615  }
616 #endif
617  SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype,
618  filename);
619 
620  return 0;
621 }
622 
623 /**
624  * \brief Reopen a regular log file with the side-affect of truncating it.
625  *
626  * This is useful to clear the log file and start a new one, or to
627  * re-open the file after its been moved by something external
628  * (eg. logrotate).
629  */
631 {
632  if (!log_ctx->is_regular) {
633  /* Not supported and not needed on non-regular files. */
634  return 0;
635  }
636 
637  if (log_ctx->filename == NULL) {
638  SCLogWarning("Can't re-open LogFileCtx without a filename.");
639  return -1;
640  }
641 
642  if (log_ctx->fp != NULL) {
643  fclose(log_ctx->fp);
644  }
645 
646  /* Reopen the file. Append is forced in case the file was not
647  * moved as part of a rotation process. */
648  SCLogDebug("Reopening log file %s.", log_ctx->filename);
649  log_ctx->fp = SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode);
650  if (log_ctx->fp == NULL) {
651  return -1; // Already logged by Open..Fp routine.
652  }
653 
654  return 0;
655 }
656 
657 /** \brief LogFileNewCtx() Get a new LogFileCtx
658  * \retval LogFileCtx * pointer if successful, NULL if error
659  * */
661 {
662  LogFileCtx* lf_ctx;
663  lf_ctx = (LogFileCtx*)SCCalloc(1, sizeof(LogFileCtx));
664 
665  if (lf_ctx == NULL)
666  return NULL;
667 
668  lf_ctx->Write = SCLogFileWrite;
669  lf_ctx->Close = SCLogFileClose;
670 
671  return lf_ctx;
672 }
673 
674 /** \brief LogFileThread2Slot() Return a file entry
675  * \retval ThreadLogFileHashEntry * file entry for caller
676  *
677  * This function returns the file entry for the calling thread.
678  * Each thread -- identified by its operating system thread-id -- has its
679  * own file entry that includes a file pointer.
680  */
681 static ThreadLogFileHashEntry *LogFileThread2Slot(LogThreadedFileCtx *parent)
682 {
683  ThreadLogFileHashEntry thread_hash_entry;
684 
685  /* Check hash table for thread id*/
686  thread_hash_entry.thread_id = SCGetThreadIdLong();
688  HashTableLookup(parent->ht, &thread_hash_entry, sizeof(thread_hash_entry));
689 
690  if (!ent) {
691  ent = SCCalloc(1, sizeof(*ent));
692  if (!ent) {
693  FatalError("Unable to allocate thread/entry entry");
694  }
695  ent->thread_id = thread_hash_entry.thread_id;
696  SCLogDebug("Trying to add thread %ld to entry %d", ent->thread_id, ent->slot_number);
697  if (0 != HashTableAdd(parent->ht, ent, 0)) {
698  FatalError("Unable to add thread/entry mapping");
699  }
700  }
701  return ent;
702 }
703 
704 /** \brief LogFileEnsureExists() Ensure a log file context for the thread exists
705  * \param parent_ctx
706  * \retval LogFileCtx * pointer if successful; NULL otherwise
707  */
709 {
710  /* threaded output disabled */
711  if (!parent_ctx->threaded)
712  return parent_ctx;
713 
714  SCMutexLock(&parent_ctx->threads->mutex);
715  /* Find this thread's entry */
716  ThreadLogFileHashEntry *entry = LogFileThread2Slot(parent_ctx->threads);
717  SCLogDebug("Adding reference for thread %ld [slot %d] to file %s [ctx %p]", SCGetThreadIdLong(),
718  entry->slot_number, parent_ctx->filename, parent_ctx);
719 
720  bool new = entry->isopen;
721  /* has it been opened yet? */
722  if (!entry->isopen) {
723  SCLogDebug("Opening new file for thread/slot %d to file %s [ctx %p]", entry->slot_number,
724  parent_ctx->filename, parent_ctx);
725  if (LogFileNewThreadedCtx(
726  parent_ctx, parent_ctx->filename, parent_ctx->threads->append, entry)) {
727  entry->isopen = true;
728  } else {
729  SCLogError(
730  "Unable to open slot %d for file %s", entry->slot_number, parent_ctx->filename);
731  (void)HashTableRemove(parent_ctx->threads->ht, entry, 0);
732  }
733  }
734  SCMutexUnlock(&parent_ctx->threads->mutex);
735 
737  if (new) {
738  SCLogDebug("Existing file for thread/entry %p reference to file %s [ctx %p]", entry,
739  parent_ctx->filename, parent_ctx);
740  }
741  }
742 
743  return entry->ctx;
744 }
745 
746 /** \brief LogFileThreadedName() Create file name for threaded EVE storage
747  *
748  */
749 static bool LogFileThreadedName(
750  const char *original_name, char *threaded_name, size_t len, uint32_t unique_id)
751 {
752  sc_errno = SC_OK;
753 
754  if (strcmp("/dev/null", original_name) == 0) {
755  strlcpy(threaded_name, original_name, len);
756  return true;
757  }
758 
759  const char *base = SCBasename(original_name);
760  if (!base) {
761  FatalError("Invalid filename for threaded mode \"%s\"; "
762  "no basename found.",
763  original_name);
764  }
765 
766  /* Check if basename has an extension */
767  char *dot = strrchr(base, '.');
768  if (dot) {
769  char *tname = SCStrdup(original_name);
770  if (!tname) {
772  return false;
773  }
774 
775  /* Fetch extension location from original, not base
776  * for update
777  */
778  dot = strrchr(original_name, '.');
779  int dotpos = dot - original_name;
780  tname[dotpos] = '\0';
781  char *ext = tname + dotpos + 1;
782  if (strlen(tname) && strlen(ext)) {
783  snprintf(threaded_name, len, "%s.%u.%s", tname, unique_id, ext);
784  } else {
785  FatalError("Invalid filename for threaded mode \"%s\"; "
786  "filenames must include an extension, e.g: \"name.ext\"",
787  original_name);
788  }
789  SCFree(tname);
790  } else {
791  snprintf(threaded_name, len, "%s.%u", original_name, unique_id);
792  }
793  return true;
794 }
795 
796 /** \brief LogFileNewThreadedCtx() Create file context for threaded output
797  * \param parent_ctx
798  * \param log_path
799  * \param append
800  * \param entry
801  */
802 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append,
803  ThreadLogFileHashEntry *entry)
804 {
805  LogFileCtx *thread = SCCalloc(1, sizeof(LogFileCtx));
806  if (!thread) {
807  SCLogError("Unable to allocate thread file context entry %p", entry);
808  return false;
809  }
810 
811  *thread = *parent_ctx;
812  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
813  char fname[LOGFILE_NAME_MAX];
814  if (!LogFileThreadedName(log_path, fname, sizeof(fname), SC_ATOMIC_ADD(eve_file_id, 1))) {
815  SCLogError("Unable to create threaded filename for log");
816  goto error;
817  }
818  SCLogDebug("Thread open -- using name %s [replaces %s]", fname, log_path);
819  thread->fp = SCLogOpenFileFp(fname, append, thread->filemode);
820  if (thread->fp == NULL) {
821  goto error;
822  }
823  thread->filename = SCStrdup(fname);
824  if (!thread->filename) {
825  SCLogError("Unable to duplicate filename for context entry %p", entry);
826  goto error;
827  }
828  thread->is_regular = true;
829  thread->Write = SCLogFileWriteNoLock;
830  thread->Close = SCLogFileCloseNoLock;
832  } else if (parent_ctx->type == LOGFILE_TYPE_PLUGIN) {
833  entry->slot_number = SC_ATOMIC_ADD(eve_file_id, 1);
834  thread->plugin.plugin->ThreadInit(
835  thread->plugin.init_data, entry->slot_number, &thread->plugin.thread_data);
836  }
837  thread->threaded = false;
838  thread->parent = parent_ctx;
839  thread->entry = entry;
840  entry->ctx = thread;
841 
842  return true;
843 
844 error:
845  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
846  SC_ATOMIC_SUB(eve_file_id, 1);
847  if (thread->fp) {
848  thread->Close(thread);
849  }
850  }
851 
852  if (thread) {
853  SCFree(thread);
854  }
855  return false;
856 }
857 
858 /** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
859  * \param lf_ctx pointer to the OutputCtx
860  * \retval int 1 if successful, 0 if error
861  * */
863 {
864  if (lf_ctx == NULL) {
865  SCReturnInt(0);
866  }
867 
868  if (lf_ctx->type == LOGFILE_TYPE_PLUGIN) {
869  lf_ctx->plugin.plugin->Deinit(lf_ctx->plugin.init_data);
870  }
871 
872  if (lf_ctx->threaded) {
873  BUG_ON(lf_ctx->threads == NULL);
874  SCMutexDestroy(&lf_ctx->threads->mutex);
875  if (lf_ctx->threads->append)
876  SCFree(lf_ctx->threads->append);
877  if (lf_ctx->threads->ht) {
878  HashTableFree(lf_ctx->threads->ht);
879  }
880  SCFree(lf_ctx->threads);
881  } else {
882  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
883  if (lf_ctx->fp != NULL) {
884  lf_ctx->Close(lf_ctx);
885  }
886  }
887  SCMutexDestroy(&lf_ctx->fp_mutex);
888  }
889 
890  if (lf_ctx->prefix != NULL) {
891  SCFree(lf_ctx->prefix);
892  lf_ctx->prefix_len = 0;
893  }
894 
895  if(lf_ctx->filename != NULL)
896  SCFree(lf_ctx->filename);
897 
898  if (lf_ctx->sensor_name)
899  SCFree(lf_ctx->sensor_name);
900 
901  if (!lf_ctx->threaded) {
903  }
904 
905  memset(lf_ctx, 0, sizeof(*lf_ctx));
906  SCFree(lf_ctx);
907 
908  SCReturnInt(1);
909 }
910 
911 int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
912 {
913  if (file_ctx->type == LOGFILE_TYPE_FILE || file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
914  file_ctx->type == LOGFILE_TYPE_UNIX_STREAM) {
915  /* append \n for files only */
916  MemBufferWriteString(buffer, "\n");
917  file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
918  MEMBUFFER_OFFSET(buffer), file_ctx);
919  } else if (file_ctx->type == LOGFILE_TYPE_PLUGIN) {
920  file_ctx->plugin.plugin->Write((const char *)MEMBUFFER_BUFFER(buffer),
921  MEMBUFFER_OFFSET(buffer), file_ctx->plugin.init_data, file_ctx->plugin.thread_data);
922  }
923 #ifdef HAVE_LIBHIREDIS
924  else if (file_ctx->type == LOGFILE_TYPE_REDIS) {
925  SCMutexLock(&file_ctx->fp_mutex);
926  LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer),
927  MEMBUFFER_OFFSET(buffer));
928  SCMutexUnlock(&file_ctx->fp_mutex);
929  }
930 #endif
931 
932  return 0;
933 }
SCParseTimeSizeString
uint64_t SCParseTimeSizeString(const char *str)
Parse string containing time size (1m, 1h, etc).
Definition: util-time.c:570
LogFileCtx_::rotation_flag
int rotation_flag
Definition: util-logopenfile.h:150
util-byte.h
LogFileCtx_::prefix_len
size_t prefix_len
Definition: util-logopenfile.h:129
LogThreadedFileCtx_::append
char * append
Definition: util-logopenfile.h:62
SCEveFileType_::Write
int(* Write)(const char *buffer, int buffer_len, void *init_data, void *thread_data)
Definition: suricata-plugin.h:54
SCFerrorUnlocked
#define SCFerrorUnlocked
Definition: suricata-common.h:531
len
uint8_t len
Definition: app-layer-dnp3.h:2
LOGFILE_TYPE_REDIS
@ LOGFILE_TYPE_REDIS
Definition: util-logopenfile.h:42
LogFileCtx_::sensor_name
char * sensor_name
Definition: util-logopenfile.h:112
LogFileCtx_::reconn_timer
uint64_t reconn_timer
Definition: util-logopenfile.h:117
SC_LOG_DEBUG
@ SC_LOG_DEBUG
Definition: util-debug.h:57
LogFileCtx_::fp_mutex
SCMutex fp_mutex
Definition: util-logopenfile.h:95
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_DECL_AND_INIT_WITH_VAL
#define SC_ATOMIC_DECL_AND_INIT_WITH_VAL(type, name, val)
wrapper for declaring an atomic variable and initializing it to a specific value
Definition: util-atomic.h:304
LogFileNewCtx
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
Definition: util-logopenfile.c:660
SCGetSecondsUntil
uint64_t SCGetSecondsUntil(const char *str, time_t epoch)
Get seconds until a time unit changes.
Definition: util-time.c:621
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
LogFileCtx_::json_flags
size_t json_flags
Definition: util-logopenfile.h:147
LogFilePluginCtx_::init_data
void * init_data
Definition: util-logopenfile.h:67
LogFilePluginCtx_::thread_data
void * thread_data
Definition: util-logopenfile.h:68
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
LogFileCtx_
Definition: util-logopenfile.h:72
LOGFILE_RECONN_MIN_TIME
#define LOGFILE_RECONN_MIN_TIME
Definition: util-logopenfile.h:163
DEFAULT_LOG_MODE_APPEND
#define DEFAULT_LOG_MODE_APPEND
Definition: output.h:30
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
HashTable_
Definition: util-hash.h:35
util-log-redis.h
LogFileCtx_::Write
int(* Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:88
LogFileCtx_::filename
char * filename
Definition: util-logopenfile.h:106
LOGFILE_NAME_MAX
#define LOGFILE_NAME_MAX
Definition: util-logopenfile.c:48
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:276
LogFileCtx_::plugin
LogFilePluginCtx plugin
Definition: util-logopenfile.h:91
m
SCMutex m
Definition: flow-hash.h:6
ThreadLogFileHashEntry::ctx
struct LogFileCtx_ * ctx
Definition: util-logopenfile.h:55
LogFileCtx_::send_flags
uint8_t send_flags
Definition: util-logopenfile.h:140
SCConfLogReopen
int SCConfLogReopen(LogFileCtx *log_ctx)
Reopen a regular log file with the side-affect of truncating it.
Definition: util-logopenfile.c:630
IsRunModeOffline
bool IsRunModeOffline(enum RunModes run_mode_to_check)
Definition: runmodes.c:574
ThreadLogFileHashEntry::isopen
bool isopen
Definition: util-logopenfile.h:54
SCClearErrUnlocked
#define SCClearErrUnlocked
Definition: suricata-common.h:530
ThreadLogFileHashEntry::slot_number
int slot_number
Definition: util-logopenfile.h:53
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:535
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
HashTableFree
void HashTableFree(HashTable *ht)
Definition: util-hash.c:80
HashTable_::array_size
uint32_t array_size
Definition: util-hash.h:37
SCConfLogOpenGeneric
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: util-logopenfile.c:453
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
LogFileCtx_::sock_type
int sock_type
Definition: util-logopenfile.h:116
OutputRegisterFileRotationFlag
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
Definition: output.c:832
SCTimeToStringPattern
int SCTimeToStringPattern(time_t epoch, const char *pattern, char *str, size_t size)
Convert epoch time to string pattern.
Definition: util-time.c:541
LogFileWrite
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
Definition: util-logopenfile.c:911
LogFileCtx_::rotate_interval
uint64_t rotate_interval
Definition: util-logopenfile.h:124
DEFAULT_LOG_FILETYPE
#define DEFAULT_LOG_FILETYPE
Definition: output.h:31
LogFileCtx_::is_sock
int is_sock
Definition: util-logopenfile.h:115
SCFwriteUnlocked
#define SCFwriteUnlocked
Definition: suricata-common.h:528
LogFileCtx_::dropped
uint64_t dropped
Definition: util-logopenfile.h:157
SCFflushUnlocked
#define SCFflushUnlocked
Definition: suricata-common.h:529
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
HashTableLookup
void * HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:194
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:708
HashTableRemove
int HashTableRemove(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:153
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
util-time.h
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
HashTableAdd
int HashTableAdd(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:114
LogFileCtx_::type
enum LogFileType type
Definition: util-logopenfile.h:103
LOGFILE_TYPE_FILE
@ LOGFILE_TYPE_FILE
Definition: util-logopenfile.h:39
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:295
LogFileCtx_::threads
LogThreadedFileCtx * threads
Definition: util-logopenfile.h:80
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
ThreadLogFileHashEntry::thread_id
uint64_t thread_id
Definition: util-logopenfile.h:52
LOGFILE_TYPE_PLUGIN
@ LOGFILE_TYPE_PLUGIN
Definition: util-logopenfile.h:43
conf.h
SC_OK
@ SC_OK
Definition: util-error.h:27
SCBasename
const char * SCBasename(const char *path)
Definition: util-path.c:235
LOGFILE_TYPE_UNIX_DGRAM
@ LOGFILE_TYPE_UNIX_DGRAM
Definition: util-logopenfile.h:40
SCLogOpenThreadedFile
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx)
Definition: util-logopenfile.c:360
LOGFILE_TYPE_UNIX_STREAM
@ LOGFILE_TYPE_UNIX_STREAM
Definition: util-logopenfile.h:41
MemBuffer_
Definition: util-buffer.h:27
LogFileCtx_::is_regular
uint8_t is_regular
Definition: util-logopenfile.h:144
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
SCGetThreadIdLong
#define SCGetThreadIdLong(...)
Definition: threads.h:255
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:784
LogFileCtx_::parent
struct LogFileCtx_ * parent
Definition: util-logopenfile.h:99
util-conf.h
LogThreadedFileCtx_
Definition: util-logopenfile.h:59
suricata-common.h
LogFilePluginCtx_::plugin
SCEveFileType * plugin
Definition: util-logopenfile.h:66
util-path.h
LogFileCtx_::output_errors
uint64_t output_errors
Definition: util-logopenfile.h:159
ThreadLogFileHashEntry
Definition: util-logopenfile.h:51
ConfNode_::name
char * name
Definition: conf.h:33
MemBufferWriteString
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
LogFileCtx_::entry
ThreadLogFileHashEntry * entry
Definition: util-logopenfile.h:100
LogFileFreeCtx
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
Definition: util-logopenfile.c:862
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
LogFileCtx_::rotate_time
time_t rotate_time
Definition: util-logopenfile.h:121
SCEveFileType_::Deinit
void(* Deinit)(void *init_data)
Definition: suricata-plugin.h:56
FatalError
#define FatalError(...)
Definition: util-debug.h:502
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
RunmodeGetCurrent
int RunmodeGetCurrent(void)
Definition: suricata.c:254
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
ConfigGetLogDirectory
const char * ConfigGetLogDirectory(void)
Definition: util-conf.c:37
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
LOGFILE_ROTATE_INTERVAL
#define LOGFILE_ROTATE_INTERVAL
Definition: util-logopenfile.h:166
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:560
LogThreadedFileCtx_::mutex
SCMutex mutex
Definition: util-logopenfile.h:60
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
LogFileCtx_::prefix
char * prefix
Definition: util-logopenfile.h:128
HashTableInit
HashTable * HashTableInit(uint32_t size, uint32_t(*Hash)(struct HashTable_ *, void *, uint16_t), char(*Compare)(void *, uint16_t, void *, uint16_t), void(*Free)(void *))
Definition: util-hash.c:35
sc_log_global_log_level
SCLogLevel sc_log_global_log_level
Holds the global log level. Is the same as sc_log_config->log_level.
Definition: util-debug.c:101
suricata.h
LogFileCtx_::flags
uint8_t flags
Definition: util-logopenfile.h:137
LogFileCtx_::Close
void(* Close)(struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:89
MEMBUFFER_BUFFER
#define MEMBUFFER_BUFFER(mem_buffer)
Get the MemBuffers underlying buffer.
Definition: util-buffer.h:50
SCEveFileType_::ThreadInit
int(* ThreadInit)(void *init_data, int thread_id, void **thread_data)
Definition: suricata-plugin.h:58
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
LogThreadedFileCtx_::ht
HashTable * ht
Definition: util-logopenfile.h:61
OutputUnregisterFileRotationFlag
void OutputUnregisterFileRotationFlag(int *flag)
Unregister a file rotation flag.
Definition: output.c:853
MEMBUFFER_OFFSET
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
Definition: util-buffer.h:55
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
LogFileCtx_::fp
FILE * fp
Definition: util-logopenfile.h:74
output.h
LogFileCtx_::threaded
bool threaded
Definition: util-logopenfile.h:98
SCCreateDirectoryTree
int SCCreateDirectoryTree(const char *path, const bool final)
Recursively create a directory.
Definition: util-path.c:123
LogFileCtx_::filemode
uint32_t filemode
Definition: util-logopenfile.h:109
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:812