suricata
util-logopenfile.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Mike Pomraning <mpomraning@qualys.com>
22  *
23  * File-like output for logging: regular files and sockets.
24  */
25 
26 #include "suricata-common.h" /* errno.h, string.h, etc. */
27 #include "util-logopenfile.h"
28 #include "suricata.h"
29 #include "conf.h" /* ConfNode, etc. */
30 #include "output.h" /* DEFAULT_LOG_* */
31 #include "util-byte.h"
32 #include "util-conf.h"
33 #include "util-path.h"
34 #include "util-time.h"
35 
36 #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H)
37 #define BUILD_WITH_UNIXSOCKET
38 #include <sys/types.h>
39 #include <sys/socket.h>
40 #include <sys/un.h>
41 #endif
42 
43 #ifdef HAVE_LIBHIREDIS
44 #include "util-log-redis.h"
45 #endif /* HAVE_LIBHIREDIS */
46 
47 #define LOGFILE_NAME_MAX 255
48 
49 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append,
50  ThreadLogFileHashEntry *entry);
51 
52 // Threaded eve.json identifier
53 static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint32_t, eve_file_id, 1);
54 
55 #ifdef BUILD_WITH_UNIXSOCKET
56 /** \brief connect to the indicated local stream socket, logging any errors
57  * \param path filesystem path to connect to
58  * \param log_err, non-zero if connect failure should be logged.
59  * \retval FILE* on success (fdopen'd wrapper of underlying socket)
60  * \retval NULL on error
61  */
62 static FILE *
63 SCLogOpenUnixSocketFp(const char *path, int sock_type, int log_err)
64 {
65  struct sockaddr_un saun;
66  int s = -1;
67  FILE * ret = NULL;
68 
69  memset(&saun, 0x00, sizeof(saun));
70 
71  s = socket(PF_UNIX, sock_type, 0);
72  if (s < 0) goto err;
73 
74  saun.sun_family = AF_UNIX;
75  strlcpy(saun.sun_path, path, sizeof(saun.sun_path));
76 
77  if (connect(s, (const struct sockaddr *)&saun, sizeof(saun)) < 0)
78  goto err;
79 
80  ret = fdopen(s, "w");
81  if (ret == NULL)
82  goto err;
83 
84  return ret;
85 
86 err:
87  if (log_err)
89  "Error connecting to socket \"%s\": %s (will keep trying)", path, strerror(errno));
90 
91  if (s >= 0)
92  close(s);
93 
94  return NULL;
95 }
96 
97 /**
98  * \brief Attempt to reconnect a disconnected (or never-connected) Unix domain socket.
99  * \retval 1 if it is now connected; otherwise 0
100  */
101 static int SCLogUnixSocketReconnect(LogFileCtx *log_ctx)
102 {
103  int disconnected = 0;
104  if (log_ctx->fp) {
105  SCLogWarning("Write error on Unix socket \"%s\": %s; reconnecting...", log_ctx->filename,
106  strerror(errno));
107  fclose(log_ctx->fp);
108  log_ctx->fp = NULL;
109  log_ctx->reconn_timer = 0;
110  disconnected = 1;
111  }
112 
113  struct timeval tv;
114  uint64_t now;
115  gettimeofday(&tv, NULL);
116  now = (uint64_t)tv.tv_sec * 1000;
117  now += tv.tv_usec / 1000; /* msec resolution */
118  if (log_ctx->reconn_timer != 0 &&
119  (now - log_ctx->reconn_timer) < LOGFILE_RECONN_MIN_TIME) {
120  /* Don't bother to try reconnecting too often. */
121  return 0;
122  }
123  log_ctx->reconn_timer = now;
124 
125  log_ctx->fp = SCLogOpenUnixSocketFp(log_ctx->filename, log_ctx->sock_type, 0);
126  if (log_ctx->fp) {
127  /* Connected at last (or reconnected) */
128  SCLogNotice("Reconnected socket \"%s\"", log_ctx->filename);
129  } else if (disconnected) {
130  SCLogWarning("Reconnect failed: %s (will keep trying)", strerror(errno));
131  }
132 
133  return log_ctx->fp ? 1 : 0;
134 }
135 
136 static int SCLogFileWriteSocket(const char *buffer, int buffer_len,
137  LogFileCtx *ctx)
138 {
139  int tries = 0;
140  int ret = 0;
141  bool reopen = false;
142  if (ctx->fp == NULL && ctx->is_sock) {
143  SCLogUnixSocketReconnect(ctx);
144  }
145 tryagain:
146  ret = -1;
147  reopen = 0;
148  errno = 0;
149  if (ctx->fp != NULL) {
150  int fd = fileno(ctx->fp);
151  ssize_t size = send(fd, buffer, buffer_len, ctx->send_flags);
152  if (size > -1) {
153  ret = 0;
154  } else {
155  if (errno == EAGAIN || errno == EWOULDBLOCK) {
156  SCLogDebug("Socket would block, dropping event.");
157  } else if (errno == EINTR) {
158  if (tries++ == 0) {
159  SCLogDebug("Interrupted system call, trying again.");
160  goto tryagain;
161  }
162  SCLogDebug("Too many interrupted system calls, "
163  "dropping event.");
164  } else {
165  /* Some other error. Assume badness and reopen. */
166  SCLogDebug("Send failed: %s", strerror(errno));
167  reopen = true;
168  }
169  }
170  }
171 
172  if (reopen && tries++ == 0) {
173  if (SCLogUnixSocketReconnect(ctx)) {
174  goto tryagain;
175  }
176  }
177 
178  if (ret == -1) {
179  ctx->dropped++;
180  }
181 
182  return ret;
183 }
184 #endif /* BUILD_WITH_UNIXSOCKET */
185 static inline void OutputWriteLock(pthread_mutex_t *m)
186 {
187  SCMutexLock(m);
188 
189 }
190 
191 /**
192  * \brief Write buffer to log file.
193  * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
194  * characters successfully written).
195  */
196 static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
197 {
198  int ret = 0;
199 
200  BUG_ON(log_ctx->is_sock);
201 
202  /* Check for rotation. */
203  if (log_ctx->rotation_flag) {
204  log_ctx->rotation_flag = 0;
205  SCConfLogReopen(log_ctx);
206  }
207 
208  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
209  time_t now = time(NULL);
210  if (now >= log_ctx->rotate_time) {
211  SCConfLogReopen(log_ctx);
212  log_ctx->rotate_time = now + log_ctx->rotate_interval;
213  }
214  }
215 
216  if (log_ctx->fp) {
217  SCClearErrUnlocked(log_ctx->fp);
218  if (1 != SCFwriteUnlocked(buffer, buffer_len, 1, log_ctx->fp)) {
219  /* Only the first error is logged */
220  if (!log_ctx->output_errors) {
221  SCLogError("%s error while writing to %s",
222  SCFerrorUnlocked(log_ctx->fp) ? strerror(errno) : "unknown error",
223  log_ctx->filename);
224  }
225  log_ctx->output_errors++;
226  } else {
227  SCFflushUnlocked(log_ctx->fp);
228  }
229  }
230 
231  return ret;
232 }
233 
234 /**
235  * \brief Write buffer to log file.
236  * \retval 0 on failure; otherwise, the return value of fwrite (number of
237  * characters successfully written).
238  */
239 static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
240 {
241  OutputWriteLock(&log_ctx->fp_mutex);
242  int ret = 0;
243 
244 #ifdef BUILD_WITH_UNIXSOCKET
245  if (log_ctx->is_sock) {
246  ret = SCLogFileWriteSocket(buffer, buffer_len, log_ctx);
247  } else
248 #endif
249  {
250 
251  /* Check for rotation. */
252  if (log_ctx->rotation_flag) {
253  log_ctx->rotation_flag = 0;
254  SCConfLogReopen(log_ctx);
255  }
256 
257  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
258  time_t now = time(NULL);
259  if (now >= log_ctx->rotate_time) {
260  SCConfLogReopen(log_ctx);
261  log_ctx->rotate_time = now + log_ctx->rotate_interval;
262  }
263  }
264 
265  if (log_ctx->fp) {
266  clearerr(log_ctx->fp);
267  if (1 != fwrite(buffer, buffer_len, 1, log_ctx->fp)) {
268  /* Only the first error is logged */
269  if (!log_ctx->output_errors) {
270  SCLogError("%s error while writing to %s",
271  ferror(log_ctx->fp) ? strerror(errno) : "unknown error",
272  log_ctx->filename);
273  }
274  log_ctx->output_errors++;
275  } else {
276  fflush(log_ctx->fp);
277  }
278  }
279  }
280 
281  SCMutexUnlock(&log_ctx->fp_mutex);
282 
283  return ret;
284 }
285 
286 /** \brief generate filename based on pattern
287  * \param pattern pattern to use
288  * \retval char* on success
289  * \retval NULL on error
290  */
291 static char *SCLogFilenameFromPattern(const char *pattern)
292 {
293  char *filename = SCMalloc(PATH_MAX);
294  if (filename == NULL) {
295  return NULL;
296  }
297 
298  int rc = SCTimeToStringPattern(time(NULL), pattern, filename, PATH_MAX);
299  if (rc != 0) {
300  SCFree(filename);
301  return NULL;
302  }
303 
304  return filename;
305 }
306 
307 static void SCLogFileCloseNoLock(LogFileCtx *log_ctx)
308 {
309  SCLogDebug("Closing %s", log_ctx->filename);
310  if (log_ctx->fp)
311  fclose(log_ctx->fp);
312 
313  if (log_ctx->output_errors) {
314  SCLogError("There were %" PRIu64 " output errors to %s", log_ctx->output_errors,
315  log_ctx->filename);
316  }
317 }
318 
319 static void SCLogFileClose(LogFileCtx *log_ctx)
320 {
321  SCMutexLock(&log_ctx->fp_mutex);
322  SCLogFileCloseNoLock(log_ctx);
323  SCMutexUnlock(&log_ctx->fp_mutex);
324 }
325 
326 static char ThreadLogFileHashCompareFunc(
327  void *data1, uint16_t datalen1, void *data2, uint16_t datalen2)
328 {
331 
332  if (p1 == NULL || p2 == NULL)
333  return 0;
334 
335  return p1->thread_id == p2->thread_id;
336 }
337 static uint32_t ThreadLogFileHashFunc(HashTable *ht, void *data, uint16_t datalen)
338 {
339  const ThreadLogFileHashEntry *ent = (ThreadLogFileHashEntry *)data;
340 
341  return ent->thread_id % ht->array_size;
342 }
343 
344 static void ThreadLogFileHashFreeFunc(void *data)
345 {
346  BUG_ON(data == NULL);
347  ThreadLogFileHashEntry *thread_ent = (ThreadLogFileHashEntry *)data;
348 
349  if (thread_ent) {
350  LogFileCtx *lf_ctx = thread_ent->ctx;
351  /* Free the leaf log file entries */
352  if (!lf_ctx->threaded) {
353  LogFileFreeCtx(lf_ctx);
354  }
355  SCFree(thread_ent);
356  }
357 }
358 
359 bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx)
360 {
361  parent_ctx->threads = SCCalloc(1, sizeof(LogThreadedFileCtx));
362  if (!parent_ctx->threads) {
363  SCLogError("Unable to allocate threads container");
364  return false;
365  }
366 
367  parent_ctx->threads->ht = HashTableInit(255, ThreadLogFileHashFunc,
368  ThreadLogFileHashCompareFunc, ThreadLogFileHashFreeFunc);
369  if (!parent_ctx->threads->ht) {
370  FatalError("Unable to initialize thread/entry hash table");
371  }
372 
373  parent_ctx->threads->append = SCStrdup(append == NULL ? DEFAULT_LOG_MODE_APPEND : append);
374  if (!parent_ctx->threads->append) {
375  SCLogError("Unable to allocate threads append setting");
376  goto error_exit;
377  }
378 
379  SCMutexInit(&parent_ctx->threads->mutex, NULL);
380  return true;
381 
382 error_exit:
383 
384  if (parent_ctx->threads->append) {
385  SCFree(parent_ctx->threads->append);
386  }
387  if (parent_ctx->threads->ht) {
388  HashTableFree(parent_ctx->threads->ht);
389  }
390  SCFree(parent_ctx->threads);
391  parent_ctx->threads = NULL;
392  return false;
393 }
394 
395 /** \brief open the indicated file, logging any errors
396  * \param path filesystem path to open
397  * \param append_setting open file with O_APPEND: "yes" or "no"
398  * \param mode permissions to set on file
399  * \retval FILE* on success
400  * \retval NULL on error
401  */
402 static FILE *
403 SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
404 {
405  FILE *ret = NULL;
406 
407  char *filename = SCLogFilenameFromPattern(path);
408  if (filename == NULL) {
409  return NULL;
410  }
411 
412  int rc = SCCreateDirectoryTree(filename, false);
413  if (rc < 0) {
414  SCFree(filename);
415  return NULL;
416  }
417 
418  if (ConfValIsTrue(append_setting)) {
419  ret = fopen(filename, "a");
420  } else {
421  ret = fopen(filename, "w");
422  }
423 
424  if (ret == NULL) {
425  SCLogError("Error opening file: \"%s\": %s", filename, strerror(errno));
426  } else {
427  if (mode != 0) {
428 #ifdef OS_WIN32
429  int r = _chmod(filename, (mode_t)mode);
430 #else
431  int r = fchmod(fileno(ret), (mode_t)mode);
432 #endif
433  if (r < 0) {
434  SCLogWarning("Could not chmod %s to %o: %s", filename, mode, strerror(errno));
435  }
436  }
437  }
438 
439  SCFree(filename);
440  return ret;
441 }
442 
443 /** \brief open a generic output "log file", which may be a regular file or a socket
444  * \param conf ConfNode structure for the output section in question
445  * \param log_ctx Log file context allocated by caller
446  * \param default_filename Default name of file to open, if not specified in ConfNode
447  * \param rotate Register the file for rotation in HUP.
448  * \retval 0 on success
449  * \retval -1 on error
450  */
451 int
453  LogFileCtx *log_ctx,
454  const char *default_filename,
455  int rotate)
456 {
457  char log_path[PATH_MAX];
458  const char *log_dir;
459  const char *filename, *filetype;
460 
461  // Arg check
462  if (conf == NULL || log_ctx == NULL || default_filename == NULL) {
463  SCLogError("SCConfLogOpenGeneric(conf %p, ctx %p, default %p) "
464  "missing an argument",
465  conf, log_ctx, default_filename);
466  return -1;
467  }
468  if (log_ctx->fp != NULL) {
469  SCLogError("SCConfLogOpenGeneric: previously initialized Log CTX "
470  "encountered");
471  return -1;
472  }
473 
474  // Resolve the given config
475  filename = ConfNodeLookupChildValue(conf, "filename");
476  if (filename == NULL)
477  filename = default_filename;
478 
479  log_dir = ConfigGetLogDirectory();
480 
481  if (PathIsAbsolute(filename)) {
482  snprintf(log_path, PATH_MAX, "%s", filename);
483  } else {
484  snprintf(log_path, PATH_MAX, "%s/%s", log_dir, filename);
485  }
486 
487  /* Rotate log file based on time */
488  const char *rotate_int = ConfNodeLookupChildValue(conf, "rotate-interval");
489  if (rotate_int != NULL) {
490  time_t now = time(NULL);
491  log_ctx->flags |= LOGFILE_ROTATE_INTERVAL;
492 
493  /* Use a specific time */
494  if (strcmp(rotate_int, "minute") == 0) {
495  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
496  log_ctx->rotate_interval = 60;
497  } else if (strcmp(rotate_int, "hour") == 0) {
498  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
499  log_ctx->rotate_interval = 3600;
500  } else if (strcmp(rotate_int, "day") == 0) {
501  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
502  log_ctx->rotate_interval = 86400;
503  }
504 
505  /* Use a timer */
506  else {
507  log_ctx->rotate_interval = SCParseTimeSizeString(rotate_int);
508  if (log_ctx->rotate_interval == 0) {
509  FatalError("invalid rotate-interval value");
510  }
511  log_ctx->rotate_time = now + log_ctx->rotate_interval;
512  }
513  }
514 
515  filetype = ConfNodeLookupChildValue(conf, "filetype");
516  if (filetype == NULL)
517  filetype = DEFAULT_LOG_FILETYPE;
518 
519  const char *filemode = ConfNodeLookupChildValue(conf, "filemode");
520  uint32_t mode = 0;
521  if (filemode != NULL && StringParseUint32(&mode, 8, (uint16_t)strlen(filemode), filemode) > 0) {
522  log_ctx->filemode = mode;
523  }
524 
525  const char *append = ConfNodeLookupChildValue(conf, "append");
526  if (append == NULL)
527  append = DEFAULT_LOG_MODE_APPEND;
528 
529  /* JSON flags */
530  log_ctx->json_flags = JSON_PRESERVE_ORDER|JSON_COMPACT|
531  JSON_ENSURE_ASCII|JSON_ESCAPE_SLASH;
532 
533  ConfNode *json_flags = ConfNodeLookupChild(conf, "json");
534 
535  if (json_flags != 0) {
536  const char *preserve_order = ConfNodeLookupChildValue(json_flags,
537  "preserve-order");
538  if (preserve_order != NULL && ConfValIsFalse(preserve_order))
539  log_ctx->json_flags &= ~(JSON_PRESERVE_ORDER);
540 
541  const char *compact = ConfNodeLookupChildValue(json_flags, "compact");
542  if (compact != NULL && ConfValIsFalse(compact))
543  log_ctx->json_flags &= ~(JSON_COMPACT);
544 
545  const char *ensure_ascii = ConfNodeLookupChildValue(json_flags,
546  "ensure-ascii");
547  if (ensure_ascii != NULL && ConfValIsFalse(ensure_ascii))
548  log_ctx->json_flags &= ~(JSON_ENSURE_ASCII);
549 
550  const char *escape_slash = ConfNodeLookupChildValue(json_flags,
551  "escape-slash");
552  if (escape_slash != NULL && ConfValIsFalse(escape_slash))
553  log_ctx->json_flags &= ~(JSON_ESCAPE_SLASH);
554  }
555 
556 #ifdef BUILD_WITH_UNIXSOCKET
557  if (log_ctx->threaded) {
558  if (strcasecmp(filetype, "unix_stream") == 0 || strcasecmp(filetype, "unix_dgram") == 0) {
559  FatalError("Socket file types do not support threaded output");
560  }
561  }
562 #endif
563  // Now, what have we been asked to open?
564  if (strcasecmp(filetype, "unix_stream") == 0) {
565 #ifdef BUILD_WITH_UNIXSOCKET
566  /* Don't bail. May be able to connect later. */
567  log_ctx->is_sock = 1;
568  log_ctx->sock_type = SOCK_STREAM;
569  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM, 1);
570 #else
571  return -1;
572 #endif
573  } else if (strcasecmp(filetype, "unix_dgram") == 0) {
574 #ifdef BUILD_WITH_UNIXSOCKET
575  /* Don't bail. May be able to connect later. */
576  log_ctx->is_sock = 1;
577  log_ctx->sock_type = SOCK_DGRAM;
578  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM, 1);
579 #else
580  return -1;
581 #endif
582  } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
583  strcasecmp(filetype, "file") == 0) {
584  log_ctx->is_regular = 1;
585  if (!log_ctx->threaded) {
586  log_ctx->fp = SCLogOpenFileFp(log_path, append, log_ctx->filemode);
587  if (log_ctx->fp == NULL)
588  return -1; // Error already logged by Open...Fp routine
589  } else {
590  if (!SCLogOpenThreadedFile(log_path, append, log_ctx)) {
591  return -1;
592  }
593  }
594  if (rotate) {
596  }
597  } else {
598  SCLogError("Invalid entry for "
599  "%s.filetype. Expected \"regular\" (default), \"unix_stream\", "
600  "or \"unix_dgram\"",
601  conf->name);
602  }
603  log_ctx->filename = SCStrdup(log_path);
604  if (unlikely(log_ctx->filename == NULL)) {
605  SCLogError("Failed to allocate memory for filename");
606  return -1;
607  }
608 
609 #ifdef BUILD_WITH_UNIXSOCKET
610  /* If a socket and running live, do non-blocking writes. */
611  if (log_ctx->is_sock && !IsRunModeOffline(RunmodeGetCurrent())) {
612  SCLogInfo("Setting logging socket of non-blocking in live mode.");
613  log_ctx->send_flags |= MSG_DONTWAIT;
614  }
615 #endif
616  SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype,
617  filename);
618 
619  return 0;
620 }
621 
622 /**
623  * \brief Reopen a regular log file with the side-affect of truncating it.
624  *
625  * This is useful to clear the log file and start a new one, or to
626  * re-open the file after its been moved by something external
627  * (eg. logrotate).
628  */
630 {
631  if (!log_ctx->is_regular) {
632  /* Not supported and not needed on non-regular files. */
633  return 0;
634  }
635 
636  if (log_ctx->filename == NULL) {
637  SCLogWarning("Can't re-open LogFileCtx without a filename.");
638  return -1;
639  }
640 
641  if (log_ctx->fp != NULL) {
642  fclose(log_ctx->fp);
643  }
644 
645  /* Reopen the file. Append is forced in case the file was not
646  * moved as part of a rotation process. */
647  SCLogDebug("Reopening log file %s.", log_ctx->filename);
648  log_ctx->fp = SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode);
649  if (log_ctx->fp == NULL) {
650  return -1; // Already logged by Open..Fp routine.
651  }
652 
653  return 0;
654 }
655 
656 /** \brief LogFileNewCtx() Get a new LogFileCtx
657  * \retval LogFileCtx * pointer if successful, NULL if error
658  * */
660 {
661  LogFileCtx* lf_ctx;
662  lf_ctx = (LogFileCtx*)SCCalloc(1, sizeof(LogFileCtx));
663 
664  if (lf_ctx == NULL)
665  return NULL;
666 
667  lf_ctx->Write = SCLogFileWrite;
668  lf_ctx->Close = SCLogFileClose;
669 
670  return lf_ctx;
671 }
672 
673 /** \brief LogFileThread2Slot() Return a file entry
674  * \retval ThreadLogFileHashEntry * file entry for caller
675  *
676  * This function returns the file entry for the calling thread.
677  * Each thread -- identified by its operating system thread-id -- has its
678  * own file entry that includes a file pointer.
679  */
680 static ThreadLogFileHashEntry *LogFileThread2Slot(LogThreadedFileCtx *parent)
681 {
682  ThreadLogFileHashEntry thread_hash_entry;
683 
684  /* Check hash table for thread id*/
685  thread_hash_entry.thread_id = SCGetThreadIdLong();
687  HashTableLookup(parent->ht, &thread_hash_entry, sizeof(thread_hash_entry));
688 
689  if (!ent) {
690  ent = SCCalloc(1, sizeof(*ent));
691  if (!ent) {
692  FatalError("Unable to allocate thread/entry entry");
693  }
694  ent->thread_id = thread_hash_entry.thread_id;
695  SCLogDebug("Trying to add thread %ld to entry %d", ent->thread_id, ent->slot_number);
696  if (0 != HashTableAdd(parent->ht, ent, 0)) {
697  FatalError("Unable to add thread/entry mapping");
698  }
699  }
700  return ent;
701 }
702 
703 /** \brief LogFileEnsureExists() Ensure a log file context for the thread exists
704  * \param parent_ctx
705  * \retval LogFileCtx * pointer if successful; NULL otherwise
706  */
708 {
709  /* threaded output disabled */
710  if (!parent_ctx->threaded)
711  return parent_ctx;
712 
713  SCMutexLock(&parent_ctx->threads->mutex);
714  /* Find this thread's entry */
715  ThreadLogFileHashEntry *entry = LogFileThread2Slot(parent_ctx->threads);
716  SCLogDebug("Adding reference for thread %ld [slot %d] to file %s [ctx %p]", SCGetThreadIdLong(),
717  entry->slot_number, parent_ctx->filename, parent_ctx);
718 
719  bool new = entry->isopen;
720  /* has it been opened yet? */
721  if (!entry->isopen) {
722  SCLogDebug("Opening new file for thread/slot %d to file %s [ctx %p]", entry->slot_number,
723  parent_ctx->filename, parent_ctx);
724  if (LogFileNewThreadedCtx(
725  parent_ctx, parent_ctx->filename, parent_ctx->threads->append, entry)) {
726  entry->isopen = true;
727  } else {
728  SCLogError(
729  "Unable to open slot %d for file %s", entry->slot_number, parent_ctx->filename);
730  (void)HashTableRemove(parent_ctx->threads->ht, entry, 0);
731  }
732  }
733  SCMutexUnlock(&parent_ctx->threads->mutex);
734 
736  if (new) {
737  SCLogDebug("Existing file for thread/entry %p reference to file %s [ctx %p]", entry,
738  parent_ctx->filename, parent_ctx);
739  }
740  }
741 
742  return entry->ctx;
743 }
744 
745 /** \brief LogFileThreadedName() Create file name for threaded EVE storage
746  *
747  */
748 static bool LogFileThreadedName(
749  const char *original_name, char *threaded_name, size_t len, uint32_t unique_id)
750 {
751  sc_errno = SC_OK;
752 
753  if (strcmp("/dev/null", original_name) == 0) {
754  strlcpy(threaded_name, original_name, len);
755  return true;
756  }
757 
758  const char *base = SCBasename(original_name);
759  if (!base) {
760  FatalError("Invalid filename for threaded mode \"%s\"; "
761  "no basename found.",
762  original_name);
763  }
764 
765  /* Check if basename has an extension */
766  char *dot = strrchr(base, '.');
767  if (dot) {
768  char *tname = SCStrdup(original_name);
769  if (!tname) {
771  return false;
772  }
773 
774  /* Fetch extension location from original, not base
775  * for update
776  */
777  dot = strrchr(original_name, '.');
778  int dotpos = dot - original_name;
779  tname[dotpos] = '\0';
780  char *ext = tname + dotpos + 1;
781  if (strlen(tname) && strlen(ext)) {
782  snprintf(threaded_name, len, "%s.%u.%s", tname, unique_id, ext);
783  } else {
784  FatalError("Invalid filename for threaded mode \"%s\"; "
785  "filenames must include an extension, e.g: \"name.ext\"",
786  original_name);
787  }
788  SCFree(tname);
789  } else {
790  snprintf(threaded_name, len, "%s.%u", original_name, unique_id);
791  }
792  return true;
793 }
794 
795 /** \brief LogFileNewThreadedCtx() Create file context for threaded output
796  * \param parent_ctx
797  * \param log_path
798  * \param append
799  * \param entry
800  */
801 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append,
802  ThreadLogFileHashEntry *entry)
803 {
804  LogFileCtx *thread = SCCalloc(1, sizeof(LogFileCtx));
805  if (!thread) {
806  SCLogError("Unable to allocate thread file context entry %p", entry);
807  return false;
808  }
809 
810  *thread = *parent_ctx;
811  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
812  char fname[LOGFILE_NAME_MAX];
813  if (!LogFileThreadedName(log_path, fname, sizeof(fname), SC_ATOMIC_ADD(eve_file_id, 1))) {
814  SCLogError("Unable to create threaded filename for log");
815  goto error;
816  }
817  SCLogDebug("Thread open -- using name %s [replaces %s]", fname, log_path);
818  thread->fp = SCLogOpenFileFp(fname, append, thread->filemode);
819  if (thread->fp == NULL) {
820  goto error;
821  }
822  thread->filename = SCStrdup(fname);
823  if (!thread->filename) {
824  SCLogError("Unable to duplicate filename for context entry %p", entry);
825  goto error;
826  }
827  thread->is_regular = true;
828  thread->Write = SCLogFileWriteNoLock;
829  thread->Close = SCLogFileCloseNoLock;
831  } else if (parent_ctx->type == LOGFILE_TYPE_PLUGIN) {
832  entry->slot_number = SC_ATOMIC_ADD(eve_file_id, 1);
833  thread->plugin.plugin->ThreadInit(
834  thread->plugin.init_data, entry->slot_number, &thread->plugin.thread_data);
835  }
836  thread->threaded = false;
837  thread->parent = parent_ctx;
838  thread->entry = entry;
839  entry->ctx = thread;
840 
841  return true;
842 
843 error:
844  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
845  SC_ATOMIC_SUB(eve_file_id, 1);
846  if (thread->fp) {
847  thread->Close(thread);
848  }
849  }
850 
851  if (thread) {
852  SCFree(thread);
853  }
854  return false;
855 }
856 
857 /** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
858  * \param lf_ctx pointer to the OutputCtx
859  * \retval int 1 if successful, 0 if error
860  * */
862 {
863  if (lf_ctx == NULL) {
864  SCReturnInt(0);
865  }
866 
867  if (lf_ctx->type == LOGFILE_TYPE_PLUGIN && lf_ctx->parent != NULL) {
868  lf_ctx->plugin.plugin->ThreadDeinit(lf_ctx->plugin.init_data, lf_ctx->plugin.thread_data);
869  }
870 
871  if (lf_ctx->threaded) {
872  BUG_ON(lf_ctx->threads == NULL);
873  SCMutexDestroy(&lf_ctx->threads->mutex);
874  if (lf_ctx->threads->append)
875  SCFree(lf_ctx->threads->append);
876  if (lf_ctx->threads->ht) {
877  HashTableFree(lf_ctx->threads->ht);
878  }
879  SCFree(lf_ctx->threads);
880  } else {
881  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
882  if (lf_ctx->fp != NULL) {
883  lf_ctx->Close(lf_ctx);
884  }
885  }
886  SCMutexDestroy(&lf_ctx->fp_mutex);
887  }
888 
889  if (lf_ctx->prefix != NULL) {
890  SCFree(lf_ctx->prefix);
891  lf_ctx->prefix_len = 0;
892  }
893 
894  if(lf_ctx->filename != NULL)
895  SCFree(lf_ctx->filename);
896 
897  if (lf_ctx->sensor_name)
898  SCFree(lf_ctx->sensor_name);
899 
900  if (!lf_ctx->threaded) {
902  }
903 
904  /* Deinitialize output plugins. We only want to call this for the
905  * parent of threaded output, or always for non-threaded
906  * output. */
907  if (lf_ctx->type == LOGFILE_TYPE_PLUGIN && lf_ctx->parent == NULL) {
908  lf_ctx->plugin.plugin->Deinit(lf_ctx->plugin.init_data);
909  }
910 
911  memset(lf_ctx, 0, sizeof(*lf_ctx));
912  SCFree(lf_ctx);
913 
914  SCReturnInt(1);
915 }
916 
917 int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
918 {
919  if (file_ctx->type == LOGFILE_TYPE_FILE || file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
920  file_ctx->type == LOGFILE_TYPE_UNIX_STREAM) {
921  /* append \n for files only */
922  MemBufferWriteString(buffer, "\n");
923  file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
924  MEMBUFFER_OFFSET(buffer), file_ctx);
925  } else if (file_ctx->type == LOGFILE_TYPE_PLUGIN) {
926  file_ctx->plugin.plugin->Write((const char *)MEMBUFFER_BUFFER(buffer),
927  MEMBUFFER_OFFSET(buffer), file_ctx->plugin.init_data, file_ctx->plugin.thread_data);
928  }
929 #ifdef HAVE_LIBHIREDIS
930  else if (file_ctx->type == LOGFILE_TYPE_REDIS) {
931  SCMutexLock(&file_ctx->fp_mutex);
932  LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer),
933  MEMBUFFER_OFFSET(buffer));
934  SCMutexUnlock(&file_ctx->fp_mutex);
935  }
936 #endif
937 
938  return 0;
939 }
SCParseTimeSizeString
uint64_t SCParseTimeSizeString(const char *str)
Parse string containing time size (1m, 1h, etc).
Definition: util-time.c:570
LogFileCtx_::rotation_flag
int rotation_flag
Definition: util-logopenfile.h:149
util-byte.h
LogFileCtx_::prefix_len
size_t prefix_len
Definition: util-logopenfile.h:128
LogThreadedFileCtx_::append
char * append
Definition: util-logopenfile.h:62
SCEveFileType_::Write
int(* Write)(const char *buffer, int buffer_len, void *init_data, void *thread_data)
Definition: suricata-plugin.h:54
SCFerrorUnlocked
#define SCFerrorUnlocked
Definition: suricata-common.h:536
len
uint8_t len
Definition: app-layer-dnp3.h:2
LOGFILE_TYPE_REDIS
@ LOGFILE_TYPE_REDIS
Definition: util-logopenfile.h:42
LogFileCtx_::sensor_name
char * sensor_name
Definition: util-logopenfile.h:111
LogFileCtx_::reconn_timer
uint64_t reconn_timer
Definition: util-logopenfile.h:116
SC_LOG_DEBUG
@ SC_LOG_DEBUG
Definition: util-debug.h:57
LogFileCtx_::fp_mutex
SCMutex fp_mutex
Definition: util-logopenfile.h:94
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_DECL_AND_INIT_WITH_VAL
#define SC_ATOMIC_DECL_AND_INIT_WITH_VAL(type, name, val)
wrapper for declaring an atomic variable and initializing it to a specific value
Definition: util-atomic.h:304
LogFileNewCtx
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
Definition: util-logopenfile.c:659
SCGetSecondsUntil
uint64_t SCGetSecondsUntil(const char *str, time_t epoch)
Get seconds until a time unit changes.
Definition: util-time.c:621
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
LogFileCtx_::json_flags
size_t json_flags
Definition: util-logopenfile.h:146
LogFilePluginCtx_::init_data
void * init_data
Definition: util-logopenfile.h:67
LogFilePluginCtx_::thread_data
void * thread_data
Definition: util-logopenfile.h:68
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
LogFileCtx_
Definition: util-logopenfile.h:72
LOGFILE_RECONN_MIN_TIME
#define LOGFILE_RECONN_MIN_TIME
Definition: util-logopenfile.h:162
DEFAULT_LOG_MODE_APPEND
#define DEFAULT_LOG_MODE_APPEND
Definition: output.h:30
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
HashTable_
Definition: util-hash.h:35
util-log-redis.h
LogFileCtx_::Write
int(* Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:87
LogFileCtx_::filename
char * filename
Definition: util-logopenfile.h:105
LOGFILE_NAME_MAX
#define LOGFILE_NAME_MAX
Definition: util-logopenfile.c:47
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:277
LogFileCtx_::plugin
LogFilePluginCtx plugin
Definition: util-logopenfile.h:90
m
SCMutex m
Definition: flow-hash.h:6
ThreadLogFileHashEntry::ctx
struct LogFileCtx_ * ctx
Definition: util-logopenfile.h:55
LogFileCtx_::send_flags
uint8_t send_flags
Definition: util-logopenfile.h:139
SCConfLogReopen
int SCConfLogReopen(LogFileCtx *log_ctx)
Reopen a regular log file with the side-affect of truncating it.
Definition: util-logopenfile.c:629
IsRunModeOffline
bool IsRunModeOffline(enum RunModes run_mode_to_check)
Definition: runmodes.c:574
ThreadLogFileHashEntry::isopen
bool isopen
Definition: util-logopenfile.h:54
SCClearErrUnlocked
#define SCClearErrUnlocked
Definition: suricata-common.h:535
ThreadLogFileHashEntry::slot_number
int slot_number
Definition: util-logopenfile.h:53
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:537
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
HashTableFree
void HashTableFree(HashTable *ht)
Definition: util-hash.c:78
HashTable_::array_size
uint32_t array_size
Definition: util-hash.h:37
SCConfLogOpenGeneric
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: util-logopenfile.c:452
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
LogFileCtx_::sock_type
int sock_type
Definition: util-logopenfile.h:115
OutputRegisterFileRotationFlag
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
Definition: output.c:834
SCTimeToStringPattern
int SCTimeToStringPattern(time_t epoch, const char *pattern, char *str, size_t size)
Convert epoch time to string pattern.
Definition: util-time.c:541
LogFileWrite
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
Definition: util-logopenfile.c:917
LogFileCtx_::rotate_interval
uint64_t rotate_interval
Definition: util-logopenfile.h:123
DEFAULT_LOG_FILETYPE
#define DEFAULT_LOG_FILETYPE
Definition: output.h:31
LogFileCtx_::is_sock
int is_sock
Definition: util-logopenfile.h:114
SCFwriteUnlocked
#define SCFwriteUnlocked
Definition: suricata-common.h:533
LogFileCtx_::dropped
uint64_t dropped
Definition: util-logopenfile.h:156
SCFflushUnlocked
#define SCFflushUnlocked
Definition: suricata-common.h:534
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
HashTableLookup
void * HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:191
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:707
HashTableRemove
int HashTableRemove(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:150
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
util-time.h
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
HashTableAdd
int HashTableAdd(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:112
LogFileCtx_::type
enum LogFileType type
Definition: util-logopenfile.h:102
LOGFILE_TYPE_FILE
@ LOGFILE_TYPE_FILE
Definition: util-logopenfile.h:39
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
LogFileCtx_::threads
LogThreadedFileCtx * threads
Definition: util-logopenfile.h:79
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
ThreadLogFileHashEntry::thread_id
uint64_t thread_id
Definition: util-logopenfile.h:52
LOGFILE_TYPE_PLUGIN
@ LOGFILE_TYPE_PLUGIN
Definition: util-logopenfile.h:43
conf.h
SC_OK
@ SC_OK
Definition: util-error.h:27
SCBasename
const char * SCBasename(const char *path)
Definition: util-path.c:285
LOGFILE_TYPE_UNIX_DGRAM
@ LOGFILE_TYPE_UNIX_DGRAM
Definition: util-logopenfile.h:40
SCLogOpenThreadedFile
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx)
Definition: util-logopenfile.c:359
LOGFILE_TYPE_UNIX_STREAM
@ LOGFILE_TYPE_UNIX_STREAM
Definition: util-logopenfile.h:41
MemBuffer_
Definition: util-buffer.h:27
LogFileCtx_::is_regular
uint8_t is_regular
Definition: util-logopenfile.h:143
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
SCGetThreadIdLong
#define SCGetThreadIdLong(...)
Definition: threads.h:255
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:786
LogFileCtx_::parent
struct LogFileCtx_ * parent
Definition: util-logopenfile.h:98
util-conf.h
LogThreadedFileCtx_
Definition: util-logopenfile.h:59
suricata-common.h
LogFilePluginCtx_::plugin
SCEveFileType * plugin
Definition: util-logopenfile.h:66
util-path.h
LogFileCtx_::output_errors
uint64_t output_errors
Definition: util-logopenfile.h:158
ThreadLogFileHashEntry
Definition: util-logopenfile.h:51
ConfNode_::name
char * name
Definition: conf.h:33
MemBufferWriteString
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
LogFileCtx_::entry
ThreadLogFileHashEntry * entry
Definition: util-logopenfile.h:99
LogFileFreeCtx
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
Definition: util-logopenfile.c:861
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
LogFileCtx_::rotate_time
time_t rotate_time
Definition: util-logopenfile.h:120
SCEveFileType_::Deinit
void(* Deinit)(void *init_data)
Definition: suricata-plugin.h:56
FatalError
#define FatalError(...)
Definition: util-debug.h:502
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
RunmodeGetCurrent
int RunmodeGetCurrent(void)
Definition: suricata.c:261
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
ConfigGetLogDirectory
const char * ConfigGetLogDirectory(void)
Definition: util-conf.c:38
SCEveFileType_::ThreadDeinit
int(* ThreadDeinit)(void *init_data, void *thread_data)
Definition: suricata-plugin.h:60
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
LOGFILE_ROTATE_INTERVAL
#define LOGFILE_ROTATE_INTERVAL
Definition: util-logopenfile.h:165
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:562
LogThreadedFileCtx_::mutex
SCMutex mutex
Definition: util-logopenfile.h:60
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
LogFileCtx_::prefix
char * prefix
Definition: util-logopenfile.h:127
HashTableInit
HashTable * HashTableInit(uint32_t size, uint32_t(*Hash)(struct HashTable_ *, void *, uint16_t), char(*Compare)(void *, uint16_t, void *, uint16_t), void(*Free)(void *))
Definition: util-hash.c:35
sc_log_global_log_level
SCLogLevel sc_log_global_log_level
Holds the global log level. Is the same as sc_log_config->log_level.
Definition: util-debug.c:101
suricata.h
LogFileCtx_::flags
uint8_t flags
Definition: util-logopenfile.h:136
LogFileCtx_::Close
void(* Close)(struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:88
MEMBUFFER_BUFFER
#define MEMBUFFER_BUFFER(mem_buffer)
Get the MemBuffers underlying buffer.
Definition: util-buffer.h:50
SCEveFileType_::ThreadInit
int(* ThreadInit)(void *init_data, int thread_id, void **thread_data)
Definition: suricata-plugin.h:58
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
LogThreadedFileCtx_::ht
HashTable * ht
Definition: util-logopenfile.h:61
OutputUnregisterFileRotationFlag
void OutputUnregisterFileRotationFlag(int *flag)
Unregister a file rotation flag.
Definition: output.c:855
MEMBUFFER_OFFSET
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
Definition: util-buffer.h:55
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
LogFileCtx_::fp
FILE * fp
Definition: util-logopenfile.h:74
output.h
LogFileCtx_::threaded
bool threaded
Definition: util-logopenfile.h:97
SCCreateDirectoryTree
int SCCreateDirectoryTree(const char *path, const bool final)
Recursively create a directory.
Definition: util-path.c:173
LogFileCtx_::filemode
uint32_t filemode
Definition: util-logopenfile.h:108
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:814