suricata
util-logopenfile.c
Go to the documentation of this file.
1 /* vi: set et ts=4: */
2 /* Copyright (C) 2007-2021 Open Information Security Foundation
3  *
4  * You can copy, redistribute or modify this Program under the terms of
5  * the GNU General Public License version 2 as published by the Free
6  * Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * version 2 along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16  * 02110-1301, USA.
17  */
18 
19 /**
20  * \file
21  *
22  * \author Mike Pomraning <mpomraning@qualys.com>
23  *
24  * File-like output for logging: regular files and sockets.
25  */
26 
27 #include "suricata-common.h" /* errno.h, string.h, etc. */
28 #include "conf.h" /* ConfNode, etc. */
29 #include "output.h" /* DEFAULT_LOG_* */
30 #include "util-byte.h"
31 #include "util-logopenfile.h"
32 
33 #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H)
34 #define BUILD_WITH_UNIXSOCKET
35 #include <sys/types.h>
36 #include <sys/socket.h>
37 #include <sys/un.h>
38 #endif
39 
40 #ifdef HAVE_LIBHIREDIS
41 #include "util-log-redis.h"
42 #endif /* HAVE_LIBHIREDIS */
43 
44 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append, int i);
45 
46 // Threaded eve.json identifier
47 static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint32_t, eve_file_id, 1);
48 
49 #ifdef BUILD_WITH_UNIXSOCKET
50 /** \brief connect to the indicated local stream socket, logging any errors
51  * \param path filesystem path to connect to
52  * \param log_err, non-zero if connect failure should be logged.
53  * \retval FILE* on success (fdopen'd wrapper of underlying socket)
54  * \retval NULL on error
55  */
56 static FILE *
57 SCLogOpenUnixSocketFp(const char *path, int sock_type, int log_err)
58 {
59  struct sockaddr_un saun;
60  int s = -1;
61  FILE * ret = NULL;
62 
63  memset(&saun, 0x00, sizeof(saun));
64 
65  s = socket(PF_UNIX, sock_type, 0);
66  if (s < 0) goto err;
67 
68  saun.sun_family = AF_UNIX;
69  strlcpy(saun.sun_path, path, sizeof(saun.sun_path));
70 
71  if (connect(s, (const struct sockaddr *)&saun, sizeof(saun)) < 0)
72  goto err;
73 
74  ret = fdopen(s, "w");
75  if (ret == NULL)
76  goto err;
77 
78  return ret;
79 
80 err:
81  if (log_err)
83  "Error connecting to socket \"%s\": %s (will keep trying)",
84  path, strerror(errno));
85 
86  if (s >= 0)
87  close(s);
88 
89  return NULL;
90 }
91 
92 /**
93  * \brief Attempt to reconnect a disconnected (or never-connected) Unix domain socket.
94  * \retval 1 if it is now connected; otherwise 0
95  */
96 static int SCLogUnixSocketReconnect(LogFileCtx *log_ctx)
97 {
98  int disconnected = 0;
99  if (log_ctx->fp) {
101  "Write error on Unix socket \"%s\": %s; reconnecting...",
102  log_ctx->filename, strerror(errno));
103  fclose(log_ctx->fp);
104  log_ctx->fp = NULL;
105  log_ctx->reconn_timer = 0;
106  disconnected = 1;
107  }
108 
109  struct timeval tv;
110  uint64_t now;
111  gettimeofday(&tv, NULL);
112  now = (uint64_t)tv.tv_sec * 1000;
113  now += tv.tv_usec / 1000; /* msec resolution */
114  if (log_ctx->reconn_timer != 0 &&
115  (now - log_ctx->reconn_timer) < LOGFILE_RECONN_MIN_TIME) {
116  /* Don't bother to try reconnecting too often. */
117  return 0;
118  }
119  log_ctx->reconn_timer = now;
120 
121  log_ctx->fp = SCLogOpenUnixSocketFp(log_ctx->filename, log_ctx->sock_type, 0);
122  if (log_ctx->fp) {
123  /* Connected at last (or reconnected) */
124  SCLogNotice("Reconnected socket \"%s\"", log_ctx->filename);
125  } else if (disconnected) {
126  SCLogWarning(SC_ERR_SOCKET, "Reconnect failed: %s (will keep trying)",
127  strerror(errno));
128  }
129 
130  return log_ctx->fp ? 1 : 0;
131 }
132 
133 static int SCLogFileWriteSocket(const char *buffer, int buffer_len,
134  LogFileCtx *ctx)
135 {
136  int tries = 0;
137  int ret = 0;
138  bool reopen = false;
139  if (ctx->fp == NULL && ctx->is_sock) {
140  SCLogUnixSocketReconnect(ctx);
141  }
142 tryagain:
143  ret = -1;
144  reopen = 0;
145  errno = 0;
146  if (ctx->fp != NULL) {
147  int fd = fileno(ctx->fp);
148  ssize_t size = send(fd, buffer, buffer_len, ctx->send_flags);
149  if (size > -1) {
150  ret = 0;
151  } else {
152  if (errno == EAGAIN || errno == EWOULDBLOCK) {
153  SCLogDebug("Socket would block, dropping event.");
154  } else if (errno == EINTR) {
155  if (tries++ == 0) {
156  SCLogDebug("Interrupted system call, trying again.");
157  goto tryagain;
158  }
159  SCLogDebug("Too many interrupted system calls, "
160  "dropping event.");
161  } else {
162  /* Some other error. Assume badness and reopen. */
163  SCLogDebug("Send failed: %s", strerror(errno));
164  reopen = true;
165  }
166  }
167  }
168 
169  if (reopen && tries++ == 0) {
170  if (SCLogUnixSocketReconnect(ctx)) {
171  goto tryagain;
172  }
173  }
174 
175  if (ret == -1) {
176  ctx->dropped++;
177  }
178 
179  return ret;
180 }
181 #endif /* BUILD_WITH_UNIXSOCKET */
182 static inline void OutputWriteLock(pthread_mutex_t *m)
183 {
184  SCMutexLock(m);
185 
186 }
187 
188 /**
189  * \brief Write buffer to log file.
190  * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
191  * characters successfully written).
192  */
193 static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
194 {
195  int ret = 0;
196 
197  BUG_ON(log_ctx->is_sock);
198 
199  /* Check for rotation. */
200  if (log_ctx->rotation_flag) {
201  log_ctx->rotation_flag = 0;
202  SCConfLogReopen(log_ctx);
203  }
204 
205  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
206  time_t now = time(NULL);
207  if (now >= log_ctx->rotate_time) {
208  SCConfLogReopen(log_ctx);
209  log_ctx->rotate_time = now + log_ctx->rotate_interval;
210  }
211  }
212 
213  if (log_ctx->fp) {
214  SCClearErrUnlocked(log_ctx->fp);
215  if (1 != SCFwriteUnlocked(buffer, buffer_len, 1, log_ctx->fp)) {
216  /* Only the first error is logged */
217  if (!log_ctx->output_errors) {
218  SCLogError(SC_ERR_LOG_OUTPUT, "%s error while writing to %s",
219  SCFerrorUnlocked(log_ctx->fp) ? strerror(errno) : "unknown error",
220  log_ctx->filename);
221  }
222  log_ctx->output_errors++;
223  } else {
224  SCFflushUnlocked(log_ctx->fp);
225  }
226  }
227 
228  return ret;
229 }
230 
231 /**
232  * \brief Write buffer to log file.
233  * \retval 0 on failure; otherwise, the return value of fwrite (number of
234  * characters successfully written).
235  */
236 static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
237 {
238  OutputWriteLock(&log_ctx->fp_mutex);
239  int ret = 0;
240 
241 #ifdef BUILD_WITH_UNIXSOCKET
242  if (log_ctx->is_sock) {
243  ret = SCLogFileWriteSocket(buffer, buffer_len, log_ctx);
244  } else
245 #endif
246  {
247 
248  /* Check for rotation. */
249  if (log_ctx->rotation_flag) {
250  log_ctx->rotation_flag = 0;
251  SCConfLogReopen(log_ctx);
252  }
253 
254  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
255  time_t now = time(NULL);
256  if (now >= log_ctx->rotate_time) {
257  SCConfLogReopen(log_ctx);
258  log_ctx->rotate_time = now + log_ctx->rotate_interval;
259  }
260  }
261 
262  if (log_ctx->fp) {
263  clearerr(log_ctx->fp);
264  if (1 != fwrite(buffer, buffer_len, 1, log_ctx->fp)) {
265  /* Only the first error is logged */
266  if (!log_ctx->output_errors) {
267  SCLogError(SC_ERR_LOG_OUTPUT, "%s error while writing to %s",
268  ferror(log_ctx->fp) ? strerror(errno) : "unknown error",
269  log_ctx->filename);
270  }
271  log_ctx->output_errors++;
272  } else {
273  fflush(log_ctx->fp);
274  }
275  }
276  }
277 
278  SCMutexUnlock(&log_ctx->fp_mutex);
279 
280  return ret;
281 }
282 
283 /** \brief generate filename based on pattern
284  * \param pattern pattern to use
285  * \retval char* on success
286  * \retval NULL on error
287  */
288 static char *SCLogFilenameFromPattern(const char *pattern)
289 {
290  char *filename = SCMalloc(PATH_MAX);
291  if (filename == NULL) {
292  return NULL;
293  }
294 
295  int rc = SCTimeToStringPattern(time(NULL), pattern, filename, PATH_MAX);
296  if (rc != 0) {
297  SCFree(filename);
298  return NULL;
299  }
300 
301  return filename;
302 }
303 
304 static void SCLogFileCloseNoLock(LogFileCtx *log_ctx)
305 {
306  SCLogDebug("Closing %s", log_ctx->filename);
307  if (log_ctx->fp)
308  fclose(log_ctx->fp);
309 
310  if (log_ctx->output_errors) {
311  SCLogError(SC_ERR_LOG_OUTPUT, "There were %" PRIu64 " output errors to %s",
312  log_ctx->output_errors, log_ctx->filename);
313  }
314 }
315 
316 static void SCLogFileClose(LogFileCtx *log_ctx)
317 {
318  SCMutexLock(&log_ctx->fp_mutex);
319  SCLogFileCloseNoLock(log_ctx);
320  SCMutexUnlock(&log_ctx->fp_mutex);
321 }
322 
324  const char *log_path, const char *append, LogFileCtx *parent_ctx, int slot_count)
325 {
326  parent_ctx->threads = SCCalloc(1, sizeof(LogThreadedFileCtx));
327  if (!parent_ctx->threads) {
328  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate threads container");
329  return false;
330  }
331 
332  parent_ctx->threads->append = SCStrdup(append == NULL ? DEFAULT_LOG_MODE_APPEND : append);
333  if (!parent_ctx->threads->append) {
334  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate threads append setting");
335  goto error_exit;
336  }
337 
338  parent_ctx->threads->slot_count = slot_count;
339  parent_ctx->threads->lf_slots = SCCalloc(slot_count, sizeof(LogFileCtx *));
340  if (!parent_ctx->threads->lf_slots) {
341  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate thread slots");
342  goto error_exit;
343  }
344  SCLogDebug("Allocated %d file context pointers for threaded array",
345  parent_ctx->threads->slot_count);
346  for (int slot = 1; slot < parent_ctx->threads->slot_count; slot++) {
347  if (!LogFileNewThreadedCtx(parent_ctx, log_path, parent_ctx->threads->append, slot)) {
348  /* TODO: clear allocated entries [1, slot) */
349  goto error_exit;
350  }
351  }
352  SCMutexInit(&parent_ctx->threads->mutex, NULL);
353  return true;
354 
355 error_exit:
356 
357  if (parent_ctx->threads->lf_slots) {
358  SCFree(parent_ctx->threads->lf_slots);
359  }
360  if (parent_ctx->threads->append) {
361  SCFree(parent_ctx->threads->append);
362  }
363  SCFree(parent_ctx->threads);
364  parent_ctx->threads = NULL;
365  return false;
366 }
367 
368 /** \brief open the indicated file, logging any errors
369  * \param path filesystem path to open
370  * \param append_setting open file with O_APPEND: "yes" or "no"
371  * \param mode permissions to set on file
372  * \retval FILE* on success
373  * \retval NULL on error
374  */
375 static FILE *
376 SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
377 {
378  FILE *ret = NULL;
379 
380  char *filename = SCLogFilenameFromPattern(path);
381  if (filename == NULL) {
382  return NULL;
383  }
384 
385  int rc = SCCreateDirectoryTree(filename, false);
386  if (rc < 0) {
387  SCFree(filename);
388  return NULL;
389  }
390 
391  if (ConfValIsTrue(append_setting)) {
392  ret = fopen(filename, "a");
393  } else {
394  ret = fopen(filename, "w");
395  }
396 
397  if (ret == NULL) {
398  SCLogError(SC_ERR_FOPEN, "Error opening file: \"%s\": %s",
399  filename, strerror(errno));
400  } else {
401  if (mode != 0) {
402  int r = chmod(filename, mode);
403  if (r < 0) {
404  SCLogWarning(SC_WARN_CHMOD, "Could not chmod %s to %o: %s",
405  filename, mode, strerror(errno));
406  }
407  }
408  }
409 
410  SCFree(filename);
411  return ret;
412 }
413 
414 /** \brief open a generic output "log file", which may be a regular file or a socket
415  * \param conf ConfNode structure for the output section in question
416  * \param log_ctx Log file context allocated by caller
417  * \param default_filename Default name of file to open, if not specified in ConfNode
418  * \param rotate Register the file for rotation in HUP.
419  * \retval 0 on success
420  * \retval -1 on error
421  */
422 int
424  LogFileCtx *log_ctx,
425  const char *default_filename,
426  int rotate)
427 {
428  char log_path[PATH_MAX];
429  const char *log_dir;
430  const char *filename, *filetype;
431 
432  // Arg check
433  if (conf == NULL || log_ctx == NULL || default_filename == NULL) {
435  "SCConfLogOpenGeneric(conf %p, ctx %p, default %p) "
436  "missing an argument",
437  conf, log_ctx, default_filename);
438  return -1;
439  }
440  if (log_ctx->fp != NULL) {
442  "SCConfLogOpenGeneric: previously initialized Log CTX "
443  "encountered");
444  return -1;
445  }
446 
447  // Resolve the given config
448  filename = ConfNodeLookupChildValue(conf, "filename");
449  if (filename == NULL)
450  filename = default_filename;
451 
452  log_dir = ConfigGetLogDirectory();
453 
454  if (PathIsAbsolute(filename)) {
455  snprintf(log_path, PATH_MAX, "%s", filename);
456  } else {
457  snprintf(log_path, PATH_MAX, "%s/%s", log_dir, filename);
458  }
459 
460  /* Rotate log file based on time */
461  const char *rotate_int = ConfNodeLookupChildValue(conf, "rotate-interval");
462  if (rotate_int != NULL) {
463  time_t now = time(NULL);
464  log_ctx->flags |= LOGFILE_ROTATE_INTERVAL;
465 
466  /* Use a specific time */
467  if (strcmp(rotate_int, "minute") == 0) {
468  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
469  log_ctx->rotate_interval = 60;
470  } else if (strcmp(rotate_int, "hour") == 0) {
471  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
472  log_ctx->rotate_interval = 3600;
473  } else if (strcmp(rotate_int, "day") == 0) {
474  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
475  log_ctx->rotate_interval = 86400;
476  }
477 
478  /* Use a timer */
479  else {
480  log_ctx->rotate_interval = SCParseTimeSizeString(rotate_int);
481  if (log_ctx->rotate_interval == 0) {
483  "invalid rotate-interval value");
484  }
485  log_ctx->rotate_time = now + log_ctx->rotate_interval;
486  }
487  }
488 
489  filetype = ConfNodeLookupChildValue(conf, "filetype");
490  if (filetype == NULL)
491  filetype = DEFAULT_LOG_FILETYPE;
492 
493  const char *filemode = ConfNodeLookupChildValue(conf, "filemode");
494  uint32_t mode = 0;
495  if (filemode != NULL &&
496  StringParseUint32(&mode, 8, strlen(filemode),
497  filemode) > 0) {
498  log_ctx->filemode = mode;
499  }
500 
501  const char *append = ConfNodeLookupChildValue(conf, "append");
502  if (append == NULL)
503  append = DEFAULT_LOG_MODE_APPEND;
504 
505  /* JSON flags */
506  log_ctx->json_flags = JSON_PRESERVE_ORDER|JSON_COMPACT|
507  JSON_ENSURE_ASCII|JSON_ESCAPE_SLASH;
508 
509  ConfNode *json_flags = ConfNodeLookupChild(conf, "json");
510 
511  if (json_flags != 0) {
512  const char *preserve_order = ConfNodeLookupChildValue(json_flags,
513  "preserve-order");
514  if (preserve_order != NULL && ConfValIsFalse(preserve_order))
515  log_ctx->json_flags &= ~(JSON_PRESERVE_ORDER);
516 
517  const char *compact = ConfNodeLookupChildValue(json_flags, "compact");
518  if (compact != NULL && ConfValIsFalse(compact))
519  log_ctx->json_flags &= ~(JSON_COMPACT);
520 
521  const char *ensure_ascii = ConfNodeLookupChildValue(json_flags,
522  "ensure-ascii");
523  if (ensure_ascii != NULL && ConfValIsFalse(ensure_ascii))
524  log_ctx->json_flags &= ~(JSON_ENSURE_ASCII);
525 
526  const char *escape_slash = ConfNodeLookupChildValue(json_flags,
527  "escape-slash");
528  if (escape_slash != NULL && ConfValIsFalse(escape_slash))
529  log_ctx->json_flags &= ~(JSON_ESCAPE_SLASH);
530  }
531 
532 #ifdef BUILD_WITH_UNIXSOCKET
533  if (log_ctx->threaded) {
534  if (strcasecmp(filetype, "unix_stream") == 0 || strcasecmp(filetype, "unix_dgram") == 0) {
535  FatalError(SC_ERR_FATAL, "Socket file types do not support threaded output");
536  }
537  }
538 #endif
539  // Now, what have we been asked to open?
540  if (strcasecmp(filetype, "unix_stream") == 0) {
541 #ifdef BUILD_WITH_UNIXSOCKET
542  /* Don't bail. May be able to connect later. */
543  log_ctx->is_sock = 1;
544  log_ctx->sock_type = SOCK_STREAM;
545  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM, 1);
546 #else
547  return -1;
548 #endif
549  } else if (strcasecmp(filetype, "unix_dgram") == 0) {
550 #ifdef BUILD_WITH_UNIXSOCKET
551  /* Don't bail. May be able to connect later. */
552  log_ctx->is_sock = 1;
553  log_ctx->sock_type = SOCK_DGRAM;
554  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM, 1);
555 #else
556  return -1;
557 #endif
558  } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
559  strcasecmp(filetype, "file") == 0) {
560  log_ctx->is_regular = 1;
561  if (!log_ctx->threaded) {
562  log_ctx->fp = SCLogOpenFileFp(log_path, append, log_ctx->filemode);
563  if (log_ctx->fp == NULL)
564  return -1; // Error already logged by Open...Fp routine
565  } else {
566  if (!SCLogOpenThreadedFile(log_path, append, log_ctx, 1)) {
567  return -1;
568  }
569  }
570  if (rotate) {
572  }
573  } else {
574  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry for "
575  "%s.filetype. Expected \"regular\" (default), \"unix_stream\", "
576  "or \"unix_dgram\"",
577  conf->name);
578  }
579  log_ctx->filename = SCStrdup(log_path);
580  if (unlikely(log_ctx->filename == NULL)) {
582  "Failed to allocate memory for filename");
583  return -1;
584  }
585 
586 #ifdef BUILD_WITH_UNIXSOCKET
587  /* If a socket and running live, do non-blocking writes. */
588  if (log_ctx->is_sock && !IsRunModeOffline(RunmodeGetCurrent())) {
589  SCLogInfo("Setting logging socket of non-blocking in live mode.");
590  log_ctx->send_flags |= MSG_DONTWAIT;
591  }
592 #endif
593  SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype,
594  filename);
595 
596  return 0;
597 }
598 
599 /**
600  * \brief Reopen a regular log file with the side-affect of truncating it.
601  *
602  * This is useful to clear the log file and start a new one, or to
603  * re-open the file after its been moved by something external
604  * (eg. logrotate).
605  */
607 {
608  if (!log_ctx->is_regular) {
609  /* Not supported and not needed on non-regular files. */
610  return 0;
611  }
612 
613  if (log_ctx->filename == NULL) {
615  "Can't re-open LogFileCtx without a filename.");
616  return -1;
617  }
618 
619  if (log_ctx->fp != NULL) {
620  fclose(log_ctx->fp);
621  }
622 
623  /* Reopen the file. Append is forced in case the file was not
624  * moved as part of a rotation process. */
625  SCLogDebug("Reopening log file %s.", log_ctx->filename);
626  log_ctx->fp = SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode);
627  if (log_ctx->fp == NULL) {
628  return -1; // Already logged by Open..Fp routine.
629  }
630 
631  return 0;
632 }
633 
634 /** \brief LogFileNewCtx() Get a new LogFileCtx
635  * \retval LogFileCtx * pointer if successful, NULL if error
636  * */
638 {
639  LogFileCtx* lf_ctx;
640  lf_ctx = (LogFileCtx*)SCCalloc(1, sizeof(LogFileCtx));
641 
642  if (lf_ctx == NULL)
643  return NULL;
644 
645  lf_ctx->Write = SCLogFileWrite;
646  lf_ctx->Close = SCLogFileClose;
647 
648  return lf_ctx;
649 }
650 
651 /** \brief LogFileEnsureExists() Ensure a log file context for the thread exists
652  * \param parent_ctx
653  * \param thread_id
654  * \retval LogFileCtx * pointer if successful; NULL otherwise
655  */
656 LogFileCtx *LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
657 {
658  /* threaded output disabled */
659  if (!parent_ctx->threaded)
660  return parent_ctx;
661 
662  SCLogDebug("Adding reference %d to file ctx %p", thread_id, parent_ctx);
663  SCMutexLock(&parent_ctx->threads->mutex);
664  /* are there enough context slots already */
665  if (thread_id < parent_ctx->threads->slot_count) {
666  /* has it been opened yet? */
667  if (!parent_ctx->threads->lf_slots[thread_id]) {
668  SCLogDebug("Opening new file for %d reference to file ctx %p", thread_id, parent_ctx);
669  LogFileNewThreadedCtx(parent_ctx, parent_ctx->filename, parent_ctx->threads->append, thread_id);
670  }
671  SCMutexUnlock(&parent_ctx->threads->mutex);
672  SCLogDebug("Existing file for %d reference to file ctx %p", thread_id, parent_ctx);
673  return parent_ctx->threads->lf_slots[thread_id];
674  }
675 
676  /* ensure there's a slot for the caller */
677  int new_size = MAX(parent_ctx->threads->slot_count << 1, thread_id + 1);
678  SCLogDebug("Increasing slot count; current %d, trying %d",
679  parent_ctx->threads->slot_count, new_size);
680  LogFileCtx **new_array = SCRealloc(parent_ctx->threads->lf_slots, new_size * sizeof(LogFileCtx *));
681  if (new_array == NULL) {
682  /* Try one more time */
683  SCLogDebug("Unable to increase file context array size to %d; trying %d",
684  new_size, thread_id + 1);
685  new_size = thread_id + 1;
686  new_array = SCRealloc(parent_ctx->threads->lf_slots, new_size * sizeof(LogFileCtx *));
687  }
688 
689  if (new_array == NULL) {
690  SCMutexUnlock(&parent_ctx->threads->mutex);
691  SCLogError(SC_ERR_MEM_ALLOC, "Unable to increase file context array size to %d", new_size);
692  return NULL;
693  }
694 
695  parent_ctx->threads->lf_slots = new_array;
696  /* initialize newly added slots */
697  for (int i = parent_ctx->threads->slot_count; i < new_size; i++) {
698  parent_ctx->threads->lf_slots[i] = NULL;
699  }
700  parent_ctx->threads->slot_count = new_size;
701  LogFileNewThreadedCtx(parent_ctx, parent_ctx->filename, parent_ctx->threads->append, thread_id);
702 
703  SCMutexUnlock(&parent_ctx->threads->mutex);
704 
705  return parent_ctx->threads->lf_slots[thread_id];
706 }
707 
708 /** \brief LogFileThreadedName() Create file name for threaded EVE storage
709  *
710  */
711 static bool LogFileThreadedName(
712  const char *original_name, char *threaded_name, size_t len, uint32_t unique_id)
713 {
714  const char *base = SCBasename(original_name);
715  if (!base) {
717  "Invalid filename for threaded mode \"%s\"; "
718  "no basename found.",
719  original_name);
720  }
721 
722  /* Check if basename has an extension */
723  char *dot = strrchr(base, '.');
724  if (dot) {
725  char *tname = SCStrdup(original_name);
726  if (!tname) {
727  return false;
728  }
729 
730  /* Fetch extension location from original, not base
731  * for update
732  */
733  dot = strrchr(original_name, '.');
734  int dotpos = dot - original_name;
735  tname[dotpos] = '\0';
736  char *ext = tname + dotpos + 1;
737  if (strlen(tname) && strlen(ext)) {
738  snprintf(threaded_name, len, "%s.%d.%s", tname, unique_id, ext);
739  } else {
741  "Invalid filename for threaded mode \"%s\"; "
742  "filenames must include an extension, e.g: \"name.ext\"",
743  original_name);
744  }
745  SCFree(tname);
746  } else {
747  snprintf(threaded_name, len, "%s.%d", original_name, unique_id);
748  }
749  return true;
750 }
751 
752 /** \brief LogFileNewThreadedCtx() Create file context for threaded output
753  * \param parent_ctx
754  * \param log_path
755  * \param append
756  * \param thread_id
757  */
758 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append, int thread_id)
759 {
760  LogFileCtx *thread = SCCalloc(1, sizeof(LogFileCtx));
761  if (!thread) {
762  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate thread file context slot %d", thread_id);
763  return false;
764  }
765 
766  *thread = *parent_ctx;
767  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
768  char fname[NAME_MAX];
769  if (!LogFileThreadedName(log_path, fname, sizeof(fname), SC_ATOMIC_ADD(eve_file_id, 1))) {
770  SCLogError(SC_ERR_MEM_ALLOC, "Unable to create threaded filename for log");
771  goto error;
772  }
773  SCLogDebug("Thread open -- using name %s [replaces %s]", fname, log_path);
774  thread->fp = SCLogOpenFileFp(fname, append, thread->filemode);
775  if (thread->fp == NULL) {
776  goto error;
777  }
778  thread->filename = SCStrdup(fname);
779  if (!thread->filename) {
780  SCLogError(SC_ERR_MEM_ALLOC, "Unable to duplicate filename for context slot %d",
781  thread_id);
782  goto error;
783  }
784  thread->is_regular = true;
785  thread->Write = SCLogFileWriteNoLock;
786  thread->Close = SCLogFileCloseNoLock;
788  } else if (parent_ctx->type == LOGFILE_TYPE_PLUGIN) {
789  thread->plugin.plugin->ThreadInit(
790  thread->plugin.init_data, thread_id, &thread->plugin.thread_data);
791  }
792  thread->threaded = false;
793  thread->parent = parent_ctx;
794  thread->id = thread_id;
795 
796  parent_ctx->threads->lf_slots[thread_id] = thread;
797  return true;
798 
799 error:
800  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
801  SC_ATOMIC_SUB(eve_file_id, 1);
802  if (thread->fp) {
803  thread->Close(thread);
804  }
805  }
806  if (thread) {
807  SCFree(thread);
808  }
809  parent_ctx->threads->lf_slots[thread_id] = NULL;
810  return false;
811 }
812 
813 /** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
814  * \param lf_ctx pointer to the OutputCtx
815  * \retval int 1 if successful, 0 if error
816  * */
818 {
819  if (lf_ctx == NULL) {
820  SCReturnInt(0);
821  }
822 
823  if (lf_ctx->threaded) {
824  BUG_ON(lf_ctx->threads == NULL);
825  SCMutexDestroy(&lf_ctx->threads->mutex);
826  for(int i = 0; i < lf_ctx->threads->slot_count; i++) {
827  if (!lf_ctx->threads->lf_slots[i]) {
828  continue;
829  }
830  LogFileCtx *this_ctx = lf_ctx->threads->lf_slots[i];
831 
832  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
834  this_ctx->Close(this_ctx);
835  } else {
836  lf_ctx->plugin.plugin->ThreadDeinit(
837  this_ctx->plugin.init_data, this_ctx->plugin.thread_data);
838  }
839  SCFree(lf_ctx->threads->lf_slots[i]->filename);
840  SCFree(lf_ctx->threads->lf_slots[i]);
841  }
842  SCFree(lf_ctx->threads->lf_slots);
843  if (lf_ctx->threads->append)
844  SCFree(lf_ctx->threads->append);
845  SCFree(lf_ctx->threads);
846  } else {
847  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
848  if (lf_ctx->fp != NULL) {
849  lf_ctx->Close(lf_ctx);
850  }
851  if (lf_ctx->parent) {
852  SCMutexLock(&lf_ctx->parent->threads->mutex);
853  lf_ctx->parent->threads->lf_slots[lf_ctx->id] = NULL;
854  SCMutexUnlock(&lf_ctx->parent->threads->mutex);
855  }
856  }
857  SCMutexDestroy(&lf_ctx->fp_mutex);
858  }
859 
860  if (lf_ctx->type == LOGFILE_TYPE_PLUGIN) {
861  lf_ctx->plugin.plugin->Deinit(lf_ctx->plugin.init_data);
862  }
863 
864  if (lf_ctx->prefix != NULL) {
865  SCFree(lf_ctx->prefix);
866  lf_ctx->prefix_len = 0;
867  }
868 
869  if(lf_ctx->filename != NULL)
870  SCFree(lf_ctx->filename);
871 
872  if (lf_ctx->sensor_name)
873  SCFree(lf_ctx->sensor_name);
874 
875  if (!lf_ctx->threaded) {
877  }
878 
879  memset(lf_ctx, 0, sizeof(*lf_ctx));
880  SCFree(lf_ctx);
881 
882  SCReturnInt(1);
883 }
884 
885 int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
886 {
887  if (file_ctx->type == LOGFILE_TYPE_SYSLOG) {
888  syslog(file_ctx->syslog_setup.alert_syslog_level, "%s",
889  (const char *)MEMBUFFER_BUFFER(buffer));
890  } else if (file_ctx->type == LOGFILE_TYPE_FILE ||
891  file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
892  file_ctx->type == LOGFILE_TYPE_UNIX_STREAM)
893  {
894  /* append \n for files only */
895  MemBufferWriteString(buffer, "\n");
896  file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
897  MEMBUFFER_OFFSET(buffer), file_ctx);
898  } else if (file_ctx->type == LOGFILE_TYPE_PLUGIN) {
899  file_ctx->plugin.plugin->Write((const char *)MEMBUFFER_BUFFER(buffer),
900  MEMBUFFER_OFFSET(buffer), file_ctx->plugin.init_data, file_ctx->plugin.thread_data);
901  }
902 #ifdef HAVE_LIBHIREDIS
903  else if (file_ctx->type == LOGFILE_TYPE_REDIS) {
904  SCMutexLock(&file_ctx->fp_mutex);
905  LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer),
906  MEMBUFFER_OFFSET(buffer));
907  SCMutexUnlock(&file_ctx->fp_mutex);
908  }
909 #endif
910 
911  return 0;
912 }
SCParseTimeSizeString
uint64_t SCParseTimeSizeString(const char *str)
Parse string containing time size (1m, 1h, etc).
Definition: util-time.c:576
LogFileCtx_::rotation_flag
int rotation_flag
Definition: util-logopenfile.h:141
util-byte.h
LogFileCtx_::prefix_len
size_t prefix_len
Definition: util-logopenfile.h:120
SyslogSetup_::alert_syslog_level
int alert_syslog_level
Definition: util-logopenfile.h:44
LogThreadedFileCtx_::append
char * append
Definition: util-logopenfile.h:52
SCFerrorUnlocked
#define SCFerrorUnlocked
Definition: suricata-common.h:530
len
uint8_t len
Definition: app-layer-dnp3.h:2
LOGFILE_TYPE_REDIS
@ LOGFILE_TYPE_REDIS
Definition: util-logopenfile.h:40
LogFileCtx_::sensor_name
char * sensor_name
Definition: util-logopenfile.h:103
LogFileCtx_::reconn_timer
uint64_t reconn_timer
Definition: util-logopenfile.h:108
syslog
#define syslog(__pri, __fmt, __param)
Definition: win32-syslog.h:78
SC_ERR_SOCKET
@ SC_ERR_SOCKET
Definition: util-error.h:232
LogFileCtx_::fp_mutex
SCMutex fp_mutex
Definition: util-logopenfile.h:86
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_DECL_AND_INIT_WITH_VAL
#define SC_ATOMIC_DECL_AND_INIT_WITH_VAL(type, name, val)
wrapper for declaring an atomic variable and initializing it to a specific value
Definition: util-atomic.h:304
LogFileNewCtx
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
Definition: util-logopenfile.c:637
SCGetSecondsUntil
uint64_t SCGetSecondsUntil(const char *str, time_t epoch)
Get seconds until a time unit changes.
Definition: util-time.c:627
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
LogThreadedFileCtx_::lf_slots
struct LogFileCtx_ ** lf_slots
Definition: util-logopenfile.h:51
LogFileCtx_::json_flags
size_t json_flags
Definition: util-logopenfile.h:138
LogFilePluginCtx_::init_data
void * init_data
Definition: util-logopenfile.h:57
LogFilePluginCtx_::thread_data
void * thread_data
Definition: util-logopenfile.h:58
SCPluginFileType_::Deinit
void(* Deinit)(void *init_data)
Definition: suricata-plugin.h:57
LogFileCtx_::id
int id
Definition: util-logopenfile.h:91
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
LogFileCtx_
Definition: util-logopenfile.h:62
LOGFILE_TYPE_SYSLOG
@ LOGFILE_TYPE_SYSLOG
Definition: util-logopenfile.h:37
LOGFILE_RECONN_MIN_TIME
#define LOGFILE_RECONN_MIN_TIME
Definition: util-logopenfile.h:157
DEFAULT_LOG_MODE_APPEND
#define DEFAULT_LOG_MODE_APPEND
Definition: output.h:30
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
util-log-redis.h
LogFileCtx_::Write
int(* Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:79
LogFileCtx_::filename
char * filename
Definition: util-logopenfile.h:97
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:258
LogFileCtx_::plugin
LogFilePluginCtx plugin
Definition: util-logopenfile.h:82
m
SCMutex m
Definition: flow-hash.h:6
LogFileCtx_::send_flags
uint8_t send_flags
Definition: util-logopenfile.h:131
MAX
#define MAX(x, y)
Definition: suricata-common.h:376
SCConfLogReopen
int SCConfLogReopen(LogFileCtx *log_ctx)
Reopen a regular log file with the side-affect of truncating it.
Definition: util-logopenfile.c:606
IsRunModeOffline
bool IsRunModeOffline(enum RunModes run_mode_to_check)
Definition: runmodes.c:517
SCClearErrUnlocked
#define SCClearErrUnlocked
Definition: suricata-common.h:529
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
SCConfLogOpenGeneric
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: util-logopenfile.c:423
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
LogFileCtx_::sock_type
int sock_type
Definition: util-logopenfile.h:107
OutputRegisterFileRotationFlag
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
Definition: output.c:830
SCTimeToStringPattern
int SCTimeToStringPattern(time_t epoch, const char *pattern, char *str, size_t size)
Convert epoch time to string pattern.
Definition: util-time.c:547
LogFileWrite
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
Definition: util-logopenfile.c:885
SC_WARN_CHMOD
@ SC_WARN_CHMOD
Definition: util-error.h:330
LogFileCtx_::rotate_interval
uint64_t rotate_interval
Definition: util-logopenfile.h:115
DEFAULT_LOG_FILETYPE
#define DEFAULT_LOG_FILETYPE
Definition: output.h:31
LogFileCtx_::is_sock
int is_sock
Definition: util-logopenfile.h:106
SCFwriteUnlocked
#define SCFwriteUnlocked
Definition: suricata-common.h:527
LogFileCtx_::dropped
uint64_t dropped
Definition: util-logopenfile.h:151
SCFflushUnlocked
#define SCFflushUnlocked
Definition: suricata-common.h:528
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:656
SCPluginFileType_::ThreadDeinit
int(* ThreadDeinit)(void *init_data, void *thread_data)
Definition: suricata-plugin.h:61
LogFilePluginCtx_::plugin
SCPluginFileType * plugin
Definition: util-logopenfile.h:56
LogFileCtx_::type
enum LogFileType type
Definition: util-logopenfile.h:94
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
LOGFILE_TYPE_FILE
@ LOGFILE_TYPE_FILE
Definition: util-logopenfile.h:36
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:277
LogFileCtx_::threads
LogThreadedFileCtx * threads
Definition: util-logopenfile.h:70
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
SC_ERR_FOPEN
@ SC_ERR_FOPEN
Definition: util-error.h:74
LOGFILE_TYPE_PLUGIN
@ LOGFILE_TYPE_PLUGIN
Definition: util-logopenfile.h:41
SCPluginFileType_::Write
int(* Write)(const char *buffer, int buffer_len, void *init_data, void *thread_data)
Definition: suricata-plugin.h:55
conf.h
SCBasename
const char * SCBasename(const char *path)
Definition: util-path.c:236
LogFileCtx_::syslog_setup
SyslogSetup syslog_setup
Definition: util-logopenfile.h:73
LOGFILE_TYPE_UNIX_DGRAM
@ LOGFILE_TYPE_UNIX_DGRAM
Definition: util-logopenfile.h:38
LOGFILE_TYPE_UNIX_STREAM
@ LOGFILE_TYPE_UNIX_STREAM
Definition: util-logopenfile.h:39
MemBuffer_
Definition: util-buffer.h:27
LogFileCtx_::is_regular
uint8_t is_regular
Definition: util-logopenfile.h:135
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
SC_ERR_INVALID_YAML_CONF_ENTRY
@ SC_ERR_INVALID_YAML_CONF_ENTRY
Definition: util-error.h:169
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:814
LogFileCtx_::parent
struct LogFileCtx_ * parent
Definition: util-logopenfile.h:90
StringParseUint32
int StringParseUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:313
LogThreadedFileCtx_
Definition: util-logopenfile.h:48
suricata-common.h
LogFileCtx_::output_errors
uint64_t output_errors
Definition: util-logopenfile.h:153
LogThreadedFileCtx_::slot_count
int slot_count
Definition: util-logopenfile.h:49
ConfNode_::name
char * name
Definition: conf.h:33
MemBufferWriteString
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
SCLogOpenThreadedFile
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx, int slot_count)
Definition: util-logopenfile.c:323
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:45
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
LogFileFreeCtx
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
Definition: util-logopenfile.c:817
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(x,...)
Definition: util-debug.h:532
LogFileCtx_::rotate_time
time_t rotate_time
Definition: util-logopenfile.h:112
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
RunmodeGetCurrent
int RunmodeGetCurrent(void)
Definition: suricata.c:274
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
ConfigGetLogDirectory
const char * ConfigGetLogDirectory()
Definition: util-conf.c:35
SCLogWarning
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:244
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
LOGFILE_ROTATE_INTERVAL
#define LOGFILE_ROTATE_INTERVAL
Definition: util-logopenfile.h:162
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:590
LogThreadedFileCtx_::mutex
SCMutex mutex
Definition: util-logopenfile.h:50
LogFileCtx_::prefix
char * prefix
Definition: util-logopenfile.h:119
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
SC_ERR_LOG_OUTPUT
@ SC_ERR_LOG_OUTPUT
Definition: util-error.h:368
LogFileCtx_::flags
uint8_t flags
Definition: util-logopenfile.h:128
SCPluginFileType_::ThreadInit
int(* ThreadInit)(void *init_data, int thread_id, void **thread_data)
Definition: suricata-plugin.h:59
LogFileCtx_::Close
void(* Close)(struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:80
MEMBUFFER_BUFFER
#define MEMBUFFER_BUFFER(mem_buffer)
Get the MemBuffers underlying buffer.
Definition: util-buffer.h:50
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:232
OutputUnregisterFileRotationFlag
void OutputUnregisterFileRotationFlag(int *flag)
Unregister a file rotation flag.
Definition: output.c:852
MEMBUFFER_OFFSET
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
Definition: util-buffer.h:55
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
LogFileCtx_::fp
FILE * fp
Definition: util-logopenfile.h:64
output.h
LogFileCtx_::threaded
bool threaded
Definition: util-logopenfile.h:89
SCCreateDirectoryTree
int SCCreateDirectoryTree(const char *path, const bool final)
Recursively create a directory.
Definition: util-path.c:124
LogFileCtx_::filemode
uint32_t filemode
Definition: util-logopenfile.h:100
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842