suricata
util-logopenfile.c
Go to the documentation of this file.
1 /* vi: set et ts=4: */
2 /* Copyright (C) 2007-2021 Open Information Security Foundation
3  *
4  * You can copy, redistribute or modify this Program under the terms of
5  * the GNU General Public License version 2 as published by the Free
6  * Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * version 2 along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16  * 02110-1301, USA.
17  */
18 
19 /**
20  * \file
21  *
22  * \author Mike Pomraning <mpomraning@qualys.com>
23  *
24  * File-like output for logging: regular files and sockets.
25  */
26 
27 #include "suricata-common.h" /* errno.h, string.h, etc. */
28 #include "util-logopenfile.h"
29 #include "suricata.h"
30 #include "conf.h" /* ConfNode, etc. */
31 #include "output.h" /* DEFAULT_LOG_* */
32 #include "util-byte.h"
33 #include "util-conf.h"
34 #include "util-path.h"
35 #include "util-time.h"
36 
37 #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H)
38 #define BUILD_WITH_UNIXSOCKET
39 #include <sys/types.h>
40 #include <sys/socket.h>
41 #include <sys/un.h>
42 #endif
43 
44 #ifdef HAVE_LIBHIREDIS
45 #include "util-log-redis.h"
46 #endif /* HAVE_LIBHIREDIS */
47 
48 #define LOGFILE_NAME_MAX 255
49 
50 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append, int i);
51 
52 // Threaded eve.json identifier
53 static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint32_t, eve_file_id, 1);
54 
55 #ifdef BUILD_WITH_UNIXSOCKET
56 /** \brief connect to the indicated local stream socket, logging any errors
57  * \param path filesystem path to connect to
58  * \param log_err, non-zero if connect failure should be logged.
59  * \retval FILE* on success (fdopen'd wrapper of underlying socket)
60  * \retval NULL on error
61  */
62 static FILE *
63 SCLogOpenUnixSocketFp(const char *path, int sock_type, int log_err)
64 {
65  struct sockaddr_un saun;
66  int s = -1;
67  FILE * ret = NULL;
68 
69  memset(&saun, 0x00, sizeof(saun));
70 
71  s = socket(PF_UNIX, sock_type, 0);
72  if (s < 0) goto err;
73 
74  saun.sun_family = AF_UNIX;
75  strlcpy(saun.sun_path, path, sizeof(saun.sun_path));
76 
77  if (connect(s, (const struct sockaddr *)&saun, sizeof(saun)) < 0)
78  goto err;
79 
80  ret = fdopen(s, "w");
81  if (ret == NULL)
82  goto err;
83 
84  return ret;
85 
86 err:
87  if (log_err)
89  "Error connecting to socket \"%s\": %s (will keep trying)",
90  path, strerror(errno));
91 
92  if (s >= 0)
93  close(s);
94 
95  return NULL;
96 }
97 
98 /**
99  * \brief Attempt to reconnect a disconnected (or never-connected) Unix domain socket.
100  * \retval 1 if it is now connected; otherwise 0
101  */
102 static int SCLogUnixSocketReconnect(LogFileCtx *log_ctx)
103 {
104  int disconnected = 0;
105  if (log_ctx->fp) {
107  "Write error on Unix socket \"%s\": %s; reconnecting...",
108  log_ctx->filename, strerror(errno));
109  fclose(log_ctx->fp);
110  log_ctx->fp = NULL;
111  log_ctx->reconn_timer = 0;
112  disconnected = 1;
113  }
114 
115  struct timeval tv;
116  uint64_t now;
117  gettimeofday(&tv, NULL);
118  now = (uint64_t)tv.tv_sec * 1000;
119  now += tv.tv_usec / 1000; /* msec resolution */
120  if (log_ctx->reconn_timer != 0 &&
121  (now - log_ctx->reconn_timer) < LOGFILE_RECONN_MIN_TIME) {
122  /* Don't bother to try reconnecting too often. */
123  return 0;
124  }
125  log_ctx->reconn_timer = now;
126 
127  log_ctx->fp = SCLogOpenUnixSocketFp(log_ctx->filename, log_ctx->sock_type, 0);
128  if (log_ctx->fp) {
129  /* Connected at last (or reconnected) */
130  SCLogNotice("Reconnected socket \"%s\"", log_ctx->filename);
131  } else if (disconnected) {
132  SCLogWarning(SC_ERR_SOCKET, "Reconnect failed: %s (will keep trying)",
133  strerror(errno));
134  }
135 
136  return log_ctx->fp ? 1 : 0;
137 }
138 
139 static int SCLogFileWriteSocket(const char *buffer, int buffer_len,
140  LogFileCtx *ctx)
141 {
142  int tries = 0;
143  int ret = 0;
144  bool reopen = false;
145  if (ctx->fp == NULL && ctx->is_sock) {
146  SCLogUnixSocketReconnect(ctx);
147  }
148 tryagain:
149  ret = -1;
150  reopen = 0;
151  errno = 0;
152  if (ctx->fp != NULL) {
153  int fd = fileno(ctx->fp);
154  ssize_t size = send(fd, buffer, buffer_len, ctx->send_flags);
155  if (size > -1) {
156  ret = 0;
157  } else {
158  if (errno == EAGAIN || errno == EWOULDBLOCK) {
159  SCLogDebug("Socket would block, dropping event.");
160  } else if (errno == EINTR) {
161  if (tries++ == 0) {
162  SCLogDebug("Interrupted system call, trying again.");
163  goto tryagain;
164  }
165  SCLogDebug("Too many interrupted system calls, "
166  "dropping event.");
167  } else {
168  /* Some other error. Assume badness and reopen. */
169  SCLogDebug("Send failed: %s", strerror(errno));
170  reopen = true;
171  }
172  }
173  }
174 
175  if (reopen && tries++ == 0) {
176  if (SCLogUnixSocketReconnect(ctx)) {
177  goto tryagain;
178  }
179  }
180 
181  if (ret == -1) {
182  ctx->dropped++;
183  }
184 
185  return ret;
186 }
187 #endif /* BUILD_WITH_UNIXSOCKET */
188 static inline void OutputWriteLock(pthread_mutex_t *m)
189 {
190  SCMutexLock(m);
191 
192 }
193 
194 /**
195  * \brief Write buffer to log file.
196  * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
197  * characters successfully written).
198  */
199 static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
200 {
201  int ret = 0;
202 
203  BUG_ON(log_ctx->is_sock);
204 
205  /* Check for rotation. */
206  if (log_ctx->rotation_flag) {
207  log_ctx->rotation_flag = 0;
208  SCConfLogReopen(log_ctx);
209  }
210 
211  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
212  time_t now = time(NULL);
213  if (now >= log_ctx->rotate_time) {
214  SCConfLogReopen(log_ctx);
215  log_ctx->rotate_time = now + log_ctx->rotate_interval;
216  }
217  }
218 
219  if (log_ctx->fp) {
220  SCClearErrUnlocked(log_ctx->fp);
221  if (1 != SCFwriteUnlocked(buffer, buffer_len, 1, log_ctx->fp)) {
222  /* Only the first error is logged */
223  if (!log_ctx->output_errors) {
224  SCLogError(SC_ERR_LOG_OUTPUT, "%s error while writing to %s",
225  SCFerrorUnlocked(log_ctx->fp) ? strerror(errno) : "unknown error",
226  log_ctx->filename);
227  }
228  log_ctx->output_errors++;
229  } else {
230  SCFflushUnlocked(log_ctx->fp);
231  }
232  }
233 
234  return ret;
235 }
236 
237 /**
238  * \brief Write buffer to log file.
239  * \retval 0 on failure; otherwise, the return value of fwrite (number of
240  * characters successfully written).
241  */
242 static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
243 {
244  OutputWriteLock(&log_ctx->fp_mutex);
245  int ret = 0;
246 
247 #ifdef BUILD_WITH_UNIXSOCKET
248  if (log_ctx->is_sock) {
249  ret = SCLogFileWriteSocket(buffer, buffer_len, log_ctx);
250  } else
251 #endif
252  {
253 
254  /* Check for rotation. */
255  if (log_ctx->rotation_flag) {
256  log_ctx->rotation_flag = 0;
257  SCConfLogReopen(log_ctx);
258  }
259 
260  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
261  time_t now = time(NULL);
262  if (now >= log_ctx->rotate_time) {
263  SCConfLogReopen(log_ctx);
264  log_ctx->rotate_time = now + log_ctx->rotate_interval;
265  }
266  }
267 
268  if (log_ctx->fp) {
269  clearerr(log_ctx->fp);
270  if (1 != fwrite(buffer, buffer_len, 1, log_ctx->fp)) {
271  /* Only the first error is logged */
272  if (!log_ctx->output_errors) {
273  SCLogError(SC_ERR_LOG_OUTPUT, "%s error while writing to %s",
274  ferror(log_ctx->fp) ? strerror(errno) : "unknown error",
275  log_ctx->filename);
276  }
277  log_ctx->output_errors++;
278  } else {
279  fflush(log_ctx->fp);
280  }
281  }
282  }
283 
284  SCMutexUnlock(&log_ctx->fp_mutex);
285 
286  return ret;
287 }
288 
289 /** \brief generate filename based on pattern
290  * \param pattern pattern to use
291  * \retval char* on success
292  * \retval NULL on error
293  */
294 static char *SCLogFilenameFromPattern(const char *pattern)
295 {
296  char *filename = SCMalloc(PATH_MAX);
297  if (filename == NULL) {
298  return NULL;
299  }
300 
301  int rc = SCTimeToStringPattern(time(NULL), pattern, filename, PATH_MAX);
302  if (rc != 0) {
303  SCFree(filename);
304  return NULL;
305  }
306 
307  return filename;
308 }
309 
310 static void SCLogFileCloseNoLock(LogFileCtx *log_ctx)
311 {
312  SCLogDebug("Closing %s", log_ctx->filename);
313  if (log_ctx->fp)
314  fclose(log_ctx->fp);
315 
316  if (log_ctx->output_errors) {
317  SCLogError(SC_ERR_LOG_OUTPUT, "There were %" PRIu64 " output errors to %s",
318  log_ctx->output_errors, log_ctx->filename);
319  }
320 }
321 
322 static void SCLogFileClose(LogFileCtx *log_ctx)
323 {
324  SCMutexLock(&log_ctx->fp_mutex);
325  SCLogFileCloseNoLock(log_ctx);
326  SCMutexUnlock(&log_ctx->fp_mutex);
327 }
328 
330  const char *log_path, const char *append, LogFileCtx *parent_ctx, int slot_count)
331 {
332  parent_ctx->threads = SCCalloc(1, sizeof(LogThreadedFileCtx));
333  if (!parent_ctx->threads) {
334  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate threads container");
335  return false;
336  }
337 
338  parent_ctx->threads->append = SCStrdup(append == NULL ? DEFAULT_LOG_MODE_APPEND : append);
339  if (!parent_ctx->threads->append) {
340  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate threads append setting");
341  goto error_exit;
342  }
343 
344  parent_ctx->threads->slot_count = slot_count;
345  parent_ctx->threads->lf_slots = SCCalloc(slot_count, sizeof(LogFileCtx *));
346  if (!parent_ctx->threads->lf_slots) {
347  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate thread slots");
348  goto error_exit;
349  }
350  SCLogDebug("Allocated %d file context pointers for threaded array",
351  parent_ctx->threads->slot_count);
352  for (int slot = 1; slot < parent_ctx->threads->slot_count; slot++) {
353  if (!LogFileNewThreadedCtx(parent_ctx, log_path, parent_ctx->threads->append, slot)) {
354  /* TODO: clear allocated entries [1, slot) */
355  goto error_exit;
356  }
357  }
358  SCMutexInit(&parent_ctx->threads->mutex, NULL);
359  return true;
360 
361 error_exit:
362 
363  if (parent_ctx->threads->lf_slots) {
364  SCFree(parent_ctx->threads->lf_slots);
365  }
366  if (parent_ctx->threads->append) {
367  SCFree(parent_ctx->threads->append);
368  }
369  SCFree(parent_ctx->threads);
370  parent_ctx->threads = NULL;
371  return false;
372 }
373 
374 /** \brief open the indicated file, logging any errors
375  * \param path filesystem path to open
376  * \param append_setting open file with O_APPEND: "yes" or "no"
377  * \param mode permissions to set on file
378  * \retval FILE* on success
379  * \retval NULL on error
380  */
381 static FILE *
382 SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
383 {
384  FILE *ret = NULL;
385 
386  char *filename = SCLogFilenameFromPattern(path);
387  if (filename == NULL) {
388  return NULL;
389  }
390 
391  int rc = SCCreateDirectoryTree(filename, false);
392  if (rc < 0) {
393  SCFree(filename);
394  return NULL;
395  }
396 
397  if (ConfValIsTrue(append_setting)) {
398  ret = fopen(filename, "a");
399  } else {
400  ret = fopen(filename, "w");
401  }
402 
403  if (ret == NULL) {
404  SCLogError(SC_ERR_FOPEN, "Error opening file: \"%s\": %s",
405  filename, strerror(errno));
406  } else {
407  if (mode != 0) {
408 #ifdef OS_WIN32
409  int r = _chmod(filename, (mode_t)mode);
410 #else
411  int r = fchmod(fileno(ret), (mode_t)mode);
412 #endif
413  if (r < 0) {
414  SCLogWarning(SC_WARN_CHMOD, "Could not chmod %s to %o: %s",
415  filename, mode, strerror(errno));
416  }
417  }
418  }
419 
420  SCFree(filename);
421  return ret;
422 }
423 
424 /** \brief open a generic output "log file", which may be a regular file or a socket
425  * \param conf ConfNode structure for the output section in question
426  * \param log_ctx Log file context allocated by caller
427  * \param default_filename Default name of file to open, if not specified in ConfNode
428  * \param rotate Register the file for rotation in HUP.
429  * \retval 0 on success
430  * \retval -1 on error
431  */
432 int
434  LogFileCtx *log_ctx,
435  const char *default_filename,
436  int rotate)
437 {
438  char log_path[PATH_MAX];
439  const char *log_dir;
440  const char *filename, *filetype;
441 
442  // Arg check
443  if (conf == NULL || log_ctx == NULL || default_filename == NULL) {
445  "SCConfLogOpenGeneric(conf %p, ctx %p, default %p) "
446  "missing an argument",
447  conf, log_ctx, default_filename);
448  return -1;
449  }
450  if (log_ctx->fp != NULL) {
452  "SCConfLogOpenGeneric: previously initialized Log CTX "
453  "encountered");
454  return -1;
455  }
456 
457  // Resolve the given config
458  filename = ConfNodeLookupChildValue(conf, "filename");
459  if (filename == NULL)
460  filename = default_filename;
461 
462  log_dir = ConfigGetLogDirectory();
463 
464  if (PathIsAbsolute(filename)) {
465  snprintf(log_path, PATH_MAX, "%s", filename);
466  } else {
467  snprintf(log_path, PATH_MAX, "%s/%s", log_dir, filename);
468  }
469 
470  /* Rotate log file based on time */
471  const char *rotate_int = ConfNodeLookupChildValue(conf, "rotate-interval");
472  if (rotate_int != NULL) {
473  time_t now = time(NULL);
474  log_ctx->flags |= LOGFILE_ROTATE_INTERVAL;
475 
476  /* Use a specific time */
477  if (strcmp(rotate_int, "minute") == 0) {
478  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
479  log_ctx->rotate_interval = 60;
480  } else if (strcmp(rotate_int, "hour") == 0) {
481  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
482  log_ctx->rotate_interval = 3600;
483  } else if (strcmp(rotate_int, "day") == 0) {
484  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
485  log_ctx->rotate_interval = 86400;
486  }
487 
488  /* Use a timer */
489  else {
490  log_ctx->rotate_interval = SCParseTimeSizeString(rotate_int);
491  if (log_ctx->rotate_interval == 0) {
493  "invalid rotate-interval value");
494  }
495  log_ctx->rotate_time = now + log_ctx->rotate_interval;
496  }
497  }
498 
499  filetype = ConfNodeLookupChildValue(conf, "filetype");
500  if (filetype == NULL)
501  filetype = DEFAULT_LOG_FILETYPE;
502 
503  const char *filemode = ConfNodeLookupChildValue(conf, "filemode");
504  uint32_t mode = 0;
505  if (filemode != NULL && StringParseUint32(&mode, 8, (uint16_t)strlen(filemode), filemode) > 0) {
506  log_ctx->filemode = mode;
507  }
508 
509  const char *append = ConfNodeLookupChildValue(conf, "append");
510  if (append == NULL)
511  append = DEFAULT_LOG_MODE_APPEND;
512 
513  /* JSON flags */
514  log_ctx->json_flags = JSON_PRESERVE_ORDER|JSON_COMPACT|
515  JSON_ENSURE_ASCII|JSON_ESCAPE_SLASH;
516 
517  ConfNode *json_flags = ConfNodeLookupChild(conf, "json");
518 
519  if (json_flags != 0) {
520  const char *preserve_order = ConfNodeLookupChildValue(json_flags,
521  "preserve-order");
522  if (preserve_order != NULL && ConfValIsFalse(preserve_order))
523  log_ctx->json_flags &= ~(JSON_PRESERVE_ORDER);
524 
525  const char *compact = ConfNodeLookupChildValue(json_flags, "compact");
526  if (compact != NULL && ConfValIsFalse(compact))
527  log_ctx->json_flags &= ~(JSON_COMPACT);
528 
529  const char *ensure_ascii = ConfNodeLookupChildValue(json_flags,
530  "ensure-ascii");
531  if (ensure_ascii != NULL && ConfValIsFalse(ensure_ascii))
532  log_ctx->json_flags &= ~(JSON_ENSURE_ASCII);
533 
534  const char *escape_slash = ConfNodeLookupChildValue(json_flags,
535  "escape-slash");
536  if (escape_slash != NULL && ConfValIsFalse(escape_slash))
537  log_ctx->json_flags &= ~(JSON_ESCAPE_SLASH);
538  }
539 
540 #ifdef BUILD_WITH_UNIXSOCKET
541  if (log_ctx->threaded) {
542  if (strcasecmp(filetype, "unix_stream") == 0 || strcasecmp(filetype, "unix_dgram") == 0) {
543  FatalError(SC_ERR_FATAL, "Socket file types do not support threaded output");
544  }
545  }
546 #endif
547  // Now, what have we been asked to open?
548  if (strcasecmp(filetype, "unix_stream") == 0) {
549 #ifdef BUILD_WITH_UNIXSOCKET
550  /* Don't bail. May be able to connect later. */
551  log_ctx->is_sock = 1;
552  log_ctx->sock_type = SOCK_STREAM;
553  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM, 1);
554 #else
555  return -1;
556 #endif
557  } else if (strcasecmp(filetype, "unix_dgram") == 0) {
558 #ifdef BUILD_WITH_UNIXSOCKET
559  /* Don't bail. May be able to connect later. */
560  log_ctx->is_sock = 1;
561  log_ctx->sock_type = SOCK_DGRAM;
562  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM, 1);
563 #else
564  return -1;
565 #endif
566  } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
567  strcasecmp(filetype, "file") == 0) {
568  log_ctx->is_regular = 1;
569  if (!log_ctx->threaded) {
570  log_ctx->fp = SCLogOpenFileFp(log_path, append, log_ctx->filemode);
571  if (log_ctx->fp == NULL)
572  return -1; // Error already logged by Open...Fp routine
573  } else {
574  if (!SCLogOpenThreadedFile(log_path, append, log_ctx, 1)) {
575  return -1;
576  }
577  }
578  if (rotate) {
580  }
581  } else {
582  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry for "
583  "%s.filetype. Expected \"regular\" (default), \"unix_stream\", "
584  "or \"unix_dgram\"",
585  conf->name);
586  }
587  log_ctx->filename = SCStrdup(log_path);
588  if (unlikely(log_ctx->filename == NULL)) {
590  "Failed to allocate memory for filename");
591  return -1;
592  }
593 
594 #ifdef BUILD_WITH_UNIXSOCKET
595  /* If a socket and running live, do non-blocking writes. */
596  if (log_ctx->is_sock && !IsRunModeOffline(RunmodeGetCurrent())) {
597  SCLogInfo("Setting logging socket of non-blocking in live mode.");
598  log_ctx->send_flags |= MSG_DONTWAIT;
599  }
600 #endif
601  SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype,
602  filename);
603 
604  return 0;
605 }
606 
607 /**
608  * \brief Reopen a regular log file with the side-affect of truncating it.
609  *
610  * This is useful to clear the log file and start a new one, or to
611  * re-open the file after its been moved by something external
612  * (eg. logrotate).
613  */
615 {
616  if (!log_ctx->is_regular) {
617  /* Not supported and not needed on non-regular files. */
618  return 0;
619  }
620 
621  if (log_ctx->filename == NULL) {
623  "Can't re-open LogFileCtx without a filename.");
624  return -1;
625  }
626 
627  if (log_ctx->fp != NULL) {
628  fclose(log_ctx->fp);
629  }
630 
631  /* Reopen the file. Append is forced in case the file was not
632  * moved as part of a rotation process. */
633  SCLogDebug("Reopening log file %s.", log_ctx->filename);
634  log_ctx->fp = SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode);
635  if (log_ctx->fp == NULL) {
636  return -1; // Already logged by Open..Fp routine.
637  }
638 
639  return 0;
640 }
641 
642 /** \brief LogFileNewCtx() Get a new LogFileCtx
643  * \retval LogFileCtx * pointer if successful, NULL if error
644  * */
646 {
647  LogFileCtx* lf_ctx;
648  lf_ctx = (LogFileCtx*)SCCalloc(1, sizeof(LogFileCtx));
649 
650  if (lf_ctx == NULL)
651  return NULL;
652 
653  lf_ctx->Write = SCLogFileWrite;
654  lf_ctx->Close = SCLogFileClose;
655 
656  return lf_ctx;
657 }
658 
659 /** \brief LogFileEnsureExists() Ensure a log file context for the thread exists
660  * \param parent_ctx
661  * \param thread_id
662  * \retval LogFileCtx * pointer if successful; NULL otherwise
663  */
664 LogFileCtx *LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
665 {
666  /* threaded output disabled */
667  if (!parent_ctx->threaded)
668  return parent_ctx;
669 
670  SCLogDebug("Adding reference %d to file ctx %p", thread_id, parent_ctx);
671  SCMutexLock(&parent_ctx->threads->mutex);
672  /* are there enough context slots already */
673  if (thread_id < parent_ctx->threads->slot_count) {
674  /* has it been opened yet? */
675  if (!parent_ctx->threads->lf_slots[thread_id]) {
676  SCLogDebug("Opening new file for %d reference to file ctx %p", thread_id, parent_ctx);
677  LogFileNewThreadedCtx(parent_ctx, parent_ctx->filename, parent_ctx->threads->append, thread_id);
678  }
679  SCMutexUnlock(&parent_ctx->threads->mutex);
680  SCLogDebug("Existing file for %d reference to file ctx %p", thread_id, parent_ctx);
681  return parent_ctx->threads->lf_slots[thread_id];
682  }
683 
684  /* ensure there's a slot for the caller */
685  int new_size = MAX(parent_ctx->threads->slot_count << 1, thread_id + 1);
686  SCLogDebug("Increasing slot count; current %d, trying %d",
687  parent_ctx->threads->slot_count, new_size);
688  LogFileCtx **new_array = SCRealloc(parent_ctx->threads->lf_slots, new_size * sizeof(LogFileCtx *));
689  if (new_array == NULL) {
690  /* Try one more time */
691  SCLogDebug("Unable to increase file context array size to %d; trying %d",
692  new_size, thread_id + 1);
693  new_size = thread_id + 1;
694  new_array = SCRealloc(parent_ctx->threads->lf_slots, new_size * sizeof(LogFileCtx *));
695  }
696 
697  if (new_array == NULL) {
698  SCMutexUnlock(&parent_ctx->threads->mutex);
699  SCLogError(SC_ERR_MEM_ALLOC, "Unable to increase file context array size to %d", new_size);
700  return NULL;
701  }
702 
703  parent_ctx->threads->lf_slots = new_array;
704  /* initialize newly added slots */
705  for (int i = parent_ctx->threads->slot_count; i < new_size; i++) {
706  parent_ctx->threads->lf_slots[i] = NULL;
707  }
708  parent_ctx->threads->slot_count = new_size;
709  LogFileNewThreadedCtx(parent_ctx, parent_ctx->filename, parent_ctx->threads->append, thread_id);
710 
711  SCMutexUnlock(&parent_ctx->threads->mutex);
712 
713  return parent_ctx->threads->lf_slots[thread_id];
714 }
715 
716 /** \brief LogFileThreadedName() Create file name for threaded EVE storage
717  *
718  */
719 static bool LogFileThreadedName(
720  const char *original_name, char *threaded_name, size_t len, uint32_t unique_id)
721 {
722  if (strcmp("/dev/null", original_name) == 0) {
723  strlcpy(threaded_name, original_name, len);
724  return true;
725  }
726 
727  const char *base = SCBasename(original_name);
728  if (!base) {
730  "Invalid filename for threaded mode \"%s\"; "
731  "no basename found.",
732  original_name);
733  }
734 
735  /* Check if basename has an extension */
736  char *dot = strrchr(base, '.');
737  if (dot) {
738  char *tname = SCStrdup(original_name);
739  if (!tname) {
740  return false;
741  }
742 
743  /* Fetch extension location from original, not base
744  * for update
745  */
746  dot = strrchr(original_name, '.');
747  int dotpos = dot - original_name;
748  tname[dotpos] = '\0';
749  char *ext = tname + dotpos + 1;
750  if (strlen(tname) && strlen(ext)) {
751  snprintf(threaded_name, len, "%s.%u.%s", tname, unique_id, ext);
752  } else {
754  "Invalid filename for threaded mode \"%s\"; "
755  "filenames must include an extension, e.g: \"name.ext\"",
756  original_name);
757  }
758  SCFree(tname);
759  } else {
760  snprintf(threaded_name, len, "%s.%u", original_name, unique_id);
761  }
762  return true;
763 }
764 
765 /** \brief LogFileNewThreadedCtx() Create file context for threaded output
766  * \param parent_ctx
767  * \param log_path
768  * \param append
769  * \param thread_id
770  */
771 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append, int thread_id)
772 {
773  LogFileCtx *thread = SCCalloc(1, sizeof(LogFileCtx));
774  if (!thread) {
775  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate thread file context slot %d", thread_id);
776  return false;
777  }
778 
779  *thread = *parent_ctx;
780  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
781  char fname[LOGFILE_NAME_MAX];
782  if (!LogFileThreadedName(log_path, fname, sizeof(fname), SC_ATOMIC_ADD(eve_file_id, 1))) {
783  SCLogError(SC_ERR_MEM_ALLOC, "Unable to create threaded filename for log");
784  goto error;
785  }
786  SCLogDebug("Thread open -- using name %s [replaces %s]", fname, log_path);
787  thread->fp = SCLogOpenFileFp(fname, append, thread->filemode);
788  if (thread->fp == NULL) {
789  goto error;
790  }
791  thread->filename = SCStrdup(fname);
792  if (!thread->filename) {
793  SCLogError(SC_ERR_MEM_ALLOC, "Unable to duplicate filename for context slot %d",
794  thread_id);
795  goto error;
796  }
797  thread->is_regular = true;
798  thread->Write = SCLogFileWriteNoLock;
799  thread->Close = SCLogFileCloseNoLock;
801  } else if (parent_ctx->type == LOGFILE_TYPE_PLUGIN) {
802  thread->plugin.plugin->ThreadInit(
803  thread->plugin.init_data, thread_id, &thread->plugin.thread_data);
804  }
805  thread->threaded = false;
806  thread->parent = parent_ctx;
807  thread->id = thread_id;
808 
809  parent_ctx->threads->lf_slots[thread_id] = thread;
810  return true;
811 
812 error:
813  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
814  SC_ATOMIC_SUB(eve_file_id, 1);
815  if (thread->fp) {
816  thread->Close(thread);
817  }
818  }
819  if (thread) {
820  SCFree(thread);
821  }
822  parent_ctx->threads->lf_slots[thread_id] = NULL;
823  return false;
824 }
825 
826 /** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
827  * \param lf_ctx pointer to the OutputCtx
828  * \retval int 1 if successful, 0 if error
829  * */
831 {
832  if (lf_ctx == NULL) {
833  SCReturnInt(0);
834  }
835 
836  if (lf_ctx->threaded) {
837  BUG_ON(lf_ctx->threads == NULL);
838  SCMutexDestroy(&lf_ctx->threads->mutex);
839  for(int i = 0; i < lf_ctx->threads->slot_count; i++) {
840  if (!lf_ctx->threads->lf_slots[i]) {
841  continue;
842  }
843  LogFileCtx *this_ctx = lf_ctx->threads->lf_slots[i];
844 
845  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
847  this_ctx->Close(this_ctx);
848  } else {
849  lf_ctx->plugin.plugin->ThreadDeinit(
850  this_ctx->plugin.init_data, this_ctx->plugin.thread_data);
851  }
852  SCFree(lf_ctx->threads->lf_slots[i]->filename);
853  SCFree(lf_ctx->threads->lf_slots[i]);
854  }
855  SCFree(lf_ctx->threads->lf_slots);
856  if (lf_ctx->threads->append)
857  SCFree(lf_ctx->threads->append);
858  SCFree(lf_ctx->threads);
859  } else {
860  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
861  if (lf_ctx->fp != NULL) {
862  lf_ctx->Close(lf_ctx);
863  }
864  if (lf_ctx->parent) {
865  SCMutexLock(&lf_ctx->parent->threads->mutex);
866  lf_ctx->parent->threads->lf_slots[lf_ctx->id] = NULL;
867  SCMutexUnlock(&lf_ctx->parent->threads->mutex);
868  }
869  }
870  SCMutexDestroy(&lf_ctx->fp_mutex);
871  }
872 
873  if (lf_ctx->type == LOGFILE_TYPE_PLUGIN) {
874  lf_ctx->plugin.plugin->Deinit(lf_ctx->plugin.init_data);
875  }
876 
877  if (lf_ctx->prefix != NULL) {
878  SCFree(lf_ctx->prefix);
879  lf_ctx->prefix_len = 0;
880  }
881 
882  if(lf_ctx->filename != NULL)
883  SCFree(lf_ctx->filename);
884 
885  if (lf_ctx->sensor_name)
886  SCFree(lf_ctx->sensor_name);
887 
888  if (!lf_ctx->threaded) {
890  }
891 
892  memset(lf_ctx, 0, sizeof(*lf_ctx));
893  SCFree(lf_ctx);
894 
895  SCReturnInt(1);
896 }
897 
898 int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
899 {
900  if (file_ctx->type == LOGFILE_TYPE_FILE || file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
901  file_ctx->type == LOGFILE_TYPE_UNIX_STREAM) {
902  /* append \n for files only */
903  MemBufferWriteString(buffer, "\n");
904  file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
905  MEMBUFFER_OFFSET(buffer), file_ctx);
906  } else if (file_ctx->type == LOGFILE_TYPE_PLUGIN) {
907  file_ctx->plugin.plugin->Write((const char *)MEMBUFFER_BUFFER(buffer),
908  MEMBUFFER_OFFSET(buffer), file_ctx->plugin.init_data, file_ctx->plugin.thread_data);
909  }
910 #ifdef HAVE_LIBHIREDIS
911  else if (file_ctx->type == LOGFILE_TYPE_REDIS) {
912  SCMutexLock(&file_ctx->fp_mutex);
913  LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer),
914  MEMBUFFER_OFFSET(buffer));
915  SCMutexUnlock(&file_ctx->fp_mutex);
916  }
917 #endif
918 
919  return 0;
920 }
SCParseTimeSizeString
uint64_t SCParseTimeSizeString(const char *str)
Parse string containing time size (1m, 1h, etc).
Definition: util-time.c:579
LogFileCtx_::rotation_flag
int rotation_flag
Definition: util-logopenfile.h:143
util-byte.h
LogFileCtx_::prefix_len
size_t prefix_len
Definition: util-logopenfile.h:122
LogThreadedFileCtx_::append
char * append
Definition: util-logopenfile.h:55
SCEveFileType_::Write
int(* Write)(const char *buffer, int buffer_len, void *init_data, void *thread_data)
Definition: suricata-plugin.h:54
SCFerrorUnlocked
#define SCFerrorUnlocked
Definition: suricata-common.h:523
len
uint8_t len
Definition: app-layer-dnp3.h:2
LOGFILE_TYPE_REDIS
@ LOGFILE_TYPE_REDIS
Definition: util-logopenfile.h:41
LogFileCtx_::sensor_name
char * sensor_name
Definition: util-logopenfile.h:105
LogFileCtx_::reconn_timer
uint64_t reconn_timer
Definition: util-logopenfile.h:110
SC_ERR_SOCKET
@ SC_ERR_SOCKET
Definition: util-error.h:232
LogFileCtx_::fp_mutex
SCMutex fp_mutex
Definition: util-logopenfile.h:88
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_DECL_AND_INIT_WITH_VAL
#define SC_ATOMIC_DECL_AND_INIT_WITH_VAL(type, name, val)
wrapper for declaring an atomic variable and initializing it to a specific value
Definition: util-atomic.h:304
LogFileNewCtx
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
Definition: util-logopenfile.c:645
SCGetSecondsUntil
uint64_t SCGetSecondsUntil(const char *str, time_t epoch)
Get seconds until a time unit changes.
Definition: util-time.c:630
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
LogThreadedFileCtx_::lf_slots
struct LogFileCtx_ ** lf_slots
Definition: util-logopenfile.h:54
LogFileCtx_::json_flags
size_t json_flags
Definition: util-logopenfile.h:140
LogFilePluginCtx_::init_data
void * init_data
Definition: util-logopenfile.h:60
LogFilePluginCtx_::thread_data
void * thread_data
Definition: util-logopenfile.h:61
LogFileCtx_::id
int id
Definition: util-logopenfile.h:93
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
LogFileCtx_
Definition: util-logopenfile.h:65
LOGFILE_RECONN_MIN_TIME
#define LOGFILE_RECONN_MIN_TIME
Definition: util-logopenfile.h:159
DEFAULT_LOG_MODE_APPEND
#define DEFAULT_LOG_MODE_APPEND
Definition: output.h:30
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
util-log-redis.h
LogFileCtx_::Write
int(* Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:81
LogFileCtx_::filename
char * filename
Definition: util-logopenfile.h:99
LOGFILE_NAME_MAX
#define LOGFILE_NAME_MAX
Definition: util-logopenfile.c:48
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:270
LogFileCtx_::plugin
LogFilePluginCtx plugin
Definition: util-logopenfile.h:84
m
SCMutex m
Definition: flow-hash.h:6
LogFileCtx_::send_flags
uint8_t send_flags
Definition: util-logopenfile.h:133
MAX
#define MAX(x, y)
Definition: suricata-common.h:384
SCConfLogReopen
int SCConfLogReopen(LogFileCtx *log_ctx)
Reopen a regular log file with the side-affect of truncating it.
Definition: util-logopenfile.c:614
IsRunModeOffline
bool IsRunModeOffline(enum RunModes run_mode_to_check)
Definition: runmodes.c:552
SCClearErrUnlocked
#define SCClearErrUnlocked
Definition: suricata-common.h:522
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:522
SCConfLogOpenGeneric
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: util-logopenfile.c:433
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
LogFileCtx_::sock_type
int sock_type
Definition: util-logopenfile.h:109
OutputRegisterFileRotationFlag
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
Definition: output.c:832
SCTimeToStringPattern
int SCTimeToStringPattern(time_t epoch, const char *pattern, char *str, size_t size)
Convert epoch time to string pattern.
Definition: util-time.c:550
LogFileWrite
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
Definition: util-logopenfile.c:898
SC_WARN_CHMOD
@ SC_WARN_CHMOD
Definition: util-error.h:330
LogFileCtx_::rotate_interval
uint64_t rotate_interval
Definition: util-logopenfile.h:117
DEFAULT_LOG_FILETYPE
#define DEFAULT_LOG_FILETYPE
Definition: output.h:31
LogFileCtx_::is_sock
int is_sock
Definition: util-logopenfile.h:108
SCFwriteUnlocked
#define SCFwriteUnlocked
Definition: suricata-common.h:520
LogFileCtx_::dropped
uint64_t dropped
Definition: util-logopenfile.h:153
SCFflushUnlocked
#define SCFflushUnlocked
Definition: suricata-common.h:521
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:664
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
util-time.h
LogFileCtx_::type
enum LogFileType type
Definition: util-logopenfile.h:96
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
LOGFILE_TYPE_FILE
@ LOGFILE_TYPE_FILE
Definition: util-logopenfile.h:38
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
LogFileCtx_::threads
LogThreadedFileCtx * threads
Definition: util-logopenfile.h:73
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
SC_ERR_FOPEN
@ SC_ERR_FOPEN
Definition: util-error.h:74
LOGFILE_TYPE_PLUGIN
@ LOGFILE_TYPE_PLUGIN
Definition: util-logopenfile.h:42
conf.h
SCBasename
const char * SCBasename(const char *path)
Definition: util-path.c:235
LOGFILE_TYPE_UNIX_DGRAM
@ LOGFILE_TYPE_UNIX_DGRAM
Definition: util-logopenfile.h:39
LOGFILE_TYPE_UNIX_STREAM
@ LOGFILE_TYPE_UNIX_STREAM
Definition: util-logopenfile.h:40
MemBuffer_
Definition: util-buffer.h:27
LogFileCtx_::is_regular
uint8_t is_regular
Definition: util-logopenfile.h:137
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:215
SC_ERR_INVALID_YAML_CONF_ENTRY
@ SC_ERR_INVALID_YAML_CONF_ENTRY
Definition: util-error.h:169
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:771
LogFileCtx_::parent
struct LogFileCtx_ * parent
Definition: util-logopenfile.h:92
util-conf.h
LogThreadedFileCtx_
Definition: util-logopenfile.h:51
suricata-common.h
LogFilePluginCtx_::plugin
SCEveFileType * plugin
Definition: util-logopenfile.h:59
util-path.h
LogFileCtx_::output_errors
uint64_t output_errors
Definition: util-logopenfile.h:155
LogThreadedFileCtx_::slot_count
int slot_count
Definition: util-logopenfile.h:52
ConfNode_::name
char * name
Definition: conf.h:33
MemBufferWriteString
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
SCLogOpenThreadedFile
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx, int slot_count)
Definition: util-logopenfile.c:329
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:255
LogFileFreeCtx
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
Definition: util-logopenfile.c:830
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(x,...)
Definition: util-debug.h:530
LogFileCtx_::rotate_time
time_t rotate_time
Definition: util-logopenfile.h:114
SCEveFileType_::Deinit
void(* Deinit)(void *init_data)
Definition: suricata-plugin.h:56
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
RunmodeGetCurrent
int RunmodeGetCurrent(void)
Definition: suricata.c:243
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCEveFileType_::ThreadDeinit
int(* ThreadDeinit)(void *init_data, void *thread_data)
Definition: suricata-plugin.h:60
ConfigGetLogDirectory
const char * ConfigGetLogDirectory()
Definition: util-conf.c:37
SCLogWarning
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:242
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
LOGFILE_ROTATE_INTERVAL
#define LOGFILE_ROTATE_INTERVAL
Definition: util-logopenfile.h:164
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:547
LogThreadedFileCtx_::mutex
SCMutex mutex
Definition: util-logopenfile.h:53
LogFileCtx_::prefix
char * prefix
Definition: util-logopenfile.h:121
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
suricata.h
SC_ERR_LOG_OUTPUT
@ SC_ERR_LOG_OUTPUT
Definition: util-error.h:368
LogFileCtx_::flags
uint8_t flags
Definition: util-logopenfile.h:130
LogFileCtx_::Close
void(* Close)(struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:82
MEMBUFFER_BUFFER
#define MEMBUFFER_BUFFER(mem_buffer)
Get the MemBuffers underlying buffer.
Definition: util-buffer.h:50
SCEveFileType_::ThreadInit
int(* ThreadInit)(void *init_data, int thread_id, void **thread_data)
Definition: suricata-plugin.h:58
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:230
OutputUnregisterFileRotationFlag
void OutputUnregisterFileRotationFlag(int *flag)
Unregister a file rotation flag.
Definition: output.c:854
MEMBUFFER_OFFSET
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
Definition: util-buffer.h:55
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:302
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
LogFileCtx_::fp
FILE * fp
Definition: util-logopenfile.h:67
output.h
LogFileCtx_::threaded
bool threaded
Definition: util-logopenfile.h:91
SCCreateDirectoryTree
int SCCreateDirectoryTree(const char *path, const bool final)
Recursively create a directory.
Definition: util-path.c:123
LogFileCtx_::filemode
uint32_t filemode
Definition: util-logopenfile.h:102
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:799