suricata
util-logopenfile.c
Go to the documentation of this file.
1 /* vi: set et ts=4: */
2 /* Copyright (C) 2007-2021 Open Information Security Foundation
3  *
4  * You can copy, redistribute or modify this Program under the terms of
5  * the GNU General Public License version 2 as published by the Free
6  * Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * version 2 along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16  * 02110-1301, USA.
17  */
18 
19 /**
20  * \file
21  *
22  * \author Mike Pomraning <mpomraning@qualys.com>
23  *
24  * File-like output for logging: regular files and sockets.
25  */
26 
27 #include "suricata-common.h" /* errno.h, string.h, etc. */
28 #include "util-logopenfile.h"
29 #include "suricata.h"
30 #include "conf.h" /* ConfNode, etc. */
31 #include "output.h" /* DEFAULT_LOG_* */
32 #include "util-byte.h"
33 #include "util-time.h"
34 
35 #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H)
36 #define BUILD_WITH_UNIXSOCKET
37 #include <sys/types.h>
38 #include <sys/socket.h>
39 #include <sys/un.h>
40 #endif
41 
42 #ifdef HAVE_LIBHIREDIS
43 #include "util-log-redis.h"
44 #endif /* HAVE_LIBHIREDIS */
45 
46 #define LOGFILE_NAME_MAX 255
47 
48 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append, int i);
49 
50 // Threaded eve.json identifier
51 static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint32_t, eve_file_id, 1);
52 
53 #ifdef BUILD_WITH_UNIXSOCKET
54 /** \brief connect to the indicated local stream socket, logging any errors
55  * \param path filesystem path to connect to
56  * \param log_err, non-zero if connect failure should be logged.
57  * \retval FILE* on success (fdopen'd wrapper of underlying socket)
58  * \retval NULL on error
59  */
60 static FILE *
61 SCLogOpenUnixSocketFp(const char *path, int sock_type, int log_err)
62 {
63  struct sockaddr_un saun;
64  int s = -1;
65  FILE * ret = NULL;
66 
67  memset(&saun, 0x00, sizeof(saun));
68 
69  s = socket(PF_UNIX, sock_type, 0);
70  if (s < 0) goto err;
71 
72  saun.sun_family = AF_UNIX;
73  strlcpy(saun.sun_path, path, sizeof(saun.sun_path));
74 
75  if (connect(s, (const struct sockaddr *)&saun, sizeof(saun)) < 0)
76  goto err;
77 
78  ret = fdopen(s, "w");
79  if (ret == NULL)
80  goto err;
81 
82  return ret;
83 
84 err:
85  if (log_err)
87  "Error connecting to socket \"%s\": %s (will keep trying)",
88  path, strerror(errno));
89 
90  if (s >= 0)
91  close(s);
92 
93  return NULL;
94 }
95 
96 /**
97  * \brief Attempt to reconnect a disconnected (or never-connected) Unix domain socket.
98  * \retval 1 if it is now connected; otherwise 0
99  */
100 static int SCLogUnixSocketReconnect(LogFileCtx *log_ctx)
101 {
102  int disconnected = 0;
103  if (log_ctx->fp) {
105  "Write error on Unix socket \"%s\": %s; reconnecting...",
106  log_ctx->filename, strerror(errno));
107  fclose(log_ctx->fp);
108  log_ctx->fp = NULL;
109  log_ctx->reconn_timer = 0;
110  disconnected = 1;
111  }
112 
113  struct timeval tv;
114  uint64_t now;
115  gettimeofday(&tv, NULL);
116  now = (uint64_t)tv.tv_sec * 1000;
117  now += tv.tv_usec / 1000; /* msec resolution */
118  if (log_ctx->reconn_timer != 0 &&
119  (now - log_ctx->reconn_timer) < LOGFILE_RECONN_MIN_TIME) {
120  /* Don't bother to try reconnecting too often. */
121  return 0;
122  }
123  log_ctx->reconn_timer = now;
124 
125  log_ctx->fp = SCLogOpenUnixSocketFp(log_ctx->filename, log_ctx->sock_type, 0);
126  if (log_ctx->fp) {
127  /* Connected at last (or reconnected) */
128  SCLogNotice("Reconnected socket \"%s\"", log_ctx->filename);
129  } else if (disconnected) {
130  SCLogWarning(SC_ERR_SOCKET, "Reconnect failed: %s (will keep trying)",
131  strerror(errno));
132  }
133 
134  return log_ctx->fp ? 1 : 0;
135 }
136 
137 static int SCLogFileWriteSocket(const char *buffer, int buffer_len,
138  LogFileCtx *ctx)
139 {
140  int tries = 0;
141  int ret = 0;
142  bool reopen = false;
143  if (ctx->fp == NULL && ctx->is_sock) {
144  SCLogUnixSocketReconnect(ctx);
145  }
146 tryagain:
147  ret = -1;
148  reopen = 0;
149  errno = 0;
150  if (ctx->fp != NULL) {
151  int fd = fileno(ctx->fp);
152  ssize_t size = send(fd, buffer, buffer_len, ctx->send_flags);
153  if (size > -1) {
154  ret = 0;
155  } else {
156  if (errno == EAGAIN || errno == EWOULDBLOCK) {
157  SCLogDebug("Socket would block, dropping event.");
158  } else if (errno == EINTR) {
159  if (tries++ == 0) {
160  SCLogDebug("Interrupted system call, trying again.");
161  goto tryagain;
162  }
163  SCLogDebug("Too many interrupted system calls, "
164  "dropping event.");
165  } else {
166  /* Some other error. Assume badness and reopen. */
167  SCLogDebug("Send failed: %s", strerror(errno));
168  reopen = true;
169  }
170  }
171  }
172 
173  if (reopen && tries++ == 0) {
174  if (SCLogUnixSocketReconnect(ctx)) {
175  goto tryagain;
176  }
177  }
178 
179  if (ret == -1) {
180  ctx->dropped++;
181  }
182 
183  return ret;
184 }
185 #endif /* BUILD_WITH_UNIXSOCKET */
186 static inline void OutputWriteLock(pthread_mutex_t *m)
187 {
188  SCMutexLock(m);
189 
190 }
191 
192 /**
193  * \brief Write buffer to log file.
194  * \retval 0 on failure; otherwise, the return value of fwrite_unlocked (number of
195  * characters successfully written).
196  */
197 static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
198 {
199  int ret = 0;
200 
201  BUG_ON(log_ctx->is_sock);
202 
203  /* Check for rotation. */
204  if (log_ctx->rotation_flag) {
205  log_ctx->rotation_flag = 0;
206  SCConfLogReopen(log_ctx);
207  }
208 
209  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
210  time_t now = time(NULL);
211  if (now >= log_ctx->rotate_time) {
212  SCConfLogReopen(log_ctx);
213  log_ctx->rotate_time = now + log_ctx->rotate_interval;
214  }
215  }
216 
217  if (log_ctx->fp) {
218  SCClearErrUnlocked(log_ctx->fp);
219  if (1 != SCFwriteUnlocked(buffer, buffer_len, 1, log_ctx->fp)) {
220  /* Only the first error is logged */
221  if (!log_ctx->output_errors) {
222  SCLogError(SC_ERR_LOG_OUTPUT, "%s error while writing to %s",
223  SCFerrorUnlocked(log_ctx->fp) ? strerror(errno) : "unknown error",
224  log_ctx->filename);
225  }
226  log_ctx->output_errors++;
227  } else {
228  SCFflushUnlocked(log_ctx->fp);
229  }
230  }
231 
232  return ret;
233 }
234 
235 /**
236  * \brief Write buffer to log file.
237  * \retval 0 on failure; otherwise, the return value of fwrite (number of
238  * characters successfully written).
239  */
240 static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
241 {
242  OutputWriteLock(&log_ctx->fp_mutex);
243  int ret = 0;
244 
245 #ifdef BUILD_WITH_UNIXSOCKET
246  if (log_ctx->is_sock) {
247  ret = SCLogFileWriteSocket(buffer, buffer_len, log_ctx);
248  } else
249 #endif
250  {
251 
252  /* Check for rotation. */
253  if (log_ctx->rotation_flag) {
254  log_ctx->rotation_flag = 0;
255  SCConfLogReopen(log_ctx);
256  }
257 
258  if (log_ctx->flags & LOGFILE_ROTATE_INTERVAL) {
259  time_t now = time(NULL);
260  if (now >= log_ctx->rotate_time) {
261  SCConfLogReopen(log_ctx);
262  log_ctx->rotate_time = now + log_ctx->rotate_interval;
263  }
264  }
265 
266  if (log_ctx->fp) {
267  clearerr(log_ctx->fp);
268  if (1 != fwrite(buffer, buffer_len, 1, log_ctx->fp)) {
269  /* Only the first error is logged */
270  if (!log_ctx->output_errors) {
271  SCLogError(SC_ERR_LOG_OUTPUT, "%s error while writing to %s",
272  ferror(log_ctx->fp) ? strerror(errno) : "unknown error",
273  log_ctx->filename);
274  }
275  log_ctx->output_errors++;
276  } else {
277  fflush(log_ctx->fp);
278  }
279  }
280  }
281 
282  SCMutexUnlock(&log_ctx->fp_mutex);
283 
284  return ret;
285 }
286 
287 /** \brief generate filename based on pattern
288  * \param pattern pattern to use
289  * \retval char* on success
290  * \retval NULL on error
291  */
292 static char *SCLogFilenameFromPattern(const char *pattern)
293 {
294  char *filename = SCMalloc(PATH_MAX);
295  if (filename == NULL) {
296  return NULL;
297  }
298 
299  int rc = SCTimeToStringPattern(time(NULL), pattern, filename, PATH_MAX);
300  if (rc != 0) {
301  SCFree(filename);
302  return NULL;
303  }
304 
305  return filename;
306 }
307 
308 static void SCLogFileCloseNoLock(LogFileCtx *log_ctx)
309 {
310  SCLogDebug("Closing %s", log_ctx->filename);
311  if (log_ctx->fp)
312  fclose(log_ctx->fp);
313 
314  if (log_ctx->output_errors) {
315  SCLogError(SC_ERR_LOG_OUTPUT, "There were %" PRIu64 " output errors to %s",
316  log_ctx->output_errors, log_ctx->filename);
317  }
318 }
319 
320 static void SCLogFileClose(LogFileCtx *log_ctx)
321 {
322  SCMutexLock(&log_ctx->fp_mutex);
323  SCLogFileCloseNoLock(log_ctx);
324  SCMutexUnlock(&log_ctx->fp_mutex);
325 }
326 
328  const char *log_path, const char *append, LogFileCtx *parent_ctx, int slot_count)
329 {
330  parent_ctx->threads = SCCalloc(1, sizeof(LogThreadedFileCtx));
331  if (!parent_ctx->threads) {
332  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate threads container");
333  return false;
334  }
335 
336  parent_ctx->threads->append = SCStrdup(append == NULL ? DEFAULT_LOG_MODE_APPEND : append);
337  if (!parent_ctx->threads->append) {
338  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate threads append setting");
339  goto error_exit;
340  }
341 
342  parent_ctx->threads->slot_count = slot_count;
343  parent_ctx->threads->lf_slots = SCCalloc(slot_count, sizeof(LogFileCtx *));
344  if (!parent_ctx->threads->lf_slots) {
345  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate thread slots");
346  goto error_exit;
347  }
348  SCLogDebug("Allocated %d file context pointers for threaded array",
349  parent_ctx->threads->slot_count);
350  for (int slot = 1; slot < parent_ctx->threads->slot_count; slot++) {
351  if (!LogFileNewThreadedCtx(parent_ctx, log_path, parent_ctx->threads->append, slot)) {
352  /* TODO: clear allocated entries [1, slot) */
353  goto error_exit;
354  }
355  }
356  SCMutexInit(&parent_ctx->threads->mutex, NULL);
357  return true;
358 
359 error_exit:
360 
361  if (parent_ctx->threads->lf_slots) {
362  SCFree(parent_ctx->threads->lf_slots);
363  }
364  if (parent_ctx->threads->append) {
365  SCFree(parent_ctx->threads->append);
366  }
367  SCFree(parent_ctx->threads);
368  parent_ctx->threads = NULL;
369  return false;
370 }
371 
372 /** \brief open the indicated file, logging any errors
373  * \param path filesystem path to open
374  * \param append_setting open file with O_APPEND: "yes" or "no"
375  * \param mode permissions to set on file
376  * \retval FILE* on success
377  * \retval NULL on error
378  */
379 static FILE *
380 SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
381 {
382  FILE *ret = NULL;
383 
384  char *filename = SCLogFilenameFromPattern(path);
385  if (filename == NULL) {
386  return NULL;
387  }
388 
389  int rc = SCCreateDirectoryTree(filename, false);
390  if (rc < 0) {
391  SCFree(filename);
392  return NULL;
393  }
394 
395  if (ConfValIsTrue(append_setting)) {
396  ret = fopen(filename, "a");
397  } else {
398  ret = fopen(filename, "w");
399  }
400 
401  if (ret == NULL) {
402  SCLogError(SC_ERR_FOPEN, "Error opening file: \"%s\": %s",
403  filename, strerror(errno));
404  } else {
405  if (mode != 0) {
406 #ifdef OS_WIN32
407  int r = _chmod(filename, (mode_t)mode);
408 #else
409  int r = fchmod(fileno(ret), (mode_t)mode);
410 #endif
411  if (r < 0) {
412  SCLogWarning(SC_WARN_CHMOD, "Could not chmod %s to %o: %s",
413  filename, mode, strerror(errno));
414  }
415  }
416  }
417 
418  SCFree(filename);
419  return ret;
420 }
421 
422 /** \brief open a generic output "log file", which may be a regular file or a socket
423  * \param conf ConfNode structure for the output section in question
424  * \param log_ctx Log file context allocated by caller
425  * \param default_filename Default name of file to open, if not specified in ConfNode
426  * \param rotate Register the file for rotation in HUP.
427  * \retval 0 on success
428  * \retval -1 on error
429  */
430 int
432  LogFileCtx *log_ctx,
433  const char *default_filename,
434  int rotate)
435 {
436  char log_path[PATH_MAX];
437  const char *log_dir;
438  const char *filename, *filetype;
439 
440  // Arg check
441  if (conf == NULL || log_ctx == NULL || default_filename == NULL) {
443  "SCConfLogOpenGeneric(conf %p, ctx %p, default %p) "
444  "missing an argument",
445  conf, log_ctx, default_filename);
446  return -1;
447  }
448  if (log_ctx->fp != NULL) {
450  "SCConfLogOpenGeneric: previously initialized Log CTX "
451  "encountered");
452  return -1;
453  }
454 
455  // Resolve the given config
456  filename = ConfNodeLookupChildValue(conf, "filename");
457  if (filename == NULL)
458  filename = default_filename;
459 
460  log_dir = ConfigGetLogDirectory();
461 
462  if (PathIsAbsolute(filename)) {
463  snprintf(log_path, PATH_MAX, "%s", filename);
464  } else {
465  snprintf(log_path, PATH_MAX, "%s/%s", log_dir, filename);
466  }
467 
468  /* Rotate log file based on time */
469  const char *rotate_int = ConfNodeLookupChildValue(conf, "rotate-interval");
470  if (rotate_int != NULL) {
471  time_t now = time(NULL);
472  log_ctx->flags |= LOGFILE_ROTATE_INTERVAL;
473 
474  /* Use a specific time */
475  if (strcmp(rotate_int, "minute") == 0) {
476  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
477  log_ctx->rotate_interval = 60;
478  } else if (strcmp(rotate_int, "hour") == 0) {
479  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
480  log_ctx->rotate_interval = 3600;
481  } else if (strcmp(rotate_int, "day") == 0) {
482  log_ctx->rotate_time = now + SCGetSecondsUntil(rotate_int, now);
483  log_ctx->rotate_interval = 86400;
484  }
485 
486  /* Use a timer */
487  else {
488  log_ctx->rotate_interval = SCParseTimeSizeString(rotate_int);
489  if (log_ctx->rotate_interval == 0) {
491  "invalid rotate-interval value");
492  }
493  log_ctx->rotate_time = now + log_ctx->rotate_interval;
494  }
495  }
496 
497  filetype = ConfNodeLookupChildValue(conf, "filetype");
498  if (filetype == NULL)
499  filetype = DEFAULT_LOG_FILETYPE;
500 
501  const char *filemode = ConfNodeLookupChildValue(conf, "filemode");
502  uint32_t mode = 0;
503  if (filemode != NULL && StringParseUint32(&mode, 8, (uint16_t)strlen(filemode), filemode) > 0) {
504  log_ctx->filemode = mode;
505  }
506 
507  const char *append = ConfNodeLookupChildValue(conf, "append");
508  if (append == NULL)
509  append = DEFAULT_LOG_MODE_APPEND;
510 
511  /* JSON flags */
512  log_ctx->json_flags = JSON_PRESERVE_ORDER|JSON_COMPACT|
513  JSON_ENSURE_ASCII|JSON_ESCAPE_SLASH;
514 
515  ConfNode *json_flags = ConfNodeLookupChild(conf, "json");
516 
517  if (json_flags != 0) {
518  const char *preserve_order = ConfNodeLookupChildValue(json_flags,
519  "preserve-order");
520  if (preserve_order != NULL && ConfValIsFalse(preserve_order))
521  log_ctx->json_flags &= ~(JSON_PRESERVE_ORDER);
522 
523  const char *compact = ConfNodeLookupChildValue(json_flags, "compact");
524  if (compact != NULL && ConfValIsFalse(compact))
525  log_ctx->json_flags &= ~(JSON_COMPACT);
526 
527  const char *ensure_ascii = ConfNodeLookupChildValue(json_flags,
528  "ensure-ascii");
529  if (ensure_ascii != NULL && ConfValIsFalse(ensure_ascii))
530  log_ctx->json_flags &= ~(JSON_ENSURE_ASCII);
531 
532  const char *escape_slash = ConfNodeLookupChildValue(json_flags,
533  "escape-slash");
534  if (escape_slash != NULL && ConfValIsFalse(escape_slash))
535  log_ctx->json_flags &= ~(JSON_ESCAPE_SLASH);
536  }
537 
538 #ifdef BUILD_WITH_UNIXSOCKET
539  if (log_ctx->threaded) {
540  if (strcasecmp(filetype, "unix_stream") == 0 || strcasecmp(filetype, "unix_dgram") == 0) {
541  FatalError(SC_ERR_FATAL, "Socket file types do not support threaded output");
542  }
543  }
544 #endif
545  // Now, what have we been asked to open?
546  if (strcasecmp(filetype, "unix_stream") == 0) {
547 #ifdef BUILD_WITH_UNIXSOCKET
548  /* Don't bail. May be able to connect later. */
549  log_ctx->is_sock = 1;
550  log_ctx->sock_type = SOCK_STREAM;
551  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM, 1);
552 #else
553  return -1;
554 #endif
555  } else if (strcasecmp(filetype, "unix_dgram") == 0) {
556 #ifdef BUILD_WITH_UNIXSOCKET
557  /* Don't bail. May be able to connect later. */
558  log_ctx->is_sock = 1;
559  log_ctx->sock_type = SOCK_DGRAM;
560  log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM, 1);
561 #else
562  return -1;
563 #endif
564  } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
565  strcasecmp(filetype, "file") == 0) {
566  log_ctx->is_regular = 1;
567  if (!log_ctx->threaded) {
568  log_ctx->fp = SCLogOpenFileFp(log_path, append, log_ctx->filemode);
569  if (log_ctx->fp == NULL)
570  return -1; // Error already logged by Open...Fp routine
571  } else {
572  if (!SCLogOpenThreadedFile(log_path, append, log_ctx, 1)) {
573  return -1;
574  }
575  }
576  if (rotate) {
578  }
579  } else {
580  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry for "
581  "%s.filetype. Expected \"regular\" (default), \"unix_stream\", "
582  "or \"unix_dgram\"",
583  conf->name);
584  }
585  log_ctx->filename = SCStrdup(log_path);
586  if (unlikely(log_ctx->filename == NULL)) {
588  "Failed to allocate memory for filename");
589  return -1;
590  }
591 
592 #ifdef BUILD_WITH_UNIXSOCKET
593  /* If a socket and running live, do non-blocking writes. */
594  if (log_ctx->is_sock && !IsRunModeOffline(RunmodeGetCurrent())) {
595  SCLogInfo("Setting logging socket of non-blocking in live mode.");
596  log_ctx->send_flags |= MSG_DONTWAIT;
597  }
598 #endif
599  SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype,
600  filename);
601 
602  return 0;
603 }
604 
605 /**
606  * \brief Reopen a regular log file with the side-affect of truncating it.
607  *
608  * This is useful to clear the log file and start a new one, or to
609  * re-open the file after its been moved by something external
610  * (eg. logrotate).
611  */
613 {
614  if (!log_ctx->is_regular) {
615  /* Not supported and not needed on non-regular files. */
616  return 0;
617  }
618 
619  if (log_ctx->filename == NULL) {
621  "Can't re-open LogFileCtx without a filename.");
622  return -1;
623  }
624 
625  if (log_ctx->fp != NULL) {
626  fclose(log_ctx->fp);
627  }
628 
629  /* Reopen the file. Append is forced in case the file was not
630  * moved as part of a rotation process. */
631  SCLogDebug("Reopening log file %s.", log_ctx->filename);
632  log_ctx->fp = SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode);
633  if (log_ctx->fp == NULL) {
634  return -1; // Already logged by Open..Fp routine.
635  }
636 
637  return 0;
638 }
639 
640 /** \brief LogFileNewCtx() Get a new LogFileCtx
641  * \retval LogFileCtx * pointer if successful, NULL if error
642  * */
644 {
645  LogFileCtx* lf_ctx;
646  lf_ctx = (LogFileCtx*)SCCalloc(1, sizeof(LogFileCtx));
647 
648  if (lf_ctx == NULL)
649  return NULL;
650 
651  lf_ctx->Write = SCLogFileWrite;
652  lf_ctx->Close = SCLogFileClose;
653 
654  return lf_ctx;
655 }
656 
657 /** \brief LogFileEnsureExists() Ensure a log file context for the thread exists
658  * \param parent_ctx
659  * \param thread_id
660  * \retval LogFileCtx * pointer if successful; NULL otherwise
661  */
662 LogFileCtx *LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
663 {
664  /* threaded output disabled */
665  if (!parent_ctx->threaded)
666  return parent_ctx;
667 
668  SCLogDebug("Adding reference %d to file ctx %p", thread_id, parent_ctx);
669  SCMutexLock(&parent_ctx->threads->mutex);
670  /* are there enough context slots already */
671  if (thread_id < parent_ctx->threads->slot_count) {
672  /* has it been opened yet? */
673  if (!parent_ctx->threads->lf_slots[thread_id]) {
674  SCLogDebug("Opening new file for %d reference to file ctx %p", thread_id, parent_ctx);
675  LogFileNewThreadedCtx(parent_ctx, parent_ctx->filename, parent_ctx->threads->append, thread_id);
676  }
677  SCMutexUnlock(&parent_ctx->threads->mutex);
678  SCLogDebug("Existing file for %d reference to file ctx %p", thread_id, parent_ctx);
679  return parent_ctx->threads->lf_slots[thread_id];
680  }
681 
682  /* ensure there's a slot for the caller */
683  int new_size = MAX(parent_ctx->threads->slot_count << 1, thread_id + 1);
684  SCLogDebug("Increasing slot count; current %d, trying %d",
685  parent_ctx->threads->slot_count, new_size);
686  LogFileCtx **new_array = SCRealloc(parent_ctx->threads->lf_slots, new_size * sizeof(LogFileCtx *));
687  if (new_array == NULL) {
688  /* Try one more time */
689  SCLogDebug("Unable to increase file context array size to %d; trying %d",
690  new_size, thread_id + 1);
691  new_size = thread_id + 1;
692  new_array = SCRealloc(parent_ctx->threads->lf_slots, new_size * sizeof(LogFileCtx *));
693  }
694 
695  if (new_array == NULL) {
696  SCMutexUnlock(&parent_ctx->threads->mutex);
697  SCLogError(SC_ERR_MEM_ALLOC, "Unable to increase file context array size to %d", new_size);
698  return NULL;
699  }
700 
701  parent_ctx->threads->lf_slots = new_array;
702  /* initialize newly added slots */
703  for (int i = parent_ctx->threads->slot_count; i < new_size; i++) {
704  parent_ctx->threads->lf_slots[i] = NULL;
705  }
706  parent_ctx->threads->slot_count = new_size;
707  LogFileNewThreadedCtx(parent_ctx, parent_ctx->filename, parent_ctx->threads->append, thread_id);
708 
709  SCMutexUnlock(&parent_ctx->threads->mutex);
710 
711  return parent_ctx->threads->lf_slots[thread_id];
712 }
713 
714 /** \brief LogFileThreadedName() Create file name for threaded EVE storage
715  *
716  */
717 static bool LogFileThreadedName(
718  const char *original_name, char *threaded_name, size_t len, uint32_t unique_id)
719 {
720  if (strcmp("/dev/null", original_name) == 0) {
721  strlcpy(threaded_name, original_name, len);
722  return true;
723  }
724 
725  const char *base = SCBasename(original_name);
726  if (!base) {
728  "Invalid filename for threaded mode \"%s\"; "
729  "no basename found.",
730  original_name);
731  }
732 
733  /* Check if basename has an extension */
734  char *dot = strrchr(base, '.');
735  if (dot) {
736  char *tname = SCStrdup(original_name);
737  if (!tname) {
738  return false;
739  }
740 
741  /* Fetch extension location from original, not base
742  * for update
743  */
744  dot = strrchr(original_name, '.');
745  int dotpos = dot - original_name;
746  tname[dotpos] = '\0';
747  char *ext = tname + dotpos + 1;
748  if (strlen(tname) && strlen(ext)) {
749  snprintf(threaded_name, len, "%s.%u.%s", tname, unique_id, ext);
750  } else {
752  "Invalid filename for threaded mode \"%s\"; "
753  "filenames must include an extension, e.g: \"name.ext\"",
754  original_name);
755  }
756  SCFree(tname);
757  } else {
758  snprintf(threaded_name, len, "%s.%u", original_name, unique_id);
759  }
760  return true;
761 }
762 
763 /** \brief LogFileNewThreadedCtx() Create file context for threaded output
764  * \param parent_ctx
765  * \param log_path
766  * \param append
767  * \param thread_id
768  */
769 static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, const char *append, int thread_id)
770 {
771  LogFileCtx *thread = SCCalloc(1, sizeof(LogFileCtx));
772  if (!thread) {
773  SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate thread file context slot %d", thread_id);
774  return false;
775  }
776 
777  *thread = *parent_ctx;
778  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
779  char fname[LOGFILE_NAME_MAX];
780  if (!LogFileThreadedName(log_path, fname, sizeof(fname), SC_ATOMIC_ADD(eve_file_id, 1))) {
781  SCLogError(SC_ERR_MEM_ALLOC, "Unable to create threaded filename for log");
782  goto error;
783  }
784  SCLogDebug("Thread open -- using name %s [replaces %s]", fname, log_path);
785  thread->fp = SCLogOpenFileFp(fname, append, thread->filemode);
786  if (thread->fp == NULL) {
787  goto error;
788  }
789  thread->filename = SCStrdup(fname);
790  if (!thread->filename) {
791  SCLogError(SC_ERR_MEM_ALLOC, "Unable to duplicate filename for context slot %d",
792  thread_id);
793  goto error;
794  }
795  thread->is_regular = true;
796  thread->Write = SCLogFileWriteNoLock;
797  thread->Close = SCLogFileCloseNoLock;
799  } else if (parent_ctx->type == LOGFILE_TYPE_PLUGIN) {
800  thread->plugin.plugin->ThreadInit(
801  thread->plugin.init_data, thread_id, &thread->plugin.thread_data);
802  }
803  thread->threaded = false;
804  thread->parent = parent_ctx;
805  thread->id = thread_id;
806 
807  parent_ctx->threads->lf_slots[thread_id] = thread;
808  return true;
809 
810 error:
811  if (parent_ctx->type == LOGFILE_TYPE_FILE) {
812  SC_ATOMIC_SUB(eve_file_id, 1);
813  if (thread->fp) {
814  thread->Close(thread);
815  }
816  }
817  if (thread) {
818  SCFree(thread);
819  }
820  parent_ctx->threads->lf_slots[thread_id] = NULL;
821  return false;
822 }
823 
824 /** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
825  * \param lf_ctx pointer to the OutputCtx
826  * \retval int 1 if successful, 0 if error
827  * */
829 {
830  if (lf_ctx == NULL) {
831  SCReturnInt(0);
832  }
833 
834  if (lf_ctx->threaded) {
835  BUG_ON(lf_ctx->threads == NULL);
836  SCMutexDestroy(&lf_ctx->threads->mutex);
837  for(int i = 0; i < lf_ctx->threads->slot_count; i++) {
838  if (!lf_ctx->threads->lf_slots[i]) {
839  continue;
840  }
841  LogFileCtx *this_ctx = lf_ctx->threads->lf_slots[i];
842 
843  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
845  this_ctx->Close(this_ctx);
846  } else {
847  lf_ctx->plugin.plugin->ThreadDeinit(
848  this_ctx->plugin.init_data, this_ctx->plugin.thread_data);
849  }
850  SCFree(lf_ctx->threads->lf_slots[i]->filename);
851  SCFree(lf_ctx->threads->lf_slots[i]);
852  }
853  SCFree(lf_ctx->threads->lf_slots);
854  if (lf_ctx->threads->append)
855  SCFree(lf_ctx->threads->append);
856  SCFree(lf_ctx->threads);
857  } else {
858  if (lf_ctx->type != LOGFILE_TYPE_PLUGIN) {
859  if (lf_ctx->fp != NULL) {
860  lf_ctx->Close(lf_ctx);
861  }
862  if (lf_ctx->parent) {
863  SCMutexLock(&lf_ctx->parent->threads->mutex);
864  lf_ctx->parent->threads->lf_slots[lf_ctx->id] = NULL;
865  SCMutexUnlock(&lf_ctx->parent->threads->mutex);
866  }
867  }
868  SCMutexDestroy(&lf_ctx->fp_mutex);
869  }
870 
871  if (lf_ctx->type == LOGFILE_TYPE_PLUGIN) {
872  lf_ctx->plugin.plugin->Deinit(lf_ctx->plugin.init_data);
873  }
874 
875  if (lf_ctx->prefix != NULL) {
876  SCFree(lf_ctx->prefix);
877  lf_ctx->prefix_len = 0;
878  }
879 
880  if(lf_ctx->filename != NULL)
881  SCFree(lf_ctx->filename);
882 
883  if (lf_ctx->sensor_name)
884  SCFree(lf_ctx->sensor_name);
885 
886  if (!lf_ctx->threaded) {
888  }
889 
890  memset(lf_ctx, 0, sizeof(*lf_ctx));
891  SCFree(lf_ctx);
892 
893  SCReturnInt(1);
894 }
895 
896 int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
897 {
898  if (file_ctx->type == LOGFILE_TYPE_FILE || file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
899  file_ctx->type == LOGFILE_TYPE_UNIX_STREAM) {
900  /* append \n for files only */
901  MemBufferWriteString(buffer, "\n");
902  file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
903  MEMBUFFER_OFFSET(buffer), file_ctx);
904  } else if (file_ctx->type == LOGFILE_TYPE_PLUGIN) {
905  file_ctx->plugin.plugin->Write((const char *)MEMBUFFER_BUFFER(buffer),
906  MEMBUFFER_OFFSET(buffer), file_ctx->plugin.init_data, file_ctx->plugin.thread_data);
907  }
908 #ifdef HAVE_LIBHIREDIS
909  else if (file_ctx->type == LOGFILE_TYPE_REDIS) {
910  SCMutexLock(&file_ctx->fp_mutex);
911  LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer),
912  MEMBUFFER_OFFSET(buffer));
913  SCMutexUnlock(&file_ctx->fp_mutex);
914  }
915 #endif
916 
917  return 0;
918 }
SCParseTimeSizeString
uint64_t SCParseTimeSizeString(const char *str)
Parse string containing time size (1m, 1h, etc).
Definition: util-time.c:579
LogFileCtx_::rotation_flag
int rotation_flag
Definition: util-logopenfile.h:142
util-byte.h
LogFileCtx_::prefix_len
size_t prefix_len
Definition: util-logopenfile.h:121
LogThreadedFileCtx_::append
char * append
Definition: util-logopenfile.h:54
SCEveFileType_::Write
int(* Write)(const char *buffer, int buffer_len, void *init_data, void *thread_data)
Definition: suricata-plugin.h:54
SCFerrorUnlocked
#define SCFerrorUnlocked
Definition: suricata-common.h:524
len
uint8_t len
Definition: app-layer-dnp3.h:2
LOGFILE_TYPE_REDIS
@ LOGFILE_TYPE_REDIS
Definition: util-logopenfile.h:40
LogFileCtx_::sensor_name
char * sensor_name
Definition: util-logopenfile.h:104
LogFileCtx_::reconn_timer
uint64_t reconn_timer
Definition: util-logopenfile.h:109
SC_ERR_SOCKET
@ SC_ERR_SOCKET
Definition: util-error.h:232
LogFileCtx_::fp_mutex
SCMutex fp_mutex
Definition: util-logopenfile.h:87
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_DECL_AND_INIT_WITH_VAL
#define SC_ATOMIC_DECL_AND_INIT_WITH_VAL(type, name, val)
wrapper for declaring an atomic variable and initializing it to a specific value
Definition: util-atomic.h:304
LogFileNewCtx
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
Definition: util-logopenfile.c:643
SCGetSecondsUntil
uint64_t SCGetSecondsUntil(const char *str, time_t epoch)
Get seconds until a time unit changes.
Definition: util-time.c:630
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
LogThreadedFileCtx_::lf_slots
struct LogFileCtx_ ** lf_slots
Definition: util-logopenfile.h:53
LogFileCtx_::json_flags
size_t json_flags
Definition: util-logopenfile.h:139
LogFilePluginCtx_::init_data
void * init_data
Definition: util-logopenfile.h:59
LogFilePluginCtx_::thread_data
void * thread_data
Definition: util-logopenfile.h:60
LogFileCtx_::id
int id
Definition: util-logopenfile.h:92
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
LogFileCtx_
Definition: util-logopenfile.h:64
LOGFILE_RECONN_MIN_TIME
#define LOGFILE_RECONN_MIN_TIME
Definition: util-logopenfile.h:158
DEFAULT_LOG_MODE_APPEND
#define DEFAULT_LOG_MODE_APPEND
Definition: output.h:27
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
util-log-redis.h
LogFileCtx_::Write
int(* Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:80
LogFileCtx_::filename
char * filename
Definition: util-logopenfile.h:98
LOGFILE_NAME_MAX
#define LOGFILE_NAME_MAX
Definition: util-logopenfile.c:46
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:262
LogFileCtx_::plugin
LogFilePluginCtx plugin
Definition: util-logopenfile.h:83
m
SCMutex m
Definition: flow-hash.h:6
LogFileCtx_::send_flags
uint8_t send_flags
Definition: util-logopenfile.h:132
MAX
#define MAX(x, y)
Definition: suricata-common.h:376
SCConfLogReopen
int SCConfLogReopen(LogFileCtx *log_ctx)
Reopen a regular log file with the side-affect of truncating it.
Definition: util-logopenfile.c:612
IsRunModeOffline
bool IsRunModeOffline(enum RunModes run_mode_to_check)
Definition: runmodes.c:545
SCClearErrUnlocked
#define SCClearErrUnlocked
Definition: suricata-common.h:523
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:521
SCConfLogOpenGeneric
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: util-logopenfile.c:431
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
LogFileCtx_::sock_type
int sock_type
Definition: util-logopenfile.h:108
OutputRegisterFileRotationFlag
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
Definition: output.c:833
SCTimeToStringPattern
int SCTimeToStringPattern(time_t epoch, const char *pattern, char *str, size_t size)
Convert epoch time to string pattern.
Definition: util-time.c:550
LogFileWrite
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
Definition: util-logopenfile.c:896
SC_WARN_CHMOD
@ SC_WARN_CHMOD
Definition: util-error.h:330
LogFileCtx_::rotate_interval
uint64_t rotate_interval
Definition: util-logopenfile.h:116
DEFAULT_LOG_FILETYPE
#define DEFAULT_LOG_FILETYPE
Definition: output.h:28
LogFileCtx_::is_sock
int is_sock
Definition: util-logopenfile.h:107
SCFwriteUnlocked
#define SCFwriteUnlocked
Definition: suricata-common.h:521
LogFileCtx_::dropped
uint64_t dropped
Definition: util-logopenfile.h:152
SCFflushUnlocked
#define SCFflushUnlocked
Definition: suricata-common.h:522
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:662
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
util-time.h
LogFileCtx_::type
enum LogFileType type
Definition: util-logopenfile.h:95
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
LOGFILE_TYPE_FILE
@ LOGFILE_TYPE_FILE
Definition: util-logopenfile.h:37
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
LogFileCtx_::threads
LogThreadedFileCtx * threads
Definition: util-logopenfile.h:72
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
SC_ERR_FOPEN
@ SC_ERR_FOPEN
Definition: util-error.h:74
LOGFILE_TYPE_PLUGIN
@ LOGFILE_TYPE_PLUGIN
Definition: util-logopenfile.h:41
conf.h
SCBasename
const char * SCBasename(const char *path)
Definition: util-path.c:236
LOGFILE_TYPE_UNIX_DGRAM
@ LOGFILE_TYPE_UNIX_DGRAM
Definition: util-logopenfile.h:38
LOGFILE_TYPE_UNIX_STREAM
@ LOGFILE_TYPE_UNIX_STREAM
Definition: util-logopenfile.h:39
MemBuffer_
Definition: util-buffer.h:27
LogFileCtx_::is_regular
uint8_t is_regular
Definition: util-logopenfile.h:136
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:215
SC_ERR_INVALID_YAML_CONF_ENTRY
@ SC_ERR_INVALID_YAML_CONF_ENTRY
Definition: util-error.h:169
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:770
LogFileCtx_::parent
struct LogFileCtx_ * parent
Definition: util-logopenfile.h:91
LogThreadedFileCtx_
Definition: util-logopenfile.h:50
suricata-common.h
LogFilePluginCtx_::plugin
SCEveFileType * plugin
Definition: util-logopenfile.h:58
LogFileCtx_::output_errors
uint64_t output_errors
Definition: util-logopenfile.h:154
LogThreadedFileCtx_::slot_count
int slot_count
Definition: util-logopenfile.h:51
ConfNode_::name
char * name
Definition: conf.h:33
MemBufferWriteString
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
SCLogOpenThreadedFile
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx, int slot_count)
Definition: util-logopenfile.c:327
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:45
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:255
LogFileFreeCtx
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
Definition: util-logopenfile.c:828
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(x,...)
Definition: util-debug.h:530
LogFileCtx_::rotate_time
time_t rotate_time
Definition: util-logopenfile.h:113
SCEveFileType_::Deinit
void(* Deinit)(void *init_data)
Definition: suricata-plugin.h:56
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:31
RunmodeGetCurrent
int RunmodeGetCurrent(void)
Definition: suricata.c:286
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCEveFileType_::ThreadDeinit
int(* ThreadDeinit)(void *init_data, void *thread_data)
Definition: suricata-plugin.h:60
ConfigGetLogDirectory
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
SCLogWarning
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:242
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
LOGFILE_ROTATE_INTERVAL
#define LOGFILE_ROTATE_INTERVAL
Definition: util-logopenfile.h:163
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:546
LogThreadedFileCtx_::mutex
SCMutex mutex
Definition: util-logopenfile.h:52
LogFileCtx_::prefix
char * prefix
Definition: util-logopenfile.h:120
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
suricata.h
SC_ERR_LOG_OUTPUT
@ SC_ERR_LOG_OUTPUT
Definition: util-error.h:368
LogFileCtx_::flags
uint8_t flags
Definition: util-logopenfile.h:129
LogFileCtx_::Close
void(* Close)(struct LogFileCtx_ *fp)
Definition: util-logopenfile.h:81
MEMBUFFER_BUFFER
#define MEMBUFFER_BUFFER(mem_buffer)
Get the MemBuffers underlying buffer.
Definition: util-buffer.h:50
SCEveFileType_::ThreadInit
int(* ThreadInit)(void *init_data, int thread_id, void **thread_data)
Definition: suricata-plugin.h:58
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:230
OutputUnregisterFileRotationFlag
void OutputUnregisterFileRotationFlag(int *flag)
Unregister a file rotation flag.
Definition: output.c:855
MEMBUFFER_OFFSET
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
Definition: util-buffer.h:55
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:302
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
LogFileCtx_::fp
FILE * fp
Definition: util-logopenfile.h:66
output.h
LogFileCtx_::threaded
bool threaded
Definition: util-logopenfile.h:90
SCCreateDirectoryTree
int SCCreateDirectoryTree(const char *path, const bool final)
Recursively create a directory.
Definition: util-path.c:124
LogFileCtx_::filemode
uint32_t filemode
Definition: util-logopenfile.h:101
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:798