suricata
log-filestore.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "debug.h"
27 #include "detect.h"
28 #include "pkt-var.h"
29 #include "conf.h"
30 
31 #include "threadvars.h"
32 #include "tm-modules.h"
33 
34 #include "threads.h"
35 
36 #include "app-layer-parser.h"
37 
38 #include "detect-filemagic.h"
39 
40 #include "stream.h"
41 
42 #include "util-print.h"
43 #include "util-unittest.h"
44 #include "util-privs.h"
45 #include "util-debug.h"
46 #include "util-atomic.h"
47 #include "util-file.h"
48 #include "util-time.h"
49 #include "util-misc.h"
50 
51 #include "output.h"
52 
53 #include "log-filestore.h"
54 #include "util-logopenfile.h"
55 
56 #include "app-layer-htp.h"
57 #include "app-layer-smtp.h"
58 #include "util-decode-mime.h"
59 #include "util-memcmp.h"
60 #include "stream-tcp-reassemble.h"
61 
62 #define MODULE_NAME "LogFilestoreLog"
63 
64 static char g_logfile_base_dir[PATH_MAX] = "/tmp";
65 static char g_working_file_suffix[PATH_MAX] = ".tmp";
66 
67 SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt); /**< Atomic counter of simultaneously open files */
68 
69 typedef struct LogFilestoreLogThread_ {
71  /** LogFilestoreCtx has the pointer to the file and a mutex to allow multithreading */
72  uint32_t file_cnt;
73  uint16_t counter_max_hits;
75 
76 static uint64_t LogFilestoreOpenFilesCounter(void)
77 {
78  uint64_t fcopy = SC_ATOMIC_GET(filestore_open_file_cnt);
79  return fcopy;
80 }
81 
82 static void LogFilestoreMetaGetUri(FILE *fp, const Packet *p, const File *ff)
83 {
84  HtpState *htp_state = (HtpState *)p->flow->alstate;
85  if (htp_state != NULL) {
86  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
87  if (tx != NULL) {
88  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
89  if (tx_ud->request_uri_normalized != NULL) {
90  PrintRawUriFp(fp, bstr_ptr(tx_ud->request_uri_normalized),
91  bstr_len(tx_ud->request_uri_normalized));
92  }
93  return;
94  }
95  }
96 
97  fprintf(fp, "<unknown>");
98 }
99 
100 static void LogFilestoreMetaGetHost(FILE *fp, const Packet *p, const File *ff)
101 {
102  HtpState *htp_state = (HtpState *)p->flow->alstate;
103  if (htp_state != NULL) {
104  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
105  if (tx != NULL && tx->request_hostname != NULL) {
106  PrintRawUriFp(fp, (uint8_t *)bstr_ptr(tx->request_hostname),
107  bstr_len(tx->request_hostname));
108  return;
109  }
110  }
111 
112  fprintf(fp, "<unknown>");
113 }
114 
115 static void LogFilestoreMetaGetReferer(FILE *fp, const Packet *p, const File *ff)
116 {
117  HtpState *htp_state = (HtpState *)p->flow->alstate;
118  if (htp_state != NULL) {
119  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
120  if (tx != NULL) {
121  htp_header_t *h = NULL;
122  h = (htp_header_t *)htp_table_get_c(tx->request_headers,
123  "Referer");
124  if (h != NULL) {
125  PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
126  bstr_len(h->value));
127  return;
128  }
129  }
130  }
131 
132  fprintf(fp, "<unknown>");
133 }
134 
135 static void LogFilestoreMetaGetUserAgent(FILE *fp, const Packet *p, const File *ff)
136 {
137  HtpState *htp_state = (HtpState *)p->flow->alstate;
138  if (htp_state != NULL) {
139  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
140  if (tx != NULL) {
141  htp_header_t *h = NULL;
142  h = (htp_header_t *)htp_table_get_c(tx->request_headers,
143  "User-Agent");
144  if (h != NULL) {
145  PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
146  bstr_len(h->value));
147  return;
148  }
149  }
150  }
151 
152  fprintf(fp, "<unknown>");
153 }
154 
155 static void LogFilestoreMetaGetSmtp(FILE *fp, const Packet *p, const File *ff)
156 {
157  SMTPState *state = (SMTPState *) p->flow->alstate;
158  if (state != NULL) {
159  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, state, ff->txid);
160  if (tx == NULL || tx->msg_tail == NULL)
161  return;
162 
163  /* Message Id */
164  if (tx->msg_tail->msg_id != NULL) {
165  fprintf(fp, "MESSAGE-ID: ");
166  PrintRawUriFp(fp, (uint8_t *) tx->msg_tail->msg_id, tx->msg_tail->msg_id_len);
167  fprintf(fp, "\n");
168  }
169 
170  /* Sender */
171  MimeDecField *field = MimeDecFindField(tx->msg_tail, "from");
172  if (field != NULL) {
173  fprintf(fp, "SENDER: ");
174  PrintRawUriFp(fp, (uint8_t *) field->value, field->value_len);
175  fprintf(fp, "\n");
176  }
177  }
178 }
179 
180 /** \brief switch to write meta file
181  */
182 static int g_file_write_meta = 1;
183 
184 static void FileWriteMetaDisable(void)
185 {
186  g_file_write_meta = 0;
187 }
188 
189 static int FileWriteMeta(void)
190 {
191  return g_file_write_meta;
192 }
193 
194 static uint32_t g_file_store_max_open_files = 0;
195 
196 static void FileSetMaxOpenFiles(uint32_t count)
197 {
198  g_file_store_max_open_files = count;
199 }
200 
201 static uint32_t FileGetMaxOpenFiles(void)
202 {
203  return g_file_store_max_open_files;
204 }
205 
206 static int g_file_store_include_pid = 0;
207 
208 static void FileIncludePidEnable(void)
209 {
210  g_file_store_include_pid = 1;
211 }
212 
213 static int FileIncludePid(void)
214 {
215  return g_file_store_include_pid;
216 }
217 
218 static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *base_filename, int ipver) {
219  if (!FileWriteMeta())
220  return;
221 
222  char metafilename[PATH_MAX] = "";
223  if (snprintf(metafilename, sizeof(metafilename), "%s.meta%s", base_filename,
224  g_working_file_suffix) == sizeof(metafilename))
225  return;
226 
227  FILE *fp = fopen(metafilename, "w+");
228  if (fp != NULL) {
229  char timebuf[64];
230 
231  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
232 
233  fprintf(fp, "TIME: %s\n", timebuf);
234  if (p->pcap_cnt > 0) {
235  fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt);
236  }
237 
238  char srcip[46], dstip[46];
239  Port sp, dp;
240  switch (ipver) {
241  case AF_INET:
242  PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
243  PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
244  break;
245  case AF_INET6:
246  PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip));
247  PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip));
248  break;
249  default:
250  strlcpy(srcip, "<unknown>", sizeof(srcip));
251  strlcpy(dstip, "<unknown>", sizeof(dstip));
252  break;
253  }
254  sp = p->sp;
255  dp = p->dp;
256 
257  fprintf(fp, "SRC IP: %s\n", srcip);
258  fprintf(fp, "DST IP: %s\n", dstip);
259  fprintf(fp, "PROTO: %" PRIu32 "\n", p->proto);
260  if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) {
261  fprintf(fp, "SRC PORT: %" PRIu16 "\n", sp);
262  fprintf(fp, "DST PORT: %" PRIu16 "\n", dp);
263  }
264 
265  fprintf(fp, "APP PROTO: %s\n",
267 
268  /* Only applicable to HTTP traffic */
269  if (p->flow->alproto == ALPROTO_HTTP) {
270  fprintf(fp, "HTTP URI: ");
271  LogFilestoreMetaGetUri(fp, p, ff);
272  fprintf(fp, "\n");
273  fprintf(fp, "HTTP HOST: ");
274  LogFilestoreMetaGetHost(fp, p, ff);
275  fprintf(fp, "\n");
276  fprintf(fp, "HTTP REFERER: ");
277  LogFilestoreMetaGetReferer(fp, p, ff);
278  fprintf(fp, "\n");
279  fprintf(fp, "HTTP USER AGENT: ");
280  LogFilestoreMetaGetUserAgent(fp, p, ff);
281  fprintf(fp, "\n");
282  } else if (p->flow->alproto == ALPROTO_SMTP) {
283  /* Only applicable to SMTP */
284  LogFilestoreMetaGetSmtp(fp, p, ff);
285  }
286 
287  fprintf(fp, "FILENAME: ");
288  PrintRawUriFp(fp, ff->name, ff->name_len);
289  fprintf(fp, "\n");
290 
291  fclose(fp);
292  }
293 }
294 
295 static void LogFilestoreLogCloseMetaFile(const File *ff)
296 {
297  char pid_expression[PATH_MAX] = "";
298  if (FileIncludePid())
299  snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
300  char final_filename[PATH_MAX] = "";
301  if (snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
302  g_logfile_base_dir, pid_expression, ff->file_store_id) == sizeof(final_filename))
303  return;
304  char final_metafilename[PATH_MAX] = "";
305  if (snprintf(final_metafilename, sizeof(final_metafilename),
306  "%s.meta", final_filename) == sizeof(final_metafilename))
307  return;
308  char working_metafilename[PATH_MAX] = "";
309  if (snprintf(working_metafilename, sizeof(working_metafilename),
310  "%s%s", final_metafilename, g_working_file_suffix) == sizeof(working_metafilename))
311  return;
312 
313  FILE *fp = fopen(working_metafilename, "a");
314  if (fp != NULL) {
315 #ifdef HAVE_MAGIC
316  fprintf(fp, "MAGIC: %s\n",
317  ff->magic ? ff->magic : "<unknown>");
318 #endif
319  switch (ff->state) {
320  case FILE_STATE_CLOSED:
321  fprintf(fp, "STATE: CLOSED\n");
322 #ifdef HAVE_NSS
323  if (ff->flags & FILE_MD5) {
324  fprintf(fp, "MD5: ");
325  size_t x;
326  for (x = 0; x < sizeof(ff->md5); x++) {
327  fprintf(fp, "%02x", ff->md5[x]);
328  }
329  fprintf(fp, "\n");
330  }
331  if (ff->flags & FILE_SHA1) {
332  fprintf(fp, "SHA1: ");
333  size_t x;
334  for (x = 0; x < sizeof(ff->sha1); x++) {
335  fprintf(fp, "%02x", ff->sha1[x]);
336  }
337  fprintf(fp, "\n");
338  }
339  if (ff->flags & FILE_SHA256) {
340  fprintf(fp, "SHA256: ");
341  size_t x;
342  for (x = 0; x < sizeof(ff->sha256); x++) {
343  fprintf(fp, "%02x", ff->sha256[x]);
344  }
345  fprintf(fp, "\n");
346  }
347 #endif
348  break;
350  fprintf(fp, "STATE: TRUNCATED\n");
351  break;
352  case FILE_STATE_ERROR:
353  fprintf(fp, "STATE: ERROR\n");
354  break;
355  default:
356  fprintf(fp, "STATE: UNKNOWN\n");
357  break;
358  }
359  fprintf(fp, "SIZE: %"PRIu64"\n", FileTrackedSize(ff));
360 
361  fclose(fp);
362  } else {
363  SCLogInfo("opening %s failed: %s", working_metafilename,
364  strerror(errno));
365  }
366 }
367 
368 static void LogFilestoreFinalizeFiles(const File *ff) {
369  char pid_expression[PATH_MAX] = "";
370  if (FileIncludePid())
371  snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
372  char final_filename[PATH_MAX] = "";
373  if (snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
374  g_logfile_base_dir, pid_expression, ff->file_store_id) == sizeof(final_filename))
375  return;
376  char working_filename[PATH_MAX] = "";
377  if (snprintf(working_filename, sizeof(working_filename), "%s%s",
378  final_filename, g_working_file_suffix) == sizeof(working_filename))
379  return;
380 
381  if (rename(working_filename, final_filename) != 0) {
382  SCLogWarning(SC_WARN_RENAMING_FILE, "renaming file %s to %s failed",
383  working_filename, final_filename);
384  return;
385  }
386  if (FileWriteMeta()) {
387  LogFilestoreLogCloseMetaFile(ff);
388  char final_metafilename[PATH_MAX] = "";
389  if (snprintf(final_metafilename, sizeof(final_metafilename),
390  "%s.meta", final_filename) == sizeof(final_metafilename))
391  return;
392  char working_metafilename[PATH_MAX] = "";
393  if (snprintf(working_metafilename, sizeof(working_metafilename),
394  "%s%s", final_metafilename, g_working_file_suffix) == sizeof(working_metafilename))
395  return;
396 
397  if (rename(working_metafilename, final_metafilename) != 0) {
399  "renaming metafile %s to %s failed", working_metafilename,
400  final_metafilename);
401  }
402  }
403 }
404 
405 static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p,
406  File *ff, const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
407 {
408  SCEnter();
409  LogFilestoreLogThread *aft = (LogFilestoreLogThread *)thread_data;
410  char filename[PATH_MAX] = "";
411  int file_fd = -1;
412  int ipver = -1;
413 
414  /* no flow, no htp state */
415  if (p->flow == NULL) {
417  }
418 
419  if (PKT_IS_IPV4(p)) {
420  ipver = AF_INET;
421  } else if (PKT_IS_IPV6(p)) {
422  ipver = AF_INET6;
423  } else {
424  return 0;
425  }
426 
427  SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
428 
429  char pid_expression[PATH_MAX] = "";
430  if (FileIncludePid())
431  snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
432  char base_filename[PATH_MAX] = "";
433  if (snprintf(base_filename, sizeof(base_filename), "%s/file%s.%u",
434  g_logfile_base_dir, pid_expression, ff->file_store_id) == sizeof(base_filename))
435  return -1;
436  if (snprintf(filename, sizeof(filename), "%s%s", base_filename,
437  g_working_file_suffix) == sizeof(filename))
438  return -1;
439 
441  aft->file_cnt++;
442 
443  /* create a .meta file that contains time, src/dst/sp/dp/proto */
444  LogFilestoreLogCreateMetaFile(p, ff, base_filename, ipver);
445 
446  if (SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
447  SC_ATOMIC_ADD(filestore_open_file_cnt, 1);
448  ff->fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644);
449  if (ff->fd == -1) {
450  SCLogDebug("failed to create file");
451  return -1;
452  }
453  file_fd = ff->fd;
454  } else {
455  file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644);
456  if (file_fd == -1) {
457  SCLogDebug("failed to create file");
458  return -1;
459  }
460  if (FileGetMaxOpenFiles() > 0) {
461  StatsIncr(tv, aft->counter_max_hits);
462  }
463  ff->fd = -1;
464  }
465  /* we can get called with a NULL ffd when we need to close */
466  } else if (data != NULL) {
467  if (ff->fd == -1) {
468  file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY);
469  if (file_fd == -1) {
470  SCLogDebug("failed to open file %s: %s", filename, strerror(errno));
471  return -1;
472  }
473  } else {
474  file_fd = ff->fd;
475  }
476  }
477 
478  if (file_fd != -1) {
479  ssize_t r = write(file_fd, (const void *)data, (size_t)data_len);
480  if (r == -1) {
481  SCLogDebug("write failed: %s", strerror(errno));
482  if (ff->fd != -1) {
483  SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
484  }
485  ff->fd = -1;
486  }
487  if (ff->fd == -1) {
488  close(file_fd);
489  }
490  }
491 
493  if (ff->fd != -1) {
494  close(ff->fd);
495  ff->fd = -1;
496  SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
497  }
498  LogFilestoreFinalizeFiles(ff);
499  }
500 
501  return 0;
502 }
503 
504 static TmEcode LogFilestoreLogThreadInit(ThreadVars *t, const void *initdata, void **data)
505 {
507  if (unlikely(aft == NULL))
508  return TM_ECODE_FAILED;
509  memset(aft, 0, sizeof(LogFilestoreLogThread));
510 
511  if (initdata == NULL)
512  {
513  SCLogDebug("Error getting context for LogFileStore. \"initdata\" argument NULL");
514  SCFree(aft);
515  return TM_ECODE_FAILED;
516  }
517 
518  /* Use the Ouptut Context (file pointer and mutex) */
519  aft->file_ctx = ((OutputCtx *)initdata)->data;
520 
521  struct stat stat_buf;
522  if (stat(g_logfile_base_dir, &stat_buf) != 0) {
523  int ret;
524  // coverity[toctou : FALSE]
525  ret = SCMkDir(g_logfile_base_dir, S_IRWXU|S_IXGRP|S_IRGRP);
526  if (ret != 0) {
527  int err = errno;
528  if (err != EEXIST) {
530  "Cannot create file drop directory %s: %s",
531  g_logfile_base_dir, strerror(err));
532  exit(EXIT_FAILURE);
533  }
534  } else {
535  SCLogInfo("Created file drop directory %s",
536  g_logfile_base_dir);
537  }
538 
539  }
540 
541  aft->counter_max_hits = StatsRegisterCounter("file_store.open_files_max_hit", t);
542 
543  *data = (void *)aft;
544  return TM_ECODE_OK;
545 }
546 
547 static TmEcode LogFilestoreLogThreadDeinit(ThreadVars *t, void *data)
548 {
550  if (aft == NULL) {
551  return TM_ECODE_OK;
552  }
553 
554  /* clear memory */
555  memset(aft, 0, sizeof(LogFilestoreLogThread));
556 
557  SCFree(aft);
558  return TM_ECODE_OK;
559 }
560 
561 static void LogFilestoreLogExitPrintStats(ThreadVars *tv, void *data)
562 {
564  if (aft == NULL) {
565  return;
566  }
567 
568  SCLogInfo("(%s) Files extracted %" PRIu32 "", tv->name, aft->file_cnt);
569 }
570 
571 /**
572  * \internal
573  *
574  * \brief deinit the log ctx and write out the waldo
575  *
576  * \param output_ctx output context to deinit
577  */
578 static void LogFilestoreLogDeInitCtx(OutputCtx *output_ctx)
579 {
580  LogFileCtx *logfile_ctx = (LogFileCtx *)output_ctx->data;
581  LogFileFreeCtx(logfile_ctx);
582  SCFree(output_ctx);
583 
584 }
585 
586 /** \brief Create a new http log LogFilestoreCtx.
587  * \param conf Pointer to ConfNode containing this loggers configuration.
588  * \return NULL if failure, LogFilestoreCtx* to the file_ctx if succesful
589  * */
590 static OutputInitResult LogFilestoreLogInitCtx(ConfNode *conf)
591 {
592  OutputInitResult result = { NULL, false };
593 
594  intmax_t version = 0;
595  if (ConfGetChildValueInt(conf, "version", &version)) {
596  if (version > 1) {
597  result.ok = true;
598  return result;
599  }
600  }
601 
604  "A file data logger is already enabled. Filestore (v1) "
605  "will not be enabled.");
606  return result;
607  }
608 
609  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
610  if (unlikely(output_ctx == NULL))
611  return result;
612 
613  output_ctx->data = NULL;
614  output_ctx->DeInit = LogFilestoreLogDeInitCtx;
615 
616  const char *s_default_log_dir = NULL;
617  s_default_log_dir = ConfigGetLogDirectory();
618 
619  const char *s_base_dir = NULL;
620  s_base_dir = ConfNodeLookupChildValue(conf, "log-dir");
621  if (s_base_dir == NULL || strlen(s_base_dir) == 0) {
622  strlcpy(g_logfile_base_dir,
623  s_default_log_dir, sizeof(g_logfile_base_dir));
624  } else {
625  if (PathIsAbsolute(s_base_dir)) {
626  strlcpy(g_logfile_base_dir,
627  s_base_dir, sizeof(g_logfile_base_dir));
628  } else {
629  snprintf(g_logfile_base_dir, sizeof(g_logfile_base_dir),
630  "%s/%s", s_default_log_dir, s_base_dir);
631  }
632  }
633 
634  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
635  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
637  SCLogInfo("forcing filestore of all files");
638  }
639 
640  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
641  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
643  SCLogInfo("forcing magic lookup for stored files");
644  }
645 
646  const char *write_meta = ConfNodeLookupChildValue(conf, "write-meta");
647  if (write_meta != NULL && !ConfValIsTrue(write_meta)) {
648  FileWriteMetaDisable();
649  SCLogInfo("File-store output will not write meta files");
650  }
651 
652  FileForceHashParseCfg(conf);
653  SCLogInfo("storing files in %s", g_logfile_base_dir);
654 
655  const char *stream_depth_str = ConfNodeLookupChildValue(conf, "stream-depth");
656  if (stream_depth_str != NULL && strcmp(stream_depth_str, "no")) {
657  uint32_t stream_depth = 0;
658  if (ParseSizeStringU32(stream_depth_str,
659  &stream_depth) < 0) {
660  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing "
661  "file-store.stream-depth "
662  "from conf file - %s. Killing engine",
663  stream_depth_str);
664  exit(EXIT_FAILURE);
665  } else {
666  FileReassemblyDepthEnable(stream_depth);
667  }
668  }
669 
670  const char *file_count_str = ConfNodeLookupChildValue(conf, "max-open-files");
671  if (file_count_str != NULL) {
672  uint32_t file_count = 0;
673  if (ParseSizeStringU32(file_count_str,
674  &file_count) < 0) {
675  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing "
676  "file-store.max-open-files "
677  "from conf file - %s. Killing engine",
678  stream_depth_str);
679  exit(EXIT_FAILURE);
680  } else {
681  if (file_count != 0) {
682  FileSetMaxOpenFiles(file_count);
683  SCLogInfo("file-store will keep a max of %d simultaneously"
684  " open files", file_count);
685  }
686  }
687  }
688 
689  const char *include_pid = ConfNodeLookupChildValue(conf, "include-pid");
690  if (include_pid != NULL && ConfValIsTrue(include_pid)) {
691  FileIncludePidEnable();
692  SCLogInfo("enabling pid as a part of all file names");
693  }
694 
695  StatsRegisterGlobalCounter("file_store.open_files",
696  LogFilestoreOpenFilesCounter);
697 
698  result.ctx = output_ctx;
699  result.ok = true;
700  SCReturnCT(result, "OutputInitResult");
701 }
702 
704 {
706  LogFilestoreLogInitCtx, LogFilestoreLogger, LogFilestoreLogThreadInit,
707  LogFilestoreLogThreadDeinit, LogFilestoreLogExitPrintStats);
709  LogFilestoreLogInitCtx, LogFilestoreLogger, LogFilestoreLogThreadInit,
710  LogFilestoreLogThreadDeinit, LogFilestoreLogExitPrintStats);
711 
712  SC_ATOMIC_INIT(filestore_open_file_cnt);
713  SC_ATOMIC_SET(filestore_open_file_cnt, 0);
714  SCLogDebug("registered");
715 }
#define OUTPUT_FILEDATA_FLAG_CLOSE
#define GET_IPV4_SRC_ADDR_PTR(p)
Definition: decode.h:212
uint16_t flags
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
MimeDecEntity * msg_tail
#define SCMkDir(a, b)
Definition: util-path.h:29
#define unlikely(expr)
Definition: util-optimize.h:35
#define GET_IPV4_DST_ADDR_PTR(p)
Definition: decode.h:213
Port sp
Definition: decode.h:413
#define FILE_SHA256
Definition: util-file.h:43
bstr * request_uri_normalized
Port dp
Definition: decode.h:421
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
#define PKT_IS_IPV6(p)
Definition: decode.h:250
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt)
uint64_t pcap_cnt
Definition: decode.h:561
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:124
int fd
Definition: util-file.h:73
#define PKT_IS_IPV4(p)
Definition: decode.h:249
void FileReassemblyDepthEnable(uint32_t size)
Definition: util-file.c:116
#define SCReturnCT(x, type)
Definition: util-debug.h:351
uint16_t StatsRegisterCounter(const char *name, struct ThreadVars_ *tv)
Registers a normal, unqualified counter.
Definition: counters.c:939
void * alstate
Definition: flow.h:438
#define O_NOFOLLOW
Definition: win32-misc.h:35
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define SCCalloc(nm, a)
Definition: util-mem.h:197
void LogFilestoreRegister(void)
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
uint16_t flags
Definition: util-file.h:65
uint32_t value_len
uint8_t proto
Definition: decode.h:428
#define GET_IPV6_DST_ADDR(p)
Definition: decode.h:218
struct LogFilestoreLogThread_ LogFilestoreLogThread
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileForceMagicEnable(void)
Definition: util-file.c:91
uint64_t txid
Definition: util-file.h:69
#define SCEnter(...)
Definition: util-debug.h:337
uint16_t name_len
Definition: util-file.h:66
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
void PrintRawUriFp(FILE *fp, uint8_t *buf, uint32_t buflen)
Definition: util-print.c:93
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:39
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1000
#define SCReturnInt(x)
Definition: util-debug.h:341
uint16_t Port
Definition: decode.h:233
#define FILE_MD5
Definition: util-file.h:39
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
#define MODULE_NAME
Definition: log-filestore.c:62
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:166
#define GET_IPV6_SRC_ADDR(p)
Definition: decode.h:217
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
int RunModeOutputFiledataEnabled(void)
Definition: runmodes.c:490
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
uint8_t version
Definition: decode-gre.h:405
#define SCFree(a)
Definition: util-mem.h:228
#define PKT_IS_TCP(p)
Definition: decode.h:251
void OutputRegisterFiledataModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FiledataLogger FiledataLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file data output module.
Definition: output.c:498
void * data
Definition: tm-modules.h:81
FileState state
Definition: util-file.h:67
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
uint32_t file_store_id
Definition: util-file.h:72
int ConfGetChildValueInt(const ConfNode *base, const char *name, intmax_t *val)
Definition: conf.c:469
char name[16]
Definition: threadvars.h:59
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
#define PKT_IS_UDP(p)
Definition: decode.h:252
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:449
int ParseSizeStringU32(const char *size, uint32_t *res)
Definition: util-misc.c:186
AppProto alproto
application level protocol
Definition: flow.h:409
#define OUTPUT_FILEDATA_FLAG_OPEN
This represents a header field name and associated value.
uint8_t * name
Definition: util-file.h:75
void CreateTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:237