suricata
log-filestore.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "debug.h"
27 #include "detect.h"
28 #include "pkt-var.h"
29 #include "conf.h"
30 
31 #include "threadvars.h"
32 #include "tm-modules.h"
33 
34 #include "threads.h"
35 
36 #include "app-layer-parser.h"
37 
38 #include "detect-filemagic.h"
39 
40 #include "stream.h"
41 
42 #include "util-print.h"
43 #include "util-unittest.h"
44 #include "util-privs.h"
45 #include "util-debug.h"
46 #include "util-atomic.h"
47 #include "util-file.h"
48 #include "util-time.h"
49 #include "util-misc.h"
50 
51 #include "output.h"
52 
53 #include "log-file.h"
54 #include "log-filestore.h"
55 #include "util-logopenfile.h"
56 
57 #include "app-layer-htp.h"
58 #include "app-layer-smtp.h"
59 #include "util-decode-mime.h"
60 #include "util-memcmp.h"
61 #include "stream-tcp-reassemble.h"
62 
63 #define MODULE_NAME "LogFilestoreLog"
64 
65 static char g_logfile_base_dir[PATH_MAX] = "/tmp";
66 static char g_working_file_suffix[PATH_MAX] = ".tmp";
67 
68 SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt); /**< Atomic counter of simultaneously open files */
69 
70 typedef struct LogFilestoreLogThread_ {
72  /** LogFilestoreCtx has the pointer to the file and a mutex to allow multithreading */
73  uint32_t file_cnt;
74  uint16_t counter_max_hits;
76 
77 static uint64_t LogFilestoreOpenFilesCounter(void)
78 {
79  uint64_t fcopy = SC_ATOMIC_GET(filestore_open_file_cnt);
80  return fcopy;
81 }
82 
83 static void LogFilestoreMetaGetUri(FILE *fp, const Packet *p, const File *ff)
84 {
85  HtpState *htp_state = (HtpState *)p->flow->alstate;
86  if (htp_state != NULL) {
87  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
88  if (tx != NULL) {
89  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
90  if (tx_ud->request_uri_normalized != NULL) {
91  PrintRawUriFp(fp, bstr_ptr(tx_ud->request_uri_normalized),
92  bstr_len(tx_ud->request_uri_normalized));
93  }
94  return;
95  }
96  }
97 
98  fprintf(fp, "<unknown>");
99 }
100 
101 static void LogFilestoreMetaGetHost(FILE *fp, const Packet *p, const File *ff)
102 {
103  HtpState *htp_state = (HtpState *)p->flow->alstate;
104  if (htp_state != NULL) {
105  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
106  if (tx != NULL && tx->request_hostname != NULL) {
107  PrintRawUriFp(fp, (uint8_t *)bstr_ptr(tx->request_hostname),
108  bstr_len(tx->request_hostname));
109  return;
110  }
111  }
112 
113  fprintf(fp, "<unknown>");
114 }
115 
116 static void LogFilestoreMetaGetReferer(FILE *fp, const Packet *p, const File *ff)
117 {
118  HtpState *htp_state = (HtpState *)p->flow->alstate;
119  if (htp_state != NULL) {
120  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
121  if (tx != NULL) {
122  htp_header_t *h = NULL;
123  h = (htp_header_t *)htp_table_get_c(tx->request_headers,
124  "Referer");
125  if (h != NULL) {
126  PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
127  bstr_len(h->value));
128  return;
129  }
130  }
131  }
132 
133  fprintf(fp, "<unknown>");
134 }
135 
136 static void LogFilestoreMetaGetUserAgent(FILE *fp, const Packet *p, const File *ff)
137 {
138  HtpState *htp_state = (HtpState *)p->flow->alstate;
139  if (htp_state != NULL) {
140  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
141  if (tx != NULL) {
142  htp_header_t *h = NULL;
143  h = (htp_header_t *)htp_table_get_c(tx->request_headers,
144  "User-Agent");
145  if (h != NULL) {
146  PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
147  bstr_len(h->value));
148  return;
149  }
150  }
151  }
152 
153  fprintf(fp, "<unknown>");
154 }
155 
156 static void LogFilestoreMetaGetSmtp(FILE *fp, const Packet *p, const File *ff)
157 {
158  SMTPState *state = (SMTPState *) p->flow->alstate;
159  if (state != NULL) {
160  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, state, ff->txid);
161  if (tx == NULL || tx->msg_tail == NULL)
162  return;
163 
164  /* Message Id */
165  if (tx->msg_tail->msg_id != NULL) {
166  fprintf(fp, "MESSAGE-ID: ");
167  PrintRawUriFp(fp, (uint8_t *) tx->msg_tail->msg_id, tx->msg_tail->msg_id_len);
168  fprintf(fp, "\n");
169  }
170 
171  /* Sender */
172  MimeDecField *field = MimeDecFindField(tx->msg_tail, "from");
173  if (field != NULL) {
174  fprintf(fp, "SENDER: ");
175  PrintRawUriFp(fp, (uint8_t *) field->value, field->value_len);
176  fprintf(fp, "\n");
177  }
178  }
179 }
180 
181 /** \brief switch to write meta file
182  */
183 static int g_file_write_meta = 1;
184 
185 static void FileWriteMetaDisable(void)
186 {
187  g_file_write_meta = 0;
188 }
189 
190 static int FileWriteMeta(void)
191 {
192  return g_file_write_meta;
193 }
194 
195 static uint32_t g_file_store_max_open_files = 0;
196 
197 static void FileSetMaxOpenFiles(uint32_t count)
198 {
199  g_file_store_max_open_files = count;
200 }
201 
202 static uint32_t FileGetMaxOpenFiles(void)
203 {
204  return g_file_store_max_open_files;
205 }
206 
207 static int g_file_store_include_pid = 0;
208 
209 static void FileIncludePidEnable(void)
210 {
211  g_file_store_include_pid = 1;
212 }
213 
214 static int FileIncludePid(void)
215 {
216  return g_file_store_include_pid;
217 }
218 
219 static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *base_filename, int ipver) {
220  if (!FileWriteMeta())
221  return;
222 
223  char metafilename[PATH_MAX] = "";
224  if (snprintf(metafilename, sizeof(metafilename), "%s.meta%s", base_filename,
225  g_working_file_suffix) == sizeof(metafilename))
226  return;
227 
228  FILE *fp = fopen(metafilename, "w+");
229  if (fp != NULL) {
230  char timebuf[64];
231 
232  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
233 
234  fprintf(fp, "TIME: %s\n", timebuf);
235  if (p->pcap_cnt > 0) {
236  fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt);
237  }
238 
239  char srcip[46], dstip[46];
240  Port sp, dp;
241  switch (ipver) {
242  case AF_INET:
243  PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
244  PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
245  break;
246  case AF_INET6:
247  PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip));
248  PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip));
249  break;
250  default:
251  strlcpy(srcip, "<unknown>", sizeof(srcip));
252  strlcpy(dstip, "<unknown>", sizeof(dstip));
253  break;
254  }
255  sp = p->sp;
256  dp = p->dp;
257 
258  fprintf(fp, "SRC IP: %s\n", srcip);
259  fprintf(fp, "DST IP: %s\n", dstip);
260  fprintf(fp, "PROTO: %" PRIu32 "\n", p->proto);
261  if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) {
262  fprintf(fp, "SRC PORT: %" PRIu16 "\n", sp);
263  fprintf(fp, "DST PORT: %" PRIu16 "\n", dp);
264  }
265 
266  fprintf(fp, "APP PROTO: %s\n",
268 
269  /* Only applicable to HTTP traffic */
270  if (p->flow->alproto == ALPROTO_HTTP) {
271  fprintf(fp, "HTTP URI: ");
272  LogFilestoreMetaGetUri(fp, p, ff);
273  fprintf(fp, "\n");
274  fprintf(fp, "HTTP HOST: ");
275  LogFilestoreMetaGetHost(fp, p, ff);
276  fprintf(fp, "\n");
277  fprintf(fp, "HTTP REFERER: ");
278  LogFilestoreMetaGetReferer(fp, p, ff);
279  fprintf(fp, "\n");
280  fprintf(fp, "HTTP USER AGENT: ");
281  LogFilestoreMetaGetUserAgent(fp, p, ff);
282  fprintf(fp, "\n");
283  } else if (p->flow->alproto == ALPROTO_SMTP) {
284  /* Only applicable to SMTP */
285  LogFilestoreMetaGetSmtp(fp, p, ff);
286  }
287 
288  fprintf(fp, "FILENAME: ");
289  PrintRawUriFp(fp, ff->name, ff->name_len);
290  fprintf(fp, "\n");
291 
292  fclose(fp);
293  }
294 }
295 
296 static void LogFilestoreLogCloseMetaFile(const File *ff)
297 {
298  char pid_expression[PATH_MAX] = "";
299  if (FileIncludePid())
300  snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
301  char final_filename[PATH_MAX] = "";
302  if (snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
303  g_logfile_base_dir, pid_expression, ff->file_store_id) == sizeof(final_filename))
304  return;
305  char final_metafilename[PATH_MAX] = "";
306  if (snprintf(final_metafilename, sizeof(final_metafilename),
307  "%s.meta", final_filename) == sizeof(final_metafilename))
308  return;
309  char working_metafilename[PATH_MAX] = "";
310  if (snprintf(working_metafilename, sizeof(working_metafilename),
311  "%s%s", final_metafilename, g_working_file_suffix) == sizeof(working_metafilename))
312  return;
313 
314  FILE *fp = fopen(working_metafilename, "a");
315  if (fp != NULL) {
316 #ifdef HAVE_MAGIC
317  fprintf(fp, "MAGIC: %s\n",
318  ff->magic ? ff->magic : "<unknown>");
319 #endif
320  switch (ff->state) {
321  case FILE_STATE_CLOSED:
322  fprintf(fp, "STATE: CLOSED\n");
323 #ifdef HAVE_NSS
324  if (ff->flags & FILE_MD5) {
325  fprintf(fp, "MD5: ");
326  size_t x;
327  for (x = 0; x < sizeof(ff->md5); x++) {
328  fprintf(fp, "%02x", ff->md5[x]);
329  }
330  fprintf(fp, "\n");
331  }
332  if (ff->flags & FILE_SHA1) {
333  fprintf(fp, "SHA1: ");
334  size_t x;
335  for (x = 0; x < sizeof(ff->sha1); x++) {
336  fprintf(fp, "%02x", ff->sha1[x]);
337  }
338  fprintf(fp, "\n");
339  }
340  if (ff->flags & FILE_SHA256) {
341  fprintf(fp, "SHA256: ");
342  size_t x;
343  for (x = 0; x < sizeof(ff->sha256); x++) {
344  fprintf(fp, "%02x", ff->sha256[x]);
345  }
346  fprintf(fp, "\n");
347  }
348 #endif
349  break;
351  fprintf(fp, "STATE: TRUNCATED\n");
352  break;
353  case FILE_STATE_ERROR:
354  fprintf(fp, "STATE: ERROR\n");
355  break;
356  default:
357  fprintf(fp, "STATE: UNKNOWN\n");
358  break;
359  }
360  fprintf(fp, "SIZE: %"PRIu64"\n", FileTrackedSize(ff));
361 
362  fclose(fp);
363  } else {
364  SCLogInfo("opening %s failed: %s", working_metafilename,
365  strerror(errno));
366  }
367 }
368 
369 static void LogFilestoreFinalizeFiles(const File *ff) {
370  char pid_expression[PATH_MAX] = "";
371  if (FileIncludePid())
372  snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
373  char final_filename[PATH_MAX] = "";
374  if (snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
375  g_logfile_base_dir, pid_expression, ff->file_store_id) == sizeof(final_filename))
376  return;
377  char working_filename[PATH_MAX] = "";
378  if (snprintf(working_filename, sizeof(working_filename), "%s%s",
379  final_filename, g_working_file_suffix) == sizeof(working_filename))
380  return;
381 
382  if (rename(working_filename, final_filename) != 0) {
383  SCLogWarning(SC_WARN_RENAMING_FILE, "renaming file %s to %s failed",
384  working_filename, final_filename);
385  return;
386  }
387  if (FileWriteMeta()) {
388  LogFilestoreLogCloseMetaFile(ff);
389  char final_metafilename[PATH_MAX] = "";
390  if (snprintf(final_metafilename, sizeof(final_metafilename),
391  "%s.meta", final_filename) == sizeof(final_metafilename))
392  return;
393  char working_metafilename[PATH_MAX] = "";
394  if (snprintf(working_metafilename, sizeof(working_metafilename),
395  "%s%s", final_metafilename, g_working_file_suffix) == sizeof(working_metafilename))
396  return;
397 
398  if (rename(working_metafilename, final_metafilename) != 0) {
400  "renaming metafile %s to %s failed", working_metafilename,
401  final_metafilename);
402  }
403  }
404 }
405 
406 static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p,
407  File *ff, const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
408 {
409  SCEnter();
410  LogFilestoreLogThread *aft = (LogFilestoreLogThread *)thread_data;
411  char filename[PATH_MAX] = "";
412  int file_fd = -1;
413  int ipver = -1;
414 
415  /* no flow, no htp state */
416  if (p->flow == NULL) {
418  }
419 
420  if (PKT_IS_IPV4(p)) {
421  ipver = AF_INET;
422  } else if (PKT_IS_IPV6(p)) {
423  ipver = AF_INET6;
424  } else {
425  return 0;
426  }
427 
428  SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
429 
430  char pid_expression[PATH_MAX] = "";
431  if (FileIncludePid())
432  snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
433  char base_filename[PATH_MAX] = "";
434  if (snprintf(base_filename, sizeof(base_filename), "%s/file%s.%u",
435  g_logfile_base_dir, pid_expression, ff->file_store_id) == sizeof(base_filename))
436  return -1;
437  if (snprintf(filename, sizeof(filename), "%s%s", base_filename,
438  g_working_file_suffix) == sizeof(filename))
439  return -1;
440 
442  aft->file_cnt++;
443 
444  /* create a .meta file that contains time, src/dst/sp/dp/proto */
445  LogFilestoreLogCreateMetaFile(p, ff, base_filename, ipver);
446 
447  if (SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
448  SC_ATOMIC_ADD(filestore_open_file_cnt, 1);
449  ff->fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644);
450  if (ff->fd == -1) {
451  SCLogDebug("failed to create file");
452  return -1;
453  }
454  file_fd = ff->fd;
455  } else {
456  file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644);
457  if (file_fd == -1) {
458  SCLogDebug("failed to create file");
459  return -1;
460  }
461  if (FileGetMaxOpenFiles() > 0) {
462  StatsIncr(tv, aft->counter_max_hits);
463  }
464  ff->fd = -1;
465  }
466  /* we can get called with a NULL ffd when we need to close */
467  } else if (data != NULL) {
468  if (ff->fd == -1) {
469  file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY);
470  if (file_fd == -1) {
471  SCLogDebug("failed to open file %s: %s", filename, strerror(errno));
472  return -1;
473  }
474  } else {
475  file_fd = ff->fd;
476  }
477  }
478 
479  if (file_fd != -1) {
480  ssize_t r = write(file_fd, (const void *)data, (size_t)data_len);
481  if (r == -1) {
482  SCLogDebug("write failed: %s", strerror(errno));
483  if (ff->fd != -1) {
484  SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
485  }
486  ff->fd = -1;
487  }
488  if (ff->fd == -1) {
489  close(file_fd);
490  }
491  }
492 
494  if (ff->fd != -1) {
495  close(ff->fd);
496  ff->fd = -1;
497  SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
498  }
499  LogFilestoreFinalizeFiles(ff);
500  }
501 
502  return 0;
503 }
504 
505 static TmEcode LogFilestoreLogThreadInit(ThreadVars *t, const void *initdata, void **data)
506 {
508  if (unlikely(aft == NULL))
509  return TM_ECODE_FAILED;
510  memset(aft, 0, sizeof(LogFilestoreLogThread));
511 
512  if (initdata == NULL)
513  {
514  SCLogDebug("Error getting context for LogFileStore. \"initdata\" argument NULL");
515  SCFree(aft);
516  return TM_ECODE_FAILED;
517  }
518 
519  /* Use the Ouptut Context (file pointer and mutex) */
520  aft->file_ctx = ((OutputCtx *)initdata)->data;
521 
522  struct stat stat_buf;
523  if (stat(g_logfile_base_dir, &stat_buf) != 0) {
524  int ret;
525  // coverity[toctou : FALSE]
526  ret = SCMkDir(g_logfile_base_dir, S_IRWXU|S_IXGRP|S_IRGRP);
527  if (ret != 0) {
528  int err = errno;
529  if (err != EEXIST) {
531  "Cannot create file drop directory %s: %s",
532  g_logfile_base_dir, strerror(err));
533  exit(EXIT_FAILURE);
534  }
535  } else {
536  SCLogInfo("Created file drop directory %s",
537  g_logfile_base_dir);
538  }
539 
540  }
541 
542  aft->counter_max_hits = StatsRegisterCounter("file_store.open_files_max_hit", t);
543 
544  *data = (void *)aft;
545  return TM_ECODE_OK;
546 }
547 
548 static TmEcode LogFilestoreLogThreadDeinit(ThreadVars *t, void *data)
549 {
551  if (aft == NULL) {
552  return TM_ECODE_OK;
553  }
554 
555  /* clear memory */
556  memset(aft, 0, sizeof(LogFilestoreLogThread));
557 
558  SCFree(aft);
559  return TM_ECODE_OK;
560 }
561 
562 static void LogFilestoreLogExitPrintStats(ThreadVars *tv, void *data)
563 {
565  if (aft == NULL) {
566  return;
567  }
568 
569  SCLogInfo("(%s) Files extracted %" PRIu32 "", tv->name, aft->file_cnt);
570 }
571 
572 /**
573  * \internal
574  *
575  * \brief deinit the log ctx and write out the waldo
576  *
577  * \param output_ctx output context to deinit
578  */
579 static void LogFilestoreLogDeInitCtx(OutputCtx *output_ctx)
580 {
581  LogFileCtx *logfile_ctx = (LogFileCtx *)output_ctx->data;
582  LogFileFreeCtx(logfile_ctx);
583  SCFree(output_ctx);
584 
585 }
586 
587 /** \brief Create a new http log LogFilestoreCtx.
588  * \param conf Pointer to ConfNode containing this loggers configuration.
589  * \return NULL if failure, LogFilestoreCtx* to the file_ctx if succesful
590  * */
591 static OutputInitResult LogFilestoreLogInitCtx(ConfNode *conf)
592 {
593  OutputInitResult result = { NULL, false };
594 
595  intmax_t version = 0;
596  if (ConfGetChildValueInt(conf, "version", &version)) {
597  if (version > 1) {
598  result.ok = true;
599  return result;
600  }
601  }
602 
605  "A file data logger is already enabled. Filestore (v1) "
606  "will not be enabled.");
607  return result;
608  }
609 
610  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
611  if (unlikely(output_ctx == NULL))
612  return result;
613 
614  output_ctx->data = NULL;
615  output_ctx->DeInit = LogFilestoreLogDeInitCtx;
616 
617  const char *s_default_log_dir = NULL;
618  s_default_log_dir = ConfigGetLogDirectory();
619 
620  const char *s_base_dir = NULL;
621  s_base_dir = ConfNodeLookupChildValue(conf, "log-dir");
622  if (s_base_dir == NULL || strlen(s_base_dir) == 0) {
623  strlcpy(g_logfile_base_dir,
624  s_default_log_dir, sizeof(g_logfile_base_dir));
625  } else {
626  if (PathIsAbsolute(s_base_dir)) {
627  strlcpy(g_logfile_base_dir,
628  s_base_dir, sizeof(g_logfile_base_dir));
629  } else {
630  snprintf(g_logfile_base_dir, sizeof(g_logfile_base_dir),
631  "%s/%s", s_default_log_dir, s_base_dir);
632  }
633  }
634 
635  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
636  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
638  SCLogInfo("forcing filestore of all files");
639  }
640 
641  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
642  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
644  SCLogInfo("forcing magic lookup for stored files");
645  }
646 
647  const char *write_meta = ConfNodeLookupChildValue(conf, "write-meta");
648  if (write_meta != NULL && !ConfValIsTrue(write_meta)) {
649  FileWriteMetaDisable();
650  SCLogInfo("File-store output will not write meta files");
651  }
652 
653  FileForceHashParseCfg(conf);
654  SCLogInfo("storing files in %s", g_logfile_base_dir);
655 
656  const char *stream_depth_str = ConfNodeLookupChildValue(conf, "stream-depth");
657  if (stream_depth_str != NULL && strcmp(stream_depth_str, "no")) {
658  uint32_t stream_depth = 0;
659  if (ParseSizeStringU32(stream_depth_str,
660  &stream_depth) < 0) {
661  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing "
662  "file-store.stream-depth "
663  "from conf file - %s. Killing engine",
664  stream_depth_str);
665  exit(EXIT_FAILURE);
666  } else {
667  FileReassemblyDepthEnable(stream_depth);
668  }
669  }
670 
671  const char *file_count_str = ConfNodeLookupChildValue(conf, "max-open-files");
672  if (file_count_str != NULL) {
673  uint32_t file_count = 0;
674  if (ParseSizeStringU32(file_count_str,
675  &file_count) < 0) {
676  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing "
677  "file-store.max-open-files "
678  "from conf file - %s. Killing engine",
679  stream_depth_str);
680  exit(EXIT_FAILURE);
681  } else {
682  if (file_count != 0) {
683  FileSetMaxOpenFiles(file_count);
684  SCLogInfo("file-store will keep a max of %d simultaneously"
685  " open files", file_count);
686  }
687  }
688  }
689 
690  const char *include_pid = ConfNodeLookupChildValue(conf, "include-pid");
691  if (include_pid != NULL && ConfValIsTrue(include_pid)) {
692  FileIncludePidEnable();
693  SCLogInfo("enabling pid as a part of all file names");
694  }
695 
696  StatsRegisterGlobalCounter("file_store.open_files",
697  LogFilestoreOpenFilesCounter);
698 
699  result.ctx = output_ctx;
700  result.ok = true;
701  SCReturnCT(result, "OutputInitResult");
702 }
703 
705 {
707  LogFilestoreLogInitCtx, LogFilestoreLogger, LogFilestoreLogThreadInit,
708  LogFilestoreLogThreadDeinit, LogFilestoreLogExitPrintStats);
710  LogFilestoreLogInitCtx, LogFilestoreLogger, LogFilestoreLogThreadInit,
711  LogFilestoreLogThreadDeinit, LogFilestoreLogExitPrintStats);
712 
713  SC_ATOMIC_INIT(filestore_open_file_cnt);
714  SC_ATOMIC_SET(filestore_open_file_cnt, 0);
715  SCLogDebug("registered");
716 }
#define OUTPUT_FILEDATA_FLAG_CLOSE
#define GET_IPV4_SRC_ADDR_PTR(p)
Definition: decode.h:213
uint16_t flags
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:444
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
MimeDecEntity * msg_tail
#define SCMkDir(a, b)
Definition: util-path.h:29
#define unlikely(expr)
Definition: util-optimize.h:35
#define GET_IPV4_DST_ADDR_PTR(p)
Definition: decode.h:214
Port sp
Definition: decode.h:414
#define FILE_SHA256
Definition: util-file.h:43
bstr * request_uri_normalized
Port dp
Definition: decode.h:422
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:108
#define PKT_IS_IPV6(p)
Definition: decode.h:251
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt)
uint64_t pcap_cnt
Definition: decode.h:566
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:125
int fd
Definition: util-file.h:73
#define PKT_IS_IPV4(p)
Definition: decode.h:250
void FileReassemblyDepthEnable(uint32_t size)
Definition: util-file.c:116
#define SCReturnCT(x, type)
Definition: util-debug.h:351
uint16_t StatsRegisterCounter(const char *name, struct ThreadVars_ *tv)
Registers a normal, unqualified counter.
Definition: counters.c:939
void * alstate
Definition: flow.h:436
#define O_NOFOLLOW
Definition: win32-misc.h:35
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:82
#define SCCalloc(nm, a)
Definition: util-mem.h:205
void LogFilestoreRegister(void)
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
uint16_t flags
Definition: util-file.h:65
uint32_t value_len
uint8_t proto
Definition: decode.h:429
#define GET_IPV6_DST_ADDR(p)
Definition: decode.h:219
struct LogFilestoreLogThread_ LogFilestoreLogThread
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileForceMagicEnable(void)
Definition: util-file.c:91
uint64_t txid
Definition: util-file.h:69
#define SCEnter(...)
Definition: util-debug.h:337
uint16_t name_len
Definition: util-file.h:66
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
void PrintRawUriFp(FILE *fp, uint8_t *buf, uint32_t buflen)
Definition: util-print.c:93
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:39
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1000
#define SCReturnInt(x)
Definition: util-debug.h:341
uint16_t Port
Definition: decode.h:234
#define FILE_MD5
Definition: util-file.h:39
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
#define MODULE_NAME
Definition: log-filestore.c:63
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:174
#define GET_IPV6_SRC_ADDR(p)
Definition: decode.h:218
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:208
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
int RunModeOutputFiledataEnabled(void)
Definition: runmodes.c:495
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
uint8_t version
Definition: decode-gre.h:405
#define SCFree(a)
Definition: util-mem.h:236
#define PKT_IS_TCP(p)
Definition: decode.h:252
void OutputRegisterFiledataModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FiledataLogger FiledataLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file data output module.
Definition: output.c:498
void * data
Definition: tm-modules.h:81
FileState state
Definition: util-file.h:67
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:193
uint32_t file_store_id
Definition: util-file.h:72
int ConfGetChildValueInt(const ConfNode *base, const char *name, intmax_t *val)
Definition: conf.c:469
char name[16]
Definition: threadvars.h:59
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
#define PKT_IS_UDP(p)
Definition: decode.h:253
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:450
int ParseSizeStringU32(const char *size, uint32_t *res)
Definition: util-misc.c:186
AppProto alproto
application level protocol
Definition: flow.h:407
#define OUTPUT_FILEDATA_FLAG_OPEN
This represents a header field name and associated value.
uint8_t * name
Definition: util-file.h:75
void CreateTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:237