suricata
output-filedata.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-filedata.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "conf.h"
34 #include "util-profiling.h"
35 #include "util-validate.h"
36 
37 typedef struct OutputLoggerThreadStore_ {
38  void *thread_data;
41 
42 /** per thread data for this module, contains a list of per thread
43  * data for the packet loggers. */
44 typedef struct OutputLoggerThreadData_ {
47 
48 /* logger instance, a module + a output ctx,
49  * it's perfectly valid that have multiple instances of the same
50  * log module (e.g. http.log) with different output ctx'. */
51 typedef struct OutputFiledataLogger_ {
55  const char *name;
61 
62 static OutputFiledataLogger *list = NULL;
63 static char g_waldo[PATH_MAX] = "";
64 static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER;
65 static int g_waldo_init = 0;
66 static int g_waldo_deinit = 0;
67 
72 {
73  OutputFiledataLogger *op = SCMalloc(sizeof(*op));
74  if (op == NULL)
75  return -1;
76  memset(op, 0x00, sizeof(*op));
77 
78  op->LogFunc = LogFunc;
79  op->output_ctx = output_ctx;
80  op->name = name;
81  op->logger_id = id;
82  op->ThreadInit = ThreadInit;
85 
86  if (list == NULL)
87  list = op;
88  else {
89  OutputFiledataLogger *t = list;
90  while (t->next)
91  t = t->next;
92  t->next = op;
93  }
94 
95  SCLogDebug("OutputRegisterFiledataLogger happy");
96  return 0;
97 }
98 
99 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
100 
101 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
102  Packet *p, File *ff,
103  const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
104 {
105  OutputFiledataLogger *logger = list;
106  OutputLoggerThreadStore *store = store_list;
107  int file_logged = 0;
108 
109  while (logger && store) {
110  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
111 
112  SCLogDebug("logger %p", logger);
114  logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
116 
117  file_logged = 1;
118 
119  logger = logger->next;
120  store = store->next;
121 
122  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
123  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
124  }
125  return file_logged;
126 }
127 
128 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadStore *store,
129  Packet *p, FileContainer *ffc, const uint8_t call_flags,
130  const bool file_close, const bool file_trunc, const uint8_t dir)
131 {
132  if (ffc != NULL) {
133  File *ff;
134  for (ff = ffc->head; ff != NULL; ff = ff->next) {
135  uint8_t file_flags = call_flags;
136 #ifdef HAVE_MAGIC
137  if (FileForceMagic() && ff->magic == NULL) {
138  FilemagicGlobalLookup(ff);
139  }
140 #endif
141  SCLogDebug("ff %p", ff);
142  if (ff->flags & FILE_STORED) {
143  SCLogDebug("stored flag set");
144  continue;
145  }
146 
147  if (!(ff->flags & FILE_STORE)) {
148  SCLogDebug("ff FILE_STORE not set");
149  continue;
150  }
151 
152  /* if we have no data chunks left to log, we should still
153  * close the logger(s) */
154  if (FileDataSize(ff) == ff->content_stored &&
155  (file_trunc || file_close)) {
156  if (ff->state < FILE_STATE_CLOSED) {
157  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
158  }
159  CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
160  ff->flags |= FILE_STORED;
161  continue;
162  }
163 
164  /* store */
165 
166  /* if file_store_id == 0, this is the first store of this file */
167  if (ff->file_store_id == 0) {
168  /* new file */
169  ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
170  file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
171  } else {
172  /* existing file */
173  }
174 
175  /* if file needs to be closed or truncated, inform
176  * loggers */
177  if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
178  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
179  }
180 
181  /* tell the logger we're closing up */
182  if (ff->state >= FILE_STATE_CLOSED)
183  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
184 
185  /* do the actual logging */
186  const uint8_t *data = NULL;
187  uint32_t data_len = 0;
188 
190  &data, &data_len,
191  ff->content_stored);
192 
193  const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
194  if (file_logged) {
195  ff->content_stored += data_len;
196 
197  /* all done */
198  if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
199  ff->flags |= FILE_STORED;
200  }
201  }
202  }
203  }
204 }
205 
206 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
207 {
208  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
209 
210  if (list == NULL) {
211  /* No child loggers. */
212  return TM_ECODE_OK;
213  }
214 
215  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
216  OutputLoggerThreadStore *store = op_thread_data->store;
217 
218  /* no flow, no files */
219  Flow * const f = p->flow;
220  if (f == NULL || f->alstate == NULL) {
222  }
223 
224  const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END) &&
226  const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END) &&
228  const bool file_trunc = StreamTcpReassembleDepthReached(p);
229 
234  SCLogDebug("ffc_ts %p", ffc_ts);
235  OutputFiledataLogFfc(tv, store, p, ffc_ts, STREAM_TOSERVER, file_close_ts, file_trunc, STREAM_TOSERVER);
236  SCLogDebug("ffc_tc %p", ffc_tc);
237  OutputFiledataLogFfc(tv, store, p, ffc_tc, STREAM_TOCLIENT, file_close_tc, file_trunc, STREAM_TOCLIENT);
238 
239  return TM_ECODE_OK;
240 }
241 
242 /**
243  * \internal
244  *
245  * \brief Open the waldo file (if available) and load the file_id
246  *
247  * \param path full path for the waldo file
248  */
249 static void LogFiledataLogLoadWaldo(const char *path)
250 {
251  char line[16] = "";
252  unsigned int id = 0;
253 
254  FILE *fp = fopen(path, "r");
255  if (fp == NULL) {
256  SCLogInfo("couldn't open waldo: %s", strerror(errno));
257  SCReturn;
258  }
259 
260  if (fgets(line, (int)sizeof(line), fp) != NULL) {
261  if (sscanf(line, "%10u", &id) == 1) {
262  SCLogInfo("id %u", id);
263  (void) SC_ATOMIC_CAS(&g_file_store_id, 0, id);
264  }
265  }
266  fclose(fp);
267 }
268 
269 /**
270  * \internal
271  *
272  * \brief Store the waldo file based on the file_id
273  *
274  * \param path full path for the waldo file
275  */
276 static void LogFiledataLogStoreWaldo(const char *path)
277 {
278  char line[16] = "";
279 
280  if (SC_ATOMIC_GET(g_file_store_id) == 0) {
281  SCReturn;
282  }
283 
284  FILE *fp = fopen(path, "w");
285  if (fp == NULL) {
286  SCLogInfo("couldn't open waldo: %s", strerror(errno));
287  SCReturn;
288  }
289 
290  snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(g_file_store_id));
291  if (fwrite(line, strlen(line), 1, fp) != 1) {
292  SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno));
293  }
294  fclose(fp);
295 }
296 
297 /** \brief thread init for the tx logger
298  * This will run the thread init functions for the individual registered
299  * loggers */
300 static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
301 {
302  OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
303  if (td == NULL)
304  return TM_ECODE_FAILED;
305  memset(td, 0x00, sizeof(*td));
306 
307  *data = (void *)td;
308 
309  SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
310 
311  OutputFiledataLogger *logger = list;
312  while (logger) {
313  if (logger->ThreadInit) {
314  void *retptr = NULL;
315  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
316  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
317 /* todo */ BUG_ON(ts == NULL);
318  memset(ts, 0x00, sizeof(*ts));
319 
320  /* store thread handle */
321  ts->thread_data = retptr;
322 
323  if (td->store == NULL) {
324  td->store = ts;
325  } else {
326  OutputLoggerThreadStore *tmp = td->store;
327  while (tmp->next != NULL)
328  tmp = tmp->next;
329  tmp->next = ts;
330  }
331 
332  SCLogDebug("%s is now set up", logger->name);
333  }
334  }
335 
336  logger = logger->next;
337  }
338 
339  SCMutexLock(&g_waldo_mutex);
340  if (g_waldo_init == 0) {
341  ConfNode *node = ConfGetNode("file-store-waldo");
342  if (node == NULL) {
343  ConfNode *outputs = ConfGetNode("outputs");
344  if (outputs) {
345  ConfNode *output;
346  TAILQ_FOREACH(output, &outputs->head, next) {
347  /* we only care about file and file-store */
348  if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0))
349  continue;
350 
351  ConfNode *file = ConfNodeLookupChild(output, output->val);
352  BUG_ON(file == NULL);
353  if (file == NULL) {
354  SCLogDebug("file-store failed, lets try 'file'");
355  file = ConfNodeLookupChild(outputs, "file");
356  if (file == NULL)
357  SCLogDebug("file failed as well, giving up");
358  }
359 
360  if (file != NULL) {
361  node = ConfNodeLookupChild(file, "waldo");
362  if (node == NULL)
363  SCLogDebug("no waldo node");
364  }
365  }
366  }
367  }
368  if (node != NULL) {
369  const char *s_default_log_dir = NULL;
370  s_default_log_dir = ConfigGetLogDirectory();
371 
372  const char *waldo = node->val;
373  SCLogDebug("loading waldo %s", waldo);
374  if (waldo != NULL && strlen(waldo) > 0) {
375  if (PathIsAbsolute(waldo)) {
376  snprintf(g_waldo, sizeof(g_waldo), "%s", waldo);
377  } else {
378  snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo);
379  }
380 
381  SCLogDebug("loading waldo file %s", g_waldo);
382  LogFiledataLogLoadWaldo(g_waldo);
383  }
384  }
385  g_waldo_init = 1;
386  }
387  SCMutexUnlock(&g_waldo_mutex);
388  return TM_ECODE_OK;
389 }
390 
391 static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
392 {
393  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
394  OutputLoggerThreadStore *store = op_thread_data->store;
395  OutputFiledataLogger *logger = list;
396 
397  while (logger && store) {
398  if (logger->ThreadDeinit) {
399  logger->ThreadDeinit(tv, store->thread_data);
400  }
401 
402  OutputLoggerThreadStore *next_store = store->next;
403  SCFree(store);
404  store = next_store;
405  logger = logger->next;
406  }
407 
408  SCMutexLock(&g_waldo_mutex);
409  if (g_waldo_deinit == 0) {
410  if (strlen(g_waldo) > 0) {
411  SCLogDebug("we have a waldo at %s", g_waldo);
412  LogFiledataLogStoreWaldo(g_waldo);
413  }
414  g_waldo_deinit = 1;
415  }
416  SCMutexUnlock(&g_waldo_mutex);
417 
418  SCFree(op_thread_data);
419  return TM_ECODE_OK;
420 }
421 
422 static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
423 {
424  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
425  OutputLoggerThreadStore *store = op_thread_data->store;
426  OutputFiledataLogger *logger = list;
427 
428  while (logger && store) {
429  if (logger->ThreadExitPrintStats) {
430  logger->ThreadExitPrintStats(tv, store->thread_data);
431  }
432 
433  logger = logger->next;
434  store = store->next;
435  }
436 }
437 
439 {
440  OutputRegisterRootLogger(OutputFiledataLogThreadInit,
441  OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
442  OutputFiledataLog);
443  SC_ATOMIC_INIT(g_file_store_id);
444 }
445 
447 {
448  OutputFiledataLogger *logger = list;
449  while (logger) {
450  OutputFiledataLogger *next_logger = logger->next;
451  SCFree(logger);
452  logger = next_logger;
453  }
454 
455  list = NULL;
456 }
#define OUTPUT_FILEDATA_FLAG_CLOSE
void OutputFiledataLoggerRegister(void)
#define SCMutex
uint16_t flags
#define PACKET_PROFILING_LOGGER_START(p, id)
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:446
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
#define BUG_ON(x)
int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
LoggerId
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:279
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
OutputLoggerThreadStore * store
Definition: output-file.c:44
#define FILE_STORE
Definition: util-file.h:46
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
struct OutputFiledataLogger_ OutputFiledataLogger
struct File_ * next
Definition: util-file.h:77
ThreadDeinitFunc ThreadDeinit
#define SCMutexLock(mut)
StreamingBuffer * sb
Definition: util-file.h:67
struct OutputFiledataLogger_ * next
void OutputFiledataShutdown(void)
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
FiledataLogger LogFunc
void * alstate
Definition: flow.h:438
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define SCMutexUnlock(mut)
uint16_t flags
Definition: util-file.h:64
uint8_t proto
Definition: decode.h:431
#define SCMUTEX_INITIALIZER
struct OutputLoggerThreadStore_ * next
Definition: output-file.c:38
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc)
Definition: output.c:1004
uint8_t flowflags
Definition: decode.h:440
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id)
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1095
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:39
ThreadExitPrintStatsFunc ThreadExitPrintStats
int(* FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir)
#define SCReturnInt(x)
Definition: util-debug.h:341
int FileForceMagic(void)
Definition: util-file.c:132
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:222
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:322
#define SC_ATOMIC_CAS(name, cmpval, newval)
atomic Compare and Switch
Definition: util-atomic.h:222
uint64_t ts
struct OutputLoggerThreadStore_ OutputLoggerThreadStore
#define STREAM_TOSERVER
Definition: stream.h:31
FileState state
Definition: util-file.h:66
struct OutputLoggerThreadData_ OutputLoggerThreadData
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
uint32_t file_store_id
Definition: util-file.h:70
uint64_t content_stored
Definition: util-file.h:88
ThreadInitFunc ThreadInit
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define FILE_TRUNCATED
Definition: util-file.h:36
#define SCReturn
Definition: util-debug.h:339
Per thread variable structure.
Definition: threadvars.h:57
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
#define PACKET_PROFILING_LOGGER_END(p, id)
uint32_t flags
Definition: decode.h:444
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
#define OUTPUT_FILEDATA_FLAG_OPEN
int FileCloseFilePtr(File *ff, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:918
Flow data structure.
Definition: flow.h:325
#define DEBUG_VALIDATE_BUG_ON(exp)