suricata
output-filedata.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-filedata.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "conf.h"
34 #include "util-profiling.h"
35 #include "util-validate.h"
36 
37 typedef struct OutputLoggerThreadStore_ {
38  void *thread_data;
41 
42 /** per thread data for this module, contains a list of per thread
43  * data for the packet loggers. */
44 typedef struct OutputLoggerThreadData_ {
47 
48 /* logger instance, a module + a output ctx,
49  * it's perfectly valid that have multiple instances of the same
50  * log module (e.g. http.log) with different output ctx'. */
51 typedef struct OutputFiledataLogger_ {
55  const char *name;
61 
62 static OutputFiledataLogger *list = NULL;
63 static char g_waldo[PATH_MAX] = "";
64 static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER;
65 static int g_waldo_init = 0;
66 static int g_waldo_deinit = 0;
67 
72 {
73  OutputFiledataLogger *op = SCMalloc(sizeof(*op));
74  if (op == NULL)
75  return -1;
76  memset(op, 0x00, sizeof(*op));
77 
78  op->LogFunc = LogFunc;
79  op->output_ctx = output_ctx;
80  op->name = name;
81  op->logger_id = id;
82  op->ThreadInit = ThreadInit;
85 
86  if (list == NULL)
87  list = op;
88  else {
89  OutputFiledataLogger *t = list;
90  while (t->next)
91  t = t->next;
92  t->next = op;
93  }
94 
95  SCLogDebug("OutputRegisterFiledataLogger happy");
96  return 0;
97 }
98 
99 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
100 
101 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
102  Packet *p, File *ff,
103  const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
104 {
105  OutputFiledataLogger *logger = list;
106  OutputLoggerThreadStore *store = store_list;
107  int file_logged = 0;
108 
109  while (logger && store) {
110  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
111 
112  SCLogDebug("logger %p", logger);
114  logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
116 
117  file_logged = 1;
118 
119  logger = logger->next;
120  store = store->next;
121 
122  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
123  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
124  }
125  return file_logged;
126 }
127 
128 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadStore *store,
129  Packet *p, FileContainer *ffc, const uint8_t call_flags,
130  const bool file_close, const bool file_trunc, const uint8_t dir)
131 {
132  if (ffc != NULL) {
133  File *ff;
134  for (ff = ffc->head; ff != NULL; ff = ff->next) {
135  uint8_t file_flags = call_flags;
136 #ifdef HAVE_MAGIC
137  if (FileForceMagic() && ff->magic == NULL) {
138  FilemagicGlobalLookup(ff);
139  }
140 #endif
141  SCLogDebug("ff %p", ff);
142  if (ff->flags & FILE_STORED) {
143  SCLogDebug("stored flag set");
144  continue;
145  }
146 
147  if (!(ff->flags & FILE_STORE)) {
148  SCLogDebug("ff FILE_STORE not set");
149  continue;
150  }
151 
152  /* if we have no data chunks left to log, we should still
153  * close the logger(s) */
154  if (FileDataSize(ff) == ff->content_stored &&
155  (file_trunc || file_close)) {
156  if (ff->state < FILE_STATE_CLOSED) {
157  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
158  }
159  CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
160  ff->flags |= FILE_STORED;
161  continue;
162  }
163 
164  /* store */
165 
166  /* if file_store_id == 0, this is the first store of this file */
167  if (ff->file_store_id == 0) {
168  /* new file */
169  ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
170  file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
171  } else {
172  /* existing file */
173  }
174 
175  /* if file needs to be closed or truncated, inform
176  * loggers */
177  if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
178  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
179  }
180 
181  /* tell the logger we're closing up */
182  if (ff->state >= FILE_STATE_CLOSED)
183  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
184 
185  /* do the actual logging */
186  const uint8_t *data = NULL;
187  uint32_t data_len = 0;
188 
190  &data, &data_len,
191  ff->content_stored);
192 
193  const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
194  if (file_logged) {
195  ff->content_stored += data_len;
196 
197  /* all done */
198  if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
199  ff->flags |= FILE_STORED;
200  }
201  }
202  }
203 
204  FilePrune(ffc);
205  }
206 }
207 
208 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
209 {
210  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
211 
212  if (list == NULL) {
213  /* No child loggers. */
214  return TM_ECODE_OK;
215  }
216 
217  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
218  OutputLoggerThreadStore *store = op_thread_data->store;
219 
220  /* no flow, no files */
221  Flow * const f = p->flow;
222  if (f == NULL || f->alstate == NULL) {
224  }
225 
226  const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END) &&
228  const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END) &&
230  const bool file_trunc = StreamTcpReassembleDepthReached(p);
231 
236  SCLogDebug("ffc_ts %p", ffc_ts);
237  OutputFiledataLogFfc(tv, store, p, ffc_ts, STREAM_TOSERVER, file_close_ts, file_trunc, STREAM_TOSERVER);
238  SCLogDebug("ffc_tc %p", ffc_tc);
239  OutputFiledataLogFfc(tv, store, p, ffc_tc, STREAM_TOCLIENT, file_close_tc, file_trunc, STREAM_TOCLIENT);
240 
241  return TM_ECODE_OK;
242 }
243 
244 /**
245  * \internal
246  *
247  * \brief Open the waldo file (if available) and load the file_id
248  *
249  * \param path full path for the waldo file
250  */
251 static void LogFiledataLogLoadWaldo(const char *path)
252 {
253  char line[16] = "";
254  unsigned int id = 0;
255 
256  FILE *fp = fopen(path, "r");
257  if (fp == NULL) {
258  SCLogInfo("couldn't open waldo: %s", strerror(errno));
259  SCReturn;
260  }
261 
262  if (fgets(line, (int)sizeof(line), fp) != NULL) {
263  if (sscanf(line, "%10u", &id) == 1) {
264  SCLogInfo("id %u", id);
265  (void) SC_ATOMIC_CAS(&g_file_store_id, 0, id);
266  }
267  }
268  fclose(fp);
269 }
270 
271 /**
272  * \internal
273  *
274  * \brief Store the waldo file based on the file_id
275  *
276  * \param path full path for the waldo file
277  */
278 static void LogFiledataLogStoreWaldo(const char *path)
279 {
280  char line[16] = "";
281 
282  if (SC_ATOMIC_GET(g_file_store_id) == 0) {
283  SCReturn;
284  }
285 
286  FILE *fp = fopen(path, "w");
287  if (fp == NULL) {
288  SCLogInfo("couldn't open waldo: %s", strerror(errno));
289  SCReturn;
290  }
291 
292  snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(g_file_store_id));
293  if (fwrite(line, strlen(line), 1, fp) != 1) {
294  SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno));
295  }
296  fclose(fp);
297 }
298 
299 /** \brief thread init for the tx logger
300  * This will run the thread init functions for the individual registered
301  * loggers */
302 static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
303 {
304  OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
305  if (td == NULL)
306  return TM_ECODE_FAILED;
307  memset(td, 0x00, sizeof(*td));
308 
309  *data = (void *)td;
310 
311  SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
312 
313  OutputFiledataLogger *logger = list;
314  while (logger) {
315  if (logger->ThreadInit) {
316  void *retptr = NULL;
317  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
318  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
319 /* todo */ BUG_ON(ts == NULL);
320  memset(ts, 0x00, sizeof(*ts));
321 
322  /* store thread handle */
323  ts->thread_data = retptr;
324 
325  if (td->store == NULL) {
326  td->store = ts;
327  } else {
328  OutputLoggerThreadStore *tmp = td->store;
329  while (tmp->next != NULL)
330  tmp = tmp->next;
331  tmp->next = ts;
332  }
333 
334  SCLogDebug("%s is now set up", logger->name);
335  }
336  }
337 
338  logger = logger->next;
339  }
340 
341  SCMutexLock(&g_waldo_mutex);
342  if (g_waldo_init == 0) {
343  ConfNode *node = ConfGetNode("file-store-waldo");
344  if (node == NULL) {
345  ConfNode *outputs = ConfGetNode("outputs");
346  if (outputs) {
347  ConfNode *output;
348  TAILQ_FOREACH(output, &outputs->head, next) {
349  /* we only care about file and file-store */
350  if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0))
351  continue;
352 
353  ConfNode *file = ConfNodeLookupChild(output, output->val);
354  BUG_ON(file == NULL);
355  if (file == NULL) {
356  SCLogDebug("file-store failed, lets try 'file'");
357  file = ConfNodeLookupChild(outputs, "file");
358  if (file == NULL)
359  SCLogDebug("file failed as well, giving up");
360  }
361 
362  if (file != NULL) {
363  node = ConfNodeLookupChild(file, "waldo");
364  if (node == NULL)
365  SCLogDebug("no waldo node");
366  }
367  }
368  }
369  }
370  if (node != NULL) {
371  const char *s_default_log_dir = NULL;
372  s_default_log_dir = ConfigGetLogDirectory();
373 
374  const char *waldo = node->val;
375  SCLogDebug("loading waldo %s", waldo);
376  if (waldo != NULL && strlen(waldo) > 0) {
377  if (PathIsAbsolute(waldo)) {
378  snprintf(g_waldo, sizeof(g_waldo), "%s", waldo);
379  } else {
380  snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo);
381  }
382 
383  SCLogDebug("loading waldo file %s", g_waldo);
384  LogFiledataLogLoadWaldo(g_waldo);
385  }
386  }
387  g_waldo_init = 1;
388  }
389  SCMutexUnlock(&g_waldo_mutex);
390  return TM_ECODE_OK;
391 }
392 
393 static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
394 {
395  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
396  OutputLoggerThreadStore *store = op_thread_data->store;
397  OutputFiledataLogger *logger = list;
398 
399  while (logger && store) {
400  if (logger->ThreadDeinit) {
401  logger->ThreadDeinit(tv, store->thread_data);
402  }
403 
404  OutputLoggerThreadStore *next_store = store->next;
405  SCFree(store);
406  store = next_store;
407  logger = logger->next;
408  }
409 
410  SCMutexLock(&g_waldo_mutex);
411  if (g_waldo_deinit == 0) {
412  if (strlen(g_waldo) > 0) {
413  SCLogDebug("we have a waldo at %s", g_waldo);
414  LogFiledataLogStoreWaldo(g_waldo);
415  }
416  g_waldo_deinit = 1;
417  }
418  SCMutexUnlock(&g_waldo_mutex);
419 
420  SCFree(op_thread_data);
421  return TM_ECODE_OK;
422 }
423 
424 static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
425 {
426  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
427  OutputLoggerThreadStore *store = op_thread_data->store;
428  OutputFiledataLogger *logger = list;
429 
430  while (logger && store) {
431  if (logger->ThreadExitPrintStats) {
432  logger->ThreadExitPrintStats(tv, store->thread_data);
433  }
434 
435  logger = logger->next;
436  store = store->next;
437  }
438 }
439 
441 {
442  OutputRegisterRootLogger(OutputFiledataLogThreadInit,
443  OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
444  OutputFiledataLog);
445  SC_ATOMIC_INIT(g_file_store_id);
446 }
447 
449 {
450  OutputFiledataLogger *logger = list;
451  while (logger) {
452  OutputFiledataLogger *next_logger = logger->next;
453  SCFree(logger);
454  logger = next_logger;
455  }
456 
457  list = NULL;
458 }
#define OUTPUT_FILEDATA_FLAG_CLOSE
void OutputFiledataLoggerRegister(void)
#define SCMutex
uint16_t flags
#define PACKET_PROFILING_LOGGER_START(p, id)
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
#define BUG_ON(x)
int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
LoggerId
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:277
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
OutputLoggerThreadStore * store
Definition: output-file.c:44
#define FILE_STORE
Definition: util-file.h:46
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
struct OutputFiledataLogger_ OutputFiledataLogger
struct File_ * next
Definition: util-file.h:79
ThreadDeinitFunc ThreadDeinit
#define SCMutexLock(mut)
StreamingBuffer * sb
Definition: util-file.h:68
struct OutputFiledataLogger_ * next
void OutputFiledataShutdown(void)
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
FiledataLogger LogFunc
void * alstate
Definition: flow.h:438
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define SCMutexUnlock(mut)
uint16_t flags
Definition: util-file.h:65
uint8_t proto
Definition: decode.h:428
#define SCMUTEX_INITIALIZER
struct OutputLoggerThreadStore_ * next
Definition: output-file.c:38
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc)
Definition: output.c:1001
uint8_t flowflags
Definition: decode.h:437
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id)
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:39
ThreadExitPrintStatsFunc ThreadExitPrintStats
int(* FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir)
#define SCReturnInt(x)
Definition: util-debug.h:341
void FilePrune(FileContainer *ffc)
Definition: util-file.c:345
int FileForceMagic(void)
Definition: util-file.c:130
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:166
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:228
#define SC_ATOMIC_CAS(name, cmpval, newval)
atomic Compare and Switch
Definition: util-atomic.h:222
uint64_t ts
struct OutputLoggerThreadStore_ OutputLoggerThreadStore
#define STREAM_TOSERVER
Definition: stream.h:31
FileState state
Definition: util-file.h:67
struct OutputLoggerThreadData_ OutputLoggerThreadData
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
uint32_t file_store_id
Definition: util-file.h:72
uint64_t content_stored
Definition: util-file.h:90
ThreadInitFunc ThreadInit
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define FILE_TRUNCATED
Definition: util-file.h:36
#define SCReturn
Definition: util-debug.h:339
Per thread variable structure.
Definition: threadvars.h:57
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
#define PACKET_PROFILING_LOGGER_END(p, id)
uint32_t flags
Definition: decode.h:441
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
#define OUTPUT_FILEDATA_FLAG_OPEN
int FileCloseFilePtr(File *ff, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:881
Flow data structure.
Definition: flow.h:325
#define DEBUG_VALIDATE_BUG_ON(exp)