suricata
output-filedata.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-filedata.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "conf.h"
34 #include "util-profiling.h"
35 
36 typedef struct OutputLoggerThreadStore_ {
37  void *thread_data;
40 
41 /** per thread data for this module, contains a list of per thread
42  * data for the packet loggers. */
43 typedef struct OutputLoggerThreadData_ {
46 
47 /* logger instance, a module + a output ctx,
48  * it's perfectly valid that have multiple instances of the same
49  * log module (e.g. http.log) with different output ctx'. */
50 typedef struct OutputFiledataLogger_ {
54  const char *name;
60 
61 static OutputFiledataLogger *list = NULL;
62 static char g_waldo[PATH_MAX] = "";
63 static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER;
64 static int g_waldo_init = 0;
65 static int g_waldo_deinit = 0;
66 
71 {
72  OutputFiledataLogger *op = SCMalloc(sizeof(*op));
73  if (op == NULL)
74  return -1;
75  memset(op, 0x00, sizeof(*op));
76 
77  op->LogFunc = LogFunc;
78  op->output_ctx = output_ctx;
79  op->name = name;
80  op->logger_id = id;
81  op->ThreadInit = ThreadInit;
84 
85  if (list == NULL)
86  list = op;
87  else {
88  OutputFiledataLogger *t = list;
89  while (t->next)
90  t = t->next;
91  t->next = op;
92  }
93 
94  SCLogDebug("OutputRegisterFiledataLogger happy");
95  return 0;
96 }
97 
98 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
99 
100 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
101  Packet *p, File *ff,
102  const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
103 {
104  OutputFiledataLogger *logger = list;
105  OutputLoggerThreadStore *store = store_list;
106  int file_logged = 0;
107 
108  while (logger && store) {
109  BUG_ON(logger->LogFunc == NULL);
110 
111  SCLogDebug("logger %p", logger);
113  logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
115 
116  file_logged = 1;
117 
118  logger = logger->next;
119  store = store->next;
120 
121  BUG_ON(logger == NULL && store != NULL);
122  BUG_ON(logger != NULL && store == NULL);
123  }
124  return file_logged;
125 }
126 
127 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadStore *store,
128  Packet *p, FileContainer *ffc, const uint8_t call_flags,
129  const bool file_close, const bool file_trunc, const uint8_t dir)
130 {
131  if (ffc != NULL) {
132  File *ff;
133  for (ff = ffc->head; ff != NULL; ff = ff->next) {
134  uint8_t file_flags = call_flags;
135 #ifdef HAVE_MAGIC
136  if (FileForceMagic() && ff->magic == NULL) {
137  FilemagicGlobalLookup(ff);
138  }
139 #endif
140  SCLogDebug("ff %p", ff);
141  if (ff->flags & FILE_STORED) {
142  SCLogDebug("stored flag set");
143  continue;
144  }
145 
146  if (!(ff->flags & FILE_STORE)) {
147  SCLogDebug("ff FILE_STORE not set");
148  continue;
149  }
150 
151  /* if we have no data chunks left to log, we should still
152  * close the logger(s) */
153  if (FileDataSize(ff) == ff->content_stored &&
154  (file_trunc || file_close)) {
155  if (ff->state < FILE_STATE_CLOSED) {
156  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
157  }
158  CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
159  ff->flags |= FILE_STORED;
160  continue;
161  }
162 
163  /* store */
164 
165  /* if file_store_id == 0, this is the first store of this file */
166  if (ff->file_store_id == 0) {
167  /* new file */
168  ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
169  file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
170  } else {
171  /* existing file */
172  }
173 
174  /* if file needs to be closed or truncated, inform
175  * loggers */
176  if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
177  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
178  }
179 
180  /* tell the logger we're closing up */
181  if (ff->state >= FILE_STATE_CLOSED)
182  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
183 
184  /* do the actual logging */
185  const uint8_t *data = NULL;
186  uint32_t data_len = 0;
187 
189  &data, &data_len,
190  ff->content_stored);
191 
192  const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
193  if (file_logged) {
194  ff->content_stored += data_len;
195 
196  /* all done */
197  if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
198  ff->flags |= FILE_STORED;
199  }
200  }
201  }
202 
203  FilePrune(ffc);
204  }
205 }
206 
207 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
208 {
209  BUG_ON(thread_data == NULL);
210 
211  if (list == NULL) {
212  /* No child loggers. */
213  return TM_ECODE_OK;
214  }
215 
216  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
217  OutputLoggerThreadStore *store = op_thread_data->store;
218 
219  /* no flow, no files */
220  Flow * const f = p->flow;
221  if (f == NULL || f->alstate == NULL) {
223  }
224 
225  const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END) &&
227  const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END) &&
229  const bool file_trunc = StreamTcpReassembleDepthReached(p);
230 
235  SCLogDebug("ffc_ts %p", ffc_ts);
236  OutputFiledataLogFfc(tv, store, p, ffc_ts, STREAM_TOSERVER, file_close_ts, file_trunc, STREAM_TOSERVER);
237  SCLogDebug("ffc_tc %p", ffc_tc);
238  OutputFiledataLogFfc(tv, store, p, ffc_tc, STREAM_TOCLIENT, file_close_tc, file_trunc, STREAM_TOCLIENT);
239 
240  return TM_ECODE_OK;
241 }
242 
243 /**
244  * \internal
245  *
246  * \brief Open the waldo file (if available) and load the file_id
247  *
248  * \param path full path for the waldo file
249  */
250 static void LogFiledataLogLoadWaldo(const char *path)
251 {
252  char line[16] = "";
253  unsigned int id = 0;
254 
255  FILE *fp = fopen(path, "r");
256  if (fp == NULL) {
257  SCLogInfo("couldn't open waldo: %s", strerror(errno));
258  SCReturn;
259  }
260 
261  if (fgets(line, (int)sizeof(line), fp) != NULL) {
262  if (sscanf(line, "%10u", &id) == 1) {
263  SCLogInfo("id %u", id);
264  (void) SC_ATOMIC_CAS(&g_file_store_id, 0, id);
265  }
266  }
267  fclose(fp);
268 }
269 
270 /**
271  * \internal
272  *
273  * \brief Store the waldo file based on the file_id
274  *
275  * \param path full path for the waldo file
276  */
277 static void LogFiledataLogStoreWaldo(const char *path)
278 {
279  char line[16] = "";
280 
281  if (SC_ATOMIC_GET(g_file_store_id) == 0) {
282  SCReturn;
283  }
284 
285  FILE *fp = fopen(path, "w");
286  if (fp == NULL) {
287  SCLogInfo("couldn't open waldo: %s", strerror(errno));
288  SCReturn;
289  }
290 
291  snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(g_file_store_id));
292  if (fwrite(line, strlen(line), 1, fp) != 1) {
293  SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno));
294  }
295  fclose(fp);
296 }
297 
298 /** \brief thread init for the tx logger
299  * This will run the thread init functions for the individual registered
300  * loggers */
301 static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
302 {
303  OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
304  if (td == NULL)
305  return TM_ECODE_FAILED;
306  memset(td, 0x00, sizeof(*td));
307 
308  *data = (void *)td;
309 
310  SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
311 
312  OutputFiledataLogger *logger = list;
313  while (logger) {
314  if (logger->ThreadInit) {
315  void *retptr = NULL;
316  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
317  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
318 /* todo */ BUG_ON(ts == NULL);
319  memset(ts, 0x00, sizeof(*ts));
320 
321  /* store thread handle */
322  ts->thread_data = retptr;
323 
324  if (td->store == NULL) {
325  td->store = ts;
326  } else {
327  OutputLoggerThreadStore *tmp = td->store;
328  while (tmp->next != NULL)
329  tmp = tmp->next;
330  tmp->next = ts;
331  }
332 
333  SCLogDebug("%s is now set up", logger->name);
334  }
335  }
336 
337  logger = logger->next;
338  }
339 
340  SCMutexLock(&g_waldo_mutex);
341  if (g_waldo_init == 0) {
342  ConfNode *node = ConfGetNode("file-store-waldo");
343  if (node == NULL) {
344  ConfNode *outputs = ConfGetNode("outputs");
345  if (outputs) {
346  ConfNode *output;
347  TAILQ_FOREACH(output, &outputs->head, next) {
348  /* we only care about file and file-store */
349  if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0))
350  continue;
351 
352  ConfNode *file = ConfNodeLookupChild(output, output->val);
353  BUG_ON(file == NULL);
354  if (file == NULL) {
355  SCLogDebug("file-store failed, lets try 'file'");
356  file = ConfNodeLookupChild(outputs, "file");
357  if (file == NULL)
358  SCLogDebug("file failed as well, giving up");
359  }
360 
361  if (file != NULL) {
362  node = ConfNodeLookupChild(file, "waldo");
363  if (node == NULL)
364  SCLogDebug("no waldo node");
365  }
366  }
367  }
368  }
369  if (node != NULL) {
370  const char *s_default_log_dir = NULL;
371  s_default_log_dir = ConfigGetLogDirectory();
372 
373  const char *waldo = node->val;
374  SCLogDebug("loading waldo %s", waldo);
375  if (waldo != NULL && strlen(waldo) > 0) {
376  if (PathIsAbsolute(waldo)) {
377  snprintf(g_waldo, sizeof(g_waldo), "%s", waldo);
378  } else {
379  snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo);
380  }
381 
382  SCLogDebug("loading waldo file %s", g_waldo);
383  LogFiledataLogLoadWaldo(g_waldo);
384  }
385  }
386  g_waldo_init = 1;
387  }
388  SCMutexUnlock(&g_waldo_mutex);
389  return TM_ECODE_OK;
390 }
391 
392 static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
393 {
394  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
395  OutputLoggerThreadStore *store = op_thread_data->store;
396  OutputFiledataLogger *logger = list;
397 
398  while (logger && store) {
399  if (logger->ThreadDeinit) {
400  logger->ThreadDeinit(tv, store->thread_data);
401  }
402 
403  OutputLoggerThreadStore *next_store = store->next;
404  SCFree(store);
405  store = next_store;
406  logger = logger->next;
407  }
408 
409  SCMutexLock(&g_waldo_mutex);
410  if (g_waldo_deinit == 0) {
411  if (strlen(g_waldo) > 0) {
412  SCLogDebug("we have a waldo at %s", g_waldo);
413  LogFiledataLogStoreWaldo(g_waldo);
414  }
415  g_waldo_deinit = 1;
416  }
417  SCMutexUnlock(&g_waldo_mutex);
418 
419  SCFree(op_thread_data);
420  return TM_ECODE_OK;
421 }
422 
423 static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
424 {
425  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
426  OutputLoggerThreadStore *store = op_thread_data->store;
427  OutputFiledataLogger *logger = list;
428 
429  while (logger && store) {
430  if (logger->ThreadExitPrintStats) {
431  logger->ThreadExitPrintStats(tv, store->thread_data);
432  }
433 
434  logger = logger->next;
435  store = store->next;
436  }
437 }
438 
440 {
441  OutputRegisterRootLogger(OutputFiledataLogThreadInit,
442  OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
443  OutputFiledataLog);
444  SC_ATOMIC_INIT(g_file_store_id);
445 }
446 
448 {
449  OutputFiledataLogger *logger = list;
450  while (logger) {
451  OutputFiledataLogger *next_logger = logger->next;
452  SCFree(logger);
453  logger = next_logger;
454  }
455 
456  list = NULL;
457 }
#define OUTPUT_FILEDATA_FLAG_CLOSE
void OutputFiledataLoggerRegister(void)
#define SCMutex
uint16_t flags
#define PACKET_PROFILING_LOGGER_START(p, id)
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
#define BUG_ON(x)
int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
LoggerId
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:277
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
OutputLoggerThreadStore * store
Definition: output-file.c:44
#define FILE_STORE
Definition: util-file.h:46
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
struct OutputFiledataLogger_ OutputFiledataLogger
struct File_ * next
Definition: util-file.h:79
ThreadDeinitFunc ThreadDeinit
#define SCMutexLock(mut)
StreamingBuffer * sb
Definition: util-file.h:68
struct OutputFiledataLogger_ * next
void OutputFiledataShutdown(void)
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
FiledataLogger LogFunc
void * alstate
Definition: flow.h:433
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define SCMutexUnlock(mut)
uint16_t flags
Definition: util-file.h:65
uint8_t proto
Definition: decode.h:428
#define SCMUTEX_INITIALIZER
struct OutputLoggerThreadStore_ * next
Definition: output-file.c:38
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc)
Definition: output.c:999
uint8_t flowflags
Definition: decode.h:437
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id)
File * head
Definition: util-file.h:99
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:200
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:39
ThreadExitPrintStatsFunc ThreadExitPrintStats
int(* FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir)
#define SCReturnInt(x)
Definition: util-debug.h:341
void FilePrune(FileContainer *ffc)
Definition: util-file.c:345
int FileForceMagic(void)
Definition: util-file.c:130
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:166
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:228
#define SC_ATOMIC_CAS(name, cmpval, newval)
atomic Compare and Switch
Definition: util-atomic.h:222
uint64_t ts
struct OutputLoggerThreadStore_ OutputLoggerThreadStore
#define STREAM_TOSERVER
Definition: stream.h:31
FileState state
Definition: util-file.h:67
struct OutputLoggerThreadData_ OutputLoggerThreadData
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
uint32_t file_store_id
Definition: util-file.h:72
uint64_t content_stored
Definition: util-file.h:90
ThreadInitFunc ThreadInit
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define FILE_TRUNCATED
Definition: util-file.h:36
#define SCReturn
Definition: util-debug.h:339
Per thread variable structure.
Definition: threadvars.h:57
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
#define FLOW_PKT_TOCLIENT
Definition: flow.h:201
AppProto alproto
application level protocol
Definition: flow.h:404
#define PACKET_PROFILING_LOGGER_END(p, id)
uint32_t flags
Definition: decode.h:441
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
#define OUTPUT_FILEDATA_FLAG_OPEN
int FileCloseFilePtr(File *ff, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:859
Flow data structure.
Definition: flow.h:324