suricata
output-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer File Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-file.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "util-profiling.h"
34 #include "util-validate.h"
35 #include "util-magic.h"
36 
37 bool g_file_logger_enabled = false;
38 
39 typedef struct OutputLoggerThreadStore_ {
40  void *thread_data;
43 
44 /** per thread data for this module, contains a list of per thread
45  * data for the packet loggers. */
46 typedef struct OutputLoggerThreadData_ {
48 #ifdef HAVE_MAGIC
49  magic_t magic_ctx;
50 #endif
52 
53 /* logger instance, a module + a output ctx,
54  * it's perfectly valid that have multiple instances of the same
55  * log module (e.g. http.log) with different output ctx'. */
56 typedef struct OutputFileLogger_ {
60  const char *name;
66 
67 static OutputFileLogger *list = NULL;
68 
73 {
74  OutputFileLogger *op = SCMalloc(sizeof(*op));
75  if (op == NULL)
76  return -1;
77  memset(op, 0x00, sizeof(*op));
78 
79  op->LogFunc = LogFunc;
80  op->output_ctx = output_ctx;
81  op->name = name;
82  op->logger_id = id;
83  op->ThreadInit = ThreadInit;
86 
87  if (list == NULL)
88  list = op;
89  else {
90  OutputFileLogger *t = list;
91  while (t->next)
92  t = t->next;
93  t->next = op;
94  }
95 
96  SCLogDebug("OutputRegisterFileLogger happy");
97 
98  g_file_logger_enabled = true;
99  return 0;
100 }
101 
102 static void CloseFile(const Packet *p, Flow *f, File *file)
103 {
104  void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, file->txid);
105  if (txv) {
106  AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
107  if (txd)
108  txd->files_logged++;
109  }
110  file->flags |= FILE_LOGGED;
111 }
112 
113 static void OutputFileLogFfc(ThreadVars *tv,
114  OutputLoggerThreadData *op_thread_data,
115  Packet *p,
116  FileContainer *ffc, const bool file_close, const bool file_trunc,
117  uint8_t dir)
118 {
119  SCLogDebug("ffc %p", ffc);
120  if (ffc != NULL) {
121  File *ff;
122  for (ff = ffc->head; ff != NULL; ff = ff->next) {
123  if (ff->flags & FILE_LOGGED)
124  continue;
125 
126  SCLogDebug("ff %p", ff);
127 
128  if (file_trunc && ff->state < FILE_STATE_CLOSED)
130 
131  if (file_close && ff->state < FILE_STATE_CLOSED)
133 
134  SCLogDebug("ff %p state %u", ff, ff->state);
135 
136  if (ff->state > FILE_STATE_OPENED) {
137  bool file_logged = false;
138 #ifdef HAVE_MAGIC
139  if (FileForceMagic() && ff->magic == NULL) {
140  FilemagicThreadLookup(&op_thread_data->magic_ctx, ff);
141  }
142 #endif
143  const OutputFileLogger *logger = list;
144  const OutputLoggerThreadStore *store = op_thread_data->store;
145  while (logger && store) {
146  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
147 
148  SCLogDebug("logger %p", logger);
150  logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, dir);
152  file_logged = true;
153 
154  logger = logger->next;
155  store = store->next;
156 
157  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
158  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
159  }
160 
161  if (file_logged) {
162  CloseFile(p, p->flow, ff);
163  }
164  }
165  }
166  }
167 }
168 
169 static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
170 {
171  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
172 
173  if (list == NULL) {
174  /* No child loggers. */
175  return TM_ECODE_OK;
176  }
177 
178  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
179 
180  /* no flow, no files */
181  Flow * const f = p->flow;
182  if (f == NULL || f->alstate == NULL) {
184  }
185 
186  if (p->proto == IPPROTO_TCP) {
187  const bool file_trunc = StreamTcpReassembleDepthReached(p);
188  if (p->flowflags & FLOW_PKT_TOSERVER) {
189  const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
191  OutputFileLogFfc(
192  tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
193  } else {
194  const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
196  OutputFileLogFfc(
197  tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
198  }
199  } else if (p->proto == IPPROTO_UDP) {
201  OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, false, false, STREAM_TOSERVER);
203  OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, false, false, STREAM_TOCLIENT);
204  }
205  return TM_ECODE_OK;
206 }
207 
208 /** \brief thread init for the tx logger
209  * This will run the thread init functions for the individual registered
210  * loggers */
211 static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
212 {
213  OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
214  if (td == NULL)
215  return TM_ECODE_FAILED;
216  memset(td, 0x00, sizeof(*td));
217 
218  *data = (void *)td;
219 
220 #ifdef HAVE_MAGIC
221  td->magic_ctx = MagicInitContext();
222  if (td->magic_ctx == NULL) {
223  SCFree(td);
224  return TM_ECODE_FAILED;
225  }
226 #endif
227 
228  SCLogDebug("OutputFileLogThreadInit happy (*data %p)", *data);
229 
230  OutputFileLogger *logger = list;
231  while (logger) {
232  if (logger->ThreadInit) {
233  void *retptr = NULL;
234  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
235  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
236 /* todo */ BUG_ON(ts == NULL);
237  memset(ts, 0x00, sizeof(*ts));
238 
239  /* store thread handle */
240  ts->thread_data = retptr;
241 
242  if (td->store == NULL) {
243  td->store = ts;
244  } else {
245  OutputLoggerThreadStore *tmp = td->store;
246  while (tmp->next != NULL)
247  tmp = tmp->next;
248  tmp->next = ts;
249  }
250 
251  SCLogDebug("%s is now set up", logger->name);
252  }
253  }
254 
255  logger = logger->next;
256  }
257 
258  return TM_ECODE_OK;
259 }
260 
261 static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data)
262 {
263  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
264  OutputLoggerThreadStore *store = op_thread_data->store;
265  OutputFileLogger *logger = list;
266 
267  while (logger && store) {
268  if (logger->ThreadDeinit) {
269  logger->ThreadDeinit(tv, store->thread_data);
270  }
271 
272  OutputLoggerThreadStore *next_store = store->next;
273  SCFree(store);
274  store = next_store;
275  logger = logger->next;
276  }
277 
278 #ifdef HAVE_MAGIC
279  MagicDeinitContext(op_thread_data->magic_ctx);
280 #endif
281 
282  SCFree(op_thread_data);
283  return TM_ECODE_OK;
284 }
285 
286 static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data)
287 {
288  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
289  OutputLoggerThreadStore *store = op_thread_data->store;
290  OutputFileLogger *logger = list;
291 
292  while (logger && store) {
293  if (logger->ThreadExitPrintStats) {
294  logger->ThreadExitPrintStats(tv, store->thread_data);
295  }
296 
297  logger = logger->next;
298  store = store->next;
299  }
300 }
301 
302 static uint32_t OutputFileLoggerGetActiveCount(void)
303 {
304  uint32_t cnt = 0;
305  for (OutputFileLogger *p = list; p != NULL; p = p->next) {
306  cnt++;
307  }
308  return cnt;
309 }
310 
312 {
313  OutputRegisterRootLogger(OutputFileLogThreadInit,
314  OutputFileLogThreadDeinit, OutputFileLogExitPrintStats,
315  OutputFileLog, OutputFileLoggerGetActiveCount);
316 }
317 
319 {
320  OutputFileLogger *logger = list;
321  while (logger) {
322  OutputFileLogger *next_logger = logger->next;
323  SCFree(logger);
324  logger = next_logger;
325  }
326 
327  list = NULL;
328 }
Packet_::proto
uint8_t proto
Definition: decode.h:436
FileContainer_
Definition: util-file.h:107
ts
uint64_t ts
Definition: source-erf-file.c:54
OutputLoggerThreadStore_
Definition: output-file.c:39
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output-file.c:41
OutputFileLogger_::logger_id
LoggerId logger_id
Definition: output-file.c:61
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:493
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
OutputFileLoggerRegister
void OutputFileLoggerRegister(void)
Definition: output-file.c:311
OutputRegisterFileLogger
int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output-file.c:69
OutputFileLogger
struct OutputFileLogger_ OutputFileLogger
Packet_::flags
uint32_t flags
Definition: decode.h:449
FILE_STATE_OPENED
@ FILE_STATE_OPENED
Definition: util-file.h:63
Flow_
Flow data structure.
Definition: flow.h:353
File_::state
FileState state
Definition: util-file.h:75
LoggerId
LoggerId
Definition: suricata-common.h:436
OutputFileLogger_::next
struct OutputFileLogger_ * next
Definition: output-file.c:59
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:225
tm-modules.h
AppLayerParserGetFiles
FileContainer * AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
Definition: app-layer-parser.c:861
FILE_STATE_TRUNCATED
@ FILE_STATE_TRUNCATED
Definition: util-file.h:66
OutputLoggerThreadData
struct OutputLoggerThreadData_ OutputLoggerThreadData
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:445
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output-file.c:40
OutputCtx_
Definition: tm-modules.h:78
OutputLoggerThreadData_
Definition: output-file.c:46
FileForceMagic
int FileForceMagic(void)
Definition: util-file.c:142
OutputFileLogger_::LogFunc
FileLogger LogFunc
Definition: output-file.c:57
output-file.h
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1154
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
detect-filemagic.h
FileContainer_::head
File * head
Definition: util-file.h:108
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
OutputFileLogger_::ThreadExitPrintStats
ThreadExitPrintStatsFunc ThreadExitPrintStats
Definition: output-file.c:64
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:277
util-profiling.h
OutputFileLogger_
Definition: output-file.c:56
Packet_
Definition: decode.h:414
OutputLoggerThreadStore
struct OutputLoggerThreadStore_ OutputLoggerThreadStore
OutputLoggerThreadData_::store
OutputLoggerThreadStore * store
Definition: output-file.c:47
OutputFileShutdown
void OutputFileShutdown(void)
Definition: output-file.c:318
util-magic.h
TmEcode
TmEcode
Definition: tm-threads-common.h:79
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1090
File_::flags
uint16_t flags
Definition: util-file.h:73
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:260
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:64
File_
Definition: util-file.h:72
Packet_::flow
struct Flow_ * flow
Definition: decode.h:451
suricata-common.h
ThreadExitPrintStatsFunc
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
File_::next
struct File_ * next
Definition: util-file.h:86
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
OutputFileLogger_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output-file.c:63
FileLogger
int(* FileLogger)(ThreadVars *, void *thread_data, const Packet *, const File *, uint8_t direction)
Definition: output-file.h:33
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1177
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
util-validate.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
Packet_::next
struct Packet_ * next
Definition: decode.h:581
FILE_LOGGED
#define FILE_LOGGED
Definition: util-file.h:53
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:486
OutputFileLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-file.c:58
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:253
OutputRegisterRootLogger
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc, OutputGetActiveCountFunc ActiveCntFunc)
Definition: output.c:962
File_::txid
uint64_t txid
Definition: util-file.h:77
g_file_logger_enabled
bool g_file_logger_enabled
Definition: output-file.c:37
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
OutputFileLogger_::name
const char * name
Definition: output-file.c:60
OutputFileLogger_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output-file.c:62
output.h
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
app-layer.h