suricata
output-tx.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer TX Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-tx.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "util-profiling.h"
33 #include "util-validate.h"
34 
35 typedef struct OutputLoggerThreadStore_ {
36  void *thread_data;
39 
40 /** per thread data for this module, contains a list of per thread
41  * data for the packet loggers. */
42 typedef struct OutputLoggerThreadData_ {
45 
46 /* logger instance, a module + a output ctx,
47  * it's perfectly valid that have multiple instances of the same
48  * log module (e.g. http.log) with different output ctx'. */
49 typedef struct OutputTxLogger_ {
55  const char *name;
57  uint32_t id;
60  TmEcode (*ThreadInit)(ThreadVars *, const void *, void **);
62  void (*ThreadExitPrintStats)(ThreadVars *, void *);
64 
65 static OutputTxLogger *list[ALPROTO_MAX] = { NULL };
66 
73  void (*ThreadExitPrintStats)(ThreadVars *, void *))
74 {
76  SCLogNotice("%s logger not enabled: protocol %s is disabled",
78  return -1;
79  }
80  OutputTxLogger *op = SCMalloc(sizeof(*op));
81  if (op == NULL)
82  return -1;
83  memset(op, 0x00, sizeof(*op));
84 
85  op->alproto = alproto;
86  op->LogFunc = LogFunc;
88  op->output_ctx = output_ctx;
89  op->name = name;
90  op->logger_id = id;
91  op->ThreadInit = ThreadInit;
94 
95  if (alproto == ALPROTO_UNKNOWN) {
96  op->tc_log_progress = 0;
97  } else if (tc_log_progress < 0) {
98  op->tc_log_progress =
101  } else {
103  }
104 
105  if (alproto == ALPROTO_UNKNOWN) {
106  op->ts_log_progress = 0;
107  } else if (ts_log_progress < 0) {
108  op->ts_log_progress =
111  } else {
113  }
114 
115  if (list[alproto] == NULL) {
116  op->id = 1;
117  list[alproto] = op;
118  } else {
119  OutputTxLogger *t = list[alproto];
120  while (t->next)
121  t = t->next;
122  if (t->id * 2 > UINT32_MAX) {
123  SCLogError(SC_ERR_FATAL, "Too many loggers registered.");
124  exit(EXIT_FAILURE);
125  }
126  op->id = t->id * 2;
127  t->next = op;
128  }
129 
130  SCLogDebug("OutputRegisterTxLogger happy");
131  return 0;
132 }
133 
134 static void OutputTxLogList0(ThreadVars *tv,
135  OutputLoggerThreadData *op_thread_data,
136  Packet *p, Flow *f, void *tx, const uint64_t tx_id)
137 {
138  const OutputTxLogger *logger = list[ALPROTO_UNKNOWN];
139  const OutputLoggerThreadStore *store = op_thread_data->store[ALPROTO_UNKNOWN];
140 
141  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
142  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
143  DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
144 
145  while (logger && store) {
146  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
147 
148  SCLogDebug("logger %p", logger);
149 
150  /* always invoke "wild card" tx loggers */
151  SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
153  logger->LogFunc(tv, store->thread_data, p, f, f->alstate, tx, tx_id);
155 
156  logger = logger->next;
157  store = store->next;
158 
159  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
160  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
161  }
162 }
163 
164 static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
165 {
166  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
167  if (p->flow == NULL)
168  return TM_ECODE_OK;
169 
170  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
171 
172  Flow * const f = p->flow;
173  const uint8_t ipproto = f->proto;
174  const AppProto alproto = f->alproto;
175 
176  if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
177  /* No child loggers registered. */
178  return TM_ECODE_OK;
179  }
180 
182  goto end;
183  const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(p->proto, alproto);
184  if (logger_expectation == 0)
185  goto end;
186 
187  void *alstate = f->alstate;
188  if (alstate == NULL) {
189  SCLogDebug("no alstate");
190  goto end;
191  }
192 
193  const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
194  const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
195  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
196  uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
197  uint64_t max_id = tx_id;
198  int logged = 0;
199  int gap = 0;
200 
203  memset(&state, 0, sizeof(state));
204 
205  while (1) {
206  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
207  if (ires.tx_ptr == NULL)
208  break;
209  void * const tx = ires.tx_ptr;
210  tx_id = ires.tx_id;
211 
212  if (list[ALPROTO_UNKNOWN] != 0) {
213  OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id);
214  if (list[alproto] == NULL)
215  goto next_tx;
216  }
217 
218  LoggerId tx_logged = AppLayerParserGetTxLogged(f, alstate, tx);
219  const LoggerId tx_logged_old = tx_logged;
220  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged);
221  if (tx_logged == logger_expectation) {
222  /* tx already fully logged */
223  goto next_tx;
224  }
225 
226  int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
227  tx, ts_disrupt_flags);
228  int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto,
229  tx, tc_disrupt_flags);
230  SCLogDebug("tx_progress_ts %d tx_progress_tc %d",
231  tx_progress_ts, tx_progress_tc);
232 
233  const OutputTxLogger *logger = list[alproto];
234  const OutputLoggerThreadStore *store = op_thread_data->store[alproto];
235 
236  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
237  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
238  DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
239 
240  while (logger && store) {
241  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
243 
244  SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
245  "tc_log_progress %d", logger, logger->alproto, logger->LogCondition,
246  logger->ts_log_progress, logger->tc_log_progress);
247  if ((tx_logged_old & (1<<logger->logger_id)) == 0) {
248  SCLogDebug("alproto match %d, logging tx_id %"PRIu64, logger->alproto, tx_id);
249 
252  if (logger->LogCondition) {
253  int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
254  if (r == FALSE) {
255  SCLogDebug("conditions not met, not logging");
256  goto next_logger;
257  }
258  } else {
259  if (tx_progress_tc < logger->tc_log_progress) {
260  SCLogDebug("progress not far enough, not logging");
261  goto next_logger;
262  }
263 
264  if (tx_progress_ts < logger->ts_log_progress) {
265  SCLogDebug("progress not far enough, not logging");
266  goto next_logger;
267  }
268  }
269  }
270 
271  SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
273  logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
275 
276  tx_logged |= (1<<logger->logger_id);
277  }
278 
279 next_logger:
280  logger = logger->next;
281  store = store->next;
282 
283  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
284  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
285  }
286 
287  if (tx_logged != tx_logged_old) {
288  SCLogDebug("logger: storing %08x (was %08x)",
289  tx_logged, tx_logged_old);
290  AppLayerParserSetTxLogged(p->proto, alproto, alstate, tx,
291  tx_logged);
292  }
293 
294  /* If all loggers logged set a flag and update the last tx_id
295  * that was logged.
296  *
297  * If not all loggers were logged we flag that there was a gap
298  * so any subsequent transactions in this loop don't increase
299  * the maximum ID that was logged. */
300  if (!gap && tx_logged == logger_expectation) {
301  logged = 1;
302  max_id = tx_id;
303  } else {
304  gap = 1;
305  }
306 next_tx:
307  if (!ires.has_next)
308  break;
309  tx_id++;
310  }
311 
312  /* Update the the last ID that has been logged with all
313  * transactions before it. */
314  if (logged) {
315  SCLogDebug("updating log tx_id %"PRIu64, max_id);
317  }
318 
319 end:
320  return TM_ECODE_OK;
321 }
322 
323 /** \brief thread init for the tx logger
324  * This will run the thread init functions for the individual registered
325  * loggers */
326 static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
327 {
328  OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
329  if (td == NULL)
330  return TM_ECODE_FAILED;
331  memset(td, 0x00, sizeof(*td));
332 
333  *data = (void *)td;
334  SCLogDebug("OutputTxLogThreadInit happy (*data %p)", *data);
335 
336  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
337  OutputTxLogger *logger = list[alproto];
338  while (logger) {
339  if (logger->ThreadInit) {
340  void *retptr = NULL;
341  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
342  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
343  /* todo */ BUG_ON(ts == NULL);
344  memset(ts, 0x00, sizeof(*ts));
345 
346  /* store thread handle */
347  ts->thread_data = retptr;
348 
349  if (td->store[alproto] == NULL) {
350  td->store[alproto] = ts;
351  } else {
353  while (tmp->next != NULL)
354  tmp = tmp->next;
355  tmp->next = ts;
356  }
357 
358  SCLogDebug("%s is now set up", logger->name);
359  }
360  }
361 
362  logger = logger->next;
363  }
364  }
365  return TM_ECODE_OK;
366 }
367 
368 static TmEcode OutputTxLogThreadDeinit(ThreadVars *tv, void *thread_data)
369 {
370  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
371 
372  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
373  OutputLoggerThreadStore *store = op_thread_data->store[alproto];
374  OutputTxLogger *logger = list[alproto];
375 
376  while (logger && store) {
377  if (logger->ThreadDeinit) {
378  logger->ThreadDeinit(tv, store->thread_data);
379  }
380 
381  OutputLoggerThreadStore *next_store = store->next;
382  SCFree(store);
383  store = next_store;
384  logger = logger->next;
385  }
386  }
387 
388  SCFree(op_thread_data);
389  return TM_ECODE_OK;
390 }
391 
392 static void OutputTxLogExitPrintStats(ThreadVars *tv, void *thread_data)
393 {
394  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
395 
396  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
397  OutputLoggerThreadStore *store = op_thread_data->store[alproto];
398  OutputTxLogger *logger = list[alproto];
399 
400  while (logger && store) {
401  if (logger->ThreadExitPrintStats) {
402  logger->ThreadExitPrintStats(tv, store->thread_data);
403  }
404 
405  logger = logger->next;
406  store = store->next;
407  }
408  }
409 }
410 
411 static uint32_t OutputTxLoggerGetActiveCount(void)
412 {
413  uint32_t cnt = 0;
414  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
415  for (OutputTxLogger *p = list[alproto]; p != NULL; p = p->next) {
416  cnt++;
417  }
418  }
419  return cnt;
420 }
421 
422 
424 {
425  OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit,
426  OutputTxLogExitPrintStats, OutputTxLog, OutputTxLoggerGetActiveCount);
427 }
428 
430 {
431  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
432  OutputTxLogger *logger = list[alproto];
433  while (logger) {
434  OutputTxLogger *next_logger = logger->next;
435  SCFree(logger);
436  logger = next_logger;
437  }
438  list[alproto] = NULL;
439  }
440 }
AppLayerParserSetTxLogged
void AppLayerParserSetTxLogged(uint8_t ipproto, AppProto alproto, void *alstate, void *tx, LoggerId logger)
Definition: app-layer-parser.c:686
OutputTxLogger_::alproto
AppProto alproto
Definition: output-tx.c:50
Packet_::proto
uint8_t proto
Definition: decode.h:433
output-tx.h
ts
uint64_t ts
Definition: source-erf-file.c:54
AppLayerParserGetTxLogged
LoggerId AppLayerParserGetTxLogged(const Flow *f, void *alstate, void *tx)
Definition: app-layer-parser.c:700
OutputLoggerThreadStore_
Definition: output-file.c:37
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output-file.c:39
AppLayerParserIsEnabled
int AppLayerParserIsEnabled(AppProto alproto)
simple way to globally test if a alproto is registered and fully enabled in the configuration.
Definition: app-layer-parser.c:1368
OutputLoggerThreadData
struct OutputLoggerThreadData_ OutputLoggerThreadData
OutputTxLoggerRegister
void OutputTxLoggerRegister(void)
Definition: output-tx.c:423
AppLayerParserProtocolHasLogger
int AppLayerParserProtocolHasLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1382
AppLayerParserSetTransactionLogId
void AppLayerParserSetTransactionLogId(AppLayerParserState *pstate, uint64_t tx_id)
Definition: app-layer-parser.c:721
OutputTxLogger_::tc_log_progress
int tc_log_progress
Definition: output-tx.c:58
OutputTxShutdown
void OutputTxShutdown(void)
Definition: output-tx.c:429
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
AppLayerGetTxIterator
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
Definition: app-layer-parser.c:678
Flow_::proto
uint8_t proto
Definition: flow.h:361
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:71
AppLayerParserGetStateProgress
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
Definition: app-layer-parser.c:1013
Flow_
Flow data structure.
Definition: flow.h:343
LoggerId
LoggerId
Definition: suricata-common.h:439
AppLayerParserGetTransactionLogId
uint64_t AppLayerParserGetTransactionLogId(AppLayerParserState *pstate)
Definition: app-layer-parser.c:714
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
logged
int logged
Definition: app-layer-htp.h:1
AppLayerParserGetStateProgressCompletionStatus
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
Definition: app-layer-parser.c:1046
tm-modules.h
TxLogger
int(* TxLogger)(ThreadVars *, void *thread_data, const Packet *, Flow *f, void *state, void *tx, uint64_t tx_id)
Definition: output-tx.h:32
OutputTxLogger_::ThreadInit
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: output-tx.c:60
OutputTxLogger_::logger_id
LoggerId logger_id
Definition: output-tx.c:56
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:79
ALPROTO_MAX
@ ALPROTO_MAX
Definition: app-layer-protos.h:66
OutputTxLogger_::ThreadDeinit
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: output-tx.c:61
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:78
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output-file.c:38
OutputCtx_
Definition: tm-modules.h:78
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:453
OutputRegisterTxLogger
int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *output_ctx, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, void(*ThreadExitPrintStats)(ThreadVars *, void *))
Definition: output-tx.c:67
AppLayerParserStateIssetFlag
int AppLayerParserStateIssetFlag(AppLayerParserState *pstate, uint8_t flag)
Definition: app-layer-parser.c:1609
OutputLoggerThreadData_
Definition: output-file.c:44
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
OutputTxLogger_::next
struct OutputTxLogger_ * next
Definition: output-tx.c:54
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
util-profiling.h
FALSE
#define FALSE
Definition: suricata-common.h:34
OutputTxLogger_::ts_log_progress
int ts_log_progress
Definition: output-tx.c:59
AppLayerGetTxIterState
Definition: app-layer-parser.h:114
Packet_
Definition: decode.h:411
OutputLoggerThreadData_::store
OutputLoggerThreadStore * store
Definition: output-file.c:45
TmEcode
TmEcode
Definition: tm-threads-common.h:77
OutputTxLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-tx.c:53
OutputTxLogger_::LogCondition
TxLoggerCondition LogCondition
Definition: output-tx.c:52
APP_LAYER_PARSER_EOF
#define APP_LAYER_PARSER_EOF
Definition: app-layer-parser.h:35
OutputTxLogger_::ThreadExitPrintStats
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: output-tx.c:62
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:260
Packet_::flow
struct Flow_ * flow
Definition: decode.h:448
suricata-common.h
OutputTxLogger
struct OutputTxLogger_ OutputTxLogger
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
util-validate.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
Packet_::next
struct Packet_ * next
Definition: decode.h:573
TxLoggerCondition
int(* TxLoggerCondition)(ThreadVars *, const Packet *, void *state, void *tx, uint64_t tx_id)
Definition: output-tx.h:37
OutputTxLogger_::LogFunc
TxLogger LogFunc
Definition: output-tx.c:51
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:454
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
OutputTxLogger_
Definition: output-tx.c:49
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:253
OutputRegisterRootLogger
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc, OutputGetActiveCountFunc ActiveCntFunc)
Definition: output.c:1011
OutputLoggerThreadStore
struct OutputLoggerThreadStore_ OutputLoggerThreadStore
OutputTxLogger_::name
const char * name
Definition: output-tx.c:55
AppLayerParserProtocolGetLoggerBits
LoggerId AppLayerParserProtocolGetLoggerBits(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1390
AppLayerGetTxIteratorFunc
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype
Definition: app-layer-parser.h:123
FlowGetDisruptionFlags
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get 'disruption' flags: GAP/DEPTH/PASS
Definition: flow.c:1093
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:232
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:425
OutputTxLogger_::id
uint32_t id
Definition: output-tx.c:57
AppLayerParserGetTxCnt
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
Definition: app-layer-parser.c:1028
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
output.h
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
app-layer.h