suricata
output-tx.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer TX Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-tx.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "util-profiling.h"
33 #include "util-validate.h"
34 
35 typedef struct OutputLoggerThreadStore_ {
36  void *thread_data;
39 
40 /** per thread data for this module, contains a list of per thread
41  * data for the packet loggers. */
42 typedef struct OutputLoggerThreadData_ {
45 
46 /* logger instance, a module + a output ctx,
47  * it's perfectly valid that have multiple instances of the same
48  * log module (e.g. http.log) with different output ctx'. */
49 typedef struct OutputTxLogger_ {
55  const char *name;
57  uint32_t id;
60  TmEcode (*ThreadInit)(ThreadVars *, const void *, void **);
62  void (*ThreadExitPrintStats)(ThreadVars *, void *);
64 
65 static OutputTxLogger *list = NULL;
66 
73  void (*ThreadExitPrintStats)(ThreadVars *, void *))
74 {
75  if (alproto != ALPROTO_UNKNOWN && !(AppLayerParserIsTxAware(alproto))) {
76  SCLogNotice("%s logger not enabled: protocol %s is disabled",
77  name, AppProtoToString(alproto));
78  return -1;
79  }
80 
81  OutputTxLogger *op = SCMalloc(sizeof(*op));
82  if (op == NULL)
83  return -1;
84  memset(op, 0x00, sizeof(*op));
85 
86  op->alproto = alproto;
87  op->LogFunc = LogFunc;
89  op->output_ctx = output_ctx;
90  op->name = name;
91  op->logger_id = id;
92  op->ThreadInit = ThreadInit;
95 
96  if (alproto == ALPROTO_UNKNOWN) {
97  op->tc_log_progress = 0;
98  } else if (tc_log_progress < 0) {
99  op->tc_log_progress =
102  } else {
104  }
105 
106  if (alproto == ALPROTO_UNKNOWN) {
107  op->ts_log_progress = 0;
108  } else if (ts_log_progress < 0) {
109  op->ts_log_progress =
112  } else {
114  }
115 
116  if (list == NULL) {
117  op->id = 1;
118  list = op;
119  } else {
120  OutputTxLogger *t = list;
121  while (t->next)
122  t = t->next;
123  if (t->id * 2 > UINT32_MAX) {
124  SCLogError(SC_ERR_FATAL, "Too many loggers registered.");
125  exit(EXIT_FAILURE);
126  }
127  op->id = t->id * 2;
128  t->next = op;
129  }
130 
131  SCLogDebug("OutputRegisterTxLogger happy");
132  return 0;
133 }
134 
135 static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
136 {
137  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
138  if (list == NULL) {
139  /* No child loggers registered. */
140  return TM_ECODE_OK;
141  }
142 
143  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
144  if (p->flow == NULL)
145  return TM_ECODE_OK;
146 
147  Flow * const f = p->flow;
148  const uint8_t ipproto = f->proto;
149  const AppProto alproto = f->alproto;
150 
151  if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
152  goto end;
153  const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(p->proto, alproto);
154  if (logger_expectation == 0)
155  goto end;
156 
157  void *alstate = f->alstate;
158  if (alstate == NULL) {
159  SCLogDebug("no alstate");
160  goto end;
161  }
162 
163  const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
164  const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
165  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
167  uint64_t max_id = tx_id;
168  int logged = 0;
169  int gap = 0;
170 
171  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
173  memset(&state, 0, sizeof(state));
174 
175  while (1) {
176  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
177  if (ires.tx_ptr == NULL)
178  break;
179  void * const tx = ires.tx_ptr;
180  tx_id = ires.tx_id;
181 
182  LoggerId tx_logged = AppLayerParserGetTxLogged(f, alstate, tx);
183  const LoggerId tx_logged_old = tx_logged;
184  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged);
185  if (tx_logged == logger_expectation) {
186  /* tx already fully logged */
187  goto next_tx;
188  }
189 
190  int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
191  tx, ts_disrupt_flags);
192  int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto,
193  tx, tc_disrupt_flags);
194  SCLogDebug("tx_progress_ts %d tx_progress_tc %d",
195  tx_progress_ts, tx_progress_tc);
196 
197  const OutputTxLogger *logger = list;
198  const OutputLoggerThreadStore *store = op_thread_data->store;
199 
200  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
201  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
202  DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
203 
204  while (logger && store) {
205  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
206 
207  SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
208  "tc_log_progress %d", logger, logger->alproto, logger->LogCondition,
209  logger->ts_log_progress, logger->tc_log_progress);
210  /* always invoke "wild card" tx loggers */
211  if (logger->alproto == ALPROTO_UNKNOWN ||
212  (logger->alproto == alproto &&
213  (tx_logged_old & (1<<logger->logger_id)) == 0)) {
214 
215  SCLogDebug("alproto match %d, logging tx_id %"PRIu64, logger->alproto, tx_id);
216 
219  if (logger->LogCondition) {
220  int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
221  if (r == FALSE) {
222  SCLogDebug("conditions not met, not logging");
223  goto next_logger;
224  }
225  } else {
226  if (tx_progress_tc < logger->tc_log_progress) {
227  SCLogDebug("progress not far enough, not logging");
228  goto next_logger;
229  }
230 
231  if (tx_progress_ts < logger->ts_log_progress) {
232  SCLogDebug("progress not far enough, not logging");
233  goto next_logger;
234  }
235  }
236  }
237 
238  SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
240  logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
242 
243  if (logger->alproto != ALPROTO_UNKNOWN) {
244  tx_logged |= (1<<logger->logger_id);
245  }
246  }
247 
248 next_logger:
249  logger = logger->next;
250  store = store->next;
251 
252  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
253  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
254  }
255 
256  if (tx_logged != tx_logged_old) {
257  SCLogDebug("logger: storing %08x (was %08x)",
258  tx_logged, tx_logged_old);
259  AppLayerParserSetTxLogged(p->proto, alproto, alstate, tx,
260  tx_logged);
261  }
262 
263  /* If all loggers logged set a flag and update the last tx_id
264  * that was logged.
265  *
266  * If not all loggers were logged we flag that there was a gap
267  * so any subsequent transactions in this loop don't increase
268  * the maximum ID that was logged. */
269  if (!gap && tx_logged == logger_expectation) {
270  logged = 1;
271  max_id = tx_id;
272  } else {
273  gap = 1;
274  }
275 next_tx:
276  if (!ires.has_next)
277  break;
278  tx_id++;
279  }
280 
281  /* Update the the last ID that has been logged with all
282  * transactions before it. */
283  if (logged) {
284  SCLogDebug("updating log tx_id %"PRIu64, max_id);
286  }
287 
288 end:
289  return TM_ECODE_OK;
290 }
291 
292 /** \brief thread init for the tx logger
293  * This will run the thread init functions for the individual registered
294  * loggers */
295 static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
296 {
297  OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
298  if (td == NULL)
299  return TM_ECODE_FAILED;
300  memset(td, 0x00, sizeof(*td));
301 
302  *data = (void *)td;
303  SCLogDebug("OutputTxLogThreadInit happy (*data %p)", *data);
304 
305  OutputTxLogger *logger = list;
306  while (logger) {
307  if (logger->ThreadInit) {
308  void *retptr = NULL;
309  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
310  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
311 /* todo */ BUG_ON(ts == NULL);
312  memset(ts, 0x00, sizeof(*ts));
313 
314  /* store thread handle */
315  ts->thread_data = retptr;
316 
317  if (td->store == NULL) {
318  td->store = ts;
319  } else {
320  OutputLoggerThreadStore *tmp = td->store;
321  while (tmp->next != NULL)
322  tmp = tmp->next;
323  tmp->next = ts;
324  }
325 
326  SCLogDebug("%s is now set up", logger->name);
327  }
328  }
329 
330  logger = logger->next;
331  }
332 
333  return TM_ECODE_OK;
334 }
335 
336 static TmEcode OutputTxLogThreadDeinit(ThreadVars *tv, void *thread_data)
337 {
338  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
339  OutputLoggerThreadStore *store = op_thread_data->store;
340  OutputTxLogger *logger = list;
341 
342  while (logger && store) {
343  if (logger->ThreadDeinit) {
344  logger->ThreadDeinit(tv, store->thread_data);
345  }
346 
347  OutputLoggerThreadStore *next_store = store->next;
348  SCFree(store);
349  store = next_store;
350  logger = logger->next;
351  }
352 
353  SCFree(op_thread_data);
354  return TM_ECODE_OK;
355 }
356 
357 static void OutputTxLogExitPrintStats(ThreadVars *tv, void *thread_data)
358 {
359  OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
360  OutputLoggerThreadStore *store = op_thread_data->store;
361  OutputTxLogger *logger = list;
362 
363  while (logger && store) {
364  if (logger->ThreadExitPrintStats) {
365  logger->ThreadExitPrintStats(tv, store->thread_data);
366  }
367 
368  logger = logger->next;
369  store = store->next;
370  }
371 }
372 
374 {
375  OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit,
376  OutputTxLogExitPrintStats, OutputTxLog);
377 }
378 
380 {
381  OutputTxLogger *logger = list;
382  while (logger) {
383  OutputTxLogger *next_logger = logger->next;
384  SCFree(logger);
385  logger = next_logger;
386  }
387  list = NULL;
388 }
LoggerId AppLayerParserGetTxLogged(const Flow *f, void *alstate, void *tx)
#define PACKET_PROFILING_LOGGER_START(p, id)
struct OutputTxLogger_ OutputTxLogger
int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *output_ctx, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, void(*ThreadExitPrintStats)(ThreadVars *, void *))
Definition: output-tx.c:67
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:445
int AppLayerParserProtocolHasLogger(uint8_t ipproto, AppProto alproto)
#define BUG_ON(x)
void AppLayerParserSetTransactionLogId(AppLayerParserState *pstate, uint64_t tx_id)
uint8_t proto
Definition: flow.h:344
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
int AppLayerParserIsTxAware(AppProto alproto)
simpler way to globally test if a alproto is registered and fully enabled in the configuration.
#define FALSE
struct OutputLoggerThreadStore_ OutputLoggerThreadStore
LoggerId
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
int logged
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
int AppLayerParserStateIssetFlag(AppLayerParserState *pstate, uint8_t flag)
OutputLoggerThreadStore * store
Definition: output-file.c:44
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
uint16_t AppProto
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: output-tx.c:61
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get &#39;disruption&#39; flags: GAP/DEPTH/PASS
Definition: flow.c:1077
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: output-tx.c:60
LoggerId logger_id
Definition: output-tx.c:56
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
void * alstate
Definition: flow.h:438
uint64_t AppLayerParserGetTransactionLogId(AppLayerParserState *pstate)
uint8_t proto
Definition: decode.h:430
OutputCtx * output_ctx
Definition: output-tx.c:53
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
struct OutputLoggerThreadStore_ * next
Definition: output-file.c:38
int ts_log_progress
Definition: output-tx.c:59
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
struct OutputLoggerThreadData_ OutputLoggerThreadData
int(* TxLogger)(ThreadVars *, void *thread_data, const Packet *, Flow *f, void *state, void *tx, uint64_t tx_id)
Definition: output-tx.h:32
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc)
Definition: output.c:1002
#define STREAM_TOCLIENT
Definition: stream.h:32
void OutputTxLoggerRegister(void)
Definition: output-tx.c:373
struct OutputTxLogger_ * next
Definition: output-tx.c:54
void OutputTxShutdown(void)
Definition: output-tx.c:379
#define SCMalloc(a)
Definition: util-mem.h:222
#define SCFree(a)
Definition: util-mem.h:322
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:269
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: output-tx.c:62
uint16_t tx_id
TxLogger LogFunc
Definition: output-tx.c:51
TxLoggerCondition LogCondition
Definition: output-tx.c:52
uint64_t ts
#define STREAM_TOSERVER
Definition: stream.h:31
int(* TxLoggerCondition)(ThreadVars *, const Packet *, void *state, void *tx, uint64_t tx_id)
Definition: output-tx.h:37
void AppLayerParserSetTxLogged(uint8_t ipproto, AppProto alproto, void *alstate, void *tx, LoggerId logger)
#define APP_LAYER_PARSER_EOF
Per thread variable structure.
Definition: threadvars.h:57
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
AppProto alproto
application level protocol
Definition: flow.h:409
AppProto alproto
Definition: output-tx.c:50
LoggerId AppLayerParserProtocolGetLoggerBits(uint8_t ipproto, AppProto alproto)
const char * name
Definition: output-tx.c:55
#define PACKET_PROFILING_LOGGER_END(p, id)
uint32_t id
Definition: output-tx.c:57
Flow data structure.
Definition: flow.h:325
AppLayerParserState * alparser
Definition: flow.h:437
int tc_log_progress
Definition: output-tx.c:58
#define DEBUG_VALIDATE_BUG_ON(exp)
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype