suricata
output-tx.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer TX Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "output.h"
28 #include "output-tx.h"
29 #include "stream.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "util-profiling.h"
33 #include "util-validate.h"
34 
35 /** per thread data for this module, contains a list of per thread
36  * data for the packet loggers. */
37 typedef struct OutputTxLoggerThreadData_ {
39 
40  /* thread local data from file api */
42  /* thread local data from filedata api */
45 
46 /* logger instance, a module + a output ctx,
47  * it's perfectly valid that have multiple instances of the same
48  * log module (e.g. http.log) with different output ctx'. */
49 typedef struct OutputTxLogger_ {
55  const char *name;
57  uint32_t id;
60  TmEcode (*ThreadInit)(ThreadVars *, const void *, void **);
62  void (*ThreadExitPrintStats)(ThreadVars *, void *);
64 
65 static OutputTxLogger *list[ALPROTO_MAX] = { NULL };
66 
73  void (*ThreadExitPrintStats)(ThreadVars *, void *))
74 {
76  SCLogDebug(
77  "%s logger not enabled: protocol %s is disabled", name, AppProtoToString(alproto));
78  return -1;
79  }
80  OutputTxLogger *op = SCMalloc(sizeof(*op));
81  if (op == NULL)
82  return -1;
83  memset(op, 0x00, sizeof(*op));
84 
85  op->alproto = alproto;
86  op->LogFunc = LogFunc;
88  op->output_ctx = output_ctx;
89  op->name = name;
90  op->logger_id = id;
91  op->ThreadInit = ThreadInit;
94 
95  if (alproto == ALPROTO_UNKNOWN) {
96  op->tc_log_progress = 0;
97  } else if (tc_log_progress < 0) {
98  op->tc_log_progress =
100  STREAM_TOCLIENT);
101  } else {
103  }
104 
105  if (alproto == ALPROTO_UNKNOWN) {
106  op->ts_log_progress = 0;
107  } else if (ts_log_progress < 0) {
108  op->ts_log_progress =
110  STREAM_TOSERVER);
111  } else {
113  }
114 
115  if (list[alproto] == NULL) {
116  op->id = 1;
117  list[alproto] = op;
118  } else {
119  OutputTxLogger *t = list[alproto];
120  while (t->next)
121  t = t->next;
122  if (t->id * 2 > UINT32_MAX) {
123  FatalError(SC_ERR_FATAL, "Too many loggers registered.");
124  }
125  op->id = t->id * 2;
126  t->next = op;
127  }
128 
129  SCLogDebug("OutputRegisterTxLogger happy");
130  return 0;
131 }
132 
133 extern bool g_file_logger_enabled;
134 extern bool g_filedata_logger_enabled;
135 
136 /** \brief run the per tx file logging
137  * \todo clean up various end of tx/stream etc indicators
138  */
139 static inline void OutputTxLogFiles(ThreadVars *tv, OutputFileLoggerThreadData *file_td,
140  OutputFiledataLoggerThreadData *filedata_td, Packet *p, Flow *f, void *tx,
141  const uint64_t tx_id, AppLayerTxData *txd, const bool tx_complete, const bool ts_ready,
142  const bool tc_ready, const bool ts_eof, const bool tc_eof, const bool eof)
143 {
144  uint8_t packet_dir;
145  uint8_t opposing_dir;
146  bool packet_dir_ready;
147  const bool opposing_dir_ready = eof;
148  bool opposing_tx_ready;
149  if (p->flowflags & FLOW_PKT_TOSERVER) {
150  packet_dir = STREAM_TOSERVER;
151  opposing_dir = STREAM_TOCLIENT;
152  packet_dir_ready = eof | ts_ready | ts_eof;
153  opposing_tx_ready = tc_ready;
154  } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
155  packet_dir = STREAM_TOCLIENT;
156  opposing_dir = STREAM_TOSERVER;
157  packet_dir_ready = eof | tc_ready | tc_eof;
158  opposing_tx_ready = ts_ready;
159  } else {
160  abort();
161  }
162 
163  SCLogDebug("eof %d ts_ready %d ts_eof %d", eof, ts_ready, ts_eof);
164  SCLogDebug("eof %d tc_ready %d tc_eof %d", eof, tc_ready, tc_eof);
165 
166  SCLogDebug("packet dir %s opposing %s packet_dir_ready %d opposing_dir_ready %d",
167  packet_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT",
168  opposing_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT", packet_dir_ready,
169  opposing_dir_ready);
170 
171  FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, packet_dir);
172  FileContainer *ffc_opposing = AppLayerParserGetTxFiles(f, tx, opposing_dir);
173 
174  /* see if opposing side is finished: if no file support in this direction, of is not
175  * files and tx is done for opposing dir. */
176  bool opposing_finished =
177  ffc_opposing == NULL || (ffc_opposing->head == NULL && opposing_tx_ready);
178  SCLogDebug("opposing_finished %d ffc_opposing %p ffc_opposing->head %p opposing_tx_ready %d",
179  opposing_finished, ffc_opposing, ffc_opposing ? ffc_opposing->head : NULL,
180  opposing_tx_ready);
181 
182  if (ffc || ffc_opposing)
183  SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
184  " ffc %p ffc_opposing %p tx_complete %d",
185  p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
186 
187  if (ffc) {
188  const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | eof;
189  const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
190  SCLogDebug("tx: calling files: ffc %p head %p file_close %d file_trunc %d", ffc, ffc->head,
191  file_close, file_trunc);
192  if (filedata_td && txd->files_opened > txd->files_stored)
193  OutputFiledataLogFfc(tv, filedata_td, p, ffc, tx, tx_id, txd, packet_dir, file_close,
194  file_trunc, packet_dir);
195  if (file_td && txd->files_opened > txd->files_logged)
197  tv, file_td, p, ffc, tx, tx_id, txd, file_close, file_trunc, packet_dir);
198  }
199  /* if EOF and we support files, do a final write out */
200  if (opposing_dir_ready && ffc_opposing != NULL) {
201  const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | tx_complete | eof;
202  const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
203  opposing_finished = true;
204  SCLogDebug("tx: calling for opposing direction files: file_close:%s file_trunc:%s",
205  file_close ? "true" : "false", file_trunc ? "true" : "false");
206  if (filedata_td && txd->files_opened > txd->files_stored)
207  OutputFiledataLogFfc(tv, filedata_td, p, ffc_opposing, tx, tx_id, txd, opposing_dir,
208  file_close, file_trunc, opposing_dir);
209  if (file_td && txd->files_opened > txd->files_logged)
210  OutputFileLogFfc(tv, file_td, p, ffc_opposing, tx, tx_id, txd, file_close, file_trunc,
211  opposing_dir);
212  }
213 
214  const bool tx_done = packet_dir_ready && opposing_finished;
215  SCLogDebug("tx_done %d packet_dir_ready %d opposing_finished %d", tx_done, packet_dir_ready,
216  opposing_finished);
217 
218  /* if not a file tx or if tx is done, set logger flags so tx can move on */
219  const bool is_file_tx = (ffc != NULL || ffc_opposing != NULL);
220  if (!is_file_tx || tx_done) {
221  SCLogDebug("is_file_tx %d tx_done %d", is_file_tx, tx_done);
222  if (file_td) {
223  txd->logged.flags |= BIT_U32(LOGGER_FILE);
224  SCLogDebug("setting LOGGER_FILE => %08x", txd->logged.flags);
225  }
226  if (filedata_td) {
227  txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
228  SCLogDebug("setting LOGGER_FILEDATA => %08x", txd->logged.flags);
229  }
230  } else {
231  SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
232  " NOT SETTING FILE FLAGS ffc %p ffc_opposing %p tx_complete %d",
233  p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
234  }
235 }
236 
237 static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data, Packet *p,
238  Flow *f, void *tx, const uint64_t tx_id)
239 {
240  const OutputTxLogger *logger = list[ALPROTO_UNKNOWN];
241  const OutputLoggerThreadStore *store = op_thread_data->store[ALPROTO_UNKNOWN];
242 
243  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
244  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
245  DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
246 
247  while (logger && store) {
248  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
249 
250  SCLogDebug("logger %p", logger);
251 
252  /* always invoke "wild card" tx loggers */
253  SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
255  logger->LogFunc(tv, store->thread_data, p, f, f->alstate, tx, tx_id);
257 
258  logger = logger->next;
259  store = store->next;
260 
261  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
262  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
263  }
264 }
265 
266 struct Ctx {
267  uint32_t tx_logged_old;
268  uint32_t tx_logged;
269 };
270 
271 static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data,
272  const OutputTxLogger *logger, const OutputLoggerThreadStore *store, Packet *p, Flow *f,
273  void *alstate, void *tx, const uint64_t tx_id, const AppProto alproto, const bool eof,
274  const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx)
275 {
276  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
277  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
278  // DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
279 
280  while (logger && store) {
281  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
282  DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto);
283 
284  SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
285  "tc_log_progress %d",
286  logger, logger->alproto, logger->LogCondition, logger->ts_log_progress,
287  logger->tc_log_progress);
288  if ((ctx->tx_logged_old & BIT_U32(logger->logger_id)) == 0) {
289  SCLogDebug("alproto match %d, logging tx_id %" PRIu64, logger->alproto, tx_id);
290 
291  SCLogDebug("pcap_cnt %" PRIu64 ", tx_id %" PRIu64 " logger %d. EOF %s", p->pcap_cnt,
292  tx_id, logger->logger_id, eof ? "true" : "false");
293 
294  if (eof) {
295  SCLogDebug("EOF, so log now");
296  } else {
297  if (logger->LogCondition) {
298  int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
299  if (r == FALSE) {
300  SCLogDebug("conditions not met, not logging");
301  goto next_logger;
302  }
303  } else {
304  if (tx_progress_tc < logger->tc_log_progress) {
305  SCLogDebug("progress not far enough, not logging");
306  goto next_logger;
307  }
308 
309  if (tx_progress_ts < logger->ts_log_progress) {
310  SCLogDebug("progress not far enough, not logging");
311  goto next_logger;
312  }
313  }
314  }
315 
316  SCLogDebug("Logging tx_id %" PRIu64 " to logger %d", tx_id, logger->logger_id);
318  logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
320 
321  ctx->tx_logged |= BIT_U32(logger->logger_id);
322  }
323 
324  next_logger:
325  logger = logger->next;
326  store = store->next;
327 
328  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
329  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
330  }
331 }
332 
333 static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
334 {
335  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
336  if (p->flow == NULL)
337  return TM_ECODE_OK;
338  if (!((PKT_IS_PSEUDOPKT(p)) || (p->flags & PKT_APPLAYER_UPDATE) != 0)) {
339  SCLogDebug("not pseudo, no app update: skip");
340  return TM_ECODE_OK;
341  }
342  SCLogDebug("pseudo, or app update: run output");
343 
344  OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;
345 
346  Flow * const f = p->flow;
347  const uint8_t ipproto = f->proto;
348  const AppProto alproto = f->alproto;
349  SCLogDebug("pcap_cnt %u tx logging %u/%s", (uint32_t)p->pcap_cnt, alproto,
350  AppProtoToString(alproto));
351 
352  const bool file_logging_active = (op_thread_data->file || op_thread_data->filedata);
353  if (!file_logging_active) {
354  if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
355  SCLogDebug("bail");
356  /* No child loggers registered. */
357  return TM_ECODE_OK;
358  }
359  if (AppLayerParserProtocolHasLogger(ipproto, alproto) == 0)
360  goto end;
361  }
362  void *alstate = f->alstate;
363  if (alstate == NULL) {
364  SCLogDebug("no alstate");
365  goto end;
366  }
367  const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(ipproto, alproto);
368  if (logger_expectation == 0) {
369  SCLogDebug("bail: logger_expectation %u. LOGGER_FILE %u LOGGER_FILEDATA %u",
370  logger_expectation, LOGGER_FILE, LOGGER_FILEDATA);
371  goto end;
372  }
373  SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
374 
375  const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0;
376  const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser,
378  const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser,
380 
381  const bool eof = last_pseudo || (ts_eof && tc_eof);
382  SCLogDebug("eof %d last_pseudo %d ts_eof %d tc_eof %d", eof, last_pseudo, ts_eof, tc_eof);
383 
384  const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
385  const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
386  SCLogDebug("ts_disrupt_flags %02x tc_disrupt_flags %02x", ts_disrupt_flags, tc_disrupt_flags);
387  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
388  uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
389  uint64_t max_id = tx_id;
390  int logged = 0;
391  int gap = 0;
392  const bool support_files = AppLayerParserSupportsFiles(ipproto, alproto);
393  const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p);
394 
395  SCLogDebug("pcap_cnt %" PRIu64 ": tx_id %" PRIu64 " total_txs %" PRIu64, p->pcap_cnt, tx_id,
396  total_txs);
397 
398  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
400  memset(&state, 0, sizeof(state));
401 
402  const int complete_ts =
403  AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER);
404  const int complete_tc =
405  AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT);
406  while (1) {
407  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
408  if (ires.tx_ptr == NULL)
409  break;
410  void * const tx = ires.tx_ptr;
411  tx_id = ires.tx_id;
412  SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx);
413 
414  const int tx_progress_ts =
415  AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags);
416  const int tx_progress_tc =
417  AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
418  const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc);
419 
420  SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
421  op_thread_data->filedata);
422 
423  AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
424  if (unlikely(txd == NULL)) {
425  SCLogDebug("NO TXD");
426  /* make sure this tx, which can't be properly logged is skipped */
427  logged = 1;
428  max_id = tx_id;
429  goto next_tx;
430  }
431 
432  if (file_logging_active) {
433  if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx,
434  // even if there are not files (yet)
435  const bool ts_ready = (tx_progress_ts == complete_ts);
436  const bool tc_ready = (tx_progress_tc == complete_tc);
437  SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready);
438 
439  const bool eval_files = ts_ready | tc_ready | tx_complete | ts_eof | tc_eof | eof;
440 
441  SCLogDebug("eval_files: %u, ts_ready %u, tc_ready %u, tx_complete %u, ts_eof %u, "
442  "tc_eof %u, eof %u",
443  eval_files, ts_ready, tc_ready, tx_complete, ts_eof, tc_eof, eof);
444  SCLogDebug("txd->file_tx & pkt_dir: %02x & %02x -> %02x", txd->file_tx, pkt_dir,
445  (txd->file_tx & pkt_dir));
446 
447  /* call only for the correct direction, except when it looks anything like a end of
448  * transaction or end of stream. Since OutputTxLogFiles has complicated logic around
449  * that, we just leave it to that function to sort things out for now. */
450  if (eval_files || AppLayerParserIsFileTxInDir(
451  txd, pkt_dir)) { // need to process each tx that might
452  // be a file tx, even if there
453  OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx,
454  tx_id, txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof);
455  }
456  } else if (support_files) {
457  if (op_thread_data->file) {
458  txd->logged.flags |= BIT_U32(LOGGER_FILE);
459  SCLogDebug("not a file_tx: setting LOGGER_FILE => %08x", txd->logged.flags);
460  }
461  if (op_thread_data->filedata) {
462  txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
463  SCLogDebug("not a file_tx: setting LOGGER_FILEDATA => %08x", txd->logged.flags);
464  }
465  }
466  }
467  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
468 
469  if (list[ALPROTO_UNKNOWN] != 0) {
470  OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id);
471  if (list[alproto] == NULL)
472  goto next_tx;
473  }
474 
475  SCLogDebug("tx %p/%" PRIu64 " txd %p: log_flags %x logger_expectation %x", tx, tx_id, txd,
476  txd->config.log_flags, logger_expectation);
477  if (txd->config.log_flags & BIT_U8(CONFIG_TYPE_TX)) {
478  SCLogDebug("SKIP tx %p/%"PRIu64, tx, tx_id);
479  goto next_tx;
480  }
481 
482  if (txd->logged.flags == logger_expectation) {
483  SCLogDebug("fully logged");
484  /* tx already fully logged */
485  goto next_tx;
486  }
487 
488  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
489  const OutputTxLogger *logger = list[alproto];
490  const OutputLoggerThreadStore *store = op_thread_data->store[alproto];
491  struct Ctx ctx = { .tx_logged = txd->logged.flags, .tx_logged_old = txd->logged.flags };
492  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
493 
494  OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, alproto,
495  eof, tx_progress_ts, tx_progress_tc, &ctx);
496 
497  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
498  if (ctx.tx_logged != ctx.tx_logged_old) {
499  SCLogDebug("logger: storing %08x (was %08x)", ctx.tx_logged, ctx.tx_logged_old);
500  DEBUG_VALIDATE_BUG_ON(txd == NULL);
501  txd->logged.flags |= ctx.tx_logged;
502  }
503 
504  /* If all loggers logged set a flag and update the last tx_id
505  * that was logged.
506  *
507  * If not all loggers were logged we flag that there was a gap
508  * so any subsequent transactions in this loop don't increase
509  * the maximum ID that was logged. */
510  if (!gap && ctx.tx_logged == logger_expectation) {
511  SCLogDebug("no gap %d, %08x == %08x", gap, ctx.tx_logged, logger_expectation);
512  logged = 1;
513  max_id = tx_id;
514  SCLogDebug("max_id %" PRIu64, max_id);
515  } else {
516  gap = 1;
517  }
518 next_tx:
519  if (!ires.has_next)
520  break;
521  tx_id++;
522  }
523 
524  /* Update the the last ID that has been logged with all
525  * transactions before it. */
526  if (logged) {
527  SCLogDebug("updating log tx_id %"PRIu64, max_id);
529  }
530 
531 end:
532  return TM_ECODE_OK;
533 }
534 
535 /** \brief thread init for the tx logger
536  * This will run the thread init functions for the individual registered
537  * loggers */
538 static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *_initdata, void **data)
539 {
540  OutputTxLoggerThreadData *td = SCMalloc(sizeof(*td));
541  if (td == NULL)
542  return TM_ECODE_FAILED;
543  memset(td, 0x00, sizeof(*td));
544 
545  *data = (void *)td;
546  SCLogDebug("OutputTxLogThreadInit happy (*data %p)", *data);
547 
548  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
549  OutputTxLogger *logger = list[alproto];
550  while (logger) {
551  if (logger->ThreadInit) {
552  void *retptr = NULL;
553  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
554  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
555  /* todo */ BUG_ON(ts == NULL);
556  memset(ts, 0x00, sizeof(*ts));
557 
558  /* store thread handle */
559  ts->thread_data = retptr;
560 
561  if (td->store[alproto] == NULL) {
562  td->store[alproto] = ts;
563  } else {
564  OutputLoggerThreadStore *tmp = td->store[alproto];
565  while (tmp->next != NULL)
566  tmp = tmp->next;
567  tmp->next = ts;
568  }
569 
570  SCLogDebug("%s is now set up", logger->name);
571  }
572  }
573 
574  logger = logger->next;
575  }
576  }
577 
578  if (g_file_logger_enabled) {
579  if (OutputFileLogThreadInit(tv, &td->file) != TM_ECODE_OK) {
580  FatalError(SC_ERR_FATAL, "failed to set up file thread data");
581  }
582  }
585  FatalError(SC_ERR_FATAL, "failed to set up filedata thread data");
586  }
587  }
588 
589  SCLogDebug("file_thread_data %p filedata_thread_data %p", td->file, td->filedata);
590 
591  return TM_ECODE_OK;
592 }
593 
594 static TmEcode OutputTxLogThreadDeinit(ThreadVars *tv, void *thread_data)
595 {
596  OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;
597 
598  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
599  OutputLoggerThreadStore *store = op_thread_data->store[alproto];
600  OutputTxLogger *logger = list[alproto];
601 
602  while (logger && store) {
603  if (logger->ThreadDeinit) {
604  logger->ThreadDeinit(tv, store->thread_data);
605  }
606 
607  OutputLoggerThreadStore *next_store = store->next;
608  SCFree(store);
609  store = next_store;
610  logger = logger->next;
611  }
612  }
613 
614  if (op_thread_data->file) {
615  OutputFileLogThreadDeinit(tv, op_thread_data->file);
616  }
617  if (op_thread_data->filedata) {
618  OutputFiledataLogThreadDeinit(tv, op_thread_data->filedata);
619  }
620 
621  SCFree(op_thread_data);
622  return TM_ECODE_OK;
623 }
624 
625 static void OutputTxLogExitPrintStats(ThreadVars *tv, void *thread_data)
626 {
627  OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;
628 
629  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
630  OutputLoggerThreadStore *store = op_thread_data->store[alproto];
631  OutputTxLogger *logger = list[alproto];
632 
633  while (logger && store) {
634  if (logger->ThreadExitPrintStats) {
635  logger->ThreadExitPrintStats(tv, store->thread_data);
636  }
637 
638  logger = logger->next;
639  store = store->next;
640  }
641  }
642 }
643 
644 static uint32_t OutputTxLoggerGetActiveCount(void)
645 {
646  uint32_t cnt = 0;
647  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
648  for (OutputTxLogger *p = list[alproto]; p != NULL; p = p->next) {
649  cnt++;
650  }
651  }
652 
653  if (g_file_logger_enabled) {
654  cnt++;
655  SCLogDebug("g_file_logger_enabled");
656  }
658  cnt++;
659  SCLogDebug("g_filedata_logger_enabled");
660  }
661 
662  return cnt;
663 }
664 
665 
667 {
668  OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit,
669  OutputTxLogExitPrintStats, OutputTxLog, OutputTxLoggerGetActiveCount);
670 }
671 
673 {
674  for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
675  OutputTxLogger *logger = list[alproto];
676  while (logger) {
677  OutputTxLogger *next_logger = logger->next;
678  SCFree(logger);
679  logger = next_logger;
680  }
681  list[alproto] = NULL;
682  }
683 }
OutputFiledataLogThreadInit
TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data)
thread init for the filedata logger This will run the thread init functions for the individual regist...
Definition: output-filedata.c:205
OutputTxLogger_::alproto
AppProto alproto
Definition: output-tx.c:50
OutputTxLoggerThreadData_::store
OutputLoggerThreadStore * store[ALPROTO_MAX]
Definition: output-tx.c:38
FileContainer_
Definition: util-file.h:113
output-tx.h
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputLoggerThreadStore_
Definition: output.h:33
AppLayerParserIsEnabled
int AppLayerParserIsEnabled(AppProto alproto)
simple way to globally test if a alproto is registered and fully enabled in the configuration.
Definition: app-layer-parser.c:1518
OutputTxLoggerRegister
void OutputTxLoggerRegister(void)
Definition: output-tx.c:666
AppLayerParserProtocolHasLogger
int AppLayerParserProtocolHasLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1528
OutputFileLoggerThreadData_
Definition: output-file.h:33
AppLayerParserSetTransactionLogId
void AppLayerParserSetTransactionLogId(AppLayerParserState *pstate, uint64_t tx_id)
Definition: app-layer-parser.c:719
CONFIG_TYPE_TX
@ CONFIG_TYPE_TX
Definition: util-config.h:37
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1058
FLOW_PKT_LAST_PSEUDO
#define FLOW_PKT_LAST_PSEUDO
Definition: flow.h:235
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
OutputTxLogger_::tc_log_progress
int tc_log_progress
Definition: output-tx.c:58
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:586
LOGGER_FILEDATA
@ LOGGER_FILEDATA
Definition: suricata-common.h:458
OutputTxShutdown
void OutputTxShutdown(void)
Definition: output-tx.c:672
Ctx
Definition: output-tx.c:266
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:594
AppLayerGetTxIterator
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
Definition: app-layer-parser.c:704
Flow_::proto
uint8_t proto
Definition: flow.h:378
Ctx::tx_logged
uint32_t tx_logged
Definition: output-tx.c:268
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:80
AppLayerParserGetStateProgress
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
Definition: app-layer-parser.c:1125
Packet_::flags
uint32_t flags
Definition: decode.h:463
Flow_
Flow data structure.
Definition: flow.h:356
OutputFiledataLogThreadDeinit
TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
Definition: output-filedata.c:253
LoggerId
LoggerId
Definition: suricata-common.h:449
AppLayerParserGetTransactionLogId
uint64_t AppLayerParserGetTransactionLogId(AppLayerParserState *pstate)
Definition: app-layer-parser.c:712
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
logged
int logged
Definition: app-layer-htp.h:1
AppLayerParserGetStateProgressCompletionStatus
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
Definition: app-layer-parser.c:1153
g_filedata_logger_enabled
bool g_filedata_logger_enabled
Definition: output-filedata.c:37
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output.h:35
AppLayerParserSupportsFiles
int AppLayerParserSupportsFiles(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1206
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:226
TxLogger
int(* TxLogger)(ThreadVars *, void *thread_data, const Packet *, Flow *f, void *state, void *tx, uint64_t tx_id)
Definition: output-tx.h:33
OutputTxLogger_::ThreadInit
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: output-tx.c:60
OutputTxLogger_::logger_id
LoggerId logger_id
Definition: output-tx.c:56
PKT_APPLAYER_UPDATE
#define PKT_APPLAYER_UPDATE
Definition: decode.h:1055
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:459
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:39
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
ALPROTO_MAX
@ ALPROTO_MAX
Definition: app-layer-protos.h:75
OutputTxLogger_::ThreadDeinit
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: output-tx.c:61
Ctx::tx_logged_old
uint32_t tx_logged_old
Definition: output-tx.c:267
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
OutputCtx_
Definition: tm-modules.h:78
OutputTxLoggerThreadData
struct OutputTxLoggerThreadData_ OutputTxLoggerThreadData
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output.h:34
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:488
OutputRegisterTxLogger
int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *output_ctx, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, void(*ThreadExitPrintStats)(ThreadVars *, void *))
Definition: output-tx.c:67
OutputFileLogThreadDeinit
TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data)
Definition: output-file.c:217
OutputTxLoggerThreadData_::filedata
OutputFiledataLoggerThreadData * filedata
Definition: output-tx.c:43
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:389
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1004
OutputTxLogger_::next
struct OutputTxLogger_ * next
Definition: output-tx.c:54
STREAM_FLAGS_FOR_PACKET
#define STREAM_FLAGS_FOR_PACKET(p)
Definition: stream.h:30
FileContainer_::head
File * head
Definition: util-file.h:114
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
AppLayerParserGetTxFiles
FileContainer * AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction)
Definition: app-layer-parser.c:890
util-profiling.h
FALSE
#define FALSE
Definition: suricata-common.h:34
stream.h
OutputTxLogger_::ts_log_progress
int ts_log_progress
Definition: output-tx.c:59
AppLayerGetTxIterState
Definition: app-layer-parser.h:142
Packet_
Definition: decode.h:428
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:83
OutputTxLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-tx.c:53
OutputTxLogger_::LogCondition
TxLoggerCondition LogCondition
Definition: output-tx.c:52
OutputFiledataLoggerThreadData_
Definition: output-filedata.h:34
OutputFiledataLogFfc
void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
Definition: output-filedata.c:129
OutputTxLogger_::ThreadExitPrintStats
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: output-tx.c:62
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:227
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:261
AppLayerTxData
struct AppLayerTxData AppLayerTxData
Definition: detect.h:1300
Packet_::flow
struct Flow_ * flow
Definition: decode.h:465
BIT_U8
#define BIT_U8(n)
Definition: suricata-common.h:387
suricata-common.h
OutputTxLogger
struct OutputTxLogger_ OutputTxLogger
OutputFileLogFfc
void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p, FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, const bool file_close, const bool file_trunc, uint8_t dir)
Definition: output-file.c:101
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1218
FatalError
#define FatalError(x,...)
Definition: util-debug.h:530
OutputTxLoggerThreadData_::file
OutputFileLoggerThreadData * file
Definition: output-tx.c:41
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputFileLogThreadInit
TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data)
thread init for the file logger This will run the thread init functions for the individual registered...
Definition: output-file.c:169
util-validate.h
LOGGER_FILE
@ LOGGER_FILE
Definition: suricata-common.h:457
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
Packet_::next
struct Packet_ * next
Definition: decode.h:603
TxLoggerCondition
int(* TxLoggerCondition)(ThreadVars *, const Packet *, void *state, void *tx, uint64_t tx_id)
Definition: output-tx.h:38
OutputTxLogger_::LogFunc
TxLogger LogFunc
Definition: output-tx.c:51
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:489
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
OutputTxLogger_
Definition: output-tx.c:49
APP_LAYER_PARSER_TRUNC_TC
#define APP_LAYER_PARSER_TRUNC_TC
Definition: app-layer-parser.h:42
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:254
OutputRegisterRootLogger
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc, OutputGetActiveCountFunc ActiveCntFunc)
Definition: output.c:963
OutputTxLogger_::name
const char * name
Definition: output-tx.c:55
AppLayerParserProtocolGetLoggerBits
LoggerId AppLayerParserProtocolGetLoggerBits(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1536
AppLayerGetTxIteratorFunc
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype
Definition: app-layer-parser.h:151
FlowGetDisruptionFlags
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get 'disruption' flags: GAP/DEPTH/PASS
Definition: flow.c:1160
APP_LAYER_PARSER_TRUNC_TS
#define APP_LAYER_PARSER_TRUNC_TS
Definition: app-layer-parser.h:41
AppLayerParserStateIssetFlag
uint16_t AppLayerParserStateIssetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1775
g_file_logger_enabled
bool g_file_logger_enabled
Definition: output-file.c:39
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:463
OutputTxLogger_::id
uint32_t id
Definition: output-tx.c:57
AppLayerParserGetTxCnt
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
Definition: app-layer-parser.c:1139
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
output.h
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
OutputTxLoggerThreadData_
Definition: output-tx.c:37
app-layer.h