suricata
output-tx.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer TX Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "output.h"
28 #include "output-tx.h"
29 #include "stream.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "util-config.h"
33 #include "util-profiling.h"
34 #include "util-validate.h"
35 
36 /** per thread data for this module, contains a list of per thread
37  * data for the packet loggers. */
38 typedef struct OutputTxLoggerThreadData_ {
39  /* thread local data from file api */
41  /* thread local data from filedata api */
45 
46 /* logger instance, a module + a output ctx,
47  * it's perfectly valid that have multiple instances of the same
48  * log module (e.g. http.log) with different output ctx'. */
49 typedef struct OutputTxLogger_ {
53  void *initdata;
55  const char *name;
57  uint32_t id;
60  TmEcode (*ThreadInit)(ThreadVars *, const void *, void **);
63 
64 static OutputTxLogger **list = NULL;
65 
69 {
70  if (list == NULL) {
71  list = SCCalloc(g_alproto_max, sizeof(OutputTxLogger *));
72  if (unlikely(list == NULL)) {
73  SCLogError("Failed to allocate OutputTx list");
74  return -1;
75  }
76  }
77 
79  SCLogDebug(
80  "%s logger not enabled: protocol %s is disabled", name, AppProtoToString(alproto));
81  return -1;
82  }
83  OutputTxLogger *op = SCCalloc(1, sizeof(*op));
84  if (op == NULL)
85  return -1;
86 
87  op->alproto = alproto;
88  op->LogFunc = LogFunc;
90  op->initdata = initdata;
91  op->name = name;
92  op->logger_id = id;
93  op->ThreadInit = ThreadInit;
95 
96  if (alproto == ALPROTO_UNKNOWN) {
97  op->tc_log_progress = 0;
98  } else if (tc_log_progress < 0) {
99  op->tc_log_progress =
101  STREAM_TOCLIENT);
102  } else {
104  }
105 
106  if (alproto == ALPROTO_UNKNOWN) {
107  op->ts_log_progress = 0;
108  } else if (ts_log_progress < 0) {
109  op->ts_log_progress =
111  STREAM_TOSERVER);
112  } else {
114  }
115 
116  if (list[alproto] == NULL) {
117  op->id = 1;
118  list[alproto] = op;
119  } else {
120  OutputTxLogger *t = list[alproto];
121  while (t->next)
122  t = t->next;
123  if (t->id * 2ULL > UINT32_MAX) {
124  FatalError("Too many loggers registered.");
125  }
126  op->id = t->id * 2;
127  t->next = op;
128  }
129 
130  SCLogDebug("OutputRegisterTxLogger happy");
131  return 0;
132 }
133 
134 extern bool g_file_logger_enabled;
135 extern bool g_filedata_logger_enabled;
136 
137 /** \brief run the per tx file logging
138  * \todo clean up various end of tx/stream etc indicators
139  */
140 static inline void OutputTxLogFiles(ThreadVars *tv, OutputFileLoggerThreadData *file_td,
141  OutputFiledataLoggerThreadData *filedata_td, Packet *p, Flow *f, void *tx,
142  const uint64_t tx_id, AppLayerTxData *txd, const bool tx_complete, const bool ts_ready,
143  const bool tc_ready, const bool ts_eof, const bool tc_eof, const bool eof)
144 {
145  uint8_t packet_dir;
146  uint8_t opposing_dir;
147  bool packet_dir_ready;
148  const bool opposing_dir_ready = eof;
149  bool opposing_tx_ready;
150  if (p->flowflags & FLOW_PKT_TOSERVER) {
151  packet_dir = STREAM_TOSERVER;
152  opposing_dir = STREAM_TOCLIENT;
153  packet_dir_ready = eof | ts_ready | ts_eof;
154  opposing_tx_ready = tc_ready;
155  } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
156  packet_dir = STREAM_TOCLIENT;
157  opposing_dir = STREAM_TOSERVER;
158  packet_dir_ready = eof | tc_ready | tc_eof;
159  opposing_tx_ready = ts_ready;
160  } else {
162  return;
163  }
164 
165  SCLogDebug("eof %d ts_ready %d ts_eof %d", eof, ts_ready, ts_eof);
166  SCLogDebug("eof %d tc_ready %d tc_eof %d", eof, tc_ready, tc_eof);
167 
168  SCLogDebug("packet dir %s opposing %s packet_dir_ready %d opposing_dir_ready %d",
169  packet_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT",
170  opposing_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT", packet_dir_ready,
171  opposing_dir_ready);
172 
173  AppLayerGetFileState app_files = AppLayerParserGetTxFiles(f, tx, packet_dir);
174  FileContainer *ffc = app_files.fc;
175  AppLayerGetFileState app_files_opposing = AppLayerParserGetTxFiles(f, tx, opposing_dir);
176  FileContainer *ffc_opposing = app_files_opposing.fc;
177 
178  /* see if opposing side is finished: if no file support in this direction, of is not
179  * files and tx is done for opposing dir. */
180  bool opposing_finished =
181  ffc_opposing == NULL || (ffc_opposing->head == NULL && opposing_tx_ready);
182  SCLogDebug("opposing_finished %d ffc_opposing %p ffc_opposing->head %p opposing_tx_ready %d",
183  opposing_finished, ffc_opposing, ffc_opposing ? ffc_opposing->head : NULL,
184  opposing_tx_ready);
185 
186  if (ffc || ffc_opposing)
187  SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
188  " ffc %p ffc_opposing %p tx_complete %d",
189  p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
190 
191  if (ffc) {
192  const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | eof;
193  const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
194  SCLogDebug("tx: calling files: ffc %p head %p file_close %d file_trunc %d", ffc, ffc->head,
195  file_close, file_trunc);
196  if (filedata_td && txd->files_opened > txd->files_stored)
197  OutputFiledataLogFfc(tv, filedata_td, p, app_files, tx, tx_id, txd, packet_dir,
198  file_close, file_trunc, packet_dir);
199  if (file_td && txd->files_opened > txd->files_logged)
201  tv, file_td, p, ffc, tx, tx_id, txd, file_close, file_trunc, packet_dir);
202  }
203  /* if EOF and we support files, do a final write out */
204  if (opposing_dir_ready && ffc_opposing != NULL) {
205  const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | tx_complete | eof;
206  const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
207  opposing_finished = true;
208  SCLogDebug("tx: calling for opposing direction files: file_close:%s file_trunc:%s",
209  file_close ? "true" : "false", file_trunc ? "true" : "false");
210  if (filedata_td && txd->files_opened > txd->files_stored)
211  OutputFiledataLogFfc(tv, filedata_td, p, app_files_opposing, tx, tx_id, txd,
212  opposing_dir, file_close, file_trunc, opposing_dir);
213  if (file_td && txd->files_opened > txd->files_logged)
214  OutputFileLogFfc(tv, file_td, p, ffc_opposing, tx, tx_id, txd, file_close, file_trunc,
215  opposing_dir);
216  }
217 
218  const bool tx_done = packet_dir_ready && opposing_finished;
219  SCLogDebug("tx_done %d packet_dir_ready %d opposing_finished %d", tx_done, packet_dir_ready,
220  opposing_finished);
221 
222  /* if not a file tx or if tx is done, set logger flags so tx can move on */
223  const bool is_file_tx = (ffc != NULL || ffc_opposing != NULL);
224  if (!is_file_tx || tx_done) {
225  SCLogDebug("is_file_tx %d tx_done %d", is_file_tx, tx_done);
226  if (file_td) {
227  txd->logged.flags |= BIT_U32(LOGGER_FILE);
228  SCLogDebug("setting LOGGER_FILE => %08x", txd->logged.flags);
229  }
230  if (filedata_td) {
231  txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
232  SCLogDebug("setting LOGGER_FILEDATA => %08x", txd->logged.flags);
233  }
234  } else {
235  SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
236  " NOT SETTING FILE FLAGS ffc %p ffc_opposing %p tx_complete %d",
237  p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
238  }
239 }
240 
241 static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data, Packet *p,
242  Flow *f, void *tx, const uint64_t tx_id)
243 {
244  const OutputTxLogger *logger = list[ALPROTO_UNKNOWN];
245  const OutputLoggerThreadStore *store = op_thread_data->store[ALPROTO_UNKNOWN];
246 
247  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
248  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
249  DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
250 
251  while (logger && store) {
252  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
253 
254  SCLogDebug("logger %p", logger);
255 
256  /* always invoke "wild card" tx loggers */
257  SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
259  logger->LogFunc(tv, store->thread_data, p, f, f->alstate, tx, tx_id);
261 
262  logger = logger->next;
263  store = store->next;
264 
265  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
266  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
267  }
268 }
269 
270 struct Ctx {
271  uint32_t tx_logged_old;
272  uint32_t tx_logged;
273 };
274 
275 static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data,
276  const OutputTxLogger *logger, const OutputLoggerThreadStore *store, Packet *p, Flow *f,
277  void *alstate, void *tx, const uint64_t tx_id, const AppProto alproto, const bool eof,
278  const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx)
279 {
280  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
281  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
282  // DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
283 
284  while (logger && store) {
285  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
286  DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto);
287 
288  SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
289  "tc_log_progress %d",
290  logger, logger->alproto, logger->LogCondition, logger->ts_log_progress,
291  logger->tc_log_progress);
292  if ((ctx->tx_logged_old & BIT_U32(logger->logger_id)) == 0) {
293  SCLogDebug("alproto match %d, logging tx_id %" PRIu64, logger->alproto, tx_id);
294 
295  SCLogDebug("pcap_cnt %" PRIu64 ", tx_id %" PRIu64 " logger %d. EOF %s", p->pcap_cnt,
296  tx_id, logger->logger_id, eof ? "true" : "false");
297 
298  if (eof) {
299  SCLogDebug("EOF, so log now");
300  } else {
301  if (logger->LogCondition) {
302  if (!logger->LogCondition(tv, p, alstate, tx, tx_id)) {
303  SCLogDebug("conditions not met, not logging");
304  goto next_logger;
305  }
306  } else {
307  if (tx_progress_tc < logger->tc_log_progress) {
308  SCLogDebug("progress not far enough, not logging");
309  goto next_logger;
310  }
311 
312  if (tx_progress_ts < logger->ts_log_progress) {
313  SCLogDebug("progress not far enough, not logging");
314  goto next_logger;
315  }
316  }
317  }
318 
319  SCLogDebug("Logging tx_id %" PRIu64 " to logger %d", tx_id, logger->logger_id);
321  logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
323 
324  ctx->tx_logged |= BIT_U32(logger->logger_id);
325  }
326 
327  next_logger:
328  logger = logger->next;
329  store = store->next;
330 
331  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
332  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
333  }
334 }
335 
336 static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
337 {
338  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
339  if (p->flow == NULL)
340  return TM_ECODE_OK;
341  if (!PKT_IS_PSEUDOPKT(p) && p->app_update_direction == 0 &&
342  ((PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATED) == 0) ||
343  (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATED) == 0))) {
344  SCLogDebug("not pseudo, no app update: skip");
345  return TM_ECODE_OK;
346  }
347  if ((p->flags & PKT_STREAM_EST) == 0 && p->proto == IPPROTO_TCP) {
348  return TM_ECODE_OK;
349  }
350  SCLogDebug("pseudo, or app update: run output");
351 
352  OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;
353 
354  Flow * const f = p->flow;
355  const uint8_t ipproto = f->proto;
356  const AppProto alproto = f->alproto;
357  SCLogDebug("pcap_cnt %u tx logging %u/%s", (uint32_t)p->pcap_cnt, alproto,
358  AppProtoToString(alproto));
359 
360  const bool file_logging_active = (op_thread_data->file || op_thread_data->filedata);
361  if (!file_logging_active) {
362  if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
363  SCLogDebug("bail");
364  /* No child loggers registered. */
365  return TM_ECODE_OK;
366  }
367  if (AppLayerParserProtocolHasLogger(ipproto, alproto) == 0)
368  goto end;
369  }
370  void *alstate = f->alstate;
371  if (alstate == NULL) {
372  SCLogDebug("no alstate");
373  goto end;
374  }
375  const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(ipproto, alproto);
376  if (logger_expectation == 0) {
377  SCLogDebug("bail: logger_expectation %u. LOGGER_FILE %u LOGGER_FILEDATA %u",
378  logger_expectation, LOGGER_FILE, LOGGER_FILEDATA);
379  goto end;
380  }
381  SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
382 
383  const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0;
386 
387  const bool eof = last_pseudo || (ts_eof && tc_eof);
388  SCLogDebug("eof %d last_pseudo %d ts_eof %d tc_eof %d", eof, last_pseudo, ts_eof, tc_eof);
389 
390  const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
391  const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
392  SCLogDebug("ts_disrupt_flags %02x tc_disrupt_flags %02x", ts_disrupt_flags, tc_disrupt_flags);
393  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
394  uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
395  uint64_t max_id = tx_id;
396  int logged = 0;
397  bool gap = false;
398  const bool support_files = AppLayerParserSupportsFiles(ipproto, alproto);
399  const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p);
400 
401  SCLogDebug("pcap_cnt %" PRIu64 ": tx_id %" PRIu64 " total_txs %" PRIu64, p->pcap_cnt, tx_id,
402  total_txs);
403 
404  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
406  memset(&state, 0, sizeof(state));
407 
408  const int complete_ts =
409  AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER);
410  const int complete_tc =
411  AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT);
412  while (1) {
413  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
414  if (ires.tx_ptr == NULL)
415  break;
416  void * const tx = ires.tx_ptr;
417  tx_id = ires.tx_id;
418  SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx);
419 
420  AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
421 
422  const int tx_progress_ts =
423  AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags);
424  const int tx_progress_tc =
425  AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
426  const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc);
427 
428  SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
429  op_thread_data->filedata);
430 
431  if (file_logging_active) {
432  if (txd->file_tx != 0) { // need to process each tx that might be a file tx,
433  // even if there are not files (yet)
434  const bool ts_ready = (tx_progress_ts == complete_ts);
435  const bool tc_ready = (tx_progress_tc == complete_tc);
436  SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready);
437 
438  const bool eval_files = ts_ready | tc_ready | tx_complete | ts_eof | tc_eof | eof;
439 
440  SCLogDebug("eval_files: %u, ts_ready %u, tc_ready %u, tx_complete %u, ts_eof %u, "
441  "tc_eof %u, eof %u",
442  eval_files, ts_ready, tc_ready, tx_complete, ts_eof, tc_eof, eof);
443  SCLogDebug("txd->file_tx & pkt_dir: %02x & %02x -> %02x", txd->file_tx, pkt_dir,
444  (txd->file_tx & pkt_dir));
445 
446  /* call only for the correct direction, except when it looks anything like a end of
447  * transaction or end of stream. Since OutputTxLogFiles has complicated logic around
448  * that, we just leave it to that function to sort things out for now. */
449  if (eval_files || ((txd->file_tx & pkt_dir) != 0)) {
450  OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx,
451  tx_id, txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof);
452  }
453  } else if (support_files) {
454  if (op_thread_data->file) {
455  txd->logged.flags |= BIT_U32(LOGGER_FILE);
456  SCLogDebug("not a file_tx: setting LOGGER_FILE => %08x", txd->logged.flags);
457  }
458  if (op_thread_data->filedata) {
459  txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
460  SCLogDebug("not a file_tx: setting LOGGER_FILEDATA => %08x", txd->logged.flags);
461  }
462  }
463  }
464  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
465  if (!txd->updated_tc && !txd->updated_ts && !(tx_progress_ts == complete_ts) &&
466  !(tx_progress_tc == complete_tc) && !ts_eof && !tc_eof) {
467  gap = true;
468  goto next_tx;
469  }
470 
471  if (list[ALPROTO_UNKNOWN] != 0) {
472  OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id);
473  if (list[alproto] == NULL)
474  goto next_tx;
475  }
476 
477  SCLogDebug("tx %p/%" PRIu64 " txd %p: log_flags %x logger_expectation %x", tx, tx_id, txd,
478  txd->config.log_flags, logger_expectation);
479  if (txd->config.log_flags & BIT_U8(CONFIG_TYPE_TX)) {
480  SCLogDebug("SKIP tx %p/%"PRIu64, tx, tx_id);
481  // so that AppLayerParserTransactionsCleanup can clean this tx
482  txd->logged.flags |= logger_expectation;
483  goto next_tx;
484  }
485 
486  if (txd->logged.flags == logger_expectation) {
487  SCLogDebug("fully logged");
488  /* tx already fully logged */
489  goto next_tx;
490  }
491 
492  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
493  const OutputTxLogger *logger = list[alproto];
494  const OutputLoggerThreadStore *store = op_thread_data->store[alproto];
495  struct Ctx ctx = { .tx_logged = txd->logged.flags, .tx_logged_old = txd->logged.flags };
496  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
497 
498  OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, alproto,
499  eof, tx_progress_ts, tx_progress_tc, &ctx);
500 
501  SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
502  if (ctx.tx_logged != ctx.tx_logged_old) {
503  SCLogDebug("logger: storing %08x (was %08x)", ctx.tx_logged, ctx.tx_logged_old);
504  DEBUG_VALIDATE_BUG_ON(txd == NULL);
505  txd->logged.flags |= ctx.tx_logged;
506  }
507 
508  /* If all loggers logged set a flag and update the last tx_id
509  * that was logged.
510  *
511  * If not all loggers were logged we flag that there was a gap
512  * so any subsequent transactions in this loop don't increase
513  * the maximum ID that was logged. */
514  if (!gap && ctx.tx_logged == logger_expectation) {
515  SCLogDebug("no gap %d, %08x == %08x", gap, ctx.tx_logged, logger_expectation);
516  logged = 1;
517  max_id = tx_id;
518  SCLogDebug("max_id %" PRIu64, max_id);
519  } else {
520  gap = true;
521  }
522 next_tx:
523  if (!ires.has_next)
524  break;
525  tx_id++;
526  }
527 
528  /* Update the last ID that has been logged with all
529  * transactions before it. */
530  if (logged) {
531  SCLogDebug("updating log tx_id %"PRIu64, max_id);
533  }
534 
535 end:
536  return TM_ECODE_OK;
537 }
538 
539 /** \brief thread init for the tx logger
540  * This will run the thread init functions for the individual registered
541  * loggers */
542 static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *_initdata, void **data)
543 {
545  SCCalloc(1, sizeof(*td) + g_alproto_max * sizeof(OutputLoggerThreadStore *));
546  if (td == NULL)
547  return TM_ECODE_FAILED;
548 
549  *data = (void *)td;
550  SCLogDebug("OutputTxLogThreadInit happy (*data %p)", *data);
551 
552  for (AppProto alproto = 0; alproto < g_alproto_max; alproto++) {
553  OutputTxLogger *logger = list[alproto];
554  while (logger) {
555  if (logger->ThreadInit) {
556  void *retptr = NULL;
557  if (logger->ThreadInit(tv, logger->initdata, &retptr) == TM_ECODE_OK) {
558  OutputLoggerThreadStore *ts = SCCalloc(1, sizeof(*ts));
559  /* todo */ BUG_ON(ts == NULL);
560 
561  /* store thread handle */
562  ts->thread_data = retptr;
563 
564  if (td->store[alproto] == NULL) {
565  td->store[alproto] = ts;
566  } else {
567  OutputLoggerThreadStore *tmp = td->store[alproto];
568  while (tmp->next != NULL)
569  tmp = tmp->next;
570  tmp->next = ts;
571  }
572 
573  SCLogDebug("%s is now set up", logger->name);
574  }
575  }
576 
577  logger = logger->next;
578  }
579  }
580 
581  if (g_file_logger_enabled) {
582  if (OutputFileLogThreadInit(tv, &td->file) != TM_ECODE_OK) {
583  FatalError("failed to set up file thread data");
584  }
585  }
588  FatalError("failed to set up filedata thread data");
589  }
590  }
591 
592  SCLogDebug("file_thread_data %p filedata_thread_data %p", td->file, td->filedata);
593 
594  return TM_ECODE_OK;
595 }
596 
597 static TmEcode OutputTxLogThreadDeinit(ThreadVars *tv, void *thread_data)
598 {
599  OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;
600 
601  for (AppProto alproto = 0; alproto < g_alproto_max; alproto++) {
602  OutputLoggerThreadStore *store = op_thread_data->store[alproto];
603  OutputTxLogger *logger = list[alproto];
604 
605  while (logger && store) {
606  if (logger->ThreadDeinit) {
607  logger->ThreadDeinit(tv, store->thread_data);
608  }
609 
610  OutputLoggerThreadStore *next_store = store->next;
611  SCFree(store);
612  store = next_store;
613  logger = logger->next;
614  }
615  }
616 
617  if (op_thread_data->file) {
618  OutputFileLogThreadDeinit(tv, op_thread_data->file);
619  }
620  if (op_thread_data->filedata) {
621  OutputFiledataLogThreadDeinit(tv, op_thread_data->filedata);
622  }
623 
624  SCFree(op_thread_data);
625  return TM_ECODE_OK;
626 }
627 
628 static uint32_t OutputTxLoggerGetActiveCount(void)
629 {
630  uint32_t cnt = 0;
631  for (AppProto alproto = 0; alproto < g_alproto_max; alproto++) {
632  for (OutputTxLogger *p = list[alproto]; p != NULL; p = p->next) {
633  cnt++;
634  }
635  }
636 
637  if (g_file_logger_enabled) {
638  cnt++;
639  SCLogDebug("g_file_logger_enabled");
640  }
642  cnt++;
643  SCLogDebug("g_filedata_logger_enabled");
644  }
645 
646  return cnt;
647 }
648 
649 
651 {
652  BUG_ON(list);
653  list = SCCalloc(g_alproto_max, sizeof(OutputTxLogger *));
654  if (unlikely(list == NULL)) {
655  FatalError("Failed to allocate OutputTx list");
656  }
657  OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit, OutputTxLog,
658  OutputTxLoggerGetActiveCount);
659 }
660 
662 {
663  // called in different places because of unix socket mode, and engine-analysis mode
664  if (list == NULL) {
665  return;
666  }
667  for (AppProto alproto = 0; alproto < g_alproto_max; alproto++) {
668  OutputTxLogger *logger = list[alproto];
669  while (logger) {
670  OutputTxLogger *next_logger = logger->next;
671  SCFree(logger);
672  logger = next_logger;
673  }
674  list[alproto] = NULL;
675  }
676  SCFree(list);
677  list = NULL;
678 }
PKT_IS_TOCLIENT
#define PKT_IS_TOCLIENT(p)
Definition: decode.h:239
OutputFiledataLogThreadInit
TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data)
thread init for the filedata logger This will run the thread init functions for the individual regist...
Definition: output-filedata.c:198
OutputTxLogger_::alproto
AppProto alproto
Definition: output-tx.c:50
Packet_::proto
uint8_t proto
Definition: decode.h:523
FileContainer_
Definition: util-file.h:113
output-tx.h
ts
uint64_t ts
Definition: source-erf-file.c:55
SCAppLayerParserStateIssetFlag
uint16_t SCAppLayerParserStateIssetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1829
OutputLoggerThreadStore_
Definition: output.h:33
AppLayerParserIsEnabled
int AppLayerParserIsEnabled(AppProto alproto)
simple way to globally test if a alproto is registered and fully enabled in the configuration.
Definition: app-layer-parser.c:1531
OutputTxLoggerRegister
void OutputTxLoggerRegister(void)
Definition: output-tx.c:650
AppLayerParserProtocolHasLogger
int AppLayerParserProtocolHasLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1541
OutputFileLoggerThreadData_
Definition: output-file.h:33
AppLayerParserSetTransactionLogId
void AppLayerParserSetTransactionLogId(AppLayerParserState *pstate, uint64_t tx_id)
Definition: app-layer-parser.c:722
CONFIG_TYPE_TX
@ CONFIG_TYPE_TX
Definition: util-config.h:37
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1321
FLOW_PKT_LAST_PSEUDO
#define FLOW_PKT_LAST_PSEUDO
Definition: flow.h:240
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
OutputTxLogger_::tc_log_progress
int tc_log_progress
Definition: output-tx.c:58
LOGGER_FILEDATA
@ LOGGER_FILEDATA
Definition: suricata-common.h:487
OutputTxShutdown
void OutputTxShutdown(void)
Definition: output-tx.c:661
Ctx
Definition: output-tx.c:270
OutputRegisterRootLogger
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, OutputLogFunc LogFunc, OutputGetActiveCountFunc ActiveCntFunc)
Definition: output.c:874
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:275
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:626
AppLayerGetTxIterator
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
Definition: app-layer-parser.c:707
Flow_::proto
uint8_t proto
Definition: flow.h:378
Ctx::tx_logged
uint32_t tx_logged
Definition: output-tx.c:272
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:86
AppLayerParserGetStateProgress
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
Definition: app-layer-parser.c:1087
Packet_::flags
uint32_t flags
Definition: decode.h:544
Flow_
Flow data structure.
Definition: flow.h:356
OutputFiledataLogThreadDeinit
TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
Definition: output-filedata.c:244
LoggerId
LoggerId
Definition: suricata-common.h:477
AppLayerParserGetTransactionLogId
uint64_t AppLayerParserGetTransactionLogId(AppLayerParserState *pstate)
Definition: app-layer-parser.c:715
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:40
ctx
struct Thresholds ctx
AppLayerParserSupportsFiles
bool AppLayerParserSupportsFiles(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1173
logged
int logged
Definition: app-layer-htp.h:1
FLOW_TC_APP_UPDATED
#define FLOW_TC_APP_UPDATED
Definition: flow.h:120
AppLayerParserGetStateProgressCompletionStatus
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
Definition: app-layer-parser.c:1116
g_filedata_logger_enabled
bool g_filedata_logger_enabled
Definition: output-filedata.c:37
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output.h:35
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:233
TxLogger
int(* TxLogger)(ThreadVars *, void *thread_data, const Packet *, Flow *f, void *state, void *tx, uint64_t tx_id)
Transaction logger function pointer type.
Definition: output-tx.h:34
AppLayerTxData
struct AppLayerTxData AppLayerTxData
Definition: app-layer-parser.h:42
OutputTxLogger_::ThreadInit
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: output-tx.c:60
OutputTxLogger_::logger_id
LoggerId logger_id
Definition: output-tx.c:56
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:532
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:53
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:82
OutputTxLogger_::ThreadDeinit
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: output-tx.c:61
Ctx::tx_logged_old
uint32_t tx_logged_old
Definition: output-tx.c:271
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:81
AppLayerParserGetTxFiles
AppLayerGetFileState AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction)
Definition: app-layer-parser.c:873
OutputTxLoggerThreadData
struct OutputTxLoggerThreadData_ OutputTxLoggerThreadData
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output.h:34
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:478
StreamTcpReassembleDepthReached
bool StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:625
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
g_alproto_max
AppProto g_alproto_max
Definition: app-layer-protos.c:29
OutputFileLogThreadDeinit
TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data)
Definition: output-file.c:211
OutputTxLoggerThreadData_::filedata
OutputFiledataLoggerThreadData * filedata
Definition: output-tx.c:42
AppLayerGetTxIterTuple
struct AppLayerGetTxIterTuple AppLayerGetTxIterTuple
Definition: app-layer-parser.h:40
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:417
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1268
AppLayerGetFileState
struct AppLayerGetFileState AppLayerGetFileState
Definition: app-layer-parser.h:41
OutputTxLogger_::next
struct OutputTxLogger_ * next
Definition: output-tx.c:54
STREAM_FLAGS_FOR_PACKET
#define STREAM_FLAGS_FOR_PACKET(p)
Definition: stream.h:30
FileContainer_::head
File * head
Definition: util-file.h:114
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:43
app-layer-parser.h
SCOutputRegisterTxLogger
int SCOutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc, void *initdata, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
Register a transaction logger.
Definition: output-tx.c:66
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
util-profiling.h
stream.h
OutputTxLogger_::ts_log_progress
int ts_log_progress
Definition: output-tx.c:59
AppLayerGetTxIterState
Definition: app-layer-parser.h:134
Packet_
Definition: decode.h:501
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:54
TmEcode
TmEcode
Definition: tm-threads-common.h:80
name
const char * name
Definition: tm-threads.c:2163
OutputTxLogger_::LogCondition
TxLoggerCondition LogCondition
Definition: output-tx.c:52
OutputFiledataLoggerThreadData_
Definition: output-filedata.h:34
OutputFiledataLogFfc
void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, AppLayerGetFileState files, void *txv, const uint64_t tx_id, AppLayerTxData *txd, const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
Definition: output-filedata.c:122
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:234
OutputTxLoggerThreadData_::store
OutputLoggerThreadStore * store[]
Definition: output-tx.c:43
TxLoggerCondition
bool(* TxLoggerCondition)(ThreadVars *, const Packet *, void *state, void *tx, uint64_t tx_id)
Transaction logger condition function pointer type.
Definition: output-tx.h:41
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:240
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
Packet_::flow
struct Flow_ * flow
Definition: decode.h:546
BIT_U8
#define BIT_U8(n)
Definition: suricata-common.h:415
suricata-common.h
OutputTxLogger
struct OutputTxLogger_ OutputTxLogger
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:535
FLOW_TS_APP_UPDATED
#define FLOW_TS_APP_UPDATED
Definition: flow.h:119
OutputFileLogFfc
void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p, FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, const bool file_close, const bool file_trunc, uint8_t dir)
Definition: output-file.c:96
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1183
FatalError
#define FatalError(...)
Definition: util-debug.h:510
OutputTxLoggerThreadData_::file
OutputFileLoggerThreadData * file
Definition: output-tx.c:40
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputFileLogThreadInit
TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data)
thread init for the file logger This will run the thread init functions for the individual registered...
Definition: output-file.c:164
util-validate.h
LOGGER_FILE
@ LOGGER_FILE
Definition: suricata-common.h:486
Packet_::next
struct Packet_ * next
Definition: decode.h:635
OutputTxLogger_::LogFunc
TxLogger LogFunc
Definition: output-tx.c:51
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:267
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:479
util-config.h
Flow_::flags
uint32_t flags
Definition: flow.h:421
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
OutputTxLogger_
Definition: output-tx.c:49
OutputTxLogger_::initdata
void * initdata
Definition: output-tx.c:53
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:233
OutputTxLogger_::name
const char * name
Definition: output-tx.c:55
AppLayerParserProtocolGetLoggerBits
LoggerId AppLayerParserProtocolGetLoggerBits(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:1549
AppLayerGetTxIteratorFunc
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype
Definition: app-layer-parser.h:143
id
uint32_t id
Definition: detect-flowbits.c:938
FlowGetDisruptionFlags
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get 'disruption' flags: GAP/DEPTH/PASS
Definition: flow.c:1141
g_file_logger_enabled
bool g_file_logger_enabled
Definition: output-file.c:39
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
OutputTxLogger_::id
uint32_t id
Definition: output-tx.c:57
AppLayerParserGetTxCnt
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
Definition: app-layer-parser.c:1102
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
output.h
PKT_STREAM_EST
#define PKT_STREAM_EST
Definition: decode.h:1262
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:44
OutputTxLoggerThreadData_
Definition: output-tx.c:38
app-layer.h