suricata
output-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer File Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-file.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "util-profiling.h"
34 #include "util-validate.h"
35 #include "util-magic.h"
36 
37 bool g_file_logger_enabled = false;
38 
39 /** per thread data for this module, contains a list of per thread
40  * data for the packet loggers. */
43 #ifdef HAVE_MAGIC
44  magic_t magic_ctx;
45 #endif
47 
48 /* logger instance, a module + a output ctx,
49  * it's perfectly valid that have multiple instances of the same
50  * log module (e.g. http.log) with different output ctx'. */
51 typedef struct OutputFileLogger_ {
55  const char *name;
61 
62 static OutputFileLogger *list = NULL;
63 
68 {
69  OutputFileLogger *op = SCMalloc(sizeof(*op));
70  if (op == NULL)
71  return -1;
72  memset(op, 0x00, sizeof(*op));
73 
74  op->LogFunc = LogFunc;
75  op->output_ctx = output_ctx;
76  op->name = name;
77  op->logger_id = id;
78  op->ThreadInit = ThreadInit;
81 
82  if (list == NULL)
83  list = op;
84  else {
85  OutputFileLogger *t = list;
86  while (t->next)
87  t = t->next;
88  t->next = op;
89  }
90 
91  SCLogDebug("OutputRegisterFileLogger happy");
92 
93  g_file_logger_enabled = true;
94  return 0;
95 }
96 
97 static void CloseFile(const Packet *p, Flow *f, File *file)
98 {
99  DEBUG_VALIDATE_BUG_ON((file->flags & FILE_LOGGED) != 0);
100  void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid);
101  if (txv) {
102  AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
103  if (txd) {
104  txd->files_logged++;
105  DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened);
106  }
107  }
108  file->flags |= FILE_LOGGED;
109 }
110 
111 static void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
112  FileContainer *ffc, const bool file_close, const bool file_trunc, uint8_t dir)
113 {
114  SCLogDebug("ffc %p", ffc);
115  if (ffc != NULL) {
116  File *ff;
117  for (ff = ffc->head; ff != NULL; ff = ff->next) {
118  if (ff->flags & FILE_LOGGED)
119  continue;
120 
121  SCLogDebug("ff %p", ff);
122 
123  if (file_trunc && ff->state < FILE_STATE_CLOSED)
125 
126  if (file_close && ff->state < FILE_STATE_CLOSED)
128 
129  SCLogDebug("ff %p state %u", ff, ff->state);
130 
131  if (ff->state > FILE_STATE_OPENED) {
132  bool file_logged = false;
133 #ifdef HAVE_MAGIC
134  if (FileForceMagic() && ff->magic == NULL) {
135  FilemagicThreadLookup(&op_thread_data->magic_ctx, ff);
136  }
137 #endif
138  const OutputFileLogger *logger = list;
139  const OutputLoggerThreadStore *store = op_thread_data->store;
140  while (logger && store) {
141  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
142 
143  SCLogDebug("logger %p", logger);
145  logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, dir);
147  file_logged = true;
148 
149  logger = logger->next;
150  store = store->next;
151 
152  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
153  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
154  }
155 
156  if (file_logged) {
157  CloseFile(p, p->flow, ff);
158  }
159  }
160  }
161  }
162 }
163 
164 static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
165 {
166  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
167 
168  if (list == NULL) {
169  /* No child loggers. */
170  return TM_ECODE_OK;
171  }
172 
173  OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
174 
175  /* no flow, no files */
176  Flow * const f = p->flow;
177  if (f == NULL || f->alstate == NULL) {
179  }
180 
181  if (p->proto == IPPROTO_TCP) {
182  const bool file_trunc = StreamTcpReassembleDepthReached(p);
183  if (p->flowflags & FLOW_PKT_TOSERVER) {
184  const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
185  FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
186  OutputFileLogFfc(
187  tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
188  } else {
189  const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
190  FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
191  OutputFileLogFfc(
192  tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
193  }
194  } else if (p->proto == IPPROTO_UDP) {
195  FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
196  OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, false, false, STREAM_TOSERVER);
197  FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
198  OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, false, false, STREAM_TOCLIENT);
199  }
200  return TM_ECODE_OK;
201 }
202 
203 /** \brief thread init for the tx logger
204  * This will run the thread init functions for the individual registered
205  * loggers */
206 static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
207 {
208  OutputFileLoggerThreadData *td = SCMalloc(sizeof(*td));
209  if (td == NULL)
210  return TM_ECODE_FAILED;
211  memset(td, 0x00, sizeof(*td));
212 
213  *data = (void *)td;
214 
215 #ifdef HAVE_MAGIC
216  td->magic_ctx = MagicInitContext();
217  if (td->magic_ctx == NULL) {
218  SCFree(td);
219  return TM_ECODE_FAILED;
220  }
221 #endif
222 
223  SCLogDebug("OutputFileLogThreadInit happy (*data %p)", *data);
224 
225  OutputFileLogger *logger = list;
226  while (logger) {
227  if (logger->ThreadInit) {
228  void *retptr = NULL;
229  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
230  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
231 /* todo */ BUG_ON(ts == NULL);
232  memset(ts, 0x00, sizeof(*ts));
233 
234  /* store thread handle */
235  ts->thread_data = retptr;
236 
237  if (td->store == NULL) {
238  td->store = ts;
239  } else {
240  OutputLoggerThreadStore *tmp = td->store;
241  while (tmp->next != NULL)
242  tmp = tmp->next;
243  tmp->next = ts;
244  }
245 
246  SCLogDebug("%s is now set up", logger->name);
247  }
248  }
249 
250  logger = logger->next;
251  }
252 
253  return TM_ECODE_OK;
254 }
255 
256 static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data)
257 {
258  OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
259  OutputLoggerThreadStore *store = op_thread_data->store;
260  OutputFileLogger *logger = list;
261 
262  while (logger && store) {
263  if (logger->ThreadDeinit) {
264  logger->ThreadDeinit(tv, store->thread_data);
265  }
266 
267  OutputLoggerThreadStore *next_store = store->next;
268  SCFree(store);
269  store = next_store;
270  logger = logger->next;
271  }
272 
273 #ifdef HAVE_MAGIC
274  MagicDeinitContext(op_thread_data->magic_ctx);
275 #endif
276 
277  SCFree(op_thread_data);
278  return TM_ECODE_OK;
279 }
280 
281 static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data)
282 {
283  OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
284  OutputLoggerThreadStore *store = op_thread_data->store;
285  OutputFileLogger *logger = list;
286 
287  while (logger && store) {
288  if (logger->ThreadExitPrintStats) {
289  logger->ThreadExitPrintStats(tv, store->thread_data);
290  }
291 
292  logger = logger->next;
293  store = store->next;
294  }
295 }
296 
297 static uint32_t OutputFileLoggerGetActiveCount(void)
298 {
299  uint32_t cnt = 0;
300  for (OutputFileLogger *p = list; p != NULL; p = p->next) {
301  cnt++;
302  }
303  return cnt;
304 }
305 
307 {
308  OutputRegisterRootLogger(OutputFileLogThreadInit,
309  OutputFileLogThreadDeinit, OutputFileLogExitPrintStats,
310  OutputFileLog, OutputFileLoggerGetActiveCount);
311 }
312 
314 {
315  OutputFileLogger *logger = list;
316  while (logger) {
317  OutputFileLogger *next_logger = logger->next;
318  SCFree(logger);
319  logger = next_logger;
320  }
321 
322  list = NULL;
323 }
Packet_::proto
uint8_t proto
Definition: decode.h:455
FileContainer_
Definition: util-file.h:110
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputLoggerThreadStore_
Definition: output-stats.c:31
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output-stats.c:33
OutputFileLoggerThreadData_
Definition: output-file.c:41
OutputFileLogger_::logger_id
LoggerId logger_id
Definition: output-file.c:56
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:584
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
OutputFileLoggerRegister
void OutputFileLoggerRegister(void)
Definition: output-file.c:306
OutputRegisterFileLogger
int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output-file.c:64
Flow_::proto
uint8_t proto
Definition: flow.h:375
OutputFileLogger
struct OutputFileLogger_ OutputFileLogger
Packet_::flags
uint32_t flags
Definition: decode.h:468
FILE_STATE_OPENED
@ FILE_STATE_OPENED
Definition: util-file.h:66
Flow_
Flow data structure.
Definition: flow.h:353
File_::state
FileState state
Definition: util-file.h:78
LoggerId
LoggerId
Definition: suricata-common.h:437
OutputFileLogger_::next
struct OutputFileLogger_ * next
Definition: output-file.c:54
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:223
tm-modules.h
AppLayerParserGetFiles
FileContainer * AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
Definition: app-layer-parser.c:895
FILE_STATE_TRUNCATED
@ FILE_STATE_TRUNCATED
Definition: util-file.h:69
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:464
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:83
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:82
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output-stats.c:32
OutputCtx_
Definition: tm-modules.h:78
OutputFileLoggerThreadData
struct OutputFileLoggerThreadData_ OutputFileLoggerThreadData
FileForceMagic
int FileForceMagic(void)
Definition: util-file.c:142
OutputFileLogger_::LogFunc
FileLogger LogFunc
Definition: output-file.c:52
output-file.h
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1178
detect-filemagic.h
FileContainer_::head
File * head
Definition: util-file.h:111
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:56
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
OutputFileLogger_::ThreadExitPrintStats
ThreadExitPrintStatsFunc ThreadExitPrintStats
Definition: output-file.c:59
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
util-profiling.h
OutputFileLogger_
Definition: output-file.c:51
Packet_
Definition: decode.h:433
OutputFileShutdown
void OutputFileShutdown(void)
Definition: output-file.c:313
util-magic.h
TmEcode
TmEcode
Definition: tm-threads-common.h:81
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1124
File_::flags
uint16_t flags
Definition: util-file.h:76
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:258
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:67
File_
Definition: util-file.h:75
Packet_::flow
struct Flow_ * flow
Definition: decode.h:470
suricata-common.h
ThreadExitPrintStatsFunc
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
File_::next
struct File_ * next
Definition: util-file.h:89
OutputFileLogger_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output-file.c:58
FileLogger
int(* FileLogger)(ThreadVars *, void *thread_data, const Packet *, const File *, uint8_t direction)
Definition: output-file.h:33
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1196
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:31
util-validate.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
Packet_::next
struct Packet_ * next
Definition: decode.h:603
FILE_LOGGED
#define FILE_LOGGED
Definition: util-file.h:53
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:486
OutputFileLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-file.c:53
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:251
OutputRegisterRootLogger
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc, OutputGetActiveCountFunc ActiveCntFunc)
Definition: output.c:965
File_::txid
uint64_t txid
Definition: util-file.h:80
g_file_logger_enabled
bool g_file_logger_enabled
Definition: output-file.c:37
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:302
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
OutputFileLogger_::name
const char * name
Definition: output-file.c:55
OutputFileLogger_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output-file.c:57
output.h
OutputFileLoggerThreadData_::store
OutputLoggerThreadStore * store
Definition: output-file.c:42
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
app-layer.h