suricata
output-filedata.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-filedata.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "conf.h"
34 #include "util-profiling.h"
35 #include "util-validate.h"
36 #include "util-magic.h"
37 
39 
40 /** per thread data for this module, contains a list of per thread
41  * data for the packet loggers. */
44 #ifdef HAVE_MAGIC
45  magic_t magic_ctx;
46 #endif
48 
49 /* logger instance, a module + a output ctx,
50  * it's perfectly valid that have multiple instances of the same
51  * log module (e.g. http.log) with different output ctx'. */
52 typedef struct OutputFiledataLogger_ {
56  const char *name;
62 
63 static OutputFiledataLogger *list = NULL;
64 
69 {
70  OutputFiledataLogger *op = SCMalloc(sizeof(*op));
71  if (op == NULL)
72  return -1;
73  memset(op, 0x00, sizeof(*op));
74 
75  op->LogFunc = LogFunc;
76  op->output_ctx = output_ctx;
77  op->name = name;
78  op->logger_id = id;
79  op->ThreadInit = ThreadInit;
82 
83  if (list == NULL)
84  list = op;
85  else {
86  OutputFiledataLogger *t = list;
87  while (t->next)
88  t = t->next;
89  t->next = op;
90  }
91 
92  SCLogDebug("OutputRegisterFiledataLogger happy");
94  return 0;
95 }
96 
97 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
98 
99 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
100  Packet *p, File *ff,
101  const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
102 {
103  OutputFiledataLogger *logger = list;
104  OutputLoggerThreadStore *store = store_list;
105  int file_logged = 0;
106 
107  while (logger && store) {
108  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
109 
110  SCLogDebug("logger %p", logger);
112  logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
114 
115  file_logged = 1;
116 
117  logger = logger->next;
118  store = store->next;
119 
120  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
121  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
122  }
123  return file_logged;
124 }
125 
126 static void CloseFile(const Packet *p, Flow *f, File *file)
127 {
128  void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid);
129  if (txv) {
130  AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
131  if (txd)
132  txd->files_stored++;
133  }
134  file->flags |= FILE_STORED;
135 }
136 
137 static void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
138  FileContainer *ffc, const uint8_t call_flags, const bool file_close, const bool file_trunc,
139  const uint8_t dir)
140 {
141  if (ffc != NULL) {
142  OutputLoggerThreadStore *store = td->store;
143  File *ff;
144  for (ff = ffc->head; ff != NULL; ff = ff->next) {
145  uint8_t file_flags = call_flags;
146 #ifdef HAVE_MAGIC
147  if (FileForceMagic() && ff->magic == NULL) {
148  FilemagicThreadLookup(&td->magic_ctx, ff);
149  }
150 #endif
151  SCLogDebug("ff %p", ff);
152  if (ff->flags & FILE_STORED) {
153  SCLogDebug("stored flag set");
154  continue;
155  }
156 
157  if (!(ff->flags & FILE_STORE)) {
158  SCLogDebug("ff FILE_STORE not set");
159  continue;
160  }
161 
162  /* if we have no data chunks left to log, we should still
163  * close the logger(s) */
164  if (FileDataSize(ff) == ff->content_stored &&
165  (file_trunc || file_close)) {
166  if (ff->state < FILE_STATE_CLOSED) {
167  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
168  }
169  CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
170  CloseFile(p, p->flow, ff);
171  continue;
172  }
173 
174  /* store */
175 
176  /* if file_store_id == 0, this is the first store of this file */
177  if (ff->file_store_id == 0) {
178  /* new file */
179  ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
180  file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
181  } else {
182  /* existing file */
183  }
184 
185  /* if file needs to be closed or truncated, inform
186  * loggers */
187  if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
188  FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
189  }
190 
191  /* tell the logger we're closing up */
192  if (ff->state >= FILE_STATE_CLOSED)
193  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
194 
195  /* do the actual logging */
196  const uint8_t *data = NULL;
197  uint32_t data_len = 0;
198 
200  &data, &data_len,
201  ff->content_stored);
202 
203  const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
204  if (file_logged) {
205  ff->content_stored += data_len;
206 
207  /* all done */
208  if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
209  CloseFile(p, p->flow, ff);
210  }
211  }
212  }
213  }
214 }
215 
216 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
217 {
218  DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
219 
220  if (list == NULL) {
221  /* No child loggers. */
222  return TM_ECODE_OK;
223  }
224 
225  OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
226 
227  /* no flow, no files */
228  Flow * const f = p->flow;
229  if (f == NULL || f->alstate == NULL) {
231  }
232  /* do not log for ICMP packets related to a TCP/UDP flow */
233  if (p->proto != IPPROTO_TCP && p->proto != IPPROTO_UDP) {
235  }
236 
237  const bool file_trunc = StreamTcpReassembleDepthReached(p);
238  if (p->flowflags & FLOW_PKT_TOSERVER) {
239  const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
240  FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
241  SCLogDebug("ffc_ts %p", ffc_ts);
242  OutputFiledataLogFfc(tv, op_thread_data, p, ffc_ts, STREAM_TOSERVER, file_close_ts,
243  file_trunc, STREAM_TOSERVER);
244  } else {
245  const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
246  FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
247  SCLogDebug("ffc_tc %p", ffc_tc);
248  OutputFiledataLogFfc(tv, op_thread_data, p, ffc_tc, STREAM_TOCLIENT, file_close_tc,
249  file_trunc, STREAM_TOCLIENT);
250  }
251 
252  return TM_ECODE_OK;
253 }
254 
255 /** \brief thread init for the tx logger
256  * This will run the thread init functions for the individual registered
257  * loggers */
258 static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
259 {
260  OutputFiledataLoggerThreadData *td = SCMalloc(sizeof(*td));
261  if (td == NULL)
262  return TM_ECODE_FAILED;
263  memset(td, 0x00, sizeof(*td));
264 
265  *data = (void *)td;
266 
267 #ifdef HAVE_MAGIC
268  td->magic_ctx = MagicInitContext();
269  if (td->magic_ctx == NULL) {
270  SCFree(td);
271  return TM_ECODE_FAILED;
272  }
273 #endif
274 
275  SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
276 
277  OutputFiledataLogger *logger = list;
278  while (logger) {
279  if (logger->ThreadInit) {
280  void *retptr = NULL;
281  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
282  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
283 /* todo */ BUG_ON(ts == NULL);
284  memset(ts, 0x00, sizeof(*ts));
285 
286  /* store thread handle */
287  ts->thread_data = retptr;
288 
289  if (td->store == NULL) {
290  td->store = ts;
291  } else {
292  OutputLoggerThreadStore *tmp = td->store;
293  while (tmp->next != NULL)
294  tmp = tmp->next;
295  tmp->next = ts;
296  }
297 
298  SCLogDebug("%s is now set up", logger->name);
299  }
300  }
301 
302  logger = logger->next;
303  }
304  return TM_ECODE_OK;
305 }
306 
307 static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
308 {
309  OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
310  OutputLoggerThreadStore *store = op_thread_data->store;
311  OutputFiledataLogger *logger = list;
312 
313  while (logger && store) {
314  if (logger->ThreadDeinit) {
315  logger->ThreadDeinit(tv, store->thread_data);
316  }
317 
318  OutputLoggerThreadStore *next_store = store->next;
319  SCFree(store);
320  store = next_store;
321  logger = logger->next;
322  }
323 
324 #ifdef HAVE_MAGIC
325  MagicDeinitContext(op_thread_data->magic_ctx);
326 #endif
327 
328  SCFree(op_thread_data);
329  return TM_ECODE_OK;
330 }
331 
332 static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
333 {
334  OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
335  OutputLoggerThreadStore *store = op_thread_data->store;
336  OutputFiledataLogger *logger = list;
337 
338  while (logger && store) {
339  if (logger->ThreadExitPrintStats) {
340  logger->ThreadExitPrintStats(tv, store->thread_data);
341  }
342 
343  logger = logger->next;
344  store = store->next;
345  }
346 }
347 
348 static uint32_t OutputFiledataLoggerGetActiveCount(void)
349 {
350  uint32_t cnt = 0;
351  for (OutputFiledataLogger *p = list; p != NULL; p = p->next) {
352  cnt++;
353  }
354  return cnt;
355 }
356 
358 {
359  OutputRegisterRootLogger(OutputFiledataLogThreadInit,
360  OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
361  OutputFiledataLog, OutputFiledataLoggerGetActiveCount);
362  SC_ATOMIC_INIT(g_file_store_id);
363  SC_ATOMIC_SET(g_file_store_id, 1);
364 }
365 
367 {
368  OutputFiledataLogger *logger = list;
369  while (logger) {
370  OutputFiledataLogger *next_logger = logger->next;
371  SCFree(logger);
372  logger = next_logger;
373  }
374 
375  list = NULL;
376 }
FILE_TRUNCATED
#define FILE_TRUNCATED
Definition: util-file.h:45
Packet_::proto
uint8_t proto
Definition: decode.h:455
FileContainer_
Definition: util-file.h:110
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputLoggerThreadStore_
Definition: output-stats.c:31
OutputFiledataLogger_::name
const char * name
Definition: output-filedata.c:56
OutputFiledataLoggerThreadData_::store
OutputLoggerThreadStore * store
Definition: output-filedata.c:43
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output-stats.c:33
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
output-filedata.h
OutputFiledataLogger_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output-filedata.c:59
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:584
OutputFiledataLogger_::logger_id
LoggerId logger_id
Definition: output-filedata.c:57
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
Flow_::proto
uint8_t proto
Definition: flow.h:375
Packet_::flags
uint32_t flags
Definition: decode.h:468
Flow_
Flow data structure.
Definition: flow.h:353
File_::state
FileState state
Definition: util-file.h:78
OutputFiledataLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-filedata.c:54
LoggerId
LoggerId
Definition: suricata-common.h:437
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
OutputFiledataShutdown
void OutputFiledataShutdown(void)
Definition: output-filedata.c:366
FILE_STORE
#define FILE_STORE
Definition: util-file.h:55
File_::file_store_id
uint32_t file_store_id
Definition: util-file.h:82
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:223
tm-modules.h
OutputFiledataLogger_::next
struct OutputFiledataLogger_ * next
Definition: output-filedata.c:55
AppLayerParserGetFiles
FileContainer * AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
Definition: app-layer-parser.c:895
FiledataLogger
int(* FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir)
Definition: output-filedata.h:36
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:847
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:464
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:83
OUTPUT_FILEDATA_FLAG_CLOSE
#define OUTPUT_FILEDATA_FLAG_CLOSE
Definition: output-filedata.h:33
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:82
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output-stats.c:32
OutputCtx_
Definition: tm-modules.h:78
File_::sb
StreamingBuffer * sb
Definition: util-file.h:79
OutputRegisterFiledataLogger
int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output-filedata.c:65
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id)
FileCloseFilePtr
int FileCloseFilePtr(File *ff, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:932
FileForceMagic
int FileForceMagic(void)
Definition: util-file.c:142
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1178
detect-filemagic.h
FileContainer_::head
File * head
Definition: util-file.h:111
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:56
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
util-profiling.h
Packet_
Definition: decode.h:433
OutputFiledataLogger_::LogFunc
FiledataLogger LogFunc
Definition: output-filedata.c:53
OutputFiledataLogger_
Definition: output-filedata.c:52
conf.h
util-magic.h
TmEcode
TmEcode
Definition: tm-threads-common.h:81
OutputFiledataLoggerThreadData_
Definition: output-filedata.c:42
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1124
FileDataSize
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:294
File_::flags
uint16_t flags
Definition: util-file.h:76
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:258
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:67
File_
Definition: util-file.h:75
File_::content_stored
uint64_t content_stored
Definition: util-file.h:98
Packet_::flow
struct Flow_ * flow
Definition: decode.h:470
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
OutputFiledataLogger
struct OutputFiledataLogger_ OutputFiledataLogger
FILE_STORED
#define FILE_STORED
Definition: util-file.h:56
ThreadExitPrintStatsFunc
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
OUTPUT_FILEDATA_FLAG_OPEN
#define OUTPUT_FILEDATA_FLAG_OPEN
Definition: output-filedata.h:32
File_::next
struct File_ * next
Definition: util-file.h:89
OutputFiledataLoggerThreadData
struct OutputFiledataLoggerThreadData_ OutputFiledataLoggerThreadData
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1196
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:31
OutputFiledataLogger_::ThreadExitPrintStats
ThreadExitPrintStatsFunc ThreadExitPrintStats
Definition: output-filedata.c:60
util-validate.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
Packet_::next
struct Packet_ * next
Definition: decode.h:603
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:486
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:251
OutputRegisterRootLogger
void OutputRegisterRootLogger(ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats, OutputLogFunc LogFunc, OutputGetActiveCountFunc ActiveCntFunc)
Definition: output.c:965
g_filedata_logger_enabled
bool g_filedata_logger_enabled
Definition: output-filedata.c:38
File_::txid
uint64_t txid
Definition: util-file.h:80
OutputFiledataLogger_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output-filedata.c:58
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:302
OutputFiledataLoggerRegister
void OutputFiledataLoggerRegister(void)
Definition: output-filedata.c:357
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
output.h
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40
app-layer.h