suricata
output-filedata.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "output.h"
28 #include "output-filedata.h"
29 #include "app-layer-parser.h"
30 #include "detect-filemagic.h"
31 #include "conf.h"
32 #include "util-profiling.h"
33 #include "util-validate.h"
34 #include "util-magic.h"
35 #include "util-path.h"
36 
38 
39 /* logger instance, a module + a output ctx,
40  * it's perfectly valid that have multiple instances of the same
41  * log module (e.g. http.log) with different output ctx'. */
42 typedef struct OutputFiledataLogger_ {
46  const char *name;
52 
53 static OutputFiledataLogger *list = NULL;
54 
59 {
60  OutputFiledataLogger *op = SCMalloc(sizeof(*op));
61  if (op == NULL)
62  return -1;
63  memset(op, 0x00, sizeof(*op));
64 
65  op->LogFunc = LogFunc;
66  op->output_ctx = output_ctx;
67  op->name = name;
68  op->logger_id = id;
69  op->ThreadInit = ThreadInit;
72 
73  if (list == NULL)
74  list = op;
75  else {
76  OutputFiledataLogger *t = list;
77  while (t->next)
78  t = t->next;
79  t->next = op;
80  }
81 
82  SCLogDebug("OutputRegisterFiledataLogger happy");
84  return 0;
85 }
86 
87 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
88 
89 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, Packet *p, File *ff,
90  void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
91  uint8_t dir)
92 {
93  OutputFiledataLogger *logger = list;
94  OutputLoggerThreadStore *store = store_list;
95  int file_logged = 0;
96 
97  while (logger && store) {
98  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
99 
100  SCLogDebug("logger %p", logger);
102  logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, tx, tx_id, data, data_len,
103  flags, dir);
105 
106  file_logged = 1;
107 
108  logger = logger->next;
109  store = store->next;
110 
111  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
112  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
113  }
114  return file_logged;
115 }
116 
117 static void CloseFile(const Packet *p, Flow *f, File *file, void *txv)
118 {
119  DEBUG_VALIDATE_BUG_ON((file->flags & FILE_STORED) != 0);
120 
122  if (txd) {
123  BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0);
124  txd->files_stored++;
125  }
126  file->flags |= FILE_STORED;
127 }
128 
130  AppLayerGetFileState files, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
131  const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
132 {
133  SCLogDebug("ffc %p", files.fc);
134 
135  OutputLoggerThreadStore *store = td->store;
136  for (File *ff = files.fc->head; ff != NULL; ff = ff->next) {
137  FileApplyTxFlags(txd, dir, ff);
138  FilePrintFlags(ff);
139 
140  uint8_t file_flags = call_flags;
141 #ifdef HAVE_MAGIC
142  if (FileForceMagic() && ff->magic == NULL) {
143  FilemagicThreadLookup(&td->magic_ctx, ff);
144  }
145 #endif
146  if (ff->flags & FILE_STORED) {
147  continue;
148  }
149 
150  if (!(ff->flags & FILE_STORE)) {
151  continue;
152  }
153 
154  /* if file_store_id == 0, this is the first store of this file */
155  if (ff->file_store_id == 0) {
156  /* new file */
157  ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
158  file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
159  }
160 
161  /* if we have no data chunks left to log, we should still
162  * close the logger(s) */
163  if (FileDataSize(ff) == ff->content_stored && (file_trunc || file_close)) {
164  if (ff->state < FILE_STATE_CLOSED) {
165  FileCloseFilePtr(ff, files.cfg, NULL, 0, FILE_TRUNCATED);
166  }
167  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
168  CallLoggers(tv, store, p, ff, txv, tx_id, NULL, 0, file_flags, dir);
169  CloseFile(p, p->flow, ff, txv);
170  continue;
171  }
172 
173  /* if file needs to be closed or truncated, inform
174  * loggers */
175  if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
176  FileCloseFilePtr(ff, files.cfg, NULL, 0, FILE_TRUNCATED);
177  }
178 
179  /* tell the logger we're closing up */
180  if (ff->state >= FILE_STATE_CLOSED)
181  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
182 
183  /* do the actual logging */
184  const uint8_t *data = NULL;
185  uint32_t data_len = 0;
186 
187  StreamingBufferGetDataAtOffset(ff->sb, &data, &data_len, ff->content_stored);
188 
189  const int file_logged =
190  CallLoggers(tv, store, p, ff, txv, tx_id, data, data_len, file_flags, dir);
191  if (file_logged) {
192  ff->content_stored += data_len;
193 
194  /* all done */
195  if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
196  CloseFile(p, p->flow, ff, txv);
197  }
198  }
199  }
200 }
201 
202 /** \brief thread init for the filedata logger
203  * This will run the thread init functions for the individual registered
204  * loggers */
206 {
207  OutputFiledataLoggerThreadData *td = SCMalloc(sizeof(*td));
208  if (td == NULL)
209  return TM_ECODE_FAILED;
210  memset(td, 0x00, sizeof(*td));
211  *data = td;
212 
213 #ifdef HAVE_MAGIC
214  td->magic_ctx = MagicInitContext();
215  if (td->magic_ctx == NULL) {
216  SCFree(td);
217  return TM_ECODE_FAILED;
218  }
219 #endif
220 
221  SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
222 
223  OutputFiledataLogger *logger = list;
224  while (logger) {
225  if (logger->ThreadInit) {
226  void *retptr = NULL;
227  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
228  OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
229  /* todo */ BUG_ON(ts == NULL);
230  memset(ts, 0x00, sizeof(*ts));
231 
232  /* store thread handle */
233  ts->thread_data = retptr;
234 
235  if (td->store == NULL) {
236  td->store = ts;
237  } else {
238  OutputLoggerThreadStore *tmp = td->store;
239  while (tmp->next != NULL)
240  tmp = tmp->next;
241  tmp->next = ts;
242  }
243 
244  SCLogDebug("%s is now set up", logger->name);
245  }
246  }
247 
248  logger = logger->next;
249  }
250  return TM_ECODE_OK;
251 }
252 
254  ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
255 {
256  OutputLoggerThreadStore *store = op_thread_data->store;
257  OutputFiledataLogger *logger = list;
258 
259  while (logger && store) {
260  if (logger->ThreadDeinit) {
261  logger->ThreadDeinit(tv, store->thread_data);
262  }
263 
264  OutputLoggerThreadStore *next_store = store->next;
265  SCFree(store);
266  store = next_store;
267  logger = logger->next;
268  }
269 
270 #ifdef HAVE_MAGIC
271  MagicDeinitContext(op_thread_data->magic_ctx);
272 #endif
273 
274  SCFree(op_thread_data);
275  return TM_ECODE_OK;
276 }
277 
279 {
280  SC_ATOMIC_INIT(g_file_store_id);
281  SC_ATOMIC_SET(g_file_store_id, 1);
282 }
283 
285 {
286  OutputFiledataLogger *logger = list;
287  while (logger) {
288  OutputFiledataLogger *next_logger = logger->next;
289  SCFree(logger);
290  logger = next_logger;
291  }
292 
293  list = NULL;
294 }
OutputFiledataLogThreadInit
TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data)
thread init for the filedata logger This will run the thread init functions for the individual regist...
Definition: output-filedata.c:205
FILE_TRUNCATED
#define FILE_TRUNCATED
Definition: util-file.h:45
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputLoggerThreadStore_
Definition: output.h:33
OutputFiledataLogger_::name
const char * name
Definition: output-filedata.c:46
OutputFiledataLoggerThreadData_::store
OutputLoggerThreadStore * store
Definition: output-filedata.h:35
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
output-filedata.h
OutputFiledataLogger_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output-filedata.c:49
OutputFiledataLogger_::logger_id
LoggerId logger_id
Definition: output-filedata.c:47
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
Flow_::proto
uint8_t proto
Definition: flow.h:365
Flow_
Flow data structure.
Definition: flow.h:343
OutputFiledataLogThreadDeinit
TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
Definition: output-filedata.c:253
File_::state
FileState state
Definition: util-file.h:82
OutputFiledataLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-filedata.c:44
LoggerId
LoggerId
Definition: suricata-common.h:455
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
OutputFiledataShutdown
void OutputFiledataShutdown(void)
Definition: output-filedata.c:284
FILE_STORE
#define FILE_STORE
Definition: util-file.h:55
File_::file_store_id
uint32_t file_store_id
Definition: util-file.h:85
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output.h:35
OutputFiledataLogger_::next
struct OutputFiledataLogger_ * next
Definition: output-filedata.c:45
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1620
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
OUTPUT_FILEDATA_FLAG_CLOSE
#define OUTPUT_FILEDATA_FLAG_CLOSE
Definition: output-filedata.h:30
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
OutputCtx_
Definition: tm-modules.h:84
File_::sb
StreamingBuffer * sb
Definition: util-file.h:83
OutputRegisterFiledataLogger
int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output-filedata.c:55
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id)
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output.h:34
FileApplyTxFlags
void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file)
Definition: util-file.c:292
FileForceMagic
int FileForceMagic(void)
Definition: util-file.c:141
FiledataLogger
int(* FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, void *tx, const uint64_t tx_id, const uint8_t *, uint32_t, uint8_t, uint8_t dir)
Definition: output-filedata.h:49
detect-filemagic.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:39
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:295
util-profiling.h
Packet_
Definition: decode.h:429
OutputFiledataLogger_::LogFunc
FiledataLogger LogFunc
Definition: output-filedata.c:43
OutputFiledataLogger_
Definition: output-filedata.c:42
conf.h
util-magic.h
TmEcode
TmEcode
Definition: tm-threads-common.h:83
OutputFiledataLoggerThreadData_
Definition: output-filedata.h:34
FileCloseFilePtr
int FileCloseFilePtr(File *ff, const StreamingBufferConfig *sbcfg, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:995
OutputFiledataLogFfc
void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, AppLayerGetFileState files, void *txv, const uint64_t tx_id, AppLayerTxData *txd, const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
Definition: output-filedata.c:129
FileDataSize
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:323
File_::flags
uint16_t flags
Definition: util-file.h:80
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:241
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:71
File_
Definition: util-file.h:79
File_::content_stored
uint64_t content_stored
Definition: util-file.h:101
AppLayerTxData
struct AppLayerTxData AppLayerTxData
Definition: detect.h:1307
Packet_::flow
struct Flow_ * flow
Definition: decode.h:466
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
util-path.h
OutputFiledataLogger
struct OutputFiledataLogger_ OutputFiledataLogger
FILE_STORED
#define FILE_STORED
Definition: util-file.h:56
ThreadExitPrintStatsFunc
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41
OUTPUT_FILEDATA_FLAG_OPEN
#define OUTPUT_FILEDATA_FLAG_OPEN
Definition: output-filedata.h:29
File_::next
struct File_ * next
Definition: util-file.h:92
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1199
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputFiledataLogger_::ThreadExitPrintStats
ThreadExitPrintStatsFunc ThreadExitPrintStats
Definition: output-filedata.c:50
FilePrintFlags
#define FilePrintFlags(file)
Definition: util-file.h:250
util-validate.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCFree
#define SCFree(p)
Definition: util-mem.h:61
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:234
g_filedata_logger_enabled
bool g_filedata_logger_enabled
Definition: output-filedata.c:37
ALPROTO_SMB
@ ALPROTO_SMB
Definition: app-layer-protos.h:37
OutputFiledataLogger_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output-filedata.c:48
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:442
OutputFiledataLoggerRegister
void OutputFiledataLoggerRegister(void)
Definition: output-filedata.c:278
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:104
output.h
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:40