suricata
output-filedata.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "output.h"
28 #include "output-filedata.h"
29 #include "app-layer-parser.h"
30 #include "detect-filemagic.h"
31 #include "conf.h"
32 #include "util-profiling.h"
33 #include "util-validate.h"
34 #include "util-magic.h"
35 #include "util-path.h"
36 
38 
39 /* logger instance, a module + a output ctx,
40  * it's perfectly valid that have multiple instances of the same
41  * log module (e.g. http.log) with different output ctx'. */
42 typedef struct OutputFiledataLogger_ {
46  const char *name;
52 
53 static OutputFiledataLogger *list = NULL;
54 
59 {
60  OutputFiledataLogger *op = SCCalloc(1, sizeof(*op));
61  if (op == NULL)
62  return -1;
63 
64  op->LogFunc = LogFunc;
65  op->output_ctx = output_ctx;
66  op->name = name;
67  op->logger_id = id;
68  op->ThreadInit = ThreadInit;
71 
72  if (list == NULL)
73  list = op;
74  else {
75  OutputFiledataLogger *t = list;
76  while (t->next)
77  t = t->next;
78  t->next = op;
79  }
80 
81  SCLogDebug("OutputRegisterFiledataLogger happy");
83  return 0;
84 }
85 
86 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
87 
88 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, Packet *p, File *ff,
89  void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
90  uint8_t dir)
91 {
92  OutputFiledataLogger *logger = list;
93  OutputLoggerThreadStore *store = store_list;
94  int file_logged = 0;
95 
96  while (logger && store) {
97  DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
98 
99  SCLogDebug("logger %p", logger);
101  logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, tx, tx_id, data, data_len,
102  flags, dir);
104 
105  file_logged = 1;
106 
107  logger = logger->next;
108  store = store->next;
109 
110  DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
111  DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
112  }
113  return file_logged;
114 }
115 
116 static void CloseFile(const Packet *p, Flow *f, File *file, void *txv)
117 {
118  DEBUG_VALIDATE_BUG_ON((file->flags & FILE_STORED) != 0);
119 
121  if (txd) {
122  BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0);
123  txd->files_stored++;
124  }
125  file->flags |= FILE_STORED;
126 }
127 
129  AppLayerGetFileState files, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
130  const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
131 {
132  SCLogDebug("ffc %p", files.fc);
133 
134  OutputLoggerThreadStore *store = td->store;
135  for (File *ff = files.fc->head; ff != NULL; ff = ff->next) {
136  FileApplyTxFlags(txd, dir, ff);
137  FilePrintFlags(ff);
138 
139  uint8_t file_flags = call_flags;
140 #ifdef HAVE_MAGIC
141  if (FileForceMagic() && ff->magic == NULL) {
142  FilemagicThreadLookup(&td->magic_ctx, ff);
143  }
144 #endif
145  if (ff->flags & FILE_STORED) {
146  continue;
147  }
148 
149  if (!(ff->flags & FILE_STORE)) {
150  continue;
151  }
152 
153  /* if file_store_id == 0, this is the first store of this file */
154  if (ff->file_store_id == 0) {
155  /* new file */
156  ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
157  file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
158  }
159 
160  /* if we have no data chunks left to log, we should still
161  * close the logger(s) */
162  if (FileDataSize(ff) == ff->content_stored && (file_trunc || file_close)) {
163  if (ff->state < FILE_STATE_CLOSED) {
164  FileCloseFilePtr(ff, files.cfg, NULL, 0, FILE_TRUNCATED);
165  }
166  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
167  CallLoggers(tv, store, p, ff, txv, tx_id, NULL, 0, file_flags, dir);
168  CloseFile(p, p->flow, ff, txv);
169  continue;
170  }
171 
172  /* if file needs to be closed or truncated, inform
173  * loggers */
174  if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
175  FileCloseFilePtr(ff, files.cfg, NULL, 0, FILE_TRUNCATED);
176  }
177 
178  /* tell the logger we're closing up */
179  if (ff->state >= FILE_STATE_CLOSED)
180  file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
181 
182  /* do the actual logging */
183  const uint8_t *data = NULL;
184  uint32_t data_len = 0;
185 
186  StreamingBufferGetDataAtOffset(ff->sb, &data, &data_len, ff->content_stored);
187 
188  const int file_logged =
189  CallLoggers(tv, store, p, ff, txv, tx_id, data, data_len, file_flags, dir);
190  if (file_logged) {
191  ff->content_stored += data_len;
192 
193  /* all done */
194  if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
195  CloseFile(p, p->flow, ff, txv);
196  }
197  }
198  }
199 }
200 
201 /** \brief thread init for the filedata logger
202  * This will run the thread init functions for the individual registered
203  * loggers */
205 {
206  OutputFiledataLoggerThreadData *td = SCCalloc(1, sizeof(*td));
207  if (td == NULL)
208  return TM_ECODE_FAILED;
209  *data = td;
210 
211 #ifdef HAVE_MAGIC
212  td->magic_ctx = MagicInitContext();
213  if (td->magic_ctx == NULL) {
214  SCFree(td);
215  return TM_ECODE_FAILED;
216  }
217 #endif
218 
219  SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
220 
221  OutputFiledataLogger *logger = list;
222  while (logger) {
223  if (logger->ThreadInit) {
224  void *retptr = NULL;
225  if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
226  OutputLoggerThreadStore *ts = SCCalloc(1, sizeof(*ts));
227  /* todo */ BUG_ON(ts == NULL);
228 
229  /* store thread handle */
230  ts->thread_data = retptr;
231 
232  if (td->store == NULL) {
233  td->store = ts;
234  } else {
235  OutputLoggerThreadStore *tmp = td->store;
236  while (tmp->next != NULL)
237  tmp = tmp->next;
238  tmp->next = ts;
239  }
240 
241  SCLogDebug("%s is now set up", logger->name);
242  }
243  }
244 
245  logger = logger->next;
246  }
247  return TM_ECODE_OK;
248 }
249 
251  ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
252 {
253  OutputLoggerThreadStore *store = op_thread_data->store;
254  OutputFiledataLogger *logger = list;
255 
256  while (logger && store) {
257  if (logger->ThreadDeinit) {
258  logger->ThreadDeinit(tv, store->thread_data);
259  }
260 
261  OutputLoggerThreadStore *next_store = store->next;
262  SCFree(store);
263  store = next_store;
264  logger = logger->next;
265  }
266 
267 #ifdef HAVE_MAGIC
268  MagicDeinitContext(op_thread_data->magic_ctx);
269 #endif
270 
271  SCFree(op_thread_data);
272  return TM_ECODE_OK;
273 }
274 
276 {
277  SC_ATOMIC_INIT(g_file_store_id);
278  SC_ATOMIC_SET(g_file_store_id, 1);
279 }
280 
282 {
283  OutputFiledataLogger *logger = list;
284  while (logger) {
285  OutputFiledataLogger *next_logger = logger->next;
286  SCFree(logger);
287  logger = next_logger;
288  }
289 
290  list = NULL;
291 }
OutputFiledataLogThreadInit
TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data)
thread init for the filedata logger This will run the thread init functions for the individual regist...
Definition: output-filedata.c:204
FILE_TRUNCATED
#define FILE_TRUNCATED
Definition: util-file.h:45
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputLoggerThreadStore_
Definition: output.h:33
OutputFiledataLogger_::name
const char * name
Definition: output-filedata.c:46
OutputFiledataLoggerThreadData_::store
OutputLoggerThreadStore * store
Definition: output-filedata.h:35
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
output-filedata.h
OutputFiledataLogger_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output-filedata.c:49
OutputFiledataLogger_::logger_id
LoggerId logger_id
Definition: output-filedata.c:47
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
Flow_::proto
uint8_t proto
Definition: flow.h:373
Flow_
Flow data structure.
Definition: flow.h:351
OutputFiledataLogThreadDeinit
TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
Definition: output-filedata.c:250
File_::state
FileState state
Definition: util-file.h:82
OutputFiledataLogger_::output_ctx
OutputCtx * output_ctx
Definition: output-filedata.c:44
LoggerId
LoggerId
Definition: suricata-common.h:460
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
OutputFiledataShutdown
void OutputFiledataShutdown(void)
Definition: output-filedata.c:281
FILE_STORE
#define FILE_STORE
Definition: util-file.h:55
File_::file_store_id
uint32_t file_store_id
Definition: util-file.h:85
OutputLoggerThreadStore_::next
struct OutputLoggerThreadStore_ * next
Definition: output.h:35
OutputFiledataLogger_::next
struct OutputFiledataLogger_ * next
Definition: output-filedata.c:45
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1771
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
OUTPUT_FILEDATA_FLAG_CLOSE
#define OUTPUT_FILEDATA_FLAG_CLOSE
Definition: output-filedata.h:30
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
OutputCtx_
Definition: tm-modules.h:85
File_::sb
StreamingBuffer * sb
Definition: util-file.h:83
OutputRegisterFiledataLogger
int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output-filedata.c:55
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id)
OutputLoggerThreadStore_::thread_data
void * thread_data
Definition: output.h:34
FileApplyTxFlags
void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file)
Definition: util-file.c:295
FileForceMagic
int FileForceMagic(void)
Definition: util-file.c:141
FiledataLogger
int(* FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, void *tx, const uint64_t tx_id, const uint8_t *, uint32_t, uint8_t, uint8_t dir)
Definition: output-filedata.h:49
detect-filemagic.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
ThreadInitFunc
TmEcode(* ThreadInitFunc)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:40
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
util-profiling.h
Packet_
Definition: decode.h:437
OutputFiledataLogger_::LogFunc
FiledataLogger LogFunc
Definition: output-filedata.c:43
OutputFiledataLogger_
Definition: output-filedata.c:42
conf.h
util-magic.h
TmEcode
TmEcode
Definition: tm-threads-common.h:83
OutputFiledataLoggerThreadData_
Definition: output-filedata.h:34
FileCloseFilePtr
int FileCloseFilePtr(File *ff, const StreamingBufferConfig *sbcfg, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:994
OutputFiledataLogFfc
void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, AppLayerGetFileState files, void *txv, const uint64_t tx_id, AppLayerTxData *txd, const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
Definition: output-filedata.c:128
FileDataSize
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:326
File_::flags
uint16_t flags
Definition: util-file.h:80
PACKET_PROFILING_LOGGER_END
#define PACKET_PROFILING_LOGGER_END(p, id)
Definition: util-profiling.h:241
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:71
File_
Definition: util-file.h:79
File_::content_stored
uint64_t content_stored
Definition: util-file.h:101
AppLayerTxData
struct AppLayerTxData AppLayerTxData
Definition: detect.h:1355
Packet_::flow
struct Flow_ * flow
Definition: decode.h:476
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
util-path.h
OutputFiledataLogger
struct OutputFiledataLogger_ OutputFiledataLogger
FILE_STORED
#define FILE_STORED
Definition: util-file.h:56
ThreadExitPrintStatsFunc
void(* ThreadExitPrintStatsFunc)(ThreadVars *, void *)
Definition: tm-modules.h:42
OUTPUT_FILEDATA_FLAG_OPEN
#define OUTPUT_FILEDATA_FLAG_OPEN
Definition: output-filedata.h:29
File_::next
struct File_ * next
Definition: util-file.h:92
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1195
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputFiledataLogger_::ThreadExitPrintStats
ThreadExitPrintStatsFunc ThreadExitPrintStats
Definition: output-filedata.c:50
FilePrintFlags
#define FilePrintFlags(file)
Definition: util-file.h:250
util-validate.h
SCFree
#define SCFree(p)
Definition: util-mem.h:61
PACKET_PROFILING_LOGGER_START
#define PACKET_PROFILING_LOGGER_START(p, id)
Definition: util-profiling.h:234
g_filedata_logger_enabled
bool g_filedata_logger_enabled
Definition: output-filedata.c:37
ALPROTO_SMB
@ ALPROTO_SMB
Definition: app-layer-protos.h:37
OutputFiledataLogger_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output-filedata.c:48
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
OutputFiledataLoggerRegister
void OutputFiledataLoggerRegister(void)
Definition: output-filedata.c:275
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:103
output.h
ThreadDeinitFunc
TmEcode(* ThreadDeinitFunc)(ThreadVars *, void *)
Definition: tm-modules.h:41