suricata
output-json-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Log files we track.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threadvars.h"
34 #include "tm-modules.h"
35 
36 #include "threads.h"
37 
38 #include "app-layer-parser.h"
39 
40 #include "detect-filemagic.h"
41 
42 #include "stream.h"
43 
44 #include "util-print.h"
45 #include "util-unittest.h"
46 #include "util-privs.h"
47 #include "util-debug.h"
48 #include "util-atomic.h"
49 #include "util-file.h"
50 #include "util-time.h"
51 #include "util-buffer.h"
52 #include "util-byte.h"
53 #include "util-validate.h"
54 
55 #include "util-logopenfile.h"
56 
57 #include "output.h"
58 #include "output-json.h"
59 #include "output-json-file.h"
60 #include "output-json-http.h"
61 #include "output-json-smtp.h"
63 #include "output-json-nfs.h"
64 #include "output-json-smb.h"
65 #include "output-json-http2.h"
66 
67 #include "app-layer-htp.h"
68 #include "app-layer-htp-xff.h"
69 #include "util-memcmp.h"
70 #include "stream-tcp-reassemble.h"
71 
72 typedef struct OutputFileCtx_ {
74  uint32_t file_cnt;
78 
79 typedef struct JsonFileLogThread_ {
84 
85 JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
86  const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
87 {
89 
90  switch(dir) {
91  case STREAM_TOCLIENT:
92  fdir = LOG_DIR_FLOW_TOCLIENT;
93  break;
94  case STREAM_TOSERVER:
95  fdir = LOG_DIR_FLOW_TOSERVER;
96  break;
97  default:
99  break;
100  }
101 
103  JsonAddrInfoInit(p, fdir, &addr);
104 
105  /* Overwrite address info with XFF if needed. */
106  int have_xff_ip = 0;
107  char xff_buffer[XFF_MAXLEN];
108  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) {
109  if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
110  have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, xff_buffer, XFF_MAXLEN);
111  }
112  if (have_xff_ip && xff_cfg->flags & XFF_OVERWRITE) {
113  if (p->flowflags & FLOW_PKT_TOCLIENT) {
114  strlcpy(addr.dst_ip, xff_buffer, JSON_ADDR_LEN);
115  } else {
116  strlcpy(addr.src_ip, xff_buffer, JSON_ADDR_LEN);
117  }
118  have_xff_ip = 0;
119  }
120  }
121 
122  JsonBuilder *js = CreateEveHeader(p, fdir, "fileinfo", &addr);
123  if (unlikely(js == NULL))
124  return NULL;
125 
126  JsonBuilderMark mark = { 0, 0, 0 };
127  switch (p->flow->alproto) {
128  case ALPROTO_HTTP:
129  jb_open_object(js, "http");
130  EveHttpAddMetadata(p->flow, ff->txid, js);
131  jb_close(js);
132  break;
133  case ALPROTO_SMTP:
134  jb_get_mark(js, &mark);
135  jb_open_object(js, "smtp");
136  if (EveSMTPAddMetadata(p->flow, ff->txid, js)) {
137  jb_close(js);
138  } else {
139  jb_restore_mark(js, &mark);
140  }
141  jb_get_mark(js, &mark);
142  jb_open_object(js, "email");
143  if (EveEmailAddMetadata(p->flow, ff->txid, js)) {
144  jb_close(js);
145  } else {
146  jb_restore_mark(js, &mark);
147  }
148  break;
149  case ALPROTO_NFS:
150  /* rpc */
151  jb_get_mark(js, &mark);
152  jb_open_object(js, "rpc");
153  if (EveNFSAddMetadataRPC(p->flow, ff->txid, js)) {
154  jb_close(js);
155  } else {
156  jb_restore_mark(js, &mark);
157  }
158  /* nfs */
159  jb_get_mark(js, &mark);
160  jb_open_object(js, "nfs");
161  if (EveNFSAddMetadata(p->flow, ff->txid, js)) {
162  jb_close(js);
163  } else {
164  jb_restore_mark(js, &mark);
165  }
166  break;
167  case ALPROTO_SMB:
168  jb_get_mark(js, &mark);
169  jb_open_object(js, "smb");
170  if (EveSMBAddMetadata(p->flow, ff->txid, js)) {
171  jb_close(js);
172  } else {
173  jb_restore_mark(js, &mark);
174  }
175  break;
176  case ALPROTO_HTTP2:
177  jb_get_mark(js, &mark);
178  jb_open_object(js, "http2");
179  if (EveHTTP2AddMetadata(p->flow, ff->txid, js)) {
180  jb_close(js);
181  } else {
182  jb_restore_mark(js, &mark);
183  }
184  break;
185  }
186 
187  jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto));
188 
189  jb_open_object(js, "fileinfo");
190  EveFileInfo(js, ff, stored);
191  jb_close(js);
192 
193  /* xff header */
194  if (have_xff_ip && xff_cfg->flags & XFF_EXTRADATA) {
195  jb_set_string(js, "xff", xff_buffer);
196  }
197 
198  return js;
199 }
200 
201 /**
202  * \internal
203  * \brief Write meta data on a single line json record
204  */
205 static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
206  const File *ff, uint32_t dir)
207 {
208  HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ?
210  JsonBuilder *js = JsonBuildFileInfoRecord(p, ff,
211  ff->flags & FILE_STORED ? true : false, dir, xff_cfg);
212  if (unlikely(js == NULL)) {
213  return;
214  }
215 
216  MemBufferReset(aft->buffer);
217  OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer);
218  jb_free(js);
219 }
220 
221 static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
222  const File *ff, uint8_t dir)
223 {
224  SCEnter();
225  JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
226 
227  BUG_ON(ff->flags & FILE_LOGGED);
228 
229  SCLogDebug("ff %p", ff);
230 
231  FileWriteJsonRecord(aft, p, ff, dir);
232  return 0;
233 }
234 
235 
236 static TmEcode JsonFileLogThreadInit(ThreadVars *t, const void *initdata, void **data)
237 {
238  JsonFileLogThread *aft = SCCalloc(1, sizeof(JsonFileLogThread));
239  if (unlikely(aft == NULL))
240  return TM_ECODE_FAILED;
241 
242  if(initdata == NULL)
243  {
244  SCLogDebug("Error getting context for EveLogFile. \"initdata\" argument NULL");
245  goto error_exit;
246  }
247 
249  if (aft->buffer == NULL) {
250  goto error_exit;
251  }
252 
253  /* Use the Ouptut Context (file pointer and mutex) */
254  aft->filelog_ctx = ((OutputCtx *)initdata)->data;
256  if (!aft->file_ctx) {
257  goto error_exit;
258  }
259 
260  *data = (void *)aft;
261  return TM_ECODE_OK;
262 
263 error_exit:
264  if (aft->buffer != NULL) {
265  MemBufferFree(aft->buffer);
266  }
267  SCFree(aft);
268  return TM_ECODE_FAILED;
269 }
270 
271 static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
272 {
273  JsonFileLogThread *aft = (JsonFileLogThread *)data;
274  if (aft == NULL) {
275  return TM_ECODE_OK;
276  }
277 
278  MemBufferFree(aft->buffer);
279  /* clear memory */
280  memset(aft, 0, sizeof(JsonFileLogThread));
281 
282  SCFree(aft);
283  return TM_ECODE_OK;
284 }
285 
286 static void OutputFileLogDeinitSub(OutputCtx *output_ctx)
287 {
288  OutputFileCtx *ff_ctx = output_ctx->data;
289  if (ff_ctx->xff_cfg != NULL) {
290  SCFree(ff_ctx->xff_cfg);
291  }
292  SCFree(ff_ctx);
293  SCFree(output_ctx);
294 }
295 
296 /** \brief Create a new http log LogFileCtx.
297  * \param conf Pointer to ConfNode containing this loggers configuration.
298  * \return NULL if failure, LogFileCtx* to the file_ctx if succesful
299  * */
300 static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
301 {
302  OutputInitResult result = { NULL, false };
303  OutputJsonCtx *ojc = parent_ctx->data;
304 
305  OutputFileCtx *output_file_ctx = SCCalloc(1, sizeof(OutputFileCtx));
306  if (unlikely(output_file_ctx == NULL))
307  return result;
308 
309  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
310  if (unlikely(output_ctx == NULL)) {
311  SCFree(output_file_ctx);
312  return result;
313  }
314 
315  output_file_ctx->file_ctx = ojc->file_ctx;
316 
317  if (conf) {
318  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
319  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
321  SCLogConfig("forcing filestore of all files");
322  }
323 
324  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
325  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
327  SCLogConfig("forcing magic lookup for logged files");
328  }
329 
330  FileForceHashParseCfg(conf);
331  }
332 
333  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
334  output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
335  if (output_file_ctx->xff_cfg != NULL) {
336  HttpXFFGetCfg(conf, output_file_ctx->xff_cfg);
337  }
338  } else if (ojc->xff_cfg) {
339  output_file_ctx->parent_xff_cfg = ojc->xff_cfg;
340  }
341 
342  output_ctx->data = output_file_ctx;
343  output_ctx->DeInit = OutputFileLogDeinitSub;
344 
346  result.ctx = output_ctx;
347  result.ok = true;
348  return result;
349 }
350 
352 {
353  /* register as child of eve-log */
354  OutputRegisterFileSubModule(LOGGER_JSON_FILE, "eve-log", "JsonFileLog",
355  "eve-log.files", OutputFileLogInitSub, JsonFileLogger,
356  JsonFileLogThreadInit, JsonFileLogThreadDeinit, NULL);
357 }
XFF_MAXLEN
#define XFF_MAXLEN
Definition: app-layer-htp-xff.h:39
util-byte.h
JsonBuildFileInfoRecord
JsonBuilder * JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
Definition: output-json-file.c:85
EveSMTPAddMetadata
bool EveSMTPAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js)
Definition: output-json-smtp.c:101
OutputJsonCtx_::xff_cfg
HttpXFFCfg * xff_cfg
Definition: output-json.h:105
XFF_EXTRADATA
#define XFF_EXTRADATA
Definition: app-layer-htp-xff.h:31
FileForceTrackingEnable
void FileForceTrackingEnable(void)
Definition: util-file.c:163
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
JSON_OUTPUT_BUFFER_SIZE
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:63
threads.h
OutputJsonCtx_
Definition: output-json.h:101
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
LogFileCtx_
Definition: util-logopenfile.h:60
JsonFileLogRegister
void JsonFileLogRegister(void)
Definition: output-json-file.c:351
EveNFSAddMetadata
bool EveNFSAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *jb)
Definition: output-json-nfs.c:63
json_addr_info_zero
const JsonAddrInfo json_addr_info_zero
Definition: output-json.c:89
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:978
OutputRegisterFileSubModule
void OutputRegisterFileSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FileLogger FileLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file output sub-module.
Definition: output.c:466
tm-modules.h
JsonFileLogThread_::buffer
MemBuffer * buffer
Definition: output-json-file.c:82
util-privs.h
stream-tcp-reassemble.h
FileForceMagicEnable
void FileForceMagicEnable(void)
Definition: util-file.c:100
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:445
OutputFileCtx_::xff_cfg
HttpXFFCfg * xff_cfg
Definition: output-json-file.c:75
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:81
util-unittest.h
EveHttpAddMetadata
bool EveHttpAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js)
Definition: output-json-http.c:528
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
OutputCtx_::data
void * data
Definition: tm-modules.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
OutputCtx_
Definition: tm-modules.h:78
app-layer-htp-xff.h
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
util-memcmp.h
app-layer-htp.h
util-debug.h
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:44
CreateEveHeader
JsonBuilder * CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr)
Definition: output-json.c:846
LOGGER_JSON_FILE
@ LOGGER_JSON_FILE
Definition: suricata-common.h:482
output-json.h
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
output-json-file.h
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
util-print.h
detect-filemagic.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
pkt-var.h
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:659
EveFileInfo
void EveFileInfo(JsonBuilder *jb, const File *ff, const bool stored)
Definition: output-json.c:141
util-atomic.h
output-json-email-common.h
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:45
app-layer-parser.h
ThreadVars_::id
int id
Definition: threadvars.h:87
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
stream.h
JsonAddrInfo_
Definition: output-json.h:49
Packet_
Definition: decode.h:414
conf.h
EveHTTP2AddMetadata
bool EveHTTP2AddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *jb)
Definition: output-json-http2.c:68
XFF_OVERWRITE
#define XFF_OVERWRITE
Definition: app-layer-htp-xff.h:33
TmEcode
TmEcode
Definition: tm-threads-common.h:79
JsonFileLogThread_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-file.c:81
EveNFSAddMetadataRPC
bool EveNFSAddMetadataRPC(const Flow *f, uint64_t tx_id, JsonBuilder *jb)
Definition: output-json-nfs.c:51
HttpXFFCfg_
Definition: app-layer-htp-xff.h:41
LOG_DIR_FLOW_TOCLIENT
@ LOG_DIR_FLOW_TOCLIENT
Definition: output-json.h:41
ALPROTO_HTTP2
@ ALPROTO_HTTP2
Definition: app-layer-protos.h:59
MemBuffer_
Definition: util-buffer.h:27
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:220
EveEmailAddMetadata
bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
Definition: output-json-email-common.c:388
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:814
File_::flags
uint16_t flags
Definition: util-file.h:64
util-file.h
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
File_
Definition: util-file.h:63
OutputInitResult_
Definition: output.h:43
Packet_::flow
struct Flow_ * flow
Definition: decode.h:451
suricata-common.h
FileForceFilestoreEnable
void FileForceFilestoreEnable(void)
Definition: util-file.c:94
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
OutputFileCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-file.c:73
JSON_ADDR_LEN
#define JSON_ADDR_LEN
Definition: output-json.h:45
output-json-nfs.h
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
FILE_STORED
#define FILE_STORED
Definition: util-file.h:47
JsonFileLogThread_
Definition: output-json-file.c:79
FileForceHashParseCfg
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:172
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
OutputJsonLogDirection
OutputJsonLogDirection
Definition: output-json.h:38
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
threadvars.h
util-validate.h
LOG_DIR_FLOW
@ LOG_DIR_FLOW
Definition: output-json.h:40
output-json-http2.h
FlowGetAppProtocol
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1097
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
JsonAddrInfo_::src_ip
char src_ip[JSON_ADDR_LEN]
Definition: output-json.h:50
HttpXFFCfg_::flags
uint8_t flags
Definition: app-layer-htp-xff.h:42
FILE_LOGGED
#define FILE_LOGGED
Definition: util-file.h:44
LOG_DIR_FLOW_TOSERVER
@ LOG_DIR_FLOW_TOSERVER
Definition: output-json.h:42
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
util-buffer.h
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:30
JsonAddrInfo_::dst_ip
char dst_ip[JSON_ADDR_LEN]
Definition: output-json.h:51
OutputJsonCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:102
JsonFileLogThread
struct JsonFileLogThread_ JsonFileLogThread
JsonAddrInfoInit
void JsonAddrInfoInit(const Packet *p, enum OutputJsonLogDirection dir, JsonAddrInfo *addr)
Definition: output-json.c:508
File_::txid
uint64_t txid
Definition: util-file.h:68
HttpXFFGetCfg
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
Definition: app-layer-htp-xff.c:202
output-json-smb.h
ALPROTO_SMB
@ ALPROTO_SMB
Definition: app-layer-protos.h:37
OutputFileCtx_
Definition: output-json-file.c:72
HttpXFFGetIPFromTx
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow.
Definition: app-layer-htp-xff.c:113
output-json-smtp.h
output-json-http.h
EveSMBAddMetadata
bool EveSMBAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *jb)
Definition: output-json-smb.c:50
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
OutputFileCtx
struct OutputFileCtx_ OutputFileCtx
XFF_DISABLED
#define XFF_DISABLED
Definition: app-layer-htp-xff.h:29
JsonFileLogThread_::filelog_ctx
OutputFileCtx * filelog_ctx
Definition: output-json-file.c:80
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
ALPROTO_NFS
@ ALPROTO_NFS
Definition: app-layer-protos.h:45
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
debug.h
OutputFileCtx_::file_cnt
uint32_t file_cnt
Definition: output-json-file.c:74
output.h
OutputFileCtx_::parent_xff_cfg
HttpXFFCfg * parent_xff_cfg
Definition: output-json-file.c:76
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842