suricata
output-json-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Log files we track.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threadvars.h"
34 #include "tm-modules.h"
35 
36 #include "threads.h"
37 
38 #include "app-layer-parser.h"
39 
40 #include "detect-filemagic.h"
41 
42 #include "stream.h"
43 
44 #include "util-print.h"
45 #include "util-unittest.h"
46 #include "util-privs.h"
47 #include "util-debug.h"
48 #include "util-atomic.h"
49 #include "util-file.h"
50 #include "util-time.h"
51 #include "util-buffer.h"
52 #include "util-byte.h"
53 #include "util-validate.h"
54 
55 #include "util-logopenfile.h"
56 
57 #include "output.h"
58 #include "output-json.h"
59 #include "output-json-file.h"
60 #include "output-json-http.h"
61 #include "output-json-smtp.h"
63 #include "output-json-nfs.h"
64 #include "output-json-smb.h"
65 
66 #include "app-layer-htp.h"
67 #include "app-layer-htp-xff.h"
68 #include "util-memcmp.h"
69 #include "stream-tcp-reassemble.h"
70 
71 #ifdef HAVE_LIBJANSSON
72 
73 typedef struct OutputFileCtx_ {
74  LogFileCtx *file_ctx;
75  uint32_t file_cnt;
76  HttpXFFCfg *xff_cfg;
77  HttpXFFCfg *parent_xff_cfg;
78 } OutputFileCtx;
79 
80 typedef struct JsonFileLogThread_ {
81  OutputFileCtx *filelog_ctx;
82  MemBuffer *buffer;
83 } JsonFileLogThread;
84 
85 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
86  const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
87 {
88  json_t *hjs = NULL;
89  enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
90 
91  switch(dir) {
92  case STREAM_TOCLIENT:
93  fdir = LOG_DIR_FLOW_TOCLIENT;
94  break;
95  case STREAM_TOSERVER:
96  fdir = LOG_DIR_FLOW_TOSERVER;
97  break;
98  default:
100  break;
101  }
102 
103  json_t *js = CreateJSONHeader(p, fdir, "fileinfo");
104  if (unlikely(js == NULL))
105  return NULL;
106 
107  switch (p->flow->alproto) {
108  case ALPROTO_HTTP:
109  hjs = JsonHttpAddMetadata(p->flow, ff->txid);
110  if (hjs)
111  json_object_set_new(js, "http", hjs);
112  break;
113  case ALPROTO_SMTP:
114  hjs = JsonSMTPAddMetadata(p->flow, ff->txid);
115  if (hjs)
116  json_object_set_new(js, "smtp", hjs);
117  hjs = JsonEmailAddMetadata(p->flow, ff->txid);
118  if (hjs)
119  json_object_set_new(js, "email", hjs);
120  break;
121 #ifdef HAVE_RUST
122  case ALPROTO_NFS:
123  hjs = JsonNFSAddMetadataRPC(p->flow, ff->txid);
124  if (hjs)
125  json_object_set_new(js, "rpc", hjs);
126  hjs = JsonNFSAddMetadata(p->flow, ff->txid);
127  if (hjs)
128  json_object_set_new(js, "nfs", hjs);
129  break;
130  case ALPROTO_SMB:
131  hjs = JsonSMBAddMetadata(p->flow, ff->txid);
132  if (hjs)
133  json_object_set_new(js, "smb", hjs);
134  break;
135 #endif
136  }
137 
138  json_object_set_new(js, "app_proto",
139  json_string(AppProtoToString(p->flow->alproto)));
140 
141  json_t *fjs = json_object();
142  if (unlikely(fjs == NULL)) {
143  json_decref(js);
144  return NULL;
145  }
146 
147  size_t filename_size = ff->name_len * 2 + 1;
148  char filename_string[filename_size];
149  BytesToStringBuffer(ff->name, ff->name_len, filename_string, filename_size);
150  json_object_set_new(fjs, "filename", SCJsonString(filename_string));
151 
152  json_t *sig_ids = json_array();
153  if (unlikely(sig_ids == NULL)) {
154  json_decref(js);
155  return NULL;
156  }
157 
158  for (uint32_t i = 0; ff->sid != NULL && i < ff->sid_cnt; i++) {
159  json_array_append(sig_ids, json_integer(ff->sid[i]));
160  }
161  json_object_set_new(fjs, "sid", sig_ids);
162 
163 #ifdef HAVE_MAGIC
164  if (ff->magic)
165  json_object_set_new(fjs, "magic", json_string((char *)ff->magic));
166 #endif
167  json_object_set_new(fjs, "gaps", json_boolean((ff->flags & FILE_HAS_GAPS)));
168  switch (ff->state) {
169  case FILE_STATE_CLOSED:
170  json_object_set_new(fjs, "state", json_string("CLOSED"));
171 #ifdef HAVE_NSS
172  if (ff->flags & FILE_MD5) {
173  size_t x;
174  int i;
175  char str[256];
176  for (i = 0, x = 0; x < sizeof(ff->md5); x++) {
177  i += snprintf(&str[i], 255-i, "%02x", ff->md5[x]);
178  }
179  json_object_set_new(fjs, "md5", json_string(str));
180  }
181  if (ff->flags & FILE_SHA1) {
182  size_t x;
183  int i;
184  char str[256];
185  for (i = 0, x = 0; x < sizeof(ff->sha1); x++) {
186  i += snprintf(&str[i], 255-i, "%02x", ff->sha1[x]);
187  }
188  json_object_set_new(fjs, "sha1", json_string(str));
189  }
190 #endif
191  break;
193  json_object_set_new(fjs, "state", json_string("TRUNCATED"));
194  break;
195  case FILE_STATE_ERROR:
196  json_object_set_new(fjs, "state", json_string("ERROR"));
197  break;
198  default:
199  json_object_set_new(fjs, "state", json_string("UNKNOWN"));
200  break;
201  }
202 
203 #ifdef HAVE_NSS
204  if (ff->flags & FILE_SHA256) {
205  size_t x;
206  int i;
207  char str[256];
208  for (i = 0, x = 0; x < sizeof(ff->sha256); x++) {
209  i += snprintf(&str[i], 255-i, "%02x", ff->sha256[x]);
210  }
211  json_object_set_new(fjs, "sha256", json_string(str));
212  }
213 #endif
214 
215  json_object_set_new(fjs, "stored", stored ? json_true() : json_false());
216  if (ff->flags & FILE_STORED) {
217  json_object_set_new(fjs, "file_id", json_integer(ff->file_store_id));
218  }
219  json_object_set_new(fjs, "size", json_integer(FileTrackedSize(ff)));
220  if (ff->end > 0) {
221  json_object_set_new(fjs, "start", json_integer(ff->start));
222  json_object_set_new(fjs, "end", json_integer(ff->end));
223  }
224  json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
225 
226  /* xff header */
227  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) {
228  int have_xff_ip = 0;
229  char buffer[XFF_MAXLEN];
230 
231  if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
232  have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, buffer, XFF_MAXLEN);
233  }
234 
235  if (have_xff_ip) {
236  if (xff_cfg->flags & XFF_EXTRADATA) {
237  json_object_set_new(js, "xff", json_string(buffer));
238  }
239  else if (xff_cfg->flags & XFF_OVERWRITE) {
240  if (p->flowflags & FLOW_PKT_TOCLIENT) {
241  json_object_set(js, "dest_ip", json_string(buffer));
242  } else {
243  json_object_set(js, "src_ip", json_string(buffer));
244  }
245  }
246  }
247  }
248 
249  /* originally just 'file', but due to bug 1127 naming it fileinfo */
250  json_object_set_new(js, "fileinfo", fjs);
251 
252  return js;
253 }
254 
255 /**
256  * \internal
257  * \brief Write meta data on a single line json record
258  */
259 static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
260  const File *ff, uint32_t dir)
261 {
262  HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ?
263  aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg;;
264  json_t *js = JsonBuildFileInfoRecord(p, ff,
265  ff->flags & FILE_STORED ? true : false, dir, xff_cfg);
266  if (unlikely(js == NULL)) {
267  return;
268  }
269 
270  MemBufferReset(aft->buffer);
271  OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer);
272  json_decref(js);
273 }
274 
275 static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
276  const File *ff, uint8_t dir)
277 {
278  SCEnter();
279  JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
280 
281  BUG_ON(ff->flags & FILE_LOGGED);
282 
283  SCLogDebug("ff %p", ff);
284 
285  FileWriteJsonRecord(aft, p, ff, dir);
286  return 0;
287 }
288 
289 
290 #define OUTPUT_BUFFER_SIZE 65535
291 static TmEcode JsonFileLogThreadInit(ThreadVars *t, const void *initdata, void **data)
292 {
293  JsonFileLogThread *aft = SCMalloc(sizeof(JsonFileLogThread));
294  if (unlikely(aft == NULL))
295  return TM_ECODE_FAILED;
296  memset(aft, 0, sizeof(JsonFileLogThread));
297 
298  if(initdata == NULL)
299  {
300  SCLogDebug("Error getting context for EveLogFile. \"initdata\" argument NULL");
301  SCFree(aft);
302  return TM_ECODE_FAILED;
303  }
304 
305  /* Use the Ouptut Context (file pointer and mutex) */
306  aft->filelog_ctx = ((OutputCtx *)initdata)->data;
307 
308  aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
309  if (aft->buffer == NULL) {
310  SCFree(aft);
311  return TM_ECODE_FAILED;
312  }
313 
314  *data = (void *)aft;
315  return TM_ECODE_OK;
316 }
317 
318 static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
319 {
320  JsonFileLogThread *aft = (JsonFileLogThread *)data;
321  if (aft == NULL) {
322  return TM_ECODE_OK;
323  }
324 
325  MemBufferFree(aft->buffer);
326  /* clear memory */
327  memset(aft, 0, sizeof(JsonFileLogThread));
328 
329  SCFree(aft);
330  return TM_ECODE_OK;
331 }
332 
333 static void OutputFileLogDeinitSub(OutputCtx *output_ctx)
334 {
335  OutputFileCtx *ff_ctx = output_ctx->data;
336  if (ff_ctx->xff_cfg != NULL) {
337  SCFree(ff_ctx->xff_cfg);
338  }
339  SCFree(ff_ctx);
340  SCFree(output_ctx);
341 }
342 
343 /** \brief Create a new http log LogFileCtx.
344  * \param conf Pointer to ConfNode containing this loggers configuration.
345  * \return NULL if failure, LogFileCtx* to the file_ctx if succesful
346  * */
347 static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
348 {
349  OutputInitResult result = { NULL, false };
350  OutputJsonCtx *ojc = parent_ctx->data;
351 
352  OutputFileCtx *output_file_ctx = SCCalloc(1, sizeof(OutputFileCtx));
353  if (unlikely(output_file_ctx == NULL))
354  return result;
355 
356  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
357  if (unlikely(output_ctx == NULL)) {
358  SCFree(output_file_ctx);
359  return result;
360  }
361 
362  output_file_ctx->file_ctx = ojc->file_ctx;
363 
364  if (conf) {
365  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
366  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
368  SCLogConfig("forcing filestore of all files");
369  }
370 
371  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
372  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
374  SCLogConfig("forcing magic lookup for logged files");
375  }
376 
377  FileForceHashParseCfg(conf);
378  }
379 
380  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
381  output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
382  if (output_file_ctx->xff_cfg != NULL) {
383  HttpXFFGetCfg(conf, output_file_ctx->xff_cfg);
384  }
385  } else if (ojc->xff_cfg) {
386  output_file_ctx->parent_xff_cfg = ojc->xff_cfg;
387  }
388 
389  output_ctx->data = output_file_ctx;
390  output_ctx->DeInit = OutputFileLogDeinitSub;
391 
393  result.ctx = output_ctx;
394  result.ok = true;
395  return result;
396 }
397 
398 void JsonFileLogRegister (void)
399 {
400  /* register as child of eve-log */
401  OutputRegisterFileSubModule(LOGGER_JSON_FILE, "eve-log", "JsonFileLog",
402  "eve-log.files", OutputFileLogInitSub, JsonFileLogger,
403  JsonFileLogThreadInit, JsonFileLogThreadDeinit, NULL);
404 }
405 
406 #else
407 
409 {
410 }
411 
412 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
json_t * JsonSMBAddMetadata(const Flow *f, uint64_t tx_id)
#define BUG_ON(x)
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
#define unlikely(expr)
Definition: util-optimize.h:35
#define FILE_SHA256
Definition: util-file.h:43
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
uint32_t sid_cnt
Definition: util-file.h:96
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1060
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
#define XFF_MAXLEN
void FileForceTrackingEnable(void)
Definition: util-file.c:150
uint64_t start
Definition: util-file.h:92
#define FILE_LOGGED
Definition: util-file.h:44
#define str(s)
#define SCCalloc(nm, a)
Definition: util-mem.h:197
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void OutputRegisterFileSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FileLogger FileLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file output sub-module.
Definition: output.c:455
uint16_t flags
Definition: util-file.h:65
uint64_t end
Definition: util-file.h:93
uint32_t * sid
Definition: util-file.h:95
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow...
#define FILE_HAS_GAPS
Definition: util-file.h:51
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileForceMagicEnable(void)
Definition: util-file.c:91
uint64_t txid
Definition: util-file.h:69
void BytesToStringBuffer(const uint8_t *bytes, size_t nbytes, char *outstr, size_t outlen)
Turn byte array into string.
Definition: util-byte.c:85
#define SCEnter(...)
Definition: util-debug.h:337
uint8_t flowflags
Definition: decode.h:437
uint16_t name_len
Definition: util-file.h:66
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FILE_MD5
Definition: util-file.h:39
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:166
#define XFF_OVERWRITE
#define SCFree(a)
Definition: util-mem.h:228
void * data
Definition: tm-modules.h:81
#define STREAM_TOSERVER
Definition: stream.h:31
FileState state
Definition: util-file.h:67
uint32_t file_store_id
Definition: util-file.h:72
void JsonFileLogRegister(void)
#define XFF_DISABLED
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
#define OUTPUT_BUFFER_SIZE
Definition: log-httplog.c:58
uint8_t * name
Definition: util-file.h:75
#define XFF_EXTRADATA
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
#define DEBUG_VALIDATE_BUG_ON(exp)