suricata
output-json-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Log files we track.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threadvars.h"
34 #include "tm-modules.h"
35 
36 #include "threads.h"
37 
38 #include "app-layer-parser.h"
39 
40 #include "detect-filemagic.h"
41 
42 #include "stream.h"
43 
44 #include "util-print.h"
45 #include "util-unittest.h"
46 #include "util-privs.h"
47 #include "util-debug.h"
48 #include "util-atomic.h"
49 #include "util-file.h"
50 #include "util-time.h"
51 #include "util-buffer.h"
52 #include "util-byte.h"
53 #include "util-validate.h"
54 
55 #include "log-file.h"
56 #include "util-logopenfile.h"
57 
58 #include "output.h"
59 #include "output-json.h"
60 #include "output-json-file.h"
61 #include "output-json-http.h"
62 #include "output-json-smtp.h"
64 #include "output-json-nfs.h"
65 #include "output-json-smb.h"
66 
67 #include "app-layer-htp.h"
68 #include "app-layer-htp-xff.h"
69 #include "util-memcmp.h"
70 #include "stream-tcp-reassemble.h"
71 
72 #ifdef HAVE_LIBJANSSON
73 
74 typedef struct OutputFileCtx_ {
75  LogFileCtx *file_ctx;
76  uint32_t file_cnt;
77  HttpXFFCfg *xff_cfg;
78  HttpXFFCfg *parent_xff_cfg;
79 } OutputFileCtx;
80 
81 typedef struct JsonFileLogThread_ {
82  OutputFileCtx *filelog_ctx;
83  MemBuffer *buffer;
84 } JsonFileLogThread;
85 
86 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
87  const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
88 {
89  json_t *hjs = NULL;
90  enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
91 
92  switch(dir) {
93  case STREAM_TOCLIENT:
94  fdir = LOG_DIR_FLOW_TOCLIENT;
95  break;
96  case STREAM_TOSERVER:
97  fdir = LOG_DIR_FLOW_TOSERVER;
98  break;
99  default:
101  break;
102  }
103 
104  json_t *js = CreateJSONHeader(p, fdir, "fileinfo");
105  if (unlikely(js == NULL))
106  return NULL;
107 
108  switch (p->flow->alproto) {
109  case ALPROTO_HTTP:
110  hjs = JsonHttpAddMetadata(p->flow, ff->txid);
111  if (hjs)
112  json_object_set_new(js, "http", hjs);
113  break;
114  case ALPROTO_SMTP:
115  hjs = JsonSMTPAddMetadata(p->flow, ff->txid);
116  if (hjs)
117  json_object_set_new(js, "smtp", hjs);
118  hjs = JsonEmailAddMetadata(p->flow, ff->txid);
119  if (hjs)
120  json_object_set_new(js, "email", hjs);
121  break;
122 #ifdef HAVE_RUST
123  case ALPROTO_NFS:
124  hjs = JsonNFSAddMetadataRPC(p->flow, ff->txid);
125  if (hjs)
126  json_object_set_new(js, "rpc", hjs);
127  hjs = JsonNFSAddMetadata(p->flow, ff->txid);
128  if (hjs)
129  json_object_set_new(js, "nfs", hjs);
130  break;
131  case ALPROTO_SMB:
132  hjs = JsonSMBAddMetadata(p->flow, ff->txid);
133  if (hjs)
134  json_object_set_new(js, "smb", hjs);
135  break;
136 #endif
137  }
138 
139  json_object_set_new(js, "app_proto",
140  json_string(AppProtoToString(p->flow->alproto)));
141 
142  json_t *fjs = json_object();
143  if (unlikely(fjs == NULL)) {
144  json_decref(js);
145  return NULL;
146  }
147 
148  char *s = BytesToString(ff->name, ff->name_len);
149  json_object_set_new(fjs, "filename", SCJsonString(s));
150  if (s != NULL)
151  SCFree(s);
152 #ifdef HAVE_MAGIC
153  if (ff->magic)
154  json_object_set_new(fjs, "magic", json_string((char *)ff->magic));
155 #endif
156  json_object_set_new(fjs, "gaps", json_boolean((ff->flags & FILE_HAS_GAPS)));
157  switch (ff->state) {
158  case FILE_STATE_CLOSED:
159  json_object_set_new(fjs, "state", json_string("CLOSED"));
160 #ifdef HAVE_NSS
161  if (ff->flags & FILE_MD5) {
162  size_t x;
163  int i;
164  char str[256];
165  for (i = 0, x = 0; x < sizeof(ff->md5); x++) {
166  i += snprintf(&str[i], 255-i, "%02x", ff->md5[x]);
167  }
168  json_object_set_new(fjs, "md5", json_string(str));
169  }
170  if (ff->flags & FILE_SHA1) {
171  size_t x;
172  int i;
173  char str[256];
174  for (i = 0, x = 0; x < sizeof(ff->sha1); x++) {
175  i += snprintf(&str[i], 255-i, "%02x", ff->sha1[x]);
176  }
177  json_object_set_new(fjs, "sha1", json_string(str));
178  }
179 #endif
180  break;
182  json_object_set_new(fjs, "state", json_string("TRUNCATED"));
183  break;
184  case FILE_STATE_ERROR:
185  json_object_set_new(fjs, "state", json_string("ERROR"));
186  break;
187  default:
188  json_object_set_new(fjs, "state", json_string("UNKNOWN"));
189  break;
190  }
191 
192 #ifdef HAVE_NSS
193  if (ff->flags & FILE_SHA256) {
194  size_t x;
195  int i;
196  char str[256];
197  for (i = 0, x = 0; x < sizeof(ff->sha256); x++) {
198  i += snprintf(&str[i], 255-i, "%02x", ff->sha256[x]);
199  }
200  json_object_set_new(fjs, "sha256", json_string(str));
201  }
202 #endif
203 
204  json_object_set_new(fjs, "stored", stored ? json_true() : json_false());
205  if (ff->flags & FILE_STORED) {
206  json_object_set_new(fjs, "file_id", json_integer(ff->file_store_id));
207  }
208  json_object_set_new(fjs, "size", json_integer(FileTrackedSize(ff)));
209  json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
210 
211  /* xff header */
212  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) {
213  int have_xff_ip = 0;
214  char buffer[XFF_MAXLEN];
215 
216  if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
217  have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, buffer, XFF_MAXLEN);
218  }
219 
220  if (have_xff_ip) {
221  if (xff_cfg->flags & XFF_EXTRADATA) {
222  json_object_set_new(js, "xff", json_string(buffer));
223  }
224  else if (xff_cfg->flags & XFF_OVERWRITE) {
225  if (p->flowflags & FLOW_PKT_TOCLIENT) {
226  json_object_set(js, "dest_ip", json_string(buffer));
227  } else {
228  json_object_set(js, "src_ip", json_string(buffer));
229  }
230  }
231  }
232  }
233 
234  /* originally just 'file', but due to bug 1127 naming it fileinfo */
235  json_object_set_new(js, "fileinfo", fjs);
236 
237  return js;
238 }
239 
240 /**
241  * \internal
242  * \brief Write meta data on a single line json record
243  */
244 static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
245  const File *ff, uint32_t dir)
246 {
247  HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ?
248  aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg;;
249  json_t *js = JsonBuildFileInfoRecord(p, ff,
250  ff->flags & FILE_STORED ? true : false, dir, xff_cfg);
251  if (unlikely(js == NULL)) {
252  return;
253  }
254 
255  MemBufferReset(aft->buffer);
256  OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer);
257  json_decref(js);
258 }
259 
260 static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
261  const File *ff, uint8_t dir)
262 {
263  SCEnter();
264  JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
265 
266  BUG_ON(ff->flags & FILE_LOGGED);
267 
268  SCLogDebug("ff %p", ff);
269 
270  FileWriteJsonRecord(aft, p, ff, dir);
271  return 0;
272 }
273 
274 
275 #define OUTPUT_BUFFER_SIZE 65535
276 static TmEcode JsonFileLogThreadInit(ThreadVars *t, const void *initdata, void **data)
277 {
278  JsonFileLogThread *aft = SCMalloc(sizeof(JsonFileLogThread));
279  if (unlikely(aft == NULL))
280  return TM_ECODE_FAILED;
281  memset(aft, 0, sizeof(JsonFileLogThread));
282 
283  if(initdata == NULL)
284  {
285  SCLogDebug("Error getting context for EveLogFile. \"initdata\" argument NULL");
286  SCFree(aft);
287  return TM_ECODE_FAILED;
288  }
289 
290  /* Use the Ouptut Context (file pointer and mutex) */
291  aft->filelog_ctx = ((OutputCtx *)initdata)->data;
292 
293  aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
294  if (aft->buffer == NULL) {
295  SCFree(aft);
296  return TM_ECODE_FAILED;
297  }
298 
299  *data = (void *)aft;
300  return TM_ECODE_OK;
301 }
302 
303 static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
304 {
305  JsonFileLogThread *aft = (JsonFileLogThread *)data;
306  if (aft == NULL) {
307  return TM_ECODE_OK;
308  }
309 
310  MemBufferFree(aft->buffer);
311  /* clear memory */
312  memset(aft, 0, sizeof(JsonFileLogThread));
313 
314  SCFree(aft);
315  return TM_ECODE_OK;
316 }
317 
318 static void OutputFileLogDeinitSub(OutputCtx *output_ctx)
319 {
320  OutputFileCtx *ff_ctx = output_ctx->data;
321  if (ff_ctx->xff_cfg != NULL) {
322  SCFree(ff_ctx->xff_cfg);
323  }
324  SCFree(ff_ctx);
325  SCFree(output_ctx);
326 }
327 
328 /** \brief Create a new http log LogFileCtx.
329  * \param conf Pointer to ConfNode containing this loggers configuration.
330  * \return NULL if failure, LogFileCtx* to the file_ctx if succesful
331  * */
332 static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
333 {
334  OutputInitResult result = { NULL, false };
335  OutputJsonCtx *ojc = parent_ctx->data;
336 
337  OutputFileCtx *output_file_ctx = SCCalloc(1, sizeof(OutputFileCtx));
338  if (unlikely(output_file_ctx == NULL))
339  return result;
340 
341  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
342  if (unlikely(output_ctx == NULL)) {
343  SCFree(output_file_ctx);
344  return result;
345  }
346 
347  output_file_ctx->file_ctx = ojc->file_ctx;
348 
349  if (conf) {
350  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
351  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
353  SCLogConfig("forcing filestore of all files");
354  }
355 
356  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
357  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
359  SCLogConfig("forcing magic lookup for logged files");
360  }
361 
362  FileForceHashParseCfg(conf);
363  }
364 
365  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
366  output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
367  if (output_file_ctx->xff_cfg != NULL) {
368  HttpXFFGetCfg(conf, output_file_ctx->xff_cfg);
369  }
370  } else if (ojc->xff_cfg) {
371  output_file_ctx->parent_xff_cfg = ojc->xff_cfg;
372  }
373 
374  output_ctx->data = output_file_ctx;
375  output_ctx->DeInit = OutputFileLogDeinitSub;
376 
378  result.ctx = output_ctx;
379  result.ok = true;
380  return result;
381 }
382 
383 void JsonFileLogRegister (void)
384 {
385  /* register as child of eve-log */
386  OutputRegisterFileSubModule(LOGGER_JSON_FILE, "eve-log", "JsonFileLog",
387  "eve-log.files", OutputFileLogInitSub, JsonFileLogger,
388  JsonFileLogThreadInit, JsonFileLogThreadDeinit, NULL);
389 }
390 
391 #else
392 
394 {
395 }
396 
397 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:444
#define BUG_ON(x)
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
#define unlikely(expr)
Definition: util-optimize.h:35
#define FILE_SHA256
Definition: util-file.h:43
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:992
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
#define XFF_MAXLEN
void FileForceTrackingEnable(void)
Definition: util-file.c:150
#define FILE_LOGGED
Definition: util-file.h:44
#define str(s)
#define SCCalloc(nm, a)
Definition: util-mem.h:205
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void OutputRegisterFileSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FileLogger FileLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file output sub-module.
Definition: output.c:457
uint16_t flags
Definition: util-file.h:65
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow...
#define FILE_HAS_GAPS
Definition: util-file.h:51
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileForceMagicEnable(void)
Definition: util-file.c:91
uint64_t txid
Definition: util-file.h:69
#define SCEnter(...)
Definition: util-debug.h:337
uint8_t flowflags
Definition: decode.h:438
uint16_t name_len
Definition: util-file.h:66
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
#define STREAM_TOCLIENT
Definition: stream.h:32
char * BytesToString(const uint8_t *bytes, size_t nbytes)
Turn byte array into string.
Definition: util-byte.c:40
#define FILE_MD5
Definition: util-file.h:39
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:174
#define XFF_OVERWRITE
#define SCFree(a)
Definition: util-mem.h:236
#define OUTPUT_BUFFER_SIZE
Definition: log-dnslog.c:58
void * data
Definition: tm-modules.h:81
#define STREAM_TOSERVER
Definition: stream.h:31
FileState state
Definition: util-file.h:67
uint32_t file_store_id
Definition: util-file.h:72
void JsonFileLogRegister(void)
#define XFF_DISABLED
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:194
AppProto alproto
application level protocol
Definition: flow.h:407
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
uint8_t * name
Definition: util-file.h:75
#define XFF_EXTRADATA
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
#define DEBUG_VALIDATE_BUG_ON(exp)