suricata
output-json-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Log files we track.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threadvars.h"
34 #include "tm-modules.h"
35 
36 #include "threads.h"
37 
38 #include "app-layer-parser.h"
39 
40 #include "detect-filemagic.h"
41 
42 #include "stream.h"
43 
44 #include "util-print.h"
45 #include "util-unittest.h"
46 #include "util-privs.h"
47 #include "util-debug.h"
48 #include "util-atomic.h"
49 #include "util-file.h"
50 #include "util-time.h"
51 #include "util-buffer.h"
52 #include "util-byte.h"
53 #include "util-validate.h"
54 
55 #include "util-logopenfile.h"
56 
57 #include "output.h"
58 #include "output-json.h"
59 #include "output-json-file.h"
60 #include "output-json-http.h"
61 #include "output-json-smtp.h"
63 #include "output-json-nfs.h"
64 #include "output-json-smb.h"
65 
66 #include "app-layer-htp.h"
67 #include "app-layer-htp-xff.h"
68 #include "util-memcmp.h"
69 #include "stream-tcp-reassemble.h"
70 
71 typedef struct OutputFileCtx_ {
73  uint32_t file_cnt;
77 
78 typedef struct JsonFileLogThread_ {
82 
83 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
84  const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
85 {
86  json_t *hjs = NULL;
87  enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
88 
89  switch(dir) {
90  case STREAM_TOCLIENT:
91  fdir = LOG_DIR_FLOW_TOCLIENT;
92  break;
93  case STREAM_TOSERVER:
94  fdir = LOG_DIR_FLOW_TOSERVER;
95  break;
96  default:
98  break;
99  }
100 
101  json_t *js = CreateJSONHeader(p, fdir, "fileinfo");
102  if (unlikely(js == NULL))
103  return NULL;
104 
105  switch (p->flow->alproto) {
106  case ALPROTO_HTTP:
107  hjs = JsonHttpAddMetadata(p->flow, ff->txid);
108  if (hjs)
109  json_object_set_new(js, "http", hjs);
110  break;
111  case ALPROTO_SMTP:
112  hjs = JsonSMTPAddMetadata(p->flow, ff->txid);
113  if (hjs)
114  json_object_set_new(js, "smtp", hjs);
115  hjs = JsonEmailAddMetadata(p->flow, ff->txid);
116  if (hjs)
117  json_object_set_new(js, "email", hjs);
118  break;
119  case ALPROTO_NFS:
120  hjs = JsonNFSAddMetadataRPC(p->flow, ff->txid);
121  if (hjs)
122  json_object_set_new(js, "rpc", hjs);
123  hjs = JsonNFSAddMetadata(p->flow, ff->txid);
124  if (hjs)
125  json_object_set_new(js, "nfs", hjs);
126  break;
127  case ALPROTO_SMB:
128  hjs = JsonSMBAddMetadata(p->flow, ff->txid);
129  if (hjs)
130  json_object_set_new(js, "smb", hjs);
131  break;
132  }
133 
134  json_object_set_new(js, "app_proto",
135  json_string(AppProtoToString(p->flow->alproto)));
136 
137  json_t *fjs = json_object();
138  if (unlikely(fjs == NULL)) {
139  json_decref(js);
140  return NULL;
141  }
142 
143  size_t filename_size = ff->name_len * 2 + 1;
144  char filename_string[filename_size];
145  BytesToStringBuffer(ff->name, ff->name_len, filename_string, filename_size);
146  json_object_set_new(fjs, "filename", SCJsonString(filename_string));
147 
148  json_t *sig_ids = json_array();
149  if (unlikely(sig_ids == NULL)) {
150  json_decref(js);
151  return NULL;
152  }
153 
154  for (uint32_t i = 0; ff->sid != NULL && i < ff->sid_cnt; i++) {
155  json_array_append_new(sig_ids, json_integer(ff->sid[i]));
156  }
157  json_object_set_new(fjs, "sid", sig_ids);
158 
159 #ifdef HAVE_MAGIC
160  if (ff->magic)
161  json_object_set_new(fjs, "magic", json_string((char *)ff->magic));
162 #endif
163  json_object_set_new(fjs, "gaps", json_boolean((ff->flags & FILE_HAS_GAPS)));
164  switch (ff->state) {
165  case FILE_STATE_CLOSED:
166  json_object_set_new(fjs, "state", json_string("CLOSED"));
167 #ifdef HAVE_NSS
168  if (ff->flags & FILE_MD5) {
169  size_t x;
170  int i;
171  char str[256];
172  for (i = 0, x = 0; x < sizeof(ff->md5); x++) {
173  i += snprintf(&str[i], 255-i, "%02x", ff->md5[x]);
174  }
175  json_object_set_new(fjs, "md5", json_string(str));
176  }
177  if (ff->flags & FILE_SHA1) {
178  size_t x;
179  int i;
180  char str[256];
181  for (i = 0, x = 0; x < sizeof(ff->sha1); x++) {
182  i += snprintf(&str[i], 255-i, "%02x", ff->sha1[x]);
183  }
184  json_object_set_new(fjs, "sha1", json_string(str));
185  }
186 #endif
187  break;
189  json_object_set_new(fjs, "state", json_string("TRUNCATED"));
190  break;
191  case FILE_STATE_ERROR:
192  json_object_set_new(fjs, "state", json_string("ERROR"));
193  break;
194  default:
195  json_object_set_new(fjs, "state", json_string("UNKNOWN"));
196  break;
197  }
198 
199 #ifdef HAVE_NSS
200  if (ff->flags & FILE_SHA256) {
201  size_t x;
202  int i;
203  char str[256];
204  for (i = 0, x = 0; x < sizeof(ff->sha256); x++) {
205  i += snprintf(&str[i], 255-i, "%02x", ff->sha256[x]);
206  }
207  json_object_set_new(fjs, "sha256", json_string(str));
208  }
209 #endif
210 
211  json_object_set_new(fjs, "stored", stored ? json_true() : json_false());
212  if (ff->flags & FILE_STORED) {
213  json_object_set_new(fjs, "file_id", json_integer(ff->file_store_id));
214  }
215  json_object_set_new(fjs, "size", json_integer(FileTrackedSize(ff)));
216  if (ff->end > 0) {
217  json_object_set_new(fjs, "start", json_integer(ff->start));
218  json_object_set_new(fjs, "end", json_integer(ff->end));
219  }
220  json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
221 
222  /* xff header */
223  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) {
224  int have_xff_ip = 0;
225  char buffer[XFF_MAXLEN];
226 
227  if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
228  have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, buffer, XFF_MAXLEN);
229  }
230 
231  if (have_xff_ip) {
232  if (xff_cfg->flags & XFF_EXTRADATA) {
233  json_object_set_new(js, "xff", json_string(buffer));
234  }
235  else if (xff_cfg->flags & XFF_OVERWRITE) {
236  if (p->flowflags & FLOW_PKT_TOCLIENT) {
237  json_object_set(js, "dest_ip", json_string(buffer));
238  } else {
239  json_object_set(js, "src_ip", json_string(buffer));
240  }
241  }
242  }
243  }
244 
245  /* originally just 'file', but due to bug 1127 naming it fileinfo */
246  json_object_set_new(js, "fileinfo", fjs);
247 
248  return js;
249 }
250 
251 /**
252  * \internal
253  * \brief Write meta data on a single line json record
254  */
255 static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
256  const File *ff, uint32_t dir)
257 {
258  HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ?
260  json_t *js = JsonBuildFileInfoRecord(p, ff,
261  ff->flags & FILE_STORED ? true : false, dir, xff_cfg);
262  if (unlikely(js == NULL)) {
263  return;
264  }
265 
266  MemBufferReset(aft->buffer);
267  OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer);
268  json_decref(js);
269 }
270 
271 static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
272  const File *ff, uint8_t dir)
273 {
274  SCEnter();
275  JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
276 
277  BUG_ON(ff->flags & FILE_LOGGED);
278 
279  SCLogDebug("ff %p", ff);
280 
281  FileWriteJsonRecord(aft, p, ff, dir);
282  return 0;
283 }
284 
285 
286 #define OUTPUT_BUFFER_SIZE 65535
287 static TmEcode JsonFileLogThreadInit(ThreadVars *t, const void *initdata, void **data)
288 {
290  if (unlikely(aft == NULL))
291  return TM_ECODE_FAILED;
292  memset(aft, 0, sizeof(JsonFileLogThread));
293 
294  if(initdata == NULL)
295  {
296  SCLogDebug("Error getting context for EveLogFile. \"initdata\" argument NULL");
297  SCFree(aft);
298  return TM_ECODE_FAILED;
299  }
300 
301  /* Use the Ouptut Context (file pointer and mutex) */
302  aft->filelog_ctx = ((OutputCtx *)initdata)->data;
303 
305  if (aft->buffer == NULL) {
306  SCFree(aft);
307  return TM_ECODE_FAILED;
308  }
309 
310  *data = (void *)aft;
311  return TM_ECODE_OK;
312 }
313 
314 static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
315 {
316  JsonFileLogThread *aft = (JsonFileLogThread *)data;
317  if (aft == NULL) {
318  return TM_ECODE_OK;
319  }
320 
321  MemBufferFree(aft->buffer);
322  /* clear memory */
323  memset(aft, 0, sizeof(JsonFileLogThread));
324 
325  SCFree(aft);
326  return TM_ECODE_OK;
327 }
328 
329 static void OutputFileLogDeinitSub(OutputCtx *output_ctx)
330 {
331  OutputFileCtx *ff_ctx = output_ctx->data;
332  if (ff_ctx->xff_cfg != NULL) {
333  SCFree(ff_ctx->xff_cfg);
334  }
335  SCFree(ff_ctx);
336  SCFree(output_ctx);
337 }
338 
339 /** \brief Create a new http log LogFileCtx.
340  * \param conf Pointer to ConfNode containing this loggers configuration.
341  * \return NULL if failure, LogFileCtx* to the file_ctx if succesful
342  * */
343 static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
344 {
345  OutputInitResult result = { NULL, false };
346  OutputJsonCtx *ojc = parent_ctx->data;
347 
348  OutputFileCtx *output_file_ctx = SCCalloc(1, sizeof(OutputFileCtx));
349  if (unlikely(output_file_ctx == NULL))
350  return result;
351 
352  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
353  if (unlikely(output_ctx == NULL)) {
354  SCFree(output_file_ctx);
355  return result;
356  }
357 
358  output_file_ctx->file_ctx = ojc->file_ctx;
359 
360  if (conf) {
361  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
362  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
364  SCLogConfig("forcing filestore of all files");
365  }
366 
367  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
368  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
370  SCLogConfig("forcing magic lookup for logged files");
371  }
372 
373  FileForceHashParseCfg(conf);
374  }
375 
376  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
377  output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
378  if (output_file_ctx->xff_cfg != NULL) {
379  HttpXFFGetCfg(conf, output_file_ctx->xff_cfg);
380  }
381  } else if (ojc->xff_cfg) {
382  output_file_ctx->parent_xff_cfg = ojc->xff_cfg;
383  }
384 
385  output_ctx->data = output_file_ctx;
386  output_ctx->DeInit = OutputFileLogDeinitSub;
387 
389  result.ctx = output_ctx;
390  result.ok = true;
391  return result;
392 }
393 
395 {
396  /* register as child of eve-log */
397  OutputRegisterFileSubModule(LOGGER_JSON_FILE, "eve-log", "JsonFileLog",
398  "eve-log.files", OutputFileLogInitSub, JsonFileLogger,
399  JsonFileLogThreadInit, JsonFileLogThreadDeinit, NULL);
400 }
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
json_t * JsonSMBAddMetadata(const Flow *f, uint64_t tx_id)
#define BUG_ON(x)
#define OUTPUT_BUFFER_SIZE
LogFileCtx * file_ctx
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
#define unlikely(expr)
Definition: util-optimize.h:35
#define FILE_SHA256
Definition: util-file.h:43
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
OutputFileCtx * filelog_ctx
uint32_t sid_cnt
Definition: util-file.h:96
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1060
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
struct OutputFileCtx_ OutputFileCtx
#define XFF_MAXLEN
json_t * JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
void FileForceTrackingEnable(void)
Definition: util-file.c:150
struct JsonFileLogThread_ JsonFileLogThread
uint64_t start
Definition: util-file.h:92
HttpXFFCfg * parent_xff_cfg
#define FILE_LOGGED
Definition: util-file.h:44
#define str(s)
#define SCCalloc(nm, a)
Definition: util-mem.h:197
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void OutputRegisterFileSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FileLogger FileLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file output sub-module.
Definition: output.c:457
uint16_t flags
Definition: util-file.h:65
uint64_t end
Definition: util-file.h:93
uint32_t * sid
Definition: util-file.h:95
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow...
#define FILE_HAS_GAPS
Definition: util-file.h:51
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileForceMagicEnable(void)
Definition: util-file.c:91
uint64_t txid
Definition: util-file.h:69
void BytesToStringBuffer(const uint8_t *bytes, size_t nbytes, char *outstr, size_t outlen)
Turn byte array into string.
Definition: util-byte.c:85
#define SCEnter(...)
Definition: util-debug.h:337
uint8_t flowflags
Definition: decode.h:437
uint16_t name_len
Definition: util-file.h:66
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FILE_MD5
Definition: util-file.h:39
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:166
#define XFF_OVERWRITE
#define SCFree(a)
Definition: util-mem.h:228
HttpXFFCfg * xff_cfg
void * data
Definition: tm-modules.h:81
#define STREAM_TOSERVER
Definition: stream.h:31
FileState state
Definition: util-file.h:67
uint32_t file_store_id
Definition: util-file.h:72
void JsonFileLogRegister(void)
#define XFF_DISABLED
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
uint8_t * name
Definition: util-file.h:75
#define XFF_EXTRADATA
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
#define DEBUG_VALIDATE_BUG_ON(exp)