suricata
output-json-flow.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Flow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-flow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 #ifdef HAVE_LIBJANSSON
53 
54 typedef struct LogJsonFileCtx_ {
55  LogFileCtx *file_ctx;
56  uint32_t flags; /** Store mode */
57  OutputJsonCommonSettings cfg;
58 } LogJsonFileCtx;
59 
60 typedef struct JsonFlowLogThread_ {
61  LogJsonFileCtx *flowlog_ctx;
62  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
63  MemBuffer *buffer;
64 } JsonFlowLogThread;
65 
66 static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type)
67 {
68  char timebuf[64];
69  char srcip[46], dstip[46];
70  Port sp, dp;
71 
72  json_t *js = json_object();
73  if (unlikely(js == NULL))
74  return NULL;
75 
76  struct timeval tv;
77  memset(&tv, 0x00, sizeof(tv));
78  TimeGet(&tv);
79 
80  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
81 
82  srcip[0] = '\0';
83  dstip[0] = '\0';
84  if (FLOW_IS_IPV4(f)) {
85  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
86  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
87  } else if (FLOW_IS_IPV6(f)) {
88  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
89  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
90  }
91 
92  sp = f->sp;
93  dp = f->dp;
94 
95  char proto[16];
96  if (SCProtoNameValid(f->proto) == TRUE) {
97  strlcpy(proto, known_proto[f->proto], sizeof(proto));
98  } else {
99  snprintf(proto, sizeof(proto), "%03" PRIu32, f->proto);
100  }
101 
102  /* time */
103  json_object_set_new(js, "timestamp", json_string(timebuf));
104 
105  CreateJSONFlowId(js, (const Flow *)f);
106 
107 #if 0 // TODO
108  /* sensor id */
109  if (sensor_id >= 0)
110  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
111 #endif
112  if (event_type) {
113  json_object_set_new(js, "event_type", json_string(event_type));
114  }
115 #if 0
116  /* vlan */
117  if (f->vlan_id[0] > 0) {
118  json_t *js_vlan;
119  switch (f->vlan_idx) {
120  case 1:
121  json_object_set_new(js, "vlan",
122  json_integer(f->vlan_id[0]));
123  break;
124  case 2:
125  js_vlan = json_array();
126  if (unlikely(js != NULL)) {
127  json_array_append_new(js_vlan,
128  json_integer(VLAN_GET_ID1(p)));
129  json_array_append_new(js_vlan,
130  json_integer(VLAN_GET_ID2(p)));
131  json_object_set_new(js, "vlan", js_vlan);
132  }
133  break;
134  default:
135  /* shouldn't get here */
136  break;
137  }
138  }
139 #endif
140  /* tuple */
141  json_object_set_new(js, "src_ip", json_string(srcip));
142  switch(f->proto) {
143  case IPPROTO_ICMP:
144  break;
145  case IPPROTO_UDP:
146  case IPPROTO_TCP:
147  case IPPROTO_SCTP:
148  json_object_set_new(js, "src_port", json_integer(sp));
149  break;
150  }
151  json_object_set_new(js, "dest_ip", json_string(dstip));
152  switch(f->proto) {
153  case IPPROTO_ICMP:
154  break;
155  case IPPROTO_UDP:
156  case IPPROTO_TCP:
157  case IPPROTO_SCTP:
158  json_object_set_new(js, "dest_port", json_integer(dp));
159  break;
160  }
161  json_object_set_new(js, "proto", json_string(proto));
162  switch (f->proto) {
163  case IPPROTO_ICMP:
164  case IPPROTO_ICMPV6:
165  json_object_set_new(js, "icmp_type",
166  json_integer(f->icmp_s.type));
167  json_object_set_new(js, "icmp_code",
168  json_integer(f->icmp_s.code));
169  if (f->tosrcpktcnt) {
170  json_object_set_new(js, "response_icmp_type",
171  json_integer(f->icmp_d.type));
172  json_object_set_new(js, "response_icmp_code",
173  json_integer(f->icmp_d.code));
174  }
175  break;
176  }
177  return js;
178 }
179 
180 void JsonAddFlow(Flow *f, json_t *js, json_t *hjs)
181 {
182  json_object_set_new(js, "app_proto",
183  json_string(AppProtoToString(f->alproto)));
184  if (f->alproto_ts != f->alproto) {
185  json_object_set_new(js, "app_proto_ts",
186  json_string(AppProtoToString(f->alproto_ts)));
187  }
188  if (f->alproto_tc != f->alproto) {
189  json_object_set_new(js, "app_proto_tc",
190  json_string(AppProtoToString(f->alproto_tc)));
191  }
192  if (f->alproto_orig != f->alproto && f->alproto_orig != ALPROTO_UNKNOWN) {
193  json_object_set_new(js, "app_proto_orig",
194  json_string(AppProtoToString(f->alproto_orig)));
195  }
196  if (f->alproto_expect != f->alproto && f->alproto_expect != ALPROTO_UNKNOWN) {
197  json_object_set_new(js, "app_proto_expected",
198  json_string(AppProtoToString(f->alproto_expect)));
199  }
200 
201  json_object_set_new(hjs, "pkts_toserver",
202  json_integer(f->todstpktcnt));
203  json_object_set_new(hjs, "pkts_toclient",
204  json_integer(f->tosrcpktcnt));
205  json_object_set_new(hjs, "bytes_toserver",
206  json_integer(f->todstbytecnt));
207  json_object_set_new(hjs, "bytes_toclient",
208  json_integer(f->tosrcbytecnt));
209 
210  char timebuf1[64];
211  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
212  json_object_set_new(hjs, "start", json_string(timebuf1));
213 }
214 
215 /* JSON format logging */
216 static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f)
217 {
218  LogJsonFileCtx *flow_ctx = aft->flowlog_ctx;
219  json_t *hjs = json_object();
220  if (hjs == NULL) {
221  return;
222  }
223 
224  JsonAddFlow(f, js, hjs);
225 
226  char timebuf2[64];
227  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
228  json_object_set_new(hjs, "end", json_string(timebuf2));
229 
230  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
231  json_object_set_new(hjs, "age",
232  json_integer(age));
233 
235  json_object_set_new(hjs, "emergency", json_true());
236  const char *state = NULL;
238  state = "new";
240  state = "established";
242  state = "closed";
244  state = "bypassed";
245  int flow_state = SC_ATOMIC_GET(f->flow_state);
246  switch (flow_state) {
248  json_object_set_new(hjs, "bypass",
249  json_string("local"));
250  break;
252  json_object_set_new(hjs, "bypass",
253  json_string("capture"));
254  break;
255  default:
257  "Invalid flow state: %d, contact developers",
258  flow_state);
259  }
260  }
261 
262  json_object_set_new(hjs, "state",
263  json_string(state));
264 
265  const char *reason = NULL;
267  reason = "timeout";
268  else if (f->flow_end_flags & FLOW_END_FLAG_FORCED)
269  reason = "forced";
271  reason = "shutdown";
272 
273  json_object_set_new(hjs, "reason",
274  json_string(reason));
275 
276  json_object_set_new(hjs, "alerted", json_boolean(FlowHasAlerts(f)));
277  if (f->flags & FLOW_WRONG_THREAD)
278  json_object_set_new(hjs, "wrong_thread", json_true());
279 
280  json_object_set_new(js, "flow", hjs);
281 
282  JsonAddCommonOptions(&flow_ctx->cfg, NULL, f, js);
283 
284  /* TCP */
285  if (f->proto == IPPROTO_TCP) {
286  json_t *tjs = json_object();
287  if (tjs == NULL) {
288  return;
289  }
290 
291  TcpSession *ssn = f->protoctx;
292 
293  char hexflags[3];
294  snprintf(hexflags, sizeof(hexflags), "%02x",
295  ssn ? ssn->tcp_packet_flags : 0);
296  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
297 
298  snprintf(hexflags, sizeof(hexflags), "%02x",
299  ssn ? ssn->client.tcp_flags : 0);
300  json_object_set_new(tjs, "tcp_flags_ts", json_string(hexflags));
301 
302  snprintf(hexflags, sizeof(hexflags), "%02x",
303  ssn ? ssn->server.tcp_flags : 0);
304  json_object_set_new(tjs, "tcp_flags_tc", json_string(hexflags));
305 
306  JsonTcpFlags(ssn ? ssn->tcp_packet_flags : 0, tjs);
307 
308  if (ssn) {
309  const char *tcp_state = NULL;
310  switch (ssn->state) {
311  case TCP_NONE:
312  tcp_state = "none";
313  break;
314  case TCP_LISTEN:
315  tcp_state = "listen";
316  break;
317  case TCP_SYN_SENT:
318  tcp_state = "syn_sent";
319  break;
320  case TCP_SYN_RECV:
321  tcp_state = "syn_recv";
322  break;
323  case TCP_ESTABLISHED:
324  tcp_state = "established";
325  break;
326  case TCP_FIN_WAIT1:
327  tcp_state = "fin_wait1";
328  break;
329  case TCP_FIN_WAIT2:
330  tcp_state = "fin_wait2";
331  break;
332  case TCP_TIME_WAIT:
333  tcp_state = "time_wait";
334  break;
335  case TCP_LAST_ACK:
336  tcp_state = "last_ack";
337  break;
338  case TCP_CLOSE_WAIT:
339  tcp_state = "close_wait";
340  break;
341  case TCP_CLOSING:
342  tcp_state = "closing";
343  break;
344  case TCP_CLOSED:
345  tcp_state = "closed";
346  break;
347  }
348  json_object_set_new(tjs, "state", json_string(tcp_state));
350  json_object_set_new(tjs, "gap_ts", json_true());
352  json_object_set_new(tjs, "gap_tc", json_true());
353  }
354 
355  json_object_set_new(js, "tcp", tjs);
356  }
357 }
358 
359 static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
360 {
361  SCEnter();
362  JsonFlowLogThread *jhl = (JsonFlowLogThread *)thread_data;
363 
364  /* reset */
365  MemBufferReset(jhl->buffer);
366 
367  json_t *js = CreateJSONHeaderFromFlow(f, "flow");
368  if (unlikely(js == NULL))
369  return TM_ECODE_OK;
370 
371  JsonFlowLogJSON(jhl, js, f);
372 
373  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
374  json_object_del(js, "http");
375 
376  json_object_clear(js);
377  json_decref(js);
378 
380 }
381 
382 static void OutputFlowLogDeinit(OutputCtx *output_ctx)
383 {
384  LogJsonFileCtx *flow_ctx = output_ctx->data;
385  LogFileCtx *logfile_ctx = flow_ctx->file_ctx;
386  LogFileFreeCtx(logfile_ctx);
387  SCFree(flow_ctx);
388  SCFree(output_ctx);
389 }
390 
391 #define DEFAULT_LOG_FILENAME "flow.json"
392 static OutputInitResult OutputFlowLogInit(ConfNode *conf)
393 {
394  OutputInitResult result = { NULL, false };
395  LogFileCtx *file_ctx = LogFileNewCtx();
396  if(file_ctx == NULL) {
397  SCLogError(SC_ERR_FLOW_LOG_GENERIC, "couldn't create new file_ctx");
398  return result;
399  }
400 
401  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
402  LogFileFreeCtx(file_ctx);
403  return result;
404  }
405 
406  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
407  if (unlikely(flow_ctx == NULL)) {
408  LogFileFreeCtx(file_ctx);
409  return result;
410  }
411 
412  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
413  if (unlikely(output_ctx == NULL)) {
414  LogFileFreeCtx(file_ctx);
415  SCFree(flow_ctx);
416  return result;
417  }
418 
419  flow_ctx->file_ctx = file_ctx;
420  output_ctx->data = flow_ctx;
421  output_ctx->DeInit = OutputFlowLogDeinit;
422 
423  result.ctx = output_ctx;
424  result.ok = true;
425  return result;
426 }
427 
428 static void OutputFlowLogDeinitSub(OutputCtx *output_ctx)
429 {
430  LogJsonFileCtx *flow_ctx = output_ctx->data;
431  SCFree(flow_ctx);
432  SCFree(output_ctx);
433 }
434 
435 static OutputInitResult OutputFlowLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
436 {
437  OutputInitResult result = { NULL, false };
438  OutputJsonCtx *ojc = parent_ctx->data;
439 
440  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
441  if (unlikely(flow_ctx == NULL))
442  return result;
443 
444  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
445  if (unlikely(output_ctx == NULL)) {
446  SCFree(flow_ctx);
447  return result;
448  }
449 
450  flow_ctx->file_ctx = ojc->file_ctx;
451  flow_ctx->cfg = ojc->cfg;
452 
453  output_ctx->data = flow_ctx;
454  output_ctx->DeInit = OutputFlowLogDeinitSub;
455 
456  result.ctx = output_ctx;
457  result.ok = true;
458  return result;
459 }
460 
461 #define OUTPUT_BUFFER_SIZE 65535
462 static TmEcode JsonFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
463 {
464  JsonFlowLogThread *aft = SCMalloc(sizeof(JsonFlowLogThread));
465  if (unlikely(aft == NULL))
466  return TM_ECODE_FAILED;
467  memset(aft, 0, sizeof(JsonFlowLogThread));
468 
469  if(initdata == NULL)
470  {
471  SCLogDebug("Error getting context for EveLogFlow. \"initdata\" argument NULL");
472  SCFree(aft);
473  return TM_ECODE_FAILED;
474  }
475 
476  /* Use the Ouptut Context (file pointer and mutex) */
477  aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
478 
479  aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
480  if (aft->buffer == NULL) {
481  SCFree(aft);
482  return TM_ECODE_FAILED;
483  }
484 
485  *data = (void *)aft;
486  return TM_ECODE_OK;
487 }
488 
489 static TmEcode JsonFlowLogThreadDeinit(ThreadVars *t, void *data)
490 {
491  JsonFlowLogThread *aft = (JsonFlowLogThread *)data;
492  if (aft == NULL) {
493  return TM_ECODE_OK;
494  }
495 
496  MemBufferFree(aft->buffer);
497  /* clear memory */
498  memset(aft, 0, sizeof(JsonFlowLogThread));
499 
500  SCFree(aft);
501  return TM_ECODE_OK;
502 }
503 
504 void JsonFlowLogRegister (void)
505 {
506  /* register as separate module */
507  OutputRegisterFlowModule(LOGGER_JSON_FLOW, "JsonFlowLog", "flow-json-log",
508  OutputFlowLogInit, JsonFlowLogger, JsonFlowLogThreadInit,
509  JsonFlowLogThreadDeinit, NULL);
510 
511  /* also register as child of eve-log */
512  OutputRegisterFlowSubModule(LOGGER_JSON_FLOW, "eve-log", "JsonFlowLog",
513  "eve-log.flow", OutputFlowLogInitSub, JsonFlowLogger,
514  JsonFlowLogThreadInit, JsonFlowLogThreadDeinit, NULL);
515 }
516 
517 #else
518 
520 {
521 }
522 
523 #endif
AppProto alproto_expect
Definition: flow.h:416
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define FLOW_IS_IPV4(f)
Definition: flow.h:131
char * known_proto[256]
uint16_t flags
void JsonFlowLogRegister(void)
uint8_t SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
#define SCLogDebug(...)
Definition: util-debug.h:335
AppProto alproto_tc
Definition: flow.h:409
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
uint64_t todstbytecnt
Definition: flow.h:460
uint32_t event_type
uint8_t proto
Definition: flow.h:346
Port sp
Definition: flow.h:333
#define FLOW_END_FLAG_STATE_ESTABLISHED
Definition: flow.h:202
#define unlikely(expr)
Definition: util-optimize.h:35
#define STREAMTCP_STREAM_FLAG_GAP
int FlowHasAlerts(const Flow *f)
Check if flow has alerts.
Definition: flow.c:208
#define FLOW_END_FLAG_TIMEOUT
Definition: flow.h:205
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define FLOW_END_FLAG_FORCED
Definition: flow.h:206
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
struct timeval startts
Definition: flow.h:456
uint32_t todstpktcnt
Definition: flow.h:458
#define FLOW_END_FLAG_STATE_BYPASSED
Definition: flow.h:208
#define TRUE
void * protoctx
Definition: flow.h:398
FlowAddress dst
Definition: flow.h:331
struct Flow_::@124::@128 icmp_s
void OutputRegisterFlowModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output module.
Definition: output.c:579
AppProto alproto_orig
Definition: flow.h:413
uint16_t vlan_id[2]
Definition: flow.h:348
AppProto alproto_ts
Definition: flow.h:408
uint64_t tosrcbytecnt
Definition: flow.h:461
#define SCCalloc(nm, a)
Definition: util-mem.h:205
#define FLOW_END_FLAG_STATE_NEW
Definition: flow.h:201
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:179
void TimeGet(struct timeval *tv)
Definition: util-time.c:138
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define VLAN_GET_ID2(p)
Definition: decode-vlan.h:41
#define SCEnter(...)
Definition: util-debug.h:337
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
#define FLOW_END_FLAG_STATE_CLOSED
Definition: flow.h:203
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
struct Flow_::@126::@129 icmp_d
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
uint16_t Port
Definition: decode.h:234
struct timeval lastts
Definition: flow.h:356
uint8_t flow_end_flags
Definition: flow.h:404
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:618
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:174
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
union FlowAddress_::@123 address
#define SCFree(a)
Definition: util-mem.h:236
#define VLAN_GET_ID1(p)
Definition: decode-vlan.h:40
Port dp
Definition: flow.h:340
#define OUTPUT_BUFFER_SIZE
Definition: log-dnslog.c:58
#define FLOW_IS_IPV6(f)
Definition: flow.h:133
#define FLOW_END_FLAG_EMERGENCY
Definition: flow.h:204
#define FLOW_WRONG_THREAD
Definition: flow.h:103
void * data
Definition: tm-modules.h:81
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:193
FlowAddress src
Definition: flow.h:331
Per thread variable structure.
Definition: threadvars.h:57
AppProto alproto
application level protocol
Definition: flow.h:407
#define FLOW_END_FLAG_SHUTDOWN
Definition: flow.h:207
uint8_t tcp_packet_flags
Flow data structure.
Definition: flow.h:327
uint32_t flags
Definition: flow.h:377
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
uint32_t tosrcpktcnt
Definition: flow.h:459