suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 #ifdef HAVE_LIBJANSSON
53 
54 typedef struct LogJsonFileCtx_ {
55  LogFileCtx *file_ctx;
56  OutputJsonCommonSettings cfg;
57 } LogJsonFileCtx;
58 
59 typedef struct JsonNetFlowLogThread_ {
60  LogJsonFileCtx *flowlog_ctx;
61  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
62 
63  MemBuffer *buffer;
64 } JsonNetFlowLogThread;
65 
66 
67 static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type, int dir)
68 {
69  char timebuf[64];
70  char srcip[46] = {0}, dstip[46] = {0};
71  Port sp, dp;
72 
73  json_t *js = json_object();
74  if (unlikely(js == NULL))
75  return NULL;
76 
77  struct timeval tv;
78  memset(&tv, 0x00, sizeof(tv));
79  TimeGet(&tv);
80 
81  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
82 
83  /* reverse header direction if the flow started out wrong */
84  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
85 
86  if (FLOW_IS_IPV4(f)) {
87  if (dir == 0) {
88  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
89  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
90  } else {
91  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
92  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
93  }
94  } else if (FLOW_IS_IPV6(f)) {
95  if (dir == 0) {
96  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
97  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
98  } else {
99  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
100  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
101  }
102  }
103 
104  if (dir == 0) {
105  sp = f->sp;
106  dp = f->dp;
107  } else {
108  sp = f->dp;
109  dp = f->sp;
110  }
111 
112  char proto[16];
113  if (SCProtoNameValid(f->proto) == TRUE) {
114  strlcpy(proto, known_proto[f->proto], sizeof(proto));
115  } else {
116  snprintf(proto, sizeof(proto), "%03" PRIu32, f->proto);
117  }
118 
119  /* time */
120  json_object_set_new(js, "timestamp", json_string(timebuf));
121 
122  CreateJSONFlowId(js, (const Flow *)f);
123 
124 #if 0 // TODO
125  /* sensor id */
126  if (sensor_id >= 0)
127  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
128 #endif
129  if (event_type) {
130  json_object_set_new(js, "event_type", json_string(event_type));
131  }
132 #if 0
133  /* vlan */
134  if (f->vlan_id[0] > 0) {
135  json_t *js_vlan;
136  switch (f->vlan_idx) {
137  case 1:
138  json_object_set_new(js, "vlan",
139  json_integer(f->vlan_id[0]));
140  break;
141  case 2:
142  js_vlan = json_array();
143  if (unlikely(js != NULL)) {
144  json_array_append_new(js_vlan,
145  json_integer(VLAN_GET_ID1(p)));
146  json_array_append_new(js_vlan,
147  json_integer(VLAN_GET_ID2(p)));
148  json_object_set_new(js, "vlan", js_vlan);
149  }
150  break;
151  default:
152  /* shouldn't get here */
153  break;
154  }
155  }
156 #endif
157  /* tuple */
158  json_object_set_new(js, "src_ip", json_string(srcip));
159  switch(f->proto) {
160  case IPPROTO_ICMP:
161  break;
162  case IPPROTO_UDP:
163  case IPPROTO_TCP:
164  case IPPROTO_SCTP:
165  json_object_set_new(js, "src_port", json_integer(sp));
166  break;
167  }
168  json_object_set_new(js, "dest_ip", json_string(dstip));
169  switch(f->proto) {
170  case IPPROTO_ICMP:
171  break;
172  case IPPROTO_UDP:
173  case IPPROTO_TCP:
174  case IPPROTO_SCTP:
175  json_object_set_new(js, "dest_port", json_integer(dp));
176  break;
177  }
178  json_object_set_new(js, "proto", json_string(proto));
179  switch (f->proto) {
180  case IPPROTO_ICMP:
181  case IPPROTO_ICMPV6: {
182  uint8_t type = f->icmp_s.type;
183  uint8_t code = f->icmp_s.code;
184  if (dir == 1) {
185  type = f->icmp_d.type;
186  code = f->icmp_d.code;
187 
188  }
189  json_object_set_new(js, "icmp_type", json_integer(type));
190  json_object_set_new(js, "icmp_code", json_integer(code));
191  break;
192  }
193  }
194  return js;
195 }
196 
197 /* JSON format logging */
198 static void JsonNetFlowLogJSONToServer(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
199 {
200  json_t *hjs = json_object();
201  if (hjs == NULL) {
202  return;
203  }
204 
205  json_object_set_new(js, "app_proto",
206  json_string(AppProtoToString(f->alproto_ts ? f->alproto_ts : f->alproto)));
207 
208  json_object_set_new(hjs, "pkts",
209  json_integer(f->todstpktcnt));
210  json_object_set_new(hjs, "bytes",
211  json_integer(f->todstbytecnt));
212 
213  char timebuf1[64], timebuf2[64];
214 
215  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
216  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
217 
218  json_object_set_new(hjs, "start", json_string(timebuf1));
219  json_object_set_new(hjs, "end", json_string(timebuf2));
220 
221  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
222  json_object_set_new(hjs, "age",
223  json_integer(age));
224 
225  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toserver));
226  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toserver));
227 
228  json_object_set_new(js, "netflow", hjs);
229 
230  /* TCP */
231  if (f->proto == IPPROTO_TCP) {
232  json_t *tjs = json_object();
233  if (tjs == NULL) {
234  return;
235  }
236 
237  TcpSession *ssn = f->protoctx;
238 
239  char hexflags[3];
240  snprintf(hexflags, sizeof(hexflags), "%02x",
241  ssn ? ssn->client.tcp_flags : 0);
242  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
243 
244  JsonTcpFlags(ssn ? ssn->client.tcp_flags : 0, tjs);
245 
246  json_object_set_new(js, "tcp", tjs);
247  }
248 }
249 
250 static void JsonNetFlowLogJSONToClient(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
251 {
252  json_t *hjs = json_object();
253  if (hjs == NULL) {
254  return;
255  }
256 
257  json_object_set_new(js, "app_proto",
258  json_string(AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)));
259 
260  json_object_set_new(hjs, "pkts",
261  json_integer(f->tosrcpktcnt));
262  json_object_set_new(hjs, "bytes",
263  json_integer(f->tosrcbytecnt));
264 
265  char timebuf1[64], timebuf2[64];
266 
267  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
268  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
269 
270  json_object_set_new(hjs, "start", json_string(timebuf1));
271  json_object_set_new(hjs, "end", json_string(timebuf2));
272 
273  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
274  json_object_set_new(hjs, "age",
275  json_integer(age));
276 
277  /* To client is zero if we did not see any packet */
278  if (f->tosrcpktcnt) {
279  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toclient));
280  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toclient));
281  }
282 
283  json_object_set_new(js, "netflow", hjs);
284 
285  /* TCP */
286  if (f->proto == IPPROTO_TCP) {
287  json_t *tjs = json_object();
288  if (tjs == NULL) {
289  return;
290  }
291 
292  TcpSession *ssn = f->protoctx;
293 
294  char hexflags[3];
295  snprintf(hexflags, sizeof(hexflags), "%02x",
296  ssn ? ssn->server.tcp_flags : 0);
297  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
298 
299  JsonTcpFlags(ssn ? ssn->server.tcp_flags : 0, tjs);
300 
301  json_object_set_new(js, "tcp", tjs);
302  }
303 }
304 
305 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
306 {
307  SCEnter();
308  JsonNetFlowLogThread *jhl = (JsonNetFlowLogThread *)thread_data;
309  LogJsonFileCtx *netflow_ctx = jhl->flowlog_ctx;
310 
311  /* reset */
312  MemBufferReset(jhl->buffer);
313  json_t *js = CreateJSONHeaderFromFlow(f, "netflow", 0);
314  if (unlikely(js == NULL))
315  return TM_ECODE_OK;
316  JsonNetFlowLogJSONToServer(jhl, js, f);
317  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
318  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
319  json_object_del(js, "netflow");
320  json_object_clear(js);
321  json_decref(js);
322 
323  /* only log a response record if we actually have seen response packets */
324  if (f->tosrcpktcnt) {
325  /* reset */
326  MemBufferReset(jhl->buffer);
327  js = CreateJSONHeaderFromFlow(f, "netflow", 1);
328  if (unlikely(js == NULL))
329  return TM_ECODE_OK;
330  JsonNetFlowLogJSONToClient(jhl, js, f);
331  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
332  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
333  json_object_del(js, "netflow");
334  json_object_clear(js);
335  json_decref(js);
336  }
338 }
339 
340 static void OutputNetFlowLogDeinit(OutputCtx *output_ctx)
341 {
342  LogJsonFileCtx *flow_ctx = output_ctx->data;
343  LogFileCtx *logfile_ctx = flow_ctx->file_ctx;
344  LogFileFreeCtx(logfile_ctx);
345  SCFree(flow_ctx);
346  SCFree(output_ctx);
347 }
348 
349 #define DEFAULT_LOG_FILENAME "netflow.json"
350 static OutputInitResult OutputNetFlowLogInit(ConfNode *conf)
351 {
352  OutputInitResult result = { NULL, false };
353  LogFileCtx *file_ctx = LogFileNewCtx();
354  if(file_ctx == NULL) {
355  SCLogError(SC_ERR_NETFLOW_LOG_GENERIC, "couldn't create new file_ctx");
356  return result;
357  }
358 
359  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
360  LogFileFreeCtx(file_ctx);
361  return result;
362  }
363 
364  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
365  if (unlikely(flow_ctx == NULL)) {
366  LogFileFreeCtx(file_ctx);
367  return result;
368  }
369 
370  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
371  if (unlikely(output_ctx == NULL)) {
372  LogFileFreeCtx(file_ctx);
373  SCFree(flow_ctx);
374  return result;
375  }
376 
377  flow_ctx->file_ctx = file_ctx;
378  output_ctx->data = flow_ctx;
379  output_ctx->DeInit = OutputNetFlowLogDeinit;
380 
381  result.ctx = output_ctx;
382  result.ok = true;
383  return result;
384 }
385 
386 static void OutputNetFlowLogDeinitSub(OutputCtx *output_ctx)
387 {
388  LogJsonFileCtx *flow_ctx = output_ctx->data;
389  SCFree(flow_ctx);
390  SCFree(output_ctx);
391 }
392 
393 static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
394 {
395  OutputInitResult result = { NULL, false };
396  OutputJsonCtx *ojc = parent_ctx->data;
397 
398  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
399  if (unlikely(flow_ctx == NULL))
400  return result;
401 
402  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
403  if (unlikely(output_ctx == NULL)) {
404  SCFree(flow_ctx);
405  return result;
406  }
407 
408  flow_ctx->file_ctx = ojc->file_ctx;
409  flow_ctx->cfg = ojc->cfg;
410 
411  output_ctx->data = flow_ctx;
412  output_ctx->DeInit = OutputNetFlowLogDeinitSub;
413 
414  result.ctx = output_ctx;
415  result.ok = true;
416  return result;
417 }
418 
419 #define OUTPUT_BUFFER_SIZE 65535
420 static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
421 {
422  JsonNetFlowLogThread *aft = SCMalloc(sizeof(JsonNetFlowLogThread));
423  if (unlikely(aft == NULL))
424  return TM_ECODE_FAILED;
425  memset(aft, 0, sizeof(JsonNetFlowLogThread));
426 
427  if(initdata == NULL)
428  {
429  SCLogDebug("Error getting context for EveLogNetflow. \"initdata\" argument NULL");
430  SCFree(aft);
431  return TM_ECODE_FAILED;
432  }
433 
434  /* Use the Ouptut Context (file pointer and mutex) */
435  aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
436 
437  aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
438  if (aft->buffer == NULL) {
439  SCFree(aft);
440  return TM_ECODE_FAILED;
441  }
442 
443  *data = (void *)aft;
444  return TM_ECODE_OK;
445 }
446 
447 static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data)
448 {
449  JsonNetFlowLogThread *aft = (JsonNetFlowLogThread *)data;
450  if (aft == NULL) {
451  return TM_ECODE_OK;
452  }
453 
454  MemBufferFree(aft->buffer);
455  /* clear memory */
456  memset(aft, 0, sizeof(JsonNetFlowLogThread));
457 
458  SCFree(aft);
459  return TM_ECODE_OK;
460 }
461 
462 void JsonNetFlowLogRegister(void)
463 {
464  /* register as separate module */
466  "netflow-json-log", OutputNetFlowLogInit, JsonNetFlowLogger,
467  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
468 
469  /* also register as child of eve-log */
470  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog",
471  "eve-log.netflow", OutputNetFlowLogInitSub, JsonNetFlowLogger,
472  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
473 }
474 
475 #else
476 
478 {
479 }
480 
481 #endif
struct Flow_::@116::@120 icmp_s
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define FLOW_IS_IPV4(f)
Definition: flow.h:133
char * known_proto[256]
uint8_t SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
#define SCLogDebug(...)
Definition: util-debug.h:335
AppProto alproto_tc
Definition: flow.h:406
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
uint64_t todstbytecnt
Definition: flow.h:457
uint32_t event_type
uint8_t proto
Definition: flow.h:343
Port sp
Definition: flow.h:330
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define FLOW_DIR_REVERSED
Definition: flow.h:105
uint8_t max_ttl_toserver
Definition: flow.h:425
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
struct timeval startts
Definition: flow.h:453
uint32_t todstpktcnt
Definition: flow.h:455
#define TRUE
void * protoctx
Definition: flow.h:395
FlowAddress dst
Definition: flow.h:328
void OutputRegisterFlowModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output module.
Definition: output.c:577
void JsonNetFlowLogRegister(void)
uint16_t vlan_id[2]
Definition: flow.h:345
AppProto alproto_ts
Definition: flow.h:405
uint64_t tosrcbytecnt
Definition: flow.h:458
#define SCCalloc(nm, a)
Definition: util-mem.h:197
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:179
union FlowAddress_::@115 address
void TimeGet(struct timeval *tv)
Definition: util-time.c:138
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
uint8_t type
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define VLAN_GET_ID2(p)
Definition: decode-vlan.h:41
#define SCEnter(...)
Definition: util-debug.h:337
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
uint16_t Port
Definition: decode.h:233
struct timeval lastts
Definition: flow.h:353
uint8_t min_ttl_toserver
Definition: flow.h:424
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:616
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:166
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCFree(a)
Definition: util-mem.h:228
#define VLAN_GET_ID1(p)
Definition: decode-vlan.h:40
Port dp
Definition: flow.h:337
#define FLOW_IS_IPV6(f)
Definition: flow.h:135
void * data
Definition: tm-modules.h:81
uint8_t max_ttl_toclient
Definition: flow.h:427
struct Flow_::@118::@121 icmp_d
uint8_t min_ttl_toclient
Definition: flow.h:426
FlowAddress src
Definition: flow.h:328
Per thread variable structure.
Definition: threadvars.h:57
uint8_t code
AppProto alproto
application level protocol
Definition: flow.h:404
#define OUTPUT_BUFFER_SIZE
Definition: log-httplog.c:58
Flow data structure.
Definition: flow.h:324
uint32_t flags
Definition: flow.h:374
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
uint32_t tosrcpktcnt
Definition: flow.h:456