suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 #ifdef HAVE_LIBJANSSON
53 
54 typedef struct LogJsonFileCtx_ {
55  LogFileCtx *file_ctx;
56  OutputJsonCommonSettings cfg;
57 } LogJsonFileCtx;
58 
59 typedef struct JsonNetFlowLogThread_ {
60  LogJsonFileCtx *flowlog_ctx;
61  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
62 
63  MemBuffer *buffer;
64 } JsonNetFlowLogThread;
65 
66 
67 static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type, int dir)
68 {
69  char timebuf[64];
70  char srcip[46] = {0}, dstip[46] = {0};
71  Port sp, dp;
72 
73  json_t *js = json_object();
74  if (unlikely(js == NULL))
75  return NULL;
76 
77  struct timeval tv;
78  memset(&tv, 0x00, sizeof(tv));
79  TimeGet(&tv);
80 
81  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
82 
83  /* reverse header direction if the flow started out wrong */
84  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
85 
86  if (FLOW_IS_IPV4(f)) {
87  if (dir == 0) {
88  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
89  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
90  } else {
91  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
92  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
93  }
94  } else if (FLOW_IS_IPV6(f)) {
95  if (dir == 0) {
96  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
97  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
98  } else {
99  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
100  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
101  }
102  }
103 
104  if (dir == 0) {
105  sp = f->sp;
106  dp = f->dp;
107  } else {
108  sp = f->dp;
109  dp = f->sp;
110  }
111 
112  char proto[16];
113  if (SCProtoNameValid(f->proto) == TRUE) {
114  strlcpy(proto, known_proto[f->proto], sizeof(proto));
115  } else {
116  snprintf(proto, sizeof(proto), "%03" PRIu32, f->proto);
117  }
118 
119  /* time */
120  json_object_set_new(js, "timestamp", json_string(timebuf));
121 
122  CreateJSONFlowId(js, (const Flow *)f);
123 
124 #if 0 // TODO
125  /* sensor id */
126  if (sensor_id >= 0)
127  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
128 #endif
129 
130  /* input interface */
131  if (f->livedev) {
132  json_object_set_new(js, "in_iface", json_string(f->livedev->dev));
133  }
134 
135  if (event_type) {
136  json_object_set_new(js, "event_type", json_string(event_type));
137  }
138 
139  /* vlan */
140  if (f->vlan_idx > 0) {
141  json_t *js_vlan = json_array();
142  json_array_append_new(js_vlan, json_integer(f->vlan_id[0]));
143  if (f->vlan_idx > 1) {
144  json_array_append_new(js_vlan, json_integer(f->vlan_id[1]));
145  }
146  json_object_set_new(js, "vlan", js_vlan);
147  }
148 
149  /* tuple */
150  json_object_set_new(js, "src_ip", json_string(srcip));
151  switch(f->proto) {
152  case IPPROTO_ICMP:
153  break;
154  case IPPROTO_UDP:
155  case IPPROTO_TCP:
156  case IPPROTO_SCTP:
157  json_object_set_new(js, "src_port", json_integer(sp));
158  break;
159  }
160  json_object_set_new(js, "dest_ip", json_string(dstip));
161  switch(f->proto) {
162  case IPPROTO_ICMP:
163  break;
164  case IPPROTO_UDP:
165  case IPPROTO_TCP:
166  case IPPROTO_SCTP:
167  json_object_set_new(js, "dest_port", json_integer(dp));
168  break;
169  }
170  json_object_set_new(js, "proto", json_string(proto));
171  switch (f->proto) {
172  case IPPROTO_ICMP:
173  case IPPROTO_ICMPV6: {
174  uint8_t type = f->icmp_s.type;
175  uint8_t code = f->icmp_s.code;
176  if (dir == 1) {
177  type = f->icmp_d.type;
178  code = f->icmp_d.code;
179 
180  }
181  json_object_set_new(js, "icmp_type", json_integer(type));
182  json_object_set_new(js, "icmp_code", json_integer(code));
183  break;
184  }
185  }
186  return js;
187 }
188 
189 /* JSON format logging */
190 static void JsonNetFlowLogJSONToServer(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
191 {
192  json_t *hjs = json_object();
193  if (hjs == NULL) {
194  return;
195  }
196 
197  json_object_set_new(js, "app_proto",
198  json_string(AppProtoToString(f->alproto_ts ? f->alproto_ts : f->alproto)));
199 
200  json_object_set_new(hjs, "pkts",
201  json_integer(f->todstpktcnt));
202  json_object_set_new(hjs, "bytes",
203  json_integer(f->todstbytecnt));
204 
205  char timebuf1[64], timebuf2[64];
206 
207  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
208  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
209 
210  json_object_set_new(hjs, "start", json_string(timebuf1));
211  json_object_set_new(hjs, "end", json_string(timebuf2));
212 
213  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
214  json_object_set_new(hjs, "age",
215  json_integer(age));
216 
217  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toserver));
218  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toserver));
219 
220  json_object_set_new(js, "netflow", hjs);
221 
222  /* TCP */
223  if (f->proto == IPPROTO_TCP) {
224  json_t *tjs = json_object();
225  if (tjs == NULL) {
226  return;
227  }
228 
229  TcpSession *ssn = f->protoctx;
230 
231  char hexflags[3];
232  snprintf(hexflags, sizeof(hexflags), "%02x",
233  ssn ? ssn->client.tcp_flags : 0);
234  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
235 
236  JsonTcpFlags(ssn ? ssn->client.tcp_flags : 0, tjs);
237 
238  json_object_set_new(js, "tcp", tjs);
239  }
240 }
241 
242 static void JsonNetFlowLogJSONToClient(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
243 {
244  json_t *hjs = json_object();
245  if (hjs == NULL) {
246  return;
247  }
248 
249  json_object_set_new(js, "app_proto",
250  json_string(AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)));
251 
252  json_object_set_new(hjs, "pkts",
253  json_integer(f->tosrcpktcnt));
254  json_object_set_new(hjs, "bytes",
255  json_integer(f->tosrcbytecnt));
256 
257  char timebuf1[64], timebuf2[64];
258 
259  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
260  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
261 
262  json_object_set_new(hjs, "start", json_string(timebuf1));
263  json_object_set_new(hjs, "end", json_string(timebuf2));
264 
265  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
266  json_object_set_new(hjs, "age",
267  json_integer(age));
268 
269  /* To client is zero if we did not see any packet */
270  if (f->tosrcpktcnt) {
271  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toclient));
272  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toclient));
273  }
274 
275  json_object_set_new(js, "netflow", hjs);
276 
277  /* TCP */
278  if (f->proto == IPPROTO_TCP) {
279  json_t *tjs = json_object();
280  if (tjs == NULL) {
281  return;
282  }
283 
284  TcpSession *ssn = f->protoctx;
285 
286  char hexflags[3];
287  snprintf(hexflags, sizeof(hexflags), "%02x",
288  ssn ? ssn->server.tcp_flags : 0);
289  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
290 
291  JsonTcpFlags(ssn ? ssn->server.tcp_flags : 0, tjs);
292 
293  json_object_set_new(js, "tcp", tjs);
294  }
295 }
296 
297 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
298 {
299  SCEnter();
300  JsonNetFlowLogThread *jhl = (JsonNetFlowLogThread *)thread_data;
301  LogJsonFileCtx *netflow_ctx = jhl->flowlog_ctx;
302 
303  /* reset */
304  MemBufferReset(jhl->buffer);
305  json_t *js = CreateJSONHeaderFromFlow(f, "netflow", 0);
306  if (unlikely(js == NULL))
307  return TM_ECODE_OK;
308  JsonNetFlowLogJSONToServer(jhl, js, f);
309  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
310  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
311  json_object_del(js, "netflow");
312  json_object_clear(js);
313  json_decref(js);
314 
315  /* only log a response record if we actually have seen response packets */
316  if (f->tosrcpktcnt) {
317  /* reset */
318  MemBufferReset(jhl->buffer);
319  js = CreateJSONHeaderFromFlow(f, "netflow", 1);
320  if (unlikely(js == NULL))
321  return TM_ECODE_OK;
322  JsonNetFlowLogJSONToClient(jhl, js, f);
323  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
324  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
325  json_object_del(js, "netflow");
326  json_object_clear(js);
327  json_decref(js);
328  }
330 }
331 
332 static void OutputNetFlowLogDeinit(OutputCtx *output_ctx)
333 {
334  LogJsonFileCtx *flow_ctx = output_ctx->data;
335  LogFileCtx *logfile_ctx = flow_ctx->file_ctx;
336  LogFileFreeCtx(logfile_ctx);
337  SCFree(flow_ctx);
338  SCFree(output_ctx);
339 }
340 
341 #define DEFAULT_LOG_FILENAME "netflow.json"
342 static OutputInitResult OutputNetFlowLogInit(ConfNode *conf)
343 {
344  OutputInitResult result = { NULL, false };
345  LogFileCtx *file_ctx = LogFileNewCtx();
346  if(file_ctx == NULL) {
347  SCLogError(SC_ERR_NETFLOW_LOG_GENERIC, "couldn't create new file_ctx");
348  return result;
349  }
350 
351  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
352  LogFileFreeCtx(file_ctx);
353  return result;
354  }
355 
356  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
357  if (unlikely(flow_ctx == NULL)) {
358  LogFileFreeCtx(file_ctx);
359  return result;
360  }
361 
362  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
363  if (unlikely(output_ctx == NULL)) {
364  LogFileFreeCtx(file_ctx);
365  SCFree(flow_ctx);
366  return result;
367  }
368 
369  flow_ctx->file_ctx = file_ctx;
370  output_ctx->data = flow_ctx;
371  output_ctx->DeInit = OutputNetFlowLogDeinit;
372 
373  result.ctx = output_ctx;
374  result.ok = true;
375  return result;
376 }
377 
378 static void OutputNetFlowLogDeinitSub(OutputCtx *output_ctx)
379 {
380  LogJsonFileCtx *flow_ctx = output_ctx->data;
381  SCFree(flow_ctx);
382  SCFree(output_ctx);
383 }
384 
385 static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
386 {
387  OutputInitResult result = { NULL, false };
388  OutputJsonCtx *ojc = parent_ctx->data;
389 
390  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
391  if (unlikely(flow_ctx == NULL))
392  return result;
393 
394  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
395  if (unlikely(output_ctx == NULL)) {
396  SCFree(flow_ctx);
397  return result;
398  }
399 
400  flow_ctx->file_ctx = ojc->file_ctx;
401  flow_ctx->cfg = ojc->cfg;
402 
403  output_ctx->data = flow_ctx;
404  output_ctx->DeInit = OutputNetFlowLogDeinitSub;
405 
406  result.ctx = output_ctx;
407  result.ok = true;
408  return result;
409 }
410 
411 static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
412 {
413  JsonNetFlowLogThread *aft = SCMalloc(sizeof(JsonNetFlowLogThread));
414  if (unlikely(aft == NULL))
415  return TM_ECODE_FAILED;
416  memset(aft, 0, sizeof(JsonNetFlowLogThread));
417 
418  if(initdata == NULL)
419  {
420  SCLogDebug("Error getting context for EveLogNetflow. \"initdata\" argument NULL");
421  SCFree(aft);
422  return TM_ECODE_FAILED;
423  }
424 
425  /* Use the Ouptut Context (file pointer and mutex) */
426  aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
427 
428  aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
429  if (aft->buffer == NULL) {
430  SCFree(aft);
431  return TM_ECODE_FAILED;
432  }
433 
434  *data = (void *)aft;
435  return TM_ECODE_OK;
436 }
437 
438 static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data)
439 {
440  JsonNetFlowLogThread *aft = (JsonNetFlowLogThread *)data;
441  if (aft == NULL) {
442  return TM_ECODE_OK;
443  }
444 
445  MemBufferFree(aft->buffer);
446  /* clear memory */
447  memset(aft, 0, sizeof(JsonNetFlowLogThread));
448 
449  SCFree(aft);
450  return TM_ECODE_OK;
451 }
452 
453 void JsonNetFlowLogRegister(void)
454 {
455  /* register as separate module */
457  "netflow-json-log", OutputNetFlowLogInit, JsonNetFlowLogger,
458  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
459 
460  /* also register as child of eve-log */
461  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog",
462  "eve-log.netflow", OutputNetFlowLogInitSub, JsonNetFlowLogger,
463  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
464 }
465 
466 #else
467 
469 {
470 }
471 
472 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define FLOW_IS_IPV4(f)
Definition: flow.h:134
char * known_proto[256]
uint8_t SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
#define SCLogDebug(...)
Definition: util-debug.h:335
AppProto alproto_tc
Definition: flow.h:411
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
uint64_t todstbytecnt
Definition: flow.h:462
uint32_t event_type
uint8_t proto
Definition: flow.h:344
Port sp
Definition: flow.h:331
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define FLOW_DIR_REVERSED
Definition: flow.h:106
uint8_t max_ttl_toserver
Definition: flow.h:430
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
struct timeval startts
Definition: flow.h:458
uint32_t todstpktcnt
Definition: flow.h:460
#define TRUE
struct Flow_::@122::@125 icmp_d
void * protoctx
Definition: flow.h:400
FlowAddress dst
Definition: flow.h:329
char * dev
Definition: util-device.h:41
void OutputRegisterFlowModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output module.
Definition: output.c:580
void JsonNetFlowLogRegister(void)
uint16_t vlan_id[2]
Definition: flow.h:346
AppProto alproto_ts
Definition: flow.h:410
uint64_t tosrcbytecnt
Definition: flow.h:463
#define SCCalloc(nm, a)
Definition: util-mem.h:253
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
void TimeGet(struct timeval *tv)
Definition: util-time.c:146
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
uint8_t type
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define SCEnter(...)
Definition: util-debug.h:337
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
uint8_t vlan_idx
Definition: flow.h:347
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
struct LiveDevice_ * livedev
Definition: flow.h:350
uint16_t Port
Definition: decode.h:235
struct timeval lastts
Definition: flow.h:358
uint8_t min_ttl_toserver
Definition: flow.h:429
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:619
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:222
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
struct Flow_::@120::@124 icmp_s
#define SCFree(a)
Definition: util-mem.h:322
Port dp
Definition: flow.h:338
#define FLOW_IS_IPV6(f)
Definition: flow.h:136
void * data
Definition: tm-modules.h:81
uint8_t max_ttl_toclient
Definition: flow.h:432
uint8_t min_ttl_toclient
Definition: flow.h:431
FlowAddress src
Definition: flow.h:329
Per thread variable structure.
Definition: threadvars.h:57
uint8_t code
AppProto alproto
application level protocol
Definition: flow.h:409
Flow data structure.
Definition: flow.h:325
uint32_t flags
Definition: flow.h:379
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
union FlowAddress_::@119 address
uint32_t tosrcpktcnt
Definition: flow.h:461