suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 #ifdef HAVE_LIBJANSSON
53 
54 typedef struct LogJsonFileCtx_ {
55  LogFileCtx *file_ctx;
56  OutputJsonCommonSettings cfg;
57 } LogJsonFileCtx;
58 
59 typedef struct JsonNetFlowLogThread_ {
60  LogJsonFileCtx *flowlog_ctx;
61  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
62 
63  MemBuffer *buffer;
64 } JsonNetFlowLogThread;
65 
66 
67 static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type, int dir)
68 {
69  char timebuf[64];
70  char srcip[46], dstip[46];
71  Port sp, dp;
72 
73  json_t *js = json_object();
74  if (unlikely(js == NULL))
75  return NULL;
76 
77  struct timeval tv;
78  memset(&tv, 0x00, sizeof(tv));
79  TimeGet(&tv);
80 
81  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
82 
83  srcip[0] = '\0';
84  dstip[0] = '\0';
85  if (FLOW_IS_IPV4(f)) {
86  if (dir == 0) {
87  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
88  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
89  } else {
90  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
91  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
92  }
93  } else if (FLOW_IS_IPV6(f)) {
94  if (dir == 0) {
95  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
96  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
97  } else {
98  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
99  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
100  }
101  }
102 
103  if (dir == 0) {
104  sp = f->sp;
105  dp = f->dp;
106  } else {
107  sp = f->dp;
108  dp = f->sp;
109  }
110 
111  char proto[16];
112  if (SCProtoNameValid(f->proto) == TRUE) {
113  strlcpy(proto, known_proto[f->proto], sizeof(proto));
114  } else {
115  snprintf(proto, sizeof(proto), "%03" PRIu32, f->proto);
116  }
117 
118  /* time */
119  json_object_set_new(js, "timestamp", json_string(timebuf));
120 
121  CreateJSONFlowId(js, (const Flow *)f);
122 
123 #if 0 // TODO
124  /* sensor id */
125  if (sensor_id >= 0)
126  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
127 #endif
128  if (event_type) {
129  json_object_set_new(js, "event_type", json_string(event_type));
130  }
131 #if 0
132  /* vlan */
133  if (f->vlan_id[0] > 0) {
134  json_t *js_vlan;
135  switch (f->vlan_idx) {
136  case 1:
137  json_object_set_new(js, "vlan",
138  json_integer(f->vlan_id[0]));
139  break;
140  case 2:
141  js_vlan = json_array();
142  if (unlikely(js != NULL)) {
143  json_array_append_new(js_vlan,
144  json_integer(VLAN_GET_ID1(p)));
145  json_array_append_new(js_vlan,
146  json_integer(VLAN_GET_ID2(p)));
147  json_object_set_new(js, "vlan", js_vlan);
148  }
149  break;
150  default:
151  /* shouldn't get here */
152  break;
153  }
154  }
155 #endif
156  /* tuple */
157  json_object_set_new(js, "src_ip", json_string(srcip));
158  switch(f->proto) {
159  case IPPROTO_ICMP:
160  break;
161  case IPPROTO_UDP:
162  case IPPROTO_TCP:
163  case IPPROTO_SCTP:
164  json_object_set_new(js, "src_port", json_integer(sp));
165  break;
166  }
167  json_object_set_new(js, "dest_ip", json_string(dstip));
168  switch(f->proto) {
169  case IPPROTO_ICMP:
170  break;
171  case IPPROTO_UDP:
172  case IPPROTO_TCP:
173  case IPPROTO_SCTP:
174  json_object_set_new(js, "dest_port", json_integer(dp));
175  break;
176  }
177  json_object_set_new(js, "proto", json_string(proto));
178  switch (f->proto) {
179  case IPPROTO_ICMP:
180  case IPPROTO_ICMPV6: {
181  uint8_t type = f->icmp_s.type;
182  uint8_t code = f->icmp_s.code;
183  if (dir == 1) {
184  type = f->icmp_d.type;
185  code = f->icmp_d.code;
186 
187  }
188  json_object_set_new(js, "icmp_type", json_integer(type));
189  json_object_set_new(js, "icmp_code", json_integer(code));
190  break;
191  }
192  }
193  return js;
194 }
195 
196 /* JSON format logging */
197 static void JsonNetFlowLogJSONToServer(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
198 {
199  json_t *hjs = json_object();
200  if (hjs == NULL) {
201  return;
202  }
203 
204  json_object_set_new(js, "app_proto",
205  json_string(AppProtoToString(f->alproto_ts ? f->alproto_ts : f->alproto)));
206 
207  json_object_set_new(hjs, "pkts",
208  json_integer(f->todstpktcnt));
209  json_object_set_new(hjs, "bytes",
210  json_integer(f->todstbytecnt));
211 
212  char timebuf1[64], timebuf2[64];
213 
214  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
215  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
216 
217  json_object_set_new(hjs, "start", json_string(timebuf1));
218  json_object_set_new(hjs, "end", json_string(timebuf2));
219 
220  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
221  json_object_set_new(hjs, "age",
222  json_integer(age));
223 
224  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toserver));
225  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toserver));
226 
227  json_object_set_new(js, "netflow", hjs);
228 
229  /* TCP */
230  if (f->proto == IPPROTO_TCP) {
231  json_t *tjs = json_object();
232  if (tjs == NULL) {
233  return;
234  }
235 
236  TcpSession *ssn = f->protoctx;
237 
238  char hexflags[3];
239  snprintf(hexflags, sizeof(hexflags), "%02x",
240  ssn ? ssn->client.tcp_flags : 0);
241  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
242 
243  JsonTcpFlags(ssn ? ssn->client.tcp_flags : 0, tjs);
244 
245  json_object_set_new(js, "tcp", tjs);
246  }
247 }
248 
249 static void JsonNetFlowLogJSONToClient(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
250 {
251  json_t *hjs = json_object();
252  if (hjs == NULL) {
253  return;
254  }
255 
256  json_object_set_new(js, "app_proto",
257  json_string(AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)));
258 
259  json_object_set_new(hjs, "pkts",
260  json_integer(f->tosrcpktcnt));
261  json_object_set_new(hjs, "bytes",
262  json_integer(f->tosrcbytecnt));
263 
264  char timebuf1[64], timebuf2[64];
265 
266  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
267  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
268 
269  json_object_set_new(hjs, "start", json_string(timebuf1));
270  json_object_set_new(hjs, "end", json_string(timebuf2));
271 
272  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
273  json_object_set_new(hjs, "age",
274  json_integer(age));
275 
276  /* To client is zero if we did not see any packet */
277  if (f->tosrcpktcnt) {
278  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toclient));
279  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toclient));
280  }
281 
282  json_object_set_new(js, "netflow", hjs);
283 
284  /* TCP */
285  if (f->proto == IPPROTO_TCP) {
286  json_t *tjs = json_object();
287  if (tjs == NULL) {
288  return;
289  }
290 
291  TcpSession *ssn = f->protoctx;
292 
293  char hexflags[3];
294  snprintf(hexflags, sizeof(hexflags), "%02x",
295  ssn ? ssn->server.tcp_flags : 0);
296  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
297 
298  JsonTcpFlags(ssn ? ssn->server.tcp_flags : 0, tjs);
299 
300  json_object_set_new(js, "tcp", tjs);
301  }
302 }
303 
304 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
305 {
306  SCEnter();
307  JsonNetFlowLogThread *jhl = (JsonNetFlowLogThread *)thread_data;
308  LogJsonFileCtx *netflow_ctx = jhl->flowlog_ctx;
309 
310  /* reset */
311  MemBufferReset(jhl->buffer);
312  json_t *js = CreateJSONHeaderFromFlow(f, "netflow", 0);
313  if (unlikely(js == NULL))
314  return TM_ECODE_OK;
315  JsonNetFlowLogJSONToServer(jhl, js, f);
316  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
317  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
318  json_object_del(js, "netflow");
319  json_object_clear(js);
320  json_decref(js);
321 
322  /* only log a response record if we actually have seen response packets */
323  if (f->tosrcpktcnt) {
324  /* reset */
325  MemBufferReset(jhl->buffer);
326  js = CreateJSONHeaderFromFlow(f, "netflow", 1);
327  if (unlikely(js == NULL))
328  return TM_ECODE_OK;
329  JsonNetFlowLogJSONToClient(jhl, js, f);
330  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
331  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
332  json_object_del(js, "netflow");
333  json_object_clear(js);
334  json_decref(js);
335  }
337 }
338 
339 static void OutputNetFlowLogDeinit(OutputCtx *output_ctx)
340 {
341  LogJsonFileCtx *flow_ctx = output_ctx->data;
342  LogFileCtx *logfile_ctx = flow_ctx->file_ctx;
343  LogFileFreeCtx(logfile_ctx);
344  SCFree(flow_ctx);
345  SCFree(output_ctx);
346 }
347 
348 #define DEFAULT_LOG_FILENAME "netflow.json"
349 static OutputInitResult OutputNetFlowLogInit(ConfNode *conf)
350 {
351  OutputInitResult result = { NULL, false };
352  LogFileCtx *file_ctx = LogFileNewCtx();
353  if(file_ctx == NULL) {
354  SCLogError(SC_ERR_NETFLOW_LOG_GENERIC, "couldn't create new file_ctx");
355  return result;
356  }
357 
358  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
359  LogFileFreeCtx(file_ctx);
360  return result;
361  }
362 
363  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
364  if (unlikely(flow_ctx == NULL)) {
365  LogFileFreeCtx(file_ctx);
366  return result;
367  }
368 
369  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
370  if (unlikely(output_ctx == NULL)) {
371  LogFileFreeCtx(file_ctx);
372  SCFree(flow_ctx);
373  return result;
374  }
375 
376  flow_ctx->file_ctx = file_ctx;
377  output_ctx->data = flow_ctx;
378  output_ctx->DeInit = OutputNetFlowLogDeinit;
379 
380  result.ctx = output_ctx;
381  result.ok = true;
382  return result;
383 }
384 
385 static void OutputNetFlowLogDeinitSub(OutputCtx *output_ctx)
386 {
387  LogJsonFileCtx *flow_ctx = output_ctx->data;
388  SCFree(flow_ctx);
389  SCFree(output_ctx);
390 }
391 
392 static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
393 {
394  OutputInitResult result = { NULL, false };
395  OutputJsonCtx *ojc = parent_ctx->data;
396 
397  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
398  if (unlikely(flow_ctx == NULL))
399  return result;
400 
401  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
402  if (unlikely(output_ctx == NULL)) {
403  SCFree(flow_ctx);
404  return result;
405  }
406 
407  flow_ctx->file_ctx = ojc->file_ctx;
408  flow_ctx->cfg = ojc->cfg;
409 
410  output_ctx->data = flow_ctx;
411  output_ctx->DeInit = OutputNetFlowLogDeinitSub;
412 
413  result.ctx = output_ctx;
414  result.ok = true;
415  return result;
416 }
417 
418 #define OUTPUT_BUFFER_SIZE 65535
419 static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
420 {
421  JsonNetFlowLogThread *aft = SCMalloc(sizeof(JsonNetFlowLogThread));
422  if (unlikely(aft == NULL))
423  return TM_ECODE_FAILED;
424  memset(aft, 0, sizeof(JsonNetFlowLogThread));
425 
426  if(initdata == NULL)
427  {
428  SCLogDebug("Error getting context for EveLogNetflow. \"initdata\" argument NULL");
429  SCFree(aft);
430  return TM_ECODE_FAILED;
431  }
432 
433  /* Use the Ouptut Context (file pointer and mutex) */
434  aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
435 
436  aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
437  if (aft->buffer == NULL) {
438  SCFree(aft);
439  return TM_ECODE_FAILED;
440  }
441 
442  *data = (void *)aft;
443  return TM_ECODE_OK;
444 }
445 
446 static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data)
447 {
448  JsonNetFlowLogThread *aft = (JsonNetFlowLogThread *)data;
449  if (aft == NULL) {
450  return TM_ECODE_OK;
451  }
452 
453  MemBufferFree(aft->buffer);
454  /* clear memory */
455  memset(aft, 0, sizeof(JsonNetFlowLogThread));
456 
457  SCFree(aft);
458  return TM_ECODE_OK;
459 }
460 
461 void JsonNetFlowLogRegister(void)
462 {
463  /* register as separate module */
465  "netflow-json-log", OutputNetFlowLogInit, JsonNetFlowLogger,
466  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
467 
468  /* also register as child of eve-log */
469  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog",
470  "eve-log.netflow", OutputNetFlowLogInitSub, JsonNetFlowLogger,
471  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
472 }
473 
474 #else
475 
477 {
478 }
479 
480 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define FLOW_IS_IPV4(f)
Definition: flow.h:131
char * known_proto[256]
uint8_t SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
#define SCLogDebug(...)
Definition: util-debug.h:335
AppProto alproto_tc
Definition: flow.h:409
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
uint64_t todstbytecnt
Definition: flow.h:460
uint32_t event_type
uint8_t proto
Definition: flow.h:346
Port sp
Definition: flow.h:333
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
uint8_t max_ttl_toserver
Definition: flow.h:428
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
struct timeval startts
Definition: flow.h:456
uint32_t todstpktcnt
Definition: flow.h:458
#define TRUE
void * protoctx
Definition: flow.h:398
FlowAddress dst
Definition: flow.h:331
struct Flow_::@124::@128 icmp_s
void OutputRegisterFlowModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output module.
Definition: output.c:579
void JsonNetFlowLogRegister(void)
uint16_t vlan_id[2]
Definition: flow.h:348
AppProto alproto_ts
Definition: flow.h:408
uint64_t tosrcbytecnt
Definition: flow.h:461
#define SCCalloc(nm, a)
Definition: util-mem.h:205
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:179
uint16_t type
void TimeGet(struct timeval *tv)
Definition: util-time.c:138
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define VLAN_GET_ID2(p)
Definition: decode-vlan.h:41
#define SCEnter(...)
Definition: util-debug.h:337
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
struct Flow_::@126::@129 icmp_d
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
uint16_t Port
Definition: decode.h:234
struct timeval lastts
Definition: flow.h:356
uint8_t min_ttl_toserver
Definition: flow.h:427
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:618
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:174
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
union FlowAddress_::@123 address
#define SCFree(a)
Definition: util-mem.h:236
#define VLAN_GET_ID1(p)
Definition: decode-vlan.h:40
Port dp
Definition: flow.h:340
#define OUTPUT_BUFFER_SIZE
Definition: log-dnslog.c:58
#define FLOW_IS_IPV6(f)
Definition: flow.h:133
void * data
Definition: tm-modules.h:81
uint8_t max_ttl_toclient
Definition: flow.h:430
uint8_t min_ttl_toclient
Definition: flow.h:429
FlowAddress src
Definition: flow.h:331
Per thread variable structure.
Definition: threadvars.h:57
uint8_t code
AppProto alproto
application level protocol
Definition: flow.h:407
Flow data structure.
Definition: flow.h:327
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
uint32_t tosrcpktcnt
Definition: flow.h:459