suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 typedef struct LogJsonFileCtx_ {
56 
57 typedef struct JsonNetFlowLogThread_ {
59  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
60 
63 
64 
65 static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type, int dir)
66 {
67  char timebuf[64];
68  char srcip[46] = {0}, dstip[46] = {0};
69  Port sp, dp;
70 
71  json_t *js = json_object();
72  if (unlikely(js == NULL))
73  return NULL;
74 
75  struct timeval tv;
76  memset(&tv, 0x00, sizeof(tv));
77  TimeGet(&tv);
78 
79  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
80 
81  /* reverse header direction if the flow started out wrong */
82  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
83 
84  if (FLOW_IS_IPV4(f)) {
85  if (dir == 0) {
86  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
87  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
88  } else {
89  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
90  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
91  }
92  } else if (FLOW_IS_IPV6(f)) {
93  if (dir == 0) {
94  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
95  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
96  } else {
97  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
98  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
99  }
100  }
101 
102  if (dir == 0) {
103  sp = f->sp;
104  dp = f->dp;
105  } else {
106  sp = f->dp;
107  dp = f->sp;
108  }
109 
110  char proto[16];
111  if (SCProtoNameValid(f->proto) == TRUE) {
112  strlcpy(proto, known_proto[f->proto], sizeof(proto));
113  } else {
114  snprintf(proto, sizeof(proto), "%03" PRIu32, f->proto);
115  }
116 
117  /* time */
118  json_object_set_new(js, "timestamp", json_string(timebuf));
119 
120  CreateJSONFlowId(js, (const Flow *)f);
121 
122 #if 0 // TODO
123  /* sensor id */
124  if (sensor_id >= 0)
125  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
126 #endif
127 
128  /* input interface */
129  if (f->livedev) {
130  json_object_set_new(js, "in_iface", json_string(f->livedev->dev));
131  }
132 
133  if (event_type) {
134  json_object_set_new(js, "event_type", json_string(event_type));
135  }
136 
137  /* vlan */
138  if (f->vlan_idx > 0) {
139  json_t *js_vlan = json_array();
140  json_array_append_new(js_vlan, json_integer(f->vlan_id[0]));
141  if (f->vlan_idx > 1) {
142  json_array_append_new(js_vlan, json_integer(f->vlan_id[1]));
143  }
144  json_object_set_new(js, "vlan", js_vlan);
145  }
146 
147  /* tuple */
148  json_object_set_new(js, "src_ip", json_string(srcip));
149  switch(f->proto) {
150  case IPPROTO_ICMP:
151  break;
152  case IPPROTO_UDP:
153  case IPPROTO_TCP:
154  case IPPROTO_SCTP:
155  json_object_set_new(js, "src_port", json_integer(sp));
156  break;
157  }
158  json_object_set_new(js, "dest_ip", json_string(dstip));
159  switch(f->proto) {
160  case IPPROTO_ICMP:
161  break;
162  case IPPROTO_UDP:
163  case IPPROTO_TCP:
164  case IPPROTO_SCTP:
165  json_object_set_new(js, "dest_port", json_integer(dp));
166  break;
167  }
168  json_object_set_new(js, "proto", json_string(proto));
169  switch (f->proto) {
170  case IPPROTO_ICMP:
171  case IPPROTO_ICMPV6: {
172  uint8_t type = f->icmp_s.type;
173  uint8_t code = f->icmp_s.code;
174  if (dir == 1) {
175  type = f->icmp_d.type;
176  code = f->icmp_d.code;
177 
178  }
179  json_object_set_new(js, "icmp_type", json_integer(type));
180  json_object_set_new(js, "icmp_code", json_integer(code));
181  break;
182  }
183  }
184  return js;
185 }
186 
187 /* JSON format logging */
188 static void JsonNetFlowLogJSONToServer(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
189 {
190  json_t *hjs = json_object();
191  if (hjs == NULL) {
192  return;
193  }
194 
195  json_object_set_new(js, "app_proto",
196  json_string(AppProtoToString(f->alproto_ts ? f->alproto_ts : f->alproto)));
197 
198  json_object_set_new(hjs, "pkts",
199  json_integer(f->todstpktcnt));
200  json_object_set_new(hjs, "bytes",
201  json_integer(f->todstbytecnt));
202 
203  char timebuf1[64], timebuf2[64];
204 
205  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
206  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
207 
208  json_object_set_new(hjs, "start", json_string(timebuf1));
209  json_object_set_new(hjs, "end", json_string(timebuf2));
210 
211  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
212  json_object_set_new(hjs, "age",
213  json_integer(age));
214 
215  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toserver));
216  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toserver));
217 
218  json_object_set_new(js, "netflow", hjs);
219 
220  /* TCP */
221  if (f->proto == IPPROTO_TCP) {
222  json_t *tjs = json_object();
223  if (tjs == NULL) {
224  return;
225  }
226 
227  TcpSession *ssn = f->protoctx;
228 
229  char hexflags[3];
230  snprintf(hexflags, sizeof(hexflags), "%02x",
231  ssn ? ssn->client.tcp_flags : 0);
232  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
233 
234  JsonTcpFlags(ssn ? ssn->client.tcp_flags : 0, tjs);
235 
236  json_object_set_new(js, "tcp", tjs);
237  }
238 }
239 
240 static void JsonNetFlowLogJSONToClient(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
241 {
242  json_t *hjs = json_object();
243  if (hjs == NULL) {
244  return;
245  }
246 
247  json_object_set_new(js, "app_proto",
248  json_string(AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)));
249 
250  json_object_set_new(hjs, "pkts",
251  json_integer(f->tosrcpktcnt));
252  json_object_set_new(hjs, "bytes",
253  json_integer(f->tosrcbytecnt));
254 
255  char timebuf1[64], timebuf2[64];
256 
257  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
258  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
259 
260  json_object_set_new(hjs, "start", json_string(timebuf1));
261  json_object_set_new(hjs, "end", json_string(timebuf2));
262 
263  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
264  json_object_set_new(hjs, "age",
265  json_integer(age));
266 
267  /* To client is zero if we did not see any packet */
268  if (f->tosrcpktcnt) {
269  json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toclient));
270  json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toclient));
271  }
272 
273  json_object_set_new(js, "netflow", hjs);
274 
275  /* TCP */
276  if (f->proto == IPPROTO_TCP) {
277  json_t *tjs = json_object();
278  if (tjs == NULL) {
279  return;
280  }
281 
282  TcpSession *ssn = f->protoctx;
283 
284  char hexflags[3];
285  snprintf(hexflags, sizeof(hexflags), "%02x",
286  ssn ? ssn->server.tcp_flags : 0);
287  json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
288 
289  JsonTcpFlags(ssn ? ssn->server.tcp_flags : 0, tjs);
290 
291  json_object_set_new(js, "tcp", tjs);
292  }
293 }
294 
295 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
296 {
297  SCEnter();
298  JsonNetFlowLogThread *jhl = (JsonNetFlowLogThread *)thread_data;
299  LogJsonFileCtx *netflow_ctx = jhl->flowlog_ctx;
300 
301  /* reset */
302  MemBufferReset(jhl->buffer);
303  json_t *js = CreateJSONHeaderFromFlow(f, "netflow", 0);
304  if (unlikely(js == NULL))
305  return TM_ECODE_OK;
306  JsonNetFlowLogJSONToServer(jhl, js, f);
307  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
308  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
309  json_object_del(js, "netflow");
310  json_object_clear(js);
311  json_decref(js);
312 
313  /* only log a response record if we actually have seen response packets */
314  if (f->tosrcpktcnt) {
315  /* reset */
316  MemBufferReset(jhl->buffer);
317  js = CreateJSONHeaderFromFlow(f, "netflow", 1);
318  if (unlikely(js == NULL))
319  return TM_ECODE_OK;
320  JsonNetFlowLogJSONToClient(jhl, js, f);
321  JsonAddCommonOptions(&netflow_ctx->cfg, NULL, f, js);
322  OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
323  json_object_del(js, "netflow");
324  json_object_clear(js);
325  json_decref(js);
326  }
328 }
329 
330 static void OutputNetFlowLogDeinit(OutputCtx *output_ctx)
331 {
332  LogJsonFileCtx *flow_ctx = output_ctx->data;
333  LogFileCtx *logfile_ctx = flow_ctx->file_ctx;
334  LogFileFreeCtx(logfile_ctx);
335  SCFree(flow_ctx);
336  SCFree(output_ctx);
337 }
338 
339 #define DEFAULT_LOG_FILENAME "netflow.json"
340 static OutputInitResult OutputNetFlowLogInit(ConfNode *conf)
341 {
342  OutputInitResult result = { NULL, false };
344  if(file_ctx == NULL) {
345  SCLogError(SC_ERR_NETFLOW_LOG_GENERIC, "couldn't create new file_ctx");
346  return result;
347  }
348 
349  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
350  LogFileFreeCtx(file_ctx);
351  return result;
352  }
353 
354  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
355  if (unlikely(flow_ctx == NULL)) {
356  LogFileFreeCtx(file_ctx);
357  return result;
358  }
359 
360  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
361  if (unlikely(output_ctx == NULL)) {
362  LogFileFreeCtx(file_ctx);
363  SCFree(flow_ctx);
364  return result;
365  }
366 
367  flow_ctx->file_ctx = file_ctx;
368  output_ctx->data = flow_ctx;
369  output_ctx->DeInit = OutputNetFlowLogDeinit;
370 
371  result.ctx = output_ctx;
372  result.ok = true;
373  return result;
374 }
375 
376 static void OutputNetFlowLogDeinitSub(OutputCtx *output_ctx)
377 {
378  LogJsonFileCtx *flow_ctx = output_ctx->data;
379  SCFree(flow_ctx);
380  SCFree(output_ctx);
381 }
382 
383 static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
384 {
385  OutputInitResult result = { NULL, false };
386  OutputJsonCtx *ojc = parent_ctx->data;
387 
388  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
389  if (unlikely(flow_ctx == NULL))
390  return result;
391 
392  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
393  if (unlikely(output_ctx == NULL)) {
394  SCFree(flow_ctx);
395  return result;
396  }
397 
398  flow_ctx->file_ctx = ojc->file_ctx;
399  flow_ctx->cfg = ojc->cfg;
400 
401  output_ctx->data = flow_ctx;
402  output_ctx->DeInit = OutputNetFlowLogDeinitSub;
403 
404  result.ctx = output_ctx;
405  result.ok = true;
406  return result;
407 }
408 
409 static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
410 {
412  if (unlikely(aft == NULL))
413  return TM_ECODE_FAILED;
414  memset(aft, 0, sizeof(JsonNetFlowLogThread));
415 
416  if(initdata == NULL)
417  {
418  SCLogDebug("Error getting context for EveLogNetflow. \"initdata\" argument NULL");
419  SCFree(aft);
420  return TM_ECODE_FAILED;
421  }
422 
423  /* Use the Ouptut Context (file pointer and mutex) */
424  aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
425 
427  if (aft->buffer == NULL) {
428  SCFree(aft);
429  return TM_ECODE_FAILED;
430  }
431 
432  *data = (void *)aft;
433  return TM_ECODE_OK;
434 }
435 
436 static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data)
437 {
439  if (aft == NULL) {
440  return TM_ECODE_OK;
441  }
442 
443  MemBufferFree(aft->buffer);
444  /* clear memory */
445  memset(aft, 0, sizeof(JsonNetFlowLogThread));
446 
447  SCFree(aft);
448  return TM_ECODE_OK;
449 }
450 
452 {
453  /* register as separate module */
455  "netflow-json-log", OutputNetFlowLogInit, JsonNetFlowLogger,
456  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
457 
458  /* also register as child of eve-log */
459  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog",
460  "eve-log.netflow", OutputNetFlowLogInitSub, JsonNetFlowLogger,
461  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
462 }
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define FLOW_IS_IPV4(f)
Definition: flow.h:134
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:44
char * known_proto[256]
uint8_t SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
#define SCLogDebug(...)
Definition: util-debug.h:335
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:809
AppProto alproto_tc
Definition: flow.h:411
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
uint64_t todstbytecnt
Definition: flow.h:462
uint32_t event_type
uint8_t proto
Definition: flow.h:344
struct JsonNetFlowLogThread_ JsonNetFlowLogThread
Port sp
Definition: flow.h:331
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define FLOW_DIR_REVERSED
Definition: flow.h:106
uint8_t max_ttl_toserver
Definition: flow.h:430
struct Flow_::@121::@125 icmp_s
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
struct timeval startts
Definition: flow.h:458
uint32_t todstpktcnt
Definition: flow.h:460
void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, json_t *js)
Definition: output-json.c:390
#define TRUE
void * protoctx
Definition: flow.h:400
FlowAddress dst
Definition: flow.h:329
char * dev
Definition: util-device.h:41
void OutputRegisterFlowModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output module.
Definition: output.c:582
void JsonNetFlowLogRegister(void)
uint16_t vlan_id[2]
Definition: flow.h:346
AppProto alproto_ts
Definition: flow.h:410
uint64_t tosrcbytecnt
Definition: flow.h:463
#define SCCalloc(nm, a)
Definition: util-mem.h:253
void JsonTcpFlags(uint8_t flags, json_t *js)
jsonify tcp flags field Only add &#39;true&#39; fields in an attempt to keep things reasonably compact...
Definition: output-json.c:428
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
LogFileCtx * file_ctx
void TimeGet(struct timeval *tv)
Definition: util-time.c:146
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
uint8_t type
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
OutputJsonCommonSettings cfg
Definition: output-json.h:81
LogFileCtx * file_ctx
Definition: output-json.h:79
#define SCEnter(...)
Definition: util-debug.h:337
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
uint8_t vlan_idx
Definition: flow.h:347
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
struct LiveDevice_ * livedev
Definition: flow.h:350
uint16_t Port
Definition: decode.h:235
LogJsonFileCtx * flowlog_ctx
struct Flow_::@123::@126 icmp_d
struct timeval lastts
Definition: flow.h:358
uint8_t min_ttl_toserver
Definition: flow.h:429
OutputJsonCommonSettings cfg
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:621
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:222
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCFree(a)
Definition: util-mem.h:322
Port dp
Definition: flow.h:338
#define FLOW_IS_IPV6(f)
Definition: flow.h:136
union FlowAddress_::@120 address
struct LogJsonFileCtx_ LogJsonFileCtx
void * data
Definition: tm-modules.h:81
uint8_t max_ttl_toclient
Definition: flow.h:432
uint8_t min_ttl_toclient
Definition: flow.h:431
FlowAddress src
Definition: flow.h:329
Per thread variable structure.
Definition: threadvars.h:57
uint8_t code
AppProto alproto
application level protocol
Definition: flow.h:409
void CreateJSONFlowId(json_t *js, const Flow *f)
Definition: output-json.c:699
Flow data structure.
Definition: flow.h:325
uint32_t flags
Definition: flow.h:379
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
uint32_t tosrcpktcnt
Definition: flow.h:461
#define DEFAULT_LOG_FILENAME