suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 typedef struct LogJsonFileCtx_ {
56 
57 typedef struct JsonNetFlowLogThread_ {
60  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
61 
64 
65 static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir)
66 {
67  char timebuf[64];
68  char srcip[46] = {0}, dstip[46] = {0};
69  Port sp, dp;
70 
71  JsonBuilder *js = jb_new_object();
72  if (unlikely(js == NULL))
73  return NULL;
74 
75  struct timeval tv;
76  memset(&tv, 0x00, sizeof(tv));
77  TimeGet(&tv);
78 
79  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
80 
81  /* reverse header direction if the flow started out wrong */
82  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
83 
84  if (FLOW_IS_IPV4(f)) {
85  if (dir == 0) {
86  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
87  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
88  } else {
89  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
90  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
91  }
92  } else if (FLOW_IS_IPV6(f)) {
93  if (dir == 0) {
94  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
95  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
96  } else {
97  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
98  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
99  }
100  }
101 
102  if (dir == 0) {
103  sp = f->sp;
104  dp = f->dp;
105  } else {
106  sp = f->dp;
107  dp = f->sp;
108  }
109 
110  /* time */
111  jb_set_string(js, "timestamp", timebuf);
112 
113  CreateEveFlowId(js, (const Flow *)f);
114 
115 #if 0 // TODO
116  /* sensor id */
117  if (sensor_id >= 0)
118  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
119 #endif
120 
121  /* input interface */
122  if (f->livedev) {
123  jb_set_string(js, "in_iface", f->livedev->dev);
124  }
125 
126  JB_SET_STRING(js, "event_type", "netflow");
127 
128  /* vlan */
129  if (f->vlan_idx > 0) {
130  jb_open_array(js, "vlan");
131  jb_append_uint(js, f->vlan_id[0]);
132  if (f->vlan_idx > 1) {
133  jb_append_uint(js, f->vlan_id[1]);
134  }
135  jb_close(js);
136  }
137 
138  /* tuple */
139  jb_set_string(js, "src_ip", srcip);
140  switch(f->proto) {
141  case IPPROTO_ICMP:
142  break;
143  case IPPROTO_UDP:
144  case IPPROTO_TCP:
145  case IPPROTO_SCTP:
146  jb_set_uint(js, "src_port", sp);
147  break;
148  }
149  jb_set_string(js, "dest_ip", dstip);
150  switch(f->proto) {
151  case IPPROTO_ICMP:
152  break;
153  case IPPROTO_UDP:
154  case IPPROTO_TCP:
155  case IPPROTO_SCTP:
156  jb_set_uint(js, "dest_port", dp);
157  break;
158  }
159 
160  if (SCProtoNameValid(f->proto)) {
161  jb_set_string(js, "proto", known_proto[f->proto]);
162  } else {
163  char proto[4];
164  snprintf(proto, sizeof(proto), "%"PRIu8"", f->proto);
165  jb_set_string(js, "proto", proto);
166  }
167 
168  switch (f->proto) {
169  case IPPROTO_ICMP:
170  case IPPROTO_ICMPV6: {
171  uint8_t type = f->icmp_s.type;
172  uint8_t code = f->icmp_s.code;
173  if (dir == 1) {
174  type = f->icmp_d.type;
175  code = f->icmp_d.code;
176 
177  }
178  jb_set_uint(js, "icmp_type", type);
179  jb_set_uint(js, "icmp_code", code);
180  break;
181  }
182  }
183  return js;
184 }
185 
186 /* JSON format logging */
187 static void NetFlowLogEveToServer(JsonNetFlowLogThread *aft, JsonBuilder *js, Flow *f)
188 {
189  jb_set_string(js, "app_proto",
191 
192  jb_open_object(js, "netflow");
193 
194  jb_set_uint(js, "pkts", f->todstpktcnt);
195  jb_set_uint(js, "bytes", f->todstbytecnt);
196 
197  char timebuf1[64], timebuf2[64];
198 
199  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
200  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
201 
202  jb_set_string(js, "start", timebuf1);
203  jb_set_string(js, "end", timebuf2);
204 
205  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
206  jb_set_uint(js, "age", age);
207 
208  jb_set_uint(js, "min_ttl", f->min_ttl_toserver);
209  jb_set_uint(js, "max_ttl", f->max_ttl_toserver);
210 
211  /* Close netflow. */
212  jb_close(js);
213 
214  /* TCP */
215  if (f->proto == IPPROTO_TCP) {
216  jb_open_object(js, "tcp");
217 
218  TcpSession *ssn = f->protoctx;
219 
220  char hexflags[3];
221  snprintf(hexflags, sizeof(hexflags), "%02x",
222  ssn ? ssn->client.tcp_flags : 0);
223  jb_set_string(js, "tcp_flags", hexflags);
224 
225  EveTcpFlags(ssn ? ssn->client.tcp_flags : 0, js);
226 
227  jb_close(js);
228  }
229 }
230 
231 static void NetFlowLogEveToClient(JsonNetFlowLogThread *aft, JsonBuilder *js, Flow *f)
232 {
233  jb_set_string(js, "app_proto",
235 
236  jb_open_object(js, "netflow");
237 
238  jb_set_uint(js, "pkts", f->tosrcpktcnt);
239  jb_set_uint(js, "bytes", f->tosrcbytecnt);
240 
241  char timebuf1[64], timebuf2[64];
242 
243  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
244  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
245 
246  jb_set_string(js, "start", timebuf1);
247  jb_set_string(js, "end", timebuf2);
248 
249  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
250  jb_set_uint(js, "age", age);
251 
252  /* To client is zero if we did not see any packet */
253  if (f->tosrcpktcnt) {
254  jb_set_uint(js, "min_ttl", f->min_ttl_toclient);
255  jb_set_uint(js, "max_ttl", f->max_ttl_toclient);
256  }
257 
258  /* Close netflow. */
259  jb_close(js);
260 
261  /* TCP */
262  if (f->proto == IPPROTO_TCP) {
263  jb_open_object(js, "tcp");
264 
265  TcpSession *ssn = f->protoctx;
266 
267  char hexflags[3];
268  snprintf(hexflags, sizeof(hexflags), "%02x",
269  ssn ? ssn->server.tcp_flags : 0);
270  jb_set_string(js, "tcp_flags", hexflags);
271 
272  EveTcpFlags(ssn ? ssn->server.tcp_flags : 0, js);
273 
274  jb_close(js);
275  }
276 }
277 
278 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
279 {
280  SCEnter();
281  JsonNetFlowLogThread *jhl = (JsonNetFlowLogThread *)thread_data;
282  LogJsonFileCtx *netflow_ctx = jhl->flowlog_ctx;
283 
284  /* reset */
285  MemBufferReset(jhl->buffer);
286  JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0);
287  if (unlikely(jb == NULL))
288  return TM_ECODE_OK;
289  NetFlowLogEveToServer(jhl, jb, f);
290  EveAddCommonOptions(&netflow_ctx->cfg, NULL, f, jb);
291  OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer);
292  jb_free(jb);
293 
294  /* only log a response record if we actually have seen response packets */
295  if (f->tosrcpktcnt) {
296  /* reset */
297  MemBufferReset(jhl->buffer);
298  jb = CreateEveHeaderFromNetFlow(f, 1);
299  if (unlikely(jb == NULL))
300  return TM_ECODE_OK;
301  NetFlowLogEveToClient(jhl, jb, f);
302  EveAddCommonOptions(&netflow_ctx->cfg, NULL, f, jb);
303  OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer);
304  jb_free(jb);
305  }
307 }
308 
309 static void OutputNetFlowLogDeinitSub(OutputCtx *output_ctx)
310 {
311  LogJsonFileCtx *flow_ctx = output_ctx->data;
312  SCFree(flow_ctx);
313  SCFree(output_ctx);
314 }
315 
316 static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
317 {
318  OutputInitResult result = { NULL, false };
319  OutputJsonCtx *ojc = parent_ctx->data;
320 
321  LogJsonFileCtx *flow_ctx = SCMalloc(sizeof(LogJsonFileCtx));
322  if (unlikely(flow_ctx == NULL))
323  return result;
324 
325  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
326  if (unlikely(output_ctx == NULL)) {
327  SCFree(flow_ctx);
328  return result;
329  }
330 
331  flow_ctx->file_ctx = ojc->file_ctx;
332  flow_ctx->cfg = ojc->cfg;
333 
334  output_ctx->data = flow_ctx;
335  output_ctx->DeInit = OutputNetFlowLogDeinitSub;
336 
337  result.ctx = output_ctx;
338  result.ok = true;
339  return result;
340 }
341 
342 static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
343 {
345  if (unlikely(aft == NULL))
346  return TM_ECODE_FAILED;
347 
348  if(initdata == NULL) {
349  SCLogDebug("Error getting context for EveLogNetflow. \"initdata\" argument NULL");
350  goto error_exit;
351  }
352 
353  /* Use the Ouptut Context (file pointer and mutex) */
354  aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
355 
357  if (aft->buffer == NULL) {
358  goto error_exit;
359  }
360 
362  if (!aft->file_ctx) {
363  goto error_exit;
364  }
365 
366  *data = (void *)aft;
367  return TM_ECODE_OK;
368 
369 error_exit:
370  if (aft->buffer != NULL) {
371  MemBufferFree(aft->buffer);
372  }
373  SCFree(aft);
374  return TM_ECODE_FAILED;
375 }
376 
377 static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data)
378 {
380  if (aft == NULL) {
381  return TM_ECODE_OK;
382  }
383 
384  MemBufferFree(aft->buffer);
385  /* clear memory */
386  memset(aft, 0, sizeof(JsonNetFlowLogThread));
387 
388  SCFree(aft);
389  return TM_ECODE_OK;
390 }
391 
393 {
394  /* register as child of eve-log */
395  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog",
396  "eve-log.netflow", OutputNetFlowLogInitSub, JsonNetFlowLogger,
397  JsonNetFlowLogThreadInit, JsonNetFlowLogThreadDeinit, NULL);
398 }
Flow_::icmp_d
struct Flow_::@122::@127 icmp_d
tm-threads.h
JsonNetFlowLogThread
struct JsonNetFlowLogThread_ JsonNetFlowLogThread
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:154
CreateIsoTimeString
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:213
Flow_::startts
struct timeval startts
Definition: flow.h:490
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:104
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
LogJsonFileCtx
struct LogJsonFileCtx_ LogJsonFileCtx
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
Flow_::proto
uint8_t proto
Definition: flow.h:365
JSON_OUTPUT_BUFFER_SIZE
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:63
threads.h
OutputJsonCtx_
Definition: output-json.h:101
Flow_
Flow data structure.
Definition: flow.h:347
OutputJsonCommonSettings_
Definition: output-json.h:91
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
LogFileCtx_
Definition: util-logopenfile.h:60
output-json-netflow.h
Flow_::vlan_id
uint16_t vlan_id[2]
Definition: flow.h:367
SCProtoNameValid
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
Definition: util-proto-name.c:98
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:978
util-privs.h
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
Definition: output-json.c:444
JsonNetFlowLogThread_::buffer
MemBuffer * buffer
Definition: output-json-netflow.c:62
Flow_::max_ttl_toserver
uint8_t max_ttl_toserver
Definition: flow.h:468
proto
uint8_t proto
Definition: decode-template.h:0
Flow_::dp
Port dp
Definition: flow.h:359
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:81
Flow_::protoctx
void * protoctx
Definition: flow.h:441
util-unittest.h
OutputCtx_::data
void * data
Definition: tm-modules.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
OutputCtx_
Definition: tm-modules.h:78
LogJsonFileCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-flow.c:55
Flow_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:495
Flow_::dst
FlowAddress dst
Definition: flow.h:350
Flow_::min_ttl_toserver
uint8_t min_ttl_toserver
Definition: flow.h:467
util-debug.h
JB_SET_STRING
#define JB_SET_STRING(jb, key, val)
Definition: rust.h:24
type
uint8_t type
Definition: decode-icmpv4.h:0
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:44
Flow_::todstpktcnt
uint32_t todstpktcnt
Definition: flow.h:492
output-json.h
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
PrintInet
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:271
pkt-var.h
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:659
EveTcpFlags
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
Definition: output-json.c:488
LogJsonFileCtx_
Definition: output-json-flow.c:54
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:45
LiveDevice_::dev
char * dev
Definition: util-device.h:41
Flow_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:494
ThreadVars_::id
int id
Definition: threadvars.h:87
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:152
LOGGER_JSON_NETFLOW
@ LOGGER_JSON_NETFLOW
Definition: suricata-common.h:485
stream-tcp-private.h
conf.h
Port
uint16_t Port
Definition: decode.h:241
TmEcode
TmEcode
Definition: tm-threads-common.h:79
JsonNetFlowLogThread_::flowlog_ctx
LogJsonFileCtx * flowlog_ctx
Definition: output-json-netflow.c:59
Flow_::vlan_idx
uint8_t vlan_idx
Definition: flow.h:375
Flow_::min_ttl_toclient
uint8_t min_ttl_toclient
Definition: flow.h:469
util-proto-name.h
FlowAddress_::address
union FlowAddress_::@119 address
MemBuffer_
Definition: util-buffer.h:27
Flow_::src
FlowAddress src
Definition: flow.h:350
Flow_::lastts
struct timeval lastts
Definition: flow.h:404
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
OutputInitResult_
Definition: output.h:43
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
Flow_::max_ttl_toclient
uint8_t max_ttl_toclient
Definition: flow.h:470
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
Flow_::livedev
struct LiveDevice_ * livedev
Definition: flow.h:396
LogJsonFileCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json-flow.c:57
threadvars.h
CreateEveFlowId
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
Definition: output-json.c:760
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
OutputRegisterFlowSubModule
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:585
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
ConfNode_
Definition: conf.h:32
util-logopenfile.h
Flow_::flags
uint32_t flags
Definition: flow.h:421
util-buffer.h
TimeGet
void TimeGet(struct timeval *tv)
Definition: util-time.c:153
OutputJsonCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:102
known_proto
char * known_proto[256]
Definition: util-proto-name.c:33
JsonNetFlowLogThread_
Definition: output-json-netflow.c:57
Flow_::sp
Port sp
Definition: flow.h:352
TcpSession_
Definition: stream-tcp-private.h:260
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:107
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
Flow_::icmp_s
struct Flow_::@120::@126 icmp_s
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
code
uint8_t code
Definition: decode-icmpv4.h:1
TcpStream_::tcp_flags
uint8_t tcp_flags
Definition: stream-tcp-private.h:99
Flow_::tosrcpktcnt
uint32_t tosrcpktcnt
Definition: flow.h:493
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
debug.h
JsonNetFlowLogThread_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-netflow.c:58
output.h
JsonNetFlowLogRegister
void JsonNetFlowLogRegister(void)
Definition: output-json-netflow.c:392