suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "detect.h"
28 #include "pkt-var.h"
29 #include "conf.h"
30 
31 #include "threads.h"
32 #include "threadvars.h"
33 #include "tm-threads.h"
34 
35 #include "util-print.h"
36 #include "util-unittest.h"
37 
38 #include "util-debug.h"
39 
40 #include "output.h"
41 #include "util-privs.h"
42 #include "util-buffer.h"
43 #include "util-device.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir)
53 {
54  char timebuf[64];
55  char srcip[46] = {0}, dstip[46] = {0};
56  Port sp, dp;
57 
58  JsonBuilder *js = jb_new_object();
59  if (unlikely(js == NULL))
60  return NULL;
61 
62  SCTime_t ts = TimeGet();
63 
64  CreateIsoTimeString(ts, timebuf, sizeof(timebuf));
65 
66  /* reverse header direction if the flow started out wrong */
67  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
68 
69  if (FLOW_IS_IPV4(f)) {
70  if (dir == 0) {
71  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
72  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
73  } else {
74  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
75  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
76  }
77  } else if (FLOW_IS_IPV6(f)) {
78  if (dir == 0) {
79  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
80  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
81  } else {
82  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
83  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
84  }
85  }
86 
87  if (dir == 0) {
88  sp = f->sp;
89  dp = f->dp;
90  } else {
91  sp = f->dp;
92  dp = f->sp;
93  }
94 
95  /* time */
96  jb_set_string(js, "timestamp", timebuf);
97 
98  CreateEveFlowId(js, (const Flow *)f);
99 
100 #if 0 // TODO
101  /* sensor id */
102  if (sensor_id >= 0)
103  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
104 #endif
105 
106  /* input interface */
107  if (f->livedev) {
108  jb_set_string(js, "in_iface", f->livedev->dev);
109  }
110 
111  JB_SET_STRING(js, "event_type", "netflow");
112 
113  /* vlan */
114  if (f->vlan_idx > 0) {
115  jb_open_array(js, "vlan");
116  jb_append_uint(js, f->vlan_id[0]);
117  if (f->vlan_idx > 1) {
118  jb_append_uint(js, f->vlan_id[1]);
119  }
120  jb_close(js);
121  }
122 
123  /* tuple */
124  jb_set_string(js, "src_ip", srcip);
125  switch(f->proto) {
126  case IPPROTO_ICMP:
127  break;
128  case IPPROTO_UDP:
129  case IPPROTO_TCP:
130  case IPPROTO_SCTP:
131  jb_set_uint(js, "src_port", sp);
132  break;
133  }
134  jb_set_string(js, "dest_ip", dstip);
135  switch(f->proto) {
136  case IPPROTO_ICMP:
137  break;
138  case IPPROTO_UDP:
139  case IPPROTO_TCP:
140  case IPPROTO_SCTP:
141  jb_set_uint(js, "dest_port", dp);
142  break;
143  }
144 
145  if (SCProtoNameValid(f->proto)) {
146  jb_set_string(js, "proto", known_proto[f->proto]);
147  } else {
148  char proto[4];
149  snprintf(proto, sizeof(proto), "%"PRIu8"", f->proto);
150  jb_set_string(js, "proto", proto);
151  }
152 
153  switch (f->proto) {
154  case IPPROTO_ICMP:
155  case IPPROTO_ICMPV6: {
156  uint8_t type = f->icmp_s.type;
157  uint8_t code = f->icmp_s.code;
158  if (dir == 1) {
159  type = f->icmp_d.type;
160  code = f->icmp_d.code;
161 
162  }
163  jb_set_uint(js, "icmp_type", type);
164  jb_set_uint(js, "icmp_code", code);
165  break;
166  }
167  case IPPROTO_ESP:
168  jb_set_uint(js, "spi", f->esp.spi);
169  break;
170  }
171  return js;
172 }
173 
174 /* JSON format logging */
175 static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f)
176 {
177  jb_set_string(js, "app_proto",
179 
180  jb_open_object(js, "netflow");
181 
182  jb_set_uint(js, "pkts", f->todstpktcnt);
183  jb_set_uint(js, "bytes", f->todstbytecnt);
184 
185  char timebuf1[64], timebuf2[64];
186 
187  CreateIsoTimeString(f->startts, timebuf1, sizeof(timebuf1));
188  CreateIsoTimeString(f->lastts, timebuf2, sizeof(timebuf2));
189 
190  jb_set_string(js, "start", timebuf1);
191  jb_set_string(js, "end", timebuf2);
192 
193  int32_t age = SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts);
194  jb_set_uint(js, "age", age);
195 
196  jb_set_uint(js, "min_ttl", f->min_ttl_toserver);
197  jb_set_uint(js, "max_ttl", f->max_ttl_toserver);
198 
199  /* Close netflow. */
200  jb_close(js);
201 
202  /* TCP */
203  if (f->proto == IPPROTO_TCP) {
204  jb_open_object(js, "tcp");
205 
206  TcpSession *ssn = f->protoctx;
207 
208  char hexflags[3];
209  snprintf(hexflags, sizeof(hexflags), "%02x",
210  ssn ? ssn->client.tcp_flags : 0);
211  jb_set_string(js, "tcp_flags", hexflags);
212 
213  EveTcpFlags(ssn ? ssn->client.tcp_flags : 0, js);
214 
215  jb_close(js);
216  }
217 }
218 
219 static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f)
220 {
221  jb_set_string(js, "app_proto",
223 
224  jb_open_object(js, "netflow");
225 
226  jb_set_uint(js, "pkts", f->tosrcpktcnt);
227  jb_set_uint(js, "bytes", f->tosrcbytecnt);
228 
229  char timebuf1[64], timebuf2[64];
230 
231  CreateIsoTimeString(f->startts, timebuf1, sizeof(timebuf1));
232  CreateIsoTimeString(f->lastts, timebuf2, sizeof(timebuf2));
233 
234  jb_set_string(js, "start", timebuf1);
235  jb_set_string(js, "end", timebuf2);
236 
237  int32_t age = SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts);
238  jb_set_uint(js, "age", age);
239 
240  /* To client is zero if we did not see any packet */
241  if (f->tosrcpktcnt) {
242  jb_set_uint(js, "min_ttl", f->min_ttl_toclient);
243  jb_set_uint(js, "max_ttl", f->max_ttl_toclient);
244  }
245 
246  /* Close netflow. */
247  jb_close(js);
248 
249  /* TCP */
250  if (f->proto == IPPROTO_TCP) {
251  jb_open_object(js, "tcp");
252 
253  TcpSession *ssn = f->protoctx;
254 
255  char hexflags[3];
256  snprintf(hexflags, sizeof(hexflags), "%02x",
257  ssn ? ssn->server.tcp_flags : 0);
258  jb_set_string(js, "tcp_flags", hexflags);
259 
260  EveTcpFlags(ssn ? ssn->server.tcp_flags : 0, js);
261 
262  jb_close(js);
263  }
264 }
265 
266 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
267 {
268  SCEnter();
269  OutputJsonThreadCtx *jhl = thread_data;
270 
271  JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0);
272  if (unlikely(jb == NULL))
273  return TM_ECODE_OK;
274  NetFlowLogEveToServer(jb, f);
275  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb);
276  OutputJsonBuilderBuffer(jb, jhl);
277  jb_free(jb);
278 
279  /* only log a response record if we actually have seen response packets */
280  if (f->tosrcpktcnt) {
281  jb = CreateEveHeaderFromNetFlow(f, 1);
282  if (unlikely(jb == NULL))
283  return TM_ECODE_OK;
284  NetFlowLogEveToClient(jb, f);
285  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb);
286  OutputJsonBuilderBuffer(jb, jhl);
287  jb_free(jb);
288  }
290 }
291 
293 {
294  /* register as child of eve-log */
295  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog", "eve-log.netflow",
296  OutputJsonLogInitSub, JsonNetFlowLogger, JsonLogThreadInit, JsonLogThreadDeinit, NULL);
297 }
tm-threads.h
FlowAddress_::address
union FlowAddress_::@115 address
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputJsonLogInitSub
OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
Definition: output-json-common.c:73
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:162
CreateIsoTimeString
void CreateIsoTimeString(const SCTime_t ts, char *str, size_t size)
Definition: util-time.c:209
OutputJsonThreadCtx_::ctx
OutputJsonCtx * ctx
Definition: output-json.h:90
Flow_::startts
SCTime_t startts
Definition: flow.h:496
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:84
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
Flow_::proto
uint8_t proto
Definition: flow.h:379
JsonLogThreadInit
TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data)
Definition: output-json-common.c:90
threads.h
Flow_
Flow data structure.
Definition: flow.h:357
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
Definition: output-json.c:934
output-json-netflow.h
Flow_::vlan_id
uint16_t vlan_id[2]
Definition: flow.h:381
SCProtoNameValid
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
Definition: util-proto-name.c:454
util-privs.h
known_proto
const char * known_proto[256]
Definition: util-proto-name.c:40
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
Definition: output-json.c:412
Flow_::max_ttl_toserver
uint8_t max_ttl_toserver
Definition: flow.h:474
proto
uint8_t proto
Definition: decode-template.h:0
Flow_::dp
Port dp
Definition: flow.h:373
Flow_::protoctx
void * protoctx
Definition: flow.h:447
Flow_::icmp_s
struct Flow_::@116::@122 icmp_s
util-unittest.h
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
OutputJsonThreadCtx_
Definition: output-json.h:89
Flow_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:501
Flow_::dst
FlowAddress dst
Definition: flow.h:360
Flow_::min_ttl_toserver
uint8_t min_ttl_toserver
Definition: flow.h:473
util-device.h
util-debug.h
JB_SET_STRING
#define JB_SET_STRING(jb, key, val)
Definition: rust.h:38
type
uint8_t type
Definition: decode-icmpv4.h:0
Flow_::todstpktcnt
uint32_t todstpktcnt
Definition: flow.h:498
output-json.h
Flow_::lastts
SCTime_t lastts
Definition: flow.h:416
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
PrintInet
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:274
pkt-var.h
EveTcpFlags
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
Definition: output-json.c:453
util-time.h
LiveDevice_::dev
char * dev
Definition: util-device.h:40
Flow_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:500
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:160
LOGGER_JSON_NETFLOW
@ LOGGER_JSON_NETFLOW
Definition: suricata-common.h:475
TimeGet
SCTime_t TimeGet(void)
Definition: util-time.c:152
stream-tcp-private.h
conf.h
Port
uint16_t Port
Definition: decode.h:229
SCTime_t
Definition: util-time.h:40
Flow_::vlan_idx
uint8_t vlan_idx
Definition: flow.h:383
Flow_::min_ttl_toclient
uint8_t min_ttl_toclient
Definition: flow.h:475
util-proto-name.h
Flow_::icmp_d
struct Flow_::@118::@124 icmp_d
Flow_::src
FlowAddress src
Definition: flow.h:360
suricata-common.h
SCTIME_SECS
#define SCTIME_SECS(t)
Definition: util-time.h:57
Flow_::max_ttl_toclient
uint8_t max_ttl_toclient
Definition: flow.h:476
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:294
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
Flow_::livedev
struct LiveDevice_ * livedev
Definition: flow.h:404
threadvars.h
Flow_::esp
struct Flow_::@116::@123 esp
CreateEveFlowId
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
Definition: output-json.c:714
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:293
OutputRegisterFlowSubModule
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:587
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:457
util-logopenfile.h
Flow_::flags
uint32_t flags
Definition: flow.h:427
util-buffer.h
IPPROTO_SCTP
#define IPPROTO_SCTP
Definition: decode.h:931
Flow_::sp
Port sp
Definition: flow.h:362
JsonLogThreadDeinit
TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data)
Definition: output-json-common.c:123
TcpSession_
Definition: stream-tcp-private.h:282
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:458
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:456
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:109
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
code
uint8_t code
Definition: decode-icmpv4.h:1
TcpStream_::tcp_flags
uint8_t tcp_flags
Definition: stream-tcp-private.h:111
Flow_::tosrcpktcnt
uint32_t tosrcpktcnt
Definition: flow.h:499
output.h
JsonNetFlowLogRegister
void JsonNetFlowLogRegister(void)
Definition: output-json-netflow.c:292